AB_testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IMport libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD DATA\n",
    "df = pd.read_csv(\"ABAdRecall.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>device_make</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0008ef63-77a7-448b-bd1e-075f42c55e39</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-10</td>\n",
       "      <td>8</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000eabc5-17ce-4137-8efe-44734d914446</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-07</td>\n",
       "      <td>10</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0016d14a-ae18-4a02-a204-6ba53b52f2ed</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-05</td>\n",
       "      <td>2</td>\n",
       "      <td>E5823</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00187412-2932-4542-a8ef-3633901c98d9</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Samsung SM-A705FN</td>\n",
       "      <td>6</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001a7785-d3fe-4e11-a344-c8735acacc2c</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             auction_id experiment        date  hour  \\\n",
       "0  0008ef63-77a7-448b-bd1e-075f42c55e39    exposed  2020-07-10     8   \n",
       "1  000eabc5-17ce-4137-8efe-44734d914446    exposed  2020-07-07    10   \n",
       "2  0016d14a-ae18-4a02-a204-6ba53b52f2ed    exposed  2020-07-05     2   \n",
       "3  00187412-2932-4542-a8ef-3633901c98d9    control  2020-07-03    15   \n",
       "4  001a7785-d3fe-4e11-a344-c8735acacc2c    control  2020-07-03    15   \n",
       "\n",
       "          device_make  platform_os                browser  yes  no  \n",
       "0  Generic Smartphone            6          Chrome Mobile    0   0  \n",
       "1  Generic Smartphone            6          Chrome Mobile    0   0  \n",
       "2               E5823            6  Chrome Mobile WebView    0   1  \n",
       "3   Samsung SM-A705FN            6               Facebook    0   0  \n",
       "4  Generic Smartphone            6          Chrome Mobile    0   0  "
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8077 entries, 0 to 8076\n",
      "Data columns (total 9 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   auction_id   8077 non-null   object\n",
      " 1   experiment   8077 non-null   object\n",
      " 2   date         8077 non-null   object\n",
      " 3   hour         8077 non-null   int64 \n",
      " 4   device_make  8077 non-null   object\n",
      " 5   platform_os  8077 non-null   int64 \n",
      " 6   browser      8077 non-null   object\n",
      " 7   yes          8077 non-null   int64 \n",
      " 8   no           8077 non-null   int64 \n",
      "dtypes: int64(4), object(5)\n",
      "memory usage: 568.0+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hour</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8077.000000</td>\n",
       "      <td>8077.000000</td>\n",
       "      <td>8077.000000</td>\n",
       "      <td>8077.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>11.615080</td>\n",
       "      <td>5.947134</td>\n",
       "      <td>0.070818</td>\n",
       "      <td>0.083075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.734879</td>\n",
       "      <td>0.224333</td>\n",
       "      <td>0.256537</td>\n",
       "      <td>0.276013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>15.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>23.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              hour  platform_os          yes           no\n",
       "count  8077.000000  8077.000000  8077.000000  8077.000000\n",
       "mean     11.615080     5.947134     0.070818     0.083075\n",
       "std       5.734879     0.224333     0.256537     0.276013\n",
       "min       0.000000     5.000000     0.000000     0.000000\n",
       "25%       7.000000     6.000000     0.000000     0.000000\n",
       "50%      13.000000     6.000000     0.000000     0.000000\n",
       "75%      15.000000     6.000000     0.000000     0.000000\n",
       "max      23.000000     7.000000     1.000000     1.000000"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>device_make</th>\n",
       "      <th>browser</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8077</td>\n",
       "      <td>8077</td>\n",
       "      <td>8077</td>\n",
       "      <td>8077</td>\n",
       "      <td>8077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>8077</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>270</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>d37ab23c-7f1d-4e93-9e2a-ff519efef3c0</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>4071</td>\n",
       "      <td>2015</td>\n",
       "      <td>4743</td>\n",
       "      <td>4554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  auction_id experiment        date  \\\n",
       "count                                   8077       8077        8077   \n",
       "unique                                  8077          2           8   \n",
       "top     d37ab23c-7f1d-4e93-9e2a-ff519efef3c0    control  2020-07-03   \n",
       "freq                                       1       4071        2015   \n",
       "\n",
       "               device_make        browser  \n",
       "count                 8077           8077  \n",
       "unique                 270             15  \n",
       "top     Generic Smartphone  Chrome Mobile  \n",
       "freq                  4743           4554  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe(include = 'O')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "control    4071\n",
       "exposed    4006\n",
       "Name: experiment, dtype: int64"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Number of control  and exposed\n",
    "df.experiment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>device_make</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0008ef63-77a7-448b-bd1e-075f42c55e39</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-10</td>\n",
       "      <td>8</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000eabc5-17ce-4137-8efe-44734d914446</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-07</td>\n",
       "      <td>10</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0016d14a-ae18-4a02-a204-6ba53b52f2ed</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-05</td>\n",
       "      <td>2</td>\n",
       "      <td>E5823</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00187412-2932-4542-a8ef-3633901c98d9</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Samsung SM-A705FN</td>\n",
       "      <td>6</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001a7785-d3fe-4e11-a344-c8735acacc2c</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             auction_id experiment        date  hour  \\\n",
       "0  0008ef63-77a7-448b-bd1e-075f42c55e39    exposed  2020-07-10     8   \n",
       "1  000eabc5-17ce-4137-8efe-44734d914446    exposed  2020-07-07    10   \n",
       "2  0016d14a-ae18-4a02-a204-6ba53b52f2ed    exposed  2020-07-05     2   \n",
       "3  00187412-2932-4542-a8ef-3633901c98d9    control  2020-07-03    15   \n",
       "4  001a7785-d3fe-4e11-a344-c8735acacc2c    control  2020-07-03    15   \n",
       "\n",
       "          device_make  platform_os                browser  yes  no  \n",
       "0  Generic Smartphone            6          Chrome Mobile    0   0  \n",
       "1  Generic Smartphone            6          Chrome Mobile    0   0  \n",
       "2               E5823            6  Chrome Mobile WebView    0   1  \n",
       "3   Samsung SM-A705FN            6               Facebook    0   0  \n",
       "4  Generic Smartphone            6          Chrome Mobile    0   0  "
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7505\n",
       "1     572\n",
       "Name: yes, dtype: int64"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#count of yes\n",
    "df.yes.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7406\n",
       "1     671\n",
       "Name: no, dtype: int64"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#count of no\n",
    "df.no.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    6834\n",
       "1    1243\n",
       "Name: engaged, dtype: int64"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Customers that engaged with the questionnaire\n",
    "df['engaged'] = df.yes + df.no ; df.engaged.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(586, 657)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Number of customers in both groups that engaged\n",
    "df[df.experiment == 'control']['engaged'].sum() , df[df.experiment == 'exposed']['engaged'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Generic Smartphone     4743\n",
       "iPhone                  433\n",
       "Samsung SM-G960F        203\n",
       "Samsung SM-G973F        154\n",
       "Samsung SM-G950F        148\n",
       "                       ... \n",
       "G3221                     1\n",
       "XiaoMi Redmi 5 Plus       1\n",
       "TA-1024                   1\n",
       "Samsung SM-J810F          1\n",
       "XiaoMi Redmi 3S           1\n",
       "Name: device_make, Length: 270, dtype: int64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.device_make.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6    7648\n",
       "5     428\n",
       "7       1\n",
       "Name: platform_os, dtype: int64"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['platform_os'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEJCAYAAABohnsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df1yV9cH/8deBgzBFm+A54syRtZy75ZuaVCPXIWsTyjEVXSkalemdlrbZPZUQZdR26xhfTecwt1zbTNfMFByjY7uzqEYWcvfIbNpaKreiHg6ICCRwOOe6/7DOHfnrojwefryfj0cPuD7nus55H7Dz5vpc13WOxTAMAxERERNCgh1AREQ6D5WGiIiYptIQERHTVBoiImKaSkNERExTaYiIiGkqDRERMc0a7ACBVlvbiM+nS1FERMwICbHQt2+v897e5UvD5zNUGiIil4imp0RExDSVhoiImNblp6fOxTAMamvdtLQ0AV1l6spCjx4R9O1rw2KxBDuMiHRR3bI0GhrqsFgs9O9/JRZL19jZMgwfJ09W09BQR+/eXw12HBHporrGK2Y7nT7dQO/eX+0yhQFgsYTQu3dfTp9uCHYUEenCus6rZjv4fF5CQ7veTlZoqBWfzxvsGCLShXXL0gC65Lx/V3xOItKxdL0/t7uBV175L154YTNr1vwm2FE6jN59IogIDwt2jHZpavZQf6op2DFE2kWlIV1CRHgYaQs3BjtGu2zKnUY9Kg3pXFQan/HGG6/xhz+sp7XVQ0REBA8//GPeeutNjh8/Rk1NNcePH8Nms7NkyRP069ePffveJy9vOa2tHr72tStxuY4xd+58Roy4ntWrV/D+++9x+vTHGIbBokVZXHfdCGpra1m2LIfKyiP06XMF0dHRDB58DQ888CCHDh1k1ao86urq8Pl8TJ58N9///ngAnn76KV566UWuuOIKrrzy60H+SYlId6XS+MThw//Db37za371q3VcccVXOXDgI+bPf4ixY+/k3Xff4ZlnNtKrVySLFs2nsPAF7r33ARYvXsiCBZkkJIzmv/97Nz/60RwA/vGPvVRXu1m37hlCQkLYsOH3PPvsH8jNHcGTT/6Sq666mtzcJ6muruaBB6YzePA1tLa2kpW1iCVLHueb3xxKQ0MDs2ffz1VXXU1tbQ2vvrqT3/9+Ez16hJOZ+ZMg/7REpLtSaXyirOwtamqq+dGPHvKPWSwhHDlymJEjR9GrVyQAQ4YM5dSpOg4c+BcACQmjAbj++niuvvoaAOLiruPf/70PhYVbqaw8wjvvlNOzZ08Adu36O7/73ZlplH79+jFmzO3AmdI6evQIy5Y97n/85uZmPvzwAw4dOkBi4hh69jzzJmLjxv2A559/LpA/DhGRc1JpfMLn8zJq1I08/vgy/5jLdZzt27dRX3+qzbqGYRAaasUw2l5NHhJy5mS00tI3WLUqjylTpnPLLYnExl7Fjh3FAGdt9+k2Pp+PXr0i+f3vN/lvO3Gihl69IsnPX9Vmm9DQ0Ev0rEVE2qfbnnL7eaNG3cjbb++iouIQAG+++Qb33juV5ubmc64fG3sVYWFh7NpVCpyZkjpw4CMsFgtlZW8xevQtTJw4maFDv8Xrr7+Kz+cDzuyZFBUVAlBXd5LXXnsVi8XC178eS3h4uL9cXK7jpKffzQcf7OPb376ZV175L+rr6/H5fDidxQH+aYiInJv2ND4xePDVLFy4mOzszE/2JEL5xS9WsHv327S0nF0cVquVn/88l1/+chnr1q1h0KBYoqKiiYiIYMKESfz0p5mkp9+N1+vlhhu+TUnJTnw+H4888ijLl/+M9PS76dPnCmJiBhAREUFYWBjLlv1/Vq3KY9OmP9La2srMmbO57roRAHz00b+YOfMeevfuwze+cS0nT9Ze7h+RiAgW4/NzLF1MTU3DWZ+ncfx4BTExsV/6vn/961VMnTqdqKhoXK7j3HdfGps3F9K7d+/zbrN16/MMGfJN4uKuo6WlhYcfnsmMGQ/6j418WZfquXU2NlvvTnnKrdtdH+wYIm2EhFiIjo487+3a0/gSYmJi+NGP5mC1WjEMyMjIumBhAFx11WBWrvwlPp8Xj6eVMWNuv2SFISISaAEtjZ07d7JmzRpOnz7N6NGjycrKorS0lGXLltHc3Mwdd9zB/PnzAdi3bx+LFy+msbGR+Ph4cnJysFqtHD16lAULFlBTU8PgwYPJy8ujV6/zfxTh5TRp0t1MmnR3u7a5/vp41q/fEKBEIiKBFbAD4YcPHyY7O5v8/Hy2b9/OP/7xD0pKSsjMzCQ/P5/i4mL27t1LSUkJAAsWLGDp0qXs2LEDwzDYvHkzADk5OaSlpeF0OomLiyM/Pz9QkUVE5CICVhp/+9vfuPPOO4mJiSEsLIyVK1fyla98hdjYWAYNGoTVaiUlJQWn00llZSVNTU2MGHHmoG9qaipOpxOPx0NZWRlJSUltxkVEJDgCNj1VUVFBWFgYs2fP5tixY9x6661ce+212Gw2/zp2ux2Xy0VVVVWbcZvNhsvlora2lsjISKxWa5txEREJjoCVhtfrZffu3WzYsIGePXsyZ84cIiIi2rx9t2EYWCwWfD7fOcc//fpZ7X3773OdBVBVFYLV2jUvUQkJCcFmu/DBeOk49LuSziZgpdGvXz8SEhKIiooC4Lvf/S5Op7PN1cxutxu73U5MTAxut9s/Xl1djd1uJyoqivr6erxeL6Ghof712+Ncp9z6fD5aW31f4tl1XD6fr1uextlZX3y74+9KOragnXI7ZswYFi1axKlTp+jVqxevv/46ycnJ/OY3v6GiooIrr7ySoqIiJk2axMCBAwkPD6e8vJxRo0ZRWFiIw+EgLCyM+Ph4iouLSUlJoaCgAIfDEajIbQTq8xnMfobCSy85+eMf19Pa2soPfziVSZPuuuRZRETaK2ClMXz4cGbOnElaWhoej4fRo0czdepUrr76aubNm0dzczOJiYkkJycDkJeXR1ZWFg0NDQwbNoz09HQAsrOzycjIYO3atQwYMIAVK1YEKnIbgfp8BjOfoeB2V/Hb3+azfv0GwsJ6MHv2DK6/Pp7Bg6++5HlERNojoNdpTJ48mcmTJ7cZS0hIYPv27WetO3ToULZs2XLW+MCBA9mwoXtd17B799tcf308ffpcAcCYMbfz6qsvqzREJOi65tHgTq662k10dD//cnR0P6qqqoKYSETkDJVGB3Sus8lCQtp31piISCCoNDogu70/NTXV/uUTJ2ro1892gS1ERC4PlUYHFB9/I+XlZdTW1tLU1MSrr+7kppsSgh1LRETvcns+Tc0eNuVOC8j9XozNZmfWrId45JEH8XhaSUkZz7/9W9wlzyIi0l4qjfOoP9V00VNjA2ns2GTGjk0O2uOLiJyLpqdERMQ0lYaIiJim0hAREdNUGiIiYppKQ0RETFNpiIiIaTrl9jz6XtEDa4/wS36/rS3N1Na1mFq3sbGB2bNnkJv7JAMGfO2SZxERaS+VxnlYe4RTnjvzkt/vqIVPAxcvjfff30tu7s84fPh/LnkGEZEvStNTHdRf/rKNRx9dpPecEpEORXsaHVRGxpJgRxAROYv2NERExDSVhoiImKbSEBER01QaIiJimg6En0drS/Mnp8de+vttjy1b/nLJM4iIfFEqjfM4cwGeuYvwRES6C01PiYiIaQHd07jnnns4ceIEVuuZh3n88cdpbGxk2bJlNDc3c8cddzB//nwA9u3bx+LFi2lsbCQ+Pp6cnBysVitHjx5lwYIF1NTUMHjwYPLy8ujVq1cgY4uIyHkEbE/DMAwOHTpEYWGh/79vfvObZGZmkp+fT3FxMXv37qWkpASABQsWsHTpUnbs2IFhGGzevBmAnJwc0tLScDqdxMXFkZ+ff8nydTVd8TmJSMcSsNI4cOAAADNmzOAHP/gBzz77LHv27CE2NpZBgwZhtVpJSUnB6XRSWVlJU1MTI0aMACA1NRWn04nH46GsrIykpKQ241+W1dqDxsZTXepF1jAMGhtPYbX2CHYUEenCAjY9derUKRISEliyZAkej4f09HRmzpyJzfZ/76Vkt9txuVxUVVW1GbfZbLhcLmpra4mMjPRPb306/mX17WujttZNQ8PJL31fHYnV2oO+ffVeVSISOAErjZEjRzJy5Ej/8uTJk1m9ejWjRo3yjxmGgcViwefzYbFYzhr/9OtnfX75YqKjI885HhPTt133IxIINlvvYEcQaZeAlcbu3bvxeDwkJCQAZ4pg4MCBuN1u/zputxu73U5MTEyb8erqaux2O1FRUdTX1+P1egkNDfWv3x41NQ34fF1nGkrOrbO++Lrd9cGOINJGSIjlvH9sQwCPadTX15Obm0tzczMNDQ1s27aNRx99lIMHD1JRUYHX66WoqAiHw8HAgQMJDw+nvLwcgMLCQhwOB2FhYcTHx1NcXAxAQUEBDocjUJFFROQiAranMWbMGN59910mTJiAz+cjLS2NkSNHsnz5cubNm0dzczOJiYkkJycDkJeXR1ZWFg0NDQwbNoz09HQAsrOzycjIYO3atQwYMIAVK1YEKrKIiFyExehKpxCdg6anugebrTdpCzcGO0a7bMqdpukp6XCCNj0lIiJdj0pDRERMU2mIiIhpKg0RETFNpSEiIqapNERExDSVhoiImKbSEBER01QaIiJimkpDRERMU2mIiIhpKg0RETFNpSEiIqapNERExDSVhoiImKbSEBER01QaIiJimkpDRERMU2mIiIhpKg0RETFNpSEiIqapNERExDSVhoiImKbSEBER0wJeGr/4xS/IyMgAoLS0lJSUFMaOHcvKlSv96+zbt4/U1FSSkpJYvHgxra2tABw9epRp06aRnJzMnDlzaGxsDHRcERG5gICWxptvvsm2bdsAaGpqIjMzk/z8fIqLi9m7dy8lJSUALFiwgKVLl7Jjxw4Mw2Dz5s0A5OTkkJaWhtPpJC4ujvz8/EDGFRGRiwhYaZw8eZKVK1cye/ZsAPbs2UNsbCyDBg3CarWSkpKC0+mksrKSpqYmRowYAUBqaipOpxOPx0NZWRlJSUltxkVEJHisgbrjpUuXMn/+fI4dOwZAVVUVNpvNf7vdbsflcp01brPZcLlc1NbWEhkZidVqbTPeXtHRkV/ymYgEjs3WO9gRRNolIKXx/PPPM2DAABISEti6dSsAPp8Pi8XiX8cwDCwWy3nHP/36WZ9fNqOmpgGfz/iCz0Q6i8764ut21wc7gkgbISGWC/6xHZDSKC4uxu12M378eOrq6vj444+prKwkNDTUv47b7cZutxMTE4Pb7faPV1dXY7fbiYqKor6+Hq/XS2hoqH99EREJnoAc03jmmWcoKiqisLCQRx55hNtuu42nn36agwcPUlFRgdfrpaioCIfDwcCBAwkPD6e8vByAwsJCHA4HYWFhxMfHU1xcDEBBQQEOhyMQcUVExKSAHdP4vPDwcJYvX868efNobm4mMTGR5ORkAPLy8sjKyqKhoYFhw4aRnp4OQHZ2NhkZGaxdu5YBAwawYsWKyxVXRETOwWIYRpee8Ncxje7BZutN2sKNwY7RLptyp+mYhnQ4FzumoSvCRUTENJWGiIiYZqo0znV9xL/+9a9LHkZERDq2C5bGyZMnOXnyJLNmzaKurs6/XF1dzdy5cy9XRhER6SAuePbUf/zHf/D3v/8dgJtuuun/NrJa/W/vISIi3ccFS2P9+vUAPPbYYyxbtuyyBBIRkY7L1HUay5Yto7Kykrq6Oj57hu6wYcMCFkxERDoeU6WxevVq1q9fT3R0tH/MYrHw8ssvByyYiIh0PKZKo6CggJdeeon+/fsHOo+IiHRgpk65HTBggApDRETM7WkkJCSQm5vL7bffTkREhH9cxzRERLoXU6Xx6WdifPaT83RMQ0Sk+zFVGjt37gx0DhER6QRMlcYzzzxzzvH777//koYREZGOzVRp/POf//R/39LSQllZGQkJCQELJSIiHZPpi/s+y+VysXjx4oAEEhGRjusLvTV6//79qaysvNRZRESkg2v3MQ3DMNi7d2+bq8NFRKR7aPcxDThzsd/ChQsDEkhERDqudh3TqKyspLW1ldjY2ICGEhGRjslUaVRUVPDQQw9RVVWFz+ejb9++rFu3jmuuuSbQ+UREpAMxdSD88ccfZ+bMmZSVlVFeXs6cOXPIyckJdDYREelgTJVGTU0NEydO9C9PmjSJ2tragIUSEZGOyVRpeL1eTp486V8+ceKEqTtftWoVd955J+PGjfOfgVVaWkpKSgpjx45l5cqV/nX37dtHamoqSUlJLF68mNbWVgCOHj3KtGnTSE5OZs6cOTQ2Npp+ciIicmmZKo3p06dz99138+STT7Jq1SqmTp3K1KlTL7jN22+/za5du9i+fTsvvPACGzZsYP/+/WRmZpKfn09xcTF79+6lpKQEgAULFrB06VJ27NiBYRhs3rwZgJycHNLS0nA6ncTFxZGfn/8ln7KIiHxRpkojMTERAI/Hw0cffYTL5eJ73/veBbe58cYb+eMf/4jVaqWmpgav18upU6eIjY1l0KBBWK1WUlJScDqdVFZW0tTUxIgRIwBITU3F6XTi8XgoKysjKSmpzbiIiASHqbOnMjIymDZtGunp6TQ3N/OnP/2JzMxMfvvb315wu7CwMFavXs3vfvc7kpOTqaqqwmaz+W+32+24XK6zxm02Gy6Xi9raWiIjI7FarW3GRUQkOEyVRm1tLenp6QCEh4dz3333UVBQYOoBHnnkEWbNmsXs2bM5dOgQFovFf5thGFgsFnw+3znHP/36WZ9fvpjo6Mh2rS9yOdlsvYMdQaRdTJWG1+vF5XL5P/K1uroawzAuuM1HH31ES0sL3/rWt/jKV77C2LFjcTqdhIaG+tdxu93Y7XZiYmJwu93+8erqaux2O1FRUdTX1+P1egkNDfWv3x41NQ34fBfOKp1fZ33xdbvrgx1BpI2QEMsF/9g2dUzjvvvuY8KECSxcuJBFixYxceJEZs6cecFtjhw5QlZWFi0tLbS0tPDyyy8zZcoUDh48SEVFBV6vl6KiIhwOBwMHDiQ8PJzy8nIACgsLcTgchIWFER8fT3FxMQAFBQU4HA6zz11ERC4xU3sakydPJi4ujl27dhEaGsoDDzzAkCFDLrhNYmIie/bsYcKECYSGhjJ27FjGjRtHVFQU8+bNo7m5mcTERJKTkwHIy8sjKyuLhoYGhg0b5p8Oy87OJiMjg7Vr1zJgwABWrFjxJZ+yiIh8URbjYvNMnZymp7oHm603aQs3BjtGu2zKnabpKelwLsn0lIiICKg0RESkHVQaIiJimkpDRERMU2mIiIhpKg0RETFNpSEiIqapNERExDSVhoiImKbSEBER01QaIiJimkpDRERMU2mIiIhpKg0RETFNpSEiIqapNERExDSVhoiImKbSEBER01QaIiJimkpDRERMU2mIiIhpKg0RETFNpSEiIqYFtDTWrFnDuHHjGDduHLm5uQCUlpaSkpLC2LFjWblypX/dffv2kZqaSlJSEosXL6a1tRWAo0ePMm3aNJKTk5kzZw6NjY2BjCwiIhcQsNIoLS3ljTfeYNu2bRQUFPD+++9TVFREZmYm+fn5FBcXs3fvXkpKSgBYsGABS5cuZceOHRiGwebNmwHIyckhLS0Np9NJXFwc+fn5gYosIiIXEbDSsNlsZGRk0KNHD8LCwrjmmms4dOgQsbGxDBo0CKvVSkpKCk6nk8rKSpqamhgxYgQAqampOJ1OPB4PZWVlJCUltRkXEZHgCFhpXHvttf4SOHToEC+++CIWiwWbzeZfx26343K5qKqqajNus9lwuVzU1tYSGRmJ1WptMy4iIsFhDfQDfPjhhzz44IMsXLiQ0NBQDh065L/NMAwsFgs+nw+LxXLW+KdfP+vzyxcTHR35pfKLBJLN1jvYEUTaJaClUV5eziOPPEJmZibjxo3j7bffxu12+293u93Y7XZiYmLajFdXV2O324mKiqK+vh6v10toaKh//faoqWnA5zMu2XOSjqmzvvi63fXBjiDSRkiI5YJ/bAdseurYsWM8/PDD5OXlMW7cOACGDx/OwYMHqaiowOv1UlRUhMPhYODAgYSHh1NeXg5AYWEhDoeDsLAw4uPjKS4uBqCgoACHwxGoyCIichEB29NYv349zc3NLF++3D82ZcoUli9fzrx582hubiYxMZHk5GQA8vLyyMrKoqGhgWHDhpGeng5AdnY2GRkZrF27lgEDBrBixYpARRYRkYuwGIbRpeduND3VPdhsvUlbuDHYMdplU+40TU9JhxO06SkREel6VBoiImKaSkNERExTaYiIiGkqDRERMU2lISIipqk0RETENJWGiIiYptIQERHTVBoiImKaSkNERExTaYiIiGkqDRERMU2lISIipqk0RETENJWGiIiYptIQERHTVBoiImKaSkNERExTaYiIiGnWYAcQ6a58rR5stt7BjmFaa0sztXUtwY4hQabSEAmSEGsY5bkzgx3DtFELnwZUGt2dpqdERMS0gJZGQ0MD3//+9zly5AgApaWlpKSkMHbsWFauXOlfb9++faSmppKUlMTixYtpbW0F4OjRo0ybNo3k5GTmzJlDY2NjIOOKiMhFBKw03n33XaZOncqhQ4cAaGpqIjMzk/z8fIqLi9m7dy8lJSUALFiwgKVLl7Jjxw4Mw2Dz5s0A5OTkkJaWhtPpJC4ujvz8/EDFFREREwJWGps3byY7Oxu73Q7Anj17iI2NZdCgQVitVlJSUnA6nVRWVtLU1MSIESMASE1Nxel04vF4KCsrIykpqc24iIgET8AOhP/85z9vs1xVVYXNZvMv2+12XC7XWeM2mw2Xy0VtbS2RkZFYrdY24yIiEjyX7ewpn8+HxWLxLxuGgcViOe/4p18/6/PLZkRHR37x0CLSRmc6RVgC47KVRkxMDG6327/sdrux2+1njVdXV2O324mKiqK+vh6v10toaKh//faqqWnA5zMuyXOQjksvZpeH210f7AgSYCEhlgv+sX3ZTrkdPnw4Bw8epKKiAq/XS1FREQ6Hg4EDBxIeHk55eTkAhYWFOBwOwsLCiI+Pp7i4GICCggIcDsfliisiIudw2fY0wsPDWb58OfPmzaO5uZnExESSk5MByMvLIysri4aGBoYNG0Z6ejoA2dnZZGRksHbtWgYMGMCKFSsuV1wRETmHgJfGzp07/d8nJCSwffv2s9YZOnQoW7ZsOWt84MCBbNiwIaD5RETEPF0RLiIipqk0RETENJWGiIiYptIQERHTVBoiImKaSkNERExTaYiIiGkqDRERMU2lISIipqk0RETENJWGiIiYptIQERHTVBoiImKaSkNEREy7bJ+n0dH17hNBRHhYsGO0S1Ozh/pTTcGOISLdiErjExHhYaQt3BjsGO2yKXca9ag0ROTy0fSUiIiYptIQERHTVBoiImKaSkNERExTaYiIiGkqDRERMU2lISIipqk0RETEtE5RGn/5y1+48847GTt2LBs3dq4L8EREupIOf0W4y+Vi5cqVbN26lR49ejBlyhRuuukmvvGNbwQ7mohIt9PhS6O0tJRvf/vbfPWrXwUgKSkJp9PJ3LlzTW0fEmIx/Vj9+vb6QhmDqT3Pr6vrjL+/Hn2igx2hXfTvreu72O+4w5dGVVUVNpvNv2y329mzZ4/p7fu244Vk9WMT2pWtI4iOjgx2hA6jM/7+/t/sXwQ7Qrvo35t0+GMaPp8Pi+X/ms8wjDbLIiJy+XT40oiJicHtdvuX3W43drs9iIlERLqvDl8aN998M2+++SYnTpzg9OnTvPTSSzgcjmDHEhHpljr8MY3+/fszf/580tPT8Xg8TJ48meuuuy7YsUREuiWLYRhGsEOIiEjn0OGnp0REpONQaYiIiGkqDRERMU2lISIipnX4s6fkbPfccw8nTpzAaj3z63v88ccZPnx4kFOJWTt37mTNmjWcPn2a0aNHk5WVFexIYtLzzz/Ps88+618+cuQI48ePZ+nSpUFMdXnp7KlOxjAMHA4Hr7zyir80pPM4fPgwaWlpPP/880RHR3Pvvffy4IMPkpiYGOxo0k4ffvghDz/8MM899xxRUVHBjnPZ6FWnkzlw4AAAM2bM4OTJk9x1111Mnz49yKnErL/97W/ceeedxMTEALBy5UrCw8ODnEq+iJ/+9KfMnz+/WxUGqDQ6nVOnTpGQkMCSJUvweDykp6czePBgRo8eHexoYkJFRQVhYWHMnj2bY8eOceutt/LjH/842LGknUpLS2lqauKOO+4IdpTLTqXRyYwcOZKRI0f6lydPnkxJSYlKo5Pwer3s3r2bDRs20LNnT+bMmcO2bdtITU0NdjRph+eee477778/2DGCQmdPdTK7d+/mzTff9C8bhqFjG51Iv379SEhIICoqioiICL773e+2663+JfhaWlooKyvjtttuC3aUoFBpdDL19fXk5ubS3NxMQ0MD27Zt43vf+16wY4lJY8aM4Y033uDUqVN4vV5ef/11hg0bFuxY0g4ffPABV111FT179gx2lKDQn6idzJgxY3j33XeZMGECPp+PtLS0NtNV0rENHz6cmTNnkpaWhsfjYfTo0UyaNCnYsaQdDh8+7D+RoTvSKbciImKapqdERMQ0lYaIiJim0hAREdNUGiIiYppKQ0RETFNpiJxDRkYG69evv+A69fX1pKen+5e3bt3KrbfeygMPPBDoeCJBo+s0RL6guro63nvvPf9yQUEB8+fPZ/z48UFMJRJYKg3p1t566y3y8vL42te+xoEDB4iIiGD58uVt1tmyZQt//vOf8Xg81NXVMWvWLNLS0njsscdoampi/Pjx3HTTTbz33nscOXKE2tpaJk2aRE5ODvv378disXDLLbfw6KOPYrVaiYuL4/bbb2f//v3k5eWRlpbG/fffT2lpKR9//DFz587F6XTyz3/+E7vdzlNPPXXRq49//etf89e//pXQ0FAGDx7MkiVLsNlsvPTSS6xduxaLxUJoaCgLFy7khhtuCOSPVLo6Q6Qb27VrlzF06FCjrKzMMAzD2LRpkzFx4kRj0aJFxtNPP200NDQYd911l3HixAnDMAzjnXfeMUaMGGEYhmEcPnzY/71hGMb06dONF1980TAMw1i4cOFUq70AAALMSURBVKHxxBNPGD6fz2hubjZmzJhhrFu3zjAMwxgyZIixbds2/3ZDhgwx/vCHPxiGYRjr1q0zRo4caRw/ftzwer3GxIkTje3bt1/wOWzZssW4++67jcbGRsMwDGP16tXGjBkzDMMwjNtvv9145513DMMwjNdff9341a9+9eV+YNLt6ZiGdHtDhw4lPj4egEmTJrFv3z5OnjwJQK9evXjqqacoKSnhySef5KmnnuLjjz++6H2+9tprTJ8+HYvFQo8ePZgyZQqvvfaa//ZPH+9TSUlJAHz9619nyJAh9O/fn5CQEK688krq6uou+lipqan+vZH09HR27dpFS0sL48aNY+7cuSxevJhTp04xa9Ys8z8YkXNQaUi3FxoaetZYSMiZ/zWOHz/OhAkTqKysZNSoUaY/+8Ln82GxWNost7a2+pc/P90UFhZ2zu+/7GPNnz+fTZs2ERcXx9atW5k2bVq77lvk81Qa0u3t37+f/fv3A/DnP/+ZkSNH0qdPHwD27t1LVFQUDz30EN/5znd45ZVXgDOfi2G1WvF6vRjnePu273znOzz77LMYhkFLSwubN2/m5ptvDkj+W265hRdeeMG/B7RhwwZuuOEGQkJCuO222zh9+jRTp04lOzubDz74gJaWloDkkO5BB8Kl2+vXrx9PPvkklZWVREVFkZuby5o1awAYPXo0W7ZsITk5GYvFwo033khUVBQVFRXExsZy3XXXMW7cODZu3NjmPrOysvjZz35GSkoKHo+HW265hdmzZwck/+TJkzl27Bg//OEP8fl8xMbGkpeXh9VqJTMzk5/85CdYrVYsFgv/+Z//SY8ePQKSQ7oHvcutdGtvvfUWTzzxBEVFRcGOItIpaE9DpBNIS0ujsbHxnLdt3LiRyMjIy5xIuivtaYiIiGk6EC4iIqapNERExDSVhoiImKbSEBER01QaIiJimkpDRERM+1/pijz7ROQmPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(df.platform_os, hue = df.engaged);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>engaged</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hour</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.840206</td>\n",
       "      <td>0.159794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.855856</td>\n",
       "      <td>0.144144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.856522</td>\n",
       "      <td>0.143478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.834586</td>\n",
       "      <td>0.165414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.836299</td>\n",
       "      <td>0.163701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.837748</td>\n",
       "      <td>0.162252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.847095</td>\n",
       "      <td>0.152905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.845144</td>\n",
       "      <td>0.154856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.829949</td>\n",
       "      <td>0.170051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.835260</td>\n",
       "      <td>0.164740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.830357</td>\n",
       "      <td>0.169643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.843972</td>\n",
       "      <td>0.156028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.848921</td>\n",
       "      <td>0.151079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.858621</td>\n",
       "      <td>0.141379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.840125</td>\n",
       "      <td>0.159875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.851715</td>\n",
       "      <td>0.148285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.868657</td>\n",
       "      <td>0.131343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.870722</td>\n",
       "      <td>0.129278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.860806</td>\n",
       "      <td>0.139194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.832599</td>\n",
       "      <td>0.167401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.829545</td>\n",
       "      <td>0.170455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.834951</td>\n",
       "      <td>0.165049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.844444</td>\n",
       "      <td>0.155556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.838710</td>\n",
       "      <td>0.161290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "engaged         0         1\n",
       "hour                       \n",
       "0        0.840206  0.159794\n",
       "1        0.855856  0.144144\n",
       "2        0.856522  0.143478\n",
       "3        0.834586  0.165414\n",
       "4        0.836299  0.163701\n",
       "5        0.837748  0.162252\n",
       "6        0.847095  0.152905\n",
       "7        0.845144  0.154856\n",
       "8        0.829949  0.170051\n",
       "9        0.835260  0.164740\n",
       "10       0.830357  0.169643\n",
       "11       0.843972  0.156028\n",
       "12       0.848921  0.151079\n",
       "13       0.858621  0.141379\n",
       "14       0.840125  0.159875\n",
       "15       0.851715  0.148285\n",
       "16       0.868657  0.131343\n",
       "17       0.870722  0.129278\n",
       "18       0.860806  0.139194\n",
       "19       0.832599  0.167401\n",
       "20       0.829545  0.170455\n",
       "21       0.834951  0.165049\n",
       "22       0.844444  0.155556\n",
       "23       0.838710  0.161290"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(df.hour, df.engaged, normalize = 'index' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2000 hours has the ratio of users that engaged with the questionnaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAAHoCAYAAAD0XLN2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde0BUdf7/8dcMVxVU1EGUzEu5+l1NSW1dtsTsomj6c6M285JrZhfbLK28JK6slqFEWX7LrG9mm+kWtYrmslimmUqZUmYq5hUvqMOAoFzkOuf3hzmr1iqXGTjo8/GPzuHwPm8Oh5nXnM/nnLEYhmEIAAAApmWt7QYAAABwaQQ2AAAAkyOwAQAAmByBDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJedd2A56Wk1Mgp5NbzQEAAPOyWi0KCmrwX79+xQc2p9MgsAEAgDqNIVEAAACTI7ABAACY3BU/JAoAAOqe8vIy5eQ4VFZWUtutuJXV6qV69QIUENBIFoulwt9HYAMAAKaTk+OQv399NWgQUqlgY2aGYai8vEx5ebnKyXGoSZPgCn8vQ6IAAMB0yspK1KBBwysmrEmSxWKRt7ePGjduqpKSokp9L4ENAACY0pUU1s5nsVglVe4OFgQ2AAAAkyOwAQAAuNm6dWv0xBOPuK0egQ0AAMDkuEoUAADUGRs3fqW//32hyspK5e/vr7/8Zbw2b/5aJ04cV3Z2lk6cOC6bLVh//evzatasmdLSdio+frbKykrVsuU1stuP64knJigsrJvmzXtFO3f+qDNnCmUYhiZPnqYuXcKUk5Oj2NgZysg4qoYNG6lp06Zq2/Y6PfTQo0pPP6jXXovXqVOn5HQ6de+9QzRw4GBJ0jvvLNBnn/1bjRo10jXXXOvWn5vABgAA6oQjRw7r7bff0P/+71tq1KixDhzYrwkTHlffvgP0ww/fa9GiJWrQIECTJ0/QihX/1J///JCioydp4sSpCg+/Wd99t1VPPTVWkrRr1w5lZTn01luLZLVatXjxe/rgg78rLi5Mr776ktq0aae4uFeVlZWlhx4aobZtr1NZWZmmTZusv/51pjp06Kj8/Hw99tiDatOmnXJysvXll2v13ntL5evrp6lTn3Xrz05gAwAAdcKWLZuVnZ2lp5563LXMYrHq6NEjuvHG7mrQIECS9JvfdNTp06d04MA+SVJ4+M2SpG7deqhdu+skSZ07d9EjjzTUihXLlJFxVN9/n6r69etLkr75ZpPefXeJJKlZs2bq0+d2SWcD47FjRxUbO9O1/eLiYu3d+5PS0w+od+8+ql//7Ae433XX/9PHH3/otp+dwAYAAOoEp7Nc3bv/TjNnxrqW2e0ntHLlcuXlnb5gXcMw5OXlLcO48PYZVuvZ6fspKRv12mvxuv/+EerVq7dat26j1auTJOkX33fue5xOpxo0CNB77y11fe3kyWw1aBCg+fNfu+B7vLy83PRT/9yDW6sBAAB4SPfuv9O3336jQ4fSJUlff71Rf/7zUBUXF//q+q1bt5GPj4+++SZF0tlh0AMH9stisWjLls26+eZeuvvue9Wx4/9ow4Yv5XQ6JZ09I7dq1QpJ0qlTufrqqy9lsVh07bWt5efn5wp2dvsJjRw5RD/9lKbf//4PWrdujfLy8uR0OpWcnOTWn50zbAAAoE5o27adJk2KVkzM1J/PoHlpzpxXtHXrtyop+WVo8/b21qxZcXrppVi99dbratWqtZo0aSp/f3/98Y/36G9/m6qRI4eovLxcN930e61fv1ZOp1NPPvm0Zs9+QSNHDlHDho0UEtJC/v7+8vHxUWzsy3rttXgtXfq+ysrKNGbMY+rSJUyStH//Po0Z84ACAxvq+uvbKzc3x20/u8W4+FzhFSY7O19O5xX9IwIAcMU5ceKQQkJaV7vOG2+8pqFDR6hJk6ay209o1KhhSkhYocDAwP/6PcuWfazf/KaDOnfuopKSEv3lL2M0evSjrrlw7nDxz2e1WtS0acB/XZ8zbAAA4IoVEhKip54aK29vbxmGNGXKtEuGNUlq06at5s59SU5nuUpLy9Snz+1uDWtVwRk2AIBbBTb0l7+fz2XXKyouVd7pyn0ANq4e7jrDZlamOsOWn5+v+++/XwsWLNA111yj77//XrGxsSooKFCHDh00e/Zs+fr6Ki0tTdHR0SooKFCPHj00Y8YMeXt769ixY5o4caKys7PVtm1bxcfHq0GDBp5sGQBQTf5+Pho2acll11saN1x5IrABFeGxq0R/+OEHDR06VOnp6ZLOhrdx48Zp5syZ+te//iVJ+uSTTyRJEydO1PTp07V69WoZhqGEhARJ0owZMzRs2DAlJyerc+fOmj9/vqfaBQAAMC2PBbaEhATFxMQoODhYkrRp0yaFhYWpY8eOkqRp06bpzjvvVEZGhoqKihQWdvYKi6ioKCUnJ6u0tFRbtmxRv379LlgOAABwtfHYkOisWbMueHzo0CHVr19fEyZM0IEDB9StWzdNmTJFu3btks1mc61ns9lkt9uVk5OjgIAAeXt7X7C8si41HgwAqF0226Unf+PqlZlplbf3lXu7WKvVWqnjv8auEi0vL9fGjRv10UcfqWXLloqOjtbbb7+tP/zhD7JYLK71DMOQxWJx/Xu+ix9XBBcdAEDNqsyLkMOR58FOUJc5nU6VlTkvu15FL3KpLE9fFON0Oi84/k1zW49mzZqpa9euatWqlSSpf//++uCDDxQVFSWHw+FaLysrS8HBwWrSpIny8vJUXl4uLy8vORwO1/AqAACAVPGLXCqrohfFfPZZst5/f6HKysr0pz8N1T333Of2XqQa/GiqW265RTt37tTx48clSevWrVOnTp0UGhoqPz8/paamSpJWrFihiIgI+fj4qEePHkpKOvvRDomJiYqIiKipdgEAAC7J4cjU//3ffM2f/44WLVqqlSuX6+DBAx7ZVo0FthYtWmjmzJl67LHHFBkZqVOnTunRRx+VJMXHxys2NlaRkZEqLCzUyJEjJUkxMTFKSEjQgAEDtHXrVo0fP76m2gUAALikrVu/VbduPdSwYSPVq1dPffrcri+//MIj2/L4kOjatWtd/7/11lt16623/mKdjh07um7xcb7Q0FAtXrzYk+0BAABUSVaWQ02bNnM9btq0mXbt2umRbV25l18AAAB4kNPp/MWFk1Zr5S+QrAgCGwAAQBUEBzdXdnaW6/HJk9lq1sx2ie+oOgIbAABAFfTo8Tulpm5RTk6OioqK9OWXa9WzZ7hHtlVjt/UAAABwt6LiUi2NG+6RupdjswXr4Ycf15NPPqrS0jINGjRYv/1tZ7f3IhHYAABAHZZ3uqhC90vzlL59I9W3b6THt8OQKAAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5LitBwAAqLOCGvnK29fP7XXLSoqVc6qkQusWFOTrscdGKy7uVbVo0dLtvUgENgAAUId5+/opNW6M2+t2n/SOpMsHtp07dygu7gUdOXLY7T2cjyFRAACAKvr00+V6+unJHvsM0XM4wwYAAFBFU6b8tUa2wxk2AAAAkyOwAQAAmByBDQAAwOQIbAAAACbHRQcAAKDOKisp/vkWHO6vWxmffPKp23s4H4ENAADUWWdvbluxG9zWZQyJAgAAmByBDQAAwOQIbAAAwJQMw6jtFjzCMJySLJX6HgIbAAAwHW9vXxUUnL6iQpthGCorK1VubpZ8ff0r9b1cdAAAAEwnKMimnByH8vNza7sVt7JavVSvXoACAhpV6vsIbAAAwHS8vLzVrFmL2m7DNBgSBQAAMDkCGwAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5AhsAAAAJkdgAwAAMDkCGwAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5Dwa2PLz8zVw4EAdPXr0guUffPCBHnjgAdfjtLQ0RUVFqV+/foqOjlZZWZkk6dixYxo+fLgiIyM1duxYFRQUeLJdAAAAU/JYYPvhhx80dOhQpaenX7B83759evvtty9YNnHiRE2fPl2rV6+WYRhKSEiQJM2YMUPDhg1TcnKyOnfurPnz53uqXQAAANPyWGBLSEhQTEyMgoODXctKSko0ffp0Pfnkk65lGRkZKioqUlhYmCQpKipKycnJKi0t1ZYtW9SvX78LlgMAAFxtvD1VeNasWb9Y9vLLL+uee+7RNddc41qWmZkpm83memyz2WS325WTk6OAgAB5e3tfsLyymjYNqEL3AICaYLMF1nYLQJ3gscB2sU2bNun48eN67rnntHnzZtdyp9Mpi8XiemwYhiwWi+vf8138uCKys/PldBpVbxwAUCmVCWEOR54HOwHqDqvVcsmTTDUW2FatWqW9e/dq8ODBKiwsVFZWlsaPH6+JEyfK4XC41svKylJwcLCaNGmivLw8lZeXy8vLSw6H44LhVQAAgKtFjQW22NhY1/83b96s119/Xa+++qokyc/PT6mpqerevbtWrFihiIgI+fj4qEePHkpKStKgQYOUmJioiIiImmoXAADANExxH7b4+HjFxsYqMjJShYWFGjlypCQpJiZGCQkJGjBggLZu3arx48fXcqcAAAA1z2IYxhU9wYs5bABQs2y2QA2btOSy6y2NG84cNuBnl5vDZoozbAAAAPjvCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAm59HAlp+fr4EDB+ro0aOSpI8++kgDBw7UoEGD9Nxzz6mkpESSlJaWpqioKPXr10/R0dEqKyuTJB07dkzDhw9XZGSkxo4dq4KCAk+2CwAAYEoeC2w//PCDhg4dqvT0dEnSwYMHtXDhQn344YdauXKlnE6nli5dKkmaOHGipk+frtWrV8swDCUkJEiSZsyYoWHDhik5OVmdO3fW/PnzPdUuAACAaXkssCUkJCgmJkbBwcGSJF9fX8XExCggIEAWi0W/+c1vdOzYMWVkZKioqEhhYWGSpKioKCUnJ6u0tFRbtmxRv379LlgOAABwtfH2VOFZs2Zd8Dg0NFShoaGSpJMnT2rJkiWKjY1VZmambDabaz2bzSa73a6cnBwFBATI29v7guUAAABXG48Ftv/GbrdrzJgxuueee9SzZ0+lpqbKYrG4vm4YhiwWi+vf8138uCKaNg2ods8AAM+w2QJruwWgTqjRwLZ//36NGTNGDzzwgEaPHi1JCgkJkcPhcK2TlZWl4OBgNWnSRHl5eSovL5eXl5ccDodreLUysrPz5XQabvsZAACXVpkQ5nDkebAToO6wWi2XPMlUY7f1yM/P10MPPaSnnnrKFdaks0Olfn5+Sk1NlSStWLFCERER8vHxUY8ePZSUlCRJSkxMVERERE21CwAAYBo1Ftg++eQTZWVladGiRRo8eLAGDx6s1157TZIUHx+v2NhYRUZGqrCwUCNHjpQkxcTEKCEhQQMGDNDWrVs1fvz4mmoXAADANCyGYVzR44UMiQJAzbLZAjVs0pLLrrc0bjhDosDPTDMkCgAAgKohsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJicRwNbfn6+Bg4cqKNHj0qSUlJSNGjQIPXt21dz5851rZeWlqaoqCj169dP0dHRKisrkyQdO3ZMw4cPV2RkpMaOHauCggJPtgsAAGBKHgtsP/zwg4YOHar09HRJUlFRkaZOnar58+crKSlJO3bs0Pr16yVJEydO1PTp07V69WoZhqGEhARJ0owZMzRs2DAlJyerc+fOmj9/vqfaBQAAMC2PBbaEhATFxMQoODhYkrR9+3a1bt1arVq1kre3twYNGqTk5GRlZGSoqKhIYWFhkqSoqCglJyertLRUW7ZsUb9+/S5YDgAAcLXx9lThWbNmXfA4MzNTNpvN9Tg4OFh2u/0Xy202m+x2u3JychQQECBvb+8LlgMAAFxtPBbYLuZ0OmWxWFyPDcOQxWL5r8vP/Xu+ix9XRNOmAVVvGgDgUTZbYG23ANQJNRbYQkJC5HA4XI8dDoeCg4N/sTwrK0vBwcFq0qSJ8vLyVF5eLi8vL9f6lZWdnS+n03DLzwAAuLzKhDCHI8+DnQB1h9VqueRJphq7rUfXrl118OBBHTp0SOXl5Vq1apUiIiIUGhoqPz8/paamSpJWrFihiIgI+fj4qEePHkpKSpIkJSYmKiIioqbaBQAAMI0aO8Pm5+en2bNna9y4cSouLlbv3r0VGRkpSYqPj9e0adOUn5+vTp06aeTIkZKkmJgYTZkyRW+++aZatGihV155pabaBQAAMA2LYRhX9HghQ6IAULNstkANm7TksustjRvOkCjwM9MMiQIAAKBqCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATK5Cgc1ut/9i2b59+9zeDAAAAH7pkoEtNzdXubm5evjhh3Xq1CnX46ysLD3xxBM11SMAAMBVzftSX3zmmWe0adMmSVLPnj3/803e3urXr59nOwMAAICkywS2hQsXSpKee+45xcbG1khDAAAAuNAlA9s5sbGxysjI0KlTp2QYhmt5p06dPNYYAAAAzqpQYJs3b54WLlyopk2bupZZLBZ98cUXHmsMAAAAZ1UosCUmJuqzzz5T8+bNPd0PAAAALlKh23q0aNGCsAYAAFBLKnSGLTw8XHFxcbr99tvl7+/vWs4cNgAAAM+rUGBbtmyZJCk5Odm1jDlsAAAANaNCgW3t2rWe7gMAAAD/RYUC26JFi351+YMPPujWZgAAAPBLFQpse/bscf2/pKREW7ZsUXh4uMeaAgAAwH9U+Ma557Pb7YqOjvZIQwAAALhQhW7rcbHmzZsrIyPD3b0AAADgV1R6DpthGNqxY8cFn3oAAAAAz6n0HDbp7I10J02a5JGGAAAAcKFKzWHLyMhQWVmZWrdu7dGmAAAA8B8VCmyHDh3S448/rszMTDmdTgUFBemtt97Sdddd5+n+AAAArnoVuuhg5syZGjNmjLZs2aLU1FSNHTtWM2bM8HRvAAAAUAUDW3Z2tu6++27X43vuuUc5OTkeawoAAAD/UaHAVl5ertzcXNfjkydPeqwhAAAAXKhCc9hGjBihIUOGqH///rJYLEpKStKf//znKm90xYoVevvttyVJERERmjx5slJSUhQbG6vi4mL1799fEyZMkCSlpaUpOjpaBQUF6tGjh2bMmCFv7wq1DQAAcEWo0Bm23r17S5JKS0u1f/9+2e123XnnnVXa4JkzZzRr1iwtXrxYK1as0NatW7V27VpNnTpV8+fPV1JSknbs2KH169dLkiZOnKjp06dr9erVMgxDCQkJVdouAABAXVWhwDZlyhQNHz5cEydO1EsvvaTx48dr6tSpVdpgeXm5nE6nzpw5o7KyMpWVlSkgIECtW7dWq1at5O3trUGDBik5OVkZGRkqKipSWFiYJCkqKkrJyclV2i4AAEBdVaGxxZycHI0cOVKS5Ofnp1GjRikxMbFKGwwICNBTTz2l/v37q169errpppuUmZkpm83mWic4OFh2u/0Xy202m+x2e6W217RpQJX6BAB4ns0WWNstAHVChQJbeXm57Ha7mjdvLknKysqSYRhV2uDu3bv1z3/+U+vWrVNgYKCeffZZpaeny2KxuNYxDEMWi0VOp/NXl1dGdna+nM6q9QoAqLzKhDCHI8+DnQB1h9VqueRJpgoFtlGjRumPf/yjevXqJYvFopSUlCp/NNXGjRsVHh7u+izSqKgoLVy4UF5eXq51HA6HgoODFRISIofD4VqelZWl4ODgKm0XAACgrqrQHLZ7771XixYt0m9/+1t17txZCxcu1KBBg6q0wY4dOyolJUWFhYUyDENr165V165ddfDgQR06dEjl5eVatWqVIiIiFBoaKj8/P6Wmpko6e3VpRERElbYLAABQV1X4/hgdO3ZUx44dq73BW265Rbt27VJUVJR8fHx0ww03aNy4cbr55ps1btw4FRcXq3fv3oqMjJQkxcfHa9q0acrPz1enTp1cc+kAAACuFhajqpPR6gjmsAFAzbLZAjVs0pLLrrc0bjhz2ICfXW4OW4WGRAEAAFB7CGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmR2ADAAAwOQIbAACAyRHYAAAATI7ABgAAYHIENgAAAJMjsAEAAJgcgQ0AAMDkCGwAAAAmVyuBbe3atYqKilL//v31wgsvSJJSUlI0aNAg9e3bV3PnznWtm5aWpqioKPXr10/R0dEqKyurjZYBAABqTY0HtiNHjigmJkbz58/XypUrtWvXLq1fv15Tp07V/PnzlZSUpB07dmj9+vWSpIkTJ2r69OlavXq1DMNQQkJCTbcMAABQq2o8sH3++ecaMGCAQkJC5OPjo7lz56pevXpq3bq1WrVqJW9vbw0aNEjJycnKyMhQUVGRwsLCJElRUVFKTk6u6ZYBAABqlXdNb/DQoUPy8fHRY489puPHj+vWW29V+/btZbPZXOsEBwfLbrcrMzPzguU2m012u72mWwYAAKhVNR7YysvLtXXrVi1evFj169fX2LFj5e/vL4vF4lrHMAxZLBY5nc5fXV4ZTZsGuK13AIB72WyBtd0CUCfUeGBr1qyZwsPD1aRJE0nSHXfcoeTkZHl5ebnWcTgcCg4OVkhIiBwOh2t5VlaWgoODK7W97Ox8OZ2Ge5oHAFxWZUKYw5HnwU6AusNqtVzyJFONz2Hr06ePNm7cqNOnT6u8vFwbNmxQZGSkDh48qEOHDqm8vFyrVq1SRESEQkND5efnp9TUVEnSihUrFBERUdMtAwAA1KoaP8PWtWtXjRkzRsOGDVNpaaluvvlmDR06VO3atdO4ceNUXFys3r17KzIyUpIUHx+vadOmKT8/X506ddLIkSNrumUAAIBaZTEM44oeL2RIFABqls0WqGGTllx2vaVxwxkSBX5muiFRAAAAVA6BDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwOQIbAAAACZHYAMAADA5AhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwORqNbDNmTNHU6ZMkSSlpKRo0KBB6tu3r+bOnetaJy0tTVFRUerXr5+io6NVVlZWW+0CAADUiloLbF9//bWWL18uSSoqKtLUqVM1f/58JSUlaceOHVq/fr0kaeLEiZo+fbpWr14twzCUkJBQWy0DAADUiloJbLm5uZo7d64ee+wxSdL27dvVunVrtWrVSt7e3ho0aJCSk5OVkZGhoqIihYWFSZKioqKUnJxcGy0DAADUmloJbNOnT9eECRPUsGFDSVJmZqZsNpvr68HBwbLb7b9YbrPZZLfba7xfAACA2uRd0xv8+OOP1aJFC4WHh2vZsmWSJKfTKYvF4lrHMAxZLJb/urwymjYNcE/jAAC3s9kCa7sFoE6o8cCWlJQkh8OhwYMH69SpUyosLFRGRoa8vLxc6zgcDgUHByskJEQOh8O1PCsrS8HBwZXaXnZ2vpxOw239AwAurTIhzOHI82AnQN1htVoueZKpxgPbokWLXP9ftmyZvv32W82YMUN9+/bVoUOHdM0112jVqlW65557FBoaKj8/P6Wmpqp79+5asWKFIiIiarplAACAWlXjge3X+Pn5afbs2Ro3bpyKi4vVu3dvRUZGSpLi4+M1bdo05efnq1OnTho5cmQtdwsAAFCzLIZhXNHjhQyJAkDNstkCNWzSksuutzRuOEOiwM8uNyTKJx0AAACYHIENAADA5AhsAAAAJkdgAwAAMDkCGwAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5AhsAAAAJkdgAwAAMDkCGwAAgMkR2AAAAEzOu7YbAOA+gQ395e/nc8l1iopLlXe6qIY6AgC4A7QIOPYAACAASURBVIENqEEVCVRS1UOVv5+Phk1acsl1lsYNV54IbABQlxDYgBpUkUAlEaoAABdiDhsAAIDJEdgAAABMjsAGAABgcgQ2AAAAkyOwAQAAmByBDQAAwOS4rQeACuGmvDALjkVcjQhsACqEm/LCLDgWcTUisAEAaoWzrFQ2W+Al1ykrKVbOqZIa6ggwLwIbcBGGW2oe+/zqZPX2UWrcmEuu033SO5IIbACBDbgIwy01j30OAJfGVaIAAAAmxxk2ALgKMQwN1C0ENgC4CjEMDdQtDIkCAACYHIENAADA5BgSRZ3E/BsAwNWEwIY6ifk3uBrwxgTAOQQ2ADAp3pgAOIfABuCKVpGzVBJnqgCYG4ENHsNwDsygImeppKqfqeI4R0VxrKA6CGzwGIZzcDXgOEdFcaygOghsAAD8jLNgMCsCGwAAP+MsGMyKwHYVYzI2AKAiOPNY+whsVzFPT8YGAFwZOPNY+2rlo6lef/113XXXXbrrrrsUFxcnSUpJSdGgQYPUt29fzZ0717VuWlqaoqKi1K9fP0VHR6usrKw2WgYAAKg1NR7YUlJStHHjRi1fvlyJiYnauXOnVq1apalTp2r+/PlKSkrSjh07tH79eknSxIkTNX36dK1evVqGYSghIaGmWwYAAKhVNR7YbDabpkyZIl9fX/n4+Oi6665Tenq6WrdurVatWsnb21uDBg1ScnKyMjIyVFRUpLCwMElSVFSUkpOTa7plAACAWlXjc9jat2/v+n96err+/e9/a8SIEbLZbK7lwcHBstvtyszMvGC5zWaT3W6v1PaaNg2oftOQzRZI7Rqs7en61K75+tSu+R7q6n4xwz6vqrrcu9nV2kUHe/fu1aOPPqpJkybJy8tL6enprq8ZhiGLxSKn0ymLxfKL5ZWRnZ0vp9NwV9s1zpNX5lTmD8vhyPNYfWrXfH1q13x9alfdxT2YoXez1fa0utx7XWG1Wi55kqlWAltqaqqefPJJTZ06VXfddZe+/fZbORwO19cdDoeCg4MVEhJywfKsrCwFBwfXRsu1hitzAABAjQe248eP6y9/+Yvmzp2r8PBwSVLXrl118OBBHTp0SNdcc41WrVqle+65R6GhofLz81Nqaqq6d++uFStWKCIioqZbvizuTwMAqE3cV/PKV+OBbeHChSouLtbs2bNdy+6//37Nnj1b48aNU3FxsXr37q3IyEhJUnx8vKZNm6b8/Hx16tRJI0eOrOmWL4uzYACA2sR9Na98NR7Ypk2bpmnTpv3q11auXPmLZR07dtQnn3zi6bYAAABMq1ZunAsAAICK46OpAABArWEeeMUQ2AAAQK1hHnjFMCQKAABgclfNGTZOuQIAgLrqqglsnHIFAAB1FUOiAAAAJkdgAwAAMDkCGwAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5AhsAAAAJkdgAwAAMLmr5qOpAADA1eVK+hxxAhsAALgiXUmfI86QKAAAgMkR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENAADA5AhsAAAAJkdgAwAAMDkCGwAAgMkR2AAAAEyOj6YCrjLOslLZbIGXXKespFg5p0pqqCMAwOUQ2ICrjNXbR6lxYy65TvdJ70gisAGAWRDYABPiLBgA4HwENsCE6upZME8GzYrUrk59ADAzAhsAt/Fk0KxI7erUxy8RkgHzILDhisXZHqB6CMmAeRDYcFm1HXyqWpuzPagM5g1eWfh91jz2uWcR2HBZtR18CD2oCZ48Fuvqm566jOeWmsc+9ywCGwB4GG96UFGenm5BwK+7CGw1pLbfYVenPgDgP2p7Kod09QV8giaB7QK1/UfIfCoAML+6GnrqMvY5ge0CHBAAAKAiAhv6y9/P57LrFRWXKu90UbW3R2ADAABXrapOK/L389GwSUsu+31L44YrTwQ2AACAKqsr04qstbblSvj00081YMAA9e3bV0uWXD7NAgAAXElMf4bNbrdr7ty5WrZsmXx9fXX//ferZ8+euv7662u7NQAAgBph+sCWkpKi3//+92rcuLEkqV+/fkpOTtYTTzxRy50BAABcWkXvQHEqr/SS65g+sGVmZspms7keBwcHa/v27RX+fqvV4vp/s6AGl13ft2HTStWsS7V/rX5Fale0fl3dL+zzK6v2r9Vnn7PPr7Tav1affW7OfW719tGPCyZfcp0bHpsjq7XskutYDMMwLt9i7XnzzTdVXFys8ePHS5ISEhK0Y8cOzZw5s5Y7AwAAqBmmv+ggJCREDofD9djhcCg4OLgWOwIAAKhZpg9sf/jDH/T111/r5MmTOnPmjD777DNFRETUdlsAAAA1xvRz2Jo3b64JEyZo5MiRKi0t1b333qsuXbrUdlsAAAA1xvRz2AAAAK52ph8SBQAAuNoR2AAAAEyOwAYAAGByBDYAAACTI7ABAACYHIENV5TCwsLabqHK9u/fX6f7B64m3GDhP87ti7q2T2qqb3fVv6oD25EjR5SWlqbS0lKVl5fXdjuVsnv3bm3cuFHp6ekqKSlxa+309HRt375dZ86ccWtd6WzfSUlJbq8rSV988YVeffVVj4We3bt3a/369Tp48KCcTqdba6ekpOjxxx+/4FM93GX37t3asGGDMjIy3N63dDZobt26VYcPH/bIMVNT6tqLTU3w5D6pq29ODh8+LEmyWH75mZV1iTt/t+d+l+f2iTufZ0pLL/2B6NWRk5NzwTbcfbzn5+e7tZ7pb5zrKZ999pnmzp2rkJAQNWnSRJ06ddKf/vQnBQYGuqX+rl27ZLPZLvjgendZs2aNXnvtNV177bXy8/NT3759FRkZ6bba8+bNU1BQkHx8fPTOO++4pa5hGCovL9fHH3+srVu3ysfHR3feeafra9V98lu3bp1ef/11zZgxQ/Xr13dHyxdYu3atXnnlFQUFBem2227TiBEjZLW65/3Oxo0b9dJLL6mwsFDbtm1T69at3VJX+s+xEhoaqtLSUr344otq3ry5W+u/8cYbateunXx8fFSvXj2NHz9ejRo1cts2jh49KqvVqqCgINWrV0+Se44ZSTp48KCys7PVo0cPWSwWt9U9Z+/evSovL5e/v7/atGnjtrrS2eBgsVjUuHFjtz1vnbN9+3Y5nU6FhYW5fZ9I0vr167Vr1y6NHDlSDRpU7APLK2rPnj0qLy+Xr6+vrrvuOrfW/uabbzRjxgwtXLhQLVu2dGvtPXv2yOl0qkGDBmrVqpVba0vSgQMHJElWq1Vt2rRx2/H+5ZdfKjExUY0bN1anTp0UERGh5s2by+l0Vvs58tyJiXvuuUdBQUHVqnWxr776SosWLVJISIhatGihkSNHqnHjxm6rv379en3yyScKCgrS4MGD1b1792rvE6+//e1vf3Nbh3VEYWGhXn/9dU2bNk0PP/ywysvLtWfPHm3btk1du3aVr69vlWuf+wOYM2eOPvnkE/Xu3dutAcLhcCguLk6zZ8/WiBEjtG/fPqWkpGjAgAFurT169Gj9/e9/V7NmzeTr6ytfX195e1c931ssFlmtVmVmZio9PV25ubnKzc3V//zP/1T7CWP37t2aOnWqPvzwQ+3fv19vvfWWPvvsMxUUFCg0NFR+fn7Vql9SUqK33npLTz/9tMaOHauWLVvq2LFjKi0tlY+PT7X2y8aNG/Xiiy8qNjZWXbp0UVZWlm688Ua3PJEWFhbqtddeU3R0tB588EElJSWpfv36qlevnry8vKp1nEtSdna25syZoxkzZmj48OGyWq1KSEjQTz/9pJtuukn+/v7Vqi+dfWM1a9Ysbdy4Ubt379aRI0fUpUuXar/YnPveV199VZs2bVJoaKhCQkLcGtq++OILvfjii9q1a5d++OEHSXJbgFizZo1efPFFbdy4UUeOHNHvfvc71wtBdfsvKSnRggUL9PHHH+uGG25wfXazu/bLunXr9PLLL2vMmDEKDQ294GvV3cbatWs1c+ZMbdmyRQ0aNFCHDh3k5eVV3ZYlSRs2bNALL7ygEydO6Pbbb1eLFi3cUlc6e6zExsZqx44d2rlzpzp27KiGDRtKcs9+X7NmjeLi4vTTTz/p22+/1a5duxQeHl7t433Pnj16+umnNX78eHl5eeno0aNauXKlwsLC1LBhw2r3np6erilTpigoKEjXXnut215Lf/zxR0VHR2vKlCkKCQlRWlqagoOD1bJlS7fs7x9//FFTpkzR+PHjtW/fPm3btk19+/ZVeXm5rFarnE5nlbZxVQ6JWiwWnTx5UseOHZMkRUZGqk+fPiouLlZiYmK1hkfP/RKsVqv27t2rSZMmKSsryy19S5K3t7dKS0tdp3Dvu+8+5ebmuuXUq7e3twoLCxUQEKDs7GwdOXJE//jHP/TCCy/orbfeUnFxcbW3ERQUpMDAQN14443aunWr3njjDS1cuFBS1U9HBwQEyOl06vXXX9f777+v+++/X61atVJaWpqWLl3qluHuI0eO6NSpU8rPz9f48eMVHx+vF154QQsWLKjyMGBJSYm2bdum559/XjfccIP8/Py0bNkyZWZmuuWF8dxxvm3bNmVnZ2vPnj1KSkpyhay8vLxq1ff29lZJSYnr2Lvtttv0hz/8QYGBgVq8eHG193teXp7ee+89Pf/883r55ZfVs2dPff3113rjjTdcP19VnfvevLw87d+/X1988YW2bNni+lp1h0bsdrvefPNNxcfHKy4uTm3atHGFtuo6V3vOnDmKjY3Vhg0btHfvXtntdknVH6rz9fVVkyZN5OXlpdjY2Av2S3WlpaVpzpw5euutt5STk6O//e1vmjt3rj799NNqb+PMmTNasmSJXnjhBS1evFh33nmnTp48qRMnTrjWqervdf369YqLi9P//u//6rnnntP+/fslyS3PLadPn9bChQs1Y8YMzZs3T5mZmbLb7a7Xp+oej5mZmXrzzTf1/PPPKzY2VgMHDtQ///lPzZkzx1W/qkpLS3XrrbcqPDxcw4YN03333ae2bdtqzpw5cjgc1T5mMjMz1aFDB23cuFGJiYk6efJkteqdc+zYMfXu3Vs33XST+vbtq/z8fKWkpEhyz3F+5MgRRUREKDw8XPfdd5++/fZbvfDCC5o9e7aOHz9e5bNsV+UZNh8fH1ksFm3evFktWrSQzWZTaGiocnNztWnTJt12223VOmsiSTt37lR0dLTS0tL0z3/+U7169XLLuwMfHx/5+vqqRYsWaty4sTIyMrR8+XINGTJEvr6+Onz4sOsMSlVqt2nTRh07dtSOHTt000036dlnn1WDBg20efNm3XjjjdUeegkICNCJEyf04IMP6vvvv9fixYvVoUMH17u9qmjYsKE6deqkd999V/fee6/uuOMOde/eXSUlJdq4caNuv/32av0+vby8VFRUpN27d7uOj0mTJql+/fravHmzOnfuXKXT9V5eXrrxxht1zTXXqLy8XM2bN9eBAwd07bXXqnnz5q53Y1Xl4+Oj5s2ba/78+UpOTtaAAQM0c+ZMXX/99dq6datCQ0OrdZbA19dXJ06c0I8//qgzZ864nlB79eqlr776SnfccYd8fHyqXP/MmTP69NNPFRUVJZvNphYtWigkJETr1q3T6dOn1blz5yrV3bVrlwzDUIMGDXTw4EH169dPGRkZ2rdvn+rXr6+WLVtW+8xDdna2/v3vf+uhhx5SvXr1ZBiGVq5cqf79+8vb27taLwoOh0MrVqzQE088obKyMs2bN0+HDx/W+vXrdezYMXXv3r3Ktc/9zEeOHFFQUJC6d++ujz76SMXFxdq7d686duxYrf2yf/9+paamqmHDhvroo4/Ut29fnTx5UqmpqbLb7QoLC6ty78XFxUpISNDw4cPldDr1xBNPaNOmTdq0aZOOHj2q7t27V6nv/Px8LV++XA888IC6dOmib775RqtWrdK9997rlmkRhYWFSk5Odk0Refvtt3XgwAF9/vnn2rFjhyIiIqp1vOTk5Ohf//qXhg8frgYNGqhRo0bKzc1VWlqasrOz1a1bt0rX3L17t/bu3auCggK9++67atWqldq1a6dGjRqpZcuWOnz4sE6fPq3f/va3lT5edu/erd27d0uSjh8/rl69eunWW2/V0qVLdebMGbVp08Y1NaIqfR88eFAOh0Pbtm3TLbfcIn9/f+3atUulpaW6+eabJZ19M12V19Bz9XNycpSdna2IiAi9++676tatmzp16qSTJ09q+/bt6tmzpywWS6V/r1dlYJPOfqj8rl27tHfvXjVp0kTBwcHq2LGjPvroI91www1q1qxZteo3bdpU1157rSIjI5WSkqJly5a5JbRZrVa1bdvW9Q7Ybrfriy++0KhRo7R8+XIlJCTolltuqdIwoNVqdc2dCA0N1W9+8xtJUtu2bZWYmKjOnTtXe/7TmTNn9Pnnn6ukpETLli3TnXfeqWPHjsnpdKp9+/ZVrhsSEqJGjRqpTZs2atmypaxWq6677jp9+OGH6tatm5o0aVKtvm02m7766ivt2bNHvXr1Ups2bdS2bVstX75c7du31zXXXFOluueeFKxWq3x9fbVlyxatWLFCd999d7VOnZ/TunVrDR48WHv37lWPHj3Url07NWvWTJ999pnatGlTrXlVFotFbdq0kd1u19dffy2r1apZs2bp+uuv10cffaTw8PAqzWU7F6iaNm2qvXv36vPPP1fPnj0VGBiooKAgeXt7a/v27frd735XqUB4/nSFjz/+WL169VJRUZH69++vsLAwbd68+RehrSq9S1LLli116tQpNW7cWEFBQTp69KhSUlI0ZMgQWa1WHTt2rNJvfs7VDg0NVUBAgDp27KhvvvlG4eHhevzxx9WsWTNt2LBBXbt2rVJtwzAUEBAg6ezvdv/+/XrooYe0bds2vfPOOwoLC1NYWFiV9svOnTtlsVjUoUMH5eXlKS4uTtHR0brjjjsUFhamgIAAfffdd+rZs2elh+rP9R4UFKS0tDR9++232rBhg+688049/PDDatasmdatW6dOnTpVeo5SWlqavL291adPH9fc0k6dOmnz5s1q3ry5WrRoUeUAe67vZs2aad++ffq///s/ffDBBxoyZIj++te/qmvXrkpMTFS7du2q9Lx77nhp0aKFduzYoQ0bNqhdu3ZasmSJLBaL+vfvr9TUVIWHh1dqn69Zs0bPP/+8Dhw4oJycHHXq1Enfffeda55m48aNtX//fu3fv1+9e/eu1L45V/vw4cP68ccf1aRJE91yyy1q06aNrr/+ev3jH/9QcXFxlYZHz+/7xIkTuvnmm10h/twQeo8ePZSUlKTNmze7pl5Upb7dbtett96qdu3aqX379urTp4/at2+vM2fOaN++fbr99tsZEq2MJk2auN6Jvf/++0pKSlJSUpJOnjxZ7bAmSe3bt3f9Ebz00ksKDQ3VI488ouzs7GrX9vPzc73Q+/v764YbbtDnn3+uxYsX67HHHnPNfaiO0tJS/fTTT9q+fbvWrVsnh8OhkJCQatcNCgqS0+nUvHnzNGXKFE2dOlW33nprtc4KSGcDz6BBg9StWzdt3rxZ3333ndasWaOcnBy3TFZt0aKFHnnkEQUGBmrt2rVav369PvvsMx07dswtFwmcG/KYNGmSGjVqpFmzZkmSW97Bn3tx//rrr7V582atWbNGBw8erFZAPqd58+YaPXq04uPj9dhjj2n//v36/PPPdfr06UqHhnP74N1339WTTz6prKwsjRgxQkFBQZo/f75yc3PVoEED9ezZUwcPHqz0kO7F0xViYmJ04403SpICAwP1+OOPy8vLS8uWLdP3339f5d6feuopORwOPfDAA7r++uvl5eXlmjNotVqVmJioefPmVXgaw6/VHjx4sCTplltu0eDBg9WgQQN169ZNRUVFlRqmO7/2hAkTXFcpO51O5ebmavPmzfr+++81aNAgJSUladu2bRWufX79RYsWady4ccrOztZDDz2ksWPHui428Pf3V+fOnXX06NFKXTn6a/vlj3/8o7y8vLR792517dpVDRs2VM+ePVVaWqqysrJK137nnXf0zDPP6NSpU67lFotFDRs21Jdffimp8kOWFx/n2dnZmjRpkt59912Fh4dr4MCB8vb21nXXXadmzZpV+jng1/bLiBEj5Ovrq5dffll2u12TJ09Wz549dfTo0UpNdXE4HPr73/+uV199VQsWLFBQUJC2bNmiLl26KDEx0XUHgMDAQJ06dUrFxcUV3jfn154/f75CQ0OVkpKixo0bq7y8XF26dNHEiRP16aefKikpqVJXol5c+7rrrtOGDRtczwl2u10tW7bUmjVrtGDBAkVERFRqv19cv02bNlq2bJmks2/2z/1dFRQU6PTp0yoqKqrSMPdVG9iksy/CDz30kMLDw7VixQqtW7dOc+bMUdOmTd22jXMHVWxsrDp37qyioiK31ZbOnqFZvny55s6dq5deesl1Vqy6nE6nvvvuO8XGxmrp0qWaNWuWW6549fLy0qhRoxQTE6Pw8HBJ0sCBA91y5aLValVhYaF27typuLg4ffjhh5ozZ061z66d07p1a8XExKhVq1ZavHixkpOT9dJLL7ll8rHFYnEdKyNGjFBJSYlyc3OrXfec22+/XY0aNdKbb76phIQEvfjii7+Y8F0dFotFu3fv1uTJk/XJJ59o9uzZlQ7K5weqAwcO6LnnnpOvr68GDhwop9OpmJgYHT58WFu2bFFRUVGVh1tbtmypjz76SA0bNtQzzzyjzMxMSWffxD388MNq0qRJpa/SO7/3ffv2acqUKSooKHC9sTIMQx06dNDq1au1aNEijRkzxnU2q7K1J0+efMHtX9LS0pSfn6+tW7cqLy+vUhd7XBxiJ0+erKysLHXu3Fm5ubmaPn26JkyYoJkzZ2rw4MGVfg64+Hf6zDPPyOFwaOzYserevbs2bdqkvLw8bd++XUVFRZWaunB+7f3792vatGkKCQnR73//e1mtVi1YsEB2u11ffvmlMjMzK7y/z6/t5eXl2i/n5mT5+vrq4Ycf1r///W+9//77F6xf2b4PHDigp59+Wg6HQ82aNVNgYKAWLFigkydP6vPPP9fhw4cr/fx18e80OjpaTZs21YwZMxQfH68JEyZIOnvFq2EYlRr6+7U51AEBAerVq5f69Omj+Ph4Pffcc3r99df1yCOPyM/Pr8L75uLaQ4YM0cmTJ5Wfn+8KTzfccINmzJih2267rVKB6uLaf/rTn1xzkqWz0zvefPNNLVq0SC+//LLatm1b4dq/Vv++++7T6dOnlZeXp6NHj2rSpEmaPHmy5s2bp8cff1z+/v5VGzkxYBiGYRQXFxslJSUeqV1eXu6RuoZhGIWFhcb48eONAwcOuL12aWmpUVBQYOTl5bm9tmEYhtPp9EjdsrIyIz8/32N9G8bZ46W4uNgjtYuKioz8/Hy313U6nUZhYaFHap9TUFBQ7fpz58419u/fbzz77LPG6NGjjRMnThjHjx835syZY4wePdp48MEHjV27dlW5/p49e1y/u2effdZ44IEHjMzMTNfXq/P3eq73SZMmXVB3/fr1RocOHYy7777b2Ldvn9tqFxQUGK+88opx9913G0OGDDHS0tLcUvvEiRPGe++9Z2zevNm1TllZWZVq/1p9u91uZGVlGc8//7yr96r+Ts8/XsaMGWOcOHHC2LJli/HXv/7VGDlypDFq1Ci37Re73e762pdffmmMHj3ayMvLq9Jz2fm1R4wYYWRmZhppaWnGc889ZwwdOtS4//77q9z3+fUnTpzoqm8YhvHee+8ZAwcONIYOHWrs3LmzUjVLS0uN5cuXG/v27TPKy8uNtLQ0o3///kZRUZFhGIaxdetWIy0tzXA4HJXu9+Lau3fvNiIjI13PJ+np6caZM2cqXbcitVeuXGnceeedVf7b/G/1z/WbmJhorFmzxjh+/HiV6p9DYLsCeCo4ALXh/ED19NNPG6NGjXK9ABQWFrpeHNxlypQpxt13312lF5mL/VoYzMrKMk6dOmU8+uijVX5B+LXaf/7zn10vwidOnDBOnz7tltpPP/208eijj7rCiTvecF7c+6hRo1y9Hz9+3K29jx492lU7Ly+vyi/yv9b3+SG8uLi4Wm8Kz6/9zDP/v727d2klDcM4fA+sFlYiCIpYWMUuvfgH+JFJnC6FWIo2QlDMIEljaWNhLxZWBtJoJVbpUmuhqNiIKMqAjUqUPFusJ+uRXZc9mTGTnN9V5YPcPHkTyJ2ZgXf5p7nv7++b/sP5efa5ubnGd/zu7u6X819eXhrl/eTkxKampszMrFQq2fLyclOf5efs6elpMzMrl8uWy+Xs8fEx9OxSqWTFYvGnP21h5P9Yl3K5bL7vh3IAgcIGINby+byl02l7eHgINfdjESkWi3Z9fR1qvtnfZbCZH8h/k8/nzfO80NfFzGx1ddVmZmZCKbFf5Uc1eyaTiSTb9/1I544q2+yv70vY63JxcWG+79vh4aF5nmdnZ2dtmX16ehpadpT5FDYAsfQdhSqqyxU+5hYKBbu5uYkkO+x1iXrN23X2ds2OOv/q6soSiYRNTk42dfS4k7KjzHfM2EAPQDyFsb1Nq0Q5e7tmR51P9vfmPz8/a21tTUtLS//7Qv1OzY4yn8IGAAB+Sa1Wa3qbu07LjiqfwgYAABBz7XmuAQAA4DdCYQMAAIg5ChsAAEDMUdgAQFK1WlUqlWr1GADwjyhsAAAAMUdhA4B3T09PyuVyymQympiYaGyovrKyolQqJdd1tbGxobe3N0lSIpFQEASN1/+4X61WlU6nlc1m5bquarVaq94SgA7xR6sHAIC4uL291ebmppLJpHZ2drS1taWBgQH19vZqf39fr6+vWlxc1Pb2tubn57/MOj8/19HRkYaGhr5pegCdjCNsAPBueHhYyWRSkjQ6OqogCFSpVDQ7OyvHcdTd3a1sNqtKpfKfWYODg5Q1AKGhsAHAu66ursZtx3FkZqrX63IcV0/usgAAAKlJREFUp/F4vV5vnBL96PNpz56enugGBfDbobABwBfGx8e1u7srM1OtVtPe3p7GxsYkSX19fTo+PpYkHRwctHJMAB2OwgYAXygUCgqCQK7rynVdjYyMaGFhofHc+vq6PM/T5eWl+vv7WzwtgE7FXqIAAAAxxxE2AACAmKOwAQAAxByFDQAAIOYobAAAADFHYQMAAIg5ChsAAEDMUdgAAABijsIGAAAQc38CP7hK33p9tMQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize  = (10,8))\n",
    "sns.countplot(df.hour, hue = df.engaged)\n",
    "plt.xticks(rotation = 45);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1500 hours has the highest number of Impression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Invariant metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_metrics(cols, df = df, experiment = None):\n",
    "    ''' Compare some invariant metrics between the groups '''\n",
    "    \n",
    "    if experiment:\n",
    "        print(f'metrics for {experiment} group')\n",
    "        df[df.experiment == experiment][cols].value_counts(normalize = True) #Get metric for a particular experiment\n",
    "    else:\n",
    "        print(f'metrics for control group')\n",
    "        print('-------------------------')\n",
    "        ex1 = df[df.experiment == 'control'][cols].value_counts(normalize = True) #Get ,=metric for control\n",
    "        print(ex1)\n",
    "        print(f'\\nmetrics for exposed group')\n",
    "        print('-------------------------')\n",
    "        ex2 = df[df.experiment == 'exposed'][cols].value_counts(normalize = True) #Get metric for exposed group\n",
    "        print(ex2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "6    0.924343\n",
      "5    0.075657\n",
      "Name: platform_os, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "6    0.969795\n",
      "5    0.029955\n",
      "7    0.000250\n",
      "Name: platform_os, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('platform_os')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is difference in the use of O between groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "Chrome Mobile                 0.591992\n",
      "Facebook                      0.137804\n",
      "Samsung Internet              0.120855\n",
      "Chrome Mobile WebView         0.071727\n",
      "Mobile Safari                 0.060427\n",
      "Chrome Mobile iOS             0.008352\n",
      "Mobile Safari UI/WKWebView    0.007369\n",
      "Pinterest                     0.000491\n",
      "Chrome                        0.000246\n",
      "Opera Mini                    0.000246\n",
      "Android                       0.000246\n",
      "Puffin                        0.000246\n",
      "Name: browser, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "Chrome Mobile                 0.535197\n",
      "Chrome Mobile WebView         0.298802\n",
      "Samsung Internet              0.082876\n",
      "Facebook                      0.050674\n",
      "Mobile Safari                 0.022716\n",
      "Chrome Mobile iOS             0.004244\n",
      "Mobile Safari UI/WKWebView    0.003495\n",
      "Opera Mobile                  0.000749\n",
      "Chrome                        0.000499\n",
      "Pinterest                     0.000250\n",
      "Edge Mobile                   0.000250\n",
      "Firefox Mobile                0.000250\n",
      "Name: browser, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('browser')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is difference in the use of browser between groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "Generic Smartphone    0.592238\n",
      "iPhone                0.076394\n",
      "Samsung SM-G960F      0.025547\n",
      "Samsung SM-G973F      0.023581\n",
      "Samsung SM-G950F      0.016212\n",
      "                        ...   \n",
      "C2105                 0.000246\n",
      "Samsung SM-A908B      0.000246\n",
      "VKY-L09               0.000246\n",
      "NEM-L51               0.000246\n",
      "XiaoMi Redmi S2       0.000246\n",
      "Name: device_make, Length: 169, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "Generic Smartphone     0.582127\n",
      "iPhone                 0.030454\n",
      "Samsung SM-G960F       0.024713\n",
      "Samsung SM-G950F       0.020469\n",
      "Samsung SM-G973F       0.014478\n",
      "                         ...   \n",
      "PLK-L01                0.000250\n",
      "XT1580                 0.000250\n",
      "CUBOT_X18_Plus         0.000250\n",
      "HTC M10h               0.000250\n",
      "XiaoMi Redmi Note 5    0.000250\n",
      "Name: device_make, Length: 218, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('device_make')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a slight difference in device use between groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "0    0.935151\n",
      "1    0.064849\n",
      "Name: yes, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "0    0.923115\n",
      "1    0.076885\n",
      "Name: yes, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('yes')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a slight difference in the choosing of yes between the two groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "0    0.920904\n",
      "1    0.079096\n",
      "Name: no, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "0    0.912881\n",
      "1    0.087119\n",
      "Name: no, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('no')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a slight difference in the choosing of yes between the two groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics for control group\n",
      "-------------------------\n",
      "0    0.856055\n",
      "1    0.143945\n",
      "Name: engaged, dtype: float64\n",
      "\n",
      "metrics for exposed group\n",
      "-------------------------\n",
      "0    0.835996\n",
      "1    0.164004\n",
      "Name: engaged, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "compare_metrics('engaged')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "rate of engagement is higher in the control group"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of customers in the control group that engaged by answer to questioonaire (1-yes, 0-no)|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    322\n",
       "1    264\n",
       "Name: yes, dtype: int64"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df.experiment == 'control') & (df.engaged == 1)]['yes'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of customers in the exposed group that engaged by answer to questioonaire (1-yes, 0-no)|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    349\n",
       "1    308\n",
       "Name: yes, dtype: int64"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df.experiment == 'exposed')& (df.engaged == 1)]['yes'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classical A/B test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> The task is to determine whether a recent advertising campaign resulted in a significant lift in brand awareness.\n",
    "Hypothesis:\n",
    "\n",
    "H0: There is a significant lift in awareness due to the recent campaign\n",
    "\n",
    "H1: There is no significant lift in awareness (not H0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we'll consider only customers with engagement(i.e answered the question in the questionnaire)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "engaged = df[df.engaged == 1].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll Use the yes column - where 1 signifies that the user answered yes and 0 signifies that the user answered no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>device_make</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "      <th>engaged</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0016d14a-ae18-4a02-a204-6ba53b52f2ed</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-05</td>\n",
       "      <td>2</td>\n",
       "      <td>E5823</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>008aafdf-deef-4482-8fec-d98e3da054da</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-04</td>\n",
       "      <td>16</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00a1384a-5118-4d1b-925b-6cdada50318d</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-06</td>\n",
       "      <td>8</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00b6fadb-10bd-49e3-a778-290da82f7a8d</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-08</td>\n",
       "      <td>4</td>\n",
       "      <td>Samsung SM-A202F</td>\n",
       "      <td>6</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00ebf4a8-060f-4b99-93ac-c62724399483</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             auction_id experiment        date  hour  \\\n",
       "0  0016d14a-ae18-4a02-a204-6ba53b52f2ed    exposed  2020-07-05     2   \n",
       "1  008aafdf-deef-4482-8fec-d98e3da054da    exposed  2020-07-04    16   \n",
       "2  00a1384a-5118-4d1b-925b-6cdada50318d    exposed  2020-07-06     8   \n",
       "3  00b6fadb-10bd-49e3-a778-290da82f7a8d    control  2020-07-08     4   \n",
       "4  00ebf4a8-060f-4b99-93ac-c62724399483    control  2020-07-03    15   \n",
       "\n",
       "          device_make  platform_os                browser  yes  no  engaged  \n",
       "0               E5823            6  Chrome Mobile WebView    0   1        1  \n",
       "1  Generic Smartphone            6          Chrome Mobile    1   0        1  \n",
       "2  Generic Smartphone            6          Chrome Mobile    0   1        1  \n",
       "3    Samsung SM-A202F            6               Facebook    1   0        1  \n",
       "4  Generic Smartphone            6          Chrome Mobile    0   1        1  "
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "engaged.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using the yes column as the target column (1-yes, 0-no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "control_group = engaged[engaged.experiment == 'control']['yes']\n",
    "exposed_group = engaged[engaged.experiment == 'exposed']['yes']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Distribution of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.549488\n",
       "1    0.450512\n",
       "Name: yes, dtype: float64"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "control_group.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.531202\n",
       "1    0.468798\n",
       "Name: yes, dtype: float64"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exposed_group.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.45051194539249145, 0.4687975646879756)"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "control_group.mean(), exposed_group.mean() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.49796997543854177, 0.49940566822721727)"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "control_group.std(), exposed_group.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    322\n",
       "1    264\n",
       "Name: yes, dtype: int64"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "control_group.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.6452687398426953, 0.7406234765948244)"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm.stats.ztest(control_group, exposed_group, alternative= 'larger')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "since the p value (0.74) is greater than the alpha (at 0.05) we thereby do not reject the null hypothesis and conclude that there is no significant lift in brand awareness even after the campaign."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAHiCAYAAAAqFoLhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXCdd53v+c/znP0crZa1y5a871uc2I5DnIQQHLJ0SG4v0H2bAapphks10xkmdAYYqEoV01SToqfqUkzRFD11b08uczOhgWaaLED2OLEd77st2bL2fTv79jzzh2Mpjp1Ith/pkY7er38S6Rwdff3TOTof/X6/5/szbNu2BQAAgJtmul0AAABAoSBYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA4hWAEAADjE63YBkjQ8HJdlfXTXh4qKIg0OxmaootmNsZjAWFyJ8ZjAWExgLK7EeExgLCZMZSxM01B5eeQj7zMrgpVl2ZMGq8v3wyWMxQTG4kqMxwTGYgJjcSXGYwJjMcGJsWApEAAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHeN0uALgZ0URG8XTO7TIcF/B55eXPHgCYcwhWmNOSqZz2n+p1uwzH3bamWt4AL08AmGv4mxgAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIp7xiTvPbKa1aaLtdhuPCnqx4eQLA3MNvbsxt2ZQGT+53uwrH1S28W7Y35HYZAIDrxFIgAACAQwhWAAAADiFYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA4hWAEAADiEYAUAAOAQghUAAIBDCFYAAAAOIVgBAAA4hGAFAADgEIIVAACAQwhWAAAADiFYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA7xTuVOP/rRj/T8889Lku666y594xvfuOr2X/ziFyopKZEk/emf/qn+4i/+wuFSAQAAZrdJg9WePXv05ptv6pe//KUMw9Bf/dVf6Xe/+53uu+++8fscP35cP/zhD7Vly5ZpLRYAAGA2mzRYVVZW6sknn5Tf75ckLVu2TF1dXVfc5/jx4/rJT36izs5O3Xbbbfq7v/s7BQKB6akYAABgljJs27aneufW1lZ99rOf1c9//nM1NTVJkuLxuP72b/9WTz75pBobG/Xkk0+qvr5ejz/++HTVDIwb6enSqTdfc7sMx6352F0qq6lzuwwAwHWacrA6d+6cvvzlL+tv/uZv9Oijj37o/U6ePKlvfvOb+tWvfjXlIgYHY7Ksjy6jsrJY/f3RKT9mIWMsJvjyUR36w+/dLsNxG3bdLTtYft1fx3NjAmMxgbG4EuMxgbGYMJWxME1DFRVFH32fqXyzAwcO6POf/7y+/vWvXxWqurq69Nxzz41/bNu2vN4p7YkHAAAoKJMGq+7ubn31q1/V008/rQcffPCq24PBoH7wgx+ovb1dtm3rmWeeuWJjOwAAwHwx6dTSz372M6XTaX3/+98f/9xnPvMZvfzyy/ra176mDRs26KmnntJXvvIVZbNZ3XLLLfrCF74wrUUDAADMRte1eX26sMfq+jAWE9hjdSWeGxMYiwmMxZUYjwmMxYQZ3WMFAACAyRGsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABzidbsAAMDs5FdayqbcLsNx+aThdgkoYAQrAMC1ZVOKNR9yuwrHhTdulxRyuwwUKJYCAQAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAh3AIMwDgmixJmVze7TIcV4j/JsweBCsAwDXl85Zau8fcLsNx5WttyeN2FShULAUCAAA4hGAFAADgEIIVAACAQwhWAAAADiFYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA4hWAEAADiEYAUAAOAQghUAAIBDCFYAAAAOIVgBAAA4hGAFAADgEIIVAACAQwhWAAAADiFYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA4hWAEAADiEYAUAAOAQghUAAIBDCFYAAAAOIVgBAAA4hGAFAADgEIIVAACAQwhWAAAADiFYAQAAOIRgBQAA4BCCFQAAgEMIVgAAAA4hWAEAADhkSsHqRz/6kR588EE9+OCD+od/+Ierbj916pQee+wx7d69W9/61reUy+UcLxQAAGC2mzRY7dmzR2+++aZ++ctf6le/+pVOnDih3/3ud1fc54knntB3vvMdvfjii7JtW88+++y0FQwAADBbTRqsKisr9eSTT8rv98vn82nZsmXq6uoav72zs1OpVEqbN2+WJD322GN64YUXpq9iAACAWco72R1WrFgx/v+tra16/vnn9fOf/3z8c319faqsrBz/uLKyUr29vddVREVF0ZTuV1lZfF2PW8gYi0tGeqIKBnxul+E4r8+rshv8GfPcmMBYTLiRsSjU15fEc+P9GIsJTozFpMHqsnPnzunLX/6yvvGNb6ipqWn885ZlyTCM8Y9t277i46kYHIzJsuyPvE9lZbH6+6PX9biFirGY4JOUSmfdLsNxuWzuhn7GPDcmMBYTbnQsjGyuIF9fknhuvIfXyYSpjIVpGpNOBk1p8/qBAwf0+c9/Xl//+tf16KOPXnFbTU2N+vv7xz8eGBhQVVXVVB4WAACgoEwarLq7u/XVr35VTz/9tB588MGrbq+vr1cgENCBAwckSb/+9a+1a9cu5ysFAACY5SZdCvzZz36mdDqt73//++Of+8xnPqOXX35ZX/va17RhwwY9/fTT+va3v61YLKZ169bpc5/73LQWDQAAMBtNGqy+/e1v69vf/vZVn//sZz87/v+rV6/Wc88952xlAAAAcwyd1wEAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwCMEKAADAIQQrAAAAhxCsAAAAHEKwAgAAcAjBCgAAwCEEKwAAAIcQrAAAABxCsAIAAHAIwQoAAMAhBCsAAACHEKwAAAAcQrACAABwyJSCVSwW00MPPaSOjo6rbvvRj36ke+65R4888ogeeeQRPfPMM44XCQAAMBd4J7vDkSNH9O1vf1utra3XvP348eP64Q9/qC1btjhdGwAAwJwy6YzVs88+q+9+97uqqqq65u3Hjx/XT37yEz388MN66qmnlE6nHS8SAABgLph0xup73/veh94Wj8e1Zs0aPfHEE2psbNSTTz6pH//4x3r88cevq4iKiqIp3a+ysvi6HreQMRaXjPREFQz43C7DcV6fV2U3+DPmuTGBsZhwI2NRqK8viefG+zEWE5wYi0mD1UeJRCL66U9/Ov7xF7/4RX3zm9+87mA1OBiTZdkfeZ/KymL190dvqM5Cw1hM8ElKpbNul+G4XDZ3Qz9jnhsTGIsJNzoWRjZXkK8vSTw33sPrZMJUxsI0jUkng27qqsCuri4999xz4x/bti2v96ayGgAAwJx1UykoGAzqBz/4gbZv366GhgY988wzuu+++5yqDbhK3rJ0oTuqzv6YOgfi6u4f0/BARCvLs1qzIKuQ96NnPgEAmE43FKy+9KUv6Wtf+5o2bNigp556Sl/5yleUzWZ1yy236Atf+ILTNQKSpL7hhP7pNyd1vmtMkuT3maqvCCprGXq1I6Q3OoNaXpbVpoUZNRTnXa4WADAfTTlYvfzyy+P///59Vbt379bu3budrQp4H9u29faJHv3LS2dlGoY+/6nVWtNYrorSoAJWTIf+8Hv1J0wdH/Tr1JBPZ4b9urshqVuqMm6XDgCYZ9gQhVktmc7pX148o3dO9mplQ6m+9PA6LQjmlT33lrKtGVnKqHi0WUWmV9V1dfpYvV/PXwjr1Y6QUjlDt9emZRhu/ysAAPMFwQqzVt6y9ONfHtOpiyN69M4lemBbg/KnX1bs4L9J6fj4/Ra899+y4VMaKVulh5Y06XftEb3TE1Qqb+iehhThCgAwIwhWmLV+8dp5nWgd1uc/tVq3F3cq+Yv/U/ZYnzz16xTY/qcyS2vky43pyGuvyJeNq3z4hCqGjqk4ekGPVK5X0NOoA30BpfOG7m9MEq4AANOOYIVZae/JXr2wt0333FKvHcYxpX7/c5nlDQp+6uvyLtowfj/DDMo2fcoEytRbvVOhZI/Kh06ouu8dPVgel9+zRm93B1UXyWtTJXuuAADTi2CFWaetN6r/67entKKhVH/SOKD0az+Xd8mtCt77FRmm58O/0DCUDNcqGapWZd9+LRg+po8vDKgrtkKvdQa1uDin8qA1c/8QAMC8c1MNQgGnRRMZ/edfHFMk5NN/2iZlX/9neerWKPjxL390qHo/w9RA5a1KBxaocuCgHq1pk8ew9XxrSJM0+AcA4KYQrDCr/D9/OKfReFp/e3eRzDd+IrOiQaFPfk2G5/rOK7NNj/qrtivri6hxaK8+XdejnoRX+3oC01Q5AAAEK8wiLZ2jevtEr/5oS5nK3v0nGZFyhe7/n2X4Qzf0eJbHr77q22WZXm1LvaEt5WN6uzugnvgUZ74AALhOBCvMCpZt67/9/pxKi/y6O/+mlMso/KnHZYZLb+px896Q+qpvlyef1aORdxXx2XqhNaQ8S4IAgGlAsMKs8M6JHl3oHtMXNmZkXzwg/y1/JLO0xpHHzvpLNFq6XKWJDv1xzUUNpT06MXh9S4sAAEwFwQquS2Vyeu7VFq2oCWppx7/LLK+Tf+OnHP0eY6UrlfVGtD59UIsiab3THVSWCwQBAA4jWMF1v33nokZiGX2hoVl2bFCBj/0PMjzOdgKxTY+GKjbKl4vrT8qOKpY1dbTf7+j3AACAYAVXDYwk9cLedt2/0lDkwqvyrbpT3tpV0/K9UqEqxcP1Wpw8o03FQ9rXG1AmPy3fCgAwTxGs4Krf7GmVadjabb8hIxBRYPufTev3G16wTrZh6j+E9ymZM3Swj/YLAADnEKzgmpFYWm+f6NF/WBaXOXhegW1/IiNYNK3fM+8NaaR8tUqzfbqntF3v9gaUzHGIIADAGQQruOb373Yob1m6LbdfRnGlvCvvmJHvGy1uUs4T1L3BY8pY0ru97LUCADiDYAVXJNM5vXKoUw8tScozfFH+zQ9O/ciam2V4NFa6XMXZQd1d3qVDfcxaAQCcQbCCK14/0qVkOqe7zEMyIuXyzdBs1WWxokblTb8+HjiunG3o+AB9rQAAN49ghRmXy1t6aX+77qmPyzfUIv+mB677LMCbZZtejZUsU2mmV7cU9+nwQIADmgEAN41ghRm371SvhqNpfTJ0XEawWL7Vu1ypI1qyRJbh1f3hY4pmTLWMOts7CwAw/xCsMKNs29YLe9u0dWFc4aEz8m28X4bXnZYHtulTtGSpqrKdWhYY1mFaLwAAbhLBCjPq+IUhdfTH9UjpKSkQkX/tx12tZ6xkqWzDo0eKj6o95lV/kpcEAODG8S6CGfXKwU41RZIqHT4p/7pPyPCHXK3H8gQUK27SYqtNC80Ys1YAgJtCsMKMGY6mdbRlUJ+u6ZAMU741d7tdkiRprHipJFsPlp3RqSEfrRcAADeMYIUZs+d4tww7p6b4MXkbt8iMlLtdkiQp7wsrFarSerNFlm3r+CCtFwAAN4ZghRlh2bbeONKtT9UOyMjE5Ft7j9slXSFa1CS/ldKdxW060h+QTesFAMAN4PpyzIgzbSPqG0lqZ9lZGf5KeerXul3SFZLhauU8Qd3pPavXok1qj3q0uCTvdlkAHBBIDao4ekGmlZVsW2PPn5bt8cu3dJu8K26XYfJWCOcwY4UZ8caRLjUGY4qMXZBvzd0yjFn21DNMxYoWa2GuR9XeqE4McX4gMNcFUkOq6tmjmp43FUz2y8xnZNo5KZeRNdav1Gs/U/y/P6nMyVdk57Nul4sCQUzHtIsls3r3TL/+06J2KeaRb9Wdbpd0TbHiRpWOntWnSs7o/x7ZqnvzSfln6PhCAM4x8xlVDBxUONmrvBnQUPk6xYqbZL83M7Xl3k8oYxYp335E6YP/pvSb/0WZQ79R6N6vyFOzwuXqMdfNsmkDFKJ3TvTIyGe0JHlC3qatMkMlbpd0TXlvWMlQtdaZ52VZts6OsIkdmGvMfEbVvXsUSvZruHytOhs+oWjp8vFQdZlhGPIu3qzwI/+bQg/8L5LXp8Rvn1au65RLlaNQEKwwrWzb1utHuvSJyl6Z2cSs27T+QbHiRvmtlLaH23RikOVAYC4x82lV97wlXyaqvqptGitdcVWg+iDDMORtWK/ww/+rzOIKJZ//oXIdx2eoYhQighWmVWtPVB39cX0sdE5GaY08tavdLukjJUOXNrHfFT6rzphXo2l6WgFzgZlLqbrnLXlzMfVVb1cqXH19Xx8uU+ihJ2WW1Sj5wv+h3MXD01QpCh3BCtPqrWPdqvXHVBRrk3/1LhnGLA8qhqlYUaNqrB6VmTGdZBM7MOsZVlbVvW/Jm0uor2qHUqGqG3ocM1Si8IN/J7NikZIv/Wflus84XCnmA4IVpk3esrT/dJ8eqOqWZMi7/Ha3S5qSeNEiGZLuLT6vk4N+eloBs1z50HH5sjH1V21XOlR5U49lBIsUfvAJGcULlXrln2RnEg5VifmCYIVpc6p1WNFERqvtc/LUrZ41ndYnk/NFlA6Ua7PvgkYzpjpjXBoIzFaheLeKY20aK12h1E2GqssMf1ihj/+17PiwUm/+iyOPifmDYIVps/dkr1aERuRPDsg3R2arLotHGlRijWqRd4ieVsAsZeZSqhg8rLS/VCNlzu7f9FQtk/+WR5RrflvZ5nccfWwUNoIVpkU2l9fBc/3avbBL8njlXXqr2yVdl3ikXrYM3VvSonPDPmUttysCcAXbVsXgYRl2TgMLt0rT0HTYv+UhmdXLlXrzv8iKDTr++ChMBCtMi6Mtg0qns1qaPSPv4s0y/GG3S7ouliegVKhSa8xWZS2pdYxeusBsUhRtVTjZq5Hydcr5iz/yvomsoWMDPv26Jax/bQ7rv/7+ol7a367D5waUSH14x3XD9Ch0z19Ltn1pv5XFX1iYHO8WmBbvnOzV5qJ+eTKxObNp/YPikQYtTB7U6kCvzg4v0IqynNslAZDkyaVUPnxCyWClosVLrnkf25ZODvl0fMCvzrhHkqESv6Wgx9abJwaVzPRJkopCPj22a6l2baqTaV591bJZUqXg7X+u1Ov/rNy5t2btyRGYPQhWcFwyndOR5kH9bV2nlA3Lu3ij2yXdkES4Vpbh0V2RFv1spFpZKykfc7yA60pHTsuwLQ1VbJKu0cIlnZd+dzGksyN+VQTz2lGT1oqyrBaGLBmGtPnj92o4E1Rnf1y/fvOC/uuLZ/TqoU79+X0rtXJR2VWP5111p8xTryr97r/Ku2y7DC/7LvHheJuA4w6e7ZeZT6s+dVa+pbfJ8MzNo2Fs06tkuEbLjXZZlqULo/wdArjNm4mqKNamaHGTcr7IVbf3J0w9c7pI50Z8urM+qc+tiWlnXVqVYWs8gxmGoeKwX6sby/WNP9+i//GRdYoms/r+Mwf17MvNsj/QY8UwDAV2/Jns+LAyx1+aiX8m5jCCFRy391Svbi/rlZnPzNllwMvikQb57Iw2BTt1dnhuBkSgkJSPnJRteDRatuqq204O+vTfzhQpaxn6k5Vx3VadudaE1hUMw9C2NdX637+0Q3dvqdcL+9r07CtXhytv7Sp5G7coc+jfZSXHnPwnocAQrOCosURGJy8M62PF7TIiC+SpXel2STclGapS3vTrznCLzo/6lM27XREwfwVSgwonejRaulyWJ3DFbS0jXr1wMaS6SF5/uTqmhqLre7EG/B795SdX6t6tDXpxX7t+8dr5q8KVf9ufSLmUMod+c9P/FhQughUcdeB0nwJKqTLRcmkvwjRcAj2jDFOJSJ0a1SXTzun8GLNWgCtsW2XDJ5TzBBQtWXbFTX0JU79tDas6nNenl8cV9t3YcQmGYejPP7FCd2+p12/fuahfvXHhits95XXyrbpL2RMvyxrtveF/CgrbHH/Xw2zz7pl+fay8T4Ztybf0NrfLcUQ8XC+P8toc7GA5EHBJKNGtYHpYo2WrZZsT+x1jWUO/aoko4LH1yLLETV9gYhiG/uMnV2rXplr9Zk+rXtzXdsXt/ls/LXk8Su//xc19IxQsghUcE0tmdaZtRNsiHTKKKmRWXvsy6LkmHaxQ3vTr9kirzo96lWE5EJhZtq3ykVPK+ooUK1o8/umsJf26Jax03tCnl8VVdIMzVR9kGoY+d/9qbV1VqedebVFrz8SeKjNcJv/GTyl3fp/yAxcd+X4oLAQrOObQuX757LQqkxfkbdoqY7Jdo3OFYSgRrlWjumXaeZ0fZdYKmEmhRLd82dilY2vet73gpYsh9SY8eqApoaqws807TcPQ5z+1WiURv/7p304q/b4Nlv4Nn5R8QWWOPO/o90RhIFjBMQfP9Gt7aZ8MKzfnjrCZTCJSJ4+d06ZQp86wHAjMqJKxFmW9YSXCteOfOzfi1Zlhv+6oTWvZNDXvjQR9+qsH16hnKKFnX2ke/7wRiMi35m7lzu+TFR2Ylu+NuYtgBUck0zmdaB3SzpIuGeEyeaqXu12So1LBhcqbPt0eblXrGMuBwEwJpIYUTA9d2rD+3mxVOi+93B5SZSiv22rS0/r91zQt0O5ti/TKwU4dbZkIUf7190kylDn24rR+f8w9BCs44mjLoMx8RjXJFnmbbpn7VwN+kGEqGapRk7ok2+LsQGCGlIydU970XbG36o3OoBJZQ59sTOgap9A47rFdy9RQGdE///a0xhIZSZJZVCHv8u3Knn5ddjo+/UVgziiwdz+45cDZfm0t7pNhZeVdUljLgJclInXy2lmtC3SreYTlQGC6ebMxhRI9ihYvGb8SsDPm0dGBgG6pyqja4X1VH8bnNfXXD69TIpXVz39/bvzz/o2fknJpZU6+PCN1YG4gWOGmZbJ5HWsZ1B2l3TKCxfLUXt0RuRAkg5WyDK92Rlp1YcynPAfdA9OqZLRZkqloyaUrjHPWpTMAS/yWdtamZrSWhqoi3b+9UXtP9qqla1SS5KlYJE/DemWP/052LjOj9WD2Iljhpp24MKR8NqP6dIu8TVtkmB63S5oepkfJcLWWGR3K5m21x1gOBKaLmU+rKNauWNEiWZ6gJGlfT0BDaY8+sTgpnwu/Zh7YsVilEb/++x8mjrzxb3pAdnJM2ea3Z74gzEoEK9y0d8/0a1OkV2Y+XbDLgJclwnXy2xmt9PeoZYRgBUyX4rHzkiyNlV7qsh7LGNrfG9Cq8oyaSqbnKsDJBP1ePbprqZo7R/XumX5JkqdujcyFjcoeeV62zTQ2CFa4Sbm8pSPNA9pV1i35w/LUrXW7pGmVDFXJMjy6I3JRzaM+2c70IwTwflZexdFWJUM1yvmKJUl7ewKyJX2sbmaXAD/oYxtq1VAZ0f/7SrOyOUuGYci/8VOyRnuU7zjham2YHQhWuCmnLw4rlc5oUfa8vI2bZXgKexbHNr1Khqq00mxXImuoJ1Ggy56Ai8KJbnmszPjeqpG0oWMDfm2oyKg04O5fM6Zp6M8+vkIDoyn94UCHJMm75FYZwWJlT73qam2YHQhWuCmHzg1oVXBQnlxS3sYtbpczI5LhWgXtlBo9/WpmORBwXHG0VVlvWKlgpSTp7e6gTEPaXju9Paumat2SBdqwtEK/2dOqaCIjw+OVb9Wdyl08JCs+7HZ5cBnBCjfMtm0daRnQnRV9kumRt2G92yXNiGSoWrYM3V7URtsFwGHeTFTB9KBixU2SYWggaerUkE+bqzKOnQXohD/9+HKlM3n9f3sunRfoW3O3ZFvKnnnd3cLgOoIVblh7X0xDY2ktsy/KU7tahj/kdkkzwvL4lQ5WaK23Q8Npj4ZSvIwApxTHWmXLGG8IuqcrKL8p3VY9O2arLqtfGNHt66r12uFOjSUyMkuq5Klfp+yp12RbbGKfz3hHwA073DygSnNMwdSAvI2b3S5nRiVCNSqxx1RhRpm1AhxiWHlFYu1KhGtleQLqiXvUPOrT1uq0Qt7ZM1t12QO3Nyqbs/T7d9slXZq1suNDynccdbkyuIlghRt2pHlAdy28dMmxd/H8ClbJcI0k6fbIRZ1jnxXgiHCiSx4re2kZUNJb3QGFvJa2Vs2u2arLaisi2rqqUn840KlEKnepj1+oVJmTr7pdGlxEsMINGYmldaE7qg2BTpnl9TJLKt0uaUblfBFlfMXa5G9Xb8KrWGYGDiwDClxRtFVZb0Sp4EL1JUxdHPNpa1VG/ll88e2Dtzcpmc7plUMdMsxLm9jz7UdkxQbdLg0uIVjhhhxtGVTIyKg00TbvlgEvS4ZrVWn3K2ykdJ5DmYGb4suMKZgeUqy4UTIMHegLyGfa2rhwds5WXdZYU6z1Sxfopf3tSmfz8q25S7Kl7Gk2sc9XBCvckMPnBnRbSZ8M25p3y4CXJcI1MiRtDXXowij7rICbURS9KCneCx0AACAASURBVFumYkWLFc0YOjPk04aFGQXnwN8sD93epGgiqzePdsssrpRn0QZlT78m28q7XRpcQLDCdctk8zrZOqRtJT0ygsUyq5a5XZIrMv4y5TwBbQ226eKYV1kuBAJujJ1XJD6xaf1g36Uu61sqZ/ds1WUrF5VpRUOpnt97Ubm8Jd/qu2QnRujEPk8RrHDdTl0cVi6XU136gjyLNsow5+nTyDCUDNVqsbol21J7dA78aQ3MQuFE73ub1hcrnZeODfi1sjzrepf16/Hg7U0aGkvr7RM98i7eJCNQpOzZN90uCy6Y0jtiLBbTQw89pI6OjqtuO3XqlB577DHt3r1b3/rWt5TLuXM4JmbOkeYBrRzvtj4/lwEvS4Rr5FFea/zdOj9KsAJuRCTWrpwnoFRwoY4N+JWxDN06y/pWTWbD0gVaVFWk3+1vv9Qwefl25S4elJ2Ou10aZtikwerIkSP67Gc/q9bW1mve/sQTT+g73/mOXnzxRdm2rWeffdbpGjGLXOq2PqhdFf3zqtv6h0kFF8oyPNoeadN5DmUGrpuZTyuU7FU8skh5mTrYF9Ciopyqw3Nrbd0wDN27tUEd/XGdbR+Rb8UdUj6n7Pn9bpeGGTZpsHr22Wf13e9+V1VVVVfd1tnZqVQqpc2bL81aPPbYY3rhhRecrxKzRltvTMPR+ddt/UOZHiVDVVphdiiWNdSfnKfLosANisQ7ZchWvGiRzg77FMua2jrHZqsu27G2WpGgV78/0CGzconMslrlzu1xuyzMsEnfBb73ve/p1ltvveZtfX19qqyc6F9UWVmp3t5e56rDrDPRbb1/3i8DXpYM1yhop9TgGVQLVwcC1yUSa1faX6qMr0QHegOqCOa1pGRubinx+zy6c1OdDp0d0HA0Le/KO5TvOStrrM/t0jCDbmpTiGVZMoyJxoi2bV/x8VRVVBRN6X6VlcXX/diFyq2xONE6pHuqB6W0VLV5p3zl7v5MRnqiCgbcDTOWp0H2wCHtLGrXG2MbdU/TzS9heH1eld3gz5jXyQTGYsKNjMV0v7486TEFMiOKVm7SUDagvqRH9y/JKBSc/tf0dD03/vgTq/TSvjbtPdOvP//YJ9W27xfydb6rBcv+bFq+nxN4nUxwYixuKljV1NSov79//OOBgYFrLhlOZnAwJsv66M0plZXF6u+PXvdjFyK3xmI4mlZzx6j+clG7zHCdRnIRyeWfiU9SKp11tQbJVDpQrjWZDj07ukUDsZyKfDe32SqXzd3Qz5jXyQTGYsKNjoWRzU3r66ts6LxsGRoJ1mh/uymfaWt5SVKpGVgJnK7nhilp0/KFen5Pqz6xpU6e+rUaPfyKcqvvl2HMvq0CvE4mTGUsTNOYdDLopn7K9fX1CgQCOnDggCTp17/+tXbt2nUzD4lZ7GjLwES39XnaFPTDJEM1WmAPq8RI6AJXBwKTs21F4h1KhqoUt0M6M+zT2gWz+/iaqbp3a4Niyaz2neqTb8VO2dF+5XvOuV0WZsgNBasvfelLOnbsmCTp6aef1t///d/r/vvvVyKR0Oc+9zlHC8TscaR5ULeV9suwLXkat7hdzqxy+VDmraF2nWefFTCpYKpf3nxK8aJFOjHoU942tLEy43ZZjljTWK66hRH9/kCHPE23SN6AcmffcrsszJAp/2n98ssvj///T3/60/H/X716tZ577jlnq8Ksc7nb+v9U2yMjVyTPPO22/mGyvmLlvGHdEmzXGyMrlbMk7+yb9QdmjUisXZbhVSJUo6OtftVFcqoMza0WCx/GMAzde0u9/uWlszrfn1H90luVPb9PgTv+owyv3+3yMM341Y8pOflet/Xa9Hl5Fs/jbusfxjCUCFWrXr0yrLza6MIOfCjDyiuc6FY8Uq+LsYBG0h5tKpDZqstuX1+jUMCjlw92yLd8p5RNKdd2xO2yMAN4d8SU0G19csn3dWFnnxXw4ULJXpl2XolIvY70+xXyWlpR5vZFKM4K+r3asa5G757uV2rBchmhEuWa33G7LMwAghUmZdu2jjQPXOq2bnjkbdjgdkmzUipYIcvwaFuYLuzARwnHO5U3A+o3F6pl1Kv1FZmCXDrftbFOubyld072ybt0m3LtR2RnEm6XhWlWgE9lOO1ib1QjsYyW6aI8davotv5hjPe6sHs6FaULO3BNhpVTKNmrRKRWxwaDsiVtXFhYy4CXNdYUq7GmWK8f6ZJ32XYpn1Ou9aDbZWGa8Zsfkzp8bkCVnjEFk/20WZhEMnS5C/sQVwcC1xBK9Mi084qF63V80K+mkpxKA4U7vbtrU506+uNqy1fJKF6oLMuBBY9ghUkdaR7UroWDksT+qkkkw9WyJe0It+k8+6yAq0QSncp5AjqTrlYsa2pDRWHOVl22Y221/D5Trx/tlm/ZDuU7T8pKjrldFqYRwQofaTia1sXeqDYGOmSW18ksuf7O+vOJ5QkoHSjXOn+HehJexbPXf8QTUKgMK6tQok+JcL2ODwUU8lpaWjo3zwWcqlDAq9tWV2nvqV5ZjbdKtqXc+X1ul4VpRLDCRzrSPKCgkVFp/CLLgFP0/i7szFoBE8KJHhmyNBSoV8uoV2sWZOWZB+9Cd22qVzqT1/4ev8zyeuWa97pdEqbRPHhK42Ycbh7QtvFu6wSrqaALO3Bt4Xincp6QDsWrZdmG1hf4MuBly+pLVFsRvrSJffkO5XvPyYoNul0WpgnBCh8qnc3r1MVhbS/plQIReaqWu13SnJD1FSvnCemWYLsuRr3KFUYzaeCmmPmMQsk+xSN1Oj4YUE04p4UF0ml9MoZh6K5NdTrfNaaBskvtarLMWhUsghU+1MnWIeXf67buXUS39SkzDCXCNeNd2Nvpwg4olOiWIVvt5iINpDxaX1FYDUEnc/v6Gnk9hl5rycisWqpcy9tul4RpwjslPtSR5gGtDA3Jk03Iy6HL1+VyF/bV/m72WQGSIvEuZb1h7R2tksewtWrB/FgGvKw47Nfm5Qu192SvPEu2yxpsV36ky+2yMA0IVrgmy7YvtVm43G190Xq3S5pTLndh3x5upws75j0zn1Yw1a9YuF6nhwNaWZ5VwON2VTNv5/paRRNZnfMsl2Qo17Lf7ZIwDQhWuKaLPVGNxjNaZl/uth52u6S55b0u7Mvf68I+QBd2zGPh95YBT+YblbHmz6b1D1q/dIGKwz69cS4hT+1K5c6zz6oQ8dse13T43IAWeqIKJvvkXbzJ7XLmpGSoRiE7qXrPkFq4OhDzWDjeqaw3ordHKlXqz6uhKO92Sa7wekxtX1utw80Dyi/aKmu4S/mhDrfLgsMIVrimI80DumvhgCSxv+oGvb8Lewv7rDBPmfmUgqkBDQUb1B71aW1FVsY87pt7x/pa5fK2DmcaJcNQroVZq0JDsMJVhsZSauuLaWOgU2YZ3dZvlOUJKBMo13p/h3oTXsXowo55KBzvkiHpcKZJkrRmnm1a/6DF1UWqXxjR62ei8tStUfb8PtlswiwoBCtcZaLbeqs8LAPelMT7urBfYNYK81Ak3qWMr1jvDC9UfSSnsgI+cHkqDMPQzg01aukcU6J6s+zRXlmDbW6XBQcRrHCVw82D2lY6IMO2OHT5Jl3uwn4LXdgxD3lySQXSg+rzNWgo7dGaebpp/YN2rK2RYUhvR+skw2Q5sMAQrHCFdOYD3dar6bZ+M67owj7mVXZ+NJoGJEnhxKVlwL2pJnkMWyvL5ldT0A9TXhzQ2qYFev3UmDz1a1kOLDAEK1zhROuQ8vn3d1ufh81mnPReF/YG9cqw6cKO+SUc71TaV6L9IxVaVppVkKf/uDvW12hwLKWBsvWyowOy+i+4XRIcQrDCFQ43D2hVaEiebJxlQIdMdGHvoQs75g1PLqFgelgdnkVK5kytWcBs1fttWVmpgN+j1warJdOjLMuBBYNghXGWbetoy6B2VQy81219g9slFYTxLuyRNrqwY96IxC8d17InsUQhr6Wm0pzLFc0uAZ9Ht62q0p5zYzLq1yt3fr9sm70ChYBghXEXusc0Fs9omd0qT+1Kuq07xfAoFarScrNTsayhPrqwYx4IxzuV9JXp0Gi5VpVn5aHbyFXu2FCjdCav9uBq2fEhWb0tbpcEB/AbHuOONA+o0hN7r9s6y4BOSoSqJ7qwj3B1IAqbNxtXIDOi88Yi5W1Da1kGvKYVi8pUURLU73oXSB6vsuf3uV0SHECwwrjD5wZ1V+XlbusEKyclwzWXurC/txwIFLJwvFOS9EZ0qRYE8qoOz88jbCZjGoZuX1+jIxfjsmrfWw60WA6c6whWkCQNjCbV0R/TBn+HzLJamaXVbpdUUMa7sPs61Zf0KJphXQSFKxLvVNxXrlPx0nl/hM1k7lhfI9uWzporZCdGlO8563ZJuEkEK0iSjrYMKqCMSuMX5WEZcFokQtVaYA+914WdWSsUJm82Kn92TGesRkm2Vs/zI2wmU70grGX1JXq+vVTy+pVjOXDOI1hB0qU2CzvKB2TYeZYBp8nlLuxbQx0cyoyCFYl3yZb0h+gSLSrKq8TPZbCT2bm+Vq2DGaWr1r23HMjS6VxGsIJSmZxOXxzWtmK6rU+nrK9kvAt7W5Qu7ChM4Xinot4KdaSKtZYjbKZk25oqeT2GjuaXyE5Fle867XZJuAkEK+jEhWHl83m6rU+397qw16tHhm2pbYxZKxQWX2ZM/mxUx/ON8hq2VnCEzZREgj5tXr5Q/36xWPIFlDtPs9C5jGAFHWke0OrQ8KVu64s3uV1OQUuGquVRXmv93WphnxUKTDjeKVvS70eXaHlZVn7+RpuynetrNZywFV2wVtkLB2RbNFSdqwhW89ylbusDunMh3dZnQiq4UJbh0bZImy6MeunCjsJh24rEOzXsqdRgLsIRNtdp/dIFKg77tD+5WErHle846XZJuEEEq3nuQteYxhJZLbMvXOq2Hoi4XVJhMye6sMdzhnoT/EmPwuDLjMmXi+tQtkkRr6XGEmZcrofXY2r7mmq90F4s+ULKshw4ZxGs5rnDzQOq8kQVTPTJ27jF7XLmhctd2Bs8Q1wdiIIRSXTKlqFXxpq0ekFWJr2rrtvt62uUzpsaKF2tXOtB2Xlm/eYigtU8d/Bsv+6putxtnWA1E5Lh6ve6sLfThR2FwbYVjneqz1OlqBXSWnpX3ZCmmmLVVoS1J9ogZZLKdxx3uyTcAILVPNYzlFD3YELrfe0yFzTILKl0u6R5wfIElfGXa52vQ/1Jj8bowo45zp8ZkS+X0P5UkxYG86oM00vkRhiGoZ3ra/RKd6lsf0TZFpqFzkUEq3ns0Ll+RYyUiuMXma2aYYnwpS7sxUaSWSvMeeF4pywZejPWRO+qm7RjbY1smeoOr1Tu4iHZOcZzriFYzWOHzg7ozoWDMmxb3qZb3C5nXrnchf22cJvOs88Kc5ltKxLvUrdRo5Tt12quBrwpFaVBrVpcpleG66RsSrn2o26XhOtEsJqnRmNptXSOamu4S0a4TObCRrdLmlcud2HfEuxQe9SrNCdYYI4KpIflzSf1dnKJGktyKvLRQ+Rm3b6+Ru8Ol8vyFynHcuCcQ7Capw43D8ijvCoTLfI2bpFh8FSYUYahZLha9falLuytLAdijrq0DGhqX2Kx1jJb5YhbV1XJ6/Wq1b9CubbDsrNpt0vCdeDddJ46dG5At5YOyshn5G1if5UbEqEaeZTXumCXmlkOxFxk2wonOnVRtbJNn5ZxhI0jQgGvtqys1B8GaqRcRrm2w26XhOtAsJqHkumcTrYOa2d5r+QLylO3xu2S5qXLXdh3hNt1YdSnHBdSYY4JpAflzaf1VnyJVpRn5eMdxTE719foRKJCOX8xy4FzDC+DeejEhSHl83k1pJvlbVgvw8MylCtMj1KhSi01OpWxpLYos1aYWyLxTuXl0ZH0InpXOWxtU7mKI0GdM5cr135EdibpdkmYIoLVPHTwXL9WhUflSUe5GtBliVCNgnZSjd4hNY8QcDGH2JbC8W41W3UK+j1qKOIKDCd5TFM71lbrd/01Uj6n3MVDbpeEKSJYzTO5vKWjzYO6a2GfZJjyLtrodknz2uUu7HcUt6ll1CuLC6owRwRTg/JYl5YB1yzIyKDPreN2rq/R+exCZXwlyrZwduBcQbCaZ862jyiRzmm5dV6e2lUygkVulzSvXe7CvtrboWTOVFeMQ5kxN4TjncrJo5PZBq4GnCaLqopUV1mk4/ZS5TuOy07H3S4JU0CwmmcOnOlXQ2BM/kSfvEu2ul0OJCUitSrND6vCjKmZtguYC2xL4USXTuYWaWFEKg9y5cV0GD/iZrBOsvLKtR50uyRMAcFqHrEsWwfP9usTlf2SJG8TwWo2SIRrJUl3FbeqecQnm+VAzHKhZJ88VlZvJ5cyWzXNdqytUXu+QklfmbLnuTpwLiBYzSPNnaMajWe02myVWb1cZqTc7ZIgKecrUsZXovW+No1lTPUneVlidovEO5SSX+dyNVpZTrCaTuXFAa1tWqCD6UblO07KTsXcLgmT4Df4PPLu6T5VeeMKxTrlY7ZqVklEarXAGlCxkdA5rg7ELGZYOYUSPTqSaVRjqa2QlynW6Xb7+hq9FW2Q7LyyF951uxxMgmA1T1i2rQNn+3VfzYAksb9qlkmE62RI2lV8kbYLmNVCiR6Zdl57U0u0jt5VM+KWlZUaMCsV9ZYrx3LgrEewmicudI1pOJrWOk+rzIrFMkuq3C4J75P1FSvrjWiTv02DKY+6hlJulwRcUyTeqZgdUrcq1VSSc7uceSHo9+qWlVXal1isfNcpWYlRt0vCRyBYzRPvnulTmSepSPSivEtudbscfJBhKBGpU5XVp5CR0v6zw25XBFzFzGcUSvZqf3qJVpbn5OEdZMbsXF+j/cnFkm0rx3LgrMbLYh6wbVvvnu7X7ppBSSwDzlaJcK0M2bqzuE37z424XQ5wlXCiW4ZsvZtewtWAM2xNY7lS4WoNmwtYDpzlCFbzQGtPVINjKW3wXZRZVitPeb3bJeEaMv4y5Twh3RJoU+dgSp0DNAPE7BKOd2jIKlbCV6rqMEfYzCTTNLRzfa3eiS9SrvusrDiz2rMVwWoeOHCmX8WetIqjF+hdNZsZhhLhWlXnexRUVu+e7nO7ImCcJ5dSMDWgveklWrsgxxE2LrhjQ60OpptkyFbu/H63y8GHIFgVONu29e6ZPn2iZkiyLXmXsr9qNktE6mTK0r3VA9pPsMIsEo53ypB0MN2kNVwN6IqaBWGV1C1Wnyo4O3AWI1gVuPa+mPqGk9rivyijeKHMika3S8JHSAcWKG8GdGu4XV0DcXX20wwQs0Mk3qGu/AIFI0Uq9tO7yi13bqjVO4lGWX0tskZ73S4H10CwKnD7T/epyEyrZKxZvqXbZDB/P7sZhhKRWpXHWhRQllkrzAq+zJgCmRG9k1qmdRXMVrnp1tVVOmotky0pe26P2+XgGghWBcy2be092avdtQMybEveZdvdLglTEI80yMhndV/tkPaf7pPN4YFwWSTeIcs2dCzfqOVlXA3oplDAq5Url6g5V6vMuT38fpiFCFYF7HzXmAZGU9ria5VRWiOzYrHbJWEK0oEFMsKlujV0Ud2DCa4OhLtsW+FYu05l67SozCMf7xqu+9jGWu1NLZWi/cr3NrtdDj6Al0gBe+dkr8q9KRWPnZdv2XaWAecKw5CvaYvKxs4pbKa5OhCuCqYG5MuntD+zVOsXsgw4G6xcVKau8Cpl5VXu7Ftul4MPIFgVqLxlaf/pPj1Q2y/JlnfZNrdLwnXwNW2RYeV1X/Wg9p1iORDuicTalbJ96vHUqTpsuV0OJBmGoW0bFulwerEyLXtl5wi8swnBqkCdbhvRWDyjDZ7zMhcsoinoHGNWLJJRXKmtgVb1DCXU1svVgZh5djatUKJbB9ONWl1BqJpN7thQq3fTS2Vkk8q1HXG7HLwPwapA7T3Rq5pAUqGxi2xan4MMw5Bv2XaVRs+rzJvS2yd63C4J81Cu/bg8dk4Hs0vpXTXLLCgJyrdoraJ2WFmWA2cVglUByubyOnC2Xw/UXtqb42MZcE7yLt8u2ZY+VTuod072Km8xY4CZlW7epyErIrOoXEGv29Xgg3ZtXqR9qSXKtR+VlRxzuxy8Z0rB6je/+Y0eeOABffKTn9Qzzzxz1e0/+tGPdM899+iRRx7RI488cs37YOYcbRlSMp3TartFZuUSmSVVbpeEG2CWN8gsr9NG73mNxTM61crZYJg5VnxY+Z5z2p9eqg0VtFiYjTYtr9ApzyoZtqVcCwczzxaT/g3S29urf/zHf9S//uu/yu/36zOf+Yy2b9+u5cuXj9/n+PHj+uEPf6gtW7ZMa7GYmr2netUUSSgQ7ZBv3WfcLgc3yDAMeZdtV/jdX6kmuE1vn+jR+qUVbpeFeSLX/I4M2TptLdEjxRy4PBt5PaZWbVynjuOvqO7U6/Kv/4TbJUFTmLHas2ePduzYobKyMoXDYe3evVsvvPDCFfc5fvy4fvKTn+jhhx/WU089pXQ6PW0F46Ml0zkdaR7Q/dWXjjrwLmUZcC7zLd0uydaDdf06cLZfqUzO7ZIwD9i2reTJV3UhW6naBUEOXJ7F7txYp33p5TKH25QfbHO7HGgKM1Z9fX2qrKwc/7iqqkpHjx4d/zgej2vNmjV64okn1NjYqCeffFI//vGP9fjjj0+5iIqKoindr7KyeMqPWeg+bCxefrdN2Vxeq/JnFFy8VtVLCvtswJGeqIIBn9tlOM7r86qssliqLFZHzTKtT59TJtuglp6Y7t666CO/ltfJBMZiwvWMRarjtGLRXu3N7tRtdZaC/sJ7jRXKc6Oysli/WrxNuYED8rS+rcrV627oMXCJE2MxabCyLOuKxpK2bV/xcSQS0U9/+tPxj7/4xS/qm9/85nUFq8HBmCzro/v0VFYWq78/OuXHLGQfNRbPv3VBW8rGZEb7pFseLvgx80lKpQtv/0cumxv/2RlLd8jc84zWlCT04jutWre47EO/jtfJBMZiwvWORXzPb5W2vVLDRnntbqUKcBGikJ4bt6xv1JGXFmvzkVdlb3pUhtc/5a/ldTJhKmNhmsakk0GTLgXW1NSov79//OP+/n5VVU1shu7q6tJzzz03/rFt2/J6uXzEDf0jSZ1uG9EnKzokr1/eJbe6XRIc4F22XTI8ur+yQycuDGk0zmXvmD52Jqlcyz4dSC/RnZvq3C4HU7Bp+UIdM9bIk0sq13rQ7XLmvUmD1c6dO/X2229raGhIyWRSL730knbt2jV+ezAY1A9+8AO1t7fLtm0988wzuu+++6a1aFzbW8e65VNetbGT8jZtleEPuV0SHGCGSuRt3KTFiRMybEv7Tva6XRIKWLZlr0wrq3PB9VrdMLVtGnCX12Oqbv1WDeaLlDjxqtvlzHuTBqvq6mo9/vjj+tznPqdPf/rTeuihh7Rx40Z96Utf0rFjx7RgwQI99dRT+spXvqL7779ftm3rC1/4wkzUjvexbFtvHevR/fVDMrJJ+VZ+zO2S4CDvyjtkpqPaVTWsPTQLxTSKH3tZXbkyrdq8hfNF55Bdm+v1Tnq5zN7TssY4X9RNU1qze/jhh/Xwww9f8bn376vavXu3du/e7WxluC5nLg5rcCyl7ZXnZZgL5Klb43ZJcJB30SYZgSLtCrXq1fMVauuNanE1G07hrPxgu7wjbdqf26ZHN9ZK4iiluaKyLKSxmltljR5R+vTrCm37Y7dLmrfovF4g3jzWrapgRsUjZ+VbsVOGyY+2kBger7zLd6hi7IyKPVm9cbTb7ZJQgJInX1XONqWmHYoEC+9KwEK347Y1Op2tU/Lk67I5qcE1vPsWgEQqpwNn+vVHdb2Sbcu38g63S8I08K38mGTl9EeLBvT28R5lsjRthHPsXEbZs3t0JLNYd9y2fPIvwKyzfukCnfKtlS8zpnzHMbfLmbcIVgVg/+leZXJ5rcmfllm1VGZZrdslYRqYCxtlljdok3FWiXROB872T/5FwBRlLxyQN5/UhchGLaktcbsc3ADTMNSweaeiVlCjh//gdjnzFsGqALx5rFubFyTkjXazab2AGYYh38o7FBi9qNVlab1xpMvtklBARg+9pIF8kVbeut3tUnATdm5apP3ZFfL2HJMVG3S7nHmJYDXHdQ/G1dI5pt0V7ZLplY8jbAqad8XtkmHowepOnW4bUe9Qwu2SUADyg20KjlzQfnuttq1lxnsuCwe9yi67U7Kl2JHfu13OvESwmuNeP9KlkJlT7dgxeZfeKiNI35lCZobL5Fm0SYuiR+X7/9u78+iq6nvv4+995oxkIAMkIRDCEGSUICTIECUEwhAN1lrxVivaWq19brV9bMVeW7yu9vY+d+nt0lqtem2FWtEyKgIiMocxQAKGAIGQEDLPOZnO8Hv+oOQWBxIxZJ9z8n2tlbU8Z2+Wn/3NPr989z57/7bmZleenLUS31zjkS10KiOBN83GbJI/C94u9ZabOOGIxXFqJ8opEwr3NfkEebEOh4s9eeUsjqu9PHdVUprekUQfsIxJQ2tvYmFcPXvzK3C65O4fcf1Uhx3twgFyHQncmiwXrfuCwQMDKA1JxuJqpePsQb3j9DvSWHmxgwWV2NudJGufYQiNwRg9Uu9Iog8YY8ehBQ1kqukUTfZOjp+V6yjE9Ws5sQOTctIUO53ggJ4/Y054ttG3pFLpCqbp6Ba9o/Q70lh5KaUU23PLmBxux9JUijkpTWZJ7ic0gwHz6Nn4NRSRGNjKLrmIXVwnpdy05W+jyBHJLanybFFfMj5xIMe0cfg1l+KsKtI7Tr8ijZWXOl/ezIWKZuaHF4PJgnlkqt6RRB8yj54JBiNZUaWcOFdLVb1cxC6+vs4Lefh11nMuaDJxkXJ9pi8xGDQiJqfRrkzUHtysd5x+RRorL7U99yIDLC4iGvIxJ05Ds/jrHUn0IYNfMKZhycS15GM1uPjkSJnekYQXqju0mUa3HwnTZukdwrctiQAAIABJREFURdwAKROHccw1AsulXNztzXrH6TeksfJCjS0dHCyoYsmQKnB2Yk66Te9IQgfmpDQ0Ryt3DKljT/4l2jqcekcSXsTVUEFA/SmOa2MYPyJK7zjiBrCYjWij0jDhovbwx3rH6TeksfJC2w6W4HS5GOs4gSFiGMaIoXpHEjowDhqFIWQwyYbPaOtwsTdfnh8oeq5y/4e4lcaAiXMwyPWZPmta6iTOOAfhPrUd5ZKDr74gjZWXcbsVH+UUM2twG8bmciwyxUK/pWka5jFpmBsuMDW6nU+OXMTtVnrHEl5AtbdgK8khz53I1Mmj9I4jbqBAPzO1MbPwd7fQkL9L7zj9gjRWXibvXC2Vda3MCSwEiz+m4fL4if7MPCIVTBbmh56jsr6N3MIqvSMJL1Cx/0PMOGFMhkwI2g9MmjmTMmcobcc2oZTMe3ejySfKy2w5UELigE6Ca09cnijSbNU7ktCRZg3APPJWQquPERvoZMMuua1aXJtydmA8s4NTrjhuSZmodxzRBwaG+nNhYCpBnTW0FuXqHcfnSWPlRYouNVJY2sB3YktBM2C+aY7ekYQHsIzLALeLu2NKOXq6mks1dr0jCQ9WdWgbfqqN1uFzsFlMescRfSRpRjp1rgDq9m/QO4rPk8bKi3y0v4SBNicR1YcxJaZgCAjVO5LwAIYBUZiGTWZI4xECTC62HbmodyThoZTbheuzrVxwRTB5RorecUQfGjIohNOByYS0ltBaWqh3HJ8mjZWXKK+1c/R0Nd8Zcgnl7MAyfp7ekYQHsYyfh9bZyr0J1ezLL6fJLg9eFV9Uk7+XIFcjtXGzCfSTx9f0NyPSFmJ3W6jYs1bvKD5NGisvseVgCVajm+EtufglTMIYFqt3JOFBjFGJGKNGMLY9F5fTydZDpXpHEh5GKUVb7odUuYKZOPt2veMIHQyNjeBMwCTCm05hryzRO47PksbKCzS0dLDvRAXfHlaH1t5ESEqW3pGEBzJPmA8tNWQPa+CT3Iu0tDn0jiQ8SPWpXEIdlVyKupUBgTa94widDJ2VhRMjF3es0TuKz5LGygt8fLgUt9vNRMcRDOHx2OLH6h1JeCBT/ETMYYOYph2no9PJtsNy1kpcppSiJefvNLj9mTBngd5xhI7ihgymyH88UQ3Haa6UR2HdCNJYebjWdic7jpaxeGgLhuZKLBPmocksyeJLaJqBAVMXY2ooZf7QdrYdviiPuREAFB3YTYTzEpcGpxEcHKB3HKGzmNnfwoWBS5/+Te8oPkkaKw/3Se7lP463cgQtaCCmhCl6RxIeLHDcLDS/YG6zHKe1w8n2XLlDsL9TSlG/8x1q3UGMnbNI7zjCA8TEDeZMwM1EN+ZTe0Hmvutt0lh5MHu7g80HSlg8pBFzYwnWm7PQDDLvjPhqBrMVy4QFWGpOkx5nZ8vBUjo6XXrHEjoqObSLcGcVtfFzCPCXa6vEZUPS7qITE2c2/lnvKD5HGisPtvlACe0dDmYbjqANiMI0IlXvSMILmMekofmHMNeSS0tbJzuPyXUU/ZXb5UIdX0+1CmHsbZl6xxEeZHBMFOeCbyGy8STV52Veq94kjZWHamzp4OPDpSxJaMTYVPaPs1VGvWMJL6CZLFgmLcJSf570wU18dKBEzlr1U+dyPiFc1eEYuwir1ax3HOFhRs79Fq3KQvXOd/WO4lOksfJQH+RcwOV0Md11EEPoYEzDp+kdSXgR8+iZaIHhzLUepdHewZZDMmdNf+NwODB/9iGVKoypCxfqHUd4oNDwUGrjZhHXeY5zecf0juMzpLHyQDWNbew4WsY9wxswNFdgmXwHmkF+VaLnNKMZy82LsTSWkBV/+axVo8zG3q+c3PYBYTTiHrcIs0muzRRfbsqSpdiVjfYDq3G53XrH8Qny19oDbdhTjFFTTHEcwBAWh2lYst6RhBcyj5yOFhzFLMMRXE4X6/ec1zuS6CON9Q1ElWymwjiYxGmz9I4jPJh/YBANw+cRpy5RsGub3nF8gjRWHqa81s7eE+X8S2INWnMV1uRsNE1+TeLr0wwmrJOzMDZe5J4Rjew6donyWrvesUQfOLvpbfzpYMDs+zHI2W7RjRFpi6jWwgku3EB7a5vecbyefOI8iFKKv247Q4jFybim3RijRmCMn6h3LOHFTMOnYQiLJbl1D/5mN+99KnPW+LrzpwoZ3pJLWcgkIoeP0juO8AJGoxHjLfcQqrVwarNMGvpNSWPlQXJPV3PyfB0/GHYeraMF6/T7ZJZ18Y1oBgPW1KVo9loeTijl2NkaCkvq9Y4lbhCXy03L7pV0YmbY/O/qHUd4kfgJUyixjSKueg9lxfI4rG9CGisP0dHp4m+fnGHCwE4GVR/AnDQL48B4vWMJH2AanIQp4RaG1u5laFAn724/i9ut9I4lboC8XdsZqkppSJiLX3Co3nGElxky7340oHzbX2SM+AaksfIQH+QUU9vUztLQXDDbsExZonck4UOs074NaDwYfZLiimY+PSqThvqa+oYWQgvXU6eFknhblt5xhBcKihxMfdwsRrnPcHDXHr3jeC1prDxARV0rmw+UcNfwZqy1p7EmZ2OwBekdS/gQQ2A4lkkLGVCbT3qcnfd3FlHX1K53LNFLlFLkbVhJuKEZ/+lLMRhlMlBxfYbNuZsWLYiBBe9TU9ekdxyvJI2VzpRSrPr4NP5mN7d27sYQFot5TJresYQPsoyfhxYUQaZxH5rbxcqtp1FKTvf7giP7c5nQdoDq0AlEjJHpWcT1M1hsWGbcT5SxkVMfvC1jxHWQxkpnh05VcfJ8HT9MuIBmr718obE8ukbcAJrJgjXlHgxN5TwyqoJjZ2vIPV2tdyzxDdU12Ak6tooOzY/4hQ/pHUf4gPDRyVSF3cy4tsMc339Y7zheRxorHdU3d/D2lkJSouzEVO3GPGompsFJescSPswUfzOm+EkMq9rOxIhOVn18mtZ2p96xxHVSSnF8/dsMNtZhTP0XjH5yCYHoHUMXLKNV8yfw2CqqauUrwa9DGiudKKX4n00F4OzkbututIAwrCnf0TuW8HGapmGd8QCaycZ9gftotrfz/k6Z28pbHc7JZVL7AWrCxjNwbIrecYQPMfoFYJ3xAFHGBvLX/RmnSx5301PSWOnk06NlnDhfx/9JLMLQUoVt9kNoFj+9Y4l+wOA/AOuM+zE3lvCD4aXsOFrGsbM1escSX9PFykaCjq+iw+DHkAUP6x1H+KCwpCk0RiWT7Mzlk61yl2BPSWOlg/JaO6u3nyU9zk5UVQ7msenyFaDoU+aEKZiGT2NU/R4mR7Tz5ocFcpegF2nrcFKw/k1ijHVYb/2ufAUobpiYed+j3RTIyOLVfFYoE4f2hDRWfczpcvP6B58RaHKygB1oA6Kw3nKX3rFEP2Sbfh+aLZClAXtwOx28tvEzebq9F1BKsW3tBqZxHPuQ6YQkTdM7kvBhmjWAAfN+RKixFfv2P9HYLM8S7I40Vn3s/R1FnC9v4okhx9Ba6/Gb/TCayap3LNEPabZAbLMexNh0iSdHnOF0aQMb9xbrHUt0Y/eeY0xt+Ihm/xii0h/QO47oB2wxo3CMz2a0sYT97/8Fh9OldySPJo1VH9qbX87WQ6U8klBCUM0JrFO/jTEqUe9Yoh8zDZmAeVwGAytyuDehho37ijl1QZ4l6KlOF1cRmf8XNKORqMX/iiYTgYo+MnDqApoGjmda534+XLtV5re6Bmms+khRWSN/3nyKjJgGkhp2YUpMwTxurt6xhMA69W6Mg5OY2rSV8aEt/HHDSWoa5XS/p6mss1P+0Z8YbKzH/7ZHMAZH6B1J9COapjF44Q9ps4VzS+16tuzI0zuSx5LGqg/UN3fw0pp8EoLayXR+jCE8DtvMB9A0Te9oQqAZjNjmPIrmP4AH/D7F4rTz3+/l0dru0Dua+IdGeyd7V7/NzcYzOMZkEjB8ot6RRD+kWfyIWPSv2Iwu4gve4mBesd6RPJI0VjdYp8PFS2vyUI42HhmwE81gxG/u43JdlfAoBlsQfnMfx9DZws9iD1JV18LLa0/I3DUeoK3DyZa/rSbdsJ/2QRMJmy43uwj9GMNi8U9/nEGmRsx7/sjpYnl6w+dJY3UDOZxu/rDuBBfL6/lFTA7Glipscx7FECSn8IXnMQ4cim3m97DVF7E88QSnLtTx582n5FoKHTldbja8t5G5zk9oC01kYOZjaAYZtoW+bEPHY5z+PRJNFdRseomzpXJd5j+TT+gN4nS5+eP6E5wsquKXww7h33geW9r3McWM0TuaEF/JPCIVS3I2oTVH+WlCAXvzy9kgdwrqwulys2bNJ9xu/5DOgGgisp6Ui9WFxwi6aQauSd9inOkC5zf8kbOlDXpH8hjSWN0ALreb1zZ+xvEzVTyTkMuAxjPYZn4Pc6LMNyM8n2XSIszj5xPbcJgfDilk/Z5zbNh7Xs5c9SGH08U7733KjNr3UdYgBmY/JU9mEB4nZMoC3EkZTDMXUrj+T3Lm6h+kseplbrfijQ8KOHKqgqeH5xHWUIA1dSnm0TP1jiZEj2iahnXq3ZjH3MboloMsiy9m3e7zrNl1TpqrPtDe6eSdv21mbsO7WKxWwu/8OQb/EL1jCfGlgm+9B/eI2cy0nKR0w8sUFsvjsaSx6kXtnU5eWpNP7mcXeSb+EBH1eVim3IVlbLre0YT4WjRNwzr9PkwjUhnfvJtHhhbxYU4xf/vkrDRXN5C93cF7q9aTaV+L0T+Y0Lv+DcOAKL1jCfGVNE0jePb9uMYuJNl8lroP/5u9x0r0jqUrk94BfEVdUzu/fz+PhppqfhW7l4CWcqyp92EZO0fvaEJcF00zYJu1jHbNQNLpPfx0SBMvHHbT4XBx39yRmIxyXNabymvtfPz3dSxSn+AMiib8jqcw+A/QO5YQ3dI0jZDUu2jxD+amg3+laO/LrKn5F+647SYMhv43rZCMjL2guKKJ5/5yGK3pEs9GbyOgswa/uf9Hmirh9TSDEdusZVgm30FcSz7LY/dx4Hgx/+9vx2iyd+odz2ccLaxk7ztvkKU+xhUaz8Aly6WpEl4ncOJcLLN/QIK5mkln/sg7q7fR1uHUO1afk8bqG1BKsfv4JX67MpfxxvP8JHgzZoPCf/HTmOJlAj/hGzRNwzr5DmyzlhHeVsKK2B00VpTx67cOcb68Se94Xs3tVmz49CQdH/+eDEsuKn4K4dk/R7MG6B1NiOtiG5lC4OJfEGQzML/xHTb8z1uc6WcXtctXgdepsaWDP28upLDoEg9HHGWksxBDWAJ+cx7DEBiudzwhep151Ay0gFDY9jK/CP2QdR0p/Galg/vmjmTG+EHyJIGvqaahjY0f7CKteSOhllZMKUsJHDtH6ii8njF6BGHfeZ7aTS+TWbOHI+sucmLMXSycNQazyffP50hjdR0On6riL1sKiXFd5NeR+7E6m7FMvgPLpEVoBqPe8YS4YUyxYwlY8hzt21/lzspPuSl8NK9/1M6RwmrunzeKsGCb3hE9nlspdhy+QO3+9dxhOY7bL4iA+U9jkgeyCx9isAUx8M7/S+vhjdx8dB1NZ17mvXPTmZa5iIQY3/6aWxqrr+FiVQvv7SjiYvEF7gs7QZL7NJpfFH63/Rhj5HC94wnRJwxBA/Fb9HM6j37AyNx1rIiuZOWlyTzzegN3pyUyc+JgDHLW5UuV19rZ/MGnTLdvY4q1AVdcMqFp92OwBekdTYhep2kGAqZk4Ro6gc5tr7OoeRuF606QE53J3NunEBHim3OzSWPVA3VN7azdfY6jJ0qYF3CSB0MLMGgalokLsUxaiGaWo3TRv2gGI9bJWRhjxqDtfJ2HOrdRbBzGyo/ryTkZx7fSEkn08aPSr6O+uYMtu04woGgr2dZCnP4DsKX9K2a5FlP0A8aIoYR/ewWt+Z+QcOh9hte8xcG399A5Mp3ZMyYQ5G/RO2KvksbqGkoqm9meW0bBZ2dJNZ/i1+FFWNztmBJTsU7JlmupRL9nih5BwF3P4zjxMUNz1/N0aAl7m27iv1dWMmJ4LNkzE4iNDNQ7pm5a2x1s23cKdXILcyynMNtcaKNuJyz1W3JAJvoVzWAgYEI6fiNuoSnnfaYW7YPiMxw+k0Bt3GympUzymbFCGqvP6XC4OHamhk+PlEDlaWb6FZIdVAoamIdMwnLzYowDh+odUwiPoRlNWCbMxzQihY6D7zPj9B5SwwrYXzmCF98azfCRCaQnxzE8JrjfXJhdXmtn74ECjEW7udXyGVarA9eQWwhKycYwIFrveELoxuA/gJDbl+Gelk3dgY1MLNqNqfINzrwXxdHAsUROmMG40bH427y3PfHe5L2oqbWT42drOHG6HOfFEyQZS/ie5SL+wR1gDcSSlIk5KQ1D0EC9owrhsQz+IfjNfgjXhPl0Ht/E9DP7STEXcPzSMFb/LZHO0OHclhzH1DFRWM2+d5OHw+kmv6iG04f3E1N/mDnmUoxWReegCQRMvxtjWIzeEYXwGIaAUAbe9l3cKXdgz/uEmII9jOj4BMeBTzmyJ46mkJEMHDmREaOHe91NMf2qsVJKYW93Ulln52JpBbWXynDUlDCgvZwhphq+Y2zA6O/GbfLDPHQS5qE3YxoyEc3kW9//CnEjGUNj8Jv9MO7kbDrztzLp1E4mmc7R6Api/46hPL99BIMTEkgeFcm44eFe3WQ5XW5OFddxOi8PU9kxxhrPs8DYjMPPD8PIdALG306QPJJGiK9k8AsmaOqdBN5yB86qc7Qe30lS6RGsrRfg2MfUHgnkpDYYV/BgAqOHMGBwPOGDBhES7OexN8n4fGPVZO/kL1sKKa+1k9qxmzHGC4QZWonU3F3rOPytEBaPLWYaptixGAeNRDP4fGmEuKEMgeHYUr6DdUo2zuJcjKf3MvfiCTLIp6Y8mJMXBrPLHYctNomRQyMYPSSUwREBHjtYwuWDs0u1rRSeLaPp3AmsdWdIMpaQbrTjtmh0hCViHns3gYlT5YBMiK9B0zTMUcOJmjscpb6Hq66MmtPH0EpOMrqpGFvLaTgLnAWnMnBJWek02HAYbDSaI8gdkE6gv4XMafG6n+Hy+e7B5VZ0OlwMHhhAlDMcMNAUGE7wwCgCwyMwhseiBUeiab4/aZkQetBMVsyJKZgTU3Db63GeP0xkSR7hlwqY5T6Fs3Y7pVWhHN0XwUeGaEyRwwgbFMuQ6CDio4IIDbLqcm2W262obWrnYkUD1SXn6ag8j6W5jDgqmGysw6CBy2KkPWwEpjEp2BImo9l84+JbIfSkaRqm8FiiU2IhZSEA7rYmGssu0FhRgqOuAkdrM+52OyZXK6bOFoorm3G5FLckRXlHY7Vx40ZeeeUVnE4n999/P0uXLr1qeUFBAcuXL8dut5OcnMyvf/1rTCbP6NlCg6w88e0rtzSP0zWLEP2dISAUy9h0LGPTUc5OXOWncJZ9Rvyls8TXncXgLoCGT+msN1KVH0yeawB1Wghuv9DLZ8BCIvALHUhQUCDBgVaCAywE+pmxmo09ntFZKUWn001ru5PWDict9naa6utpa6jD0VyHq6kGo70am6OegVoTQ41NJP7jDLfTbKYtMBbXkFQCEsZhjBhGiJyZEuKGM/gFE5o4jtDEL/87fnMf57mWbrufyspKXnjhBdasWYPFYuGee+5h6tSpJCb+7yzBP/vZz/j3f/93Jk6cyNNPP83q1au59957b2hwIYR300wWTHHjMcWNxwYotxN33UVc1cUYasuIqL5IRGM51s5icAGN//i5cPmrALuy0uS2UqnMdGDCiQm3ZsatGUHT0AwGlAINhaZcGP7xY1IObFonNs2Bv9ZJlNbBYE1dlc2FkbaAUFz+0XSETcYaNxxb1DC04ChCDXJ2Wwjx1bptrPbt28e0adMICQkBICMjg82bN/OjH/0IgLKyMtrb25k48fJZoezsbH7/+99/rcbKYOjZaf6ertcfSC0uM2DEFuB7D6w1GI2o6/wde+2+YTBjjByGOXIYNuDKXOTK7US1NqLsDbhbG2hvbkCzt2Brb8bS3kqIox3N5QBXJ5rbiQEnABqglBvQcGtGFGaUwYrbYEGZrGC2oZmtdNoCsQSG4BccgiUwBC0wFM0vmDAfuzzgevYLzei7ny+v/ZzcAFKL/9VdLXpSq24bq6qqKiIiIrpeR0ZGkpeX95XLIyIiqKys7PZ//M9CQ3v2wQ0Pl+sXrpBaXBFIyh1L9A7hUXxz3wgB4vUO4dWub7+Qz1d/4JtjxvXpjVp0e0jmdruvunBUKXXV6+6WCyGEEEL0F902VtHR0VRXV3e9rq6uJjIy8iuX19TUXLVcCCGEEKK/6LaxSk1NJScnh7q6Otra2ti6dSszZ87sWh4TE4PVauXIkSMArF+//qrlQgghhBD9haaUUt2ttHHjRl599VUcDgd33XUXDz/8MA8//DA//vGPGTduHKdOneKZZ56hpaWFm266id/85jdYLHILshBCCCH6lx41VkIIIYQQonu+dT+xEEIIIYSOpLESQgghhOgl0lgJIYQQQvQSaayEEEIIIXqJNFZCCCGEEL3EoxqrS5cusXTpUubNm8cPf/hD7Hb7F9YpKytj0qRJZGVlkZWVxbJly4DLM77/x3/8B/PmzSMzM7NrXi1v1ZNaVFVVsWzZMrKysrjzzjvJyckBwOFwcPPNN3fVKCsrC5fL1deb8I1t3LiRzMxM5s6dy6pVq76wvKCggOzsbDIyMli+fDlO5+VnxPWkdt6ou3ps27aNrKwsFi9ezKOPPkpjYyMAa9eu5dZbb+3aF1544YW+jt7ruqvFSy+9RFpaWtc2X1nHF/eNa9WioKDgqnFgxowZLFy4EPDN/QKgpaWFhQsXcvHixS8s629jxrVq0Z/GiyuuVY9eHTOUB/n+97+vPvjgA6WUUi+99JL63e9+94V1Nm/erH75y19+4f2PPvpIPfzww8rlcqlz586p9PR05XA4bnjmG6UntXjyySfVypUrlVJKFRUVqdTUVOV0OlV+fr568MEH+zRvb6uoqFBpaWmqvr5e2e12tWjRInXmzJmr1lmwYIE6evSoUkqpX/ziF2rVqlVKqZ7Vztt0V4/m5mY1ffp0VVFRoZRS6sUXX1TPPfecUkqpFStWqI0bN+qS+0boyb7xgx/8QOXm5n7h3/ravtGTWlzR2tqqFixYoA4dOqSU8r39Qimljh07phYuXKhuuukmVVpa+oXl/WnMuFYt+tN4cUV3+0Zvjhkec8bK4XBw6NAhMjIyAMjOzmbz5s1fWC8/P5/Tp0+TlZXFd7/7XQoLCwHYuXMnmZmZGAwGhg0bxqBBgzh69GifbkNv6Wkt0tPTu44+4+Pj6ejooLW1lfz8fOrq6sjOzubuu+/m4MGDfZq/N+zbt49p06YREhKCv78/GRkZV9WgrKyM9vZ2Jk6cCPxvjXpaO2/TXT0cDgfPPvssUVFRAIwaNYry8nLg8mdm7dq1LFq0iJ/+9KddR6beqrtaAJw4cYJXX32VRYsWsWLFCjo6Onxy3+hJLa549dVXmTJlCsnJyYDv7RcAq1ev5tlnn/3Sx6r1tzHjWrXoT+PFFdeqB/TumOExjVV9fT2BgYGYTCYAIiIiqKys/MJ6VquVxYsXs3btWpYtW8Zjjz1GZ2cnVVVVVxUsIiKCioqKPsvfm3pai4yMDAYMGADAG2+8QVJSEkFBQWiaxu233867777Lr371K37yk59QV1fXp9vwTVVVVREREdH1OjIy8qoafH75lRr1tHbeprt6hIaGkp6eDkB7ezuvvfYac+bMAS7X4NFHH2XDhg0MGjSIFStW9G34XtZdLex2O0lJSfzsZz9j7dq1NDU18Yc//MEn943uanFFc3Mzq1ev5kc/+lHXe762XwA8//zzXY3j5/W3MeNatehP48UV16pHb48Zpl5N3kMfffQRv/nNb656Lz4+Hk3Trnrv868BHn/88a7/njVrFv/1X//FuXPncLvdV62vlMJg8Ji+8St9k1pc8dZbb/Huu++ycuVKAO65556uZWPGjGH8+PHk5uZ2fXC8wZf9Pv/59Vct//x6cO3aeYvu6nFFc3Mzjz32GKNHj+bOO+8E4OWXX+5a/tBDD3UNqN6qu1oEBATwpz/9qev1gw8+yNNPP829997rc/tGT/eLDRs2MGfOHMLDw7ve87X9ojv9bczoif4wXvREb48ZunQe8+fPZ9euXVf9vPnmmzQ3N3ddZF1dXf2lp+zefvtt6uvru14rpTCZTERHR1NVVdX1fk1NzVee8vMk36QWAL/73e947733WLVqFYMGDQJg3bp1lJSUdK2jlMJsNt/4jelF0dHRVFdXd73+fA0+v/zK7zssLKzHtfMm3dUDLh+R33vvvYwaNYrnn38euDxwvvXWW13rKKUwGo19kvlG6a4Wly5d4v333+96fWWM8MV9oyf7BVy+UDkzM7PrtS/uF93pb2NGd/rLeNETvT1meMwpHbPZTHJyMps2bQIuNwczZ878wnqHDh3qKsDBgwdxu90kJCQwc+ZMNm7ciMvl4sKFCxQXFzNu3Lg+3Ybe0tNavPXWWxw4cIB33nmH6OjorvcLCwt58803ATh37hwFBQVMnjy5b8L3ktTUVHJycqirq6OtrY2tW7deVYOYmBisVmvX3Z/r169n5syZPa6dt+muHi6Xi0ceeYT58+ezfPnyrqMqf39/Xn/9dY4fPw7AypUrvf4ItLta2Gw2/vM//5PS0lKUUqxatYr09HSf3De6qwVc/iNx8uRJJk2a1PWeL+4X3elvY8a19Kfxoid6fcz4mhfW31AXL15U9913n5o/f7568MEHVUNDg1JKqb/+9a/qxRdfVEpdvgvmgQceUAsWLFDZ2dmqoKBAKaWU2+1Wv/3tb1VmZqbKzMxUu3fv1m07ekN3tXC73So5OVnNnj1bLV68uOunoqJCNTc3q8cff1wtWLBALVy4UOXk5Oi8Nddnw4YNasGCBWru3LnvLLA8AAAC80lEQVTqtddeU0op9dBDD6m8vDyllFIFBQVqyZIlKiMjQz3xxBOqo6NDKfXVtfN216rH1q1b1ahRo67aF55++mmllFKHDh1Sd9xxh5o3b5565JFHVFNTk56b0Su62zc2b97ctfznP/+5T+8b3dWipqZGpaamfuHf+eJ+cUVaWlrXnV/9ecxQ6str0d/Gi3/2VftGb44ZmlJK3eBmUAghhBCiX/CYrwKFEEIIIbydNFZCCCGEEL1EGishhBBCiF4ijZUQQgghRC+RxkoIIYQQopdIYyWEEEII0UuksRJCCCGE6CW6PCtQCCG+iWeeeYbw8HB+8pOfAJdn0d66dStLlizhlVdeweFwYLPZeOqpp5g0aRJFRUUsX76czs5OlFLcddddLF26VOetEEL4IjljJYTwOkuXLuXvf/87TqcTgNWrVzNjxgxeeOEFXnvtNdatW8dzzz3H448/TmtrK2+88Qa33XYba9as4bXXXuPw4cO43W6dt0II4YvkjJUQwuskJSURGxvLjh07GDZsGFVVVbhcLqqqqnjggQe61tM0jZKSEtLT03nqqafIy8sjJSWFZ555BoNBjiuFEL1PHmkjhPBKGzduZNOmTQwdOpSwsDBsNhtHjhzhxRdf7FqnvLycyMhIjEYj9fX17Nu3j5ycHLZv386aNWuueni5EEL0BjlkE0J4pYyMDAoKCtiyZQtLliwhJSWFvXv3UlRUBMDOnTtZvHgx7e3tPPnkk2zatIkFCxbw7LPPEhgYSElJic5bIITwRfJVoBDCK1ksFjIyMqipqSEsLIywsDBWrFjBE088gVIKk8nEK6+8QkBAAI8++ijLly/n3XffxWg0MmfOHKZMmaL3JgghfJB8FSiE8Eqtra3cd999/Nu//RsTJ07UO44QQgDyVaAQwgvt3r2b2bNnM2PGDGmqhBAeRc5YCSGEEEL0EjljJYQQQgjRS6SxEkIIIYToJdJYCSGEEEL0EmmshBBCCCF6iTRWQgghhBC95P8DALObqIZlVHwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (10,8))\n",
    "sns.distplot(control_group)\n",
    "sns.distplot(exposed_group);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "convert to Binomial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'probability')"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAAF2CAYAAAC28fR/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xU1b3///fkNmGSYIxOgBMUEAQqt9hGyQNrOPWnRAkRjYB4ISAVpe0BTG0syMUjhaZgOODBqsix1AsU+CFNikLAUpEiFCXVIyhe0FI1YkiISMh1ktnfPzhME5IJkyEzOzt5PR+PPB7Za82e+ewsZvJmZe29bYZhGAIAAAAQcCFmFwAAAAB0FoRvAAAAIEgI3wAAAECQEL4BAACAICF8AwAAAEFC+AYAAACChPANAAAABEmY2QUE07ffVsjt7pyXNb/kkmidOHHa7DIQYIxz58A4dw6Mc+fAOHc8ISE2XXxxlNf+ThW+3W6j04ZvSZ362DsTxrlzYJw7B2/jHLHtNUlS7S1pwSwHAcL7uXNh2QkAABYT/tZuhb+12+wyAPiB8A0AAAAECeEbAAAACBLCNwAAABAkhG8AAAAgSDrV1U4AAOgIKhYtMbsEAH5i5hsAAAAIEsI3AAAWY897Rfa8V8wuA4AfCN8AAFhM2IG3FXbgbbPLAOAHwjcAAAAQJIRvAAAAIEgI3wAAAECQcKlBAACsJsJudgUA/ET4BoBOpl5Sjcvt2baHhyi0mfaGfWhfKhYsNLsEAH4ifANAJ1PjcmvH3456tkcl95YjPKRJ+9k++//1NUQoBwD/EL4BoANqy1lsb6HcEc5pQ2axb1gnSaq5826TKwHQWoRvAOiACMwdW9jB/5VE+AasiE9hAAAAIEiY+QYA+IUTNAGg9QjfAAC/sLQFAFqP8A0AFuZt9hkdmxEdY3YJAPxE+AYAC/M2+4yOrXL2PLNLAOAnpkcAAACAICF8AwBgMZEv/V6RL/3e7DIA+IFlJwCANsVVUAIv9OPDZpcAwE+EbwBAm+IqKADgHeEbACyg4WxycVmlql1urmoCABZE+AYAC2g4mxwVZVdFRQ1XNQEACyJ8AwBgMcYll5pdAgA/Eb4BALCYyqxss0sA4CcWDAIAAABBQvgGAMBiIp9fpcjnV5ldBgA/sOwEABAUXP+77YT+43OzSwDgJ8I3ACAouP43ALDsBAAAAAgaZr4BoB3xtjQDANAxEL4BoB3xtjQDaMj9bz3NLgGAnwjfAABYTNVPZ5hdAgA/BfRvmVu2bNHo0aM1atQorV27tkn/4cOHlZGRodTUVM2dO1d1dXWSpAMHDigjI0Pp6emaPn26vvvuO0nSqVOn9MADD+iWW27RPffco5KSkkCWDwAAALSpgIXv4uJiLV++XOvWrVNeXp42bNigI0eONHpMdna2FixYoO3bt8swDG3cuFGSNGfOHC1dulRbtmxRv3799Pzzz0uSVqxYoaSkJG3btk3jx4/X4sWLA1U+AADtVpenV6rL0yvNLgOAHwIWvvfu3avk5GTFxsbK4XAoNTVVBQUFnv6ioiJVV1crMTFRkpSRkeHp37p1q/r16yeXy6Xi4mJ17dpVkrRr1y6lp6dLksaMGaPdu3fL5XIF6hAAAGiXQr7+SiFff2V2GQD8ELDwffz4cTmdTs92fHy8iouLvfY7nU5Pf3h4uD7++GONHDlS+/fvV1paWpN9wsLCFB0drbKyskAdAgAAANCmAnbCpdvtls1m82wbhtFo+3z9AwYM0N69e7V+/XplZWVp/fr1TV7DMAyFhPj+/4dLLolu7WF0KE5njNklIAgYZ2srLqtUVJS9UVtkZLgkNWqPirI32+7t8WfbnXGOJq/hrf18z9VW7ZH2MFXW1DVqd9jDFBMVoc7O6/u5y5mfjYP3e4fA53bnErDw3b17dx04cMCzXVJSovj4+Eb9DU+YLC0tVXx8vGpqavTXv/5VN954oyTp1ltv1ZIlSySdmT0vLS1V9+7dVVdXp4qKCsXGxvpc04kTp+V2Gxd6aJbkdMaopKTc7DIQYIyz9VW73KqoqGncVn1med3Z9qgouyoqapq0e3t8w/aSkvImr+Gt/XzP1Vbt1dWuZi+vWF3Z+LGdTUvv56iqWklSBe93y+Nzu+MJCbG1OOEbsGUnI0aM0L59+1RWVqaqqirt2LFDKSkpnv6EhATZ7XYVFhZKkvLz85WSkqKwsDA9/vjjOnTokCRp27Zt+v73vy9JGjlypPLy8iSdWReelJSk8PDwQB0CAADtUn2fK1Tf5wqzywDgh4DNfHfr1k1ZWVnKzMyUy+XSuHHjNHToUE2bNk0zZ87UkCFDlJubq3nz5un06dMaNGiQMjMzFRoaquXLl2vBggWqr69Xt27dPFc1mTVrlmbPnq20tDTFxMQoNzc3UOUDQEBxJ0tciOofP2h2CQD8FNCb7KSnp3uuTnLW6tWrPd8PHDhQmzZtarJfUlKSNm/e3KQ9NjZWzz77bNsXCgBBxp0sAaBzYpoFAACLcSx/Qo7lT5hdBgA/cHt5AAAsxnai1OwSAPiJmW8AAAAgSAjfAAAAQJAQvgEAAIAgYc03AAAWUz/ge2aXAMBPhG8AQLvk7VrooeaU065UT5pidgkA/ET4BgC0S96uhe7gZkQALIxPMAAALMbxm0Vy/GaR2WUA8AMz3wAQQNxGHoFgO11udgkA/ET4BoAA4jbyAICGmH4BAAAAgoTwDQAAAAQJy04AALCYuiHDzC4BgJ8I3wAAWEzNnXebXQIAP7HsBAAAAAgSwjcAABYTtXCBohYuMLsMAH5g2QkAAFZTW2N2BQD8RPgGAFiKtxsXhZpTDgC0CuEbAGAp3m5c5ODOoQAsgE8qAAAAIEiY+QYAwGLqkq41uwQAfiJ8A0Ab8LYOGQiEmtvuMLsEAH4ifANAG/C2DhkAgIaYlgEAwGKi5v1SUfN+aXYZAPxA+AYAAACChPANAAAABAnhGwAAAAgSwjcAAAAQJFztBAAAi3Fdl2J2CQD8RPgGAMBiam9JM7sEAH5i2QkAAFZTU3PmC4DlMPMNAOgQvN1lNNSccgIq6lcLJEkVi5aYXAmA1iJ8AwA6BG93GXWE80deAO0Hn0gAAABAkBC+AQAAgCBh2QkAtIK3dcUAAPiC8A0AreBtXTEQTLU33Gh2CQD8FNDpmi1btmj06NEaNWqU1q5d26T/8OHDysjIUGpqqubOnau6ujpJUmFhocaNG6exY8dq8uTJKioqkiS9/fbbGj58uMaOHauxY8dqzpw5gSwfAIB2yXXDTXLdcJPZZQDwQ8DCd3FxsZYvX65169YpLy9PGzZs0JEjRxo9Jjs7WwsWLND27dtlGIY2btzoaV+0aJHy8/OVnp6uRYsWSZIOHTqkqVOnKj8/X/n5+crJyQlU+QAAtFu2U9/Jduo7s8sA4IeAhe+9e/cqOTlZsbGxcjgcSk1NVUFBgae/qKhI1dXVSkxMlCRlZGSooKBAtbW1mjVrlgYOHChJGjBggI4dOyZJOnjwoPbs2aP09HRNnz7d0w4AQGfiWPprOZb+2uwyAPghYOH7+PHjcjqdnu34+HgVFxd77Xc6nSouLlZERITGjh0rSXK73Xrqqad0441n1rbFxMRo0qRJ2rJli0aOHKmsrKxAlQ8AAAC0uYCdcOl2u2Wz2TzbhmE02j5ff21trWbPnq26ujo9+OCDkqSFCxd6+u+66y4tW7ZM5eXliomJ8ammSy6J9vt4OgKn07efE6yNcQ6s4rJKRUXZG7VFRoZLUlDbo6Lsfj2PM87R5Bi8tZt1bG3d7oxzyKq8vp+7REiSHLzfOwQ+tzuXgIXv7t2768CBA57tkpISxcfHN+ovKSnxbJeWlnr6Kyoq9JOf/ESxsbF65plnFB4eLrfbrVWrVumBBx5QaOi/bhbc8PvzOXHitNxu40IOy7KczhiVlJSbXQYCjHEOvGqXWxUVNY3bql2SFLT2qCi7Kipq/HqekpLyJsfgrd2MYwtEu1XfEy29n6OqaiVJFRY9NvwLn9sdT0iIrcUJ34AtOxkxYoT27dunsrIyVVVVaceOHUpJSfH0JyQkyG63q7CwUJKUn5/v6c/OzlavXr20YsUKRURE/N+BhOj111/X9u3bJUl5eXkaNmyYHA7rzmgAAACgcwnYzHe3bt2UlZWlzMxMuVwujRs3TkOHDtW0adM0c+ZMDRkyRLm5uZo3b55Onz6tQYMGKTMzUx9++KF27typfv366fbbb5d0Zr346tWrtWTJEs2fP1+//e1vFRcXp6VLlwaqfAAA2q3am9PMLgGAnwJ6k5309HSlp6c3alu9erXn+4EDB2rTpk2N+q+66ip9/PHHzT7flVdeqfXr17d9oQCADu3cO5Paw0Pk+6LF9sf1w5TzPwhAu8QdLgEAHd65dyYdldxbjvCA3mcuoGz/d86U0eCqYQCswbqfPAAAdFKOJ3PleDLX7DIA+IHwDQAAAAQJ4RsAAAAIEtZ8A0Azzj1BTzpzkh4AABeC8A0AzTj3BD3pzEl6AABcCMI3AAAWU3NrhtklAPAT4RsAAIupu3a42SUA8BMLGAEAsJiQoq8UUvSV2WUA8APhGwAAi+nyzEp1eWal2WUA8APhGwAAAAgSwjcAAAAQJIRvAAAAIEgI3wAAAECQcKlBAECn5e1OpqHmlOOzmvETzS4BgJ8I3wCATsvbnUwd4e37D8N1w642uwQAfmrfny4AAKCJkM8/U8jnn5ldBgA/EL4BALCYLr97Tl1+95zZZQDwA+EbAAAACBLWfAPo1M494c4KJ9sBAKyL8A2gUzv3hDsrnGwHALAufsMAAAAAQcLMNwAAFlN97xSzSwDgJ8I3AAAWUz/we2aXAMBPLDsBAMBiQj86rNCPDptdBgA/EL4BALCYyJd/r8iXf292GQD8QPgGAAAAgoTwDQAAAAQJ4RsAAAAIEsI3AAAAECRcahAAgHPU68zdTxuyh4co1Jxymqia+oDZJQDwE+EbAIBz1Ljc2vG3o43aRiX3liO8ffzB2H1FX7NLAOCn9vEpAgAAfBb2v+8q7H/fNbsMAH5g5hsAAIux///rJUl1w642uRIArcXMNwAAABAkzHwD6PDa+8lzAIDOg/ANoMNr7yfPAQA6D37zAAAAAEES0PC9ZcsWjR49WqNGjdLatWub9B8+fFgZGRlKTU3V3LlzVVdXJ0kqLCzUuHHjNHbsWE2ePFlFRUWSpFOnTumBBx7QLbfconvuuUclJSWBLB8AgHap6iczVPWTGWaXAcAPAQvfxcXFWr58udatW6e8vDxt2LBBR44cafSY7OxsLViwQNu3b5dhGNq4caOnfdGiRcrPz1d6eroWLVokSVqxYoWSkpK0bds2jR8/XosXLw5U+QAAtFvuhJ5yJ/Q0uwwAfvApfH/77betfuK9e/cqOTlZsbGxcjgcSk1NVUFBgae/qKhI1dXVSkxMlCRlZGSooKBAtbW1mjVrlgYOHChJGjBggI4dOyZJ2rVrl9LT0yVJY8aM0e7du+VyuVpdGwAAVhb29n6Fvb3f7DIA+MGn8J2WlqaHH35YBw4c8PmJjx8/LqfT6dmOj49XcXGx136n06ni4mJFRERo7NixkiS3262nnnpKN954Y5N9wsLCFB0drbKyMp9rAgCgI7D/abPsf9psdhkA/ODT1U7+8pe/6LXXXtPSpUtVVVWliRMnauzYsYqOjva6j9vtls1m82wbhtFo+3z9tbW1mj17turq6vTggw82+xqGYSgkxPeVM5dc4r3ezsDpjDG7BAQB49xUcVmloqLsjdoiI8PljHM06fPWfrZPUrtoj4qy+/U8Vj5mf9vb8pidcQ4Fk9f3c5cISZKD93uHwOd25+JT+I6MjNQdd9yhO+64Q/v379ejjz6q3Nxc3XbbbZo5c6YuvvjiJvt079690Ux5SUmJ4uPjG/U3PGGytLTU019RUaGf/OQnio2N1TPPPKPw8DMfhPHx8SotLVX37t1VV1eniooKxcbG+nywJ06clttt+Pz4jsTpjFFJSbnZZSDAGOfmVbvcqqioadxW7VJJSXmTPm/tZ/skmd4eFWVXRUWNX89j1WO+kPa2POZgvr9aej9HVdVKkip4v1sen9sdT0iIrcUJX5+njXfv3q0ZM2YoKytLN954o9avX68ePXropz/9abOPHzFihPbt26eysjJVVVVpx44dSklJ8fQnJCTIbrersLBQkpSfn+/pz87OVq9evbRixQpFRER49hk5cqTy8vIkSVu3blVSUpInmAMAAADtnU8z3z/60Y8UGxuru+++W0888YQiIyMlnTkZcsOGDc3u061bN2VlZSkzM1Mul0vjxo3T0KFDNW3aNM2cOVNDhgxRbm6u5s2bp9OnT2vQoEHKzMzUhx9+qJ07d6pfv366/fbbJZ2Z8V69erVmzZql2bNnKy0tTTExMcrNzW2jHwMAAOfH3VIBXCifwvfSpUt1zTXXNGo7cuSI+vXrp507d3rdLz093XN1krNWr17t+X7gwIHatGlTo/6rrrpKH3/8cbPPFxsbq2effdaXkgEAaHPt5W6plbN+EdTXA9B2Wvy0OHnypE6ePKlf/epX+u677zzbpaWl+o//+I9g1QgAABownE4ZDa4YBsA6Wpz5fvjhh/XWW29JkoYPH/6vncLClJqaGtjKAABAs8L37JYkuX6Ycp5HAmhvWgzfzz//vCRpzpw5ysnJCUpBAACgZREFr0kifANW1GL4/uyzz9S3b1/de++9+uCDD5r0Dxo0KGCFAUBreTsZDgCA9qLF8L1kyRI999xzmjFjRpM+m83W4smWABBs3k6GAwCgvWgxfD/33HOSztzhEgAAAMCFaTF8L1q0qMWd582b16bFAAAAAB1Zi+G7NbduBwAAwVH5yKNmlwDATy2Gb67lDQBA+2N0vcjsEgD4qcXwfdddd+kPf/iDrr76atlstib9f//73wNWGAAAaF74X16XJLluuMnkSgC0Vovh+8knn5Qkvfrqq0EpBgAAnF/EX/4sifANWFGL4Ts+Pl6SlJCQoL/+9a/au3evwsLClJKSomuuuSYoBQIAAAAdhU93n3j22WeVk5OjyMhIhYSEaN68eVq7dm2gawMAAAA6lBZnvs969dVXtXHjRkVHR0uSpk6dqrvvvlv33HNPQIsDAAAAOhKfwrfdbldUVJRn+6KLLpLdbg9YUQAAWEm9ztxhtSF7eIhCzSkHQDvWYvjesWOHJKlPnz766U9/qvHjxys0NFR5eXkaPHhwUAoEAKC9q3G5teNvRxu1jUruLUe4T6s7W61i/sKAPC+AwGsxfL/00kuNttesWeP5/sSJE4GpCAAAtIy/PgOW1arwDQAAzBex7TVJUu0taSZXAqC1fFrzffToUb388suqrKyUYRhyu9365z//qfXr1we6PgAAcI7wt3ZLInwDVuTTYrSHH35YLpdL7777rhISEnTkyBH1798/0LUBQLPqJVW63I2+6s0uCgAAH/g0811RUaHHH39cixcvVkpKijIzM3XvvfcGujYAaJa3k9sAAGjvfJr5jo2NlST16tVLn376qbp27SqbzRbQwgAAAICOxqeZ7169emnx4sW6/fbbNXfuXFVWVqquri7QtQEAAAAdik/h+z//8z+1e/duXXXVVZowYYL27NmjhQu5xigAAGaoWLTE7BIA+Mmn8N2lSxdde+21euONN9SzZ0/l5OSoa9euga4NAAAA6FB8Ct+7du3SL3/5S/Xr109ut1tffvmlli9frmuuuSbQ9QEAgHPY816RJNXcdofJlQBoLZ/C95NPPqmXX35ZV155pSTpgw8+0Pz587V58+aAFgcAAJoKO/C2JMI3YEU+Xe3EZrN5grckDRo0SIZhBKwoAAAAoCNqMXyfPHlSJ0+e1ODBg/X888+roqJCVVVVWrt2rZKTk4NVIwAAANAhtLjsJDk5WTabzTPL/cQTT3j6bDabfvnLXwa2OgAALKxeZ24K1ZA9PESh5pQDoB1oMXx/9NFHwaoDAIAOx9vdWB3hPq369C7CfmH7AzCNTydcut1uPf/889q9e7fq6up03XXXafr06QoL82l3AADQhioWcK8NwKp8+q/3smXL9Le//U2TJ0/Wfffdp3fffVdLlnCBfwAAAKA1fJq6/utf/6pXXnlF4eHhkqR///d/16233hrQwgAAQPPsG9ZJkmruvNvkSgC0lk8z34ZheIK3JEVERDTaBoBAqJdU6XI3+qo3uyigHQg7+L8KO/i/ZpcBwA8+zXwPHDhQv/71r3XvvffKZrPp5ZdfVv/+/QNdG4BOztvJagAAWJVPM9+PPfaYTp06pYkTJ2rChAkqKyvT/PnzA10bAAAA0KH4NPO9atUq/eY3vwl0LQAAAECH5tPM965du/x68i1btmj06NEaNWqU1q5d26T/8OHDysjIUGpqqubOnau6urpG/StWrNDKlSs922+//baGDx+usWPHauzYsZozZ45fdQEAYGVGdIyM6BizywDgB59mvnv27KmpU6fq+9//vqKiojzt9913n9d9iouLtXz5cm3evFkRERGaOHGihg8frn79+nkek52drUWLFikxMVGPPvqoNm7cqLvvvlvl5eXKycnRa6+9pvvvv9/z+EOHDmnq1Kl68MEH/TlWAAA6hMrZ88wuAYCffJr5jo2NVbdu3fTBBx+osLBQn3zyiT755JMW99m7d6+Sk5MVGxsrh8Oh1NRUFRQUePqLiopUXV2txMRESVJGRoanf+fOnerdu3eTcH/w4EHt2bNH6enpmj59uo4dO9aqgwUAAADM5FP4vv/++/XJJ5/ozTff1F/+8hd9+eWXmjFjRov7HD9+XE6n07MdHx+v4uJir/1Op9PTf9ttt+mBBx5QaGhoo+eMiYnRpEmTtGXLFo0cOVJZWVm+lA8AQIcS+dLvFfnS780uA4AffFp28uijj2rChAnKyMiQYRjasGGD5s6dqzVr1njdx+12y2azebYNw2i0fb7+5ixc+K/b6d51111atmyZysvLFRPj27q3Sy6J9ulxHZXTyfrAzqAjjXNxWaWiouyN2iIjz9xjoC3anXGOJq/hrb2tX/tC26Oi7J3umP1tb4/H7IxzyBde389ffCZJiulA7/fOrCN9buP8fArfVVVVuvPOOz3bkyZN0saNG1vcp3v37jpw4IBnu6SkRPHx8Y36S0pKPNulpaWN+s/ldru1atWqJjPi586Ot+TEidNyuw2fH9+ROJ0xKikpN7sMBFhHG+dql1sVFTWN26pdktQm7SUl5U1ew1t7W7/2hbRHRdlVUVHTqY75Qtrb4zH78j5t6f0cVVV75rk70Pu9s+pon9uQQkJsLU74+rTs5IorrtDf//53z/Ynn3yinj17trjPiBEjtG/fPpWVlamqqko7duxQSkqKpz8hIUF2u12FhYWSpPz8/Eb9TQ8kRK+//rq2b98uScrLy9OwYcPkcPg2ewAAQHvB3VuBzsunme+vv/5akyZN0oABAxQWFqYPP/xQTqdT6enpks5cUvBc3bp1U1ZWljIzM+VyuTRu3DgNHTpU06ZN08yZMzVkyBDl5uZq3rx5On36tAYNGqTMzMwW61iyZInmz5+v3/72t4qLi9PSpUv9OGQAAMzl7e6tjnCf5sQAWJhP4fsXv/iFX0+enp7uCehnrV692vP9wIEDtWnTJq/7n3tS55VXXqn169f7VQsAAB2FccmlZpcAwE8+he9rr7020HUAAAAfVWZlm10CAD/x9y0AAAAgSHya+QaAQKrXmTWwDdlZ+wp4Ffn8KklS9Y+54zNgNYRvAKbzdvIZgOaF/uNzs0sA4CemlgAAAIAgIXwDAAAAQUL4BgAAAIKENd8AAFiM+99avss0gPaL8A0AgMVU/XTG+R8EoF1i2QkAAAAQJIRvAAAspsvTK9Xl6ZVmlwHADyw7AQCgnWh4w6niskpVu9yyh4co9JzHhXz9VdBrA9A2CN8AALQTDW84FRVlV0VFjUYl95aDO74CHQbvZgAAACBICN8AAABAkLDsBAAAi6nvc4XZJQDwE+EbQNA0PJnsLDtrWYFWq/7xg2aXAMBPhG8AQdPwZLKzRiX3NqUWAADMwJQTAAAW41j+hBzLnzC7DAB+YOYbAACLsZ0oNbsEAH5i5hsAAAAIEsI3AAAAECSEbwAAACBIWPMNAIDF1A/4ntklAPAT4RsAAIupnjTF7BIA+InwDQBAO+ftBlWh5pQD4AIQvgEAaOfOvUHVD/7wtK7oeZFcj843rygAfiF8AwBgMeFVpxVSzjUTACvinQsAAAAECTPfANqct/WpAAB0doRvAG3u3PWpkjQqubcptQAA0J4QvgEAsJgTV3xPPfpeanYZAPxA+AYAwGI+/fd09UnuLYfZhQBoNRZhAgAAAEFC+AYAwGKufWmFYhc/ZnYZAPxA+AYAwGJCXLWy1dSYXQYAPxC+AQAAgCDhhEsAACzK2zX1Q80pB4APAjrzvWXLFo0ePVqjRo3S2rVrm/QfPnxYGRkZSk1N1dy5c1VXV9eof8WKFVq5cqVn+9SpU3rggQd0yy236J577lFJSUkgywcAoF07e039hl/nhnEA7UvAwndxcbGWL1+udevWKS8vTxs2bNCRI0caPSY7O1sLFizQ9u3bZRiGNm7cKEkqLy/Xo48+qjVr1jR6/IoVK5SUlKRt27Zp/PjxWrx4caDKBwCg3To+YJhqfnCt2WUA8EPAwvfevXuVnJys2NhYORwOpaamqqCgwNNfVFSk6upqJSYmSpIyMjI8/Tt37lTv3r113333NXrOXbt2KT09XZI0ZswY7d69Wy6XK1CHAABAu/T5damqHJthdhkA/BCwNd/Hjx+X0+n0bMfHx+v999/32u90OlVcXCxJuu222ySp0ZKTc/cJCwtTdHS0ysrK1K1bN59quuSSaP8OpoNwOmPMLgFB0B7GubisUlFR9kZtkZHhktSu2p1xjia1ems3u9Zz26Oi7J3umP1tt/IxX8g4wzraw+c2gidg4dvtdstms3m2DcNotH2+fl8YhqGQEN8n70+cOC2322jVa3QUTmeMSkrKzS4DARbscfZ2sleNy62KisaXQauuPvNXqvbUXlJSrupzavXWbnatDdujomAvUFgAABdTSURBVOyqqKjpVMd8Ie1WPeaWxjn5d0vVpXtXfbvwN16PGdbA7+eOJyTE1uKEb8DCd/fu3XXgwAHPdklJieLj4xv1NzxhsrS0tFF/c+Lj41VaWqru3burrq5OFRUVio2NbfviAfjk7MleDY1K7m1KLQAAWEHA1nyPGDFC+/btU1lZmaqqqrRjxw6lpKR4+hMSEmS321VYWChJys/Pb9TfnJEjRyovL0+StHXrViUlJSk8PDxQhwAAAAC0qYCF727duikrK0uZmZm67bbbNGbMGA0dOlTTpk3TwYMHJUm5ubnKycnRzTffrMrKSmVmZrb4nLNmzdJ7772ntLQ0rVu3TgsWLAhU+QAAAECbC+hNdtLT0z1XJzlr9erVnu8HDhyoTZs2ed1/xowZjbZjY2P17LPPtm2RAAAAQJBwh0sAACzm2OBr1G1Ay+dJAWifCN8AAFjMP6/9kQZwcjNgSYRvAAAsJsRVK9XUSHa718eceylQe3iIQoNQG4CWEb4BALCYa19aoYtfP3Odb2/OvRToqOTecoQH7DoLAHzEuxAAAAAIEsI3AAAAECSEbwAAACBIWPMN4LzOPXFLOnPyFgAAaB3CN4DzOvfELenMyVsAzPHV1dcpfmA3s8sA4AfCNwAAFvPV1dfpKv4DDFgSfzcGAMBiwivKZTv1ndllAPAD4RsAAIv5wYZnFJubY3YZAPxA+AYAAACChDXfAAB0It6uXsSt54HgIHwDANCJeLt6EbeeB4KDdxoAAAAQJMx8AwBgMf+85kdyXsV1vgErInwD8Dh3LSjrQIH26diQa1TDdb4BSyJ8A/A4dy0o60CB9inyuzKFlEbJfanT7FIAtBK/VQEAsJjEV/5HF/33MrPLAOAHwjcAAAAQJIRvAAAAIEgI3wAAAECQcMIlAADgzpdAkBC+AQCwmM9HjJJzUI82fU7ufAkEB+EbAACLOT4wUTXX9Da7DAB+4L+zAABYTFTpNwot+srsMgD4gfANdDL1kipd7kZf9WYXBaBVhvzpRXVd9ZTZZQDwA8tOgE6GdZ0AAJiH37YAAABAkBC+AQAAgCAhfAMAAABBwppvAAAs5sjIMbp0yL8F5bW4+Q7QtgjfAABYTGnfq1Q7tHdQXouTtIG2xTsHAACL6XrsC4X943OzywDgB8I3AAAWc9W29YpZ85zZZQDwA8tOgA7K2zpNAABgnoCG7y1btuiZZ55RXV2dJk+erHvuuadR/+HDhzV37lxVVFQoKSlJjz/+uMLCwvT1118rOztbJ06cUJ8+fZSbm6uoqCi9/fbbmjFjhrp37y5Juuqqq5STkxPIQwAsy9s6TQAAYJ6ATYMVFxdr+fLlWrdunfLy8rRhwwYdOXKk0WOys7O1YMECbd++XYZhaOPGjZKkxx9/XHfffbcKCgo0ePBgPf3005KkQ4cOaerUqcrPz1d+fj7BGwAAAJYSsPC9d+9eJScnKzY2Vg6HQ6mpqSooKPD0FxUVqbq6WomJiZKkjIwMFRQUyOVy6Z133lFqamqjdkk6ePCg9uzZo/T0dE2fPl3Hjh0LVPkAAABAmwtY+D5+/LicTqdnOz4+XsXFxV77nU6niouL9e233yo6OlphYWGN2iUpJiZGkyZN0pYtWzRy5EhlZWUFqnwAANqtj2/M0Ol7JptaQ72kSpe70Ve9qRUB1hCwNd9ut1s2m82zbRhGo21v/ec+TpJne+HChZ62u+66S8uWLVN5ebliYmJ8qumSS6L9OpaOwun07ecEazs7zsVllYqKsjfqi4wMl6Rm251xjib7eGs/33NZpd3KxxwVZe90x+xvu5WP2ds4135vkEKHXa5Qk4/5rb8XNWr//665XM44h9A6/H7uXAIWvrt3764DBw54tktKShQfH9+ov6SkxLNdWlqq+Ph4xcXFqby8XPX19QoNDfXs53a7tWrVKj3wwAMKDf3XfbUafn8+J06cltttXOCRWZPTGaOSknKzy0CANRznapdbFRU1jfqrq12S1Gx7SUl5k328tZ/vuazSbtVjjoqyq6KiplMd84W0W/WYWxrni784ovqIk3IN+F67O2Z+17QOv587npAQW4sTvgFbdjJixAjt27dPZWVlqqqq0o4dO5SSkuLpT0hIkN1uV2FhoSQpPz9fKSkpCg8PV1JSkrZu3SpJysvLU0pKikJCQvT6669r+/btnvZhw4bJ4eB/2ACAzmXAnzcreu0LZpcBwA8BC9/dunVTVlaWMjMzddttt2nMmDEaOnSopk2bpoMHD0qScnNzlZOTo5tvvlmVlZXKzMyUJD322GPauHGjRo8erQMHDuihhx6SJC1ZskQvvvii0tLS9Morr2jRokWBKh+wjIbrLovLKll3CQBAOxbQ63ynp6crPT29Udvq1as93w8cOFCbNm1qsl9CQoJeeumlJu1XXnml1q9f3/aFAhbW8HreZ/9MzfW8AQBon7jdHQAAABAkhG8AAAAgSAK67AQAALS9D2+ZqLjEnmaX0ax6nVkO15A9PES+X5sM6NgI3wAAWMypHperrk9vs8toVsPzUM4aldxbjnD+2A5IhG/AMrzNJgHofC797ENFOE6qdmii2aUAaCXCN2AR3maTAHQ+/d58VVEfdyV8AxbEtBkAAAAQJIRvAAAAIEhYdgIAAAKOq6AAZxC+AQBAwHEVFOAMwjcAABZz8NZMXXx1+7zON4CWEb6BdoZLCgI4n4pLu6s+gfANWBHhG2hnuKQggPOJ/+g92UOLVXPNcLNLAdBKhG8AACzmir075Pi8a4cI35yIic6G8A0AAEzDiZjobPiXDQAAAAQJM9+ASTixEgCAzofwDZiEEysBAOh8CN8AAFjMe3fcr9gfXGZ2GQHFiZjoqAjfAABYTPVFcXJf6jS7jIDiREx0VIRvIMBY2w2grfU4+I7s9V+o5roUs0sB0EqEbyDAWNsNoK31eucNOb7sSvgGLIjwDQAALIO14LA6wjcAALAM1oLD6gjfQBthbTcAADgfwjfQRljbDQAAzofwDQCAxRTe+RNddM3lZpfR7pz7F0jWgqM9InwDAGAxrqgYGV0vMruMdufcv0CyFhztEf8igVaql1Tpcjf6qje7KACdSs9331LkG382uwwAfmDmG2gl1nYDMFvPd99Sl2NdVf2jG80uxRK4PCHaE8I34AVrBwGgY+DyhGhPCN+AF6wdBAAAbY3wDQAAOiWWo8AMhG90anzwAkDnxXIUmIHwjU6ND14AVvT2pIfU9dpeZpfRYTExg0AifKNT4NbvADoSd3iEZLebXUaH5W1ixh4eQijHBSN8o0PxFrK5PCCAjqTX22+oy3fxqkpNM7uUToW/lqItEL7RoRCyAXQGPQ69o8jSroTvdoJlKmiNgIbvLVu26JlnnlFdXZ0mT56se+65p1H/4cOHNXfuXFVUVCgpKUmPP/64wsLC9PXXXys7O1snTpxQnz59lJubq6ioKJ06dUq/+MUv9OWXXyouLk4rVqyQ0+kM5CGgnWIZCQCgvWCZClojYGmluLhYy5cv17p165SXl6cNGzboyJEjjR6TnZ2tBQsWaPv27TIMQxs3bpQkPf7447r77rtVUFCgwYMH6+mnn5YkrVixQklJSdq2bZvGjx+vxYsXB6p8BJm3W7Z7az/7Qdfw69wPOAAAzOTtd1XD323FZZWe323oHAIWvvfu3avk5GTFxsbK4XAoNTVVBQUFnv6ioiJVV1crMTFRkpSRkaGCggK5XC698847Sk1NbdQuSbt27VJ6erokacyYMdq9e7dcLlegDgEB0NowTcgGAHQ0DX+37Xzni2ZDuS8TUbCmgC07OX78eKMlIfHx8Xr//fe99judThUXF+vbb79VdHS0wsLCGrWfu09YWJiio6NVVlambt26+VRTSIjtgo/Lyi7k+N2Sauoa3Go9LEQhzbSf7ZOX9ro6t/a+X9So/fqreyo01KZoR3ij9tDQM/Wa0R4SYmtSk7d2s2tt2O6IDJfNcHeqY76QdqseM+PcOY65pXEOj79UtkuivdZv1WO+kHarHnPDca5zG83+jpTUbHt4WEib/X5m4WbbOV/eshmGYQTihZ955hnV1NTooYcekiRt3LhRhw4d0sKFCyVJhYWFWrZsmdatWydJOnr0qKZPn64XXnhBEyZM0JtvvilJqqur09VXX62DBw9q8ODBeu+99zzB/Prrr9fmzZtZ9w0AAABLCNh/dLp3766SkhLPdklJieLj4732l5aWKj4+XnFxcSovL1d9fX2T/eLj41VaWirpTCivqKhQbGxsoA4BAAAAaFMBC98jRozQvn37VFZWpqqqKu3YsUMpKSme/oSEBNntdhUWFkqS8vPzlZKSovDwcCUlJWnr1q2SpLy8PM9+I0eOVF5eniRp69atSkpKUnh4uAAAAAArCNiyE+nMpQZXrVoll8ulcePGadq0aZo2bZpmzpypIUOG6KOPPtK8efN0+vRpDRo0SDk5OYqIiFBRUZFmz56tEydOqEePHvqv//ovXXTRRTp58qRmz56tL7/8UjExMcrNzVXPnj0DVT4AAADQpgIavgEAAAD8Cye3AgAAAEFC+AYAAACChPANAAAABAnhGwAAAAgSwjcAAAAQJITvDuKpp55SWlqa0tLStHTpUknShg0bNGbMGKWnp2vOnDmqra2VJB0+fFgZGRlKTU3V3LlzVVdXZ2bpaIXmxvmsl19+WZMmTfJsM87W1dw4v/vuu5owYYLS0tL085//nPdzB9DcOO/Zs0e33nqrxowZo0ceeYRx7gCefPJJjR49WmlpaVqzZo0kae/evUpPT9eoUaO0fPlyz2MZ507CgOW99dZbxp133mnU1NQYtbW1RmZmprFq1SrjpptuMsrLyw2322088sgjxpo1awzDMIy0tDTj3XffNQzDMObMmWOsXbvWxOrhq+bGeceOHYZhGMann35qXH/99ca9997reTzjbE3NjfPmzZuN6667zjh8+LBhGIaRlZXlGU/G2Zq8vZ9TUlKMI0eOGIZhGDNmzDA2btxoGAbjbFX79+83Jk6caLhcLqOqqsr40Y9+ZBw+fNgYOXKk8cUXXxgul8uYOnWqsWvXLsMwGOfOgpnvDsDpdGr27NmKiIhQeHi4+vbtq9raWj322GOKjo6WzWZT//799fXXX6uoqEjV1dVKTEyUJGVkZKigoMDkI4Avmhvnr7/+WrW1tVqwYIFmzpzpeSzjbF3NjXNRUZESExM1cOBASdK8efN00003Mc4W5u39XF9fr9OnT6u+vl41NTWy2+2Ms4Vde+21evHFFxUWFqYTJ06ovr5ep06dUq9evXTZZZcpLCxM6enpKigoYJw7kTCzC8CFu/LKKz3fHz16VNu2bdMf/vAH9e7dW5JUVlamtWvXKicnR8ePH5fT6fQ83ul0qri4ONglww/exnnZsmW64447Gt3tlXG2rubG+b777pPD4VBWVpY+//xzff/739fs2bP14YcfMs4W5e39nJCQoEmTJik6Olo9e/bUzTffrA8++IBxtrDw8HD993//t373u9/p5ptvbvL5HB8fr+LiYj63OxFmvjuQTz/9VFOnTtUjjzziCd7FxcWaPHmy7rjjDg0fPlxut1s2m82zj2EYjbbR/jUc56KiIh07dkx33HFHo8cwztbXcJzr6+u1Z88e/fznP9fmzZtVVVWl5557jnHuABqOc1RUlHJzc/Xqq69qz549GjZsmHJychjnDmDmzJnat2+fjh07pqNHjzY7noxz50H47iAKCws1ZcoUPfzww7r99tslSZ999pkmTpyo22+/XT/72c8kSd27d1dJSYlnv9LSUsXHx5tSM1rv3HF+9dVX9emnn2rs2LGaN2+eDh06pIceeohxtrhzx/nSSy/VsGHDdNlllyk0NFS33HKL3n//fcbZ4s4d5wMHDqh///66/PLLFRISogkTJujtt99mnC3ss88+0+HDhyVJXbp00ahRo7R///5G41lSUqL4+HjGuRMhfHcAx44d089+9jPl5uYqLS1NknT69Gn9+Mc/1qxZszR16lTPYxMSEmS321VYWChJys/PV0pKiil1o3WaG+ecnBxt27ZN+fn5WrRokQYPHqwVK1YwzhbW3Dj/8Ic/1AcffKBjx45Jkt544w0NGjSIcbaw5sa5f//+ev/991VaWipJ2rlzp4YMGcI4W9hXX32lefPmqba2VrW1tdq5c6cmTpyof/zjH/rnP/+p+vp6vfrqq0pJSWGcOxGbYRiG2UXgwixatEivvPKKLr/8ck/b6NGjtXLlSvXt29fTdsMNN2jWrFn66KOPNG/ePJ0+fVqDBg1STk6OIiIizCgdrdDcOE+cOFF33XWXJGn//v166qmn9NJLL0kS42xR3sa5R48eWr58uWpqavS9731Pv/71r9WlSxfG2aK8jXNkZKRWr16t0NBQ9erVSwsXLlRcXBzjbGErV67Utm3bFBoaqlGjRmnGjBnat2+fcnJyVFNTo5EjR2rOnDmy2WyMcydB+AYAAACChGUnAAAAQJAQvgEAAIAgIXwDAAAAQUL4BgAAAIKE8A0AAAAECeEbAHDBpk6dqrKyslbtc/DgQd1www0BqggA2ifCNwDggr311ltmlwAAlhBmdgEAAP9s2rRJa9asUUhIiC6++GItWbJEu3fv1ksvvaSQkBBdeumlmj9/vvr06aPZs2crOjpaH3/8sb755hsNGDBAS5Ys0auvvqo33nhDzz77rKQzt8OeMmWKdu3apaNHj2rx4sU6efKk6uvrNWnSJI0bN0779+/X4sWL5XA4VFFRocGDB0uSJk+erOeee04hISFauHChjh07JpfLpbS0NE2fPl2StG7dOr3wwguKjo5W//79TfvZAYBZCN8AYEEfffSRcnNz9cc//lE9evTQ73//e02ZMkVut1sbNmxQXFycNm/erJ/97Gd67bXXJEmHDh3Siy++KJvNpgkTJqigoEBpaWnKzc1VSUmJnE6nNm/erIyMDBmGoZkzZ2rp0qUaNGiQysvLdeedd6pfv36SpE8//VR//vOflZCQIEnavHmzXnjhBcXFxSkzM1NTpkzRDTfcoJqaGk2bNk2XX365+vTpo6eeekr5+flyOp1asGCBaT8/ADAL4RsALGjfvn364Q9/qB49ekiSpkyZouPHjys8PFxxcXGSpIyMDC1evFhfffWVJOn666/33Kq6f//++u677xQdHa2bbrpJf/rTnzRlyhRt2bJFa9eu1dGjR/XFF1/o0Ucf9bxmdXW1PvzwQ/Xt21c9evTwBO+GKisr9c477+i7777Tk08+6Wn76KOP9M033+i6666T0+mUJN15553as2dP4H5IANAOEb4BwIJCQ0Nls9k829XV1fryyy91xRVXNHqcYRiqq6uTJEVGRnrabTabDMOQJE2YMEHz589X37591bdvX1122WX6+OOPFRMTo/z8fM8+paWliomJ0XvvvSeHw9FsXW63W4ZhaP369erSpYskqaysTHa7XRs2bPC85tljAIDOhhMuAcCChg8frn379un48eOSpPXr1+vNN9/U1q1bPVcdeeWVVxQbG6tevXq1+FyJiYmSpN/+9rcaP368JKlPnz6KjIz0hO9jx45pzJgxOnToULPPERoaqrq6OkVHRysxMVFr1qyRJJ06dUp33XWXdu7cqeuuu05vvfWWvvnmG0nSH//4xwv8KQCA9TDzDQAWNGDAAGVnZ+v++++XJDmdTr3++uv685//rMmTJ8vtdisuLk6rVq1SSMj551nGjx+vp59+WjfeeKMkKSIiQk8//bQWL16s//mf/1FdXZ1mzZqlH/zgB9q/f3+T/W+++WZNmjRJK1euVG5urn71q18pPT1dtbW1GjNmjG699VZJUnZ2tiZPnqyoqCgNHTq0DX8iAGANNqPh3wABAAAABAzLTgAAAIAgIXwDAAAAQUL4BgAAAIKE8A0AAAAECeEbAAAACBLCNwAAABAkhG8AAAAgSAjfAAAAQJD8P9rwrZLIwRqhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,6))\n",
    "x = np.linspace(264-49, 264+50, 100)\n",
    "y = stats.binom(586, 0.45).pmf(x)\n",
    "ax.bar(x, y, alpha=0.5)\n",
    "ax.axvline(x=0.48 * 586, c='red', alpha=0.75, linestyle='--')\n",
    "plt.xlabel('converted')\n",
    "plt.ylabel('probability')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'probability')"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAAF2CAYAAAC28fR/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xU9Z3/8ffMJJmYi0TsBFy0YEVkC2LoppIfatiqTZQQgxEQtQTLFkvbFZpaWhCklcJSaFxwtSqyPtheYCEPSpPFQojFUoqASoorKF7wsVQbMSREbrnO5fz+CBlmQi6TZObMZPJ6Ph55mO+cc758zicD8/Gb7/l+LYZhGAIAAAAQctZwBwAAAAD0FxTfAAAAgEkovgEAAACTUHwDAAAAJqH4BgAAAExC8Q0AAACYhOIbAAAAMElMuAMw0+ef18njia5lza+8MkmnTp0PdxhRg3wGHzkNPnIaXOQz+Mhp8JHT4AplPq1Wi664IrHD4/2q+PZ4jKgrviVF5T2FE/kMPnIafOQ0uMhn8JHT4COnwRWufDLtBAAAADAJxTcAAABgEopvAAAAwCQU3wAAAIBJKL4BAAAAk1B8AwAAACah+AYAAABMQvENAAAAmITiGwAAADAJxTcAAABgEopvAAAAwCQU3wAAAIBJKL4BAAAAk8SEOwAAaOWW1OT0SJLssS1jA63t1tds4QgMAIAgofgGEDGanB6VHzguScrKGCZJ3rYk3TX+WjV5DG+bYhwA0NdQfAMIm/ZGujvT7PLo1Tf+5m1nZQxTQgDXAQAQKSi+AYRNeyPdAABEM4aMAAAAAJMw8g2gT2s7dYU54ACASMbIN4A+rXXqSvmB434rowAAEIkY+QZgmu4+YAkAQLSh+AZgGh6wBAD0dww9AQAAACah+AYAAABMwrQTAFGF1U8AAJGMkW8AUYXVTwAAkYziG0BIuCXVOz2qqq1XvdMjd7gDAgAgAjDtBEBItI5AJybaVVfXxOomAACIkW8AAADANBTfAAAAgEkovgEAAACTMOcbAIAwirc0Sa6miy/E2Fv+2/pajF2Nht38wACEBMU3gKjmu+63xNrfiECuJp05tMvbHDD2DknyvjZg7B2KjxHFOBAlKL4BRLXWVVdaZWUMU0IsM+4QPm1Hum2WABbi9CnQB4y9Q7JRfAN9FcU3AABmajPSPTAtM4zBADAbxTeAoGi7rTuA0GlvnjhTUYC+geIbQFD4Tu9gQx0gxNqbJ85UFKBPoPgGACCE/EapYyiQgf4upL8b3rZtmyZOnKisrCxt2LDhkuNHjx5Vfn6+srOztWjRIrlcLknSwYMHlZ+fr9zcXM2ZM0dnzpyRJJ09e1aPPPKI7r77bj300EOqrq4OZfgAAPTehVHqM4d2+U8VAdAvhaz4rqqq0urVq7Vx40aVlJRo8+bNOnbsmN858+fP15IlS7Rz504ZhqHi4mJJ0sKFC7Vq1Spt27ZNw4cP10svvSRJWrNmjdLT07Vjxw5NnTpVy5cvD1X4AAAAQNCFrPjet2+fMjIylJKSooSEBGVnZ6usrMx7vLKyUo2NjUpLS5Mk5efne49v375dw4cPl9PpVFVVlS6//HJJ0u7du5WbmytJmjRpkvbs2SOn0xmqWwAAoE+IsVkU7z578cvCCDsQqUJWfJ88eVIOh8PbTk1NVVVVVYfHHQ6H93hsbKzef/99TZgwQa+//rpycnIuuSYmJkZJSUmqra0N1S0AANAnGK5m79QWprcAkS1kD1x6PB5ZLBZv2zAMv3ZXx2+44Qbt27dPmzZtUmFhoTZt2nTJn2EYhqzWwP//4cork7p7G32Cw5Ec7hCiCvnsmaraeiUmtjxMFh8fK0nedmKi/ZLXutuWJJvN6tfuSR9xcTHyWC/+W5Ngj1FyYlyP7ztceJ8GVyjz6TzdIGeb96Szzfs6oZ33dUfXBNpHckp43yO8R4OPnAZXuPIZsuJ78ODBOnjwoLddXV2t1NRUv+O+D0zW1NQoNTVVTU1N+stf/qI777xTknTPPfdo5cqVklpGz2tqajR48GC5XC7V1dUpJSUl4JhOnTovj8fo7a1FFIcjWdXV58IdRtQgnz3X6PSorq5ltK2xsWU6WF1dkxIT7aqra/J7re05gbQlye32+LV70kd9o1OvvvE3bzsrY5ga6/vWKCHv0+AKdj4vWd3E5VT9hfdg7IX3ZL3PezLe7fFrtz2nq3ZHfZwO43uE92jwkdPgCmU+rVZLpwO+IZt2Mn78eO3fv1+1tbVqaGhQeXm5MjMv7uI1ZMgQ2e12VVRUSJJKS0uVmZmpmJgYPfnkkzpy5IgkaceOHfrKV74iSZowYYJKSkoktcwLT09PV2xsbKhuAQCA7mN1EwCdCNnI96BBg1RYWKiCggI5nU5NmTJFY8aM0ezZszV37lzdeOONKioq0uLFi3X+/HmNGjVKBQUFstlsWr16tZYsWSK3261BgwZ5VzWZN2+eFixYoJycHCUnJ6uoqChU4QPoAjtaAgDQfSHdZCc3N9e7OkmrdevWeb8fOXKktmzZcsl16enp2rp16yWvp6Sk6IUXXgh+oAC6jR0tgcjWdvoL288DkYEdLgEAiEY+W9Cz/TwQOfhdMQAAAGASim8AAADAJEw7AQCgFy5ZWhAAOkHxDQBAb7SdWw0AnWDaCQAAAGASRr4B9Htt1yy3hTccAEAUo/gG0O+1XbM8gU2DEIVY9xuIDBTfAAD0B6z7DUQEim8AAWE7eQAAeo/iG0BA2E4eAIDeY/gKAAAAMAkj3wAABMjvoUWJTXUAdBvFNwAAgfJ5aFFiUx0A3ce0EwAAAMAkFN8AAACASZh2AgBAP9Te/HU23gFCj+IbAID+qL3562y8A4QcxTcAtNF2QyFbeMMBAEQR5nwDQButGwqVHzjuLcIBAAgGim8AAADAJBTfAAAAgEmY8w3gEr5znqWWec9Af+S3Igi7WQIIAopvAJdonfPcKitjWNhiAcLKZ0UQdrMEEAwMZwEAAAAmofgGAAAATELxDQAAAJiEOd8AAEAxNovi3Wd9XmC7eSAUKL4BAIAMV7POvLXH22a7eSA0mHYCAAAAmITiGwAAADAJxTcAAABgEuZ8A0AXfHf8tMdaZQtvOACAPoziGwC64LvjZ1bGMCXE8kvDaMV28gBCjeIbAIBWbCcPIMQYvgEAAABMwsg3AL85zZJkhC8UAACiGsU3AL85zZJ0+81DwxcMAABRLKTTTrZt26aJEycqKytLGzZsuOT40aNHlZ+fr+zsbC1atEgul0uSVFFRoSlTpigvL08zZ85UZWWlJOmNN97QuHHjlJeXp7y8PC1cuDCU4QMA0K/FW5oU7z7b8mVpCnc4QFQIWfFdVVWl1atXa+PGjSopKdHmzZt17Ngxv3Pmz5+vJUuWaOfOnTIMQ8XFxd7Xly1bptLSUuXm5mrZsmWSpCNHjmjWrFkqLS1VaWmpVqxYEarwAQDAhQdQzxzadXEVGAC9ErLie9++fcrIyFBKSooSEhKUnZ2tsrIy7/HKyko1NjYqLS1NkpSfn6+ysjI1Nzdr3rx5GjlypCTphhtu0IkTJyRJhw8f1t69e5Wbm6s5c+Z4XwcAAAD6gpAV3ydPnpTD4fC2U1NTVVVV1eFxh8OhqqoqxcXFKS8vT5Lk8Xj07LPP6s4775QkJScna8aMGdq2bZsmTJigwsLCUIUPAAAABF3IHrj0eDyyWCzetmEYfu2ujjc3N2vBggVyuVz69re/LUlaunSp9/gDDzygp556SufOnVNycnJAMV15ZVKP7yeSORyB3T8C0x/zWVVbr8TEixuK2GxWv3Z8fKwkeV/rbjsx0d7rPoIRV7DuzTEwQeHWH9+nodSaT+fpBjnb/Ow7agdyTiB92GxWJfSiTzP7SE4J/H3HezT4yGlwhSufISu+Bw8erIMHD3rb1dXVSk1N9TteXV3tbdfU1HiP19XV6Tvf+Y5SUlL0/PPPKzY2Vh6PR2vXrtUjjzwim+3i5s6+33fl1Knz8niiaxE1hyNZ1dXnwh1G1Oiv+Wx0elRXd3E+p9vt325sdEqS97XutBMT7aqra+pVH8GKK1j3Fu73SH99n4aKbz7j3U7VX/hZx1742XfUDuScQPqId3t61aeZfZwO8H3HezT4yGlwhTKfVqul0wHfkE07GT9+vPbv36/a2lo1NDSovLxcmZmZ3uNDhgyR3W5XRUWFJKm0tNR7fP78+Ro6dKjWrFmjuLi4Czdi1SuvvKKdO3dKkkpKSnTTTTcpISH8I1AAAABAIEI28j1o0CAVFhaqoKBATqdTU6ZM0ZgxYzR79mzNnTtXN954o4qKirR48WKdP39eo0aNUkFBgd59913t2rVLw4cP17333iupZb74unXrtHLlSj3xxBP65S9/qYEDB2rVqlWhCh8A0A/EW5rkPN2geLdTirF3fQEA9FJIN9nJzc1Vbm6u32vr1q3zfj9y5Eht2bLF7/iXv/xlvf/+++32d/3112vTpk3BDxQAuqHtjqD2WKsCnwCHiOJq0rl396q+rkkDxt4R7mgA9APscAkA3dR2R9CsjGFKiA3pnmUAgCjBpwUAAABgEopvAAAAwCQU3wAAAIBJmPMN9EO+DwzamasMIEDxlibJdWEt8Bi7Gg1WiAG6i+Ib6Id8HxjMyhgW1lgA9CGuJp05tEuSWlaHsVF8A93FkBcAAABgEopvAAAAwCQU3wAAAIBJKL4BAAAAk1B8AwAAACZhtRMAQL/gt0yeJMWwUgcA81F8AwD6B59l8qQLS+UBgMmYdgIAAACYhJFvAAiCtruG2sIbDgAgQjHyDQBB0LpraPmB494iHACAtii+AQAAAJMw7QQAAPSI3woyMXY1GqwgA3SF4hsAAPSMzwoyA8beIdkovoGuUHwD/UDbhwEBAEB4UHwD/UDrw4CSlJUxLKyxAADQnzEEBgAAAJiE4hsAAAAwCcU3AAAAYBLmfAMAopLfMniSbBZ3GKMBgBYU3wCA6OSzDJ4kDUzLDGMwANCCaScAAACASSi+AQAAAJNQfAMAAAAmofgGAAAATMIDlwAQAm617CwqSfZYq2zhDQcwResKM87TDYp3O6UYuxoNe7jDAiIKxTcAhECT06PyA8clSVkZw5QQyy8a0Q9cWGHGmWhXfV2TBoy9Q7JRfAO++DQAAAAATELxDQAAAJiE4hsAAAAwCXO+gSjj+6Cf1PKwHwAAiAwU30CU8X3QT2p52A8AAEQGhsQAAAAAk4S0+N62bZsmTpyorKwsbdiw4ZLjR48eVX5+vrKzs7Vo0SK5XC5JUkVFhaZMmaK8vDzNnDlTlZWVkqSzZ8/qkUce0d13362HHnpI1dXVoQwfANCHxFuaFO8+2/JlaQp3OADQrpAV31VVVVq9erU2btyokpISbd68WceOHfM7Z/78+VqyZIl27twpwzBUXFzsfX3ZsmUqLS1Vbm6uli1bJklas2aN0tPTtWPHDk2dOlXLly8PVfgAgL7mwhrTZw7tklwU3wAiU0DF9+eff97tjvft26eMjAylpKQoISFB2dnZKisr8x6vrKxUY2Oj0tLSJEn5+fkqKytTc3Oz5s2bp5EjR0qSbrjhBp04cUKStHv3buXm5kqSJk2apD179sjpdHY7NgAAACAcAiq+c3Jy9Nhjj+ngwYMBd3zy5Ek5HA5vOzU1VVVVVR0edzgcqqqqUlxcnPLy8iRJHo9Hzz77rO68885LromJiVFSUpJqa2sDjgkAAAAIp4BWO3n11Vf1hz/8QatWrVJDQ4OmT5+uvLw8JSUldXiNx+ORxWLxtg3D8Gt3dby5uVkLFiyQy+XSt7/97Xb/DMMwZLUGPnPmyis7jrcvcziSwx1CVOnr+ayqrVdi4sXtnOPjYyXJ+1pXbUmy2ay96qNtOzHR3us+ghFXKO4t0GscAxMUTH39fRoKztMNcrb5GTjb/OwT2vm5OSUl+LxHO+qjvT67e0234upjfbRtt+Y0OYX3arDw9z64wpXPgIrv+Ph43Xfffbrvvvv0+uuv6/HHH1dRUZEmT56suXPn6oorrrjkmsGDB/uNlFdXVys1NdXvuO8DkzU1Nd7jdXV1+s53vqOUlBQ9//zzio1t+QudmpqqmpoaDR48WC6XS3V1dUpJSQn4Zk+dOi+Pxwj4/L7A4UhWdfW5cIcRNaIhn41Oj+rqLs53bWxsmZrV+lpXbUlyu3vXh287MdGuurqmXvURrLiCfW/d6SOY76toeJ+GQrzbqfoLOY+98DOo9/k5xbs9fu3Wc1rPa3tNV+1Azgmkj47i6mt9+LYTEu3enJ7mvRoU/L0PrlDm02q1dDrgG/Cw8Z49e/Too4+qsLBQd955pzZt2qSrrrpK3/3ud9s9f/z48dq/f79qa2vV0NCg8vJyZWZmeo8PGTJEdrtdFRUVkqTS0lLv8fnz52vo0KFas2aN4uLivNdMmDBBJSUlkqTt27crPT3dW5gDAIDIwyo0gL+ARr6/9rWvKSUlRQ8++KB+8YtfKD4+XlLLw5CbN29u95pBgwapsLBQBQUFcjqdmjJlisaMGaPZs2dr7ty5uvHGG1VUVKTFixfr/PnzGjVqlAoKCvTuu+9q165dGj58uO69915JLSPe69at07x587RgwQLl5OQoOTlZRUVFQUoDAISW786j9lirbOENBzDPhVVoJGnA2Dskm72LC4DoFlDxvWrVKn31q1/1e+3YsWMaPny4du3a1eF1ubm53tVJWq1bt877/ciRI7Vlyxa/41/+8pf1/vvvt9tfSkqKXnjhhUBCBoCI4rvzaFbGMCXEsscZAPRHnf7rf/r0aZ0+fVo/+9nPdObMGW+7pqZG//qv/2pWjAAAAEBU6HTk+7HHHtNrr70mSRo3btzFi2JilJ2dHdrIAAAAgCjTafH90ksvSZIWLlyoFStWmBIQAAAAEK06Lb4/+ugjXXfddfrGN76hd95555Ljo0aNCllgAALj+yCfJEXXYpoAAESXTovvlStX6sUXX9Sjjz56yTGLxdLpw5YAzOH7IJ8k3X7z0PAFA5go3tIkuS4sXRfDChoA+oZOi+8XX3xRUssOlwAARJS2S9gBQB/QafG9bNmyTi9evHhxUIMBAAAAolmnxXd3tm4HAAAA0LlOi2/W8gYAAACCp9Pi+4EHHtB///d/a+zYsbJYLJcc/+tf/xqywAAAAIBo02nx/fTTT0uSXn75ZVOCAQAAAKJZp8V3amqqJGnIkCH6y1/+on379ikmJkaZmZn66le/akqAAAAgerRdIrLRYJlI9C/WQE564YUXtGLFCsXHx8tqtWrx4sXasGFDqGMDAADR5sISkWcO7bpYhAP9SKcj361efvllFRcXKykpSZI0a9YsPfjgg3rooYdCGhwAAAAQTQIqvu12uxITE73tAQMGyG7n10QA0FNutexO2soea5UtfOEAAEzSafFdXl4uSbr22mv13e9+V1OnTpXNZlNJSYlGjx5tSoAAEI2anB6VHzjubWdlDFNCbEAzAQEAfVinxfdvfvMbv/b69eu93586dSo0EQEAAABRqlvFNwAAAICeC2jO9/Hjx/Xb3/5W9fX1MgxDHo9Hf/vb37Rp06ZQxwcAgP/ydJIUw3NHAPqmgCYYPvbYY3I6nTp06JCGDBmiY8eOacSIEaGODUA73JLqnR7VOz1yhzsYwCw+y9OxRB2Aviyg4ruurk5PPvmkbr31VmVmZmr9+vV66623Qh0bgHa0PqhXfuC432oZAAAg8gVUfKekpEiShg4dqg8//FCXX365LBZLSAMDAAAAok1Ac76HDh2q5cuX695779WiRYtUX18vl8sV6tgAAACAqBLQyPdPf/pTpaen68tf/rKmTZumAwcOaOnSpaGODQAAAIgqAY18X3bZZbr55pv1pz/9SVdffbVWrFihyy+/PNSxAQAAAFEloOJ79+7d+vGPf6zhw4fL4/Hok08+0erVq/XVr3411PEBAIAo5reMZIxdjQbLSCK6BVR8P/300/rtb3+r66+/XpL0zjvv6IknntDWrVtDGhwAAIhyF5aRlKQBY++QbBTfiG4Bzfm2WCzewluSRo0aJcMwQhYUAAAAEI06Lb5Pnz6t06dPa/To0XrppZdUV1enhoYGbdiwQRkZGWbFCAAAAESFTqedZGRkyGKxeEe5f/GLX3iPWSwW/fjHPw5tdADQT1itFtX7bJpkj7XKFsZ4AACh0Wnx/d5775kVBwD0a80uj15942/edlbGMCXEBjQzEADQhwT0wKXH49FLL72kPXv2yOVy6ZZbbtGcOXMUExPQ5QAAAAAU4AOXTz31lA4cOKCZM2fqm9/8pg4dOqSVK1eGOjYAQD8Vb2lSvPus98tmcYc7JAAIioCGrv/yl7/od7/7nWJjYyVJ//zP/6x77rknpIEBAPoxn+XnJGlgWmYYgwGA4Amo+DYMw1t4S1JcXJxfG0DouCU1XXgQz84cYAAA+rSAPslHjhypf/u3f9PHH3+sTz75RCtWrNCIESNCHRsAtRTe5QeOq/zAcW8RDgAA+qaAiu+f/OQnOnv2rKZPn65p06aptrZWTzzxRKhjAwAAAKJKQNNO1q5dq5///OehjgUAAACIagGNfO/evbtHnW/btk0TJ05UVlaWNmzYcMnxo0ePKj8/X9nZ2Vq0aJFcLpff8TVr1uiZZ57xtt944w2NGzdOeXl5ysvL08KFC3sUFwAAiExtV7qJtzSFOyQgqAIa+b766qs1a9YsfeUrX1FiYqL39W9+85sdXlNVVaXVq1dr69atiouL0/Tp0zVu3DgNHz7ce878+fO1bNkypaWl6fHHH1dxcbEefPBBnTt3TitWrNAf/vAHfetb3/Kef+TIEc2aNUvf/va3e3KvAAAg0rVZ6WbA2Dskmz2MAQHBFdDId0pKigYNGqR33nlHFRUV+uCDD/TBBx90es2+ffuUkZGhlJQUJSQkKDs7W2VlZd7jlZWVamxsVFpamiQpPz/fe3zXrl0aNmzYJcX94cOHtXfvXuXm5mrOnDk6ceJEt24WAAAACKeAiu9vfetb+uCDD/TnP/9Zr776qj755BM9+uijnV5z8uRJORwObzs1NVVVVVUdHnc4HN7jkydP1iOPPCKbzebXZ3JysmbMmKFt27ZpwoQJKiwsDCR8AAAAICIENO3k8ccf17Rp05Sfny/DMLR582YtWrRI69ev7/Aaj8cji8XibRuG4dfu6nh7li5d6v3+gQce0FNPPaVz584pOTk5kNvQlVcmBXReX+NwBHb/CEyk5bOqtl6JiS2/co2Pb1lfv6O2JNlsVr92V9eEoo+27cREe6/7iNR7C1Yf7cXlGJigjkTa+zTYnKcb5GyTn4R2cu5sk9OO2p32ISnB5z3anT67e0234upjfbRt9ySnHeU4OSW63++Biva/92YLVz4DKr4bGhp0//33e9szZsxQcXFxp9cMHjxYBw8e9Larq6uVmprqd7y6utrbrqmp8Tvelsfj0dq1ay8ZEW87Ot6ZU6fOy+MxAj6/L3A4klVdfS7cYUSNSMxno9OjurqWB44aG52S1GFbktxuj1+7q2tC0YdvOzHRrrq6pl71Ean3Fsw+2ouro/diJL5Pgy3e7VS9Tz7i3R6/duyFHLa+1lW7sz5az+tJn929pjtx9bU+fNsJifYe5bSjHJ+O8vd7IPrD33szhTKfVqul0wHfgKadfOlLX9Jf//pXb/uDDz7Q1Vdf3ek148eP1/79+1VbW6uGhgaVl5crM/Pi9sBDhgyR3W5XRUWFJKm0tNTv+KU3YtUrr7yinTt3SpJKSkp00003KSGh45EhAOjL3JLqnR7VOz1yhzsYAEBQBDTy/emnn2rGjBm64YYbFBMTo3fffVcOh0O5ubmSWpYUbGvQoEEqLCxUQUGBnE6npkyZojFjxmj27NmaO3eubrzxRhUVFWnx4sU6f/68Ro0apYKCgk7jWLlypZ544gn98pe/1MCBA7Vq1aoe3DIA9A2tu5tKUlbGMCXEBjRe0ifFW5ok14XRzhhWtgAQvQIqvn/4wx/2qPPc3Fxvgd5q3bp13u9HjhypLVu2dHh924c6r7/+em3atKlHsQAAIpjP8nIDxt4R5mAAIHQCKr5vvvnmUMcBAAAARL3o/R0mAAAAEGECGvkGYB63Wub6SpI9iuf4AgDQH1F8AxGm7UN2AAAgelB8AwCAiBVjsyjefdbnBbsaDVbEQd9F8Q0AACKW4WrWmbf2eNsDxt4h2Si+0XcxoRQAAAAwCcU3AAAAYBKKbwAAAMAkFN8AAACASSi+AQAAAJOw2gkA9BGtGzBV1dbLkGQLd0C9EG9pklxNLY0YVq4A0H8w8g0AfUTrBky73vzYuwtqn+Vq0plDu3Tm0K6LRTgA9AMU3wAAAIBJKL4BAAAAk1B8AwAAACbhgUsgjFofoGtlj+X/hwEAiGYU30AYtT5A1yorY1jYYgGAvqLtajmNBivmoO+g+AYAAH3LhdVyJGnA2DskG8U3+g5+xw0AAACYhOIbAAAAMAnFNwAAAGASim8AAADAJBTfAAAAgElY7QQAEHJtl4YDgP6K4hsA+qj2NmmyhS+czrVdGg4A+imKbwDoo9rbpCmBXVIBIKLxrzQAAABgEopvAAAAwCRMOwFM5DtH1870AAAIirYP9DYaPNSLyEXxDZjId45uVsawsMYCAFGj7QO9NopvRC6G3gAAAACTUHwDAAAAJqH4BgAAAExC8Q0AAACYhOIbAAAAMAmrnQAAgspv2TdJimHlCQBoRfENAFHCarWo/sI68lLLWvK2cATis+ybdGHpNwCApBBPO9m2bZsmTpyorKwsbdiw4ZLjR48eVX5+vrKzs7Vo0SK5XC6/42vWrNEzzzzjbZ89e1aPPPKI7r77bj300EOqrq4OZfgA0Kc0u1rWkW/9avIpxAEAkSFkxXdVVZVWr16tjRs3qqSkRJs3b9axY8f8zpk/f76WLFminTt3yjAMFRcXS5LOnTunxx9/XOvXr/c7f82aNUpPT9eOHTs0depULV++PFThAwAAAEEXsuJ73759ysjIUEpKihISEpSdna2ysjLv8crKSjU2NiotLU2SlJ+f7z2+a9cuDRs2TN/85jf9+ty9e7dyc3MlSZMmTdKePXvkdDpDdbTEvtEAABtvSURBVAsAAABAUIVszvfJkyflcDi87dTUVL399tsdHnc4HKqqqpIkTZ48WZL8ppy0vSYmJkZJSUmqra3VoEGDAorpyiuTenYzEc7hSA53CFEllPmsqq1XYmLLw2fx8bGS5G2391p325Jks1l71Wcw+mjbTky0R+29BauPUN2bY2CCzOY83SBnO/fibBN7oG2p5d4SetFnp31ISvB5j3anz4i/NxP7aNvuSU6DFVeCXfI0NXivsdovk+2yvv9Zyed9cIUrnyErvj0ejywWi7dtGIZfu6vjgTAMQ1Zr4IP3p06dl8djdOvPiHQOR7Kqq8+FO4yoEex8uiXvvFt7rFVNTo/q6lpWgWhsbPmtTWu7vde625Ykt9vTqz6D0YdvOzHRrrq6pqi8t2D20Z24EhPtAfcRjn8f4t1O1fvEEXsh1tbXuttu6dPTqz4766P1vJ70Gen3ZmYfvu2ERHuPchq0uBqdlzz023hefRqf98EVynxarZZOB3xDNu1k8ODBfg9EVldXKzU1tcPjNTU1fsfbk5qaqpqaGkmSy+VSXV2dUlJSghw5EDxNTg8PvwEAAK+QFd/jx4/X/v37VVtbq4aGBpWXlyszM9N7fMiQIbLb7aqoqJAklZaW+h1vz4QJE1RSUiJJ2r59u9LT0xUbGxuqWwAAAACCKmTF96BBg1RYWKiCggJNnjxZkyZN0pgxYzR79mwdPnxYklRUVKQVK1borrvuUn19vQoKCjrtc968eXrrrbeUk5OjjRs3asmSJaEKHwAAAAi6kG6yk5ub612dpNW6deu8348cOVJbtmzp8PpHH33Ur52SkqIXXnghuEECAAAAJmGHSwBAr/htJ89W8gDQKYpvAEDv+Gwnz1byANA5im8AiGJtl7u0hTccAOj3QvbAJQAg/FjuEgAiCyPfAAAgqsXYLIp3n/V5wa5Gg+cTEB4U3wAAIKoZrmadeWuPtz1g7B2SjeIb4cG0EwAAAMAkjHwDQdT24TYAAABfFN9AELU+3CZJWRnDwhoLAACIPAzNAQAAACZh5BsA0C3saAkAPUfxDQDoHna0BIAeY9oJAAAAYBJGvgGgH2G7eQAIL4pvAOhH2q7Ik8CSmOin2j67wI6XMAvFNwAA6H/aPrvAjpcwCUMeAAAAgEkY+QZ6yHfurMSOlgAAoGsU30AP+c6dldjREgAAdI2hOgAAAMAkjHwDADrFjpYAEDwU3wCAzrGjJQAEDdNOAAAAAJMw8g0A/Vh7q/aw6yX6IzbdgVkovgGgH2tv1R52vUS/xKY7MAn/wgIAAAAmofgGAAAATMK0EyBAvnNj2c0SAAD0BMU3ECDfubHsZololRzjVKy76eILrOsNAEFF8Q0A8LK6GnXmf1/1tlnXGwCCi9+dAwAAACah+AYAAABMwrQTAMBFFoucHsPbNCRZwhcNEDZsuoNQofgGAHh5DEMf/f20t33FWEM2K+U3+iE23UGIMO0EAAAAMAnFNwAAAGASpp0AHWBTHfQHyTFOjb/u4q/T4yyeMEYDANEvpBXFtm3bNHHiRGVlZWnDhg2XHD969Kjy8/OVnZ2tRYsWyeVySZI+/fRTPfTQQ7rrrrv0ne98R3V1dZKkN954Q+PGjVNeXp7y8vK0cOHCUIaPfq51U53yA8e9RTgQbayuRp04sNP7JcMd7pAAIKqFrPiuqqrS6tWrtXHjRpWUlGjz5s06duyY3znz58/XkiVLtHPnThmGoeLiYknSk08+qQcffFBlZWUaPXq0nnvuOUnSkSNHNGvWLJWWlqq0tFQrVqwIVfgAAABA0IWs+N63b58yMjKUkpKihIQEZWdnq6yszHu8srJSjY2NSktLkyTl5+errKxMTqdTb775prKzs/1el6TDhw9r7969ys3N1Zw5c3TixIlQhQ8AAOAVb2lSvPvsxS9LU7hDQh8VsuL75MmTcjgc3nZqaqqqqqo6PO5wOFRVVaXPP/9cSUlJiomJ8XtdkpKTkzVjxgxt27ZNEyZMUGFhYajCBwBcYEhyegw5PYaMLs8GotSFpQdbv7xrgAPdFLIHLj0ejyyWi2vDGobh1+7oeNvzJHnbS5cu9b72wAMP6KmnntK5c+eUnJwcUExXXpnUo3uJdA5HYPePwLTms6q2XomJLQ+ixcfHSlKH7UDOMasPm83aqz6D0UfbdmKiPWrvLVh9hOvebE6r4mIvfhRYLBa/ts3WMkbzSdV5SdIXLBbF22PkbNNnR+1AzgmkD5vNqoRe9NlpH5ISfN6jUXVvJvbRtt2TnPaVe2t9LTnF3M9fPu+DK1z5DFnxPXjwYB08eNDbrq6uVmpqqt/x6upqb7umpkapqakaOHCgzp07J7fbLZvN5r3O4/Fo7dq1euSRR2Sz2bzX+X7flVOnzsvjia5xG4cjWdXV58IdRtTwzWej06O6upaRjcZGpyR12A7kHLP6cLs9veozGH34thMT7aqra4rKewtmH92JKzHRHrR7s8ujZqfLe41hGH5tt7vlYePW19xujxobnaq/0EfshT47agdyTiB9xLs9veqzsz5az+tJn5F+b2b24dtOSLT3KKd94d58+zht4ucvn/fBFcp8Wq2WTgd8QzbtZPz48dq/f79qa2vV0NCg8vJyZWZmeo8PGTJEdrtdFRUVkqTS0lJlZmYqNjZW6enp2r59uySppKREmZmZslqteuWVV7Rz507v6zfddJMSEhJCdQsAAABAUIWs+B40aJAKCwtVUFCgyZMna9KkSRozZoxmz56tw4cPS5KKioq0YsUK3XXXXaqvr1dBQYEk6Sc/+YmKi4s1ceJEHTx4UN///vclSStXrtSvf/1r5eTk6He/+52WLVsWqvDRD7nVMtWk3ukRi60hWrWu6z3+OruSY5xdXwAACKqQbrKTm5ur3Nxcv9fWrVvn/X7kyJHasmXLJdcNGTJEv/nNby55/frrr9emTZuCHyiglnW9X/trperqmpSVMSzc4QAh0bqutyQNyb0vzNEAQP/DDpcAAAA9EG9purjqSYxdjYa98wsAUXwDAAD0zIXlByVpwNg7JBvFN7pG8Q0A6JbWdb9bv7d0ejYAwBfFNwCgWzweQx/9/bQk6YqxhmxWym8ACBTFN/olt1oesGxljw3Zwj9AWLWubtL6vVxdXAAACCmKb/RLTU6Pyg8c97ZZ3QTRitVNACCyUHwDAAAEAaufIBAU3wAAAMHA6icIAMU3AKBXWP0EAAJH8Q0A6BVWPwGAwFF8A0CUGBB7cWUTidVNACASUXyjX2i7tKARvlCAkLG6m7wrm0isbgIAkYjiG/1C26UFb795aPiCAQD0C6x+gvZQfAMAgsr3AczWNrPA0S+x+gnaQfENAAgq3wcwJR7CBABfFN8A0Ef5bh0vSXEWTydnAwAiAcU3opLvA5b2WGt4gwFCxHfreEkamntPGKMB0BW/OeAS88D7KYpvRCXfByyzMoaFNRYAACT5zQGXmAfeX1F8AwBCjl0wAaAFxTcA9BGtc7xjY2x9bgMddsEEgBZMhkVUcEuqd3pU7/TIHe5ggBBpneP92evlsroawx0OAKAHGPlGVGCONwCgr4mxWRTvPuvzAg9g9gcU3wAA0zEHHJAMV7POvLXH2+YBzP6B4hsAIpTvOt59bY53V5gDDqC/ovhGn+O7hrfEOt6IXr7reA/JvS/M0QAwg99a4ExDiUoU3+hzfOd3S8zxBgBEEZ+1wJmGEp0ovgEgAgyI9d8qPtqmmXSFOeAA+guKbwCIAFZ3k99W8f1tmglzwIH2tU5DcZ5uULzFyjSUKEDxjYjXdo63Eb5QAAAw14VpKM5Eu2JH3Mo0lChA8Y2I13aO9+03Dw1fMECQ+K5kIklxFk8nZ/c/vtNQACCaUHwj4viOdLOSCaKV70omkjQ0954wRhN5fKehSFLqWApxQGqzGorEiih9EMU3Ig67VSIaRfOa3QBM5LMaisSKKH0RxTcAmIA1u3vJYvGbhsKKKEALtqjveyi+EXZMM0E0YqQ7uAzDfxoKK6IALdiivu+h+EbYMc0E0YiR7tBjbXCgfeySGdkovmE6RroRjRjpNl/btcGtVgvFOCCxS2aEo/hGSLVdo9sea2WkG1GJke7wY6MeoH2MhEcWim+EVNs1uim2EQ3artHNSHdkYq1w4II2I+HxMWK5wjAKafG9bds2Pf/883K5XJo5c6Yeeughv+NHjx7VokWLVFdXp/T0dD355JOKiYnRp59+qvnz5+vUqVO69tprVVRUpMTERJ09e1Y//OEP9cknn2jgwIFas2aNHA5HKG8B3cSUEkSjtlNK2q7RzUh3ZLpkrfCviBVTAOmS5QqvTL9T8W6KcbOErDqqqqrS6tWrtXHjRpWUlGjz5s06duyY3znz58/XkiVLtHPnThmGoeLiYknSk08+qQcffFBlZWUaPXq0nnvuOUnSmjVrlJ6erh07dmjq1Klavnx5qMJHB9yS6p0e1Ts9crfTbh3pLj9w3G+6CRDJWovr8dfZlRzjvKTdWmyfOLBTVldjuMNFD7WumNL65fEYMiTVNTjlvPA90B8ZrmadObTL+yVXk+ItTYp3n235sjR13QkCFrLie9++fcrIyFBKSooSEhKUnZ2tsrIy7/HKyko1NjYqLS1NkpSfn6+ysjI5nU69+eabys7O9ntdknbv3q3c3FxJ0qRJk7Rnzx45nc5Q3UK/51tYd1RcU2yjrxkQe7Gw7qi4ptjuPzweQ/9XecavGHd6DG8x3rYN9BsXRsc7KsYpznsuZNNOTp486TclJDU1VW+//XaHxx0Oh6qqqvT5558rKSlJMTExfq+3vSYmJkZJSUmqra3VoEGDAorJGqUP3wR6Xx5JTa4LU0JiWv6/q6N2q31vV3q/v23s1bLZLEpKiJUk2Wwtf26g7Z5c014fMTZrr/rsrI/E+FhZDE/Y7i3S89OTuBJ6mNP24rrC7taEkUne9oA4tyR5X+uqLUmXWZr1+ZF93vY1g++WbDbFJyVLkqw2myQF3O7JNe31YbHFBNxnbGxMr/sIRhyRmp/u9mFPSpbV6fK2K0+3vG+utFgvbVstMuwJ3j504Rrva+20LZJslyVeiKHleGu7NS7/tv85XbUjpQ/fttUeJ5snpld9ROq9BbePwOOy2uPCmx/DpXPvHZAkJY+6RZL82pfZrJK72duHbHEt/219rZ12kxGncApVXdhVvxbDMELyP/PPP/+8mpqa9P3vf1+SVFxcrCNHjmjp0qWSpIqKCj311FPauHGjJOn48eOaM2eOfvWrX2natGn685//LElyuVwaO3asDh8+rNGjR+utt97yFua33Xabtm7dyrxvAAAA9Akhm3YyePBgVVdXe9vV1dVKTU3t8HhNTY1SU1M1cOBAnTt3Tm63+5LrUlNTVVNTI6mlKK+rq1NKSkqobgEAAAAIqpAV3+PHj9f+/ftVW1urhoYGlZeXKzMz03t8yJAhstvtqqiokCSVlpYqMzNTsbGxSk9P1/bt2yVJJSUl3usmTJigkpISSdL27duVnp6u2NhYAQAAAH1ByKadSC1LDa5du1ZOp1NTpkzR7NmzNXv2bM2dO1c33nij3nvvPS1evFjnz5/XqFGjtGLFCsXFxamyslILFizQqVOndNVVV+nf//3fNWDAAJ0+fVoLFizQJ598ouTkZBUVFenqq68OVfgAAABAUIW0+AYAAABwEbugAAAAACah+AYAAABMQvENAAAAmITiGwAAADAJxTcAAABgEorvCPfss88qJydHOTk5WrVqlSRp8+bNmjRpknJzc7Vw4UI1N7ds1Xr06FHl5+crOztbixYtksvlCmfoEam9fLb67W9/qxkzZnjb5DMw7eX00KFDmjZtmnJycvSDH/yA92g3tZfTvXv36p577tGkSZP0ox/9iJx209NPP62JEycqJydH69evlyTt27dPubm5ysrK0urVq73nktOutZdPPpt6p72ctuLzqWfay2lEfD4ZiFivvfaacf/99xtNTU1Gc3OzUVBQYKxdu9b4+te/bpw7d87weDzGj370I2P9+vWGYRhGTk6OcejQIcMwDGPhwoXGhg0bwhh95Gkvn+Xl5YZhGMaHH35o3HbbbcY3vvEN7/nks2vt5XTr1q3GLbfcYhw9etQwDMMoLCz05o6cdq2j92lmZqZx7NgxwzAM49FHHzWKi4sNwyCngXj99deN6dOnG06n02hoaDC+9rWvGUePHjUmTJhgfPzxx4bT6TRmzZpl7N692zAMctqV9vL50Ucf8dnUCx3l1DD4fOqpjv7eR8LnEyPfEczhcGjBggWKi4tTbGysrrvuOjU3N+snP/mJkpKSZLFYNGLECH366aeqrKxUY2Oj0tLSJEn5+fkqKysL8x1Elvby+emnn6q5uVlLlizR3LlzveeSz8C0l9PKykqlpaVp5MiRkqTFixfr61//OjkNUEfvU7fbrfPnz8vtdqupqUl2u52cBujmm2/Wr3/9a8XExOjUqVNyu906e/ashg4dqmuuuUYxMTHKzc1VWVkZOQ1Ae/m02+18NvVCezlNSEjg86kX2svp0aNHI+LzieI7gl1//fXeN8Lx48e1Y8cOTZo0Sbfccoskqba2Vhs2bNAdd9yhkydPyuFweK91OByqqqoKS9yRqr18TpgwQU899ZTuu+8+XXPNNd5zyWdg2stpXFycEhISVFhYqLy8PD3zzDO6/PLLyWmAOnqf/vSnP9WMGTN022236fPPP9ddd91FTrshNjZW//Ef/6GcnBz9v//3/y7JXWpqqqqqqshpgNrm8x/+4R/4bOqltjkdNGgQn0+91Dan1dXVEfH5RPHdB3z44YeaNWuWfvSjH2nYsGGSpKqqKs2cOVP33Xefxo0bJ4/HI4vF4r3GMAy/Ni7yzWdlZaVOnDih++67z+8c8tk9vjl1u93au3evfvCDH2jr1q1qaGjQiy++SE67yTeniYmJKioq0ssvv6y9e/fqpptu0ooVK8hpN82dO1f79+/XiRMndPz48XZzR04D55vP4uJiSXw29ZZvTjdv3sznUxD45rS5uTkiPp8oviNcRUWFHn74YT322GO69957JUkfffSRpk+frnvvvVff+973JEmDBw9WdXW197qamhqlpqaGJeZI1jafL7/8sj788EPl5eVp8eLFOnLkiL7//e+Tz25om9MvfOELuummm3TNNdfIZrPp7rvv1ttvv01Ou6FtTg8ePKgRI0boi1/8oqxWq6ZNm6Y33niDnAboo48+0tGjRyVJl112mbKysvT666/75a66ulqpqankNADt5fP999/ns6kX2svp//7v//L51Avt5fTFF1+MiM8niu8IduLECX3ve99TUVGRcnJyJEnnz5/Xv/zLv2jevHmaNWuW99whQ4bIbreroqJCklRaWqrMzMywxB2p2svnihUrtGPHDpWWlmrZsmUaPXq01qxZQz4D1F5Ob731Vr3zzjs6ceKEJOlPf/qTRo0aRU4D1F5OR4wYobfffls1NTWSpF27dunGG28kpwH6+9//rsWLF6u5uVnNzc3atWuXpk+frv/7v//T3/72N7ndbr388svKzMwkpwFoL59jxozhs6kX2svprbfeyudTL7SX06VLl0bE51NMyHpGr7300ktqamrSz3/+c+9rEydOVE1NjdavX+9dNuf222/XvHnzVFRUpMWLF+v8+fMaNWqUCgoKwhV6RGovn9OnT9cDDzzQ7vnks2sd5XTp0qWaM2eOmpqa9I//+I/68Y9/LImcBqKjnM6bN08FBQWy2WwaOnSoli5dKomcBmLChAl6++23NXnyZNlsNmVlZSknJ0cDBw7Uo48+qqamJk2YMEF33XWXJHLalfbyefr0aT6beqGj92hHyGnX2svp5MmTlZKSEvbPJ4thGEbIegcAAADgxbQTAAAAwCQU3wAAAIBJKL4BAAAAk1B8AwAAACah+AYAAABMQvENAOi1WbNmqba2tlvXHD58WLfffnuIIgKAyETxDQDotddeey3cIQBAn8AmOwDQR23ZskXr16+X1WrVFVdcoZUrV2rPnj36zW9+I6vVqi984Qt64okndO2112rBggVKSkrS+++/r88++0w33HCDVq5cqZdffll/+tOf9MILL0hq2ZL54Ycf1u7du3X8+HEtX75cp0+fltvt1owZMzRlyhS9/vrrWr58uRISElRXV6fRo0dLkmbOnKkXX3xRVqtVS5cu1YkTJ+R0OpWTk6M5c+ZIkjZu3Khf/epXSkpK0ogRI8KWOwAIF4pvAOiD3nvvPRUVFen3v/+9rrrqKv3Xf/2XHn74YXk8Hm3evFkDBw7U1q1b9b3vfU9/+MMfJElHjhzRr3/9a1ksFk2bNk1lZWXKyclRUVGRqqur5XA4tHXrVuXn58swDM2dO1erVq3SqFGjdO7cOd1///0aPny4JOnDDz/UH//4Rw0ZMkSStHXrVv3qV7/SwIEDVVBQoIcffli33367mpqaNHv2bH3xi1/Utddeq2effValpaVyOBxasmRJ2PIHAOFC8Q0AfdD+/ft166236qqrrpIkPfzwwzp58qRiY2M1cOBASVJ+fr6WL1+uv//975Kk2267TXFxcZKkESNG6MyZM0pKStLXv/51/c///I8efvhhbdu2TRs2bNDx48f18ccf6/HHH/f+mY2NjXr33Xd13XXX6aqrrvIW3r7q6+v15ptv6syZM3r66ae9r7333nv67LPPdMstt8jhcEiS7r//fu3duzd0SQKACETxDQB9kM1mk8Vi8bYbGxv1ySef6Etf+pLfeYZhyOVySZLi4+O9r1ssFhmGIUmaNm2annjiCV133XW67rrrdM011+j9999XcnKySktLvdfU1NQoOTlZb731lhISEtqNy+PxyDAMbdq0SZdddpkkqba2Vna7XZs3b/b+ma33AAD9DQ9cAkAfNG7cOO3fv18nT56UJG3atEl//vOftX37du+qI7/73e+UkpKioUOHdtpXWlqaJOmXv/ylpk6dKkm69tprFR8f7y2+T5w4oUmTJunIkSPt9mGz2eRyuZSUlKS0tDStX79eknT27Fk98MAD2rVrl2655Ra99tpr+uyzzyRJv//973uZBQDoexj5BoA+6IYbbtD8+fP1rW99S5LkcDj0yiuv6I9//KNmzpwpj8ejgQMHau3atbJaux5nmTp1qp577jndeeedkqS4uDg999xzWr58uf7zP/9TLpdL8+bN0z/90z/p9ddfv+T6u+66SzNmzNAzzzyjoqIi/exnP1Nubq6am5s1adIk3XPPPZKk+fPna+bMmUpMTNSYMWOCmBEA6Bsshu/vAAEAAACEDNNOAAAAAJNQfAMAAAAmofgGAAAATELxDQAAAJiE4hsAAAAwCcU3AAAAYBKKbwAAAMAkFN8AAACASf4/qZm7mZdVdqUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,6))\n",
    "xA = np.linspace(264-49, 264+50, 100)\n",
    "yA = stats.binom(586, 0.45).pmf(x)\n",
    "ax.bar(xA, yA, alpha=0.5)\n",
    "xB = np.linspace(308-49, 308+50, 100)\n",
    "yB = stats.binom(657, 0.47).pmf(xB)\n",
    "ax.bar(xB, yB, alpha=0.5)\n",
    "plt.xlabel('converted')\n",
    "plt.ylabel('probability')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate Power and minimum sample size needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ci(lift, alpha, sd):\n",
    "    '''compute Confidence Interval'''\n",
    "    \n",
    "    val = abs(stats.norm.ppf((1-alpha)/2))\n",
    "    \n",
    "    #Calculate bounds\n",
    "    lwr_bnd = lift - val * sd\n",
    "    upr_bnd = lift + val * sd\n",
    "    \n",
    "    #return lower and upper bounds\n",
    "    return (lwr_bnd, upr_bnd)\n",
    "\n",
    "def get_power(n, p1, p2, alpha):\n",
    "    \n",
    "    '''Calculate Power'''\n",
    "    \n",
    "    qu = stats.norm.ppf(1 - alpha/2)\n",
    "    diff = abs(p2-p1)\n",
    "    bp = (p1+p2) / 2\n",
    "    \n",
    "    v1 = p1 * (1-p1)\n",
    "    v2 = p2 * (1-p2)\n",
    "    bv = bp * (1-bp)\n",
    "    \n",
    "    power_part_one = stats.norm.cdf((n**0.5 * diff - qu * (2 * bv)**0.5) / (v1+v2) ** 0.5)\n",
    "    power_part_two = 1 - stats.norm.cdf((n**0.5 * diff + qu * (2 * bv)**0.5) / (v1+v2) ** 0.5)\n",
    "    \n",
    "    power = power_part_one + power_part_two\n",
    "    \n",
    "    return (power)\n",
    "\n",
    "def min_sample_size(bcr, mde, power, sig_level=0.05):\n",
    "    \"\"\"Returns the minimum sample size to set up a split test\n",
    "   \n",
    "   Arguments:\n",
    "        bcr (float): probability of success for control\n",
    "        mde (float): minimum change in measurement between control\n",
    "        group and test group \n",
    "        power (float): probability of rejecting the null hypothesis when the\n",
    "        null hypothesis is false, typically 0.8\n",
    "        sig_level (float): significance level often denoted as alpha,\n",
    "        typically 0.05\n",
    "   \n",
    "   Returns:\n",
    "        min_N: minimum sample size (float)\n",
    "    References:\n",
    "        Stanford lecture on sample sizes\n",
    "        http://statweb.stanford.edu/~susan/courses/s141/hopower.pdf\n",
    "    \"\"\"\n",
    "    # standard normal distribution to determine z-values\n",
    "    standard_norm = stats.norm(0, 1)\n",
    "\n",
    "    # find Z_beta from desired power\n",
    "    Z_beta = standard_norm.ppf(power)\n",
    "\n",
    "    # find Z_alpha\n",
    "    Z_alpha = standard_norm.ppf(1-sig_level/2)\n",
    "\n",
    "    # average of probabilities from both groups\n",
    "    pooled_prob = (bcr + bcr+mde) / 2\n",
    "\n",
    "    min_N = (2 * pooled_prob * (1 - pooled_prob) * (Z_beta + Z_alpha)**2\n",
    "             / mde**2)\n",
    "\n",
    "    return min_N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_conv = exposed_group.value_counts(normalize= True)[1]\n",
    "con_conv = control_group.value_counts(normalize= True)[1]\n",
    "\n",
    "test_size = exposed_group.shape[0]\n",
    "con_size = control_group.shape[0]\n",
    "\n",
    "lift_mean = test_conv - con_conv\n",
    "lift_variance = (1 - test_conv) * test_conv /test_size + (1 - con_conv) * con_conv / con_size\n",
    "lift_sd = lift_variance**0.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.03720170164125786, 0.07377294023222619)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lower bound, upper bound\n",
    "get_ci(lift_mean, 0.95, lift_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14992904553585906"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Compute power\n",
    "N= con_size + test_size\n",
    "power = get_power(N, con_conv, test_conv, 0.05) ; power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1257.0"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Minimum sample size needed\n",
    "d = control_group.mean() - exposed_group.mean()\n",
    "round(min_sample_size(con_conv, d, power))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The optimum sample size needed is 1257 while the sample size tha engaged in the data is 1243"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11576.0"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(min_sample_size(con_conv, d, 0.8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To hold a test with a higher probability of rejecting null hypothesis when it is to be rejected, the optimum sample size need is 11576"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EML-L29                  1\n",
       "CUBOT_X18_Plus           1\n",
       "X4_Soul_Infinity_Z       1\n",
       "VFD 500                  1\n",
       "Samsung SM-G986U1        1\n",
       "XiaoMi MI 6              1\n",
       "XiaoMi Redmi 8           1\n",
       "H3113                    1\n",
       "Samsung SM-G850F         1\n",
       "HTC U11 $2               1\n",
       "VFD 700                  1\n",
       "Samsung SM-G973U         1\n",
       "Samsung SM-A908B         1\n",
       "OnePlus ONEPLUS A3000    1\n",
       "Samsung SM-G960U1        1\n",
       "G3221                    1\n",
       "XiaoMi Redmi 5 Plus      1\n",
       "TA-1024                  1\n",
       "Samsung SM-J810F         1\n",
       "XiaoMi Redmi 3S          1\n",
       "Name: device_make, dtype: int64"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.device_make.value_counts()[-20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Generic Smartphone       4743\n",
       "iPhone                    433\n",
       "LG-$2                      43\n",
       "POT-LX1                    40\n",
       "CLT-L09                    26\n",
       "VOG-L09                    23\n",
       "MAR-LX1A                   21\n",
       "Moto $2                    19\n",
       "FIG-LX1                    18\n",
       "ANE-LX1                    18\n",
       "I3312                      18\n",
       "LYA-L09                    17\n",
       "OnePlus ONEPLUS A5010      15\n",
       "PRA-LX1                    14\n",
       "OnePlus ONEPLUS A6013      14\n",
       "VOG-L29                    12\n",
       "MRD-LX1                    12\n",
       "OnePlus ONEPLUS A6003      11\n",
       "VTR-L09                     9\n",
       "SNE-LX1                     9\n",
       "Name: device_make, dtype: int64"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.device_make[df.device_make.str.contains('Samsung') | df.device_make.str.contains('Nokia')  \\\n",
    "               | df.device_make.str.contains('Pixel') == False].value_counts()[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Consider only device with more than 30 occurence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['device'] = df.device_make.apply(lambda x: 'iPhone' if x.startswith('iPhone') else 'Samsung' if x.startswith('Samsung') else \\\n",
    "                                   'Nokia' if x.startswith('Nokia') else 'OnePlus' if x.startswith('OnePlus') else 'LG' if x.startswith('LG') \\\n",
    "                                   else 'Pixel' if x.startswith('Pixel') else 'LX1' if x.startswith('POT') else 'others')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "engaged['device'] = engaged.device_make.apply(lambda x: 'iPhone' if x.startswith('iPhone') else 'Samsung' if x.startswith('Samsung') else \\\n",
    "                                   'Nokia' if x.startswith('Nokia') else 'OnePlus' if x.startswith('OnePlus') else 'LG' if x.startswith('LG') \\\n",
    "                                   else 'Pixel' if x.startswith('Pixel') else 'LX1' if x.startswith('POT') else 'others')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract Day of week from DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.date = pd.to_datetime(df.date)\n",
    "df['dow'] = df.date.apply(lambda x: x.dayofweek)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "engaged.date = pd.to_datetime(engaged.date)\n",
    "engaged['dow'] = engaged.date.apply(lambda x: x.dayofweek)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>device_make</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "      <th>engaged</th>\n",
       "      <th>device</th>\n",
       "      <th>dow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0008ef63-77a7-448b-bd1e-075f42c55e39</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-10</td>\n",
       "      <td>8</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>others</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000eabc5-17ce-4137-8efe-44734d914446</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-07</td>\n",
       "      <td>10</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>others</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0016d14a-ae18-4a02-a204-6ba53b52f2ed</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-05</td>\n",
       "      <td>2</td>\n",
       "      <td>E5823</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>others</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00187412-2932-4542-a8ef-3633901c98d9</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Samsung SM-A705FN</td>\n",
       "      <td>6</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001a7785-d3fe-4e11-a344-c8735acacc2c</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>others</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             auction_id experiment       date  hour  \\\n",
       "0  0008ef63-77a7-448b-bd1e-075f42c55e39    exposed 2020-07-10     8   \n",
       "1  000eabc5-17ce-4137-8efe-44734d914446    exposed 2020-07-07    10   \n",
       "2  0016d14a-ae18-4a02-a204-6ba53b52f2ed    exposed 2020-07-05     2   \n",
       "3  00187412-2932-4542-a8ef-3633901c98d9    control 2020-07-03    15   \n",
       "4  001a7785-d3fe-4e11-a344-c8735acacc2c    control 2020-07-03    15   \n",
       "\n",
       "          device_make  platform_os                browser  yes  no  engaged  \\\n",
       "0  Generic Smartphone            6          Chrome Mobile    0   0        0   \n",
       "1  Generic Smartphone            6          Chrome Mobile    0   0        0   \n",
       "2               E5823            6  Chrome Mobile WebView    0   1        1   \n",
       "3   Samsung SM-A705FN            6               Facebook    0   0        0   \n",
       "4  Generic Smartphone            6          Chrome Mobile    0   0        0   \n",
       "\n",
       "    device  dow  \n",
       "0   others    4  \n",
       "1   others    1  \n",
       "2   others    6  \n",
       "3  Samsung    4  \n",
       "4   others    4  "
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Problem formulation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Building machine models to predict the response of a custoomer. 1-yes, 0-no"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import libraries\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import QuantileTransformer, StandardScaler, PowerTransformer, PolynomialFeatures\n",
    "from xgboost import XGBClassifier,DMatrix, train, cv\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV\n",
    "from category_encoders.polynomial import PolynomialEncoder\n",
    "from category_encoders.target_encoder import TargetEncoder\n",
    "from sklearn.metrics import log_loss, accuracy_score, f1_score, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auction_id</th>\n",
       "      <th>experiment</th>\n",
       "      <th>date</th>\n",
       "      <th>hour</th>\n",
       "      <th>device_make</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>yes</th>\n",
       "      <th>no</th>\n",
       "      <th>engaged</th>\n",
       "      <th>device</th>\n",
       "      <th>dow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0016d14a-ae18-4a02-a204-6ba53b52f2ed</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-05</td>\n",
       "      <td>2</td>\n",
       "      <td>E5823</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>others</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>008aafdf-deef-4482-8fec-d98e3da054da</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-04</td>\n",
       "      <td>16</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>others</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00a1384a-5118-4d1b-925b-6cdada50318d</td>\n",
       "      <td>exposed</td>\n",
       "      <td>2020-07-06</td>\n",
       "      <td>8</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>others</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00b6fadb-10bd-49e3-a778-290da82f7a8d</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-08</td>\n",
       "      <td>4</td>\n",
       "      <td>Samsung SM-A202F</td>\n",
       "      <td>6</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00ebf4a8-060f-4b99-93ac-c62724399483</td>\n",
       "      <td>control</td>\n",
       "      <td>2020-07-03</td>\n",
       "      <td>15</td>\n",
       "      <td>Generic Smartphone</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>others</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             auction_id experiment       date  hour  \\\n",
       "0  0016d14a-ae18-4a02-a204-6ba53b52f2ed    exposed 2020-07-05     2   \n",
       "1  008aafdf-deef-4482-8fec-d98e3da054da    exposed 2020-07-04    16   \n",
       "2  00a1384a-5118-4d1b-925b-6cdada50318d    exposed 2020-07-06     8   \n",
       "3  00b6fadb-10bd-49e3-a778-290da82f7a8d    control 2020-07-08     4   \n",
       "4  00ebf4a8-060f-4b99-93ac-c62724399483    control 2020-07-03    15   \n",
       "\n",
       "          device_make  platform_os                browser  yes  no  engaged  \\\n",
       "0               E5823            6  Chrome Mobile WebView    0   1        1   \n",
       "1  Generic Smartphone            6          Chrome Mobile    1   0        1   \n",
       "2  Generic Smartphone            6          Chrome Mobile    0   1        1   \n",
       "3    Samsung SM-A202F            6               Facebook    1   0        1   \n",
       "4  Generic Smartphone            6          Chrome Mobile    0   1        1   \n",
       "\n",
       "    device  dow  \n",
       "0   others    6  \n",
       "1   others    5  \n",
       "2   others    0  \n",
       "3  Samsung    2  \n",
       "4   others    4  "
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "engaged.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this machine learning modelling part, we'll try to predict the response of the questionnaires and also obtain the variable that drives customers engagement\n",
    "\n",
    "* Class 0: Customers that will answer NO\n",
    "* Class 1: Customers that will answer YES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = engaged.drop(['auction_id', 'date', 'device_make', 'yes', 'no', 'engaged'], axis = 1)\n",
    "y = engaged.yes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .1, random_state = 10) #split dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>hour</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>browser</th>\n",
       "      <th>device</th>\n",
       "      <th>dow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1186</th>\n",
       "      <td>exposed</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>973</th>\n",
       "      <td>exposed</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile WebView</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1087</th>\n",
       "      <td>exposed</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>Samsung Internet</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>exposed</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>others</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>752</th>\n",
       "      <td>control</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>Chrome Mobile</td>\n",
       "      <td>others</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     experiment  hour  platform_os                browser   device  dow\n",
       "1186    exposed    17            6  Chrome Mobile WebView  Samsung    3\n",
       "973     exposed    21            6  Chrome Mobile WebView  Samsung    3\n",
       "1087    exposed    15            6       Samsung Internet  Samsung    2\n",
       "373     exposed     3            6          Chrome Mobile   others    0\n",
       "752     control     7            6          Chrome Mobile   others    0"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encode categorical variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc = TargetEncoder(cols = ['experiment', 'device', 'browser']).fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = enc.transform(X_train)\n",
    "X_test = enc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(X_train, X_test, n_poly = 3):\n",
    "    '''Transform data for linear model'''\n",
    "    \n",
    "    Q = QuantileTransformer(output_distribution= 'normal')\n",
    "    S = StandardScaler()\n",
    "    P = PowerTransformer(standardize= False)\n",
    "    #Scale Data and transform\n",
    "    X_train_std = S.fit_transform(X_train)\n",
    "    X_test_std = S.transform(X_test)\n",
    "    #Transformer to have a normal like distribution\n",
    "    X_train_nom = P.fit_transform(X_train_std)\n",
    "    X_test_nom = P.transform(X_test_std)\n",
    "    #Add polynomial features\n",
    "    poly = PolynomialFeatures(degree=n_poly)\n",
    "    X_train_poly = poly.fit_transform(X_train_nom)\n",
    "    X_test_poly = poly.transform(X_test_nom)\n",
    "\n",
    "    return X_train_poly, X_test_poly\n",
    "    \n",
    "def logit(X_train, X_test, y_train, y_test,multi_class = 'raise'):\n",
    "    '''Fit Logistic regression model'''\n",
    "    \n",
    "    #Call the model\n",
    "    lr = LogisticRegression(random_state= 10, max_iter = 10000)\n",
    "    #fit model\n",
    "    lr.fit(X_train, y_train)\n",
    "    #Obtain and print AUC Score for test and train\n",
    "    train_score = roc_auc_score(y_train, lr.predict_proba(X_train)[:,1], multi_class=multi_class)\n",
    "    test_score = roc_auc_score(y_test, lr.predict_proba(X_test)[:,1],multi_class=multi_class )\n",
    "    print(f'train score: {train_score}, test score: {test_score}')\n",
    "    \n",
    "    \n",
    "def dt(X_train, X_test, y_train, y_test, multi_class = 'raise'):\n",
    "    '''Fit Decision Tree model'''\n",
    "    #Call model\n",
    "    dt = DecisionTreeClassifier(random_state=10)\n",
    "    \n",
    "    #Fit model\n",
    "    dt.fit(X_train, y_train)\n",
    "    \n",
    "    #Obtain and print train and test score\n",
    "    train_score = roc_auc_score(y_train, dt.predict_proba(X_train)[:,1], multi_class=multi_class)\n",
    "    test_score = roc_auc_score(y_test, dt.predict_proba(X_test)[:,1], multi_class=multi_class)\n",
    "    print(f'train score: {train_score}, test score: {test_score}')\n",
    "    \n",
    "    imp = pd.DataFrame(dt.feature_importances_, index = X_train.columns, columns = ['coef'])\n",
    "\n",
    "    return imp.sort_values(by = 'coef', ascending =False)\n",
    "    \n",
    "def xgboost(X_train, X_test, y_train, y_test):\n",
    "    \n",
    "    '''Fit XGBoost model'''\n",
    "    #Call model\n",
    "    xgb = XGBClassifier(random_state= 10)\n",
    "    \n",
    "    #fit model\n",
    "    xgb.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=100, eval_metric= 'auc', verbose=False)\n",
    "    \n",
    "    #Calculate and print train and test score\n",
    "    train_score = roc_auc_score(y_train, xgb.predict_proba(X_train)[:,1])\n",
    "    test_score = roc_auc_score(y_test, xgb.predict_proba(X_test)[:,1])\n",
    "    print(f'train score: {train_score}, test score: {test_score}')\n",
    "    \n",
    "    imp = pd.DataFrame(xgb.feature_importances_, index = X_train.columns, columns = ['coef'])\n",
    "    return imp.sort_values(by = 'coef', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train score: 0.6587537855991143, test score: 0.5272727272727273\n"
     ]
    }
   ],
   "source": [
    "imp = xgboost(X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>device</th>\n",
       "      <td>0.287747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>experiment</th>\n",
       "      <td>0.209806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dow</th>\n",
       "      <td>0.171368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hour</th>\n",
       "      <td>0.169532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>browser</th>\n",
       "      <td>0.161548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>platform_os</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 coef\n",
       "device       0.287747\n",
       "experiment   0.209806\n",
       "dow          0.171368\n",
       "hour         0.169532\n",
       "browser      0.161548\n",
       "platform_os  0.000000"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train score: 0.6303243787755418, test score: 0.46571428571428575\n"
     ]
    }
   ],
   "source": [
    "logit(*transform(X_train, X_test), y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train score: 0.9050824383603085, test score: 0.46545454545454545\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>hour</th>\n",
       "      <td>0.433597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dow</th>\n",
       "      <td>0.242032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>browser</th>\n",
       "      <td>0.139907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>experiment</th>\n",
       "      <td>0.131323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>device</th>\n",
       "      <td>0.050736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>platform_os</th>\n",
       "      <td>0.002404</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 coef\n",
       "hour         0.433597\n",
       "dow          0.242032\n",
       "browser      0.139907\n",
       "experiment   0.131323\n",
       "device       0.050736\n",
       "platform_os  0.002404"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt(X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using XGBoost feature imoportance, the device variable has the highest importance while platform os contributes nothing\n",
    "\n",
    "For Decision Tree feature importance, the hour column has the highest importance\n",
    "\n",
    ">> Conclusion: The decision tree classifier overfitted the data hence XGBoost feature importance is more reliable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hyperparameter tuning and cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logit_hyper(X_train_trans, X_test_trans, y_train, y_test):\n",
    "    \n",
    "    '''Hyper Parameter Search and Cross Validation for logistic Regression'''\n",
    "    \n",
    "    print('Searching for best hyperparameter... ')\n",
    "    #Set params\n",
    "    params = {'C': np.linspace(0.0001,0.001,20)}\n",
    "    \n",
    "    #Init and fit grid search\n",
    "    lr_grid = GridSearchCV(LogisticRegression(random_state = 10, max_iter = 10000), params, scoring='roc_auc', cv =5)\n",
    "    lr_grid.fit(X_train_trans, y_train)\n",
    "    print('--------------------------------------\\nDONE')\n",
    "    print(f'Best Score {lr_grid.best_score_} Best Param {lr_grid.best_params_}')\n",
    "    print(f'AUC score on test set {roc_auc_score(y_test, lr_grid.best_estimator_.predict(X_test_trans))}')\n",
    "    \n",
    "    print('Running 5fold stratified Cross Val \\nReturning 5fold CV Scores')\n",
    "    print('--------------------------------------')\n",
    "    \n",
    "    error1 = [] #container to update score\n",
    "    y_pred_lr = [] #container to update predictions\n",
    "    \n",
    "    #stratified kfold\n",
    "    fold=StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    \n",
    "    #Get index of splits and assing\n",
    "    for train_index, test_index in fold.split(X_train_trans, y_train):\n",
    "        X_t, X_val = X_train_trans[train_index], X_train_trans[test_index]\n",
    "        y_t, y_val = y_train.iloc[train_index], y_train.iloc[test_index]\n",
    "        \n",
    "        #train split with best estimator from grid_search  and predict\n",
    "        lr_grid.best_estimator_.fit(X_t, y_t)\n",
    "        preds=lr_grid.best_estimator_.predict_proba(X_val)[:, 1]\n",
    "        \n",
    "        #Auc score and append\n",
    "        print(\"AUC SCORE: \",roc_auc_score(y_val,preds))\n",
    "        error1.append(roc_auc_score(y_val,preds))\n",
    "        #predict on test set\n",
    "        prob = lr_grid.best_estimator_.predict_proba(X_test_trans)[:, 1]\n",
    "        y_pred_lr.append(prob)\n",
    "    print('--------------------------------------\\nDONE')\n",
    "    print(f'Average AUC SCORE: {np.mean(error1)}')\n",
    "    print('AUC ScORE ON HOLDOUT SET: {}'.format(roc_auc_score(y_test, np.mean(y_pred_lr,0))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching for best hyperparameter... \n",
      "--------------------------------------\n",
      "DONE\n",
      "Best Score 0.5668463485249673 Best Param {'C': 0.0001}\n",
      "AUC score on test set 0.4967532467532468\n",
      "Running 5fold stratified Cross Val \n",
      "Returning 5fold CV Scores\n",
      "--------------------------------------\n",
      "AUC SCORE:  0.617948717948718\n",
      "AUC SCORE:  0.553886217948718\n",
      "AUC SCORE:  0.5476209580357859\n",
      "AUC SCORE:  0.5610032362459546\n",
      "AUC SCORE:  0.4818770226537217\n",
      "--------------------------------------\n",
      "DONE\n",
      "Average AUC SCORE: 0.5524672305665798\n",
      "AUC ScORE ON HOLDOUT SET: 0.452987012987013\n"
     ]
    }
   ],
   "source": [
    "logit_hyper(*transform(X_train, X_test), y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dt_hyper(X_train, X_test, y_train, y_test):\n",
    "    \n",
    "    '''Hyper Parameter Search and Cross Validation for logistic Regression'''\n",
    "\n",
    "    \n",
    "    print('Searching for best hyperparameter... ')\n",
    "    params = {\n",
    "              'criterion': ['gini', 'entropy'], \n",
    "              'max_depth': [2,4,6,8,10], \n",
    "              'max_features': [0.1, 0.2, 0.3, 0.4, 0.5],                \n",
    "              'min_samples_split': [2,3,4,5,6,7]}\n",
    "\n",
    "    dt_grid = GridSearchCV(DecisionTreeClassifier(random_state = 10), params, scoring='roc_auc', cv =5)\n",
    "    dt_grid.fit(X_train, y_train)\n",
    "    \n",
    "    print('--------------------------------------\\nDONE')\n",
    "    print(f'Best Score {dt_grid.best_score_} Best Param {dt_grid.best_params_}')\n",
    "    print(f'AUC score on test set {roc_auc_score(y_test, dt_grid.best_estimator_.predict(X_test))}')\n",
    "    \n",
    "    print('Running 5fold stratified Cross Val \\nReturning 5fold CV Scores')\n",
    "    print('--------------------------------------')\n",
    "\n",
    "    \n",
    "    error_dt = []\n",
    "    y_pred_dt = []\n",
    "    fold=StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    for train_index, test_index in fold.split(X_train, y_train):\n",
    "        X_t, X_val = X_train.iloc[train_index], X_train.iloc[test_index]\n",
    "        y_t, y_val = y_train.iloc[train_index], y_train.iloc[test_index]\n",
    "        dt_grid.best_estimator_.fit(X_t, y_t)\n",
    "        preds=dt_grid.best_estimator_.predict_proba(X_val)[:, 1]\n",
    "        print(\"err: \",roc_auc_score(y_val,preds))\n",
    "        error_dt.append(roc_auc_score(y_val,preds))\n",
    "        prob = dt_grid.best_estimator_.predict_proba(X_test)[:, 1]\n",
    "        y_pred_dt.append(prob)\n",
    "    print('--------------------------------------\\nDONE')\n",
    "    print(f'Average AUC SCORE: {np.mean(error_dt)}')\n",
    "    print('AUC ScORE ON HOLDOUT SET: {}'.format(roc_auc_score(y_test, np.mean(y_pred_dt,0))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching for best hyperparameter... \n",
      "--------------------------------------\n",
      "DONE\n",
      "Best Score 0.5577747510837205 Best Param {'criterion': 'gini', 'max_depth': 6, 'max_features': 0.5, 'min_samples_split': 7}\n",
      "AUC score on test set 0.5441558441558442\n",
      "Running 5fold stratified Cross Val \n",
      "Returning 5fold CV Scores\n",
      "--------------------------------------\n",
      "err:  0.611258012820513\n",
      "err:  0.4983974358974359\n",
      "err:  0.5511915269196822\n",
      "err:  0.49093851132686084\n",
      "err:  0.5484223300970874\n",
      "--------------------------------------\n",
      "DONE\n",
      "Average AUC SCORE: 0.5400415634123158\n",
      "AUC ScORE ON HOLDOUT SET: 0.5424675324675325\n"
     ]
    }
   ],
   "source": [
    "dt_hyper(X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CV and tuning XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtrain = DMatrix(X_train, label=y_train)\n",
    "dtest = DMatrix(X_test, label=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    # Parameters that we are going to tune.\n",
    "    'max_depth':6,\n",
    "    'min_child_weight': 1,\n",
    "    'eta':.3,\n",
    "    'subsample': 1,\n",
    "    'colsample_bytree': 1,\n",
    "    'eval_metric':'auc'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tTrain-auc:0.640094\tTest-auc:0.50013\n",
      "Multiple eval metrics have been passed: 'Test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until Test-auc hasn't improved in 100 rounds.\n",
      "[1]\tTrain-auc:0.665845\tTest-auc:0.484026\n",
      "[2]\tTrain-auc:0.669651\tTest-auc:0.493766\n",
      "[3]\tTrain-auc:0.687499\tTest-auc:0.483117\n",
      "[4]\tTrain-auc:0.701035\tTest-auc:0.515195\n",
      "[5]\tTrain-auc:0.716448\tTest-auc:0.519351\n",
      "[6]\tTrain-auc:0.726571\tTest-auc:0.495584\n",
      "[7]\tTrain-auc:0.729732\tTest-auc:0.500779\n",
      "[8]\tTrain-auc:0.74396\tTest-auc:0.502078\n",
      "[9]\tTrain-auc:0.744359\tTest-auc:0.501299\n",
      "[10]\tTrain-auc:0.770563\tTest-auc:0.505714\n",
      "[11]\tTrain-auc:0.783147\tTest-auc:0.497662\n",
      "[12]\tTrain-auc:0.796282\tTest-auc:0.472987\n",
      "[13]\tTrain-auc:0.804481\tTest-auc:0.451818\n",
      "[14]\tTrain-auc:0.808116\tTest-auc:0.460909\n",
      "[15]\tTrain-auc:0.813697\tTest-auc:0.45026\n",
      "[16]\tTrain-auc:0.827261\tTest-auc:0.459351\n",
      "[17]\tTrain-auc:0.827605\tTest-auc:0.457792\n",
      "[18]\tTrain-auc:0.827602\tTest-auc:0.453896\n",
      "[19]\tTrain-auc:0.828561\tTest-auc:0.452338\n",
      "[20]\tTrain-auc:0.832751\tTest-auc:0.44961\n",
      "[21]\tTrain-auc:0.832748\tTest-auc:0.453247\n",
      "[22]\tTrain-auc:0.836327\tTest-auc:0.46\n",
      "[23]\tTrain-auc:0.836646\tTest-auc:0.455065\n",
      "[24]\tTrain-auc:0.836816\tTest-auc:0.452727\n",
      "[25]\tTrain-auc:0.83806\tTest-auc:0.455065\n",
      "[26]\tTrain-auc:0.840789\tTest-auc:0.450909\n",
      "[27]\tTrain-auc:0.843935\tTest-auc:0.452468\n",
      "[28]\tTrain-auc:0.844905\tTest-auc:0.453506\n",
      "[29]\tTrain-auc:0.845083\tTest-auc:0.453506\n",
      "[30]\tTrain-auc:0.845153\tTest-auc:0.448831\n",
      "[31]\tTrain-auc:0.846441\tTest-auc:0.452468\n",
      "[32]\tTrain-auc:0.847084\tTest-auc:0.452727\n",
      "[33]\tTrain-auc:0.847857\tTest-auc:0.45013\n",
      "[34]\tTrain-auc:0.848317\tTest-auc:0.449351\n",
      "[35]\tTrain-auc:0.851268\tTest-auc:0.452727\n",
      "[36]\tTrain-auc:0.855944\tTest-auc:0.454805\n",
      "[37]\tTrain-auc:0.8586\tTest-auc:0.456623\n",
      "[38]\tTrain-auc:0.860656\tTest-auc:0.456364\n",
      "[39]\tTrain-auc:0.86065\tTest-auc:0.458182\n",
      "[40]\tTrain-auc:0.861992\tTest-auc:0.458961\n",
      "[41]\tTrain-auc:0.862777\tTest-auc:0.458701\n",
      "[42]\tTrain-auc:0.865027\tTest-auc:0.46026\n",
      "[43]\tTrain-auc:0.869214\tTest-auc:0.452468\n",
      "[44]\tTrain-auc:0.870501\tTest-auc:0.457143\n",
      "[45]\tTrain-auc:0.872004\tTest-auc:0.455844\n",
      "[46]\tTrain-auc:0.872236\tTest-auc:0.450649\n",
      "[47]\tTrain-auc:0.873571\tTest-auc:0.452727\n",
      "[48]\tTrain-auc:0.876713\tTest-auc:0.455065\n",
      "[49]\tTrain-auc:0.878866\tTest-auc:0.448312\n",
      "[50]\tTrain-auc:0.878978\tTest-auc:0.447013\n",
      "[51]\tTrain-auc:0.879171\tTest-auc:0.448052\n",
      "[52]\tTrain-auc:0.880382\tTest-auc:0.445195\n",
      "[53]\tTrain-auc:0.880549\tTest-auc:0.448831\n",
      "[54]\tTrain-auc:0.880774\tTest-auc:0.44961\n",
      "[55]\tTrain-auc:0.881064\tTest-auc:0.446753\n",
      "[56]\tTrain-auc:0.881196\tTest-auc:0.446494\n",
      "[57]\tTrain-auc:0.881598\tTest-auc:0.444156\n",
      "[58]\tTrain-auc:0.88284\tTest-auc:0.443896\n",
      "[59]\tTrain-auc:0.883033\tTest-auc:0.441558\n",
      "[60]\tTrain-auc:0.884095\tTest-auc:0.442078\n",
      "[61]\tTrain-auc:0.885048\tTest-auc:0.443117\n",
      "[62]\tTrain-auc:0.885399\tTest-auc:0.447792\n",
      "[63]\tTrain-auc:0.886059\tTest-auc:0.45013\n",
      "[64]\tTrain-auc:0.88629\tTest-auc:0.451169\n",
      "[65]\tTrain-auc:0.886718\tTest-auc:0.45039\n",
      "[66]\tTrain-auc:0.886889\tTest-auc:0.44961\n",
      "[67]\tTrain-auc:0.888135\tTest-auc:0.448312\n",
      "[68]\tTrain-auc:0.889293\tTest-auc:0.450909\n",
      "[69]\tTrain-auc:0.890362\tTest-auc:0.449091\n",
      "[70]\tTrain-auc:0.890497\tTest-auc:0.44961\n",
      "[71]\tTrain-auc:0.890329\tTest-auc:0.450649\n",
      "[72]\tTrain-auc:0.891626\tTest-auc:0.454805\n",
      "[73]\tTrain-auc:0.892035\tTest-auc:0.454805\n",
      "[74]\tTrain-auc:0.89247\tTest-auc:0.452208\n",
      "[75]\tTrain-auc:0.893078\tTest-auc:0.455325\n",
      "[76]\tTrain-auc:0.893078\tTest-auc:0.453766\n",
      "[77]\tTrain-auc:0.893406\tTest-auc:0.450649\n",
      "[78]\tTrain-auc:0.893545\tTest-auc:0.450649\n",
      "[79]\tTrain-auc:0.89366\tTest-auc:0.44987\n",
      "[80]\tTrain-auc:0.894163\tTest-auc:0.450909\n",
      "[81]\tTrain-auc:0.894488\tTest-auc:0.450909\n",
      "[82]\tTrain-auc:0.894919\tTest-auc:0.451429\n",
      "[83]\tTrain-auc:0.895308\tTest-auc:0.44987\n",
      "[84]\tTrain-auc:0.895485\tTest-auc:0.44987\n",
      "[85]\tTrain-auc:0.895431\tTest-auc:0.44961\n",
      "[86]\tTrain-auc:0.895852\tTest-auc:0.448571\n",
      "[87]\tTrain-auc:0.895987\tTest-auc:0.442857\n",
      "[88]\tTrain-auc:0.896171\tTest-auc:0.441558\n",
      "[89]\tTrain-auc:0.896187\tTest-auc:0.440519\n",
      "[90]\tTrain-auc:0.896329\tTest-auc:0.438182\n",
      "[91]\tTrain-auc:0.896679\tTest-auc:0.438182\n",
      "[92]\tTrain-auc:0.897069\tTest-auc:0.438442\n",
      "[93]\tTrain-auc:0.897056\tTest-auc:0.438701\n",
      "[94]\tTrain-auc:0.897111\tTest-auc:0.440519\n",
      "[95]\tTrain-auc:0.897516\tTest-auc:0.438182\n",
      "[96]\tTrain-auc:0.897719\tTest-auc:0.438182\n",
      "[97]\tTrain-auc:0.897983\tTest-auc:0.437143\n",
      "[98]\tTrain-auc:0.89816\tTest-auc:0.438961\n",
      "[99]\tTrain-auc:0.898343\tTest-auc:0.437143\n",
      "[100]\tTrain-auc:0.898678\tTest-auc:0.441558\n",
      "[101]\tTrain-auc:0.8989\tTest-auc:0.443636\n",
      "[102]\tTrain-auc:0.899035\tTest-auc:0.443377\n",
      "[103]\tTrain-auc:0.899312\tTest-auc:0.446494\n",
      "[104]\tTrain-auc:0.899283\tTest-auc:0.447273\n",
      "[105]\tTrain-auc:0.899373\tTest-auc:0.444935\n",
      "Stopping. Best iteration:\n",
      "[5]\tTrain-auc:0.716448\tTest-auc:0.519351\n",
      "\n"
     ]
    }
   ],
   "source": [
    "num_boost_round = 999\n",
    "model = train(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    evals=[(dtrain, \"Train\"),(dtest, \"Test\")],\n",
    "    early_stopping_rounds=100,\n",
    "    verbose_eval=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-auc-mean</th>\n",
       "      <th>train-auc-std</th>\n",
       "      <th>test-auc-mean</th>\n",
       "      <th>test-auc-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.654480</td>\n",
       "      <td>0.012569</td>\n",
       "      <td>0.503825</td>\n",
       "      <td>0.052362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.687613</td>\n",
       "      <td>0.010774</td>\n",
       "      <td>0.524366</td>\n",
       "      <td>0.047041</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train-auc-mean  train-auc-std  test-auc-mean  test-auc-std\n",
       "0        0.654480       0.012569       0.503825      0.052362\n",
       "1        0.687613       0.010774       0.524366      0.047041"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cv(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    seed=42,\n",
    "    nfold=5,\n",
    "    metrics={'auc'},\n",
    "    early_stopping_rounds=100\n",
    ")\n",
    "cv_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimize max depth and min child weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV with max_depth=9, min_child_weight=5\n",
      "\tAUC 0.49943279999999995 for 1 rounds\n",
      "CV with max_depth=9, min_child_weight=6\n",
      "\tAUC 0.49828920000000004 for 0 rounds\n",
      "CV with max_depth=9, min_child_weight=7\n",
      "\tAUC 0.5023626 for 1 rounds\n",
      "CV with max_depth=10, min_child_weight=5\n",
      "\tAUC 0.498765 for 0 rounds\n",
      "CV with max_depth=10, min_child_weight=6\n",
      "\tAUC 0.49969840000000004 for 0 rounds\n",
      "CV with max_depth=10, min_child_weight=7\n",
      "\tAUC 0.4964334 for 5 rounds\n",
      "CV with max_depth=11, min_child_weight=5\n",
      "\tAUC 0.49371020000000004 for 0 rounds\n",
      "CV with max_depth=11, min_child_weight=6\n",
      "\tAUC 0.5001106 for 2 rounds\n",
      "CV with max_depth=11, min_child_weight=7\n",
      "\tAUC 0.49666499999999997 for 1 rounds\n",
      "Best params: 9, 7, AUC: 0.5023626\n"
     ]
    }
   ],
   "source": [
    "gridsearch_params = [\n",
    "    (max_depth, min_child_weight)\n",
    "    for max_depth in range(9,12)\n",
    "    for min_child_weight in range(5,8)\n",
    "]\n",
    "\n",
    "# Define initial best params and MAE\n",
    "max_auc = 0\n",
    "best_params = None\n",
    "for max_depth, min_child_weight in gridsearch_params:\n",
    "    print(\"CV with max_depth={}, min_child_weight={}\".format(\n",
    "                             max_depth,\n",
    "                             min_child_weight))\n",
    "    # Update our parameters\n",
    "    params['max_depth'] = max_depth\n",
    "    params['min_child_weight'] = min_child_weight\n",
    "    # Run CV\n",
    "    cv_results = cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        metrics={'auc'},\n",
    "        early_stopping_rounds=100\n",
    "    )\n",
    "    # Update best MAE\n",
    "    mean_auc = cv_results['test-auc-mean'].max()\n",
    "    boost_rounds = cv_results['test-auc-mean'].argmin()\n",
    "    print(\"\\tAUC {} for {} rounds\".format(mean_auc, boost_rounds))\n",
    "    if mean_auc > max_auc:\n",
    "        max_auc = mean_auc\n",
    "        best_params = (max_depth,min_child_weight)\n",
    "print(\"Best params: {}, {}, AUC: {}\".format(best_params[0], best_params[1], max_auc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['max_depth'] = 9\n",
    "params['min_child_weight'] = 7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimize subsample and col sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV with subsample=1.0, colsample=1.0\n",
      "\tAUC 0.5023626 for 1 rounds\n",
      "CV with subsample=1.0, colsample=0.9\n",
      "\tAUC 0.4981288000000001 for 1 rounds\n",
      "CV with subsample=1.0, colsample=0.8\n",
      "\tAUC 0.4993666 for 0 rounds\n",
      "CV with subsample=1.0, colsample=0.7\n",
      "\tAUC 0.4993666 for 0 rounds\n",
      "CV with subsample=0.9, colsample=1.0\n",
      "\tAUC 0.49570119999999995 for 1 rounds\n",
      "CV with subsample=0.9, colsample=0.9\n",
      "\tAUC 0.4951502 for 0 rounds\n",
      "CV with subsample=0.9, colsample=0.8\n",
      "\tAUC 0.5103432 for 0 rounds\n",
      "CV with subsample=0.9, colsample=0.7\n",
      "\tAUC 0.5103432 for 0 rounds\n",
      "CV with subsample=0.8, colsample=1.0\n",
      "\tAUC 0.5138301999999999 for 0 rounds\n",
      "CV with subsample=0.8, colsample=0.9\n",
      "\tAUC 0.5154628 for 0 rounds\n",
      "CV with subsample=0.8, colsample=0.8\n",
      "\tAUC 0.5197068 for 0 rounds\n",
      "CV with subsample=0.8, colsample=0.7\n",
      "\tAUC 0.5197068 for 0 rounds\n",
      "CV with subsample=0.7, colsample=1.0\n",
      "\tAUC 0.5068968 for 0 rounds\n",
      "CV with subsample=0.7, colsample=0.9\n",
      "\tAUC 0.5143066000000001 for 1 rounds\n",
      "CV with subsample=0.7, colsample=0.8\n",
      "\tAUC 0.5162922 for 0 rounds\n",
      "CV with subsample=0.7, colsample=0.7\n",
      "\tAUC 0.5162922 for 0 rounds\n",
      "Best params: 0.8, 0.8, MAE: 0.5197068\n"
     ]
    }
   ],
   "source": [
    "gridsearch_params = [\n",
    "    (subsample, colsample)\n",
    "    for subsample in [i/10. for i in range(7,11)]\n",
    "    for colsample in [i/10. for i in range(7,11)]\n",
    "]\n",
    "\n",
    "max_auc = 0\n",
    "best_params = None\n",
    "# We start by the largest values and go down to the smallest\n",
    "for subsample, colsample in reversed(gridsearch_params):\n",
    "    print(\"CV with subsample={}, colsample={}\".format(\n",
    "                             subsample,\n",
    "                             colsample))\n",
    "    # We update our parameters\n",
    "    params['subsample'] = subsample\n",
    "    params['colsample_bytree'] = colsample\n",
    "    # Run CV\n",
    "    cv_results = cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        metrics={'auc'},\n",
    "        early_stopping_rounds=100\n",
    "    )\n",
    "    # Update best score\n",
    "    mean_auc = cv_results['test-auc-mean'].max()\n",
    "    boost_rounds = cv_results['test-auc-mean'].argmin()\n",
    "    print(\"\\tAUC {} for {} rounds\".format(mean_auc, boost_rounds))\n",
    "    if mean_auc > max_auc:\n",
    "        max_auc = mean_auc\n",
    "        best_params = (subsample,colsample)\n",
    "print(\"Best params: {}, {}, MAE: {}\".format(best_params[0], best_params[1], max_auc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['subsample'] = .8\n",
    "params['colsample_bytree'] = .8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimize eta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n",
      "CV with eta=0.5\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5132944 for 2 rounds\n",
      "\n",
      "CV with eta=0.5111111111111111\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5102372 for 2 rounds\n",
      "\n",
      "CV with eta=0.5222222222222223\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5072791999999999 for 2 rounds\n",
      "\n",
      "CV with eta=0.5333333333333333\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5102806 for 1 rounds\n",
      "\n",
      "CV with eta=0.5444444444444444\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5128744 for 1 rounds\n",
      "\n",
      "CV with eta=0.5555555555555556\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5133989999999999 for 1 rounds\n",
      "\n",
      "CV with eta=0.5666666666666667\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5142782 for 0 rounds\n",
      "\n",
      "CV with eta=0.5777777777777777\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5146092 for 0 rounds\n",
      "\n",
      "CV with eta=0.5888888888888889\n",
      "Wall time: 0 ns\n",
      "\tauc 0.5176824 for 0 rounds\n",
      "\n",
      "CV with eta=0.6\n",
      "Wall time: 0 ns\n",
      "\tauc 0.515546 for 0 rounds\n",
      "\n",
      "Best params: 0.5888888888888889, MAE: 0.5176824\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "# This can take some time…\n",
    "max_auc = 0\n",
    "best_params = None\n",
    "for eta in np.linspace(0.5, 0.6, 10):\n",
    "    print(\"CV with eta={}\".format(eta))\n",
    "    # We update our parameters\n",
    "    params['eta'] = eta\n",
    "    # Run and time CV\n",
    "    %time\n",
    "    cv_results = cv(\n",
    "            params,\n",
    "            dtrain,\n",
    "            num_boost_round=num_boost_round,\n",
    "            seed=42,\n",
    "            nfold=5,\n",
    "            metrics=['auc'],\n",
    "            early_stopping_rounds=100)\n",
    "    # Update best score\n",
    "    mean_auc = cv_results['test-auc-mean'].max()\n",
    "    boost_rounds = cv_results['test-auc-mean'].argmin()\n",
    "    print(\"\\tauc {} for {} rounds\\n\".format(mean_auc, boost_rounds))\n",
    "    if mean_auc > max_auc:\n",
    "        max_auc = mean_auc\n",
    "        best_params = eta\n",
    "print(\"Best params: {}, MAE: {}\".format(best_params, max_auc))\n",
    "params['eta'] = best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5fold Cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 9,\n",
       " 'min_child_weight': 7,\n",
       " 'eta': 0.5888888888888889,\n",
       " 'subsample': 0.8,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'eval_metric': 'auc'}"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-auc-mean</th>\n",
       "      <th>train-auc-std</th>\n",
       "      <th>test-auc-mean</th>\n",
       "      <th>test-auc-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.670154</td>\n",
       "      <td>0.022946</td>\n",
       "      <td>0.503866</td>\n",
       "      <td>0.042888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.695766</td>\n",
       "      <td>0.024052</td>\n",
       "      <td>0.505545</td>\n",
       "      <td>0.012185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.717339</td>\n",
       "      <td>0.020412</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.025563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.741373</td>\n",
       "      <td>0.016464</td>\n",
       "      <td>0.517682</td>\n",
       "      <td>0.026579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train-auc-mean  train-auc-std  test-auc-mean  test-auc-std\n",
       "0        0.670154       0.022946       0.503866      0.042888\n",
       "1        0.695766       0.024052       0.505545      0.012185\n",
       "2        0.717339       0.020412       0.510793      0.025563\n",
       "3        0.741373       0.016464       0.517682      0.026579"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cv(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    seed=42,\n",
    "    nfold=5,\n",
    "    metrics={'auc'},\n",
    "    early_stopping_rounds=100\n",
    ")\n",
    "cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Include class with no engagement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dell\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.03 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df['target'] = -1\n",
    "for i in range(df.shape[0]):\n",
    "    if (df.yes[i] == df.no[i]):\n",
    "        df['target'].loc[i] = 2\n",
    "    elif df.yes[i] == 1:\n",
    "        df['target'].loc[i] = 1\n",
    "    else:\n",
    "        df['target'].loc[i] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The distribution is greatly imbalanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    6834\n",
       "0     671\n",
       "1     572\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.target.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The goal of including users with no engagement is to have a multiclass classification of customers\n",
    "* Class 0 - customers that will answer no to the questionnaire\n",
    "* Class 1 - customers that will answer yes to the questionnaire\n",
    "* Class 2 - Customers that won't engage the questionnaire"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = df.drop(['auction_id', 'date', 'device_make', 'yes', 'no', 'engaged', 'target'], axis = 1)\n",
    "y = df.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment</th>\n",
       "      <th>hour</th>\n",
       "      <th>platform_os</th>\n",
       "      <th>dow</th>\n",
       "      <th>browser_Android</th>\n",
       "      <th>browser_Chrome</th>\n",
       "      <th>browser_Chrome Mobile</th>\n",
       "      <th>browser_Chrome Mobile WebView</th>\n",
       "      <th>browser_Chrome Mobile iOS</th>\n",
       "      <th>browser_Edge Mobile</th>\n",
       "      <th>...</th>\n",
       "      <th>browser_Puffin</th>\n",
       "      <th>browser_Samsung Internet</th>\n",
       "      <th>device_LG</th>\n",
       "      <th>device_LX1</th>\n",
       "      <th>device_Nokia</th>\n",
       "      <th>device_OnePlus</th>\n",
       "      <th>device_Pixel</th>\n",
       "      <th>device_Samsung</th>\n",
       "      <th>device_iPhone</th>\n",
       "      <th>device_others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   experiment  hour  platform_os  dow  browser_Android  browser_Chrome  \\\n",
       "0           1     8            6    4                0               0   \n",
       "1           1    10            6    1                0               0   \n",
       "2           1     2            6    6                0               0   \n",
       "3           0    15            6    4                0               0   \n",
       "4           0    15            6    4                0               0   \n",
       "\n",
       "   browser_Chrome Mobile  browser_Chrome Mobile WebView  \\\n",
       "0                      1                              0   \n",
       "1                      1                              0   \n",
       "2                      0                              1   \n",
       "3                      0                              0   \n",
       "4                      1                              0   \n",
       "\n",
       "   browser_Chrome Mobile iOS  browser_Edge Mobile  ...  browser_Puffin  \\\n",
       "0                          0                    0  ...               0   \n",
       "1                          0                    0  ...               0   \n",
       "2                          0                    0  ...               0   \n",
       "3                          0                    0  ...               0   \n",
       "4                          0                    0  ...               0   \n",
       "\n",
       "   browser_Samsung Internet  device_LG  device_LX1  device_Nokia  \\\n",
       "0                         0          0           0             0   \n",
       "1                         0          0           0             0   \n",
       "2                         0          0           0             0   \n",
       "3                         0          0           0             0   \n",
       "4                         0          0           0             0   \n",
       "\n",
       "   device_OnePlus  device_Pixel  device_Samsung  device_iPhone  device_others  \n",
       "0               0             0               0              0              1  \n",
       "1               0             0               0              0              1  \n",
       "2               0             0               0              0              1  \n",
       "3               0             0               1              0              0  \n",
       "4               0             0               0              0              1  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Turn categorical variables to dummies\n",
    "d = {'control': 0, 'exposed':1}\n",
    "X.experiment = X.experiment.replace(d)\n",
    "\n",
    "X = pd.get_dummies(X) ; X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .1, random_state = 10) #split dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(colsample_bylevel=0.8, objective='multi:softprob',\n",
       "              random_state=42)"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb = XGBClassifier(random_state=42, colsample_bylevel=.8, subsample=1)\n",
    "xgb.fit(X_train, y_train, eval_set= [(X_train, y_train), (X_test, y_test)], early_stopping_rounds = 100, eval_metric = 'mlogloss', verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_1 = xgb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[745,   0],\n",
       "        [ 63,   0]],\n",
       "\n",
       "       [[744,   0],\n",
       "        [ 64,   0]],\n",
       "\n",
       "       [[  0, 127],\n",
       "        [  0, 681]]], dtype=int64)"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multilabel_confusion_matrix(y_test, pred_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    681\n",
       "1     64\n",
       "0     63\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "comparing the disribution of the test set and the confusion matrix, we find out that the model overfitted the train set and only predicted class 2 which we can confirm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2], dtype=int64)"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(pred_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This can be due to the distribution of the target class. We aim to fix this and slightly improve our model by oversampling the minority classes\n",
    "\n",
    "> By Oversampling, synthethic data will be added to increase the number of the minority classes (class o and class 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTE(k_neighbors=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_m, y_m = smote.fit_sample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(colsample_bylevel=0.8, objective='multi:softprob',\n",
       "              random_state=42)"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_m = XGBClassifier(random_state=42, colsample_bylevel=.8, subsample=1)\n",
    "xgb_m.fit(X_m, y_m, eval_set= [(X_train, y_train), (X_test, y_test)], early_stopping_rounds = 100, eval_metric = 'mlogloss', verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_m = xgb_m.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[473, 272],\n",
       "        [ 48,  15]],\n",
       "\n",
       "       [[558, 186],\n",
       "        [ 49,  15]],\n",
       "\n",
       "       [[ 75,  52],\n",
       "        [413, 268]]], dtype=int64)"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multilabel_confusion_matrix(y_test, pred_m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We notice a slight improvement in our model, thou some class 2 points were misclassified, the model overfitting was reduced. More improvement can be made by optimizing the model parameters and the number of Neighbors but we stop here for now\n",
    "> Bias - variance tradeoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip freeze > requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}