Extraction_Et_Analyse_Des_Donnees.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"color:blue; text-align:center\">Extract & Merge tables</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Import de librairies pour l'extraction et la transformation\n",
    "import gzip\n",
    "import shutil\n",
    "import os\n",
    "import time\n",
    "import pathlib\n",
    "import pickle\n",
    "import feather\n",
    "import pandas as pd \n",
    "import multiprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_files(path, pattern):\n",
    "    \"\"\"\n",
    "    Return the List of filenames in the <path> directory that respect\n",
    "    the given <pattern> (i.e. extension)\n",
    "    \"\"\"\n",
    "    \n",
    "    list_files=[]\n",
    "    currentDirectory = pathlib.Path(path)\n",
    "    for currentFile in currentDirectory.glob(pattern):\n",
    "        list_files.append(str(currentFile))\n",
    "    return list_files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_4Dec(path_ins, path_out, date=b\"2017-12-04\"):\n",
    "    \n",
    "    \"\"\" read all the files in the <path_ins> list\n",
    "    then Keep and write only data for the given <date> in <path_out>\"\"\"\n",
    "    \n",
    "    with open(path_out, 'wb') as f_out:\n",
    "        for path in path_ins:\n",
    "            with gzip.open(path, 'rb') as f_in:\n",
    "                for line in f_in:\n",
    "                    if date in line: f_out.write(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style=\"color:blue; text-align:center\">Processing \"display\" tables</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/display_12\"\n",
    "path_out_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/display_12.csv\"\n",
    "list_affichage_4dec = get_files(path_4dec, \"*.gz\")\n",
    "extract_4Dec(path_ins=list_affichage_4dec, path_out=path_out_4dec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style=\"color:blue; text-align:center\">Processing \"requests\" tables</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/request_12\"\n",
    "path_out_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/request_12.csv\"\n",
    "list_affichage_4dec = get_files(path_4dec, \"*.gz\")\n",
    "extract_4Dec(path_ins=list_affichage_4dec, path_out=path_out_4dec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style=\"color:blue; text-align:center\">Processing \"clics_12\" tables</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/clics_12\"\n",
    "path_out_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/clics_12.csv\"\n",
    "list_affichage_4dec = get_files(path_4dec, \"*.gz\")\n",
    "with open(path_out_4dec, 'wb') as f_out:\n",
    "    for path in list_affichage_4dec:\n",
    "        with gzip.open(path, 'rb') as f_in:\n",
    "            shutil.copyfileobj(f_in, f_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style=\"color:blue; text-align:center\">Processing \"affichage\" tables</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/affichages_12\"\n",
    "path_out_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/affichages.csv\"\n",
    "list_affichage_4dec = get_files(path_4dec, \"*.gz\")\n",
    "extract_4Dec(path_ins=list_affichage_4dec, path_out=path_out_4dec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 style=\"color:blue; text-align:center\">Merging the 4 tables in one pandas dataFrame</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load affichage table in pandas dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "cols_aff=['date_aff', 'uniq_id', 'is_mobile', 'uniq_id_widget', 'pos_widget', 'widget_width', 'widget_height']\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_aff={'is_mobile':'category', 'pos_widget':'category', \n",
    "               'widget_width':'uint16', 'widget_height':'uint16'}\n",
    "##Reading options\n",
    "options_aff = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_aff,\n",
    "            'dtype':col_types_aff, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_aff'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "path_aff= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/affichages.csv\"\n",
    "\n",
    "df_Affichage = pd.read_csv(path_aff, **options_aff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_aff</th>\n",
       "      <th>uniq_id</th>\n",
       "      <th>is_mobile</th>\n",
       "      <th>uniq_id_widget</th>\n",
       "      <th>pos_widget</th>\n",
       "      <th>widget_width</th>\n",
       "      <th>widget_height</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-12-04 00:00:01</td>\n",
       "      <td>pjsle-1921681430-5a2481f03587e1.82984907</td>\n",
       "      <td>false</td>\n",
       "      <td>pjsle-1921681430-5a2481f03a52b2.96551034</td>\n",
       "      <td>pjsleads</td>\n",
       "      <td>780</td>\n",
       "      <td>419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-12-04 00:00:07</td>\n",
       "      <td>pjsle-19216814130-5a2481f680f1a3.75029244</td>\n",
       "      <td>true</td>\n",
       "      <td>pjsle-19216814130-5a2481f684db84.62324602</td>\n",
       "      <td>pjsleads</td>\n",
       "      <td>356</td>\n",
       "      <td>472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-12-04 00:00:09</td>\n",
       "      <td>pjsle-19216814130-5a2481f73fbd54.22395195</td>\n",
       "      <td>false</td>\n",
       "      <td>pjsle-19216814130-5a2481f73fe579.95228700</td>\n",
       "      <td>pjsleads</td>\n",
       "      <td>622</td>\n",
       "      <td>757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-12-04 00:00:10</td>\n",
       "      <td>pjsle-19216814134-5a2481f73434c5.39442554</td>\n",
       "      <td>false</td>\n",
       "      <td>pjsle-19216814134-5a2481f7347681.22320918</td>\n",
       "      <td>pjsleads</td>\n",
       "      <td>835</td>\n",
       "      <td>410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-12-04 00:00:10</td>\n",
       "      <td>pjsle-1921681415-5a2481f8f18e06.98170193</td>\n",
       "      <td>false</td>\n",
       "      <td>pjsle-1921681415-5a2481f8f33084.60941479</td>\n",
       "      <td>pjsleads</td>\n",
       "      <td>1000</td>\n",
       "      <td>648</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             date_aff                                    uniq_id is_mobile  \\\n",
       "0 2017-12-04 00:00:01   pjsle-1921681430-5a2481f03587e1.82984907     false   \n",
       "1 2017-12-04 00:00:07  pjsle-19216814130-5a2481f680f1a3.75029244      true   \n",
       "2 2017-12-04 00:00:09  pjsle-19216814130-5a2481f73fbd54.22395195     false   \n",
       "3 2017-12-04 00:00:10  pjsle-19216814134-5a2481f73434c5.39442554     false   \n",
       "4 2017-12-04 00:00:10   pjsle-1921681415-5a2481f8f18e06.98170193     false   \n",
       "\n",
       "                              uniq_id_widget pos_widget  widget_width  \\\n",
       "0   pjsle-1921681430-5a2481f03a52b2.96551034   pjsleads           780   \n",
       "1  pjsle-19216814130-5a2481f684db84.62324602   pjsleads           356   \n",
       "2  pjsle-19216814130-5a2481f73fe579.95228700   pjsleads           622   \n",
       "3  pjsle-19216814134-5a2481f7347681.22320918   pjsleads           835   \n",
       "4   pjsle-1921681415-5a2481f8f33084.60941479   pjsleads          1000   \n",
       "\n",
       "   widget_height  \n",
       "0            419  \n",
       "1            472  \n",
       "2            757  \n",
       "3            410  \n",
       "4            648  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_Affichage.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load display table in pandas dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "cols_disp=['date_disp', 'uniq_id', 'pos', 'merchant_id', 'an9', \"uniq_id_widget\"]\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_disp={'pos':'category', 'merchant_id':'uint32'}\n",
    "\n",
    "##Reading options\n",
    "options_disp = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_disp,\n",
    "            'dtype':col_types_disp, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_disp'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "path_disp= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/display_12.csv\"\n",
    "\n",
    "df_Disp = pd.read_csv(path_disp, **options_disp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_disp</th>\n",
       "      <th>uniq_id</th>\n",
       "      <th>pos</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>an9</th>\n",
       "      <th>uniq_id_widget</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-12-04 00:00:01</td>\n",
       "      <td>pjsle-1921681415-5a2481f1368e13.99248011</td>\n",
       "      <td>1</td>\n",
       "      <td>359499</td>\n",
       "      <td>58251900;58303600;58155100</td>\n",
       "      <td>pjsle-1921681415-5a2481f1390e07.85048495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-12-04 00:00:01</td>\n",
       "      <td>pjsle-1921681415-5a2481f1368e13.99248011</td>\n",
       "      <td>2</td>\n",
       "      <td>57063793</td>\n",
       "      <td>58251900;58203000;58303600;58155100</td>\n",
       "      <td>pjsle-1921681415-5a2481f1390e07.85048495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-12-04 00:00:01</td>\n",
       "      <td>pjsle-1921681415-5a2481f1368e13.99248011</td>\n",
       "      <td>3</td>\n",
       "      <td>5795832</td>\n",
       "      <td>58251900;58203000;58303600;58155100</td>\n",
       "      <td>pjsle-1921681415-5a2481f1390e07.85048495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-12-04 00:00:02</td>\n",
       "      <td>pjsle-1921681415-5a2481f2348313.04486093</td>\n",
       "      <td>1</td>\n",
       "      <td>5981923</td>\n",
       "      <td>56051400</td>\n",
       "      <td>pjsle-1921681415-5a2481f2379246.78851340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-12-04 00:00:02</td>\n",
       "      <td>pjsle-1921681415-5a2481f2348313.04486093</td>\n",
       "      <td>2</td>\n",
       "      <td>3926636</td>\n",
       "      <td>56051400</td>\n",
       "      <td>pjsle-1921681415-5a2481f2379246.78851340</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            date_disp                                   uniq_id pos  \\\n",
       "0 2017-12-04 00:00:01  pjsle-1921681415-5a2481f1368e13.99248011   1   \n",
       "1 2017-12-04 00:00:01  pjsle-1921681415-5a2481f1368e13.99248011   2   \n",
       "2 2017-12-04 00:00:01  pjsle-1921681415-5a2481f1368e13.99248011   3   \n",
       "3 2017-12-04 00:00:02  pjsle-1921681415-5a2481f2348313.04486093   1   \n",
       "4 2017-12-04 00:00:02  pjsle-1921681415-5a2481f2348313.04486093   2   \n",
       "\n",
       "   merchant_id                                  an9  \\\n",
       "0       359499           58251900;58303600;58155100   \n",
       "1     57063793  58251900;58203000;58303600;58155100   \n",
       "2      5795832  58251900;58203000;58303600;58155100   \n",
       "3      5981923                             56051400   \n",
       "4      3926636                             56051400   \n",
       "\n",
       "                             uniq_id_widget  \n",
       "0  pjsle-1921681415-5a2481f1390e07.85048495  \n",
       "1  pjsle-1921681415-5a2481f1390e07.85048495  \n",
       "2  pjsle-1921681415-5a2481f1390e07.85048495  \n",
       "3  pjsle-1921681415-5a2481f2379246.78851340  \n",
       "4  pjsle-1921681415-5a2481f2379246.78851340  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_Disp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load request table in pandas dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "cols_req=['date_req', 'uniq_id', 'at_partner', 'at_pagetype', 'uniq_id_widget', 'pos_widget']\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_req={'at_pagetype':'category', 'pos_widget':'category', 'at_partner':'category'}\n",
    "\n",
    "##Reading options\n",
    "options_req = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_req,\n",
    "            'dtype':col_types_req, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_req'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "path_req= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/request_12.csv\"\n",
    "\n",
    "df_Req = pd.read_csv(path_req, **options_req)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_req</th>\n",
       "      <th>uniq_id</th>\n",
       "      <th>at_partner</th>\n",
       "      <th>at_pagetype</th>\n",
       "      <th>uniq_id_widget</th>\n",
       "      <th>pos_widget</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-12-04 00:00:00</td>\n",
       "      <td>pjsle-19216814134-5a2481f0261a99.37202975</td>\n",
       "      <td>Avisdedeces</td>\n",
       "      <td>LR</td>\n",
       "      <td>pjsle-19216814134-5a2481f0289c77.79748604</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-12-04 00:00:00</td>\n",
       "      <td>pjsle-19216814134-5a2481f069d2c2.90766901</td>\n",
       "      <td>Justacote</td>\n",
       "      <td>FD</td>\n",
       "      <td>pjsle-19216814134-5a2481f069f308.49543759</td>\n",
       "      <td>pj_middle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-12-04 00:00:00</td>\n",
       "      <td>pjsle-19216814134-5a2481f069d2c2.90766901</td>\n",
       "      <td>Justacote</td>\n",
       "      <td>FD</td>\n",
       "      <td>pjsle-19216814134-5a2481f06c7e99.82505453</td>\n",
       "      <td>pj_right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-12-04 00:00:01</td>\n",
       "      <td>pjsle-19216814134-5a2481f118d804.96015430</td>\n",
       "      <td>PetitFuteUp</td>\n",
       "      <td>LR</td>\n",
       "      <td>pjsle-19216814134-5a2481f11917a7.82571839</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-12-04 00:00:02</td>\n",
       "      <td>pjsle-19216814134-5a2481f1e4f1d6.00644870</td>\n",
       "      <td>Oubruncher</td>\n",
       "      <td>LR</td>\n",
       "      <td>pjsle-19216814134-5a2481f1e8bd41.31767876</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             date_req                                    uniq_id   at_partner  \\\n",
       "0 2017-12-04 00:00:00  pjsle-19216814134-5a2481f0261a99.37202975  Avisdedeces   \n",
       "1 2017-12-04 00:00:00  pjsle-19216814134-5a2481f069d2c2.90766901    Justacote   \n",
       "2 2017-12-04 00:00:00  pjsle-19216814134-5a2481f069d2c2.90766901    Justacote   \n",
       "3 2017-12-04 00:00:01  pjsle-19216814134-5a2481f118d804.96015430  PetitFuteUp   \n",
       "4 2017-12-04 00:00:02  pjsle-19216814134-5a2481f1e4f1d6.00644870   Oubruncher   \n",
       "\n",
       "  at_pagetype                             uniq_id_widget pos_widget  \n",
       "0          LR  pjsle-19216814134-5a2481f0289c77.79748604   pjsleads  \n",
       "1          FD  pjsle-19216814134-5a2481f069f308.49543759  pj_middle  \n",
       "2          FD  pjsle-19216814134-5a2481f06c7e99.82505453   pj_right  \n",
       "3          LR  pjsle-19216814134-5a2481f11917a7.82571839   pjsleads  \n",
       "4          LR  pjsle-19216814134-5a2481f1e8bd41.31767876   pjsleads  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_Req.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load click table in pandas dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "###Column names\n",
    "cols_click=['uniq_id', 'pos', 'merchant_id', 'top_annonceur', 'event_target', 'uniq_id_widget', 'pos_widget']\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_click={'pos':'category', 'merchant_id':'uint32', 'top_annonceur':'category', \n",
    "                    'event_target':'category', 'pos_widget':'category'}\n",
    "\n",
    "\n",
    "##Reading options\n",
    "options_click = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_click,\n",
    "            'dtype':col_types_click, \n",
    "            'index_col':False\n",
    "        }\n",
    "\n",
    "path_click= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/clics_12.csv\"\n",
    "\n",
    "df_click = pd.read_csv(path_click, **options_click)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uniq_id</th>\n",
       "      <th>pos</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>top_annonceur</th>\n",
       "      <th>event_target</th>\n",
       "      <th>uniq_id_widget</th>\n",
       "      <th>pos_widget</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pjsle-19216814134-5a27250cd3ca37.98709992</td>\n",
       "      <td>1</td>\n",
       "      <td>7860052</td>\n",
       "      <td>\\N</td>\n",
       "      <td>click2call</td>\n",
       "      <td>pjsle-19216814134-5a27250cd679e8.83105518</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pjsle-1921681415-5a27244c3a2864.10515859</td>\n",
       "      <td>2</td>\n",
       "      <td>1629382</td>\n",
       "      <td>1</td>\n",
       "      <td>Adresse</td>\n",
       "      <td>pjsle-1921681415-5a27244c3a52d8.55501948</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pjsle-1921681415-5a2725064cc307.92537863</td>\n",
       "      <td>2</td>\n",
       "      <td>51966694</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Adresse</td>\n",
       "      <td>pjsle-1921681415-5a2725064cd226.35052992</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pjsle-19216814134-5a2725e18fbf61.95799090</td>\n",
       "      <td>2</td>\n",
       "      <td>5473398</td>\n",
       "      <td>\\N</td>\n",
       "      <td>Adresse</td>\n",
       "      <td>pjsle-19216814134-5a2725e1900d61.30525464</td>\n",
       "      <td>pjsleads</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pjsle-1921681430-5a2725e347be41.77778799</td>\n",
       "      <td>4</td>\n",
       "      <td>1945929</td>\n",
       "      <td>1</td>\n",
       "      <td>Adresse</td>\n",
       "      <td>pjsle-1921681430-5a2725e34a1fa4.87381388</td>\n",
       "      <td>pj_middle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     uniq_id pos  merchant_id top_annonceur  \\\n",
       "0  pjsle-19216814134-5a27250cd3ca37.98709992   1      7860052            \\N   \n",
       "1   pjsle-1921681415-5a27244c3a2864.10515859   2      1629382             1   \n",
       "2   pjsle-1921681415-5a2725064cc307.92537863   2     51966694            \\N   \n",
       "3  pjsle-19216814134-5a2725e18fbf61.95799090   2      5473398            \\N   \n",
       "4   pjsle-1921681430-5a2725e347be41.77778799   4      1945929             1   \n",
       "\n",
       "  event_target                             uniq_id_widget pos_widget  \n",
       "0   click2call  pjsle-19216814134-5a27250cd679e8.83105518   pjsleads  \n",
       "1      Adresse   pjsle-1921681415-5a27244c3a52d8.55501948   pjsleads  \n",
       "2      Adresse   pjsle-1921681415-5a2725064cd226.35052992   pjsleads  \n",
       "3      Adresse  pjsle-19216814134-5a2725e1900d61.30525464   pjsleads  \n",
       "4      Adresse   pjsle-1921681430-5a2725e34a1fa4.87381388  pj_middle  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_click.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load ref_partenaire table in pandas dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "#cols_ref_partenaire=['at_partner', 'partenaire', 'categorie']\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_ref_partenaire={'at_partner':'category', 'partenaire':'category', 'categorie':'category'}\n",
    "\n",
    "\n",
    "##Reading options\n",
    "options_ref_partenaire = {\n",
    "            'sep':';',\n",
    "            'header':0,\n",
    "            'encoding' :'ISO-8859-1', \n",
    "            'index_col':False\n",
    "        }\n",
    "\n",
    "path_ref_partenaire= \"/home/alka/Documents/PROJET_FIN_ETUDES/ref_partenaire_20171201.csv\"\n",
    "\n",
    "df_ref_partenaire = pd.read_csv(path_ref_partenaire, **options_ref_partenaire)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(295, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ref_partenaire.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>at_partner</th>\n",
       "      <th>partenaire</th>\n",
       "      <th>categorie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>118000</td>\n",
       "      <td>118000</td>\n",
       "      <td>Annuaires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>123cuisine</td>\n",
       "      <td>Utopia Web</td>\n",
       "      <td>Annuaires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>123pages</td>\n",
       "      <td>123pages</td>\n",
       "      <td>Annuaires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3evie</td>\n",
       "      <td>Conapt</td>\n",
       "      <td>Annuaires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AZ-FRANCE</td>\n",
       "      <td>AZ-France</td>\n",
       "      <td>Annuaires</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   at_partner  partenaire  categorie\n",
       "0      118000      118000  Annuaires\n",
       "1  123cuisine  Utopia Web  Annuaires\n",
       "2    123pages    123pages  Annuaires\n",
       "3       3evie      Conapt  Annuaires\n",
       "4   AZ-FRANCE   AZ-France  Annuaires"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ref_partenaire = df_ref_partenaire[df_ref_partenaire[\"categorie\"]==\"Annuaires\"]\n",
    "df_ref_partenaire.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Merging all the dataframes**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Merge between request & ref_partenaire tables\n",
    "df_dec4 = pd.merge(df_ref_partenaire, df_Req, how=\"inner\", left_on=\"at_partner\", right_on=\"at_partner\")\n",
    "\n",
    "##Merge : add click table\n",
    "df_dec4 = pd.merge(df_dec4, df_click, how=\"left\",\n",
    "                   left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                   right_on=[\"uniq_id\", \"uniq_id_widget\"])\n",
    "\n",
    "##Merge : add affichage table\n",
    "df_dec4 = pd.merge(df_dec4, df_Affichage, how=\"left\",\n",
    "                   left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                   right_on=[\"uniq_id\", \"uniq_id_widget\"])\n",
    "\n",
    "##Merge : add display table\n",
    "df_dec4 = pd.merge(df_dec4, df_Disp, how=\"left\",\n",
    "                   left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                   right_on=[\"uniq_id\", \"uniq_id_widget\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  at_partner partenaire  categorie            date_req  \\\n",
      "0     118000     118000  Annuaires 2017-12-04 00:00:11   \n",
      "1     118000     118000  Annuaires 2017-12-04 00:00:11   \n",
      "2     118000     118000  Annuaires 2017-12-04 00:00:24   \n",
      "3     118000     118000  Annuaires 2017-12-04 00:01:22   \n",
      "4     118000     118000  Annuaires 2017-12-04 00:01:29   \n",
      "\n",
      "                                     uniq_id at_pagetype  \\\n",
      "0  pjsle-19216814134-5a2481fae0bce4.11528640          FD   \n",
      "1  pjsle-19216814134-5a2481fae0bce4.11528640          FD   \n",
      "2  pjsle-19216814134-5a248208817cd8.31345439          FD   \n",
      "3  pjsle-19216814134-5a2482421eda44.12100670          FD   \n",
      "4  pjsle-19216814134-5a248249bfa352.91478634          FD   \n",
      "\n",
      "                              uniq_id_widget pos_widget_x pos_x  \\\n",
      "0  pjsle-19216814134-5a2481fae2ddd9.20270842     pjsleads   NaN   \n",
      "1  pjsle-19216814134-5a2481fae2ddd9.20270842     pjsleads   NaN   \n",
      "2  pjsle-19216814134-5a24820881c865.59860143     pjsleads   NaN   \n",
      "3  pjsle-19216814134-5a24824221ba81.19748565     pjsleads   NaN   \n",
      "4  pjsle-19216814134-5a248249bfd260.22632257     pjsleads   NaN   \n",
      "\n",
      "   merchant_id_x        ...         pos_widget_y            date_aff  \\\n",
      "0            NaN        ...                  NaN 2017-12-04 00:00:13   \n",
      "1            NaN        ...                  NaN 2017-12-04 00:00:13   \n",
      "2            NaN        ...                  NaN 2017-12-04 00:00:28   \n",
      "3            NaN        ...                  NaN                 NaT   \n",
      "4            NaN        ...                  NaN 2017-12-04 00:01:30   \n",
      "\n",
      "  is_mobile pos_widget widget_width widget_height           date_disp  pos_y  \\\n",
      "0     false   pjsleads        430.0         326.0 2017-12-04 00:00:11      1   \n",
      "1     false   pjsleads        430.0         326.0 2017-12-04 00:00:11      2   \n",
      "2      true   pjsleads        265.0         174.0 2017-12-04 00:00:24      1   \n",
      "3       NaN        NaN          NaN           NaN                 NaT    NaN   \n",
      "4      true   pjsleads        289.0         333.0 2017-12-04 00:01:29      1   \n",
      "\n",
      "  merchant_id_y                an9  \n",
      "0    54898299.0           58451500  \n",
      "1      527466.0  58451500;68050700  \n",
      "2    56191642.0           72257700  \n",
      "3           NaN                NaN  \n",
      "4    51917465.0           72259800  \n",
      "\n",
      "[5 rows x 22 columns]\n",
      "             at_partner   partenaire  categorie            date_req  \\\n",
      "1951637  animaux_online  Buena-Media  Annuaires 2017-12-04 13:20:35   \n",
      "1951638  animaux_online  Buena-Media  Annuaires 2017-12-04 13:20:35   \n",
      "1951639  animaux_online  Buena-Media  Annuaires 2017-12-04 13:20:35   \n",
      "1951640  animaux_online  Buena-Media  Annuaires 2017-12-04 13:20:35   \n",
      "1951641  animaux_online  Buena-Media  Annuaires 2017-12-04 13:20:35   \n",
      "\n",
      "                                          uniq_id at_pagetype  \\\n",
      "1951637  pjsle-1921681415-5a253d934194d4.53258329          LR   \n",
      "1951638  pjsle-1921681415-5a253d934194d4.53258329          LR   \n",
      "1951639  pjsle-1921681415-5a253d934194d4.53258329          LR   \n",
      "1951640  pjsle-1921681415-5a253d934194d4.53258329          LR   \n",
      "1951641  pjsle-1921681415-5a253d934194d4.53258329          LR   \n",
      "\n",
      "                                   uniq_id_widget pos_widget_x pos_x  \\\n",
      "1951637  pjsle-1921681415-5a253d9343b779.80624643       PJHAUT   NaN   \n",
      "1951638  pjsle-1921681415-5a253d9343b779.80624643       PJHAUT   NaN   \n",
      "1951639  pjsle-1921681415-5a253d934c92e9.69669795        PJBAS   NaN   \n",
      "1951640  pjsle-1921681415-5a253d934c92e9.69669795        PJBAS   NaN   \n",
      "1951641  pjsle-1921681415-5a253d934c92e9.69669795        PJBAS   NaN   \n",
      "\n",
      "         merchant_id_x    ...    pos_widget_y            date_aff is_mobile  \\\n",
      "1951637            NaN    ...             NaN 2017-12-04 13:20:35     false   \n",
      "1951638            NaN    ...             NaN 2017-12-04 13:20:35     false   \n",
      "1951639            NaN    ...             NaN 2017-12-04 13:20:35     false   \n",
      "1951640            NaN    ...             NaN 2017-12-04 13:20:35     false   \n",
      "1951641            NaN    ...             NaN 2017-12-04 13:20:35     false   \n",
      "\n",
      "        pos_widget widget_width widget_height           date_disp  pos_y  \\\n",
      "1951637     PJHAUT        660.0         409.0 2017-12-04 13:20:35      2   \n",
      "1951638     PJHAUT        660.0         409.0 2017-12-04 13:20:35      3   \n",
      "1951639      PJBAS        660.0         409.0 2017-12-04 13:20:35      1   \n",
      "1951640      PJBAS        660.0         409.0 2017-12-04 13:20:35      2   \n",
      "1951641      PJBAS        660.0         409.0 2017-12-04 13:20:35      3   \n",
      "\n",
      "        merchant_id_y       an9  \n",
      "1951637      216615.0  72101100  \n",
      "1951638    51258320.0  72101100  \n",
      "1951639    53403881.0  72101100  \n",
      "1951640     8675220.0  72101100  \n",
      "1951641    50103020.0  72101100  \n",
      "\n",
      "[5 rows x 22 columns]\n",
      "(1951642, 22)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1951642 entries, 0 to 1951641\n",
      "Data columns (total 22 columns):\n",
      "at_partner        object\n",
      "partenaire        object\n",
      "categorie         object\n",
      "date_req          datetime64[ns]\n",
      "uniq_id           object\n",
      "at_pagetype       category\n",
      "uniq_id_widget    object\n",
      "pos_widget_x      category\n",
      "pos_x             category\n",
      "merchant_id_x     float64\n",
      "top_annonceur     category\n",
      "event_target      category\n",
      "pos_widget_y      category\n",
      "date_aff          datetime64[ns]\n",
      "is_mobile         category\n",
      "pos_widget        category\n",
      "widget_width      float64\n",
      "widget_height     float64\n",
      "date_disp         datetime64[ns]\n",
      "pos_y             category\n",
      "merchant_id_y     float64\n",
      "an9               object\n",
      "dtypes: category(9), datetime64[ns](3), float64(4), object(6)\n",
      "memory usage: 1.1 GB\n",
      "None\n",
      "at_partner              0\n",
      "partenaire              0\n",
      "categorie               0\n",
      "date_req                0\n",
      "uniq_id                 0\n",
      "at_pagetype             0\n",
      "uniq_id_widget          0\n",
      "pos_widget_x            0\n",
      "pos_x             1859572\n",
      "merchant_id_x     1859572\n",
      "top_annonceur     1859572\n",
      "event_target      1859572\n",
      "pos_widget_y      1859572\n",
      "date_aff           184406\n",
      "is_mobile          184406\n",
      "pos_widget         184406\n",
      "widget_width       184406\n",
      "widget_height      184406\n",
      "date_disp           56255\n",
      "pos_y               56255\n",
      "merchant_id_y       56255\n",
      "an9                 56982\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df_dec4.head())\n",
    "print(df_dec4.tail())\n",
    "print(df_dec4.shape )\n",
    "print(df_dec4.info(memory_usage=\"deep\"))\n",
    "print(df_dec4.isnull().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Save the data Frame\n",
    "with gzip.open(\"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/data_dec4.pklz\",\"w\") as f:\n",
    "    pickle.dump(file=f, obj=df_dec4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##To load the dataFrame\n",
    "\n",
    "with gzip.open(\"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/data_dec4.pklz\", \"r\") as f:\n",
    "    df = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data for 5, 6, 7, 8, 9, 10 december 2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "###dates to extract data for\n",
    "dates = [b\"2017-12-05\", b\"2017-12-06\", b\"2017-12-07\", b\"2017-12-08\", b\"2017-12-09\", b\"2017-12-10\"]\n",
    "##Tables to extract\n",
    "tables = [\"display_12\", \"request_12\", \"affichages_12\"]\n",
    "\n",
    "for date in dates:\n",
    "    for table in tables:\n",
    "        path_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/\" + table\n",
    "        path_out_4dec = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/\" + table + str(date)[2:-1] + \".csv\"\n",
    "        list_affichage_4dec = get_files(path_4dec, \"*.gz\")\n",
    "        extract_4Dec(path_ins=list_affichage_4dec, path_out=path_out_4dec, date=date)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merging dataframes for all the dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Column names\n",
    "cols_aff=['date_aff', 'uniq_id', 'is_mobile', 'uniq_id_widget', 'pos_widget', 'widget_width', 'widget_height']\n",
    "cols_disp=['date_disp', 'uniq_id', 'pos', 'merchant_id', 'an9', \"uniq_id_widget\"]\n",
    "cols_req=['date_req', 'uniq_id', 'at_partner', 'at_pagetype', 'uniq_id_widget', 'pos_widget']\n",
    "\n",
    "##Specify some column types to reduce memory overflow \n",
    "col_types_aff={'is_mobile':'category', 'pos_widget':'category', \n",
    "               'widget_width':'uint16', 'widget_height':'uint16'}\n",
    "col_types_disp={'pos':'category', 'merchant_id':'uint32'}\n",
    "col_types_req={'at_pagetype':'category', 'pos_widget':'category', 'at_partner':'category'}\n",
    "\n",
    "\n",
    "##Reading options\n",
    "options_aff = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_aff,\n",
    "            'dtype':col_types_aff, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_aff'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "\n",
    "options_disp = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_disp,\n",
    "            'dtype':col_types_disp, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_disp'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "\n",
    "options_req = {\n",
    "            'sep':',',\n",
    "            'header':None,\n",
    "            'names':cols_req,\n",
    "            'dtype':col_types_req, \n",
    "            'index_col':False,\n",
    "            'parse_dates':['date_req'],\n",
    "            'infer_datetime_format':True}\n",
    "\n",
    "\n",
    "dates = [b\"2017-12-05\", b\"2017-12-06\", b\"2017-12-07\", b\"2017-12-08\", b\"2017-12-09\", b\"2017-12-10\"]\n",
    "\n",
    "for date in dates:\n",
    "    \n",
    "    path_aff= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/affichages_12\" + str(date)[2:-1] + \".csv\"\n",
    "    path_req= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/request_12\" + str(date)[2:-1] + \".csv\"\n",
    "    path_disp= \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/display_12\" + str(date)[2:-1] + \".csv\"\n",
    "\n",
    "\n",
    "    df_Req = pd.read_csv(path_req, **options_req)\n",
    "    df_Disp = pd.read_csv(path_disp, **options_disp)\n",
    "    df_Affichage = pd.read_csv(path_aff, **options_aff)\n",
    "\n",
    "    ##Merge between request & ref_partenaire tables\n",
    "    df_dec4 = pd.merge(df_ref_partenaire, df_Req, how=\"inner\", left_on=\"at_partner\", right_on=\"at_partner\")\n",
    "\n",
    "    ##Merge : add click table\n",
    "    df_dec4 = pd.merge(df_dec4, df_click, how=\"left\",\n",
    "                       left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                       right_on=[\"uniq_id\", \"uniq_id_widget\"])\n",
    "\n",
    "    ##Merge : add affichage table\n",
    "    df_dec4 = pd.merge(df_dec4, df_Affichage, how=\"left\",\n",
    "                       left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                       right_on=[\"uniq_id\", \"uniq_id_widget\"])\n",
    "\n",
    "    ##Merge : add display table\n",
    "    df_dec4 = pd.merge(df_dec4, df_Disp, how=\"left\",\n",
    "                       left_on=[\"uniq_id\", \"uniq_id_widget\"], \n",
    "                       right_on=[\"uniq_id\", \"uniq_id_widget\"])\n",
    "\n",
    "    ###Save the data Frame\n",
    "    out_path = \"/home/alka/Documents/PROJET_FIN_ETUDES/Data/Data4dec/data_\" + str(date)[2:-1] + \".pklz\"\n",
    "    with gzip.open(out_path,\"w\") as f:\n",
    "        pickle.dump(file=f, obj=df_dec4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2017-12-06'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(b\"2017-12-06\")[2:-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}