data-preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NIRS (preprocessing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gzip\n",
    "import json\n",
    "import utils\n",
    "\n",
    "utils.seed_everything(seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading data (skip if you already have the sampled data you need)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reviews = utils.getDF('data/Office_Products_5.json.gz').drop_duplicates(subset=['reviewerID', 'asin', 'reviewText', 'summary'])\n",
    "df_products = utils.getDF('data/meta_Office_Products.json.gz').drop_duplicates(subset=['asin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique products: 306617\n",
      "Number of unique users: 101501\n",
      "Number of unique reviews: 750068\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of unique products: {df_products[\"asin\"].drop_duplicates().shape[0]}')\n",
    "print(f'Number of unique users: {df_reviews[\"reviewerID\"].drop_duplicates().shape[0]}')\n",
    "print(f'Number of unique reviews: {df_reviews[[\"reviewerID\", \"asin\", \"reviewText\", \"summary\"]].drop_duplicates().shape[0]}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (750068, 12)\n",
      "Products df shape: (306617, 19)\n"
     ]
    }
   ],
   "source": [
    "utils.print_shapes(df_reviews, df_products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (750068, 12)\n",
      "Products df shape: (306617, 19)\n"
     ]
    }
   ],
   "source": [
    "df_reviews = df_reviews.drop_duplicates(subset=['reviewerID', 'asin', 'reviewText', 'summary'])\n",
    "df_products = df_products.drop_duplicates(subset=['asin'])\n",
    "\n",
    "utils.print_shapes(df_reviews, df_products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>style</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0</td>\n",
       "      <td>True</td>\n",
       "      <td>11 7, 2017</td>\n",
       "      <td>A2NIJTYWADLK57</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>{'Format:': ' Board book'}</td>\n",
       "      <td>cotton clay</td>\n",
       "      <td>kids like story BUT while i really wanted a bo...</td>\n",
       "      <td>good story, small size book though</td>\n",
       "      <td>1510012800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>True</td>\n",
       "      <td>03 7, 2017</td>\n",
       "      <td>A2827D8EEURMP4</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>{'Format:': ' Hardcover'}</td>\n",
       "      <td>emankcin</td>\n",
       "      <td>Bought this used and it came in great conditio...</td>\n",
       "      <td>Good</td>\n",
       "      <td>1488844800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>06 25, 2016</td>\n",
       "      <td>APB6087F4J09J</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>{'Format:': ' Board book'}</td>\n",
       "      <td>Starbucks Fan</td>\n",
       "      <td>Every story and book about Corduroy is Fantast...</td>\n",
       "      <td>Best Books for All Children</td>\n",
       "      <td>1466812800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified   reviewTime      reviewerID        asin                       style   reviewerName                                         reviewText                             summary  unixReviewTime vote image\n",
       "0      4.0      True   11 7, 2017  A2NIJTYWADLK57  0140503528  {'Format:': ' Board book'}    cotton clay  kids like story BUT while i really wanted a bo...  good story, small size book though      1510012800  NaN   NaN\n",
       "1      4.0      True   03 7, 2017  A2827D8EEURMP4  0140503528   {'Format:': ' Hardcover'}       emankcin  Bought this used and it came in great conditio...                                Good      1488844800  NaN   NaN\n",
       "2      5.0      True  06 25, 2016   APB6087F4J09J  0140503528  {'Format:': ' Board book'}  Starbucks Fan  Every story and book about Corduroy is Fantast...         Best Books for All Children      1466812800  NaN   NaN"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>tech1</th>\n",
       "      <th>description</th>\n",
       "      <th>fit</th>\n",
       "      <th>title</th>\n",
       "      <th>also_buy</th>\n",
       "      <th>tech2</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>also_view</th>\n",
       "      <th>main_cat</th>\n",
       "      <th>similar_item</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>asin</th>\n",
       "      <th>imageURL</th>\n",
       "      <th>imageURLHighRes</th>\n",
       "      <th>details</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[Office Products, Office &amp; School Supplies, Ed...</td>\n",
       "      <td>class=\"a-keyvalue prodDetTable\" role=\"present...</td>\n",
       "      <td>[Sequential Spelling is based on the classic O...</td>\n",
       "      <td></td>\n",
       "      <td>Sequential Spelling Level 1 Bundle with Studen...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>STL Distributors</td>\n",
       "      <td>[]</td>\n",
       "      <td>[&gt;#439,654 in Office Products (See top 100), &gt;...</td>\n",
       "      <td>[1935943065, 1935943073, B00IJH9Q4M, 002115021...</td>\n",
       "      <td>Office Products</td>\n",
       "      <td>class=\"a-bordered a-horizontal-stripes  a-spa...</td>\n",
       "      <td>August 15, 2014</td>\n",
       "      <td>$32.90</td>\n",
       "      <td>0012624861</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Office Products, Office &amp;amp; School Supplies...</td>\n",
       "      <td></td>\n",
       "      <td>[Unusual book, , ]</td>\n",
       "      <td></td>\n",
       "      <td>Mathematics, Applications and Concepts, Course...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>bailey</td>\n",
       "      <td>[]</td>\n",
       "      <td>3,839,628 in Books (</td>\n",
       "      <td>[]</td>\n",
       "      <td>Books</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>$8.62</td>\n",
       "      <td>0078652669</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[]</td>\n",
       "      <td>class=\"a-keyvalue prodDetTable\" role=\"present...</td>\n",
       "      <td>[Pearson MyHistoryLab Online Access Code for A...</td>\n",
       "      <td></td>\n",
       "      <td>Pearson MyHistoryLab Online Access Code for Am...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td>Pearson MyHistoryLab</td>\n",
       "      <td>[Pearson MyHistoryLab Online Access Code for A...</td>\n",
       "      <td>[&gt;#1,925,354 in Office Products (See top 100)]</td>\n",
       "      <td>[]</td>\n",
       "      <td>Office Products</td>\n",
       "      <td></td>\n",
       "      <td>June 21, 2012</td>\n",
       "      <td>$0.99</td>\n",
       "      <td>0136039847</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            category                                              tech1                                        description fit                                              title also_buy tech2                 brand                                            feature                                               rank                                          also_view         main_cat                                       similar_item             date   price        asin imageURL imageURLHighRes details\n",
       "0  [Office Products, Office & School Supplies, Ed...   class=\"a-keyvalue prodDetTable\" role=\"present...  [Sequential Spelling is based on the classic O...      Sequential Spelling Level 1 Bundle with Studen...       []            STL Distributors                                                 []  [>#439,654 in Office Products (See top 100), >...  [1935943065, 1935943073, B00IJH9Q4M, 002115021...  Office Products   class=\"a-bordered a-horizontal-stripes  a-spa...  August 15, 2014  $32.90  0012624861       []              []     NaN\n",
       "1  [Office Products, Office &amp; School Supplies...                                                                                    [Unusual book, , ]      Mathematics, Applications and Concepts, Course...       []                      bailey                                                 []                               3,839,628 in Books (                                                 []            Books                                                                       $8.62  0078652669       []              []     NaN\n",
       "2                                                 []   class=\"a-keyvalue prodDetTable\" role=\"present...  [Pearson MyHistoryLab Online Access Code for A...      Pearson MyHistoryLab Online Access Code for Am...       []        Pearson MyHistoryLab  [Pearson MyHistoryLab Online Access Code for A...     [>#1,925,354 in Office Products (See top 100)]                                                 []  Office Products                                                       June 21, 2012   $0.99  0136039847       []              []     NaN"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_products.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling data (skip if you already have the sampled data you need)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min_reviews_count = 10\n",
    "# max_users = 1000\n",
    "# frac_products = 0.1\n",
    "\n",
    "# df_reviews_sampled, df_products_sampled = utils.sample_data(\n",
    "#   df_reviews, df_products, min_reviews_count=min_reviews_count, max_users=max_users, frac_products=frac_products)\n",
    "\n",
    "# # save sampled data as checkpoint\n",
    "# # utils.save_data(df_reviews_sampled, df_products_sampled, 'data/reviews_sampled.csv', 'data/products_sampled.csv')\n",
    "# df_reviews_sampled['index'] = df_reviews_sampled.index\n",
    "# df_products_sampled['index'] = df_products_sampled.index\n",
    "# utils.save_data(df_reviews_sampled, 'data/reviews_sampled.csv')\n",
    "# utils.save_data(df_products_sampled, 'data/products_sampled.csv')\n",
    "\n",
    "df_reviews_sampled, df_products_sampled = df_reviews, df_products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (750068, 12)\n",
      "Products df shape: (306617, 19)\n"
     ]
    }
   ],
   "source": [
    "utils.print_shapes(df_reviews_sampled, df_products_sampled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data cleaning (missing values handling / removing irrelevant features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "notebookRunGroups": {
     "groupValue": "1"
    }
   },
   "outputs": [],
   "source": [
    "import data_cleaning as cleaning\n",
    "\n",
    "# optional checkpoint to skip the initial reading and sampling step\n",
    "# df_reviews_sampled = pd.read_csv('data/reviews_sampled.csv')\n",
    "# df_products_sampled = pd.read_csv('data/products_sampled.csv')\n",
    "\n",
    "# df_reviews_cleaned = df_reviews_sampled.copy()\n",
    "# df_products_cleaned = df_products_sampled.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process for reviews dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total nan values:  overall                0\n",
      "verified               0\n",
      "reviewTime             0\n",
      "reviewerID             0\n",
      "asin                   0\n",
      "style             281098\n",
      "reviewerName         133\n",
      "reviewText           202\n",
      "summary              122\n",
      "unixReviewTime         0\n",
      "vote              667035\n",
      "image             739274\n",
      "dtype: int64\n",
      "reviews shape: (750068, 12)\n"
     ]
    }
   ],
   "source": [
    "print(\"Total nan values: \" , df_reviews_sampled.isna().sum())\n",
    "utils.print_shape(df_reviews_sampled, 'reviews')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reviews_cleaned = cleaning.clean_reviews_data(df_reviews_sampled)\n",
    "df_reviews_cleaned.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total nan values:  overall         0\n",
      "reviewTime      0\n",
      "reviewerID      0\n",
      "asin            0\n",
      "reviewerName    0\n",
      "reviewText      0\n",
      "summary         0\n",
      "dtype: int64\n",
      "reviews shape: (749626, 7)\n"
     ]
    }
   ],
   "source": [
    "print(\"Total nan values: \" , df_reviews_cleaned.isna().sum())\n",
    "utils.print_shape(df_reviews_cleaned, 'reviews')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process for the product dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nan values per feature: \n",
      " details    7147\n",
      "dtype: int64\n",
      "products shape: (306617, 19)\n"
     ]
    }
   ],
   "source": [
    "print('Nan values per feature: \\n', utils.count_nan_values(df_products_sampled))\n",
    "utils.print_shape(df_products_sampled, 'products')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_products_cleaned = cleaning.clean_products_data(df_products_sampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nan values per feature: \n",
      " Series([], dtype: int64)\n",
      "products shape: (303665, 8)\n"
     ]
    }
   ],
   "source": [
    "print('Nan values per feature: \\n', utils.count_nan_values(df_products_cleaned))\n",
    "utils.print_shape(df_products_cleaned, 'products')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import text_preprocessing as tp\n",
    "\n",
    "df_reviews_text_processed = df_reviews_cleaned.copy()\n",
    "df_products_text_processed = df_products_cleaned.copy()\n",
    "\n",
    "preprocessor = tp.TextPreprocessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reviews_text_processed['summary'] = df_reviews_text_processed['summary'].astype(str)\n",
    "df_reviews_text_processed['summary'] = preprocessor.fit_transform(df_reviews_text_processed['summary'])\n",
    "\n",
    "df_reviews_text_processed = df_reviews_text_processed[df_reviews_text_processed['summary'] != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reviews_text_processed['reviewText'] = df_reviews_text_processed['reviewText'].astype(str)\n",
    "df_reviews_text_processed['reviewText'] = preprocessor.fit_transform(df_reviews_text_processed['reviewText'])\n",
    "\n",
    "df_reviews_text_processed = df_reviews_text_processed[df_reviews_text_processed['reviewText'] != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_products_text_processed['description'] = df_products_text_processed['description'].astype(str)\n",
    "df_products_text_processed['description'] = preprocessor.fit_transform(df_products_text_processed['description'])\n",
    "df_products_text_processed['description'] = df_products_text_processed['description'].str[1:-1]\n",
    "\n",
    "df_products_text_processed = df_products_text_processed[df_products_text_processed['description'] != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_products_text_processed['feature'] = df_products_text_processed['feature'].astype(str)\n",
    "df_products_text_processed['feature'] = df_products_text_processed['feature'].apply(lambda x: \". \".join(eval(x)).strip())\n",
    "df_products_text_processed['feature'] = preprocessor.fit_transform(df_products_text_processed['feature'])\n",
    "\n",
    "df_products_text_processed = df_products_text_processed[df_products_text_processed['feature'] != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess title\n",
    "df_products_text_processed['title'] = df_products_text_processed['title'].astype(str)\n",
    "df_products_text_processed['title'] = preprocessor.fit_transform(df_products_text_processed['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess brand\n",
    "df_products_text_processed['brand'] = df_products_text_processed['brand'].astype(str)\n",
    "df_products_text_processed[\"brand\"] = preprocessor.fit_transform(df_products_text_processed[\"brand\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save checkpoint data preprocessed\n",
    "utils.save_data(df_reviews_text_processed, 'data/reviews_processed_checkpoint.csv')\n",
    "utils.save_data(df_products_text_processed, 'data/products_processed_checkpoint.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling \"rank\" feature from product dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We decided just to keep the rank of the product in the office category (the rank will be -1 if the product doen't have a rank in hte Office category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "df_products_rank_managed = df_products_text_processed.copy()\n",
    "\n",
    "def extract_office_product_rank(rank):\n",
    "    match = re.search(r'>#(\\d+)', rank)\n",
    "    if match and 'Office Products' in rank:\n",
    "        return int(match.group(1))\n",
    "    else:\n",
    "        return -1\n",
    "\n",
    "df_products_rank_managed['rank'] = df_products_rank_managed['rank'].astype(str)\n",
    "df_products_rank_managed['rank'] = df_products_rank_managed['rank'].apply(extract_office_product_rank)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Renaming features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_products_ranaming_cols = df_products_rank_managed.copy()\n",
    "\n",
    "df_products_ranaming_cols.rename(columns={'date': 'productPublishedDate'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot encoding for \"main category\" feature of product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>productPublishedDate</th>\n",
       "      <th>asin</th>\n",
       "      <th>main_cat_</th>\n",
       "      <th>main_cat_All Beauty</th>\n",
       "      <th>main_cat_All Electronics</th>\n",
       "      <th>main_cat_Amazon Devices</th>\n",
       "      <th>main_cat_Amazon Home</th>\n",
       "      <th>main_cat_Appliances</th>\n",
       "      <th>main_cat_Arts, Crafts &amp; Sewing</th>\n",
       "      <th>main_cat_Arts, Crafts &amp;amp; Sewing</th>\n",
       "      <th>main_cat_Automotive</th>\n",
       "      <th>main_cat_Baby</th>\n",
       "      <th>main_cat_Books</th>\n",
       "      <th>main_cat_Camera &amp; Photo</th>\n",
       "      <th>main_cat_Camera &amp;amp; Photo</th>\n",
       "      <th>main_cat_Car Electronics</th>\n",
       "      <th>main_cat_Cell Phones &amp; Accessories</th>\n",
       "      <th>main_cat_Cell Phones &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Computers</th>\n",
       "      <th>main_cat_GPS &amp; Navigation</th>\n",
       "      <th>main_cat_Gift Cards</th>\n",
       "      <th>main_cat_Grocery</th>\n",
       "      <th>main_cat_Health &amp; Personal Care</th>\n",
       "      <th>main_cat_Health &amp;amp; Personal Care</th>\n",
       "      <th>main_cat_Home Audio &amp; Theater</th>\n",
       "      <th>main_cat_Home Audio &amp;amp; Theater</th>\n",
       "      <th>main_cat_Industrial &amp; Scientific</th>\n",
       "      <th>main_cat_Industrial &amp;amp; Scientific</th>\n",
       "      <th>main_cat_Musical Instruments</th>\n",
       "      <th>main_cat_Office Products</th>\n",
       "      <th>main_cat_Pet Supplies</th>\n",
       "      <th>main_cat_Portable Audio &amp; Accessories</th>\n",
       "      <th>main_cat_Portable Audio &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Software</th>\n",
       "      <th>main_cat_Sports &amp; Outdoors</th>\n",
       "      <th>main_cat_Sports &amp;amp; Outdoors</th>\n",
       "      <th>main_cat_Tools &amp; Home Improvement</th>\n",
       "      <th>main_cat_Tools &amp;amp; Home Improvement</th>\n",
       "      <th>main_cat_Toys &amp; Games</th>\n",
       "      <th>main_cat_Toys &amp;amp; Games</th>\n",
       "      <th>main_cat_Video Games</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>earson myhistorylab online access code america...</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>pearson myhistorylab</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>1</td>\n",
       "      <td>June 21, 2012</td>\n",
       "      <td>0136039847</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>orduroy bear go launderette lisa overhears mot...</td>\n",
       "      <td>pocket corduroy</td>\n",
       "      <td>ingram book distributor</td>\n",
       "      <td>9780140503524</td>\n",
       "      <td>422</td>\n",
       "      <td>September 14, 2006</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description                                              title                    brand                                            feature  rank productPublishedDate        asin  main_cat_  main_cat_All Beauty  main_cat_All Electronics  main_cat_Amazon Devices  main_cat_Amazon Home  main_cat_Appliances  main_cat_Arts, Crafts & Sewing  main_cat_Arts, Crafts &amp; Sewing  main_cat_Automotive  main_cat_Baby  main_cat_Books  main_cat_Camera & Photo  main_cat_Camera &amp; Photo  main_cat_Car Electronics  main_cat_Cell Phones & Accessories  main_cat_Cell Phones &amp; Accessories  main_cat_Computers  main_cat_GPS & Navigation  main_cat_Gift Cards  main_cat_Grocery  main_cat_Health & Personal Care  main_cat_Health &amp; Personal Care  main_cat_Home Audio & Theater  main_cat_Home Audio &amp; Theater  main_cat_Industrial & Scientific  main_cat_Industrial &amp; Scientific  main_cat_Musical Instruments  main_cat_Office Products  main_cat_Pet Supplies  main_cat_Portable Audio & Accessories  main_cat_Portable Audio &amp; Accessories  main_cat_Software  main_cat_Sports & Outdoors  main_cat_Sports &amp; Outdoors  main_cat_Tools & Home Improvement  main_cat_Tools &amp; Home Improvement  main_cat_Toys & Games  main_cat_Toys &amp; Games  main_cat_Video Games\n",
       "2  earson myhistorylab online access code america...  pearson myhistorylab online access code americ...     pearson myhistorylab  pearson myhistorylab online access code americ...     1        June 21, 2012  0136039847          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0\n",
       "3  orduroy bear go launderette lisa overhears mot...                                    pocket corduroy  ingram book distributor                                      9780140503524   422   September 14, 2006  0140503528          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_products_main_cat_onehot = df_products_ranaming_cols.copy()\n",
    "\n",
    "print(df_products_main_cat_onehot[\"main_cat\"].nunique())\n",
    "\n",
    "df_products_main_cat_onehot = pd.get_dummies(df_products_main_cat_onehot, columns=['main_cat'], prefix='main_cat')\n",
    "\n",
    "one_hot_columns = [col for col in df_products_main_cat_onehot.columns if col.startswith('main_cat_')]\n",
    "df_products_main_cat_onehot[one_hot_columns] = df_products_main_cat_onehot[one_hot_columns].astype(int)\n",
    "\n",
    "df_products_main_cat_onehot.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Managing dates for products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "most recent date: 2019-01-14 00:00:00\n",
      "most oudated date: 1941-12-07 00:00:00\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>productPublishedDate</th>\n",
       "      <th>asin</th>\n",
       "      <th>main_cat_</th>\n",
       "      <th>main_cat_All Beauty</th>\n",
       "      <th>main_cat_All Electronics</th>\n",
       "      <th>main_cat_Amazon Devices</th>\n",
       "      <th>main_cat_Amazon Home</th>\n",
       "      <th>main_cat_Appliances</th>\n",
       "      <th>main_cat_Arts, Crafts &amp; Sewing</th>\n",
       "      <th>main_cat_Arts, Crafts &amp;amp; Sewing</th>\n",
       "      <th>main_cat_Automotive</th>\n",
       "      <th>main_cat_Baby</th>\n",
       "      <th>main_cat_Books</th>\n",
       "      <th>main_cat_Camera &amp; Photo</th>\n",
       "      <th>main_cat_Camera &amp;amp; Photo</th>\n",
       "      <th>main_cat_Car Electronics</th>\n",
       "      <th>main_cat_Cell Phones &amp; Accessories</th>\n",
       "      <th>main_cat_Cell Phones &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Computers</th>\n",
       "      <th>main_cat_GPS &amp; Navigation</th>\n",
       "      <th>main_cat_Gift Cards</th>\n",
       "      <th>main_cat_Grocery</th>\n",
       "      <th>main_cat_Health &amp; Personal Care</th>\n",
       "      <th>main_cat_Health &amp;amp; Personal Care</th>\n",
       "      <th>main_cat_Home Audio &amp; Theater</th>\n",
       "      <th>main_cat_Home Audio &amp;amp; Theater</th>\n",
       "      <th>main_cat_Industrial &amp; Scientific</th>\n",
       "      <th>main_cat_Industrial &amp;amp; Scientific</th>\n",
       "      <th>main_cat_Musical Instruments</th>\n",
       "      <th>main_cat_Office Products</th>\n",
       "      <th>main_cat_Pet Supplies</th>\n",
       "      <th>main_cat_Portable Audio &amp; Accessories</th>\n",
       "      <th>main_cat_Portable Audio &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Software</th>\n",
       "      <th>main_cat_Sports &amp; Outdoors</th>\n",
       "      <th>main_cat_Sports &amp;amp; Outdoors</th>\n",
       "      <th>main_cat_Tools &amp; Home Improvement</th>\n",
       "      <th>main_cat_Tools &amp;amp; Home Improvement</th>\n",
       "      <th>main_cat_Toys &amp; Games</th>\n",
       "      <th>main_cat_Toys &amp;amp; Games</th>\n",
       "      <th>main_cat_Video Games</th>\n",
       "      <th>dayDifferenceProduct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>earson myhistorylab online access code america...</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>pearson myhistorylab</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>1</td>\n",
       "      <td>June 21, 2012</td>\n",
       "      <td>0136039847</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>orduroy bear go launderette lisa overhears mot...</td>\n",
       "      <td>pocket corduroy</td>\n",
       "      <td>ingram book distributor</td>\n",
       "      <td>9780140503524</td>\n",
       "      <td>422</td>\n",
       "      <td>September 14, 2006</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.637309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description                                              title                    brand                                            feature  rank productPublishedDate        asin  main_cat_  main_cat_All Beauty  main_cat_All Electronics  main_cat_Amazon Devices  main_cat_Amazon Home  main_cat_Appliances  main_cat_Arts, Crafts & Sewing  main_cat_Arts, Crafts &amp; Sewing  main_cat_Automotive  main_cat_Baby  main_cat_Books  main_cat_Camera & Photo  main_cat_Camera &amp; Photo  main_cat_Car Electronics  main_cat_Cell Phones & Accessories  main_cat_Cell Phones &amp; Accessories  main_cat_Computers  main_cat_GPS & Navigation  main_cat_Gift Cards  main_cat_Grocery  main_cat_Health & Personal Care  main_cat_Health &amp; Personal Care  main_cat_Home Audio & Theater  main_cat_Home Audio &amp; Theater  main_cat_Industrial & Scientific  main_cat_Industrial &amp; Scientific  main_cat_Musical Instruments  main_cat_Office Products  main_cat_Pet Supplies  main_cat_Portable Audio & Accessories  main_cat_Portable Audio &amp; Accessories  main_cat_Software  main_cat_Sports & Outdoors  main_cat_Sports &amp; Outdoors  main_cat_Tools & Home Improvement  main_cat_Tools &amp; Home Improvement  main_cat_Toys & Games  main_cat_Toys &amp; Games  main_cat_Video Games  dayDifferenceProduct\n",
       "2  earson myhistorylab online access code america...  pearson myhistorylab online access code americ...     pearson myhistorylab  pearson myhistorylab online access code americ...     1        June 21, 2012  0136039847          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.786785\n",
       "3  orduroy bear go launderette lisa overhears mot...                                    pocket corduroy  ingram book distributor                                      9780140503524   422   September 14, 2006  0140503528          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.637309"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_products_dates_managed = df_products_main_cat_onehot.copy()\n",
    "\n",
    "def exponential_decay(time_diff, decay_rate=0.0001):\n",
    "    return np.exp(-decay_rate * time_diff)\n",
    "\n",
    "dates = df_products_dates_managed['productPublishedDate'].apply(lambda x: pd.to_datetime(x, errors='coerce'))\n",
    "most_recent_date = dates.max()\n",
    "most_outdated_date = dates.min()\n",
    "print(\"most recent date:\", most_recent_date)\n",
    "print(\"most oudated date:\", most_outdated_date)\n",
    "\n",
    "\n",
    "df_products_dates_managed['dayDifferenceProduct'] = dates.apply(lambda x: (most_recent_date - x).days if pd.notnull(x) else (most_recent_date - most_outdated_date).days)\n",
    "\n",
    "df_products_dates_managed['dayDifferenceProduct'] = df_products_dates_managed['dayDifferenceProduct'].apply(exponential_decay)\n",
    "df_products_dates_managed.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Managing dates for reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reviews: most recent date: 2018-10-02 00:00:00\n",
      "reviews: most oudated date: 1999-10-11 00:00:00\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>dayDifferenceReview</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0</td>\n",
       "      <td>November 07, 2017</td>\n",
       "      <td>A2NIJTYWADLK57</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>cotton clay</td>\n",
       "      <td>kid like story really wanted board book one sm...</td>\n",
       "      <td>good story small size book though</td>\n",
       "      <td>0.967635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>March 07, 2017</td>\n",
       "      <td>A2827D8EEURMP4</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>emankcin</td>\n",
       "      <td>bought used came great condition almost like n...</td>\n",
       "      <td>good</td>\n",
       "      <td>0.944216</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall         reviewTime      reviewerID        asin reviewerName                                         reviewText                            summary  dayDifferenceReview\n",
       "0      4.0  November 07, 2017  A2NIJTYWADLK57  0140503528  cotton clay  kid like story really wanted board book one sm...  good story small size book though             0.967635\n",
       "1      4.0     March 07, 2017  A2827D8EEURMP4  0140503528     emankcin  bought used came great condition almost like n...                               good             0.944216"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews_dates_managed = df_reviews_text_processed.copy()\n",
    "\n",
    "rev_dates = df_reviews_cleaned['reviewTime'].apply(lambda x: pd.to_datetime(x, errors='coerce'))\n",
    "\n",
    "\n",
    "rev_most_recent_date = rev_dates.max()\n",
    "rev_most_outdated_date = rev_dates.min()\n",
    "print(\"reviews: most recent date:\", rev_most_recent_date)\n",
    "print(\"reviews: most oudated date:\", rev_most_outdated_date)\n",
    "\n",
    "df_reviews_dates_managed['dayDifferenceReview'] = rev_dates.apply(lambda x: (rev_most_recent_date - x).days if pd.notnull(x) else (rev_most_recent_date - rev_most_outdated_date).days)\n",
    "df_reviews_dates_managed['dayDifferenceReview'] = df_reviews_dates_managed['dayDifferenceReview'].apply(exponential_decay)\n",
    "df_reviews_dates_managed.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Consistency checks of the samples of products and reviews data after the preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (746577, 8)\n",
      "Products df shape: (233469, 47)\n"
     ]
    }
   ],
   "source": [
    "utils.print_shapes(df_reviews_dates_managed, df_products_dates_managed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reviews shape: (661471, 8)\n",
      "products shape: (233469, 47)\n"
     ]
    }
   ],
   "source": [
    "df_products_final = df_products_dates_managed.copy().drop_duplicates(subset=['asin'])\n",
    "df_reviews_final = df_reviews_dates_managed.copy().drop_duplicates(subset=['reviewerID', 'asin', 'reviewText', 'summary'])\n",
    "\n",
    "df_reviews_final = df_reviews_final[df_reviews_final['asin'].isin(df_products_final['asin'])]\n",
    "\n",
    "\n",
    "# # Last check to make sure users has at least \"max_reviews_count\" (+ drop duplicates)\n",
    "# user_reviews_count = df_reviews_final['reviewerID'].value_counts()\n",
    "# selected_users = user_reviews_count[user_reviews_count >= min_reviews_count]\n",
    "# df_reviews_final = df_reviews_final[df_reviews_final['reviewerID'].isin(selected_users.keys())]\n",
    "\n",
    "assert (df_reviews_final['asin'].isin(df_products_final['asin'])).all()\n",
    "utils.print_shape(df_reviews_final, 'reviews')\n",
    "utils.print_shape(df_products_final, 'products')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge products and reviews dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the following dataset (`merged_df`) will be the actual data used in NCF. This since, other than the reviews themselves, It just contains the related products information with which the users interacted already."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reviews_final: pd.DataFrame = df_reviews_final.copy().drop_duplicates(subset=['reviewerID', 'asin', 'reviewText', 'summary'])\n",
    "df_products_final: pd.DataFrame = df_products_dates_managed.copy().drop_duplicates(subset=['asin'])\n",
    "\n",
    "merged_df = pd.merge(df_reviews_final, df_products_final, on='asin', how='inner')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a separate dataset for unreviewed products (items with no interaction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "unreviewed_products_df = df_products_final[~df_products_final['asin'].isin(df_reviews_final['asin'])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the prepared data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>dayDifferenceReview</th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>productPublishedDate</th>\n",
       "      <th>main_cat_</th>\n",
       "      <th>main_cat_All Beauty</th>\n",
       "      <th>main_cat_All Electronics</th>\n",
       "      <th>main_cat_Amazon Devices</th>\n",
       "      <th>main_cat_Amazon Home</th>\n",
       "      <th>main_cat_Appliances</th>\n",
       "      <th>main_cat_Arts, Crafts &amp; Sewing</th>\n",
       "      <th>main_cat_Arts, Crafts &amp;amp; Sewing</th>\n",
       "      <th>main_cat_Automotive</th>\n",
       "      <th>main_cat_Baby</th>\n",
       "      <th>main_cat_Books</th>\n",
       "      <th>main_cat_Camera &amp; Photo</th>\n",
       "      <th>main_cat_Camera &amp;amp; Photo</th>\n",
       "      <th>main_cat_Car Electronics</th>\n",
       "      <th>main_cat_Cell Phones &amp; Accessories</th>\n",
       "      <th>main_cat_Cell Phones &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Computers</th>\n",
       "      <th>main_cat_GPS &amp; Navigation</th>\n",
       "      <th>main_cat_Gift Cards</th>\n",
       "      <th>main_cat_Grocery</th>\n",
       "      <th>main_cat_Health &amp; Personal Care</th>\n",
       "      <th>main_cat_Health &amp;amp; Personal Care</th>\n",
       "      <th>main_cat_Home Audio &amp; Theater</th>\n",
       "      <th>main_cat_Home Audio &amp;amp; Theater</th>\n",
       "      <th>main_cat_Industrial &amp; Scientific</th>\n",
       "      <th>main_cat_Industrial &amp;amp; Scientific</th>\n",
       "      <th>main_cat_Musical Instruments</th>\n",
       "      <th>main_cat_Office Products</th>\n",
       "      <th>main_cat_Pet Supplies</th>\n",
       "      <th>main_cat_Portable Audio &amp; Accessories</th>\n",
       "      <th>main_cat_Portable Audio &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Software</th>\n",
       "      <th>main_cat_Sports &amp; Outdoors</th>\n",
       "      <th>main_cat_Sports &amp;amp; Outdoors</th>\n",
       "      <th>main_cat_Tools &amp; Home Improvement</th>\n",
       "      <th>main_cat_Tools &amp;amp; Home Improvement</th>\n",
       "      <th>main_cat_Toys &amp; Games</th>\n",
       "      <th>main_cat_Toys &amp;amp; Games</th>\n",
       "      <th>main_cat_Video Games</th>\n",
       "      <th>dayDifferenceProduct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0</td>\n",
       "      <td>November 07, 2017</td>\n",
       "      <td>A2NIJTYWADLK57</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>cotton clay</td>\n",
       "      <td>kid like story really wanted board book one sm...</td>\n",
       "      <td>good story small size book though</td>\n",
       "      <td>0.967635</td>\n",
       "      <td>orduroy bear go launderette lisa overhears mot...</td>\n",
       "      <td>pocket corduroy</td>\n",
       "      <td>ingram book distributor</td>\n",
       "      <td>9780140503524</td>\n",
       "      <td>422</td>\n",
       "      <td>September 14, 2006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.637309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>March 07, 2017</td>\n",
       "      <td>A2827D8EEURMP4</td>\n",
       "      <td>0140503528</td>\n",
       "      <td>emankcin</td>\n",
       "      <td>bought used came great condition almost like n...</td>\n",
       "      <td>good</td>\n",
       "      <td>0.944216</td>\n",
       "      <td>orduroy bear go launderette lisa overhears mot...</td>\n",
       "      <td>pocket corduroy</td>\n",
       "      <td>ingram book distributor</td>\n",
       "      <td>9780140503524</td>\n",
       "      <td>422</td>\n",
       "      <td>September 14, 2006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.637309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall         reviewTime      reviewerID        asin reviewerName                                         reviewText                            summary  dayDifferenceReview                                        description            title                    brand        feature  rank productPublishedDate  main_cat_  main_cat_All Beauty  main_cat_All Electronics  main_cat_Amazon Devices  main_cat_Amazon Home  main_cat_Appliances  main_cat_Arts, Crafts & Sewing  main_cat_Arts, Crafts &amp; Sewing  main_cat_Automotive  main_cat_Baby  main_cat_Books  main_cat_Camera & Photo  main_cat_Camera &amp; Photo  main_cat_Car Electronics  main_cat_Cell Phones & Accessories  main_cat_Cell Phones &amp; Accessories  main_cat_Computers  main_cat_GPS & Navigation  main_cat_Gift Cards  main_cat_Grocery  main_cat_Health & Personal Care  main_cat_Health &amp; Personal Care  main_cat_Home Audio & Theater  main_cat_Home Audio &amp; Theater  main_cat_Industrial & Scientific  main_cat_Industrial &amp; Scientific  main_cat_Musical Instruments  main_cat_Office Products  main_cat_Pet Supplies  main_cat_Portable Audio & Accessories  main_cat_Portable Audio &amp; Accessories  main_cat_Software  main_cat_Sports & Outdoors  main_cat_Sports &amp; Outdoors  main_cat_Tools & Home Improvement  main_cat_Tools &amp; Home Improvement  main_cat_Toys & Games  main_cat_Toys &amp; Games  main_cat_Video Games  dayDifferenceProduct\n",
       "0      4.0  November 07, 2017  A2NIJTYWADLK57  0140503528  cotton clay  kid like story really wanted board book one sm...  good story small size book though             0.967635  orduroy bear go launderette lisa overhears mot...  pocket corduroy  ingram book distributor  9780140503524   422   September 14, 2006          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.637309\n",
       "1      4.0     March 07, 2017  A2827D8EEURMP4  0140503528     emankcin  bought used came great condition almost like n...                               good             0.944216  orduroy bear go launderette lisa overhears mot...  pocket corduroy  ingram book distributor  9780140503524   422   September 14, 2006          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.637309"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>brand</th>\n",
       "      <th>feature</th>\n",
       "      <th>rank</th>\n",
       "      <th>productPublishedDate</th>\n",
       "      <th>asin</th>\n",
       "      <th>main_cat_</th>\n",
       "      <th>main_cat_All Beauty</th>\n",
       "      <th>main_cat_All Electronics</th>\n",
       "      <th>main_cat_Amazon Devices</th>\n",
       "      <th>main_cat_Amazon Home</th>\n",
       "      <th>main_cat_Appliances</th>\n",
       "      <th>main_cat_Arts, Crafts &amp; Sewing</th>\n",
       "      <th>main_cat_Arts, Crafts &amp;amp; Sewing</th>\n",
       "      <th>main_cat_Automotive</th>\n",
       "      <th>main_cat_Baby</th>\n",
       "      <th>main_cat_Books</th>\n",
       "      <th>main_cat_Camera &amp; Photo</th>\n",
       "      <th>main_cat_Camera &amp;amp; Photo</th>\n",
       "      <th>main_cat_Car Electronics</th>\n",
       "      <th>main_cat_Cell Phones &amp; Accessories</th>\n",
       "      <th>main_cat_Cell Phones &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Computers</th>\n",
       "      <th>main_cat_GPS &amp; Navigation</th>\n",
       "      <th>main_cat_Gift Cards</th>\n",
       "      <th>main_cat_Grocery</th>\n",
       "      <th>main_cat_Health &amp; Personal Care</th>\n",
       "      <th>main_cat_Health &amp;amp; Personal Care</th>\n",
       "      <th>main_cat_Home Audio &amp; Theater</th>\n",
       "      <th>main_cat_Home Audio &amp;amp; Theater</th>\n",
       "      <th>main_cat_Industrial &amp; Scientific</th>\n",
       "      <th>main_cat_Industrial &amp;amp; Scientific</th>\n",
       "      <th>main_cat_Musical Instruments</th>\n",
       "      <th>main_cat_Office Products</th>\n",
       "      <th>main_cat_Pet Supplies</th>\n",
       "      <th>main_cat_Portable Audio &amp; Accessories</th>\n",
       "      <th>main_cat_Portable Audio &amp;amp; Accessories</th>\n",
       "      <th>main_cat_Software</th>\n",
       "      <th>main_cat_Sports &amp; Outdoors</th>\n",
       "      <th>main_cat_Sports &amp;amp; Outdoors</th>\n",
       "      <th>main_cat_Tools &amp; Home Improvement</th>\n",
       "      <th>main_cat_Tools &amp;amp; Home Improvement</th>\n",
       "      <th>main_cat_Toys &amp; Games</th>\n",
       "      <th>main_cat_Toys &amp;amp; Games</th>\n",
       "      <th>main_cat_Video Games</th>\n",
       "      <th>dayDifferenceProduct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>earson myhistorylab online access code america...</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>pearson myhistorylab</td>\n",
       "      <td>pearson myhistorylab online access code americ...</td>\n",
       "      <td>1</td>\n",
       "      <td>June 21, 2012</td>\n",
       "      <td>0136039847</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ood helper help record reading progress br mak...</td>\n",
       "      <td>ekloen mixed design antiqued bronze colour ele...</td>\n",
       "      <td>ekloen</td>\n",
       "      <td>bookmark also art inspirational gift occasion ...</td>\n",
       "      <td>43</td>\n",
       "      <td>November 08, 2015</td>\n",
       "      <td>0245109919</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.890208</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description                                              title                 brand                                            feature  rank productPublishedDate        asin  main_cat_  main_cat_All Beauty  main_cat_All Electronics  main_cat_Amazon Devices  main_cat_Amazon Home  main_cat_Appliances  main_cat_Arts, Crafts & Sewing  main_cat_Arts, Crafts &amp; Sewing  main_cat_Automotive  main_cat_Baby  main_cat_Books  main_cat_Camera & Photo  main_cat_Camera &amp; Photo  main_cat_Car Electronics  main_cat_Cell Phones & Accessories  main_cat_Cell Phones &amp; Accessories  main_cat_Computers  main_cat_GPS & Navigation  main_cat_Gift Cards  main_cat_Grocery  main_cat_Health & Personal Care  main_cat_Health &amp; Personal Care  main_cat_Home Audio & Theater  main_cat_Home Audio &amp; Theater  main_cat_Industrial & Scientific  main_cat_Industrial &amp; Scientific  main_cat_Musical Instruments  main_cat_Office Products  main_cat_Pet Supplies  main_cat_Portable Audio & Accessories  main_cat_Portable Audio &amp; Accessories  main_cat_Software  main_cat_Sports & Outdoors  main_cat_Sports &amp; Outdoors  main_cat_Tools & Home Improvement  main_cat_Tools &amp; Home Improvement  main_cat_Toys & Games  main_cat_Toys &amp; Games  main_cat_Video Games  dayDifferenceProduct\n",
       "2  earson myhistorylab online access code america...  pearson myhistorylab online access code americ...  pearson myhistorylab  pearson myhistorylab online access code americ...     1        June 21, 2012  0136039847          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.786785\n",
       "5  ood helper help record reading progress br mak...  ekloen mixed design antiqued bronze colour ele...                ekloen  bookmark also art inspirational gift occasion ...    43    November 08, 2015  0245109919          0                    0                         0                        0                     0                    0                               0                                   0                    0              0               0                        0                            0                         0                                   0                                       0                   0                          0                    0                 0                                0                                    0                              0                                  0                                 0                                     0                             0                         1                      0                                      0                                          0                  0                           0                               0                                  0                                      0                      0                          0                     0              0.890208"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unreviewed_products_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique products: 233469\n",
      "Number of unique users: 101359\n",
      "Number of unique reviewed products: 23884\n",
      "Number of unique unreviewed products: 209585\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of unique products: {df_products_final[\"asin\"].nunique()}')\n",
    "print(f'Number of unique users: {df_reviews_final[\"reviewerID\"].nunique()}')\n",
    "print(f'Number of unique reviewed products: {merged_df[\"asin\"].nunique()}')\n",
    "print(f'Number of unique unreviewed products: {unreviewed_products_df[\"asin\"].nunique()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (661471, 8)\n",
      "Products df shape: (233469, 47)\n"
     ]
    }
   ],
   "source": [
    "utils.print_shapes(df_reviews_final, df_products_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews df shape: (661471, 8)\n",
      "Products df shape: (233469, 47)\n"
     ]
    }
   ],
   "source": [
    "utils.print_shapes(df_reviews_final.drop_duplicates(subset=['reviewerID', 'asin', 'reviewText', 'summary']), df_products_final.drop_duplicates(subset=['asin']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(661471, 54)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.save_data(merged_df, 'data/merged_data_processed.csv')\n",
    "utils.save_data(unreviewed_products_df, 'data/unreviewed_products_processed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.save_data(df_reviews_final, 'data/reviews_sampled_processed.csv')\n",
    "utils.save_data(df_products_final, 'data/products_sampled_processed.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "projects",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}