user_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import database_utils\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "myObject = database_utils.DatabaseConnector()\n",
    "engine = myObject.init_db_engine()\n",
    "df = myObject.read_rds_table(engine,\"legacy_users\")\n",
    "\n",
    "type(df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df is a dataframe that now contains the data from our table\n",
    "These are the first & last few lines of the database then!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>date_of_birth</th>\n",
       "      <th>company</th>\n",
       "      <th>email_address</th>\n",
       "      <th>address</th>\n",
       "      <th>country</th>\n",
       "      <th>country_code</th>\n",
       "      <th>phone_number</th>\n",
       "      <th>join_date</th>\n",
       "      <th>user_uuid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Sigfried</td>\n",
       "      <td>Noack</td>\n",
       "      <td>1990-09-30</td>\n",
       "      <td>Heydrich Junitz KG</td>\n",
       "      <td>rudi79@winkler.de</td>\n",
       "      <td>Zimmerstr. 1/0\\n59015 Gießen</td>\n",
       "      <td>Germany</td>\n",
       "      <td>DE</td>\n",
       "      <td>+49(0) 047905356</td>\n",
       "      <td>2018-10-10</td>\n",
       "      <td>93caf182-e4e9-4c6e-bebb-60a1a9dcf9b8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Guy</td>\n",
       "      <td>Allen</td>\n",
       "      <td>1940-12-01</td>\n",
       "      <td>Fox Ltd</td>\n",
       "      <td>rhodesclifford@henderson.com</td>\n",
       "      <td>Studio 22a\\nLynne terrace\\nMcCarthymouth\\nTF0 9GH</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>(0161) 496 0674</td>\n",
       "      <td>2001-12-20</td>\n",
       "      <td>8fe96c3a-d62d-4eb5-b313-cf12d9126a49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Harry</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>1995-08-02</td>\n",
       "      <td>Johnson, Jones and Harris</td>\n",
       "      <td>glen98@bryant-marshall.co.uk</td>\n",
       "      <td>92 Ann drive\\nJoanborough\\nSK0 6LR</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>+44(0)121 4960340</td>\n",
       "      <td>2016-12-16</td>\n",
       "      <td>fc461df4-b919-48b2-909e-55c95a03fe6b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Darren</td>\n",
       "      <td>Hussain</td>\n",
       "      <td>1972-09-23</td>\n",
       "      <td>Wheeler LLC</td>\n",
       "      <td>daniellebryan@thompson.org</td>\n",
       "      <td>19 Robinson meadow\\nNew Tracy\\nW22 2QG</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>(0306) 999 0871</td>\n",
       "      <td>2004-02-23</td>\n",
       "      <td>6104719f-ef14-4b09-bf04-fb0c4620acb0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Garry</td>\n",
       "      <td>Stone</td>\n",
       "      <td>1952-12-20</td>\n",
       "      <td>Warner Inc</td>\n",
       "      <td>billy14@long-warren.com</td>\n",
       "      <td>3 White pass\\nHunterborough\\nNN96 4UE</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>0121 496 0225</td>\n",
       "      <td>2006-09-01</td>\n",
       "      <td>9523a6d3-b2dd-4670-a51a-36aebc89f579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index first_name last_name date_of_birth                    company  \\\n",
       "0      0   Sigfried     Noack    1990-09-30         Heydrich Junitz KG   \n",
       "1      1        Guy     Allen    1940-12-01                    Fox Ltd   \n",
       "2      2      Harry  Lawrence    1995-08-02  Johnson, Jones and Harris   \n",
       "3      3     Darren   Hussain    1972-09-23                Wheeler LLC   \n",
       "4      4      Garry     Stone    1952-12-20                 Warner Inc   \n",
       "\n",
       "                  email_address  \\\n",
       "0             rudi79@winkler.de   \n",
       "1  rhodesclifford@henderson.com   \n",
       "2  glen98@bryant-marshall.co.uk   \n",
       "3    daniellebryan@thompson.org   \n",
       "4       billy14@long-warren.com   \n",
       "\n",
       "                                             address         country  \\\n",
       "0                       Zimmerstr. 1/0\\n59015 Gießen         Germany   \n",
       "1  Studio 22a\\nLynne terrace\\nMcCarthymouth\\nTF0 9GH  United Kingdom   \n",
       "2                 92 Ann drive\\nJoanborough\\nSK0 6LR  United Kingdom   \n",
       "3             19 Robinson meadow\\nNew Tracy\\nW22 2QG  United Kingdom   \n",
       "4              3 White pass\\nHunterborough\\nNN96 4UE  United Kingdom   \n",
       "\n",
       "  country_code       phone_number   join_date  \\\n",
       "0           DE   +49(0) 047905356  2018-10-10   \n",
       "1           GB    (0161) 496 0674  2001-12-20   \n",
       "2           GB  +44(0)121 4960340  2016-12-16   \n",
       "3           GB    (0306) 999 0871  2004-02-23   \n",
       "4           GB      0121 496 0225  2006-09-01   \n",
       "\n",
       "                              user_uuid  \n",
       "0  93caf182-e4e9-4c6e-bebb-60a1a9dcf9b8  \n",
       "1  8fe96c3a-d62d-4eb5-b313-cf12d9126a49  \n",
       "2  fc461df4-b919-48b2-909e-55c95a03fe6b  \n",
       "3  6104719f-ef14-4b09-bf04-fb0c4620acb0  \n",
       "4  9523a6d3-b2dd-4670-a51a-36aebc89f579  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>date_of_birth</th>\n",
       "      <th>company</th>\n",
       "      <th>email_address</th>\n",
       "      <th>address</th>\n",
       "      <th>country</th>\n",
       "      <th>country_code</th>\n",
       "      <th>phone_number</th>\n",
       "      <th>join_date</th>\n",
       "      <th>user_uuid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15315</th>\n",
       "      <td>14913</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>Jenkins</td>\n",
       "      <td>1943-08-09</td>\n",
       "      <td>Thornton, Carroll and Newman</td>\n",
       "      <td>s.jenkins@smith.com</td>\n",
       "      <td>Studio 41I\\nJones lodge\\nOliviaborough\\nE8 3DU</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>+44(0)292018946</td>\n",
       "      <td>2016-04-15</td>\n",
       "      <td>2bd3a12f-a92d-4cdd-b99c-fc70572db302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15316</th>\n",
       "      <td>14994</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>Smith</td>\n",
       "      <td>1948-08-20</td>\n",
       "      <td>Robinson-Harris</td>\n",
       "      <td>s.smith@smith.com</td>\n",
       "      <td>530 Young parkway\\nMillsfurt\\nL4G 7NX</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>+44(0)1144960977</td>\n",
       "      <td>2020-07-20</td>\n",
       "      <td>d234c04b-c07c-46a5-a902-526f91478ecc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15317</th>\n",
       "      <td>15012</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>Losekann</td>\n",
       "      <td>1940-10-09</td>\n",
       "      <td>Rosenow</td>\n",
       "      <td>s.losekann@smith.com</td>\n",
       "      <td>Viviane-Fritsch-Straße 3/5\\n15064 Bad Liebenwerda</td>\n",
       "      <td>Germany</td>\n",
       "      <td>DE</td>\n",
       "      <td>02984 08192</td>\n",
       "      <td>2021-03-07</td>\n",
       "      <td>1a0a8b7b-7c17-42d8-a946-8a85d5495651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15318</th>\n",
       "      <td>15269</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>Rivera</td>\n",
       "      <td>1952-06-04</td>\n",
       "      <td>Taylor, Fry and Jones</td>\n",
       "      <td>s.rivera@smith.com</td>\n",
       "      <td>660 Ross Falls Suite 357\\nAnthonymouth, MA 09610</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>239.711.3836</td>\n",
       "      <td>2011-01-03</td>\n",
       "      <td>187fe06e-bd5f-4381-af2f-d7ac37ca7572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15319</th>\n",
       "      <td>1249</td>\n",
       "      <td>Stephen</td>\n",
       "      <td>Duncan</td>\n",
       "      <td>1994-03-27</td>\n",
       "      <td>Phillips, Brown and Powell</td>\n",
       "      <td>s.duncan@smith.com</td>\n",
       "      <td>Studio 4\\nHancock road\\nPhillipsview\\nST1X 3XB</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>GB</td>\n",
       "      <td>+44(0)1314960870</td>\n",
       "      <td>2015-08-28</td>\n",
       "      <td>0589bbca-1d58-4b1f-9d0a-04ed4c57aaa1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       index first_name last_name date_of_birth                       company  \\\n",
       "15315  14913    Stephen   Jenkins    1943-08-09  Thornton, Carroll and Newman   \n",
       "15316  14994    Stephen     Smith    1948-08-20               Robinson-Harris   \n",
       "15317  15012    Stephen  Losekann    1940-10-09                       Rosenow   \n",
       "15318  15269    Stephen    Rivera    1952-06-04         Taylor, Fry and Jones   \n",
       "15319   1249    Stephen    Duncan    1994-03-27    Phillips, Brown and Powell   \n",
       "\n",
       "              email_address  \\\n",
       "15315   s.jenkins@smith.com   \n",
       "15316     s.smith@smith.com   \n",
       "15317  s.losekann@smith.com   \n",
       "15318    s.rivera@smith.com   \n",
       "15319    s.duncan@smith.com   \n",
       "\n",
       "                                                 address         country  \\\n",
       "15315     Studio 41I\\nJones lodge\\nOliviaborough\\nE8 3DU  United Kingdom   \n",
       "15316              530 Young parkway\\nMillsfurt\\nL4G 7NX  United Kingdom   \n",
       "15317  Viviane-Fritsch-Straße 3/5\\n15064 Bad Liebenwerda         Germany   \n",
       "15318   660 Ross Falls Suite 357\\nAnthonymouth, MA 09610   United States   \n",
       "15319     Studio 4\\nHancock road\\nPhillipsview\\nST1X 3XB  United Kingdom   \n",
       "\n",
       "      country_code      phone_number   join_date  \\\n",
       "15315           GB   +44(0)292018946  2016-04-15   \n",
       "15316           GB  +44(0)1144960977  2020-07-20   \n",
       "15317           DE       02984 08192  2021-03-07   \n",
       "15318           US      239.711.3836  2011-01-03   \n",
       "15319           GB  +44(0)1314960870  2015-08-28   \n",
       "\n",
       "                                  user_uuid  \n",
       "15315  2bd3a12f-a92d-4cdd-b99c-fc70572db302  \n",
       "15316  d234c04b-c07c-46a5-a902-526f91478ecc  \n",
       "15317  1a0a8b7b-7c17-42d8-a946-8a85d5495651  \n",
       "15318  187fe06e-bd5f-4381-af2f-d7ac37ca7572  \n",
       "15319  0589bbca-1d58-4b1f-9d0a-04ed4c57aaa1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 15320 entries, 0 to 15319\n",
      "Data columns (total 12 columns):\n",
      " #   Column         Non-Null Count  Dtype \n",
      "---  ------         --------------  ----- \n",
      " 0   index          15320 non-null  int64 \n",
      " 1   first_name     15320 non-null  object\n",
      " 2   last_name      15320 non-null  object\n",
      " 3   date_of_birth  15320 non-null  object\n",
      " 4   company        15320 non-null  object\n",
      " 5   email_address  15320 non-null  object\n",
      " 6   address        15320 non-null  object\n",
      " 7   country        15320 non-null  object\n",
      " 8   country_code   15320 non-null  object\n",
      " 9   phone_number   15320 non-null  object\n",
      " 10  join_date      15320 non-null  object\n",
      " 11  user_uuid      15320 non-null  object\n",
      "dtypes: int64(1), object(11)\n",
      "memory usage: 1.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index             int64\n",
       "first_name       object\n",
       "last_name        object\n",
       "date_of_birth    object\n",
       "company          object\n",
       "email_address    object\n",
       "address          object\n",
       "country          object\n",
       "country_code     object\n",
       "phone_number     object\n",
       "join_date        object\n",
       "user_uuid        object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "index seems to be an integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['index'] = df['index'].astype('int64')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have some date columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['date_of_birth']=pd.to_datetime(df['date_of_birth'],infer_datetime_format=True, errors='coerce')\n",
    "df['join_date']=pd.to_datetime(df['join_date'],infer_datetime_format=True, errors='coerce')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "inspect some potentially categorical data:\n",
    "Country\n",
    "Country Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count              15320\n",
       "unique                19\n",
       "top       United Kingdom\n",
       "freq                9371\n",
       "Name: country, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"country\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     15320\n",
       "unique       20\n",
       "top          GB\n",
       "freq       9365\n",
       "Name: country_code, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"country_code\"].describe()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not bad, though there's 19 countries and 20 codes!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"country\"] = df[\"country\"].astype(\"category\")\n",
    "df[\"country_code\"] = df[\"country_code\"].astype(\"category\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Companies as categories?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count         15320\n",
       "unique        12105\n",
       "top       Smith Inc\n",
       "freq             22\n",
       "Name: company, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"company\"].describe()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nope. What about duplicate lines?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "None! ok. Let's just check on the critical columns only - not system generated ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated(subset=[\"first_name\", \"last_name\", \"date_of_birth\", \"country\"],keep=\"first\").sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "20 duplicates. There's nothing in the other columns to average or anything like that - hopefully dropping 20 won't kill us."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 15300 entries, 0 to 15319\n",
      "Data columns (total 12 columns):\n",
      " #   Column         Non-Null Count  Dtype         \n",
      "---  ------         --------------  -----         \n",
      " 0   index          15300 non-null  int64         \n",
      " 1   first_name     15300 non-null  object        \n",
      " 2   last_name      15300 non-null  object        \n",
      " 3   date_of_birth  15284 non-null  datetime64[ns]\n",
      " 4   company        15300 non-null  object        \n",
      " 5   email_address  15300 non-null  object        \n",
      " 6   address        15300 non-null  object        \n",
      " 7   country        15300 non-null  category      \n",
      " 8   country_code   15300 non-null  category      \n",
      " 9   phone_number   15300 non-null  object        \n",
      " 10  join_date      15284 non-null  datetime64[ns]\n",
      " 11  user_uuid      15300 non-null  object        \n",
      "dtypes: category(2), datetime64[ns](2), int64(1), object(7)\n",
      "memory usage: 1.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df.drop_duplicates(subset=[\"first_name\", \"last_name\", \"date_of_birth\", \"country\"],keep=\"first\",inplace=True)\n",
    "df.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "standardise the phone numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardise_phone_number(number: str):\n",
    "    if number[:1] == \"+\" :\n",
    "        number = number[1:]\n",
    "    number = number.replace(\" \",\"\")\n",
    "    number = number.replace(\"-\",\"\")\n",
    "\n",
    "    if number[:2] != \"00\":\n",
    "        number = \"00\" + number\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    row[\"phone_number\"] = standardise_phone_number(row[\"phone_number\"])\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Should I do something about the null values? What about email parsing with regex?\n",
    "what about that index column and the uuid one - are they still relevant?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now put the code into a local database maybe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "myObject.upload_to_db(df,\"dim_users\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ideas for future:\n",
    "Misspellings: https://medium.com/geekculture/how-to-fix-misspelled-words-for-your-next-nlp-project-with-one-line-of-code-98888a5ff207\n",
    "Common variants of companies - \"ltd.\" vs \"ltd\" vs \"limited\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "retail-data",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}