notebook_sample_dataset_normalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Install required packages\n",
    "!pip install lshkrepresentatives numpy scikit-learn pandas kmodes networkx termcolor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from LSHkRepresentatives.LSHkRepresentatives import CategoricalDatasetNormalization,LSHkRepresentatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Example of non normalized categorical dataset (there are strings, numbers, and nan values)\n",
    "X = np.array([['a',np.nan,45],\n",
    "              [0,1,1],\n",
    "              ['g',0,0],\n",
    "              [1,0,1],\n",
    "              ['g',2,2],\n",
    "              [2,3,2],\n",
    "              ['fg',3,2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3, 4, 3],\n",
       "       [0, 1, 1],\n",
       "       [5, 0, 0],\n",
       "       [1, 0, 1],\n",
       "       [5, 2, 2],\n",
       "       [2, 3, 2],\n",
       "       [4, 3, 2]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Normalize it by CategoricalDatasetNormalization instance\n",
    "normalizer = CategoricalDatasetNormalization(X)\n",
    "X_categorical = normalizer.Normalize(X)\n",
    "X_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SKIP LOADING distMatrix because IS_LOAD_AUTO=False or dbname=None; If you cluster a dataset multiple times, make sure to include the 'dbname' parameter during initialization to cache the distance matrix of the dataset. Calculating distance matrix can take a lot of time with high categirical values dataset.\n",
      "Generating disMatrix for DILCA\n",
      "Saving DILCA to: saved_dist_matrices/json/DILCA_None.json\n",
      "Generating LSH hash table:   hbits: 2(4)  k 1  d 3  n= 7\n",
      "LSH time: 0.008334099999046884 Score:  11.333333333333334  Time: 0.00031906000076560305\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 1, 1, 1], dtype=uint16)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Do clustering \n",
    "kreps = LSHkRepresentatives(n_clusters=2,n_init=5) \n",
    "kreps.fit(X_categorical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3, 2, 0],\n",
       "       [2, 2, 2]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Normlize test/query dataset using the trained normalizer\n",
    "x2 = np.array([['a',2,0],[2,2,2]])\n",
    "x2_normalized = normalizer.Normalize(x2)\n",
    "x2_normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int64)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Predic the normalized test/query \n",
    "labels_test = kreps.predict(x2_normalized)\n",
    "labels_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}