universal-sentence-encoder-clustering-poc.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Universal Sentence Encoder Clustering POC\n_by Nick Brooks, Feb 2020_\n\n**Resources:** <br>\n- [Word Embeddings Guide Tensorflow](https://www.tensorflow.org/tutorials/text/word_embeddings)\n- [Universal Sentence Encoder Tensorflow Hub](https://tfhub.dev/google/universal-sentence-encoder-large/5)\n\n\n**Similar Notebooks:** <br>\n- [Universal Sentence Encoder Semantic Similarity](https://www.kaggle.com/nicapotato/universal-sentence-encoder-semantic-similarity/)\n- [Clasify Tweets with BERT](https://www.kaggle.com/nicapotato/bert-oof-with-dense-features)\n\n### Table of Content\n1. Universal Sentence Encoder Clustering\n1. Universal Sentence Encoder Fine-Tuning"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport tensorflow_hub as hub\nfrom sklearn.cluster import KMeans\nimport time\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold\n\npd.options.display.max_colwidth = 500\nprint(\"Tensorflow Hub Version: \", hub.__version__)\n\nSEED = 42\n\nnotebookstart = time.time()\nmodule_url = 'https://tfhub.dev/google/universal-sentence-encoder-large/4'\nUSE_embed = hub.KerasLayer(module_url, trainable=False, name='USE_embedding')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Prepare Toy Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"toy_data = [\n    [\"My flight got cancelled and I didn't get a refund.\", \"travel\"],\n    [\"The pilot said the delays were due to ice on the wings.\", \"travel\"],\n    [\"The airport was closed due to a terrorist threat.\", \"travel\"],\n    [\"The plane coudn't take off, so the airline booked us into the Marriott nearby.\", \"travel\"],\n    [\"Room service was friendly and professional, I will definitely be back!\", \"hotel\"],\n    [\"Hotel was having a huge function and I had no room to enjoy the facilities.\", \"hotel\"],\n    [\"I was charged 10$ for a water in the mini fridge, ridiculous!!!\", \"hotel\"],\n    [\"The soccer and basketball events were badly organised.\", \"activities\"],\n    [\"I wish that they would offer surfing in the itinerary, the weather was perfect for it.\", \"activities\"],\n    [\"I swim at 8 AM every day to train for the competition\", \"activities\"],\n    [\"Lets get a together an plan a giant ski trip in France\", \"activities\"],\n    [\"Today is gonna be the day that we're gonna throw it back to you.\", \"other\"],\n    [\"I wish the duty free stores had more liquor options\", \"travel\"],\n    [\"There was no more room at the gate, so I was forced to stand up for 30 minutes\", \"travel\"],\n    [\"The airport security held me up for a petty reason and wasted my time\",\"travel\"],\n    [\"I had a great experience at the Aspire Lounge, I really enjoyed the food\", \"travel\"],\n    [\"I was once again unable to enter the lounge, I was turned down due to capacity\",\"travel\"],\n    [\"Flights prices during the holiday are way to high, this is outrageous.\", \"travel\"],\n    [\"Prices on this website seem to change every 10 seconds, I don't like it.\", \"travel\"],\n    [\"I had a hard time finding the lounge, I am thankful for the Priority Pass navigation system\", \"travel\"],\n    [\"This hotel was very full over the weekend, I wish they had more space\", \"hotel\"],\n    [\"I was able to check into my room in 5 minutes, super easy\", \"hotel\"],\n    [\"This hotel was full of corporate types, they ruined my holiday.\", \"hotel\"],\n    [\"I loved my UCPA ski holiday, the food was great, and I learned lots of snowboarding tricks\", \"activities\"],\n    [\"Next time I go to the beach, I will definitely try to surf\", \"activities\"]\n]\n\ndf = pd.DataFrame(toy_data, columns = ['review','category'])\nprint(df.category.value_counts().to_dict())\nprint(\"\")\ndisplay(df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Fit Universal Sentence Encoder and Fit Kmeans Algorithm"},{"metadata":{"trusted":true},"cell_type":"code","source":"train, validation = train_test_split(df.values, test_size=0.40, random_state=23)\ntrain = pd.DataFrame(train, columns = df.columns)\nvalidation = pd.DataFrame(validation, columns = df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_embeddings = USE_embed(train.review)['outputs'].numpy()\n\nkmeans = KMeans(n_clusters=3, random_state=SEED).fit(train_embeddings)\ntrain.loc[:,'Clusters'] = None\ntrain.loc[:,'Clusters'] = kmeans.labels_\n\ndisplay(train.sort_values(by = 'category'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(train.category, kmeans.labels_))\nprint(\"Completeness: %0.3f\" % metrics.completeness_score(train.category, kmeans.labels_))\nprint(\"V-measure: %0.3f\" % metrics.v_measure_score(train.category, kmeans.labels_))\nprint(\"Adjusted Rand-Index: %.3f\"\n      % metrics.adjusted_rand_score(train.category, kmeans.labels_))\nprint(\"Silhouette Coefficient: %0.3f\"\n      % metrics.silhouette_score(train_embeddings, train.category, sample_size=1000))\n\ndisplay(pd.crosstab(train['Clusters'], train['category']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Predict on New cases.."},{"metadata":{"trusted":true},"cell_type":"code","source":"def kmeans_predict(text_input, fitted_kmeans):\n    tmp_embd = USE_embed([text_input])['outputs'].numpy()\n    return fitted_kmeans.predict(tmp_embd)[0]\n\nvalidation[\"Clusters\"] = validation.review.apply(lambda t: kmeans_predict(text_input=t,fitted_kmeans=kmeans))\ndisplay(pd.crosstab(validation['Clusters'], validation['category']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fine Tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nfrom keras import backend as K\n\nfrom tensorflow.keras.layers import Dense, Input, BatchNormalization, Dropout, concatenate, GlobalAveragePooling1D\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.optimizers import Adam, SGD\nfrom keras.utils import to_categorical\nimport itertools\n\nimport matplotlib.pyplot as plt\n\nprint(\"Tensorflow Version: \", tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['labels'] = df['category'].map({\"travel\":0,\"hotel\":1,\"activities\":2})\nclass_names = [\"trave\",\"hotel\",\"activities\"]\ntrain_data = df.loc[df.labels.notnull(),:].reset_index().copy()\ntrain_data['labels'] = train_data['labels'].astype(int)\ntrain_data['review'] = train_data['review'].astype(str)\n\ncategorical_labels = to_categorical(train_data['labels'].values, num_classes=None)\noutput_size = categorical_labels.shape[1]\n\nprint('Inpus Shape: {}, Output Shape: {}'.format(train_data['review'].shape, categorical_labels.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(embed, output_size=output_size):\n    \n    model = Sequential([\n        Input(shape=[], dtype=tf.string),\n        embed,\n        Dense(output_size, activation='softmax')\n    ])\n    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n    \n    return model\n\nmodel = build_model(USE_embed)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds = np.zeros([train_data.shape[0], output_size])\n\nn_splits = 3\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=SEED)\nplot_metrics = ['loss','accuracy']\n\nfold_hist = {}\nfor i, (trn_idx, val_idx) in enumerate(folds.split(train_data)):\n    modelstart = time.time()\n    model = build_model(USE_embed)\n    \n    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,\n                                 mode='min', baseline=None, restore_best_weights=True)\n    \n    history = model.fit(\n        train_data.review[trn_idx].values,\n        categorical_labels[trn_idx],\n        validation_data=(\n            train_data.review[val_idx].values,\n            categorical_labels[val_idx]),\n        epochs=40,\n        batch_size=6,\n        callbacks = [es],\n        verbose=1)\n    \n    best_index = np.argmin(history.history['val_loss'])\n    fold_hist[i] = history\n    \n    oof_preds[val_idx] = model.predict(train_data.review[val_idx].values)\n    \n    f, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])\n    for p_i,metric in enumerate(plot_metrics):\n        ax[p_i].plot(history.history[metric], label='Train ' + metric)\n        ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)\n        ax[p_i].set_title(\"{} Fold Loss Curve - {}\\nBest Epoch {}\".format(i, metric, best_index))\n        ax[p_i].legend()\n        ax[p_i].axvline(x=best_index, c='black')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confusion matrix \ndef plot_confusion_matrix(cm, classes,\n                          title = 'Confusion matrix\"',\n                          cmap = plt.cm.Blues) :\n    plt.imshow(cm, interpolation = 'nearest', cmap = cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation = 0)\n    plt.yticks(tick_marks, classes)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])) :\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment = 'center',\n                 color = 'white' if cm[i, j] > thresh else 'black')\n \n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = oof_preds.argmax(axis = 1)\nprint(\"Accuracy: {:.2f}\".format(metrics.accuracy_score(train_data['labels'], preds)))\nprint(\"Confusion Matrix\")\nplot_confusion_matrix(metrics.confusion_matrix(train_data['labels'], preds),\n                          class_names,\n                          title = 'Confusion matrix\"',\n                          cmap = plt.cm.Blues)\ncm = metrics.confusion_matrix(train_data['labels'], preds, normalize = 'true').round(3)\nplot_confusion_matrix(cm,\n                          class_names,\n                          title = 'Normalised Confusion matrix\"',\n                          cmap = plt.cm.Blues)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}