nlp_workshop_participant.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In your own words: Computing customer similarity using website text data\n",
    "#### Workshop developed for DSS Austin '19\n",
    "#### By: Ben Batorsky [github](https://github.com/bpben)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import NMF, TruncatedSVD\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from collections import Counter\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optional, can run exercise code without these\n",
    "# uncomment system commands for colab notebook\n",
    "#!pip install PyStemmer\n",
    "import Stemmer\n",
    "stemmer = Stemmer.Stemmer('english')\n",
    "#!pip install spacy\n",
    "import spacy\n",
    "#!python -m spacy download en_core_web_sm\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "#!pip install gensim\n",
    "from gensim.models.phrases import Phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if you use colab, modify this\n",
    "DATA_PATH = './data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data ingestion\n",
    "The data for the workshop comes from a random set of business website text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_data = pd.read_csv(DATA_PATH+'website_text.csv')\n",
    "w_content = full_data['content']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# capitalization\n",
    "text = \"We're Cowboys fans, but we're not cowboys\"\n",
    "print(Counter(text.split()))\n",
    "print(Counter(text.lower().split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# punctuation\n",
    "text = \"We're Cowboys fans, but we're not cowboys\"\n",
    "strip_punct = '[^A-Za-z0-9 ]'\n",
    "print(text)\n",
    "print(re.sub(strip_punct, '', text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numbers\n",
    "text = 'Call 867-5309'\n",
    "strip_num = '[0-9]'\n",
    "print(text)\n",
    "print(re.sub(strip_num, '', text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# urls\n",
    "# regex from textacy: https://github.com/chartbeat-labs/textacy\n",
    "SHORT_URL_REGEX = re.compile(\n",
    "    r\"(?:^|(?<![\\w/.]))\"\n",
    "    # optional scheme\n",
    "    r\"(?:(?:https?://)?)\"\n",
    "    # domain\n",
    "    r\"(?:\\w-?)*?\\w+(?:\\.[a-z]{2,12}){1,3}\"\n",
    "    r\"/+\"\n",
    "    # hash\n",
    "    r\"[^\\s.,?!'\\\"|+]{2,12}\"\n",
    "    r\"(?:$|(?![\\w?!+&/]))\",\n",
    "    flags=re.IGNORECASE)\n",
    "text = 'Check out this conference: https://datascience.salon/austin/'\n",
    "print(text)\n",
    "print(SHORT_URL_REGEX.sub('', text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Write your preprocessing script\n",
    "Combine some of the regex expressions (or write your own!) to process the text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    ## code here\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_processed = w_content.apply(preprocess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From text to vectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n-grams\n",
    "# note: 1-letter words are dropped by default\n",
    "text = ['I have a lovely bunch of coconuts']\n",
    "for n in range(1, 4):\n",
    "    vec = CountVectorizer(ngram_range=(1, n))\n",
    "    print('1 to {}-grams: '.format(n), list(vec.fit(text).vocabulary_.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choosing cutoffs\n",
    "texts = ['I have a lovely bunch of coconuts']\n",
    "texts = texts*9\n",
    "texts.append('I have a lovely bunch of pears')\n",
    "vec = CountVectorizer()\n",
    "print('Default (no minimum): ', list(vec.fit(texts).vocabulary_.keys()))\n",
    "vec = CountVectorizer(min_df=.2)\n",
    "print('Appear in >=20% of docs: ', list(vec.fit(texts).vocabulary_.keys()))\n",
    "vec = CountVectorizer(max_df=.1)\n",
    "print('Appear in <=10% of docs: ', list(vec.fit(texts).vocabulary_.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stemming and lemmatizing\n",
    "words = ['ponies', 'operation', 'are']\n",
    "for w in words:\n",
    "    print('Stem of {}: {}'.format(w, stemmer.stemWord(w)))\n",
    "    print('Lemma of {}: {}'.format(w, nlp(w)[0].lemma_))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# entities with spaCy\n",
    "text = \"I'm a Cowboys fan, but I'm not a cowboy\"\n",
    "ents = nlp(text).ents\n",
    "for e in ents:\n",
    "    print(e, e.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# entities with gensim - co-location\n",
    "# This example is a bit odd: Likely not dealing with a bunch of duplicates\n",
    "# Worth noting that, all likelihood being equal, gensim picks the first in a series of bigrams\n",
    "texts = ['have a lovely bunch of coconuts']\n",
    "texts = texts*4\n",
    "texts.append('have a lovely bunch of pears')\n",
    "texts.append('have a lovely bunch of pears')\n",
    "split_texts = [x.split() for x in texts]\n",
    "phrases = Phrases(split_texts, min_count=1, threshold=1)\n",
    "phrases_m2 = Phrases(split_texts, min_count=2, threshold=1)\n",
    "# scores are based on https://radimrehurek.com/gensim/models/phrases.html#gensim.models.phrases.original_scorer\n",
    "# higher = more likely to be a bigram\n",
    "print('with min count = 1:') \n",
    "print([x for x in phrases.export_phrases([split_texts[-1]])])\n",
    "print('with min count = 2')\n",
    "print([x for x in phrases_m2.export_phrases([split_texts[-1]])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# creating count vectors\n",
    "texts = []\n",
    "text = \"We are Cowboys fans, but we are not cowboys\"\n",
    "text_tagged = \"We are CowboysORG fans, but we are not cowboys\"\n",
    "# phrase without tags, lowercase\n",
    "texts.append(text.lower())\n",
    "# phrase with tags, lowercase\n",
    "texts.append(text_tagged.lower())\n",
    "# utility to display vectorizer\n",
    "def display_vec(vec, data):\n",
    "    df = pd.DataFrame(data.toarray(),\n",
    "                     columns=vec.get_feature_names())\n",
    "    return(df)\n",
    "# count vector\n",
    "vec = CountVectorizer()\n",
    "data = vec.fit_transform(texts)\n",
    "print('count vectors \\n', display_vec(vec, data))\n",
    "# binary count vector\n",
    "b_vec = CountVectorizer(binary=True)\n",
    "data = b_vec.fit_transform(texts)\n",
    "print('binary vectors \\n', display_vec(b_vec, data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Remove the stopwords from the above texts\n",
    "Use what we explored above to remove the stopwords from the count vectors of the following texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['we are cowboys fans, but we are not cowboys',\n",
    " 'we are cowboysorg fans, but we are not cowboys']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Set limits on the vocabulary to remove potentially irrelevant words\n",
    "With the following set of texts, set a limit to remove unimportant words like \"Patriots\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['We are Cowboys fans',\n",
    "        'We are cowboys',\n",
    "        'We are Patriots fans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF-IDF weighting\n",
    "TfidfVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['We are Cowboys fans',\n",
    "         'We are Patriots fans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate term frequency\n",
    "vec = CountVectorizer()\n",
    "count_vectors = vec.fit_transform(texts)\n",
    "count_df = display_vec(vec, count_vectors)\n",
    "print(count_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Formula for inverse document frequency weight:\n",
    "\n",
    "$$log(\\frac{N}{df(t)}) + 1$$\n",
    "\n",
    "\"smooth\" option ensures no zero-divisions:\n",
    "\n",
    "$$log(\\frac{N+1}{df(t)+1}) + 1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get inverse document frequency\n",
    "df = np.log(3/(1+count_df.sum()))+1\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate tfidf\n",
    "tfidf_df = count_df*df\n",
    "# normalize: default in scikit-learn accounts for different-length documents\n",
    "print(tfidf_df.apply(\n",
    "    lambda x: x/np.sqrt(x.dot(x)), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now with scikit-learn\n",
    "tfidf = TfidfVectorizer()\n",
    "data = tfidf.fit_transform(texts)\n",
    "print(display_vec(tfidf, data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Turn text to vectors\n",
    "Using what we've gone through above, create your own count vectorizer and TFIDF vectorizer.  Apply these vectorizers to the data, and store the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_params = {'min_df': .005, 'max_df': .3, 'stop_words':'english'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer = CountVectorizer(**vector_params)\n",
    "tfidf_vectorizer = TfidfVectorizer(**vector_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix factorization and topic modelling\n",
    "\n",
    "#### Latent Semantic Indexing\n",
    "In scikit-learn this is implemented as TruncatedSVD, a version of SVD where the top k elements are retained\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TruncatedSVD()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# couple examples website text\n",
    "# choose some from pretty opposite industries\n",
    "random_state = 9\n",
    "example_inds = ['Health and Fitness', 'Home & Home Improvement']\n",
    "example_idxs = []\n",
    "for ind in example_inds:\n",
    "    ind_data = full_data[full_data.type==ind]\n",
    "    idxs = ind_data.sample(n=2, random_state=random_state).index\n",
    "    example_idxs.extend(idxs.tolist())\n",
    "example_texts = w_content.loc[example_idxs]\n",
    "example_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSI requires tfidf-weighted vectors, use from above\n",
    "tfidf_example = tfidf_vectorizer.transform(example_texts)\n",
    "# create display of examples\n",
    "display_example = display_vec(tfidf_vectorizer, tfidf_example)\n",
    "# for clarity, drop vocab that does not occur\n",
    "display_example.loc[:, (display_example.sum(axis=0)>0).values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_components(model, word_features, top_display=5):\n",
    "    # utility for displaying respresentative words per component\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic %d:\" % (topic_idx))\n",
    "        top_words_idx = topic.argsort()[::-1][:top_display]\n",
    "        top_words = [word_features[i] for i in top_words_idx]\n",
    "        print(\" \".join(top_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify number of components\n",
    "n_components = 4\n",
    "svd = TruncatedSVD(n_components=n_components, random_state=random_state)\n",
    "svd_example = svd.fit_transform(tfidf_example)\n",
    "display_components(svd, tfidf_vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for display\n",
    "pd.DataFrame(svd_example,\n",
    "             index=[t[:50] for t in example_texts])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Create LSI vectors\n",
    "Using the TFIDF vectors from above, create LSI vectors for the website text data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# likely better to use more than 4 components\n",
    "n_components = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Non-negative matrix factorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NMF()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(example_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NMF also requires tfidf-weighted vectors\n",
    "tfidf_example = tfidf_vectorizer.transform(example_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify number of components\n",
    "# with NMF, n_components must be <= number of documents\n",
    "n_components = 4\n",
    "nmf = NMF(n_components=n_components)\n",
    "nmf_example = nmf.fit_transform(tfidf_example)\n",
    "display_components(nmf, tfidf_vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(nmf_example,\n",
    "             index=[t[:50] for t in example_texts])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Create NMF vectors\n",
    "Using the TFIDF vectors, create NMF vectors for the website text data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# likely better to use more than 4 components\n",
    "n_components = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing similarity\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# taking inventory of the vectors we have\n",
    "vector_sets = {'count':count_vecs,\n",
    "               'tfidf':tfidf_vecs,\n",
    "               'lsi':lsi_vecs,\n",
    "               'nmf':nmf_vecs}\n",
    "for k, v in vector_sets.items():\n",
    "    print(k, 'shape:',  v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cosine similarity\n",
    "# looking at our examples from above\n",
    "print(example_texts)\n",
    "print('tfidf shape:', tfidf_example.shape)\n",
    "example_sim = cosine_similarity(tfidf_example)\n",
    "# truncate descriptions\n",
    "trunc_example_texts = [x[:20] for x in example_texts.values]\n",
    "pd.DataFrame(example_sim,\n",
    "             index=trunc_example_texts,\n",
    "             columns=trunc_example_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# have industry category for subset of businesses\n",
    "full_data.type.value_counts(dropna=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Which of the four techniques appears to work best?\n",
    "For this more open-ended question, here's a suggestion for a workflow:\n",
    "\n",
    "1) Take inventory of available vectorized data\n",
    "\n",
    "2) Assess sources for \"ground truth\"\n",
    "\n",
    "3) Determine a metric of performance for the techniques\n",
    "\n",
    "4) Analyze and compare"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:devpy3]",
   "language": "python",
   "name": "conda-env-devpy3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}