Tutorial 1 - Word Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial 1: Word Embeddings\n",
    "## Set-Up\n",
    "1. Follow the instructions [here](https://github.com/alligilmore-sum/sum_data_science/blob/master/setup_instructions.md) if you haven't already.\n",
    "2. Open your Terminal and run: `pip install spacy`\n",
    "\n",
    "## References\n",
    "[spacy documentation](https://spacy.io/docs/usage/word-vectors-similarities)\n",
    "\n",
    "[GloVe paper](https://nlp.stanford.edu/pubs/glove.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import numpy as np\n",
    "import en_vectors_glove_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nlp = en_vectors_glove_md.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# See the vectors\n",
    "1. Try tab-completion on the objects below.\n",
    "2. Define objects for your own words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'mega-operas'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab[111111].orth_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apples, lions, oranges = nlp(u'apples'), nlp(u'lions'), nlp(u'oranges')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300L,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apples.vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.63340002,  0.18980999, -0.53544003, -0.52657998, -0.30001   ,\n",
       "        0.30559   , -0.49303001,  0.14636   ,  0.012273  ,  0.96802002,\n",
       "        0.0040354 ,  0.25233999, -0.29864001, -0.014646  , -0.24905001,\n",
       "       -0.67124999, -0.053366  ,  0.59425998, -0.068034  ,  0.10315   ,\n",
       "        0.66759002,  0.024617  , -0.37548   ,  0.52556998,  0.054449  ,\n",
       "       -0.36748001, -0.28013   ,  0.090898  , -0.025687  , -0.59469998,\n",
       "       -0.24269   ,  0.28602999,  0.68599999,  0.29736999,  0.30421999,\n",
       "        0.69032001,  0.042784  ,  0.023701  , -0.57165003,  0.70581001,\n",
       "       -0.20813   , -0.03204   , -0.12494   , -0.42932999,  0.31270999,\n",
       "        0.30351999,  0.09421   , -0.15493   ,  0.071356  ,  0.15022001,\n",
       "       -0.41791999,  0.066394  , -0.034546  , -0.45772001,  0.57177001,\n",
       "       -0.82754999, -0.27884999,  0.71801001, -0.12425   ,  0.18550999,\n",
       "        0.41341999, -0.53996998,  0.55864   , -0.015805  , -0.1074    ,\n",
       "       -0.29980999, -0.17271   ,  0.27066001,  0.043996  ,  0.60106999,\n",
       "       -0.35299999,  0.68309999,  0.20703   ,  0.12068   ,  0.24852   ,\n",
       "       -0.15605   ,  0.25812   ,  0.007004  , -0.10741   , -0.097053  ,\n",
       "        0.085628  ,  0.096307  ,  0.20857   , -0.23338   , -0.077905  ,\n",
       "       -0.030906  ,  1.04939997,  0.55368   , -0.10703   ,  0.052234  ,\n",
       "        0.43406999, -0.13925999,  0.38115001,  0.021104  , -0.40922001,\n",
       "        0.35971999, -0.28898001,  0.30618   ,  0.060807  , -0.023517  ,\n",
       "        0.58192998, -0.3098    ,  0.21013001, -0.15557   , -0.56913   ,\n",
       "       -1.13639998,  0.36598   , -0.032666  ,  1.19260001,  0.12825   ,\n",
       "       -0.090486  , -0.47964999, -0.61163998, -0.16484   , -0.41134   ,\n",
       "        0.19925   ,  0.059183  , -0.20841999,  0.45223001,  0.27697   ,\n",
       "       -0.20745   ,  0.025404  , -0.28874001,  0.040478  , -0.22274999,\n",
       "       -0.43323001,  0.76956999, -0.054327  , -0.35213   , -0.30842   ,\n",
       "       -0.48791   , -0.35563999,  0.19813   , -0.094767  , -0.50918001,\n",
       "        0.18763   , -0.087555  ,  0.37709001, -0.1322    , -0.096913  ,\n",
       "       -1.9102    ,  0.55813003,  0.27390999, -0.077744  , -0.43933001,\n",
       "       -0.10367   , -0.24408001,  0.41869   ,  0.11659   ,  0.27454001,\n",
       "        0.81020999, -0.11006   ,  0.43131   ,  0.29095   , -0.49548   ,\n",
       "       -0.31957999, -0.072506  ,  0.020286  ,  0.21789999,  0.22032   ,\n",
       "       -0.29212001,  0.75638998,  0.13598   ,  0.019736  , -0.83104002,\n",
       "        0.22836   , -0.28669   , -1.05289996,  0.052771  ,  0.41266   ,\n",
       "        0.50149   ,  0.5323    ,  0.51573002, -0.31806001, -0.4619    ,\n",
       "        0.21739   , -0.43584001, -0.41382   ,  0.042237  , -0.57178998,\n",
       "        0.067623  , -0.27853999,  0.090044  ,  0.20633   ,  0.024678  ,\n",
       "       -0.57703   , -0.020183  , -0.53147   , -0.37548   , -0.12795   ,\n",
       "       -0.093662  , -0.0061183 ,  0.20220999, -0.62295997, -0.29745999,\n",
       "        0.26934999,  0.59008998, -0.50382   , -0.69757003,  0.20157   ,\n",
       "       -0.33592001, -0.45765999,  0.14060999,  0.22982   ,  0.044046  ,\n",
       "        0.26385999,  0.02942   ,  0.34095001,  1.14960003, -0.15555   ,\n",
       "       -0.064071  ,  0.30138999,  0.024211  , -0.63515002, -0.73347002,\n",
       "       -0.10346   , -0.22637001, -0.056392  , -0.16734999, -0.097331  ,\n",
       "       -0.19205999, -0.18866   ,  0.15116   , -0.038048  ,  0.70204997,\n",
       "        0.11586   , -0.14813   ,  0.0095166 , -0.33803999, -0.10158   ,\n",
       "       -0.23829   , -0.22758999,  0.092504  , -0.29839   , -0.39721   ,\n",
       "        0.26091999,  0.34593999, -0.47396001, -0.25725001, -0.19257   ,\n",
       "       -0.53070998,  0.1692    , -0.47251999, -0.17332999, -0.40505001,\n",
       "        0.046446  , -0.04473   ,  0.33555001, -0.5693    ,  0.31591001,\n",
       "       -0.21167   , -0.31298   , -0.45923001, -0.083091  ,  0.086822  ,\n",
       "        0.01264   ,  0.43779001,  0.12650999,  0.30156001,  0.022061  ,\n",
       "        0.26549   , -0.29455   , -0.14838   ,  0.033692  , -0.37345999,\n",
       "       -0.075343  , -0.56497997, -0.24207   , -0.69351   , -0.20276999,\n",
       "       -0.0081185 ,  0.030971  ,  0.53614998, -0.16613001, -0.84087002,\n",
       "        0.74660999,  0.029132  ,  0.46935999, -0.49755001,  0.40954   ,\n",
       "       -0.022558  ,  0.21496999, -0.049528  , -0.039799  ,  0.46165001,\n",
       "        0.26456001,  0.32984999, -0.04219   , -0.099599  , -0.17312001,\n",
       "       -0.47600001, -0.019048  , -0.41887999, -0.2685    , -0.65280998,\n",
       "        0.068773  , -0.23881   , -1.17840004,  0.25503999,  0.61171001], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apples.vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Measure Cosine Similarity\n",
    "1. Check that the similarity method is computing cosine similarity from the expected definition.\n",
    "2. Try measuring similarity of some familiar words to convince yourself that cosine similarity captures semantic similarity in this vector space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.77809423"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apples_dot_oranges = np.dot(apples.vector, oranges.vector)\n",
    "apples_dot_apples = np.dot(apples.vector, apples.vector)\n",
    "oranges_dot_oranges = np.dot(oranges.vector, oranges.vector)\n",
    "apples_dot_oranges/(np.sqrt(apples_dot_apples)*np.sqrt(oranges_dot_oranges))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.77809419767910315"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apples.similarity(oranges)\n",
    "# Notice the difference in precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36175017677213472"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apples.similarity(nlp(u'fish'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lions, tigers, bears, pianos = nlp(u'lions tigers bears pianos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78141738771626912"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lions.similarity(tigers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11503109708333417"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lions.similarity(pianos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find Similar Words\n",
    "1. Try the most_similar_to_word() and least_similar_to_word() functions below on your own sample words.\n",
    "2. Modify these functions to return a specified number of similar words using a new argument called \"top_n\".\n",
    "3. Run some experiments to see how well the embedding handles words with ambiguous meanings.\n",
    "4. Run some experiments to see how well the embedding handles words from specialized vocabularies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_words = [w for w in nlp.vocab if w.orth_.islower()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def most_similar_to_word(target_word, vocab=[w for w in nlp.vocab if w.orth_.islower()]):\n",
    "    \"\"\"Find the top_n most similar words to target_word in vocab.\n",
    "        target_word : str\n",
    "        vocab : list\n",
    "        top_n : int\"\"\"\n",
    "    vocab.sort(key=lambda x: x.similarity(nlp(unicode(target_word))))\n",
    "    vocab.reverse()\n",
    "    print \"Top %d most similar words to %s:\" % (10, target_word)\n",
    "    # ignore the first when taking top n, since a word is most similar to itself\n",
    "    for word in vocab[1:11]:   \n",
    "        print(word.orth_)\n",
    "#     return vocab[1:11]\n",
    "\n",
    "def least_similar_to_word(target_word, vocab=[w for w in nlp.vocab if w.orth_.islower()]):\n",
    "    \"\"\"Find the top_n most similar words to target_word in vocab.\n",
    "        target_word : str\n",
    "        vocab : list\n",
    "        top_n : int\"\"\"\n",
    "    vocab.sort(key=lambda x: x.similarity(nlp(unicode(target_word))))\n",
    "    print \"Top %d least similar words to %s:\" % (10, target_word)\n",
    "    for word in vocab[:10]:   \n",
    "        print(word.orth_)\n",
    "#     return vocab[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 10 most similar words to apples:\n",
      "pears\n",
      "peaches\n",
      "oranges\n",
      "strawberries\n",
      "cherries\n",
      "apple\n",
      "bananas\n",
      "carrots\n",
      "apricots\n",
      "plums\n",
      "\n",
      "\n",
      "Top 10 least similar words to apples:\n",
      "incivil\n",
      "bkm\n",
      "leporello\n",
      "neo-modern\n",
      "extraordinario\n",
      "responsum\n",
      "crime-plagued\n",
      "homofobia\n",
      "320d\n",
      "dymanic\n"
     ]
    }
   ],
   "source": [
    "# Some experiments\n",
    "most_similar_to_word(u'apples', all_words)\n",
    "print '\\n'\n",
    "least_similar_to_word(u'apples', all_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Vector Arithmetic\n",
    "In this section we'll test the much-advertised \"king - man + woman = queen\" equation that GloVe vectors satisfy.\n",
    "\n",
    "1. The functions below modify the most_similar_to_word() and least_similar_to_word() functions to accept any 300-dimensional vector as input. Why is this necessary?\n",
    "2. Use these functions to test vector arithmetic with some of your sample words. \n",
    "3. Can you identify semantic regularities the vector space captures and does not capture? Some ideas: plurals, adjectives vs. adverbs, opposites, analogies. Why does it capture some of these better than others?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity as cs\n",
    "\n",
    "def cosine_similarity(a, b):\n",
    "    return cs(a.reshape(1, -1), b.reshape(1, -1))\n",
    "\n",
    "def most_similar(target, vocab=[w for w in nlp.vocab if w.orth_.islower()], top_n=10):\n",
    "    \"\"\"Find the top_n most similar words to target_vector in vocab.\n",
    "        target_vector : string, unicode, or 300-dimensional vector\n",
    "        vocab : list\n",
    "        top_n : int\"\"\"\n",
    "    if type(target) in [str, unicode]:\n",
    "        target_vector = nlp(unicode(target)).vector\n",
    "    else:\n",
    "        target_vector = target\n",
    "    vocab.sort(key=lambda x: cosine_similarity(x.vector, target_vector))\n",
    "    vocab.reverse()\n",
    "    print \"Top %d most similar words to target:\" % top_n\n",
    "    for word in vocab[1:top_n+1]:   \n",
    "        print(word.orth_, cosine_similarity(word.vector, target_vector), cosine_similarity(word.vector, target_vector))\n",
    "    return vocab[1:top_n+1]\n",
    "\n",
    "def least_similar(target, vocab=[w for w in nlp.vocab if w.orth_.islower()], top_n=10):\n",
    "    \"\"\"Find the top_n most similar words to target_word in vocab.\n",
    "        target_vector : string, unicode, or 300-dimensional vector\n",
    "        vocab : list\n",
    "        top_n : int\"\"\"\n",
    "    if type(target) in [str, unicode]:\n",
    "        target_vector = nlp(unicode(target)).vector\n",
    "    else:\n",
    "        target_vector = target\n",
    "    vocab.sort(key=lambda x: cosine_similarity(x.vector, target_vector))\n",
    "    print \"Top %d least similar words to target:\" % top_n\n",
    "    for word in vocab[:top_n]:   \n",
    "        print(word.orth_, cosine_similarity(word.vector, target_vector), cosine_similarity(word.vector, target_vector))\n",
    "    return vocab[:top_n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "most_similar(apples.vector + oranges.vector, all_words, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "most_similar(nlp(u'king').vector - nlp(u'man').vector + nlp(u'woman').vector, all_words, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization\n",
    "This section contains a few functions that allow you to plot a dimensionally-reduced version of the word embedding space.\n",
    "\n",
    "1. Plot the neighborhoods of some of your sample words.\n",
    "2. Plot lists of words that showcase the semantic regularities you experimented with in the prior section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "import seaborn as sns\n",
    "sns.set_style(\"ticks\")\n",
    "sns.set_style(\"darkgrid\")\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pca_plot_word_vectors(word_list):\n",
    "    word_vectors = [nlp(unicode(word)).vector for word in word_list]\n",
    "    pca = PCA(n_components=2)\n",
    "    word_vectors_pca = pca.fit_transform(word_vectors)\n",
    "    word_vectors_pca = np.c_[word_list, word_vectors_pca]\n",
    "    words_pca = pd.DataFrame(word_vectors_pca, columns=['word', 'pca_1', 'pca_2']).set_index('word')\n",
    "    words_pca['pca_1'] = words_pca['pca_1'].apply(float)\n",
    "    words_pca['pca_2'] = words_pca['pca_2'].apply(float)\n",
    "    ax = sns.regplot('pca_1', 'pca_2',\n",
    "               data=words_pca,\n",
    "               fit_reg=False,\n",
    "               scatter_kws={\"marker\": \"D\",\n",
    "                           \"s\": 100},\n",
    "    )\n",
    "\n",
    "    for word in words_pca.index:\n",
    "        ax.text(words_pca.loc[word, 'pca_1'] + .1, words_pca.loc[word, 'pca_2'], word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trucks = ['dump truck', 'garbage truck', 'cement mixer', 'excavator', 'impact hammer', 'bus', 'bulldozer', 'car', 'airplane', 'helicopter']\n",
    "pca_plot_word_vectors(trucks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_nearest_neighbors(word, num_neighbors=10):\n",
    "    words_to_plot = [word] + [neighbor.orth_ for neighbor in most_similar(word, all_words, num_neighbors)]\n",
    "    pca_plot_word_vectors(words_to_plot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_nearest_neighbors('truck', 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pairplot_word_list(word_list, pca_comps):\n",
    "    word_vectors = [nlp(unicode(word)).vector for word in word_list]\n",
    "    pca = PCA(n_components=pca_comps)\n",
    "    word_vectors_pca = pca.fit_transform(word_vectors)\n",
    "    word_vectors_pca = np.c_[word_list, word_vectors_pca]\n",
    "    words_pca = pd.DataFrame(word_vectors_pca, columns=['word'] + ['pca_' + str(x) for x in range(1,pca_comps+1)]).set_index('word')\n",
    "    for col in words_pca.columns:\n",
    "        words_pca[col] = words_pca[col].apply(float)\n",
    "    plotgrid = sns.pairplot(words_pca, size=4, plot_kws={\"s\": 80})\n",
    "    text_offset = .1\n",
    "    for i in range(len(plotgrid.axes)):\n",
    "        for j in range(len(plotgrid.axes)):\n",
    "            if i != j:\n",
    "                for word in words_pca.index:\n",
    "                    plotgrid.axes[i][j].text(words_pca.loc[word, 'pca_' + str(j+1)] + text_offset, words_pca.loc[word, 'pca_' + str(i+1)], word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pairplot_word_list(trucks, 3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}