02_Short_Intro_to_Embeddings_with_Tensorflow_Rendered.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Short Intro To Embeddings with Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Goals\n",
    "- Understand Embedding \n",
    "- Perform Embedding Lookup using Tensorflow\n",
    "- Use Pre-Trained Embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version : 1.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "print('Tensorflow version : {0}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.8474672  0.63517459 0.20856734 0.62158914 0.1706803 ]\n",
      " [0.01427033 0.29586182 0.15395211 0.8114454  0.92883602]\n",
      " [0.07006209 0.93486385 0.87660798 0.93495023 0.93246372]\n",
      " [0.75529191 0.45670366 0.83832113 0.96170286 0.20207429]\n",
      " [0.60661999 0.79176031 0.84172283 0.90355146 0.82368189]\n",
      " [0.9636877  0.76228184 0.55074808 0.30381757 0.82599705]\n",
      " [0.34921163 0.17196946 0.79534164 0.39571298 0.43468079]\n",
      " [0.8603585  0.87752959 0.3065835  0.02131077 0.3528051 ]\n",
      " [0.81300056 0.17322652 0.32041377 0.74049448 0.97602482]\n",
      " [0.36918957 0.52890363 0.56712384 0.8195898  0.97569215]]\n"
     ]
    }
   ],
   "source": [
    "embedding_size = 5\n",
    "vocabulary_size = 10\n",
    "\n",
    "# create a sample embedding matrix of size 5 for vocab of size 10\n",
    "embedding = np.random.rand(vocabulary_size, embedding_size)\n",
    "print(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "# create one-hot encoding for one of element in vocabulary\n",
    "i = 4\n",
    "one_hot = np.zeros(10)\n",
    "one_hot[i] = 1.0\n",
    "print(one_hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.60661999 0.79176031 0.84172283 0.90355146 0.82368189]\n"
     ]
    }
   ],
   "source": [
    "# embedding vector can be extracted by taking a dot product between the one_hot vector and embedding matrix\n",
    "embedding_vector = np.dot(one_hot, embedding)\n",
    "print(embedding_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.60661999 0.79176031 0.84172283 0.90355146 0.82368189]\n"
     ]
    }
   ],
   "source": [
    "# cross validate from the embedding matrix\n",
    "print(embedding[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow Embedding Lookup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    # provide input indices \n",
    "    x = tf.placeholder(shape=[None], dtype=tf.int32, name='x')\n",
    "    \n",
    "    # create a constant initializer\n",
    "    weights_initializer = tf.constant_initializer(embedding)\n",
    "    embedding_weights = tf.get_variable(\n",
    "                            name='embedding_weights', \n",
    "                            shape=(vocabulary_size, embedding_size), \n",
    "                            initializer=weights_initializer,\n",
    "                            trainable=False)\n",
    "    # emebedding Lookup \n",
    "    embedding_lookup = tf.nn.embedding_lookup(embedding_weights, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.60662    0.7917603  0.84172285 0.90355146 0.8236819 ]]\n"
     ]
    }
   ],
   "source": [
    "# Getting Single Row\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(embedding_lookup, feed_dict={x : [4]}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.07006209 0.93486387 0.87660795 0.93495023 0.9324637 ]\n",
      " [0.60662    0.7917603  0.84172285 0.90355146 0.8236819 ]\n",
      " [0.34921163 0.17196946 0.7953417  0.39571297 0.4346808 ]]\n"
     ]
    }
   ],
   "source": [
    "# Getting Multiple Rows\n",
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(embedding_lookup, feed_dict={x : [2,4,6]}))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using GloVe Pre-Trained Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the following glove weight file : processed/glove/glove.6B.100d.txt\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIMENSION=100 # Available dimensions for 6B data is 50, 100, 200, 300\n",
    "glove_weights_file_path = os.path.join('processed','glove', 'glove.6B.{0}d.txt'.format(EMBEDDING_DIMENSION))\n",
    "print('Using the following glove weight file : {0}'.format(glove_weights_file_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the -0.038194 -0.24487 0.72812 -0.39961 0.083172 0.043953 -0.39141 0.3344 -0.57545 0.087459 0.28787 -0.06731 0.30906 -0.26384 -0.13231 -0.20757 0.33395 -0.33848 -0.31743 -0.48336 0.1464 -0.37304 0.34577 0.052041 0.44946 -0.46971 0.02628 -0.54155 -0.15518 -0.14107 -0.039722 0.28277 0.14393 0.23464 -0.31021 0.086173 0.20397 0.52624 0.17164 -0.082378 -0.71787 -0.41531 0.20335 -0.12763 0.41367 0.55187 0.57908 -0.33477 -0.36559 -0.54857 -0.062892 0.26584 0.30205 0.99775 -0.80481 -3.0243 0.01254 -0.36942 2.2167 0.72201 -0.24978 0.92136 0.034514 0.46745 1.1079 -0.19358 -0.074575 0.23353 -0.052062 -0.22044 0.057162 -0.15806 -0.30798 -0.41625 0.37972 0.15006 -0.53212 -0.2055 -1.2526 0.071624 0.70565 0.49744 -0.42063 0.26148 -1.538 -0.30223 -0.073438 -0.28312 0.37104 -0.25217 0.016215 -0.017099 -0.38984 0.87424 -0.72569 -0.51058 -0.52028 -0.1459 0.8278 0.27062\n",
      ", -0.10767 0.11053 0.59812 -0.54361 0.67396 0.10663 0.038867 0.35481 0.06351 -0.094189 0.15786 -0.81665 0.14172 0.21939 0.58505 -0.52158 0.22783 -0.16642 -0.68228 0.3587 0.42568 0.19021 0.91963 0.57555 0.46185 0.42363 -0.095399 -0.42749 -0.16567 -0.056842 -0.29595 0.26037 -0.26606 -0.070404 -0.27662 0.15821 0.69825 0.43081 0.27952 -0.45437 -0.33801 -0.58184 0.22364 -0.5778 -0.26862 -0.20425 0.56394 -0.58524 -0.14365 -0.64218 0.0054697 -0.35248 0.16162 1.1796 -0.47674 -2.7553 -0.1321 -0.047729 1.0655 1.1034 -0.2208 0.18669 0.13177 0.15117 0.7131 -0.35215 0.91348 0.61783 0.70992 0.23955 -0.14571 -0.37859 -0.045959 -0.47368 0.2385 0.20536 -0.18996 0.32507 -1.1112 -0.36341 0.98679 -0.084776 -0.54008 0.11726 -1.0194 -0.24424 0.12771 0.013884 0.080374 -0.35414 0.34951 -0.7226 0.37549 0.4441 -0.99059 0.61214 -0.35111 -0.83155 0.45293 0.082577\n",
      ". -0.33979 0.20941 0.46348 -0.64792 -0.38377 0.038034 0.17127 0.15978 0.46619 -0.019169 0.41479 -0.34349 0.26872 0.04464 0.42131 -0.41032 0.15459 0.022239 -0.64653 0.25256 0.043136 -0.19445 0.46516 0.45651 0.68588 0.091295 0.21875 -0.70351 0.16785 -0.35079 -0.12634 0.66384 -0.2582 0.036542 -0.13605 0.40253 0.14289 0.38132 -0.12283 -0.45886 -0.25282 -0.30432 -0.11215 -0.26182 -0.22482 -0.44554 0.2991 -0.85612 -0.14503 -0.49086 0.0082973 -0.17491 0.27524 1.4401 -0.21239 -2.8435 -0.27958 -0.45722 1.6386 0.78808 -0.55262 0.65 0.086426 0.39012 1.0632 -0.35379 0.48328 0.346 0.84174 0.098707 -0.24213 -0.27053 0.045287 -0.40147 0.11395 0.0062226 0.036673 0.018518 -1.0213 -0.20806 0.64072 -0.068763 -0.58635 0.33476 -1.1432 -0.1148 -0.25091 -0.45907 -0.096819 -0.17946 -0.063351 -0.67412 -0.068895 0.53604 -0.87773 0.31802 -0.39242 -0.23394 0.47298 -0.028803\n"
     ]
    }
   ],
   "source": [
    "# look at some sample rows\n",
    "!head -3 processed/glove/glove.6B.100d.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_weights = []\n",
    "word2idx = {}\n",
    "vocabulary_size = 40000 # limit vocab to top 40K terms\n",
    "vocabulary = []\n",
    "\n",
    "\n",
    "with open(glove_weights_file_path,'r') as file:\n",
    "    for index, line in enumerate(file):\n",
    "        values = line.split() # Word and weights separated by space\n",
    "        word = values[0] # Word is first symbol on each line\n",
    "        vocabulary.append(word)\n",
    "        word_weights = np.asarray(values[1:], dtype=np.float32) # Remainder of line is weights for word\n",
    "        word2idx[word] = index \n",
    "        glove_weights.append(word_weights)\n",
    "        \n",
    "        if index + 1 == vocabulary_size:\n",
    "            break\n",
    "glove_weights = np.asarray(glove_weights, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 100)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove_weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[300, 787]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = [\"man\", \"woman\"]\n",
    "#words = [\"paris\", \"london\",\"rome\",\"berlin\"]\n",
    "words_indices = [word2idx[word] for word in words]\n",
    "words_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = tf.Graph()\n",
    "\n",
    "with g.as_default():\n",
    "    # provide input indices \n",
    "    x = tf.placeholder(shape=[None], dtype=tf.int32, name='x')\n",
    "    \n",
    "    # create a constant initializer\n",
    "    weights_initializer = tf.constant_initializer(glove_weights)\n",
    "    embedding_weights = tf.get_variable(\n",
    "                            name='embedding_weights', \n",
    "                            shape=(vocabulary_size, EMBEDDING_DIMENSION), \n",
    "                            initializer=weights_initializer,\n",
    "                            trainable=False)\n",
    "    # emebedding Lookup \n",
    "    embedding_lookup = tf.nn.embedding_lookup(embedding_weights, x)\n",
    "    \n",
    "    # We use the cosine distance:\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embedding_weights), 1, keepdims=True))\n",
    "    normalized_embeddings = embedding_weights / norm\n",
    "    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, x)\n",
    "    similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Similarity Matrix: (2, 40000)\n",
      "Nearest to man : woman, boy, one, person, another, old, life, father, turned, who,\n",
      "Nearest to woman : girl, man, mother, boy, she, child, wife, her, herself, daughter,\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=g) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    result = sess.run(embedding_lookup, feed_dict={x : words_indices})\n",
    "    sim = sess.run(similarity, feed_dict={x : words_indices})\n",
    "    print('Shape of Similarity Matrix: {0}'.format(sim.shape))\n",
    "    for i,word_index in enumerate(words_indices):\n",
    "       \n",
    "        top_k = 10 # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k+1]\n",
    "        log = 'Nearest to {0} :'.format(vocabulary[word_index])\n",
    "        \n",
    "        for k in range(top_k):\n",
    "       \n",
    "            close_word = vocabulary[nearest[k]]\n",
    "            log = '{0} {1},'.format(log, close_word)\n",
    "        print(log)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}