text-similarity/2.sentence-similarity-birnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import collections\n",
    "import random\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['GO', 0], ['PAD', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "def str_idx(corpus, dic, maxlen, UNK=3):\n",
    "    X = np.zeros((len(corpus),maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen][::-1]):\n",
    "            val = dic[k] if k in dic else UNK\n",
    "            X[i,-1 - no]= val\n",
    "    return X\n",
    "\n",
    "def load_data(filepath):\n",
    "    x1=[]\n",
    "    x2=[]\n",
    "    y=[]\n",
    "    for line in open(filepath):\n",
    "        l=line.strip().split(\"\\t\")\n",
    "        if len(l)<2:\n",
    "            continue\n",
    "        if random.random() > 0.5:\n",
    "            x1.append(l[0].lower())\n",
    "            x2.append(l[1].lower())\n",
    "        else:\n",
    "            x1.append(l[1].lower())\n",
    "            x2.append(l[0].lower())\n",
    "        y.append(int(l[2]))\n",
    "    return np.array(x1),np.array(x2),np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1_text, X2_text, Y = load_data('train_snli.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([183966, 183407]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(Y,return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 47170\n",
      "Most common words [('a', 959179), ('the', 341846), ('in', 273772), ('is', 248868), ('man', 173742), ('on', 154293)]\n",
      "Sample data [4, 38, 7, 17, 4, 16662, 2698, 20, 27512, 4] ['a', 'person', 'is', 'at', 'a', 'diner,', 'ordering', 'an', 'omelette.', 'a']\n"
     ]
    }
   ],
   "source": [
    "concat = (' '.join(X1_text.tolist() + X2_text.tolist())).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, learning_rate, dropout):\n",
    "        \n",
    "        def cells(size, reuse=False):\n",
    "            cell = tf.nn.rnn_cell.LSTMCell(size,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "            return tf.contrib.rnn.DropoutWrapper(cell,output_keep_prob=dropout)\n",
    "        \n",
    "        def birnn(inputs, scope):\n",
    "            with tf.variable_scope(scope):\n",
    "                for n in range(num_layers):\n",
    "                    (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                        cell_fw = cells(size_layer // 2),\n",
    "                        cell_bw = cells(size_layer // 2),\n",
    "                        inputs = inputs,\n",
    "                        dtype = tf.float32,\n",
    "                        scope = 'bidirectional_rnn_%d'%(n))\n",
    "                    inputs = tf.concat((out_fw, out_bw), 2)\n",
    "                return inputs[:,-1]\n",
    "        \n",
    "        self.X_left = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_right = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.float32, [None])\n",
    "        self.batch_size = tf.shape(self.X_left)[0]\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        embedded_left = tf.nn.embedding_lookup(encoder_embeddings, self.X_left)\n",
    "        embedded_right = tf.nn.embedding_lookup(encoder_embeddings, self.X_right)\n",
    "        \n",
    "        def contrastive_loss(y,d):\n",
    "            tmp= y * tf.square(d)\n",
    "            tmp2 = (1-y) * tf.square(tf.maximum((1 - d),0))\n",
    "            return tf.reduce_sum(tmp +tmp2)/tf.cast(self.batch_size,tf.float32)/2\n",
    "        \n",
    "        self.output_left = birnn(embedded_left, 'left')\n",
    "        self.output_right = birnn(embedded_right, 'right')\n",
    "        self.distance = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(self.output_left,self.output_right)),1,keep_dims=True))\n",
    "        self.distance = tf.div(self.distance, tf.add(tf.sqrt(tf.reduce_sum(tf.square(self.output_left),1,keep_dims=True)),\n",
    "                                                     tf.sqrt(tf.reduce_sum(tf.square(self.output_right),1,keep_dims=True))))\n",
    "        self.distance = tf.reshape(self.distance, [-1])\n",
    "        self.cost = contrastive_loss(self.Y,self.distance)\n",
    "        \n",
    "        self.temp_sim = tf.subtract(tf.ones_like(self.distance),\n",
    "                                    tf.rint(self.distance))\n",
    "        correct_predictions = tf.equal(self.temp_sim, self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, \"float\"))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128\n",
    "dropout = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-191127e2e043>:36: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),learning_rate,dropout)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "vectors_left = str_idx(X1_text, dictionary, maxlen)\n",
    "vectors_right = str_idx(X2_text, dictionary, maxlen)\n",
    "train_X_left, test_X_left, train_X_right, test_X_right, train_Y, test_Y = train_test_split(vectors_left,\n",
    "                                                                                           vectors_right,\n",
    "                                                                                           Y,\n",
    "                                                                                           test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2297/2297 [07:41<00:00,  5.12it/s, accuracy=0.4, cost=0.121]   \n",
      "test minibatch loop: 100%|██████████| 575/575 [00:46<00:00, 12.98it/s, accuracy=0.667, cost=0.11]  \n",
      "train minibatch loop:   0%|          | 0/2297 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 507.81807231903076\n",
      "epoch: 0, training loss: 0.111454, training acc: 0.635320, valid loss: 0.102844, valid acc: 0.684101\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2297/2297 [07:41<00:00,  5.09it/s, accuracy=0.7, cost=0.0912]  \n",
      "test minibatch loop: 100%|██████████| 575/575 [00:46<00:00, 12.28it/s, accuracy=0.667, cost=0.111] \n",
      "train minibatch loop:   0%|          | 0/2297 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 508.3038227558136\n",
      "epoch: 1, training loss: 0.099547, training acc: 0.699636, valid loss: 0.097585, valid acc: 0.710069\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2297/2297 [07:41<00:00,  5.19it/s, accuracy=0.7, cost=0.089]   \n",
      "test minibatch loop: 100%|██████████| 575/575 [00:46<00:00, 12.31it/s, accuracy=0.333, cost=0.121] \n",
      "train minibatch loop:   0%|          | 0/2297 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 508.48236751556396\n",
      "epoch: 2, training loss: 0.095110, training acc: 0.722028, valid loss: 0.095254, valid acc: 0.720867\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2297/2297 [07:41<00:00,  5.22it/s, accuracy=0.9, cost=0.0622]  \n",
      "test minibatch loop: 100%|██████████| 575/575 [00:46<00:00, 12.26it/s, accuracy=0.667, cost=0.103] \n",
      "train minibatch loop:   0%|          | 0/2297 [00:00<?, ?it/s, accuracy=0.695, cost=0.0968]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 508.11622738838196\n",
      "epoch: 3, training loss: 0.092058, training acc: 0.735736, valid loss: 0.093679, valid acc: 0.728484\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2297/2297 [07:41<00:00,  5.07it/s, accuracy=0.9, cost=0.0628]  \n",
      "test minibatch loop: 100%|██████████| 575/575 [00:46<00:00, 12.33it/s, accuracy=0.667, cost=0.12]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 507.789755821228\n",
      "epoch: 4, training loss: 0.089936, training acc: 0.745521, valid loss: 0.093175, valid acc: 0.730239\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "for EPOCH in range(5):\n",
    "    lasttime = time.time()\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(range(0, len(train_X_left), batch_size), desc='train minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x_left = train_X_left[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_x_right = train_X_right[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_y = train_Y[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X_left : batch_x_left, \n",
    "                                        model.X_right: batch_x_right,\n",
    "                                        model.Y : batch_y})\n",
    "        assert not np.isnan(loss)\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, len(test_X_left), batch_size), desc='test minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x_left = test_X_left[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_x_right = test_X_right[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_y = test_Y[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X_left : batch_x_left, \n",
    "                                        model.X_right: batch_x_right,\n",
    "                                        model.Y : batch_y})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    train_loss /= (len(train_X_left) / batch_size)\n",
    "    train_acc /= (len(train_X_left) / batch_size)\n",
    "    test_loss /= (len(test_X_left) / batch_size)\n",
    "    test_acc /= (len(test_X_left) / batch_size)\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([1.], dtype=float32), array([0.69642884], dtype=float32)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left = str_idx(['a person is outdoors, on a horse.'], dictionary, maxlen)\n",
    "right = str_idx(['a person on a horse jumps over a broken down airplane.'], dictionary, maxlen)\n",
    "sess.run([model.temp_sim,1-model.distance], feed_dict = {model.X_left : left, \n",
    "                                        model.X_right: right})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.], dtype=float32), array([0.37782538], dtype=float32)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left = str_idx(['i love you'], dictionary, maxlen)\n",
    "right = str_idx(['you love i'], dictionary, maxlen)\n",
    "sess.run([model.temp_sim,1-model.distance], feed_dict = {model.X_left : left, \n",
    "                                        model.X_right: right})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}