Lab12/Lab12-6_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lab-12-6 sequence to sequence with attention (Keras + eager version)\n",
    "\n",
    "### simple neural machine translation training\n",
    "\n",
    "* sequence to sequence\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)\n",
    "* [Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025)\n",
    "* [Neural Machine Translation with Attention from Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "### matplotlib 한글 폰트 설정 #############################\n",
    "from matplotlib import font_manager, rc\n",
    "## for window #####\n",
    "font_name = font_manager.FontProperties(fname=\"c:/Windows/Fonts/malgun.ttf\").get_name()\n",
    "rc('font', family=font_name)\n",
    "## for mac #####\n",
    "#rc('font', family='AppleGothic') #for mac\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from pprint import pprint\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "targets = [['나는', '배가', '고프다'],\n",
    "           ['텐서플로우는', '매우', '어렵다'],\n",
    "           ['텐서플로우는', '딥러닝을', '위한', '프레임워크이다'],\n",
    "           ['텐서플로우는', '매우', '빠르게', '변화한다']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0,\n",
      " 'I': 1,\n",
      " 'a': 2,\n",
      " 'changing': 3,\n",
      " 'deep': 4,\n",
      " 'difficult': 5,\n",
      " 'fast': 6,\n",
      " 'feel': 7,\n",
      " 'for': 8,\n",
      " 'framework': 9,\n",
      " 'hungry': 10,\n",
      " 'is': 11,\n",
      " 'learning': 12,\n",
      " 'tensorflow': 13,\n",
      " 'very': 14}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for sources\n",
    "s_vocab = list(set(sum(sources, [])))\n",
    "s_vocab.sort()\n",
    "s_vocab = ['<pad>'] + s_vocab\n",
    "source2idx = {word : idx for idx, word in enumerate(s_vocab)}\n",
    "idx2source = {idx : word for idx, word in enumerate(s_vocab)}\n",
    "\n",
    "pprint(source2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<bos>': 1,\n",
      " '<eos>': 2,\n",
      " '<pad>': 0,\n",
      " '고프다': 3,\n",
      " '나는': 4,\n",
      " '딥러닝을': 5,\n",
      " '매우': 6,\n",
      " '배가': 7,\n",
      " '변화한다': 8,\n",
      " '빠르게': 9,\n",
      " '어렵다': 10,\n",
      " '위한': 11,\n",
      " '텐서플로우는': 12,\n",
      " '프레임워크이다': 13}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for targets\n",
    "t_vocab = list(set(sum(targets, [])))\n",
    "t_vocab.sort()\n",
    "t_vocab = ['<pad>', '<bos>', '<eos>'] + t_vocab\n",
    "target2idx = {word : idx for idx, word in enumerate(t_vocab)}\n",
    "idx2target = {idx : word for idx, word in enumerate(t_vocab)}\n",
    "\n",
    "pprint(target2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sequences, max_len, dic, mode = 'source'):\n",
    "    assert mode in ['source', 'target'], 'source와 target 중에 선택해주세요.'\n",
    "    \n",
    "    if mode == 'source':\n",
    "        # preprocessing for source (encoder)\n",
    "        s_input = list(map(lambda sentence : [dic.get(token) for token in sentence], sequences))\n",
    "        s_len = list(map(lambda sentence : len(sentence), s_input))\n",
    "        s_input = pad_sequences(sequences = s_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        return s_len, s_input\n",
    "    \n",
    "    elif mode == 'target':\n",
    "        # preprocessing for target (decoder)\n",
    "        # input\n",
    "        t_input = list(map(lambda sentence : ['<bos>'] + sentence + ['<eos>'], sequences))\n",
    "        t_input = list(map(lambda sentence : [dic.get(token) for token in sentence], t_input))\n",
    "        t_len = list(map(lambda sentence : len(sentence), t_input))\n",
    "        t_input = pad_sequences(sequences = t_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        # output\n",
    "        t_output = list(map(lambda sentence : sentence + ['<eos>'], sequences))\n",
    "        t_output = list(map(lambda sentence : [dic.get(token) for token in sentence], t_output))\n",
    "        t_output = pad_sequences(sequences = t_output, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        return t_len, t_input, t_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 7, 5] [[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for source\n",
    "s_max_len = 10\n",
    "s_len, s_input = preprocess(sequences = sources,\n",
    "                            max_len = s_max_len, dic = source2idx, mode = 'source')\n",
    "print(s_len, s_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 6, 6] [[ 1  4  7  3  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  6 10  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  5 11 13  2  0  0  0  0  0  0]\n",
      " [ 1 12  6  9  8  2  0  0  0  0  0  0]] [[ 4  7  3  2  0  0  0  0  0  0  0  0]\n",
      " [12  6 10  2  0  0  0  0  0  0  0  0]\n",
      " [12  5 11 13  2  0  0  0  0  0  0  0]\n",
      " [12  6  9  8  2  0  0  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for target\n",
    "t_max_len = 12\n",
    "t_len, t_input, t_output = preprocess(sequences = targets,\n",
    "                                      max_len = t_max_len, dic = target2idx, mode = 'target')\n",
    "print(t_len, t_input, t_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hyper-param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper-parameters\n",
    "epochs = 100\n",
    "batch_size = 4\n",
    "learning_rate = .005\n",
    "total_step = epochs / batch_size\n",
    "buffer_size = 100\n",
    "n_batch = buffer_size//batch_size\n",
    "embedding_dim = 32\n",
    "units = 128\n",
    "\n",
    "# input\n",
    "data = tf.data.Dataset.from_tensor_slices((s_len, s_input, t_len, t_input, t_output))\n",
    "data = data.shuffle(buffer_size = buffer_size)\n",
    "data = data.batch(batch_size = batch_size)\n",
    "# s_mb_len, s_mb_input, t_mb_len, t_mb_input, t_mb_output = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_activation='sigmoid', \n",
    "                               recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "#         print(\"state: {}\".format(state.shape))\n",
    "#         print(\"output: {}\".format(state.shape))\n",
    "              \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        # * `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "                \n",
    "        #* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        # * `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        # * `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        # * `merged vector = concat(embedding output, context vector)`\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(len(source2idx), embedding_dim, units, batch_size)\n",
    "decoder = Decoder(len(target2idx), embedding_dim, units, batch_size)\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)\n",
    "\n",
    "# creating optimizer\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "# creating check point (Object-based saving)\n",
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                encoder=encoder,\n",
    "                                decoder=decoder)\n",
    "\n",
    "# create writer for tensorboard\n",
    "summary_writer = tf.summary.create_file_writer(logdir=checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Loss 0.0395 Batch Loss 0.9880\n",
      "Epoch 10 Loss 0.0375 Batch Loss 0.9386\n",
      "Epoch 20 Loss 0.0344 Batch Loss 0.8592\n",
      "Epoch 30 Loss 0.0327 Batch Loss 0.8178\n",
      "Epoch 40 Loss 0.0303 Batch Loss 0.7585\n",
      "Epoch 50 Loss 0.0259 Batch Loss 0.6464\n",
      "Epoch 60 Loss 0.0176 Batch Loss 0.4402\n",
      "Epoch 70 Loss 0.0112 Batch Loss 0.2791\n",
      "Epoch 80 Loss 0.0069 Batch Loss 0.1713\n",
      "Epoch 90 Loss 0.0038 Batch Loss 0.0962\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for i, (s_len, s_input, t_len, t_input, t_output) in enumerate(data):\n",
    "        loss = 0\n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(s_input, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([target2idx['<bos>']] * batch_size, 1)\n",
    "            \n",
    "            #Teacher Forcing: feeding the target as the next input\n",
    "            for t in range(1, t_input.shape[1]):\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(t_input[:, t], predictions)\n",
    "            \n",
    "                dec_input = tf.expand_dims(t_input[:, t], 1) #using teacher forcing\n",
    "                \n",
    "        batch_loss = (loss / int(t_input.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradient = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradient, variables))\n",
    "        \n",
    "    if epoch % 10 == 0:\n",
    "        #save model every 10 epoch\n",
    "        print('Epoch {} Loss {:.4f} Batch Loss {:.4f}'.format(epoch,\n",
    "                                            total_loss / n_batch,\n",
    "                                            batch_loss.numpy()))\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "#     sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang['<bos>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += idx2target[predicted_id] + ' '\n",
    "\n",
    "        if idx2target.get(predicted_id) == '<eos>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot\n",
    "\n",
    "# result, sentence, attention_plot = evaluate(sentence, encoder, decoder, source2idx, target2idx,\n",
    "#                                             s_max_len, t_max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x2c143ab7eb0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#restore checkpoint\n",
    "\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: I feel hungry\n",
      "Predicted translation: 나는 배가 고프다 <eos> \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tnfls\\AppData\\Local\\Temp/ipykernel_29352/1800777997.py:9: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
      "C:\\Users\\tnfls\\AppData\\Local\\Temp/ipykernel_29352/1800777997.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAJlCAYAAAA1j+5XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUVUlEQVR4nO3dfbBtB1nf8d9DLkkgIRISCkTUEMNLa6GmBFDAKtahI0JrLX8U+8aMGsGh0lJLRzvtdDqMxTIqviDOLZZIawqOIyhQR0GLFrFC0FQBoeUlECDYBASSYF64PP3j7Dw9Ob3n3nNuzj7r3nM+n5k9c/Zae+393Nlwvllr7b1OdXcAIEnus/QAAJw+RAGAIQoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAFgk6p6VVX9laXnWIooANzTx5K8oap+t6qeU1VHlh5oP5VrHwHcU1VVkqcn+a4kT0rymiQ/292fWHSwfSAKACdQVV+Z5JokT0zyq0le0t3XLTrUGjl8BHAcVfXUqvoPSd6Z5ENJvjnJW7NxaOkfLjrcGtlTANikqv51kn+Q5EtJjiZ5dXd/ZtP6hyR5Z3d/1UIjrtWhOoECsANfm+T7uvstx1vZ3X9aVW/c35H2jz0FgE2q6m939+uXnmMpogCwSVV99KAeGtoJJ5oB7ul1VfWcpYdYij0FgE2q6i1JHp/k00k+mo0TzkmS7n76UnPtFyeaAe7pP69uh5I9BQCGPQWATarqO7dZdVuS/93d79vPefabPQWATarqt5M8Ocl7k/xZkkckuTDJHyd55Gr5szd/oe0g8ekjgHv6wyQv6O6v7e6ndfelSX4yyS8kuSQbUXjJgvOtlT0FgE2q6iPd/YgtyyrJe7r7a6rqfkn+Z3c/apkJ18ueAsA9dVWdvWXZfZI8KEm6+8+TbF1/YIgCwD39WpKfq6oLkqSqzkny8iTXru6fn+TAHmIRBYB7+udJKsnNVXVDks8muTLJ81frvz7J1YtMtg+cUwA4jqp6UJLLkny6uz+y9Dz7RRQAGA4fwR6qqruq6s4T3O6qqjuXnpPtVdXXVNXbquqWqjq2un2pqo4tPdt+8I1m2FuXLz0A99rVSX4nyfOy8eW1Q8XhI1iz1WfcL+7um5aehZOrqk9095cvPcdSHD6CNamqC6vqtUn+PMn7V8ueVlXftuxknMQHVieZDyWHj2B9fibJJ5N8RZJ3rpb9UZL/muTNSw3FSb0mya9U1cuT3Lh5RXe/Y5GJ9pHDR7AmVfXh7r7sOD9/sLudezhNVdV2Hz/tu9/Dg8yeAqzPHVV1fnffmo0vQ6Wq7p/krGXH4kS2XvfosHFOAdbnPyX55aq6PBvX07koyc8medOyY8H27CnA+rw0GxdOe3eSByS5IRuhePGSQ3Fiq8NHxz2ufhgOHzmnAPugqi7OxuUS/B/uNFdVT9qy6KIk35vkt7v7xxYYaV+JAqzRKgZ/IxvfU/iJpefh1FTVkSSv7+5nLT3LujmnAGtSVU9L8idJnpPkB1fLvqOqfnzRwdi17v5ikvOWnmM/2FOANamqdyd5Xne/6+6/5lVVZyV5b3c/Zun5OL6qumTLovOTfHuSv9XdT9n/ifaXE82wPg/q7netfu4k6e5jqz/awunr41vu35qNP7Dz3QvMsu9EAdbnxqp6wqYwpKr+UpJbFpyJk+juQ31Y/VD/42HNfiDJm6vqJUnOq6p/lo1LXPzbZcfiRKrqiqp6+5ZLZx87LJfOFgXYQ1V1dNPdL0/y1CQXJHlXkkcn+c7u/qUlZmPHfi7J72XjT3BesuV24DnRDHuoqj6a5PLuvmvz9Y44cxz2S2c7pwB765eTvL+qPpTkoVX1G8d7UHc/fX/HYhfeX1UXdfenlx5kCaJwmquqH9rJ47r7h9c9CyfX3f+0qt6UjctlX5HkFxYeiR2oqidvuvvzSd5QVT+RjUufj8Nw6WxROP09cgePcQzwNNLdv5kkVfXo7v75pedhR44X75dtud9JDvzhQOcUABg+fQTAEAUAhiicwarqqqVnYOe8X2eew/ieicKZ7dD9D/YM5/068xy690wUABgH/tNHZ9c5fe4BvQz6Xbkj940Lbp4pvF9nnoP6nt2e23Jn31HHW3fgv6dwbs7Lk+qvLz0GwGnj9ze+SnNcDh8BMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYi0ehqq6vqsuXngOANUehqh5TVdef4rafqqqH7/FIAJzA4nsKAJw+RAGAcWTpAU7gc0murart1n93d79pH+cBOPBOlygcraovJHlld785Sbr70QvPBHDonC5R+JkkH0/ykb14sqq6KslVSXJu7r8XTwlwKJwuUbiuuz9YVQ+rqpt3ue0V3X3D5gXdfTTJ0SS5oB7UezUkwEG3H1F4SFX9VpJe3Y4kOT/JOd392M0P7O4bk1y8DzMBcBzrjsKHkzxh9XMnOba63Zbk/5xow9X3Gx7X3Z9f54AA/D9rjUJ335nkPae4+UPjI7MA+8ovXQDG6XKieTsfqqrtThR/T3e/fl+nATjgTocoPCPJx7Yu7O5zF5gF4FBbPArd/b6lZwBgg3MKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAOLL0AOv2qMd9Ib/+69ctPQa7cPOx25YegV36zJeWnoDdePa33brtOnsKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAME6LKFRVV9W5u3j89VV1+TpnAjiM1hqFqnpuVV296f6lVfWp1c9XV9Vzj7PNk6vqU1tuN1fVB9c5KwDJkaUH2Kq735HkoZuXVdUzk7xomYkADo/T4vDRDjw1yTuWHgLgoNuPPYXHVNULVj8/aLcbV9U5Sf5ekm/d06kA+P/sRxTOTvLA1c8XbFn37VV16Um2f1GSd3f3e7Ysf2VV3ZbkF7v7mns9JQD7EoU/6u6XJBsnmpP8/U3rbkly83YbVtU3JnlhkiceZ/Wrknwiycf2bFKAQ27pcwq/2d0/fbwVVfUtSV6X5O929/F+8b+7u99+vHVVdVVVXVtV19706WN7PDLAwbUfUTh/9VHUS5M8/GQPrqoHVNUPJ7k6ybO7+227fcHuPtrdV3b3lQ++6Kzdbg5waK378NHNSS5N8kubll13km2el+QvJnlCd9+4nrEAOJ61RqG735TkTbvc5mVrGgeAk1j6nAIApxFRAGAsdpmL7n7uprtXJLljF5s/Iz6KCrDnTotrH3X3dbt8/PvWNArAoebwEQBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMI4sPcC6/cknHpwn/tDzlx6DXbjzAbX0COzSXecvPQG7cf1NP7btOnsKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgHFGRqGqrq+qy5eeA+CgObLTB1bVs5K8+gQPuTDJV3f39Zu2eWuSv3ySp35/d3/Tpm0ek+S9SW7a+sDufuhO5wVg93Yche5+Y5KLt1tfVR8/zjbfcopz3dDdl57itgCcon05fFRV11TVBZvun1dVr92P1wZg5/YyCmcl+eI2674hyQM33b8wyVP28LUB2AOnFIWquqiqnrxl8f2S3LHNJrclOX/T/QckueVUXnuTd1bVzVX1/ffyeQBY2fE5hS0em+QlSZ6aJFVVSc7L9r/ojxeFW0/xte/29Uk+lOTYvXweAFZONQpbXZjkC919++aFVXV9knNXd391ox33WP+pJF/s7oefwmse6+7jHq6qqquSXJUkZ5934Sk8NcDhtFdRuD3J92xdeC8+QXRhVb00SWXjENfZ2YjLZ7v7X5xs4+4+muRokpx38Vf0Kc4AcOjsSRS6+wtJfnEvnivJjUleePdTr25fTPLZJJ/co9cA4DhOGoWqeliSP95m3c3bbHZFknftcpYndPcN3f25JFfvclsA9sBJo9DdN+YEX1o7Ad8+BjjDnJHXPgJgPc7UKDwjyceWHgLgoNmrTx/tq+5+39IzABxEZ+qeAgBrIAoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAGAIQoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAGAIQoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAGAIQoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAGAIQoADFEAYIgCAEMUABiiAMAQBQCGKAAwRAGAIQoADFEAYIgCAEMUABhHlh5g3R74kFvyN1/035Yeg124bx1begQ40F7xhs9vu86eAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAxr2KQlV95V4NcoLXuLCq7r/u1wHgFKNQVQ+vqv+Y5OiWZa+vqg9X1f+qqhdu2eaZVfX7VfWRqvpgVf1IVZ27WndWVb20qj5QVZ+sqrdt2vSSJNdW1VVVddapzAvAzuwqCqv/av/3SX4tyW8k+dbV8nOSvDXJW7v7siTfkOT7quru9d+cjYA8v7sfkeTxSR6b5GWrp/5HSZ6S5LHdfUmS7737Nbv7vUmemuRRSd5dVd+xgzmvqqprq+ra2/7szt38EwEOtR1FoaruW1UvTvK7Sa5P8le7+7Xd3auHPDPJZ7r7FUnS3X+a5FVJnr1a/0+S/Lvu/oPV+s8l+f4k31VVleT2JH8hyWWr9R/Y/Prd/Znu/oEkz0ryzKp6e1U9Zbt5u/tod1/Z3Veed+HZO/knApDkyA4fd1aSi5McS1Kr7e7atP6yJI+tqus3LTs7ye+tfv7qJD+15Tk/nOTc1fP+lyRfluTNVfXhJP+qu//HNvPenuR+q+0A2EM72lPo7tu7+8VJvjHJg5P8QVX9YFV92eohn0zyO9196abbJd39d1brb0jyyC1Pe2mSW7v7pt7wytVjXpXkrVX1sLsfWFWPq6prshGPtyS5srt/5dT+yQBsZ1fnFFaHcf5Nkicm+VKSu38xvznJFZuP91fVE6rq0tXdVyT5l1V1xWrdBUl+PMmPru4/vqou6u4vJfmt1XOfs1r3dUl+Oslruvvruvv1mw5bAbCHdnr46B66+5YkP1JVP7q6/9mqekaSl1fVTya5I8kfJvnHq/VvXH2s9NVVdWGSW5O8OhthSJJHJ3lDVR1L8tkkL+ju61frru3uv3YqcwKwO6cUhbt19xc3/Xxdkm86wWNfl+R126y7Jsk1J3sNANbLN5oBGKIAwBAFAIYoADBEAYAhCgAMUQBgiAIAQxQAGKIAwBAFAIYoADBEAYAhCgAMUQBgiAIAQxQAGKIAwBAFAIYoADBEAYAhCgAMUQBgiAIAQxQAGKIAwBAFAIYoADBEAYAhCgAMUQBgiAIAQxQAGKIAwBAFAIYoADBEAYAhCgAMUQBgiAIAQxQAGKIAwBAFAIYoADBEAYAhCgAMUQBgHFl6gHW75X33yX9/3LlLjwFw2vh8n7XtOnsKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAxpGlB1iHqroqyVVJcm7uv/A0AGeOA7mn0N1Hu/vK7r7yvjln6XEAzhgHMgoAnBpRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAYogDAEAUAhigAMEQBgCEKAAxRAGCIAgBDFAAY1d1Lz7BWVXVTko8uPceaXJzk5qWHYMe8X2eeg/qefVV3P/h4Kw58FA6yqrq2u69ceg52xvt15jmM75nDRwAMUQBgiMKZ7ejSA7Ar3q8zz6F7z5xTAGDYUwBgiAIAQxQAGKIAwBAFAMb/Bfna7PDEInEqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sentence = 'I feel hungry'\n",
    "# sentence = 'tensorflow is a framework for deep learning'\n",
    "\n",
    "translate(sentence, encoder, decoder, source2idx, target2idx, s_max_len, t_max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}