Lab12/lab-12-5_JijoongHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-5 sequence to sequence (Keras + eager version)\n",
    "\n",
    "### simple neural machine translation training\n",
    "\n",
    "* sequence to sequence\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from matplotlib import font_manager, rc\n",
    "\n",
    "rc('font', family='AppleGothic') #for mac\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from pprint import pprint\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "targets = [['나는', '배가', '고프다'],\n",
    "           ['텐서플로우는', '매우', '어렵다'],\n",
    "           ['텐서플로우는', '딥러닝을', '위한', '프레임워크이다'],\n",
    "           ['텐서플로우는', '매우', '빠르게', '변화한다']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0,\n",
      " 'I': 1,\n",
      " 'a': 2,\n",
      " 'changing': 3,\n",
      " 'deep': 4,\n",
      " 'difficult': 5,\n",
      " 'fast': 6,\n",
      " 'feel': 7,\n",
      " 'for': 8,\n",
      " 'framework': 9,\n",
      " 'hungry': 10,\n",
      " 'is': 11,\n",
      " 'learning': 12,\n",
      " 'tensorflow': 13,\n",
      " 'very': 14}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for sources\n",
    "s_vocab = list(set(sum(sources, [])))\n",
    "s_vocab.sort()\n",
    "s_vocab = ['<pad>'] + s_vocab\n",
    "source2idx = {word : idx for idx, word in enumerate(s_vocab)}\n",
    "idx2source = {idx : word for idx, word in enumerate(s_vocab)}\n",
    "\n",
    "pprint(source2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<bos>': 1,\n",
      " '<eos>': 2,\n",
      " '<pad>': 0,\n",
      " '고프다': 3,\n",
      " '나는': 4,\n",
      " '딥러닝을': 5,\n",
      " '매우': 6,\n",
      " '배가': 7,\n",
      " '변화한다': 8,\n",
      " '빠르게': 9,\n",
      " '어렵다': 10,\n",
      " '위한': 11,\n",
      " '텐서플로우는': 12,\n",
      " '프레임워크이다': 13}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for targets\n",
    "t_vocab = list(set(sum(targets, [])))\n",
    "t_vocab.sort()\n",
    "t_vocab = ['<pad>', '<bos>', '<eos>'] + t_vocab\n",
    "target2idx = {word : idx for idx, word in enumerate(t_vocab)}\n",
    "idx2target = {idx : word for idx, word in enumerate(t_vocab)}\n",
    "\n",
    "pprint(target2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sequences, max_len, dic, mode = 'source'):\n",
    "    assert mode in ['source', 'target'], 'source와 target 중에 선택해주세요.'\n",
    "    \n",
    "    if mode == 'source':\n",
    "        # preprocessing for source (encoder)\n",
    "        s_input = list(map(lambda sentence : [dic.get(token) for token in sentence], sequences))\n",
    "        s_len = list(map(lambda sentence : len(sentence), s_input))\n",
    "        s_input = pad_sequences(sequences = s_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        return s_len, s_input\n",
    "    \n",
    "    elif mode == 'target':\n",
    "        # preprocessing for target (decoder)\n",
    "        # input\n",
    "        t_input = list(map(lambda sentence : ['<bos>'] + sentence + ['<eos>'], sequences))\n",
    "        t_input = list(map(lambda sentence : [dic.get(token) for token in sentence], t_input))\n",
    "        t_len = list(map(lambda sentence : len(sentence), t_input))\n",
    "        t_input = pad_sequences(sequences = t_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        # output\n",
    "        t_output = list(map(lambda sentence : sentence + ['<eos>'], sequences))\n",
    "        t_output = list(map(lambda sentence : [dic.get(token) for token in sentence], t_output))\n",
    "        t_output = pad_sequences(sequences = t_output, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        return t_len, t_input, t_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 7, 5] [[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for source\n",
    "s_max_len = 10\n",
    "s_len, s_input = preprocess(sequences = sources,\n",
    "                            max_len = s_max_len, dic = source2idx, mode = 'source')\n",
    "print(s_len, s_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 6, 6] [[ 1  4  7  3  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  6 10  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  5 11 13  2  0  0  0  0  0  0]\n",
      " [ 1 12  6  9  8  2  0  0  0  0  0  0]] [[ 4  7  3  2  0  0  0  0  0  0  0  0]\n",
      " [12  6 10  2  0  0  0  0  0  0  0  0]\n",
      " [12  5 11 13  2  0  0  0  0  0  0  0]\n",
      " [12  6  9  8  2  0  0  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for target\n",
    "t_max_len = 12\n",
    "t_len, t_input, t_output = preprocess(sequences = targets,\n",
    "                                      max_len = t_max_len, dic = target2idx, mode = 'target')\n",
    "print(t_len, t_input, t_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hyper-param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper-parameters\n",
    "epochs = 200\n",
    "batch_size = 4\n",
    "learning_rate = .005\n",
    "total_step = epochs / batch_size\n",
    "buffer_size = 100\n",
    "n_batch = buffer_size//batch_size\n",
    "embedding_dim = 32\n",
    "units = 32\n",
    "\n",
    "# input\n",
    "data = tf.data.Dataset.from_tensor_slices((s_len, s_input, t_len, t_input, t_output))\n",
    "data = data.shuffle(buffer_size = buffer_size)\n",
    "data = data.batch(batch_size = batch_size)\n",
    "# s_mb_len, s_mb_input, t_mb_len, t_mb_input, t_mb_output = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "                \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        \n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(len(source2idx), embedding_dim, units, batch_size)\n",
    "decoder = Decoder(len(target2idx), embedding_dim, units, batch_size)\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)\n",
    "\n",
    "# creating optimizer\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "# creating check point (Object-based saving)\n",
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                encoder=encoder,\n",
    "                                decoder=decoder)\n",
    "\n",
    "# create writer for tensorboard\n",
    "#summary_writer = tf.summary.create_file_writer(logdir=checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Loss 0.0397 Batch Loss 0.9913\n",
      "Epoch 10 Loss 0.0386 Batch Loss 0.9657\n",
      "Epoch 20 Loss 0.0372 Batch Loss 0.9303\n",
      "Epoch 30 Loss 0.0347 Batch Loss 0.8685\n",
      "Epoch 40 Loss 0.0311 Batch Loss 0.7785\n",
      "Epoch 50 Loss 0.0276 Batch Loss 0.6899\n",
      "Epoch 60 Loss 0.0242 Batch Loss 0.6041\n",
      "Epoch 70 Loss 0.0211 Batch Loss 0.5279\n",
      "Epoch 80 Loss 0.0184 Batch Loss 0.4607\n",
      "Epoch 90 Loss 0.0160 Batch Loss 0.4001\n",
      "Epoch 100 Loss 0.0137 Batch Loss 0.3433\n",
      "Epoch 110 Loss 0.0116 Batch Loss 0.2911\n",
      "Epoch 120 Loss 0.0098 Batch Loss 0.2448\n",
      "Epoch 130 Loss 0.0082 Batch Loss 0.2058\n",
      "Epoch 140 Loss 0.0070 Batch Loss 0.1741\n",
      "Epoch 150 Loss 0.0060 Batch Loss 0.1489\n",
      "Epoch 160 Loss 0.0052 Batch Loss 0.1296\n",
      "Epoch 170 Loss 0.0046 Batch Loss 0.1157\n",
      "Epoch 180 Loss 0.0042 Batch Loss 0.1061\n",
      "Epoch 190 Loss 0.0040 Batch Loss 0.0993\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for i, (s_len, s_input, t_len, t_input, t_output) in enumerate(data):\n",
    "        loss = 0\n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(s_input, hidden)\n",
    "\n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([target2idx['<bos>']] * batch_size, 1)\n",
    "            \n",
    "            #Teacher Forcing: feeding the target as the next input\n",
    "            for t in range(1, t_input.shape[1]):\n",
    "                \n",
    "                predictions, dec_hidden = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(t_input[:, t], predictions)\n",
    "            \n",
    "                dec_input = tf.expand_dims(t_input[:, t], 1) #using teacher forcing\n",
    "                \n",
    "        batch_loss = (loss / int(t_input.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradient = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradient, variables))\n",
    "        \n",
    "    if epoch % 10 == 0:\n",
    "        #save model every 10 epoch\n",
    "        print('Epoch {} Loss {:.4f} Batch Loss {:.4f}'.format(epoch,\n",
    "                                            total_loss / n_batch,\n",
    "                                            batch_loss.numpy()))\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x219dd01bbe0>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#restore checkpoint\n",
    "\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'I feel hungry'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I feel hungry\n",
      "나는 배가 고프다 <eos> \n"
     ]
    }
   ],
   "source": [
    "def prediction(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    \n",
    "    inputs = [inp_lang[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "        \n",
    "    result = ''\n",
    "    \n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "        \n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang['<bos>']], 0)\n",
    "    \n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += idx2target[predicted_id] + ' '\n",
    "\n",
    "        if idx2target.get(predicted_id) == '<eos>':\n",
    "            return result, sentence\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)    \n",
    "    \n",
    "    return result, sentence\n",
    "    \n",
    "result, output_sentence = prediction(sentence, encoder, decoder, source2idx, target2idx, s_max_len, t_max_len)\n",
    "\n",
    "print(sentence)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}