Lab12/lab-12-7 JihoHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab-12-7-bonus: sequence to sequence with attention chatbot \n",
    "### simple neural machine translation chatbot\n",
    "\n",
    "* sequence to sequence|\n",
    "* variable input sequence length\n",
    "* variable output sequence length\n",
    "* Luong attention\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)\n",
    "* [Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025)\n",
    "* [Neural Machine Translation with Attention from Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "# Import TensorFlow >= 1.10 and enable eager execution\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chat data load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from konlpy.tag import Twitter\n",
    "import pandas as pd\n",
    "import enum\n",
    "import os\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadData():\n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    dataDF = pd.read_csv('./data_in/chat_ko/ChatBotData.csv', header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(dataDF['Q']), list(dataDF['A'])\n",
    "#     dataset = dataDF['Q'] + '\\t' + dataDF['A']\n",
    "#     dataset = list(dataset)\n",
    "    # skleran에서 지원하는 함수를 통해서 학습 셋과 \n",
    "    # 테스트 셋을 나눈다.\n",
    "#     xTrain, xTest, yTrain, yTest = train_test_split(question, answer, test_size=0.33, random_state=42)\n",
    "    # 그 값을 리턴한다.\n",
    "#     return xTrain, yTrain, xTest, yTest\n",
    "    return question, answer\n",
    "\n",
    "def preproLikeMorphlized(data):\n",
    "    # 형태소 분석 모듈 객체를\n",
    "    # 생성합니다.\n",
    "\n",
    "    morphAnalyzer = Twitter()\n",
    "    # 형태소 토크나이즈 결과 문장을 받을\n",
    "    #  리스트를 생성합니다.\n",
    "    result_data = list()\n",
    "    # 데이터에 있는 매 문장에 대해 토크나이즈를\n",
    "    # 할 수 있도록 반복문을 선언합니다.\n",
    "    for seq in data:\n",
    "        # Twitter.morphs 함수를 통해 토크나이즈 된\n",
    "        # 리스트 객체를 받고 다시 공백문자를 기준으로\n",
    "        # 하여 문자열로 재구성 해줍니다.\n",
    "        morphlizedSeq = \" \".join(morphAnalyzer.morphs(seq.replace(' ', '')))\n",
    "        result_data.append(morphlizedSeq)\n",
    "\n",
    "    return result_data\n",
    "\n",
    "def dataTokenizer(data):\n",
    "    # 토크나이징 해서 담을 배열 생성\n",
    "    words = []\n",
    "    for sentence in data:\n",
    "        # FILTERS = \"([~.,!?\\\"':;)(])\"\n",
    "        # 위 필터와 같은 값들을 정규화 표현식을 \n",
    "        # 통해서 모두 \"\" 으로 변환 해주는 부분이다.\n",
    "        sentence = re.sub(change_filter, \"\", sentence)\n",
    "        for word in sentence.split():\n",
    "            words.append(word)\n",
    "    # 토그나이징과 정규표현식을 통해 만들어진 \n",
    "    # 값들을 넘겨 준다.\n",
    "    return [word for word in words if word]\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "    \n",
    "    # creating a space between a word and the punctuation following it\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\" \n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    \n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    \n",
    "    return w\n",
    "\n",
    "def create_dataset(data, num_examples):\n",
    "#     word_pairs = [[preprocess_sentence(w) for w in l]  for l in data[:num_examples]]\n",
    "    word_pairs = [[preprocess_sentence(l)]  for l in data[:num_examples]]\n",
    "    return word_pairs\n",
    "\n",
    "class LanguageIndex():\n",
    "    def __init__(self, lang):\n",
    "        self.lang = lang\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "        self.vocab = set()\n",
    "\n",
    "        self.create_index()\n",
    "        \n",
    "    def create_index(self):\n",
    "        for phrase in self.lang:\n",
    "#             if self.vocab == '<start>':\n",
    "#                 pass\n",
    "            self.vocab.update(phrase.split(' '))\n",
    "    \n",
    "        self.vocab = sorted(self.vocab)\n",
    "    \n",
    "        self.word2idx['<pad>'] = 0\n",
    "#         self.word2idx['<start>'] = 1\n",
    "#         self.word2idx['<end>'] = 2\n",
    "#         self.word2idx['<unk>'] = 3\n",
    "\n",
    "        for index, word in enumerate(self.vocab):\n",
    "            self.word2idx[word] = index + 1\n",
    "\n",
    "        for word, index in self.word2idx.items():\n",
    "            self.idx2word[index] = word\n",
    "            \n",
    "def max_length(tensor):\n",
    "    return max(len(t) for t in tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(path, num_examples):\n",
    "    \n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    data_df = pd.read_csv(path, header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(data_df['Q']), list(data_df['A'])\n",
    "    \n",
    "    question = create_dataset(question, num_examples)\n",
    "    answer = create_dataset(answer, num_examples)\n",
    "    \n",
    "    print(len(question))\n",
    "    \n",
    "    pairs = [question[i] + answer[i] for i in range(num_examples)]\n",
    "    \n",
    "    # index language using the class defined above    \n",
    "    inp_lang = LanguageIndex(q for q, a in pairs)\n",
    "    targ_lang = LanguageIndex(a for q, a in pairs)\n",
    "    \n",
    "    # Vectorize the input and target languages\n",
    "    \n",
    "    # question\n",
    "    input_tensor = [[inp_lang.word2idx[s] for s in q.split(' ')] for q, a in pairs]\n",
    "    \n",
    "    # answer\n",
    "    target_tensor = [[targ_lang.word2idx[s] for s in a.split(' ')] for q, a in pairs]\n",
    "\n",
    "    # Calculate max_length of input and output tensor\n",
    "    # Here, we'll set those to the longest sentence in the dataset\n",
    "    max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n",
    "    \n",
    "    # Padding the input and output tensor to the maximum length\n",
    "    input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n",
    "                                                                 maxlen=max_length_inp,\n",
    "                                                                 padding='post')\n",
    "    \n",
    "    target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n",
    "                                                                  maxlen=max_length_tar, \n",
    "                                                                  padding='post')\n",
    "    \n",
    "    return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11823\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9458, 9458, 2365, 2365)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset with limit\n",
    "\n",
    "num_examples = 11823\n",
    "\n",
    "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset('./data_in/ChatBotData.csv', num_examples)\n",
    "\n",
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length_targ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rgCLkfv5uO3d"
   },
   "source": [
    "### Create a tf.data dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TqHsArVZ3jFS"
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "N_BATCH = BUFFER_SIZE//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word2idx)\n",
    "vocab_tar_size = len(targ_lang.word2idx)\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) #drop_reainder가 True이면, batch size가 적지 않도록 수정\n",
    "\n",
    "# The tensors in the resulting element will have an additional outer dimension, which will be batch_size (or N % batch_size for the last element if batch_size does not divide the number of input elements N evenly and drop_remainder is False). If your program depends on the batches having the same outer dimension, you should set the drop_remainder argument to True to prevent the smaller batch from being produced."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TNfHIF71ulLu"
   },
   "source": [
    "## Write the encoder and decoder model\n",
    "\n",
    "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://www.tensorflow.org/tutorials/seq2seq). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://www.tensorflow.org/tutorials/seq2seq#background_on_the_attention_mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\">\n",
    "\n",
    "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n",
    "\n",
    "Here are the equations that are implemented:\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\">\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\">\n",
    "\n",
    "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n",
    "\n",
    "* FC = Fully connected (dense) layer\n",
    "* EO = Encoder output\n",
    "* H = hidden state\n",
    "* X = input to the decoder\n",
    "\n",
    "And the pseudo-code:\n",
    "\n",
    "* `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "* `merged vector = concat(embedding output, context vector)`\n",
    "* This merged vector is then given to the GRU\n",
    "  \n",
    "The shapes of all the vectors at each step have been specified in the comments in the code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "avyJ_4VIUoHb"
   },
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
    "  # the code automatically does that.\n",
    "    if tf.test.is_gpu_available():\n",
    "        return tf.keras.layers.CuDNNGRU(units, \n",
    "                                        return_sequences=True, \n",
    "                                        return_state=True, \n",
    "                                        recurrent_initializer='glorot_uniform')\n",
    "    else:\n",
    "        return tf.keras.layers.GRU(units, \n",
    "                                   return_sequences=True, \n",
    "                                   return_state=True, \n",
    "                                   recurrent_activation='sigmoid', \n",
    "                                   recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nZ2rI24i3jFg"
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        print(\"state: {}\".format(state.shape))\n",
    "        print(\"output: {}\".format(state.shape))\n",
    "              \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yJ_B3mhW3jFk"
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        # * `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "                \n",
    "        #* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        # * `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        # * `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        # * `merged vector = concat(embedding output, context vector)`\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P5UY8wko3jFp"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13422 9856 256 1024 64\n"
     ]
    }
   ],
   "source": [
    "print(vocab_inp_size, vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_ch_71VbIRfK"
   },
   "source": [
    "## Define the optimizer and the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WmTHr5iV3jFr"
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer()\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DMVWzzsfNl4e"
   },
   "source": [
    "## Checkpoints (Object-based saving)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zj8bXQTgNwrF"
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hpObfY22IddU"
   },
   "source": [
    "## Training\n",
    "\n",
    "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n",
    "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n",
    "3. The decoder returns the *predictions* and the *decoder hidden state*.\n",
    "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
    "5. Use *teacher forcing* to decide the next input to the decoder.\n",
    "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n",
    "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ddefjBMa3jF0",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Batch 0 Loss 2.0115\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 1\n",
    "\n",
    "#GPU 설정\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch, (inp, targ)) in enumerate(dataset):\n",
    "        loss = 0\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(inp, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([targ_lang.word2idx['<start>']] * BATCH_SIZE, 1)\n",
    "            \n",
    "            # Teacher forcing - feeding the target as the next input\n",
    "            for t in range(1, targ.shape[1]):\n",
    "                # passing enc_output to the decoder\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(targ[:, t], predictions)\n",
    "                \n",
    "                # using teacher forcing\n",
    "                dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "        \n",
    "        batch_loss = (loss / int(targ.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradients = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradients, variables))\n",
    "        \n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                         batch,\n",
    "                                                         batch_loss.numpy()))\n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    if (epoch + 1) % 2 == 0:\n",
    "          checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "    \n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                        total_loss / N_BATCH))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mU3Ce8M6I3rz"
   },
   "source": [
    "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
    "* Stop predicting when the model predicts the *end token*.\n",
    "* And store the *attention weights for every time step*.\n",
    "\n",
    "Note: The encoder output is calculated only once for one input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EbQpyYs13jF_"
   },
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "    sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang.word2idx['<start>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += targ_lang.idx2word[predicted_id] + ' '\n",
    "\n",
    "        if targ_lang.idx2word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s5hQWlbN3jGF"
   },
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sl9zUHzg3jGI"
   },
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n250XbnjOaqP"
   },
   "source": [
    "## Restore the latest checkpoint and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UJpT9D5_OgP6"
   },
   "outputs": [],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WrAM0FDomq3E"
   },
   "outputs": [],
   "source": [
    "translate('뭐라도 배워볼까.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 한글 폰트가 깨질 때\n",
    "\n",
    "https://financedata.github.io/posts/matplotlib-hangul-for-ubuntu-linux.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#실행중인 운영체제 확인\n",
    "\n",
    "import platform\n",
    "print(platform.platform())\n",
    "\n",
    "# 파이썬 버전\n",
    "import sys\n",
    "print(sys.version_info)\n",
    "\n",
    "# matplotlib 주요 설치 정보\n",
    "\n",
    "import matplotlib\n",
    "\n",
    "print ('버전: ', matplotlib.__version__)\n",
    "print ('설치위치: ', matplotlib.__file__)\n",
    "print ('설정: ', matplotlib.get_configdir())\n",
    "print ('캐시: ', matplotlib.get_cachedir())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 우분투에 폰트는 /usr/share/fonts/ 디렉토리에 있다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 12\r\n",
      "drwxr-xr-x 4 root root 4096 Oct 29 02:34 \u001b[0m\u001b[01;34mtruetype\u001b[0m/\r\n",
      "drwxr-xr-x 3 root root 4096 Oct 29 00:06 \u001b[01;34mtype1\u001b[0m/\r\n",
      "drwxr-xr-x 6 root root 4096 Oct 29 00:06 \u001b[01;34mX11\u001b[0m/\r\n"
     ]
    }
   ],
   "source": [
    "ls -l /usr/share/fonts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 8\r\n",
      "drwxr-xr-x 2 root root 4096 Jun 11 15:16 \u001b[0m\u001b[01;34mdejavu\u001b[0m/\r\n",
      "drwxr-xr-x 2 root root 4096 Oct 29 02:34 \u001b[01;34mnanum\u001b[0m/\r\n"
     ]
    }
   ],
   "source": [
    "ls -l /usr/share/fonts/truetype/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 나눔 글꼴 설치\n",
    "# apt-get 명령으로 나눔글꼴(fonts-nanum)을 설치하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo apt-get install fonts-nanum*\n",
    "# $ sudo fc-cache -fv\n",
    "# 만일 다른 ttf 폰트를 가져왔다면 다음과 같이 복사하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo cp new_font.ttf /usr/share/fonts/\n",
    "# $ sudo fc-cache -fv\n",
    "# matplotlib 나눔 글꼴을 추가\n",
    "# 나눔 글꼴을 matplotlib 에 복사하고, matplotlib의 폰트 캐시를 삭제\n",
    "\n",
    "# $ sudo cp /usr/share/fonts/truetype/nanum/Nanum* /usr/local/lib/python3.4/dist-packages/matplotlib/mpl-data/fonts/ttf/\n",
    "# $ rm -rf /home/ubuntu/.cache/matplotlib/*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설치확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 133056\r\n",
      "-rw-r--r-- 1 root root  4413912 Oct  3  2014 NanumBarunGothicBold.ttf\r\n",
      "-rw-r--r-- 1 root root  4917332 Oct  3  2014 NanumBarunGothicLight.ttf\r\n",
      "-rw-r--r-- 1 root root  4184028 Oct  3  2014 NanumBarunGothic.ttf\r\n",
      "-rw-r--r-- 1 root root  4790084 Oct  3  2014 NanumBarunGothicUltraLight.ttf\r\n",
      "-rw-r--r-- 1 root root 10028540 Oct  3  2014 NanumBarunpenB.ttf\r\n",
      "-rw-r--r-- 1 root root 11033840 Oct  3  2014 NanumBarunpenR.ttf\r\n",
      "-rw-r--r-- 1 root root  3745376 Oct  3  2014 NanumBrush.ttf\r\n",
      "-rw-r--r-- 1 root root  4288380 Oct  3  2014 NanumGothicBold.ttf\r\n",
      "-rw-r--r-- 1 root root  2246240 May  6  2010 NanumGothic_Coding_Bold.ttf\r\n",
      "-rw-r--r-- 1 root root  2315924 May  6  2010 NanumGothic_Coding.ttf\r\n",
      "-rw-r--r-- 1 root root 11021172 Oct 29  2011 NanumGothicEcoBold.ttf\r\n",
      "-rw-r--r-- 1 root root 11571064 Oct 29  2011 NanumGothicEcoExtraBold.ttf\r\n",
      "-rw-r--r-- 1 root root  7783984 Oct 29  2011 NanumGothicEco.ttf\r\n",
      "-rw-r--r-- 1 root root  4171172 Oct  3  2014 NanumGothicExtraBold.ttf\r\n",
      "-rw-r--r-- 1 root root  1523188 Oct  3  2014 NanumGothicLight.ttf\r\n",
      "-rw-r--r-- 1 root root  4343844 Oct  3  2014 NanumGothic.ttf\r\n",
      "-rw-r--r-- 1 root root  4183592 Oct  3  2014 NanumMyeongjoBold.ttf\r\n",
      "-rw-r--r-- 1 root root  9809868 Oct 29  2011 NanumMyeongjoEcoBold.ttf\r\n",
      "-rw-r--r-- 1 root root 10635180 Oct 29  2011 NanumMyeongjoEcoExtraBold.ttf\r\n",
      "-rw-r--r-- 1 root root  7024632 Oct 29  2011 NanumMyeongjoEco.ttf\r\n",
      "-rw-r--r-- 1 root root  4788696 Oct  3  2014 NanumMyeongjoExtraBold.ttf\r\n",
      "-rw-r--r-- 1 root root  3839464 Oct  3  2014 NanumMyeongjo.ttf\r\n",
      "-rw-r--r-- 1 root root  3548656 Oct  3  2014 NanumPen.ttf\r\n"
     ]
    }
   ],
   "source": [
    "ls -l /usr/share/fonts/truetype/nanum/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('NanumGothicCoding',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothic_Coding_Bold.ttf'),\n",
       " ('NanumBarunpen',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunpenR.ttf'),\n",
       " ('NanumMyeongjo',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjo.ttf'),\n",
       " ('NanumMyeongjo Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjoEco.ttf'),\n",
       " ('Nanum Brush Script',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBrush.ttf'),\n",
       " ('NanumMyeongjo Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjoEcoBold.ttf'),\n",
       " ('NanumGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicExtraBold.ttf'),\n",
       " ('NanumBarunpen',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunpenB.ttf'),\n",
       " ('NanumBarunGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunGothicLight.ttf'),\n",
       " ('NanumBarunGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunGothicBold.ttf'),\n",
       " ('NanumGothic Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicEcoBold.ttf'),\n",
       " ('NanumGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicLight.ttf'),\n",
       " ('NanumGothic Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicEco.ttf'),\n",
       " ('NanumMyeongjo Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjoEcoExtraBold.ttf'),\n",
       " ('NanumGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothic.ttf'),\n",
       " ('NanumGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicBold.ttf'),\n",
       " ('NanumBarunGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunGothicUltraLight.ttf'),\n",
       " ('NanumBarunGothic',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumBarunGothic.ttf'),\n",
       " ('Nanum Pen Script',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumPen.ttf'),\n",
       " ('NanumMyeongjo',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjoExtraBold.ttf'),\n",
       " ('NanumGothic Eco',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothicEcoExtraBold.ttf'),\n",
       " ('NanumGothicCoding',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumGothic_Coding.ttf'),\n",
       " ('NanumMyeongjo',\n",
       "  '/home/companyai8way/tf110/lib/python3.5/site-packages/matplotlib/mpl-data/fonts/ttf/NanumMyeongjoBold.ttf')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# matplotlib.font_manager.fontManager.ttflist\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.font_manager\n",
    "\n",
    "[f.fname for f in matplotlib.font_manager.fontManager.ttflist]\n",
    "[(f.name, f.fname) for f in matplotlib.font_manager.fontManager.ttflist if 'Nanum' in f.name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NanumGothic\n"
     ]
    }
   ],
   "source": [
    "from matplotlib import font_manager, rc\n",
    "\n",
    "font_fname = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'\n",
    "font_name = font_manager.FontProperties(fname=font_fname).get_name()\n",
    "\n",
    "rc('font', family=font_name)\n",
    "print(font_name)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}