Lab12/lab-12-7_JijoongHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab-12-7-bonus: sequence to sequence with attention chatbot \n",
    "### simple neural machine translation chatbot\n",
    "\n",
    "* sequence to sequence|\n",
    "* variable input sequence length\n",
    "* variable output sequence length\n",
    "* Luong attention\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)\n",
    "* [Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025)\n",
    "* [Neural Machine Translation with Attention from Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "### matplotlib 한글 폰트 설정 #############################\n",
    "from matplotlib import font_manager, rc\n",
    "## for window #####\n",
    "font_name = font_manager.FontProperties(fname=\"c:/Windows/Fonts/malgun.ttf\").get_name()\n",
    "rc('font', family=font_name)\n",
    "## for mac #####\n",
    "#rc('font', family='AppleGothic') #for mac\n",
    "## linux는 맨 아래 code 참고 #####\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chat data load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### konlpy 사전 설치가 필요함\n",
    "#### http://konlpy.org/ko/v0.5.2/install/ 참고"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from konlpy.tag import Twitter\n",
    "import pandas as pd\n",
    "import enum\n",
    "import os\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadData():\n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    dataDF = pd.read_csv('./data_in/chat_ko/ChatBotData.csv', header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(dataDF['Q']), list(dataDF['A'])\n",
    "#     dataset = dataDF['Q'] + '\\t' + dataDF['A']\n",
    "#     dataset = list(dataset)\n",
    "    # skleran에서 지원하는 함수를 통해서 학습 셋과 \n",
    "    # 테스트 셋을 나눈다.\n",
    "#     xTrain, xTest, yTrain, yTest = train_test_split(question, answer, test_size=0.33, random_state=42)\n",
    "    # 그 값을 리턴한다.\n",
    "#     return xTrain, yTrain, xTest, yTest\n",
    "    return question, answer\n",
    "\n",
    "def preproLikeMorphlized(data):\n",
    "    # 형태소 분석 모듈 객체를\n",
    "    # 생성합니다.\n",
    "\n",
    "    morphAnalyzer = Twitter()\n",
    "    # 형태소 토크나이즈 결과 문장을 받을\n",
    "    #  리스트를 생성합니다.\n",
    "    result_data = list()\n",
    "    # 데이터에 있는 매 문장에 대해 토크나이즈를\n",
    "    # 할 수 있도록 반복문을 선언합니다.\n",
    "    for seq in data:\n",
    "        # Twitter.morphs 함수를 통해 토크나이즈 된\n",
    "        # 리스트 객체를 받고 다시 공백문자를 기준으로\n",
    "        # 하여 문자열로 재구성 해줍니다.\n",
    "        morphlizedSeq = \" \".join(morphAnalyzer.morphs(seq.replace(' ', '')))\n",
    "        result_data.append(morphlizedSeq)\n",
    "\n",
    "    return result_data\n",
    "\n",
    "def dataTokenizer(data):\n",
    "    # 토크나이징 해서 담을 배열 생성\n",
    "    words = []\n",
    "    for sentence in data:\n",
    "        # FILTERS = \"([~.,!?\\\"':;)(])\"\n",
    "        # 위 필터와 같은 값들을 정규화 표현식을 \n",
    "        # 통해서 모두 \"\" 으로 변환 해주는 부분이다.\n",
    "        sentence = re.sub(change_filter, \"\", sentence)\n",
    "        for word in sentence.split():\n",
    "            words.append(word)\n",
    "    # 토그나이징과 정규표현식을 통해 만들어진 \n",
    "    # 값들을 넘겨 준다.\n",
    "    return [word for word in words if word]\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "    \n",
    "    # creating a space between a word and the punctuation following it\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\" \n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    \n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    \n",
    "    return w\n",
    "\n",
    "def create_dataset(data, num_examples):\n",
    "#     word_pairs = [[preprocess_sentence(w) for w in l]  for l in data[:num_examples]]\n",
    "    word_pairs = [[preprocess_sentence(l)]  for l in data[:num_examples]]\n",
    "    return word_pairs\n",
    "\n",
    "class LanguageIndex():\n",
    "    def __init__(self, lang):\n",
    "        self.lang = lang\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "        self.vocab = set()\n",
    "\n",
    "        self.create_index()\n",
    "        \n",
    "    def create_index(self):\n",
    "        for phrase in self.lang:\n",
    "#             if self.vocab == '<start>':\n",
    "#                 pass\n",
    "            self.vocab.update(phrase.split(' '))\n",
    "    \n",
    "        self.vocab = sorted(self.vocab)\n",
    "    \n",
    "        self.word2idx['<pad>'] = 0\n",
    "#         self.word2idx['<start>'] = 1\n",
    "#         self.word2idx['<end>'] = 2\n",
    "#         self.word2idx['<unk>'] = 3\n",
    "\n",
    "        for index, word in enumerate(self.vocab):\n",
    "            self.word2idx[word] = index + 1\n",
    "\n",
    "        for word, index in self.word2idx.items():\n",
    "            self.idx2word[index] = word\n",
    "            \n",
    "def max_length(tensor):\n",
    "    return max(len(t) for t in tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(path, num_examples):\n",
    "    \n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    data_df = pd.read_csv(path, header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(data_df['Q']), list(data_df['A'])\n",
    "    \n",
    "    question = create_dataset(question, num_examples)\n",
    "    answer = create_dataset(answer, num_examples)\n",
    "    \n",
    "    print(len(question))\n",
    "    \n",
    "    pairs = [question[i] + answer[i] for i in range(num_examples)]\n",
    "    \n",
    "    # index language using the class defined above    \n",
    "    inp_lang = LanguageIndex(q for q, a in pairs)\n",
    "    targ_lang = LanguageIndex(a for q, a in pairs)\n",
    "    \n",
    "    # Vectorize the input and target languages\n",
    "    \n",
    "    # question\n",
    "    input_tensor = [[inp_lang.word2idx[s] for s in q.split(' ')] for q, a in pairs]\n",
    "    \n",
    "    # answer\n",
    "    target_tensor = [[targ_lang.word2idx[s] for s in a.split(' ')] for q, a in pairs]\n",
    "\n",
    "    # Calculate max_length of input and output tensor\n",
    "    # Here, we'll set those to the longest sentence in the dataset\n",
    "    max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n",
    "    \n",
    "    # Padding the input and output tensor to the maximum length\n",
    "    input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n",
    "                                                                 maxlen=max_length_inp,\n",
    "                                                                 padding='post')\n",
    "    \n",
    "    target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n",
    "                                                                  maxlen=max_length_tar, \n",
    "                                                                  padding='post')\n",
    "    \n",
    "    return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11823\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9458, 9458, 2365, 2365)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset with limit\n",
    "\n",
    "num_examples = 11823\n",
    "\n",
    "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset('./data_in/ChatBotData.csv', num_examples)\n",
    "\n",
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length_targ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rgCLkfv5uO3d"
   },
   "source": [
    "### Create a tf.data dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TqHsArVZ3jFS"
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "N_BATCH = BUFFER_SIZE//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word2idx)\n",
    "vocab_tar_size = len(targ_lang.word2idx)\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) #drop_reainder가 True이면, batch size가 적지 않도록 수정\n",
    "\n",
    "# The tensors in the resulting element will have an additional outer dimension, which will be batch_size (or N % batch_size for the last element if batch_size does not divide the number of input elements N evenly and drop_remainder is False). If your program depends on the batches having the same outer dimension, you should set the drop_remainder argument to True to prevent the smaller batch from being produced."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TNfHIF71ulLu"
   },
   "source": [
    "## Write the encoder and decoder model\n",
    "\n",
    "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://www.tensorflow.org/tutorials/seq2seq). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://www.tensorflow.org/tutorials/seq2seq#background_on_the_attention_mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\">\n",
    "\n",
    "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n",
    "\n",
    "Here are the equations that are implemented:\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\">\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\">\n",
    "\n",
    "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n",
    "\n",
    "* FC = Fully connected (dense) layer\n",
    "* EO = Encoder output\n",
    "* H = hidden state\n",
    "* X = input to the decoder\n",
    "\n",
    "And the pseudo-code:\n",
    "\n",
    "* `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "* `merged vector = concat(embedding output, context vector)`\n",
    "* This merged vector is then given to the GRU\n",
    "  \n",
    "The shapes of all the vectors at each step have been specified in the comments in the code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "avyJ_4VIUoHb"
   },
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_activation='sigmoid', \n",
    "                               recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nZ2rI24i3jFg"
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        print(\"state: {}\".format(state.shape))\n",
    "        print(\"output: {}\".format(state.shape))\n",
    "              \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yJ_B3mhW3jFk"
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        # * `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "                \n",
    "        #* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        # * `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        # * `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        # * `merged vector = concat(embedding output, context vector)`\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P5UY8wko3jFp"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13422 9856 256 1024 64\n"
     ]
    }
   ],
   "source": [
    "print(vocab_inp_size, vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_ch_71VbIRfK"
   },
   "source": [
    "## Define the optimizer and the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WmTHr5iV3jFr"
   },
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DMVWzzsfNl4e"
   },
   "source": [
    "## Checkpoints (Object-based saving)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zj8bXQTgNwrF"
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hpObfY22IddU"
   },
   "source": [
    "## Training\n",
    "\n",
    "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n",
    "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n",
    "3. The decoder returns the *predictions* and the *decoder hidden state*.\n",
    "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
    "5. Use *teacher forcing* to decide the next input to the decoder.\n",
    "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n",
    "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ddefjBMa3jF0",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Batch 0 Loss 2.0834\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Batch 100 Loss 1.2099\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Loss 1.3388\n",
      "Time taken for 1 epoch 65.33734321594238 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 1\n",
    "\n",
    "#GPU 설정\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch, (inp, targ)) in enumerate(dataset):\n",
    "        loss = 0\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(inp, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([targ_lang.word2idx['<start>']] * BATCH_SIZE, 1)\n",
    "            \n",
    "            # Teacher forcing - feeding the target as the next input\n",
    "            for t in range(1, targ.shape[1]):\n",
    "                # passing enc_output to the decoder\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(targ[:, t], predictions)\n",
    "                \n",
    "                # using teacher forcing\n",
    "                dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "        \n",
    "        batch_loss = (loss / int(targ.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradients = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradients, variables))\n",
    "        \n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                         batch,\n",
    "                                                         batch_loss.numpy()))\n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    if (epoch + 1) % 2 == 0:\n",
    "          checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "    \n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                        total_loss / N_BATCH))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mU3Ce8M6I3rz"
   },
   "source": [
    "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
    "* Stop predicting when the model predicts the *end token*.\n",
    "* And store the *attention weights for every time step*.\n",
    "\n",
    "Note: The encoder output is calculated only once for one input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EbQpyYs13jF_"
   },
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "    sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang.word2idx['<start>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += targ_lang.idx2word[predicted_id] + ' '\n",
    "\n",
    "        if targ_lang.idx2word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s5hQWlbN3jGF"
   },
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sl9zUHzg3jGI"
   },
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n250XbnjOaqP"
   },
   "source": [
    "## Restore the latest checkpoint and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UJpT9D5_OgP6",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.InitializationOnlyStatus at 0x162185e1cc8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WrAM0FDomq3E"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: (1, 1024)\n",
      "output: (1, 1024)\n",
      "Input: <start> 뭐라도 배워볼까 . <end>\n",
      "Predicted translation: 좋은 . <end> \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm8AAAIPCAYAAAAlwjnSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAayElEQVR4nO3debStB1nf8d9D7s0EJRISEDCKIQp2WSsaEBEQtSpasA5Ii8WgUFOtRS1FpVoV55o6sEBrGylYFcQBtLJQRgcEx2ip1KGkJIwJkpAEEMj89I+9L+6c3Js75Z73POd8Pmuddfd+997nPHe9957zPe9+h+ruAAAww12WHgAAgCMn3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gCOUlU9tqrOWHoOYG8SbwBH7+lJ7rf0EMDetG/pAQB2sqp61EEW3yPJQ6rqrEO87JLu/tAJHAvYw8qF6QEOrap+/yhf0km+prsvOxHzAIg3AIBBvG0KcBhV9aQk70tybZIrbFUDlmTLG8BhVNWHkrwsyVlZHahwjyR/nOSFSV7S3TcvOB6wx4g3gMOoqiu7+z4b989M8sgkT03y6Um+q7ufv9R8wN4i3gAOo6pe0t1fcYjHzkvyvCRv7+4LtncyYC8SbwCHUVWvSXJ1kvck+aMkr+7uqzYeryQP6u6/XmhEYA9xkt5doKrOrKpzlp4DdrFPTfLzSd6U5DFJ/qqqfqGqzk2SXhFuwLaw5W0XqKrnJrlnd3/V0rPAbnSQfd5OTvItBz66+5cXGw7Yc2x5G279m/9Dk3z0et8b4M53m4MRuvvG7r4oyaOSPOsOrrQAcKez5W24qnpRkv+RVYg/vrufuvBIHKOqul93v2vpOTg6VbW/u29aeg5g77DlbbCq+rQk9+7uV3b3byX5FPu+jfanSw/Akauq+ySJcIPtV1WfWVWPW3qOpYi32X44yTM37v94km9baBbuQFXd7SDLHrF10TaNw53j95YeAPawi5J819JDLEW8DVVV903yuu7e3FrzS0luqirrdec52OWUXrrlvn0YdqiqOn3j49QDi9eP3buq7ntgS1xVfe1ig8IeUFVfmuT/JvmT9e09xz5vsA2q6qruPvuOllXVFd193+2fjsOpqg8n2Z/kpqxOxvvAqnpzd39iVf1tkluTnN7dZxxYvujAsEutN05ckuRxWf2ffFF3P3zZqbafLTSDHezo0qq6S1V9/BLzcIcO9luS35yG6O7Tkvz1+s+t3zffvz6NyIGtq97+hhPnKUle293v6u63Jrmsqj5/4Zm23b6lB+C4vCrJuZsLuvvWqvqtJA9aZiSOQlXVDx24neR2+8Wxo/SWPw+3HLgTVdVpSf5dks/aWPwjSZ6b5NWLDLUQ8TZQVT0hq3V3t6raemLeByQ5Zfun4hjdsvQAHLV7VtX3xfdP2G6nJPnK7r7uwILuflNVfXtV3aW7b11wtm3lm89Mn5vkgVltqfm6LY9dk+RJ2z4Rx6K7+yNHS1XVv1pyGI6K6IZtto626w6y/I8XGGdR4m2g7v76JKmqy7v7c5aehzuNt91meG93f29V/culBwH2JvE2VFVVkj3328ZgZ1XVjRv3Kw4Y2q1EONyJqurVOYL/V939Bdswzo4g3obq7q6qT66qfd1989LzcFinLT0Ax66qbkpy0vrPy7c+vP7ztKp6R5J7b+twsPv9wsbtByV5QlYHKVyb1UF7FyT5sQXmWozzvA22ftvmi5L8lyRvz+pcU0mS7r5iqbk4NlV15fqUE+wwVbV5ENCt3X3Txnne7tvdV6xPnL0/Sbr7bctMCrtbVb0hyQXd/ZaNZecn+dbu/ufLTba9xNtgVXWoI2u6u0/a1mE4blX10d397qXn4MhU1aXd/QlLzwF7SVW9rbs/7iDLL+/uPXOOU/vcDNbddznEh3Db4arq4VX1gM1lwm2crdemBU6866rqNucxraqPzR7rmT31l4Ud5IlJPmPpITh23f23S88Ae9D3J3llVX1jVX1uVX1NVies/+/LjrW9HLAw2HrLzX9O8g+TnLz5WHefe9AXse3WvxVudbesjkA92GNJcoUDUQBuq7t/taquTfK0JN+Q5Ookz05y8aKDbTP7vA1WVa9L8kdJPpTkfklemuTpSV7c3Xvqt5CdbH0EYufIr3nZSb6gu//mxE3FkaqqK3Pkp/+o9XM/r7v/+sRNBexl4m2wAztuVtVDknxdd19YVfdI8vLufvjS88FuUFW32zn6IL4jyZuSvGx9/4ruvunETQV7V1Wdm+TBSe66uby7f26Zibaft01nu6WqTkry5iSflCTdfW1VfcyyY3G0quoyb3XvTHd02o+q+sEkL0ry/iTvcYoQOLGq6puSXJTkjVmd5+2ATiLeGOGVWZ3v5gVVdWtVfVeSM5NcufBcbFFVz9+y6PLu/v6N+7dW1X5ba3auqnpMki/J6pel53b3LUnOSnLPRQeDveXbkjy8u/986UGW5GjT2Z6W5MXr2xckOS/JfZI8ebGJOJQvT/KG9cdfJXnclsevS3L37R6KI1NVX5nVGdz/JslDk/z8+qEPZXX1jJ9O8jvLTAd7yvV7PdwSW96me0R3/27ykbd2nrx+G/Vhi07FwXz4wEEkVXW/rC7vcpvHk5ye5L3bPRhH5NuTPK67L0vynKr6g/W+cDckOWW9HDjxXlZVn9/dr156kCWJt9men9V13T6iu2+pqp9N4szvO8vhjgy6Kckph3kOy7nHlkD7k6z+j92Y5Geq6tnr5ZXVFU7svwgnxhlJfqWqXp/kNpeB7O4Llxlp+4m3garq6UlOTfJRVfUdWx5+QFYhwCy3JHFljJ3r5qo6ubtvXN+/V5Jrsvq/9gNJfnOxyWBveVuSn1h6iKWJt5lOy2r/tn25/Ra2a5J82bZPxOFsPcfbJ1XVH2w89qDYB3Une1mSi9YHBX1mkk/O6mi3L0xy9eZFsoETp7u/d+kZdgLxNlB3/2CSVNV53f21S8/DEfnyjdtXJXnsQZ7z1u0ZhWPw3Un+a5K3J7k0yVd1961V1fF9FLZVVX1hkscnOaO7n7C+Us0Ne+mSdb7pDNbdj9y8X1X3zmrn6bcvNBKH0N1/uHH7xiS/t+A4HKXu/lBWR3Tf7qH4PgrbZn2et2/I6lqm37JefK+sTiGy9UCwXcsVFgarqt9O8q+7+9Kq+owkr8lq36lv7O4XLjsd7H5VdWpWv/H7RgrboKr+Jskju/uqqrq8uz9+vfzS7t4zB+r5jXG2B3b3pevb35fkqVmda+qVScTbDlFVt+bQR5vWlscO3H9wd//FiZ6Nw6uqk+/g4VuT7K+67S6NGwc2AHeu/d191fr25vfOPXW0vh2kZ/tAVZ1UVQ9M8oDu/uX1P+qPWnowbmN/kpMP8XHK+uNXknzOgecKtx3l+qzOw3f9xseHD7LsI49V1acsMyrsev+7qr5+c0FVfVmSPXXQkC1vs/1akt/O6v3+H02Sqjoz1uuOsr6M0kFV1f/L6lx9neTmO3ouy+ju2/2SW1Xfk9VRpj+1wEiwlz0tyWur6klJzqqql2V1kfovXnas7eWH/GzfmdWlsK7v7l9cL7t/VmeDZwepqu/esuia7v7JrC6vdPoCI3EUqmrrwQr/OKst3x842PO7e89cIBu2U3e/a71l+7FZ/eJ7RZIndff7lp1sezlgAbZBVb0vyXPWd09K8vju/sSq+tMkj8lq36m/c2H6namqXnAUT+/ufsoJGwbY88TbUFV1TpL/1t1fvLHs7kl+Pck/6e5bFxuO26mqd3T3Oevbpyb5i3W8vT7JE5O8a/P51h/Abfm59/ccsDBUd78jyZVVtXlem29I8oq99A94kD7E7RuzutzLTeuPm+PyZjtOVV1ZVVfcwcfLl54Rdjs/9/6efd5m+54kL6+ql2R1lOIFSR667EgcpZuSPKi737z0INyhh23cviDJ3ZP85Pr+OUl+fNsngr3Jz72It9G6+51V9YokX5fV+cF+ubs/uPBYHNxdN3Z63zxv2M1xQfodr7vfduB2VV2T5JYDy7ae4w04cfzcWxFv8/1wkjdkFQGfvfAsHNpvZHUetwNesv6zI95gcVV1z+5+79JzcET2/M898TZcd19XVT+b5J7dfd3S83Bw3f21h3jo5qyONGUHWx9temBfxU/K6qoK563v/4NlpuJO9qu57S9Y7FB+7jnadFeoqn1JTuruG5aeBXajqnryYZ5ydXc7aGGwqjp747JL7HB7/eeeeAMAGMSpQgAABhFvAACDiLddoqouXHoGjo11N5v1N5d1N9teXn/ibffYs/+IdwHrbjbrby7rbrY9u/7EGwDAIHvmaNOT65Q+NXddeowT5qbckP05ZekxOAbW3WzW31y7fd3VXXb39pkb+/qcXKcuPcYJ8/5b33t1d599sMf2zEl6T81d8xn1eUuPATCLy3+NdZfTTlt6BI7Dqz74c2871GO7O8sBAHYZ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIPu26wtV1cVJPndj0dlJKsl7Npa9rrufsuV1Zyb590n+WZLTk3SS65P8RpIf7e73nsi5AQB2km2Lt+6+MEmq6u5JvirJE5PckOTXk7ywu993iJe+KsnvJnlUd1+z/hz3SPLMJL9dVZ/a3X2CxwcA2BG2Jd6q6n5JLkpyvyTXJfmlJN+Z1Va0eyd5flWdneSdSf5jd1+2ft0ZST49yaO7++8OfL7uvraqvjfJt2a1BW9z6x0AwK61LfHW3e+qqmckefeBrWRV9QNJ9nX3M5O8tKpOSnLegXBbv+59VfWbSZ5XVf8pyZuzCr5PyGrL22u7W7gBAHvGCY+3qvroJH+0cf/AzY9a3a1/seX5SfKI7n7netGXJXlykh9Kcs562Tuzerv1Z0/U3AAAO9EJj7fufneS+29dvt73Ld39/sO8/sYkP7P+AADY07bzaNNzkpy0sehbkpxUVT+2seyW7n5HVd0nyZuO8ks8uLvfseVrXpjkwiQ5Nacfw9QAADvLtsVbkm9OcreDLH/mxu2/S/KM7r4yyVnH+wW7++IkFyfJ3etMR6QCAONt56lCnlFVpyX5t0kendVRosnqSNHfSfJT3X395muq6q5JXtDdT9iuOQEAdrLt3PKWJM9LcnNWW+HemdWRox+b5Luz2qftq7c8f3+SL9nOAQEAdrLtvjzWWUkuS3J5d1/f3TckeUuS/5PkY7Z5FgCAcbZ7y9tTszrlx5uq6uastrztT/LGJE85xGtOqap338HnfHp3v+jOHRMAYGfa1nhbn7vtgqN4/nVZXf8UAIBs/9umAAAcB/EGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCD7lh4AjsTlP/yZS4/AMXrzk3966RE4Dl/4pV+99Agco3rLu5YegePxwUM/ZMsbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIPsW3qAE6mqLkxyYZKcmtMXngYA4Pjt6i1v3X1xd5/f3efvzylLjwMAcNx2dbwBAOw24g0AYBDxBgAwyK6It6q6V1X9XlWdvfQsAAAn0q442rS735Pks5eeAwDgRNsVW94AAPYK8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIPuWHmC71L59OenMs5ceg2P0q0/8iaVH4Bj95Y299Agch5M+cP3SI3CM+oMfWnoEThBb3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEF2bLxV1ddU1WuWngMAYCfZsfEGAMDtHVe8VdXH3lmDHMHXukdVnb5dXw8AYCc6pnirqo+pqucnuXjLsl+rqsuq6s1V9c0bjz2rql5QVc+uqrdU1RWbj6+f8/iq+ouqekdV/WGSB235svdNcklVXVhVJx3L3AAA0x1VvK23fl2U5LeSvCrJF62Xn5LktUle093nJnlkkn9TVV+08fKvSPKK7n5Akscluaiqzlu//vOS/ESSJ3b3OUkuSPKEza/d3X+Z5BFJPjHJn1XVlx/tXxYAYLojireq2l9V35bkDUnemuTTuvvF3d3rpzw2yXu7+6eSpLv/Nsnzkjx+49O8vrtfsX78z5K8McmD1499U5KL1oGW7r40yXO3ztHd13T3M7KKv39aVa+vqs+6g7kvrKpLquqSG2/98JH8VQEAdrR9R/i8k5LcM8ktSWr9ups2Hj83yT+qqrduLDs5yR9u3L9iy+e8Nsld17fPy+1j7do7mGdfkuuTnJrkrEM9qbsvzvqt3TP236sP9TwAgCmOaMtbd1/f3d+e5LOTnJ3kz6vqP1TVGeunXJHkdd19/42P+3b3VxzhHFcn2Xrww7lbn1RVn1JVL0zyi1m9TfuQ7v6fR/g1AADGO6p93tZvWz4ryUOT3JrkQDi9PMmDN/dDq6qHVNX9j/BTvzjJM6vq49avPT+r/d4+oqoeluQnk/x8dz+su1+68bYtAMCecKRvm95Gd38gyY9U1Y+t719XVV+c5NlV9ZwkNyT5X0medoSf8uIk907y+1XVSf4kyY8n+ZKN51zS3Y86lnkBAHaLY4q3A7r75o3bb0zy6EM871kHWfaYjdu3JHnW+mPTcw72tQAA9ipXWAAAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGGTf0gNsl7755txy1VVLj8Ex+tb7P2zpEWCPunTpAYAtbHkDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwyL6lBziRqurCJBcmyak5feFpAACO367e8tbdF3f3+d19/v6csvQ4AADHbVfHGwDAbiPeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg1d1Lz7AtquqqJG9beo4T6KwkVy89BMfEupvN+pvLupttt6+/j+vusw/2wJ6Jt92uqi7p7vOXnoOjZ93NZv3NZd3NtpfXn7dNAQAGEW8AAIOIt93j4qUH4JhZd7NZf3NZd7Pt2fVnnzcAgEFseQMAGES8AQAMIt4AAAYRbwAAg4g3AIBB/j+2QfIPZnZHCAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translate('뭐라도 배워볼까.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 한글 폰트가 깨질 때\n",
    "\n",
    "https://financedata.github.io/posts/matplotlib-hangul-for-ubuntu-linux.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#실행중인 운영체제 확인\n",
    "\n",
    "import platform\n",
    "print(platform.platform())\n",
    "\n",
    "# 파이썬 버전\n",
    "import sys\n",
    "print(sys.version_info)\n",
    "\n",
    "# matplotlib 주요 설치 정보\n",
    "\n",
    "import matplotlib\n",
    "\n",
    "print ('버전: ', matplotlib.__version__)\n",
    "print ('설치위치: ', matplotlib.__file__)\n",
    "print ('설정: ', matplotlib.get_configdir())\n",
    "print ('캐시: ', matplotlib.get_cachedir())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 우분투에 폰트는 /usr/share/fonts/ 디렉토리에 있다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/truetype/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 나눔 글꼴 설치\n",
    "# apt-get 명령으로 나눔글꼴(fonts-nanum)을 설치하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo apt-get install fonts-nanum*\n",
    "# $ sudo fc-cache -fv\n",
    "# 만일 다른 ttf 폰트를 가져왔다면 다음과 같이 복사하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo cp new_font.ttf /usr/share/fonts/\n",
    "# $ sudo fc-cache -fv\n",
    "# matplotlib 나눔 글꼴을 추가\n",
    "# 나눔 글꼴을 matplotlib 에 복사하고, matplotlib의 폰트 캐시를 삭제\n",
    "\n",
    "# $ sudo cp /usr/share/fonts/truetype/nanum/Nanum* /usr/local/lib/python3.4/dist-packages/matplotlib/mpl-data/fonts/ttf/\n",
    "# $ rm -rf /home/ubuntu/.cache/matplotlib/*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설치확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/truetype/nanum/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# matplotlib.font_manager.fontManager.ttflist\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.font_manager\n",
    "\n",
    "[f.fname for f in matplotlib.font_manager.fontManager.ttflist]\n",
    "[(f.name, f.fname) for f in matplotlib.font_manager.fontManager.ttflist if 'Nanum' in f.name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import font_manager, rc\n",
    "\n",
    "font_fname = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'\n",
    "font_name = font_manager.FontProperties(fname=font_fname).get_name()\n",
    "\n",
    "rc('font', family=font_name)\n",
    "print(font_name)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}