Lab12/Lab12-7_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab-12-7-bonus: sequence to sequence with attention chatbot \n",
    "### simple neural machine translation chatbot\n",
    "\n",
    "* sequence to sequence|\n",
    "* variable input sequence length\n",
    "* variable output sequence length\n",
    "* Luong attention\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)\n",
    "* [Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025)\n",
    "* [Neural Machine Translation with Attention from Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "### matplotlib 한글 폰트 설정 #############################\n",
    "from matplotlib import font_manager, rc\n",
    "## for window #####\n",
    "font_name = font_manager.FontProperties(fname=\"c:/Windows/Fonts/malgun.ttf\").get_name()\n",
    "rc('font', family=font_name)\n",
    "## for mac #####\n",
    "#rc('font', family='AppleGothic') #for mac\n",
    "## linux는 맨 아래 code 참고 #####\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chat data load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### konlpy 사전 설치가 필요함\n",
    "#### http://konlpy.org/ko/v0.5.2/install/ 참고"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from konlpy.tag import Twitter\n",
    "import pandas as pd\n",
    "import enum\n",
    "import os\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadData():\n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    dataDF = pd.read_csv('./data_in/chat_ko/ChatBotData.csv', header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(dataDF['Q']), list(dataDF['A'])\n",
    "#     dataset = dataDF['Q'] + '\\t' + dataDF['A']\n",
    "#     dataset = list(dataset)\n",
    "    # skleran에서 지원하는 함수를 통해서 학습 셋과 \n",
    "    # 테스트 셋을 나눈다.\n",
    "#     xTrain, xTest, yTrain, yTest = train_test_split(question, answer, test_size=0.33, random_state=42)\n",
    "    # 그 값을 리턴한다.\n",
    "#     return xTrain, yTrain, xTest, yTest\n",
    "    return question, answer\n",
    "\n",
    "def preproLikeMorphlized(data):\n",
    "    # 형태소 분석 모듈 객체를\n",
    "    # 생성합니다.\n",
    "\n",
    "    morphAnalyzer = Twitter()\n",
    "    # 형태소 토크나이즈 결과 문장을 받을\n",
    "    #  리스트를 생성합니다.\n",
    "    result_data = list()\n",
    "    # 데이터에 있는 매 문장에 대해 토크나이즈를\n",
    "    # 할 수 있도록 반복문을 선언합니다.\n",
    "    for seq in data:\n",
    "        # Twitter.morphs 함수를 통해 토크나이즈 된\n",
    "        # 리스트 객체를 받고 다시 공백문자를 기준으로\n",
    "        # 하여 문자열로 재구성 해줍니다.\n",
    "        morphlizedSeq = \" \".join(morphAnalyzer.morphs(seq.replace(' ', '')))\n",
    "        result_data.append(morphlizedSeq)\n",
    "\n",
    "    return result_data\n",
    "\n",
    "def dataTokenizer(data):\n",
    "    # 토크나이징 해서 담을 배열 생성\n",
    "    words = []\n",
    "    for sentence in data:\n",
    "        # FILTERS = \"([~.,!?\\\"':;)(])\"\n",
    "        # 위 필터와 같은 값들을 정규화 표현식을 \n",
    "        # 통해서 모두 \"\" 으로 변환 해주는 부분이다.\n",
    "        sentence = re.sub(change_filter, \"\", sentence)\n",
    "        for word in sentence.split():\n",
    "            words.append(word)\n",
    "    # 토그나이징과 정규표현식을 통해 만들어진 \n",
    "    # 값들을 넘겨 준다.\n",
    "    return [word for word in words if word]\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "    \n",
    "    # creating a space between a word and the punctuation following it\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\" \n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    \n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    \n",
    "    return w\n",
    "\n",
    "def create_dataset(data, num_examples):\n",
    "#     word_pairs = [[preprocess_sentence(w) for w in l]  for l in data[:num_examples]]\n",
    "    word_pairs = [[preprocess_sentence(l)]  for l in data[:num_examples]]\n",
    "    return word_pairs\n",
    "\n",
    "class LanguageIndex():\n",
    "    def __init__(self, lang):\n",
    "        self.lang = lang\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "        self.vocab = set()\n",
    "\n",
    "        self.create_index()\n",
    "        \n",
    "    def create_index(self):\n",
    "        for phrase in self.lang:\n",
    "#             if self.vocab == '<start>':\n",
    "#                 pass\n",
    "            self.vocab.update(phrase.split(' '))\n",
    "    \n",
    "        self.vocab = sorted(self.vocab)\n",
    "    \n",
    "        self.word2idx['<pad>'] = 0\n",
    "#         self.word2idx['<start>'] = 1\n",
    "#         self.word2idx['<end>'] = 2\n",
    "#         self.word2idx['<unk>'] = 3\n",
    "\n",
    "        for index, word in enumerate(self.vocab):\n",
    "            self.word2idx[word] = index + 1\n",
    "\n",
    "        for word, index in self.word2idx.items():\n",
    "            self.idx2word[index] = word\n",
    "            \n",
    "def max_length(tensor):\n",
    "    return max(len(t) for t in tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(path, num_examples):\n",
    "    \n",
    "    # 판다스를 통해서 데이터를 불러온다.\n",
    "    data_df = pd.read_csv(path, header=0)\n",
    "    # 질문과 답변 열을 가져와 question과 answer에 넣는다.\n",
    "    question, answer = list(data_df['Q']), list(data_df['A'])\n",
    "    \n",
    "    question = create_dataset(question, num_examples)\n",
    "    answer = create_dataset(answer, num_examples)\n",
    "    \n",
    "    print(len(question))\n",
    "    \n",
    "    pairs = [question[i] + answer[i] for i in range(num_examples)]\n",
    "    \n",
    "    # index language using the class defined above    \n",
    "    inp_lang = LanguageIndex(q for q, a in pairs)\n",
    "    targ_lang = LanguageIndex(a for q, a in pairs)\n",
    "    \n",
    "    # Vectorize the input and target languages\n",
    "    \n",
    "    # question\n",
    "    input_tensor = [[inp_lang.word2idx[s] for s in q.split(' ')] for q, a in pairs]\n",
    "    \n",
    "    # answer\n",
    "    target_tensor = [[targ_lang.word2idx[s] for s in a.split(' ')] for q, a in pairs]\n",
    "\n",
    "    # Calculate max_length of input and output tensor\n",
    "    # Here, we'll set those to the longest sentence in the dataset\n",
    "    max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n",
    "    \n",
    "    # Padding the input and output tensor to the maximum length\n",
    "    input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n",
    "                                                                 maxlen=max_length_inp,\n",
    "                                                                 padding='post')\n",
    "    \n",
    "    target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n",
    "                                                                  maxlen=max_length_tar, \n",
    "                                                                  padding='post')\n",
    "    \n",
    "    return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11823\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9458, 9458, 2365, 2365)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset with limit\n",
    "\n",
    "num_examples = 11823\n",
    "\n",
    "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset('./data_in/ChatBotData.csv', num_examples)\n",
    "\n",
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length_targ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rgCLkfv5uO3d"
   },
   "source": [
    "### Create a tf.data dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TqHsArVZ3jFS"
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "N_BATCH = BUFFER_SIZE//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word2idx)\n",
    "vocab_tar_size = len(targ_lang.word2idx)\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) #drop_reainder가 True이면, batch size가 적지 않도록 수정\n",
    "\n",
    "# The tensors in the resulting element will have an additional outer dimension, which will be batch_size (or N % batch_size for the last element if batch_size does not divide the number of input elements N evenly and drop_remainder is False). If your program depends on the batches having the same outer dimension, you should set the drop_remainder argument to True to prevent the smaller batch from being produced."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TNfHIF71ulLu"
   },
   "source": [
    "## Write the encoder and decoder model\n",
    "\n",
    "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://www.tensorflow.org/tutorials/seq2seq). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://www.tensorflow.org/tutorials/seq2seq#background_on_the_attention_mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\">\n",
    "\n",
    "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n",
    "\n",
    "Here are the equations that are implemented:\n",
    "\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\">\n",
    "<img src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\">\n",
    "\n",
    "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n",
    "\n",
    "* FC = Fully connected (dense) layer\n",
    "* EO = Encoder output\n",
    "* H = hidden state\n",
    "* X = input to the decoder\n",
    "\n",
    "And the pseudo-code:\n",
    "\n",
    "* `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "* `merged vector = concat(embedding output, context vector)`\n",
    "* This merged vector is then given to the GRU\n",
    "  \n",
    "The shapes of all the vectors at each step have been specified in the comments in the code:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "avyJ_4VIUoHb"
   },
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_activation='sigmoid', \n",
    "                               recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nZ2rI24i3jFg"
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        print(\"state: {}\".format(state.shape))\n",
    "        print(\"output: {}\".format(state.shape))\n",
    "              \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yJ_B3mhW3jFk"
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        # * `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "                \n",
    "        #* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        # * `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        # * `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        # * `merged vector = concat(embedding output, context vector)`\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P5UY8wko3jFp"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13422 9856 256 1024 64\n"
     ]
    }
   ],
   "source": [
    "print(vocab_inp_size, vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_ch_71VbIRfK"
   },
   "source": [
    "## Define the optimizer and the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WmTHr5iV3jFr"
   },
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DMVWzzsfNl4e"
   },
   "source": [
    "## Checkpoints (Object-based saving)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zj8bXQTgNwrF"
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hpObfY22IddU"
   },
   "source": [
    "## Training\n",
    "\n",
    "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n",
    "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n",
    "3. The decoder returns the *predictions* and the *decoder hidden state*.\n",
    "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n",
    "5. Use *teacher forcing* to decide the next input to the decoder.\n",
    "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n",
    "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ddefjBMa3jF0",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Batch 0 Loss 2.2049\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Batch 100 Loss 1.2208\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "state: (64, 1024)\n",
      "output: (64, 1024)\n",
      "Epoch 1 Loss 1.3362\n",
      "Time taken for 1 epoch 701.6589603424072 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 1\n",
    "\n",
    "#GPU 설정\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch, (inp, targ)) in enumerate(dataset):\n",
    "        loss = 0\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(inp, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([targ_lang.word2idx['<start>']] * BATCH_SIZE, 1)\n",
    "            \n",
    "            # Teacher forcing - feeding the target as the next input\n",
    "            for t in range(1, targ.shape[1]):\n",
    "                # passing enc_output to the decoder\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(targ[:, t], predictions)\n",
    "                \n",
    "                # using teacher forcing\n",
    "                dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "        \n",
    "        batch_loss = (loss / int(targ.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradients = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradients, variables))\n",
    "        \n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                         batch,\n",
    "                                                         batch_loss.numpy()))\n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    if (epoch + 1) % 2 == 0:\n",
    "          checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "    \n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                        total_loss / N_BATCH))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mU3Ce8M6I3rz"
   },
   "source": [
    "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n",
    "* Stop predicting when the model predicts the *end token*.\n",
    "* And store the *attention weights for every time step*.\n",
    "\n",
    "Note: The encoder output is calculated only once for one input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EbQpyYs13jF_"
   },
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "    sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang.word2idx['<start>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += targ_lang.idx2word[predicted_id] + ' '\n",
    "\n",
    "        if targ_lang.idx2word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s5hQWlbN3jGF"
   },
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sl9zUHzg3jGI"
   },
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n250XbnjOaqP"
   },
   "source": [
    "## Restore the latest checkpoint and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UJpT9D5_OgP6"
   },
   "outputs": [],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WrAM0FDomq3E"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state: (1, 1024)\n",
      "output: (1, 1024)\n",
      "Input: <start> 뭐라도 배워볼까 . <end>\n",
      "Predicted translation: . . <end> \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tnfls\\AppData\\Local\\Temp/ipykernel_16220/1800777997.py:9: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
      "C:\\Users\\tnfls\\AppData\\Local\\Temp/ipykernel_16220/1800777997.py:10: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
      "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm8AAAIPCAYAAAAlwjnSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZAklEQVR4nO3de7RmB1nf8d9DZpJJoFwTkCACIeViK0gJV7m0ReVSsBWQlpbFzZKWsqKWWmx1oailtdQLBewltWBFLktAWlm0INQWBbEQLIUl0KBAoIRCQhLEhpBJ8vSP9x08OZnJXJJz9nnO+XzWOmved+/3zDyz9sw537PffanuDgAAM9xi6QEAADh24g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g3gOFXVE6vqNkvPAexN4g3g+L0wyV2WHgLYm/YtPQDATlZVjzrM4tsleVBVnX6ET7ugu6/cwrGAPazcmB7gyKrqd47zUzrJs7v7U1sxD4B4AwAYxNumAEdRVc9I8pUklye52F41YEn2vAEcRVVdmeRtSU7P6kSF2yf5vSSvS/KW7r5mwfGAPUa8ARxFVX2hu++84fntkzwyyfcneWCSF3f3q5eaD9hbxBvAUVTVW7r7KUdYd3aSX0pyUXc/a3snA/Yi8QZwFFX17iSXJvlSVm+Xvqu7L9mwvpLcp7s/vtCIwB7iIr27QFX9lap6ztJzwC727Ulem+SjSR6X5GNV9atVdVaS9IpwA7aFPW/DVdUtknwgqzOHH9jd1y48Euw6hznm7eQkP7T++MHuftNCowF7kHgbbr3H7c8luTrJ/+7u/7DwSLDrVNVLu/vHDrP87CS/keRR3X3p9k8G7EXibbCqOpDkg0kelWR/knd29wOWnYoTVVV36e7PLz0Hx6eq9nf3waXnAPYOx7zNdl6SN3T35d39pSTvr6qnLj0UJ+yDSw/AsauqOyeJcIPtV1UPq6onLT3HUux5G6qqbpvkfVkd53bVetk9kry5ux+45GzcUFXdqrv/ZNOyR3T3ezc8v95xVexsVXVhd99r6TlgL1rfc/hAdz9o6VmWYM/bXFdldZzNVYcWdPenkzxpfdkCdpbD3U7p1zc995PUDlVVp234OHBo8XrdnarqzEN74pz5DVurqv5akguTfKCq/urC4yzCnjfYBlV1SXefcWPLquri7j5z+6fjaKrqa1kdV3owyWe7+96H9rxV1ReTXJfktO6+jT1ysHXWV1i4IMmTsvo/+frufviyU20/e94Gq6pXHWbZSVX1s0vMw4063E9JfnIaortPTfLx9a+bv27+8frt7kN7V+35hq3z3CT/tbs/392fSfLpqvquhWfadvuWHoCb5AmHWXYgyd9I8sPbPAvHr6rqnx56nORWSw7DUfWmX4+2HLgZVdWpSf5+ku/YsPhnkrwyybsWGWoh4m2gqvrtJN+U5C5VdeGm1XfM6rpTzOCiyvPcoap+Kr5+wnY7Jcn3dfcVhxZ090er6keq6hbdfd1yo20vX3xmekGSOyR5Q5LnbVp3WXd/dPtH4gR0d7/40JOq+ttLDsNxEd2wzdbRdsVhlv+PbR9mYeJtoENxVlU/093vWXoebjbedpvhy939k1X1t5YeBNibxNtQ68uBPCzJv1x6Fo7J6VV19YbnFScM7VYiHG5GVfWuHMP/q+7+7m0YZ0cQb0N1d1fVt1XVvu6+Zul5OKpTlx6AE1dVB5OctP7105tXr389tao+l+RO2zoc7H6/uuHxfZI8LauTFC5PclaSZyb5uQXmWozrvA22ftvm8Un+VZLPZnWtqSRJd1+81FycGHdY2Lmq6pQNT6/r7oMbrvN2ZndfXFVnZnXdqXT3RctMCrtbVb0vyTO7+482LDsnyT/s7r++3GTbS7wNVlVHOrOmu/ukbR2Gm6yqvqm7/+/Sc3BsquqT3f1nl54D9pKquqi773aY5Z/u7nssMdMSHHMzWHff4ggfwm2Hq6qHV9U9Ny4TbuM8YukBYA+6oqrus3FBVX1L9ljP7Km/LOwgT0/ykKWH4MR19xeXngH2oJ9O8s6qekFV/eWqenaS30zy75cda3s5YWGw9Z6bf5HkW5OcvHFdd5+1yFDcwPqnws1uldUZqIdblyQXOxEF4Pq6+81VdXmS85I8P8mlSV6e5Pwl59pujnkbbH2nhd9LcmWSuyT59SQvTPLG7t5TP4XsZOszEDvHfs/LTvLd3f2JrZuKY1VVX8ixX/6j1q99THd/fOumAvYy8TZYVX2mu+9eVQ9K8rzuPreqbpfk7d398KXng92gqm5wcPRh/GiSjyZ52/r5xd19cOumgr2rqs5K8oAkt9y4vLt/ZZmJtp+3TWe7rqpOSnJhkvsmSXdfXlXfvOxYHK+q+pS3unemG7vsR1W9NMnrk/xxki+5RAhsrar6gSQvS/LhrK7zdkgnEW+M8M6srnfzmqq6rqpenOT2Sb6w8FxsUlWv3rTo09390xueX1dV++2t2bmq6nFJvierH5Ze2d3XJjk9q/sMA9vjRUke3t2/v/QgS3K26WznJXnj+vEzk5yd5M5JnrXYRBzJk5O8b/3xsSRP2rT+iiS33uaZOEZV9X1ZXcH9E0kenOS161VXZnX3jH+d5L8tMx3sKVft9XBL7Hmb7hHd/d+Tb7y186z126gPXXQqDudrh04iqaq7ZHV7l+utT3Jaki9v92Ackx9J8qTu/lSSV1TV766Phft6klPWy4Gt97aq+q7uftfSgyxJvM326qzu6/YN3X1tVf1yEld+31mOdmbQwSSnHOU1LOd2mwLtA1n9H7s6yb+rqpevl1dWdzhx/CJsjdskeVNVvTfJ9W4D2d3nLjPS9hNvA1XVC5McSHLbqvrRTavvmVUIMMu1SdwZY+e6pqpO7u6r18/vmOSyrP6v/ZMk/3mxyWBvuSjJLyw9xNLE20ynZnV8277ccA/bZUm+d9sn4mg2X+PtvlX1uxvW3SeOQd3J3pbkZeuTgh6W5M9ndbbbY5NcuvEm2cDW6e6fXHqGnUC8DdTdL02Sqjq7u5+z9DwckydveHxJkice5jWf2Z5ROAE/nuTfJPlskk8m+ZvdfV1VdXwdhW1VVY9N8tQkt+nup63vVPP1vXTLOl90BuvuR258XlV3yurg6c8uNBJH0N3v3/D46iTvWXAcjlN3X5nVGd03WBVfR2HbrK/z9vys7mX6Q+vFd8zqEiKbTwTbtdxhYbCq+q0kf6e7P1lVD0ny7qyOnXpBd79u2elg96uqA1n9xO8LKWyDqvpEkkd29yVV9enuvsd6+Se7e8+cqOcnxtnu3d2fXD/+qSTfn9W1pt6ZRLztEFV1XY58tmltWnfo+QO6+yNbPRtHV1Un38jq65Lsr7r+IY0bTmwAbl77u/uS9eONXzv31Nn6DpCe7atVdVJV3TvJPbv719b/qG+78Fxc3/4kJx/h45T1x5uS/KVDrxVuO8pVWV2H76oNH187zLJvrKuq+y0zKux6/6uq/u7GBVX1vUn21ElD9rzN9tYkv5XV+/0/myRVdfvYrjvK+jZKh1VVf5jVtfo6yTU39lqW0d03+CG3qn4iq7NMf3GBkWAvOy/Ju6vqGUlOr6q3ZXWT+icsO9b28k1+th/L6lZYV3X3G9bL7p7V1eDZQarqxzctuqy7X5XV7ZVOW2AkjkNVbT5Z4f5Z7fn+6uFe39175gbZsJ26+/NVdf+sztg/K6sL9T6ju7+y7GTbywkLsA2q6itJXrF+elKSp3b3varqg0kel9WxU3/ixvQ7U1W95jhe3t393C0bBtjzxNtQVXXXJP+2u5+wYdmtk/zHJN/Z3dctNRs3VFWf6+67rh8fSPKRdby9N8nTk3x+4+ttP4Dr833vTzlhYaju/lySL1TVxuvaPD/JO/bSP+BB+giPr87qdi8H1x/XxO3Ndpyq+kJVXXwjH29fekbY7Xzf+1OOeZvtJ5K8varektVZis9M8uBlR+I4HUxyn+6+cOlBuFEP3fD4mUluneRV6+d3TfLz2z4R7E2+70W8jdbd/6eq3pHkeVldH+zXuvv/LTwWh3fLDQe9b7xu2DVxQ/odr7svOvS4qi5Lcu2hZZuv8QZsHd/3VsTbfP8syfuyioBHLzwLR/YbWV3H7ZC3rH/tiDdYXFXdobu/vPQcHJM9/31PvA3X3VdU1S8nOb27r1h4HI6gu59zhFXXZHWmKTvY+mzTQ8cq3jeruyqcvX7+Z5aZipvZm3P9H7DYoXzfc7bprlBV+5Kc1N1fX3oW2I2q6llHecml3e2khcGq6owNt11ih9vr3/fEGwDAIC4VAgAwiHgDABhEvO0SVXXu0jNwYmy72Wy/uWy72fby9hNvu8ee/Ue8C9h2s9l+c9l2s+3Z7SfeAAAG2TNnm55cp/SB3HLpMbbMwXw9+3PK0mNwAmy72Wy/uXb9ttvld/842Fdlfx1Yeowt89W+7NLuPuNw6/bMRXoP5JZ5SD1m6TEAZrmFG4BMVfv3zLf4XeldV73uoiOt87YpAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAG2bf0AFupqs5Ncm6SHMhpC08DAHDT7eo9b919fnef093n7M8pS48DAHCT7ep4AwDYbcQbAMAguyLequqOVfWeqjpj6VkAALbSrjhhobu/lOTRS88BALDVdsWeNwCAvUK8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGGTf0gPAsfgHf/gHS4/ACTrvQ09fegRugrNeevXSI3CC6uC1S4/ATfGxI6+y5w0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQfYtPcBWqqpzk5ybJAdy2sLTAADcdLt6z1t3n9/d53T3OftzytLjAADcZLs63gAAdhvxBgAwiHgDABhkV8RbVd2xqt5TVWcsPQsAwFbaFWebdveXkjx66TkAALbartjzBgCwV4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAbZt/QA2+Ve97sy73znh5ceA/ac537r+5cegZvgzQ/8zqVH4ATd4bUfXHoEtog9bwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCDiDQBgEPEGADCIeAMAGES8AQAMIt4AAAYRbwAAg4g3AIBBxBsAwCA7Nt6q6tlV9e6l5wAA2El2bLwBAHBDNynequpbbq5BjuHPul1VnbZdfx4AwE50QvFWVd9cVa9Ocv6mZW+tqk9V1YVV9YMb1r2kql5TVS+vqj+qqos3rl+/5qlV9ZGq+lxVvT/JfTb9sWcmuaCqzq2qk05kbgCA6Y4r3tZ7v16W5L8k+c0kj18vPyXJu5O8u7vPSvLIJH+vqh6/4dOfkuQd3X3PJE9K8rKqOnv9+Y9J8gtJnt7dd03yzCRP2/hnd/cfJHlEknsl+VBVPfl4/7IAANMdU7xV1f6qelGS9yX5TJK/0N1v7O5ev+SJSS7r7l9Mku7+YpJfSvLUDb/Ne7v7Hev1H0ry4SQPWK/7gSQvWwdauvuTSV65eY7uvqy7fzir+HtiVb23qr7jRuY+t6ouqKoLLvnytcfyVwUA2NH2HePrTkpyepJrk9T68w5uWH9Wkm+rqs9sWHZykvdveH7xpt/z8iS3XD8+OzeMtctvZJ59Sa5Kcup6rsPq7vOzfmv3nPsf6CO9DgBgimPa89bdV3X3i5I8OskZSX6/qv5xVd1m/ZKLk/x2d999w8eZ3f2UY5zj0iSbT344a/OLqup+VfX6JG9I8q4k53T3fzrGPwMAYLzjOuZt/bblS5I8OMl1SQ6F09uTPGDjcWhV9aCquvsx/tZvTPKPqupu6889J6vj3r6hqh6a5FVJfqW7H9rdb93wti0AwJ5wrG+bXk93fzXJP6+qn1s/v6KqnpDk5VX1iiRfT/I/k5x3jL/l+UnulOR3qqqTfCDJzyf5ng2vuaC7H3Ui8wIA7BYnFG+HdPc1Gx5/OMlfPMLrXnKYZY/b8PjaJC9Zf2z0isP9WQAAe5U7LAAADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADCLeAAAGEW8AAIOINwCAQcQbAMAg4g0AYBDxBgAwiHgDABhEvAEADLJv6QG2y4UfOS2PPfPblx4DYJTb5/1Lj8AJ6qUHYMvY8wYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGCQfUsPsJWq6twk5ybJgZy28DQAADfdrt7z1t3nd/c53X3O/pyy9DgAADfZro43AIDdRrwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEHEGwDAIOINAGAQ8QYAMIh4AwAYRLwBAAwi3gAABhFvAACDiDcAgEGqu5eeYVtU1SVJLlp6ji10epJLlx6CE2LbzWb7zWXbzbbbt9/duvuMw63YM/G221XVBd19ztJzcPxsu9lsv7lsu9n28vbztikAwCDiDQBgEPG2e5y/9ACcMNtuNttvLttutj27/RzzBgAwiD1vAACDiDcAgEHEGwDAIOINAGAQ8QYAMMj/B/qudUTbSI/NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translate('뭐라도 배워볼까.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 한글 폰트가 깨질 때\n",
    "\n",
    "https://financedata.github.io/posts/matplotlib-hangul-for-ubuntu-linux.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Windows-10-10.0.18362-SP0\n",
      "sys.version_info(major=3, minor=8, micro=8, releaselevel='final', serial=0)\n",
      "버전:  3.4.2\n",
      "설치위치:  C:\\Users\\tnfls\\anaconda3\\lib\\site-packages\\matplotlib\\__init__.py\n",
      "설정:  C:\\Users\\tnfls\\.matplotlib\n",
      "캐시:  C:\\Users\\tnfls\\.matplotlib\n"
     ]
    }
   ],
   "source": [
    "#실행중인 운영체제 확인\n",
    "\n",
    "import platform\n",
    "print(platform.platform())\n",
    "\n",
    "# 파이썬 버전\n",
    "import sys\n",
    "print(sys.version_info)\n",
    "\n",
    "# matplotlib 주요 설치 정보\n",
    "\n",
    "import matplotlib\n",
    "\n",
    "print ('버전: ', matplotlib.__version__)\n",
    "print ('설치위치: ', matplotlib.__file__)\n",
    "print ('설정: ', matplotlib.get_configdir())\n",
    "print ('캐시: ', matplotlib.get_cachedir())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 우분투에 폰트는 /usr/share/fonts/ 디렉토리에 있다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/truetype/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 나눔 글꼴 설치\n",
    "# apt-get 명령으로 나눔글꼴(fonts-nanum)을 설치하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo apt-get install fonts-nanum*\n",
    "# $ sudo fc-cache -fv\n",
    "# 만일 다른 ttf 폰트를 가져왔다면 다음과 같이 복사하고, fc-cache 명령으로 폰트 캐시 삭제\n",
    "\n",
    "# $ sudo cp new_font.ttf /usr/share/fonts/\n",
    "# $ sudo fc-cache -fv\n",
    "# matplotlib 나눔 글꼴을 추가\n",
    "# 나눔 글꼴을 matplotlib 에 복사하고, matplotlib의 폰트 캐시를 삭제\n",
    "\n",
    "# $ sudo cp /usr/share/fonts/truetype/nanum/Nanum* /usr/local/lib/python3.4/dist-packages/matplotlib/mpl-data/fonts/ttf/\n",
    "# $ rm -rf /home/ubuntu/.cache/matplotlib/*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설치확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls -l /usr/share/fonts/truetype/nanum/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# matplotlib.font_manager.fontManager.ttflist\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.font_manager\n",
    "\n",
    "[f.fname for f in matplotlib.font_manager.fontManager.ttflist]\n",
    "[(f.name, f.fname) for f in matplotlib.font_manager.fontManager.ttflist if 'Nanum' in f.name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import font_manager, rc\n",
    "\n",
    "font_fname = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'\n",
    "font_name = font_manager.FontProperties(fname=font_fname).get_name()\n",
    "\n",
    "rc('font', family=font_name)\n",
    "print(font_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}