word2vec - continuous bag of words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [\n",
    "    'This is an excellent movie',\n",
    "    'The move was fantastic I like it',\n",
    "    'You should watch it is brilliant',\n",
    "    'Exceptionally good',\n",
    "    'Wonderfully directed and executed I like it',\n",
    "    'Its a fantastic series',\n",
    "    'Never watched such a brillent movie',\n",
    "    'It is a Wonderful movie',\n",
    "    \"horrible acting\",\n",
    "    'waste of money',\n",
    "    'pathetic picture',\n",
    "    'It was very boring',\n",
    "    'I did not like the movie',\n",
    "    'The movie was horrible',\n",
    "    'I will not recommend',\n",
    "    'The acting is pathetic'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Exceptionally',\n",
       " 'I',\n",
       " 'It',\n",
       " 'Its',\n",
       " 'Never',\n",
       " 'The',\n",
       " 'This',\n",
       " 'Wonderful',\n",
       " 'Wonderfully',\n",
       " 'You',\n",
       " 'a',\n",
       " 'acting',\n",
       " 'an',\n",
       " 'and',\n",
       " 'boring',\n",
       " 'brillent',\n",
       " 'brilliant',\n",
       " 'did',\n",
       " 'directed',\n",
       " 'excellent',\n",
       " 'executed',\n",
       " 'fantastic',\n",
       " 'good',\n",
       " 'horrible',\n",
       " 'is',\n",
       " 'it',\n",
       " 'like',\n",
       " 'money',\n",
       " 'move',\n",
       " 'movie',\n",
       " 'not',\n",
       " 'of',\n",
       " 'pathetic',\n",
       " 'picture',\n",
       " 'recommend',\n",
       " 'series',\n",
       " 'should',\n",
       " 'such',\n",
       " 'the',\n",
       " 'very',\n",
       " 'was',\n",
       " 'waste',\n",
       " 'watch',\n",
       " 'watched',\n",
       " 'will'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = set(re.sub(r' +', ' ', ' '.join(corpus)).strip().split(' '))\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'did': 0,\n",
       " 'such': 1,\n",
       " 'will': 2,\n",
       " 'It': 3,\n",
       " 'it': 4,\n",
       " 'picture': 5,\n",
       " 'Wonderful': 6,\n",
       " 'watch': 7,\n",
       " 'like': 8,\n",
       " 'I': 9,\n",
       " 'watched': 10,\n",
       " 'brillent': 11,\n",
       " 'money': 12,\n",
       " 'very': 13,\n",
       " 'recommend': 14,\n",
       " 'good': 15,\n",
       " 'Its': 16,\n",
       " 'pathetic': 17,\n",
       " 'You': 18,\n",
       " 'should': 19,\n",
       " 'move': 20,\n",
       " 'Exceptionally': 21,\n",
       " 'a': 22,\n",
       " 'movie': 23,\n",
       " 'not': 24,\n",
       " 'executed': 25,\n",
       " 'fantastic': 26,\n",
       " 'is': 27,\n",
       " 'boring': 28,\n",
       " 'Wonderfully': 29,\n",
       " 'and': 30,\n",
       " 'directed': 31,\n",
       " 'excellent': 32,\n",
       " 'of': 33,\n",
       " 'the': 34,\n",
       " 'Never': 35,\n",
       " 'This': 36,\n",
       " 'was': 37,\n",
       " 'an': 38,\n",
       " 'acting': 39,\n",
       " 'horrible': 40,\n",
       " 'waste': 41,\n",
       " 'brilliant': 42,\n",
       " 'series': 43,\n",
       " 'The': 44}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2id = {word:i for i, word in enumerate(vocab)}\n",
    "word2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['did',\n",
       " 'such',\n",
       " 'will',\n",
       " 'It',\n",
       " 'it',\n",
       " 'picture',\n",
       " 'Wonderful',\n",
       " 'watch',\n",
       " 'like',\n",
       " 'I',\n",
       " 'watched',\n",
       " 'brillent',\n",
       " 'money',\n",
       " 'very',\n",
       " 'recommend',\n",
       " 'good',\n",
       " 'Its',\n",
       " 'pathetic',\n",
       " 'You',\n",
       " 'should',\n",
       " 'move',\n",
       " 'Exceptionally',\n",
       " 'a',\n",
       " 'movie',\n",
       " 'not',\n",
       " 'executed',\n",
       " 'fantastic',\n",
       " 'is',\n",
       " 'boring',\n",
       " 'Wonderfully',\n",
       " 'and',\n",
       " 'directed',\n",
       " 'excellent',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Never',\n",
       " 'This',\n",
       " 'was',\n",
       " 'an',\n",
       " 'acting',\n",
       " 'horrible',\n",
       " 'waste',\n",
       " 'brilliant',\n",
       " 'series',\n",
       " 'The']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id2word = list(vocab)\n",
    "id2word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### one hot encoding for word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ohe(word, word2id, dim=45):\n",
    "    vec = [0]*dim\n",
    "    vec[word2id[word]] = 1\n",
    "    return vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[['This', 'an'], 'is'],\n",
       " [['is', 'excellent'], 'an'],\n",
       " [['an', 'movie'], 'excellent'],\n",
       " [['The', 'was'], 'move'],\n",
       " [['move', 'fantastic'], 'was'],\n",
       " [['was', 'I'], 'fantastic'],\n",
       " [['fantastic', 'like'], 'I'],\n",
       " [['I', 'it'], 'like'],\n",
       " [['You', 'watch'], 'should'],\n",
       " [['should', 'it'], 'watch'],\n",
       " [['watch', 'is'], 'it'],\n",
       " [['it', 'brilliant'], 'is'],\n",
       " [['Wonderfully', 'and'], 'directed'],\n",
       " [['directed', 'executed'], 'and'],\n",
       " [['and', 'I'], 'executed'],\n",
       " [['executed', 'like'], 'I'],\n",
       " [['I', 'it'], 'like'],\n",
       " [['Its', 'fantastic'], 'a'],\n",
       " [['a', 'series'], 'fantastic'],\n",
       " [['Never', 'such'], 'watched'],\n",
       " [['watched', 'a'], 'such'],\n",
       " [['such', 'brillent'], 'a'],\n",
       " [['a', 'movie'], 'brillent'],\n",
       " [['It', 'a'], 'is'],\n",
       " [['is', 'Wonderful'], 'a'],\n",
       " [['a', 'movie'], 'Wonderful'],\n",
       " [['waste', 'money'], 'of'],\n",
       " [['It', 'very'], 'was'],\n",
       " [['was', 'boring'], 'very'],\n",
       " [['I', 'not'], 'did'],\n",
       " [['did', 'like'], 'not'],\n",
       " [['not', 'the'], 'like'],\n",
       " [['like', 'movie'], 'the'],\n",
       " [['The', 'was'], 'movie'],\n",
       " [['movie', 'horrible'], 'was'],\n",
       " [['I', 'not'], 'will'],\n",
       " [['will', 'recommend'], 'not'],\n",
       " [['The', 'is'], 'acting'],\n",
       " [['acting', 'pathetic'], 'is']]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_training_data_from_a_sentence(words, window_size):\n",
    "    ans = []\n",
    "    lookup = (window_size-1)//2\n",
    "    for i in range(len(words)):\n",
    "        if (i-lookup) >= 0 and (i+lookup) < len(words):\n",
    "            x = []\n",
    "            j = i-lookup\n",
    "            while j <= (i+lookup):\n",
    "                if j != i:\n",
    "                    x.append(words[j])\n",
    "                j += 1\n",
    "            ans.append([x, words[i]])\n",
    "    return ans\n",
    "\n",
    "window_size = 3 # keep it odd\n",
    "training_data = [] # [[x1,x2],y1]\n",
    "for i, line in enumerate(corpus):\n",
    "    words = line.split(' ')\n",
    "    training_data.extend(create_training_data_from_a_sentence(words, window_size))\n",
    "\n",
    "training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "\n",
    "X  = []\n",
    "Y = []\n",
    "\n",
    "for x, y in training_data:\n",
    "    Y.append(ohe(y, word2id, 45))\n",
    "    X.append(list(reduce(lambda u, v: [i|j for i, j in zip(u, v)], [ohe(word, word2id, 45) for word in x])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39, 39, 45, 45)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(Y), len(X[0]), len(Y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['executed', 'directed'], ['and']]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check of ith sample\n",
    "i = 13\n",
    "[[id2word[j] for j in range(len(X[i])) if X[i][j] == 1], [id2word[j] for j in range(len(Y[i])) if Y[i][j] == 1]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 1, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 1],\n",
       "        [0, 0, 0, ..., 0, 0, 0]]),\n",
       " array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = np.array(X)\n",
    "Y_train = np.array(Y)\n",
    "X_train, Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((39, 45), (39, 45))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, Y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/500\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 3.8038 - accuracy: 0.0513\n",
      "Epoch 2/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 3.7951 - accuracy: 0.0513\n",
      "Epoch 3/500\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 3.7877 - accuracy: 0.0513\n",
      "Epoch 4/500\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 3.7808 - accuracy: 0.0513\n",
      "Epoch 5/500\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 3.7740 - accuracy: 0.0513\n",
      "Epoch 6/500\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 3.7671 - accuracy: 0.0513\n",
      "Epoch 7/500\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 3.7606 - accuracy: 0.0769\n",
      "Epoch 8/500\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 3.7541 - accuracy: 0.0769\n",
      "Epoch 9/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 3.7477 - accuracy: 0.0769\n",
      "Epoch 10/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7414 - accuracy: 0.0769\n",
      "Epoch 11/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7349 - accuracy: 0.0769\n",
      "Epoch 12/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7286 - accuracy: 0.0769\n",
      "Epoch 13/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7222 - accuracy: 0.0769\n",
      "Epoch 14/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.7158 - accuracy: 0.0769\n",
      "Epoch 15/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7096 - accuracy: 0.1026\n",
      "Epoch 16/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.7032 - accuracy: 0.1026\n",
      "Epoch 17/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 3.6970 - accuracy: 0.1026\n",
      "Epoch 18/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6907 - accuracy: 0.1026\n",
      "Epoch 19/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6844 - accuracy: 0.1026\n",
      "Epoch 20/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6780 - accuracy: 0.1026\n",
      "Epoch 21/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6715 - accuracy: 0.1026\n",
      "Epoch 22/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6652 - accuracy: 0.1538\n",
      "Epoch 23/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6587 - accuracy: 0.1538\n",
      "Epoch 24/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6522 - accuracy: 0.1538\n",
      "Epoch 25/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6457 - accuracy: 0.1538\n",
      "Epoch 26/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6392 - accuracy: 0.1538\n",
      "Epoch 27/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6325 - accuracy: 0.1538\n",
      "Epoch 28/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6260 - accuracy: 0.1538\n",
      "Epoch 29/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6192 - accuracy: 0.1538\n",
      "Epoch 30/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.6124 - accuracy: 0.1538\n",
      "Epoch 31/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.6057 - accuracy: 0.1538\n",
      "Epoch 32/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5988 - accuracy: 0.1538\n",
      "Epoch 33/500\n",
      "2/2 [==============================] - 0s 986us/step - loss: 3.5918 - accuracy: 0.1538\n",
      "Epoch 34/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5849 - accuracy: 0.1795\n",
      "Epoch 35/500\n",
      "2/2 [==============================] - 0s 997us/step - loss: 3.5781 - accuracy: 0.1795\n",
      "Epoch 36/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5710 - accuracy: 0.1795\n",
      "Epoch 37/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5640 - accuracy: 0.1795\n",
      "Epoch 38/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5568 - accuracy: 0.1795\n",
      "Epoch 39/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.5499 - accuracy: 0.1795\n",
      "Epoch 40/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5427 - accuracy: 0.2051\n",
      "Epoch 41/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.5354 - accuracy: 0.2051\n",
      "Epoch 42/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5283 - accuracy: 0.2051\n",
      "Epoch 43/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 3.5210 - accuracy: 0.2051\n",
      "Epoch 44/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.5139 - accuracy: 0.2051\n",
      "Epoch 45/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 3.5066 - accuracy: 0.2051\n",
      "Epoch 46/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4993 - accuracy: 0.1795\n",
      "Epoch 47/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4920 - accuracy: 0.2308\n",
      "Epoch 48/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4847 - accuracy: 0.2821\n",
      "Epoch 49/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4771 - accuracy: 0.2821\n",
      "Epoch 50/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4697 - accuracy: 0.2564\n",
      "Epoch 51/500\n",
      "2/2 [==============================] - 0s 997us/step - loss: 3.4620 - accuracy: 0.3077\n",
      "Epoch 52/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4544 - accuracy: 0.3077\n",
      "Epoch 53/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4465 - accuracy: 0.3333\n",
      "Epoch 54/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.4390 - accuracy: 0.3333\n",
      "Epoch 55/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4310 - accuracy: 0.3333\n",
      "Epoch 56/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4230 - accuracy: 0.3333\n",
      "Epoch 57/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4150 - accuracy: 0.3333\n",
      "Epoch 58/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.4069 - accuracy: 0.3333\n",
      "Epoch 59/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.3990 - accuracy: 0.3333\n",
      "Epoch 60/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3908 - accuracy: 0.3333\n",
      "Epoch 61/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3825 - accuracy: 0.3333\n",
      "Epoch 62/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.3743 - accuracy: 0.3333\n",
      "Epoch 63/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.3660 - accuracy: 0.3333\n",
      "Epoch 64/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3577 - accuracy: 0.3333\n",
      "Epoch 65/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3495 - accuracy: 0.3333\n",
      "Epoch 66/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3411 - accuracy: 0.3333\n",
      "Epoch 67/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3329 - accuracy: 0.3333\n",
      "Epoch 68/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.3243 - accuracy: 0.3333\n",
      "Epoch 69/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.3157 - accuracy: 0.3333\n",
      "Epoch 70/500\n",
      "2/2 [==============================] - 0s 986us/step - loss: 3.3069 - accuracy: 0.3333\n",
      "Epoch 71/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2984 - accuracy: 0.3333\n",
      "Epoch 72/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2897 - accuracy: 0.3590\n",
      "Epoch 73/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2809 - accuracy: 0.3590\n",
      "Epoch 74/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2724 - accuracy: 0.3590\n",
      "Epoch 75/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2634 - accuracy: 0.3590\n",
      "Epoch 76/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2546 - accuracy: 0.3590\n",
      "Epoch 77/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2456 - accuracy: 0.3846\n",
      "Epoch 78/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2365 - accuracy: 0.3846\n",
      "Epoch 79/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2276 - accuracy: 0.3846\n",
      "Epoch 80/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.2184 - accuracy: 0.3846\n",
      "Epoch 81/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2093 - accuracy: 0.3846\n",
      "Epoch 82/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.2000 - accuracy: 0.3846\n",
      "Epoch 83/500\n",
      "2/2 [==============================] - 0s 880us/step - loss: 3.1908 - accuracy: 0.4103\n",
      "Epoch 84/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1813 - accuracy: 0.4359\n",
      "Epoch 85/500\n",
      "2/2 [==============================] - 0s 998us/step - loss: 3.1723 - accuracy: 0.4359\n",
      "Epoch 86/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1629 - accuracy: 0.4359\n",
      "Epoch 87/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1534 - accuracy: 0.4359\n",
      "Epoch 88/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1439 - accuracy: 0.4359\n",
      "Epoch 89/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1342 - accuracy: 0.4359\n",
      "Epoch 90/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1245 - accuracy: 0.4103\n",
      "Epoch 91/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1148 - accuracy: 0.4103\n",
      "Epoch 92/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.1050 - accuracy: 0.4103\n",
      "Epoch 93/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.0947 - accuracy: 0.4103\n",
      "Epoch 94/500\n",
      "2/2 [==============================] - 0s 994us/step - loss: 3.0849 - accuracy: 0.4359\n",
      "Epoch 95/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.0751 - accuracy: 0.4359\n",
      "Epoch 96/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.0649 - accuracy: 0.4359\n",
      "Epoch 97/500\n",
      "2/2 [==============================] - 0s 961us/step - loss: 3.0549 - accuracy: 0.4359\n",
      "Epoch 98/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.0446 - accuracy: 0.4359\n",
      "Epoch 99/500\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 3.0346 - accuracy: 0.4359\n",
      "Epoch 100/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.0247 - accuracy: 0.4359\n",
      "Epoch 101/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 3.0147 - accuracy: 0.4359\n",
      "Epoch 102/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 3.0047 - accuracy: 0.4359\n",
      "Epoch 103/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.9947 - accuracy: 0.4359\n",
      "Epoch 104/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.9843 - accuracy: 0.4359\n",
      "Epoch 105/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.9744 - accuracy: 0.4359\n",
      "Epoch 106/500\n",
      "2/2 [==============================] - 0s 882us/step - loss: 2.9643 - accuracy: 0.4359\n",
      "Epoch 107/500\n",
      "2/2 [==============================] - 0s 828us/step - loss: 2.9543 - accuracy: 0.4615\n",
      "Epoch 108/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.9439 - accuracy: 0.4615\n",
      "Epoch 109/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.9337 - accuracy: 0.4615\n",
      "Epoch 110/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.9234 - accuracy: 0.4872\n",
      "Epoch 111/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.9134 - accuracy: 0.4872\n",
      "Epoch 112/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.9032 - accuracy: 0.4872\n",
      "Epoch 113/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8930 - accuracy: 0.5128\n",
      "Epoch 114/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8826 - accuracy: 0.5128\n",
      "Epoch 115/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8725 - accuracy: 0.5128\n",
      "Epoch 116/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8623 - accuracy: 0.5128\n",
      "Epoch 117/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.8520 - accuracy: 0.5128\n",
      "Epoch 118/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8419 - accuracy: 0.5128\n",
      "Epoch 119/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.8318 - accuracy: 0.5128\n",
      "Epoch 120/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8215 - accuracy: 0.5128\n",
      "Epoch 121/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.8116 - accuracy: 0.5128\n",
      "Epoch 122/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.8014 - accuracy: 0.5128\n",
      "Epoch 123/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.7916 - accuracy: 0.5128\n",
      "Epoch 124/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.7816 - accuracy: 0.5128\n",
      "Epoch 125/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.7715 - accuracy: 0.5128\n",
      "Epoch 126/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.7614 - accuracy: 0.5128\n",
      "Epoch 127/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.7513 - accuracy: 0.5128\n",
      "Epoch 128/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.7410 - accuracy: 0.5128\n",
      "Epoch 129/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.7306 - accuracy: 0.5128\n",
      "Epoch 130/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.7205 - accuracy: 0.5128\n",
      "Epoch 131/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.7102 - accuracy: 0.5385\n",
      "Epoch 132/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.7000 - accuracy: 0.5385\n",
      "Epoch 133/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.6895 - accuracy: 0.5385\n",
      "Epoch 134/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6797 - accuracy: 0.5128\n",
      "Epoch 135/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6694 - accuracy: 0.5128\n",
      "Epoch 136/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6588 - accuracy: 0.5128\n",
      "Epoch 137/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6486 - accuracy: 0.4872\n",
      "Epoch 138/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6383 - accuracy: 0.4872\n",
      "Epoch 139/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.6280 - accuracy: 0.4872\n",
      "Epoch 140/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.6178 - accuracy: 0.4872\n",
      "Epoch 141/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.6078 - accuracy: 0.4872\n",
      "Epoch 142/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.5975 - accuracy: 0.4872\n",
      "Epoch 143/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5873 - accuracy: 0.5128\n",
      "Epoch 144/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5773 - accuracy: 0.5128\n",
      "Epoch 145/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5668 - accuracy: 0.5128\n",
      "Epoch 146/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.5566 - accuracy: 0.5385\n",
      "Epoch 147/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5468 - accuracy: 0.5385\n",
      "Epoch 148/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5369 - accuracy: 0.5385\n",
      "Epoch 149/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5267 - accuracy: 0.5385\n",
      "Epoch 150/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.5166 - accuracy: 0.5128\n",
      "Epoch 151/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.5070 - accuracy: 0.5128\n",
      "Epoch 152/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.4968 - accuracy: 0.5128\n",
      "Epoch 153/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4871 - accuracy: 0.5128\n",
      "Epoch 154/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.4773 - accuracy: 0.5128\n",
      "Epoch 155/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4674 - accuracy: 0.5128\n",
      "Epoch 156/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4577 - accuracy: 0.5385\n",
      "Epoch 157/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.4479 - accuracy: 0.5385\n",
      "Epoch 158/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4379 - accuracy: 0.5385\n",
      "Epoch 159/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4281 - accuracy: 0.5385\n",
      "Epoch 160/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.4180 - accuracy: 0.5385\n",
      "Epoch 161/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.4082 - accuracy: 0.5385\n",
      "Epoch 162/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3984 - accuracy: 0.5385\n",
      "Epoch 163/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3884 - accuracy: 0.5385\n",
      "Epoch 164/500\n",
      "2/2 [==============================] - 0s 513us/step - loss: 2.3784 - accuracy: 0.5385\n",
      "Epoch 165/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3685 - accuracy: 0.5385\n",
      "Epoch 166/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3588 - accuracy: 0.5385\n",
      "Epoch 167/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3492 - accuracy: 0.5385\n",
      "Epoch 168/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3395 - accuracy: 0.5385\n",
      "Epoch 169/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3300 - accuracy: 0.5385\n",
      "Epoch 170/500\n",
      "2/2 [==============================] - 0s 922us/step - loss: 2.3198 - accuracy: 0.5641\n",
      "Epoch 171/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.3102 - accuracy: 0.5641\n",
      "Epoch 172/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.3005 - accuracy: 0.5641\n",
      "Epoch 173/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.2908 - accuracy: 0.5641\n",
      "Epoch 174/500\n",
      "2/2 [==============================] - 0s 926us/step - loss: 2.2810 - accuracy: 0.5641\n",
      "Epoch 175/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2714 - accuracy: 0.5641\n",
      "Epoch 176/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2617 - accuracy: 0.5641\n",
      "Epoch 177/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2520 - accuracy: 0.5641\n",
      "Epoch 178/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2423 - accuracy: 0.5641\n",
      "Epoch 179/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2325 - accuracy: 0.5641\n",
      "Epoch 180/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2226 - accuracy: 0.5641\n",
      "Epoch 181/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2132 - accuracy: 0.5641\n",
      "Epoch 182/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.2033 - accuracy: 0.5641\n",
      "Epoch 183/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1937 - accuracy: 0.5641\n",
      "Epoch 184/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1839 - accuracy: 0.5897\n",
      "Epoch 185/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1744 - accuracy: 0.5641\n",
      "Epoch 186/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 2.1647 - accuracy: 0.5641\n",
      "Epoch 187/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 2.1552 - accuracy: 0.5641\n",
      "Epoch 188/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1460 - accuracy: 0.6154\n",
      "Epoch 189/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1365 - accuracy: 0.6410\n",
      "Epoch 190/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.1268 - accuracy: 0.6410\n",
      "Epoch 191/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.1175 - accuracy: 0.6410\n",
      "Epoch 192/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.1084 - accuracy: 0.6410\n",
      "Epoch 193/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.0988 - accuracy: 0.6410\n",
      "Epoch 194/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.0890 - accuracy: 0.6410\n",
      "Epoch 195/500\n",
      "2/2 [==============================] - 0s 852us/step - loss: 2.0798 - accuracy: 0.6410\n",
      "Epoch 196/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.0704 - accuracy: 0.6410\n",
      "Epoch 197/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.0614 - accuracy: 0.6410\n",
      "Epoch 198/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.0518 - accuracy: 0.6410\n",
      "Epoch 199/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.0426 - accuracy: 0.6410\n",
      "Epoch 200/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.0333 - accuracy: 0.6410\n",
      "Epoch 201/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 2.0242 - accuracy: 0.6410\n",
      "Epoch 202/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.0149 - accuracy: 0.6410\n",
      "Epoch 203/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 2.0056 - accuracy: 0.6410\n",
      "Epoch 204/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9963 - accuracy: 0.6154\n",
      "Epoch 205/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9871 - accuracy: 0.6154\n",
      "Epoch 206/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9778 - accuracy: 0.6154\n",
      "Epoch 207/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9685 - accuracy: 0.6154\n",
      "Epoch 208/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9595 - accuracy: 0.6154\n",
      "Epoch 209/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9504 - accuracy: 0.6410\n",
      "Epoch 210/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9416 - accuracy: 0.6410\n",
      "Epoch 211/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9325 - accuracy: 0.6410\n",
      "Epoch 212/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9236 - accuracy: 0.6410\n",
      "Epoch 213/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.9149 - accuracy: 0.6410\n",
      "Epoch 214/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.9055 - accuracy: 0.6410\n",
      "Epoch 215/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8966 - accuracy: 0.6410\n",
      "Epoch 216/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.8877 - accuracy: 0.6667\n",
      "Epoch 217/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8787 - accuracy: 0.6667\n",
      "Epoch 218/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8693 - accuracy: 0.6667\n",
      "Epoch 219/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8604 - accuracy: 0.6667\n",
      "Epoch 220/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8513 - accuracy: 0.6667\n",
      "Epoch 221/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.8423 - accuracy: 0.6667\n",
      "Epoch 222/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.8332 - accuracy: 0.6667\n",
      "Epoch 223/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8242 - accuracy: 0.6667\n",
      "Epoch 224/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.8154 - accuracy: 0.6667\n",
      "Epoch 225/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.8065 - accuracy: 0.6667\n",
      "Epoch 226/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.7976 - accuracy: 0.6667\n",
      "Epoch 227/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.7888 - accuracy: 0.6667\n",
      "Epoch 228/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.7800 - accuracy: 0.6667\n",
      "Epoch 229/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7712 - accuracy: 0.6923\n",
      "Epoch 230/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7626 - accuracy: 0.6923\n",
      "Epoch 231/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7537 - accuracy: 0.6667\n",
      "Epoch 232/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7451 - accuracy: 0.6667\n",
      "Epoch 233/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7363 - accuracy: 0.6667\n",
      "Epoch 234/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 1.7277 - accuracy: 0.6667\n",
      "Epoch 235/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.7189 - accuracy: 0.6667\n",
      "Epoch 236/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.7107 - accuracy: 0.6667\n",
      "Epoch 237/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.7022 - accuracy: 0.6923\n",
      "Epoch 238/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6935 - accuracy: 0.6923\n",
      "Epoch 239/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6851 - accuracy: 0.6923\n",
      "Epoch 240/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.6766 - accuracy: 0.6923\n",
      "Epoch 241/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.6683 - accuracy: 0.6923\n",
      "Epoch 242/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6596 - accuracy: 0.6923\n",
      "Epoch 243/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6516 - accuracy: 0.6923\n",
      "Epoch 244/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6429 - accuracy: 0.6923\n",
      "Epoch 245/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6347 - accuracy: 0.7179\n",
      "Epoch 246/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6263 - accuracy: 0.7436\n",
      "Epoch 247/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6180 - accuracy: 0.7436\n",
      "Epoch 248/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6099 - accuracy: 0.7436\n",
      "Epoch 249/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.6017 - accuracy: 0.7436\n",
      "Epoch 250/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5934 - accuracy: 0.7692\n",
      "Epoch 251/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5855 - accuracy: 0.7692\n",
      "Epoch 252/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.5770 - accuracy: 0.7692\n",
      "Epoch 253/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5692 - accuracy: 0.7692\n",
      "Epoch 254/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5612 - accuracy: 0.7949\n",
      "Epoch 255/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.5532 - accuracy: 0.7949\n",
      "Epoch 256/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5455 - accuracy: 0.7949\n",
      "Epoch 257/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.5376 - accuracy: 0.7949\n",
      "Epoch 258/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5296 - accuracy: 0.7949\n",
      "Epoch 259/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.5220 - accuracy: 0.7949\n",
      "Epoch 260/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.5144 - accuracy: 0.7949\n",
      "Epoch 261/500\n",
      "2/2 [==============================] - 0s 971us/step - loss: 1.5063 - accuracy: 0.7949\n",
      "Epoch 262/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.4987 - accuracy: 0.7949\n",
      "Epoch 263/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.4908 - accuracy: 0.7949\n",
      "Epoch 264/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 1.4832 - accuracy: 0.7949\n",
      "Epoch 265/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.4755 - accuracy: 0.7949\n",
      "Epoch 266/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4679 - accuracy: 0.7949\n",
      "Epoch 267/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4603 - accuracy: 0.7949\n",
      "Epoch 268/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4526 - accuracy: 0.7949\n",
      "Epoch 269/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.4449 - accuracy: 0.7949\n",
      "Epoch 270/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4372 - accuracy: 0.7949\n",
      "Epoch 271/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4300 - accuracy: 0.7949\n",
      "Epoch 272/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4225 - accuracy: 0.7949\n",
      "Epoch 273/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4148 - accuracy: 0.7949\n",
      "Epoch 274/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.4075 - accuracy: 0.7949\n",
      "Epoch 275/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3997 - accuracy: 0.7949\n",
      "Epoch 276/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3922 - accuracy: 0.7949\n",
      "Epoch 277/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3847 - accuracy: 0.7949\n",
      "Epoch 278/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3772 - accuracy: 0.7949\n",
      "Epoch 279/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3696 - accuracy: 0.8205\n",
      "Epoch 280/500\n",
      "2/2 [==============================] - 0s 997us/step - loss: 1.3623 - accuracy: 0.8205\n",
      "Epoch 281/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3550 - accuracy: 0.8205\n",
      "Epoch 282/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.3480 - accuracy: 0.8205\n",
      "Epoch 283/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3406 - accuracy: 0.8205\n",
      "Epoch 284/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3335 - accuracy: 0.8205\n",
      "Epoch 285/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.3265 - accuracy: 0.8205\n",
      "Epoch 286/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.3196 - accuracy: 0.8205\n",
      "Epoch 287/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3131 - accuracy: 0.8205\n",
      "Epoch 288/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.3060 - accuracy: 0.8205\n",
      "Epoch 289/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2994 - accuracy: 0.8205\n",
      "Epoch 290/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2928 - accuracy: 0.8205\n",
      "Epoch 291/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2862 - accuracy: 0.8205\n",
      "Epoch 292/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2795 - accuracy: 0.8205\n",
      "Epoch 293/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2730 - accuracy: 0.8205\n",
      "Epoch 294/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2665 - accuracy: 0.8205\n",
      "Epoch 295/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2598 - accuracy: 0.8205\n",
      "Epoch 296/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 1.2531 - accuracy: 0.8205\n",
      "Epoch 297/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2465 - accuracy: 0.8205\n",
      "Epoch 298/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.2401 - accuracy: 0.8205\n",
      "Epoch 299/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2334 - accuracy: 0.8205\n",
      "Epoch 300/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2269 - accuracy: 0.8205\n",
      "Epoch 301/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2206 - accuracy: 0.8205\n",
      "Epoch 302/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2145 - accuracy: 0.8205\n",
      "Epoch 303/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 1.2079 - accuracy: 0.8205\n",
      "Epoch 304/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.2016 - accuracy: 0.8205\n",
      "Epoch 305/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1952 - accuracy: 0.8205\n",
      "Epoch 306/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1887 - accuracy: 0.8205\n",
      "Epoch 307/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1828 - accuracy: 0.8205\n",
      "Epoch 308/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1767 - accuracy: 0.8205\n",
      "Epoch 309/500\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 1.1703 - accuracy: 0.8205\n",
      "Epoch 310/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1643 - accuracy: 0.8205\n",
      "Epoch 311/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 1.1584 - accuracy: 0.8205\n",
      "Epoch 312/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1525 - accuracy: 0.8205\n",
      "Epoch 313/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1469 - accuracy: 0.8205\n",
      "Epoch 314/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1411 - accuracy: 0.8205\n",
      "Epoch 315/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.1351 - accuracy: 0.8205\n",
      "Epoch 316/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1295 - accuracy: 0.8205\n",
      "Epoch 317/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.1239 - accuracy: 0.8205\n",
      "Epoch 318/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1182 - accuracy: 0.8205\n",
      "Epoch 319/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1127 - accuracy: 0.8205\n",
      "Epoch 320/500\n",
      "2/2 [==============================] - 0s 935us/step - loss: 1.1069 - accuracy: 0.8205\n",
      "Epoch 321/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.1015 - accuracy: 0.8205\n",
      "Epoch 322/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0958 - accuracy: 0.8205\n",
      "Epoch 323/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0903 - accuracy: 0.8205\n",
      "Epoch 324/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0848 - accuracy: 0.8205\n",
      "Epoch 325/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0793 - accuracy: 0.8205\n",
      "Epoch 326/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0739 - accuracy: 0.8205\n",
      "Epoch 327/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0685 - accuracy: 0.8205\n",
      "Epoch 328/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0631 - accuracy: 0.8205\n",
      "Epoch 329/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0579 - accuracy: 0.8205\n",
      "Epoch 330/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0525 - accuracy: 0.8205\n",
      "Epoch 331/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0472 - accuracy: 0.8205\n",
      "Epoch 332/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0421 - accuracy: 0.8205\n",
      "Epoch 333/500\n",
      "2/2 [==============================] - 0s 986us/step - loss: 1.0369 - accuracy: 0.8205\n",
      "Epoch 334/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0317 - accuracy: 0.8205\n",
      "Epoch 335/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0268 - accuracy: 0.8205\n",
      "Epoch 336/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0218 - accuracy: 0.8205\n",
      "Epoch 337/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0168 - accuracy: 0.8205\n",
      "Epoch 338/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0117 - accuracy: 0.8205\n",
      "Epoch 339/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 1.0070 - accuracy: 0.8462\n",
      "Epoch 340/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 1.0018 - accuracy: 0.8462\n",
      "Epoch 341/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9966 - accuracy: 0.8462\n",
      "Epoch 342/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9920 - accuracy: 0.8462\n",
      "Epoch 343/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9866 - accuracy: 0.8462\n",
      "Epoch 344/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.9818 - accuracy: 0.8462\n",
      "Epoch 345/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9768 - accuracy: 0.8462\n",
      "Epoch 346/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9719 - accuracy: 0.8462\n",
      "Epoch 347/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9670 - accuracy: 0.8462\n",
      "Epoch 348/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9623 - accuracy: 0.8462\n",
      "Epoch 349/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9574 - accuracy: 0.8462\n",
      "Epoch 350/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9526 - accuracy: 0.8462\n",
      "Epoch 351/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9480 - accuracy: 0.8462\n",
      "Epoch 352/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9432 - accuracy: 0.8462\n",
      "Epoch 353/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9387 - accuracy: 0.8462\n",
      "Epoch 354/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9342 - accuracy: 0.8462\n",
      "Epoch 355/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9297 - accuracy: 0.8462\n",
      "Epoch 356/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9250 - accuracy: 0.8462\n",
      "Epoch 357/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9205 - accuracy: 0.8462\n",
      "Epoch 358/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9160 - accuracy: 0.8462\n",
      "Epoch 359/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.9114 - accuracy: 0.8462\n",
      "Epoch 360/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9068 - accuracy: 0.8462\n",
      "Epoch 361/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.9026 - accuracy: 0.8462\n",
      "Epoch 362/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8982 - accuracy: 0.8462\n",
      "Epoch 363/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8935 - accuracy: 0.8462\n",
      "Epoch 364/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8893 - accuracy: 0.8462\n",
      "Epoch 365/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8852 - accuracy: 0.8462\n",
      "Epoch 366/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8809 - accuracy: 0.8462\n",
      "Epoch 367/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8766 - accuracy: 0.8462\n",
      "Epoch 368/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8725 - accuracy: 0.8462\n",
      "Epoch 369/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8685 - accuracy: 0.8462\n",
      "Epoch 370/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8642 - accuracy: 0.8462\n",
      "Epoch 371/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8599 - accuracy: 0.8462\n",
      "Epoch 372/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8557 - accuracy: 0.8462\n",
      "Epoch 373/500\n",
      "2/2 [==============================] - 0s 998us/step - loss: 0.8515 - accuracy: 0.8462\n",
      "Epoch 374/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8475 - accuracy: 0.8462\n",
      "Epoch 375/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8432 - accuracy: 0.8462\n",
      "Epoch 376/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8391 - accuracy: 0.8462\n",
      "Epoch 377/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8351 - accuracy: 0.8462\n",
      "Epoch 378/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8309 - accuracy: 0.8462\n",
      "Epoch 379/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8269 - accuracy: 0.8462\n",
      "Epoch 380/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8227 - accuracy: 0.8462\n",
      "Epoch 381/500\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 0.8186 - accuracy: 0.8462\n",
      "Epoch 382/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.8148 - accuracy: 0.8462\n",
      "Epoch 383/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8106 - accuracy: 0.8462\n",
      "Epoch 384/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.8068 - accuracy: 0.8462\n",
      "Epoch 385/500\n",
      "2/2 [==============================] - 0s 962us/step - loss: 0.8025 - accuracy: 0.8462\n",
      "Epoch 386/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7988 - accuracy: 0.8462\n",
      "Epoch 387/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7949 - accuracy: 0.8462\n",
      "Epoch 388/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7908 - accuracy: 0.8718\n",
      "Epoch 389/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7872 - accuracy: 0.8718\n",
      "Epoch 390/500\n",
      "2/2 [==============================] - 0s 821us/step - loss: 0.7834 - accuracy: 0.8718\n",
      "Epoch 391/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.7798 - accuracy: 0.8718\n",
      "Epoch 392/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7762 - accuracy: 0.8718\n",
      "Epoch 393/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7726 - accuracy: 0.8718\n",
      "Epoch 394/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.7691 - accuracy: 0.8718\n",
      "Epoch 395/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7654 - accuracy: 0.8718\n",
      "Epoch 396/500\n",
      "2/2 [==============================] - 0s 982us/step - loss: 0.7620 - accuracy: 0.8718\n",
      "Epoch 397/500\n",
      "2/2 [==============================] - 0s 906us/step - loss: 0.7583 - accuracy: 0.8718\n",
      "Epoch 398/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7549 - accuracy: 0.8718\n",
      "Epoch 399/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7511 - accuracy: 0.8718\n",
      "Epoch 400/500\n",
      "2/2 [==============================] - 0s 874us/step - loss: 0.7476 - accuracy: 0.8718\n",
      "Epoch 401/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7441 - accuracy: 0.8718\n",
      "Epoch 402/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7406 - accuracy: 0.8718\n",
      "Epoch 403/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.7372 - accuracy: 0.8718\n",
      "Epoch 404/500\n",
      "2/2 [==============================] - 0s 999us/step - loss: 0.7337 - accuracy: 0.8718\n",
      "Epoch 405/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7302 - accuracy: 0.8718\n",
      "Epoch 406/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7267 - accuracy: 0.8718\n",
      "Epoch 407/500\n",
      "2/2 [==============================] - 0s 912us/step - loss: 0.7234 - accuracy: 0.8718\n",
      "Epoch 408/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7198 - accuracy: 0.8718\n",
      "Epoch 409/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7166 - accuracy: 0.8718\n",
      "Epoch 410/500\n",
      "2/2 [==============================] - 0s 822us/step - loss: 0.7133 - accuracy: 0.8718\n",
      "Epoch 411/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.7098 - accuracy: 0.8718\n",
      "Epoch 412/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7068 - accuracy: 0.8718\n",
      "Epoch 413/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.7033 - accuracy: 0.8718\n",
      "Epoch 414/500\n",
      "2/2 [==============================] - ETA: 0s - loss: 0.7231 - accuracy: 0.84 - 0s 997us/step - loss: 0.6999 - accuracy: 0.8718\n",
      "Epoch 415/500\n",
      "2/2 [==============================] - 0s 499us/step - loss: 0.6967 - accuracy: 0.8718\n",
      "Epoch 416/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6934 - accuracy: 0.8718\n",
      "Epoch 417/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6901 - accuracy: 0.8718\n",
      "Epoch 418/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6867 - accuracy: 0.8718\n",
      "Epoch 419/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6838 - accuracy: 0.8718\n",
      "Epoch 420/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6806 - accuracy: 0.8718\n",
      "Epoch 421/500\n",
      "2/2 [==============================] - 0s 902us/step - loss: 0.6772 - accuracy: 0.8718\n",
      "Epoch 422/500\n",
      "2/2 [==============================] - 0s 998us/step - loss: 0.6741 - accuracy: 0.8718\n",
      "Epoch 423/500\n",
      "2/2 [==============================] - 0s 908us/step - loss: 0.6709 - accuracy: 0.8718\n",
      "Epoch 424/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6679 - accuracy: 0.8718\n",
      "Epoch 425/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6647 - accuracy: 0.8718\n",
      "Epoch 426/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6619 - accuracy: 0.8718\n",
      "Epoch 427/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6587 - accuracy: 0.8718\n",
      "Epoch 428/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6558 - accuracy: 0.8718\n",
      "Epoch 429/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6529 - accuracy: 0.8718\n",
      "Epoch 430/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6497 - accuracy: 0.8718\n",
      "Epoch 431/500\n",
      "2/2 [==============================] - 0s 991us/step - loss: 0.6469 - accuracy: 0.8718\n",
      "Epoch 432/500\n",
      "2/2 [==============================] - 0s 998us/step - loss: 0.6440 - accuracy: 0.8718\n",
      "Epoch 433/500\n",
      "2/2 [==============================] - 0s 984us/step - loss: 0.6413 - accuracy: 0.8718\n",
      "Epoch 434/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6383 - accuracy: 0.8718\n",
      "Epoch 435/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6355 - accuracy: 0.8718\n",
      "Epoch 436/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6328 - accuracy: 0.8718\n",
      "Epoch 437/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6299 - accuracy: 0.8718\n",
      "Epoch 438/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6270 - accuracy: 0.8718\n",
      "Epoch 439/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6244 - accuracy: 0.8718\n",
      "Epoch 440/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6217 - accuracy: 0.8718\n",
      "Epoch 441/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6191 - accuracy: 0.8718\n",
      "Epoch 442/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6164 - accuracy: 0.8718\n",
      "Epoch 443/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6138 - accuracy: 0.8718\n",
      "Epoch 444/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6114 - accuracy: 0.8718\n",
      "Epoch 445/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.6088 - accuracy: 0.8718\n",
      "Epoch 446/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6063 - accuracy: 0.8718\n",
      "Epoch 447/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6037 - accuracy: 0.8974\n",
      "Epoch 448/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.6009 - accuracy: 0.8974\n",
      "Epoch 449/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5984 - accuracy: 0.8974\n",
      "Epoch 450/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5960 - accuracy: 0.8974\n",
      "Epoch 451/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5934 - accuracy: 0.8974\n",
      "Epoch 452/500\n",
      "2/2 [==============================] - 0s 691us/step - loss: 0.5908 - accuracy: 0.8974\n",
      "Epoch 453/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5883 - accuracy: 0.8974\n",
      "Epoch 454/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5858 - accuracy: 0.8974\n",
      "Epoch 455/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5833 - accuracy: 0.8974\n",
      "Epoch 456/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5810 - accuracy: 0.8974\n",
      "Epoch 457/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5784 - accuracy: 0.8974\n",
      "Epoch 458/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5762 - accuracy: 0.8974\n",
      "Epoch 459/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5737 - accuracy: 0.8974\n",
      "Epoch 460/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5712 - accuracy: 0.8974\n",
      "Epoch 461/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5689 - accuracy: 0.8974\n",
      "Epoch 462/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.5666 - accuracy: 0.8974\n",
      "Epoch 463/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5641 - accuracy: 0.8974\n",
      "Epoch 464/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.5617 - accuracy: 0.8974\n",
      "Epoch 465/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5594 - accuracy: 0.8974\n",
      "Epoch 466/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5571 - accuracy: 0.8974\n",
      "Epoch 467/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5548 - accuracy: 0.8974\n",
      "Epoch 468/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5523 - accuracy: 0.8974\n",
      "Epoch 469/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5501 - accuracy: 0.8974\n",
      "Epoch 470/500\n",
      "2/2 [==============================] - 0s 960us/step - loss: 0.5478 - accuracy: 0.8974\n",
      "Epoch 471/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5454 - accuracy: 0.8974\n",
      "Epoch 472/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5431 - accuracy: 0.8974\n",
      "Epoch 473/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5410 - accuracy: 0.8974\n",
      "Epoch 474/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5385 - accuracy: 0.8974\n",
      "Epoch 475/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5361 - accuracy: 0.8974\n",
      "Epoch 476/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.5340 - accuracy: 0.8974\n",
      "Epoch 477/500\n",
      "2/2 [==============================] - 0s 915us/step - loss: 0.5318 - accuracy: 0.8974\n",
      "Epoch 478/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5297 - accuracy: 0.8974\n",
      "Epoch 479/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5275 - accuracy: 0.8974\n",
      "Epoch 480/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5252 - accuracy: 0.8974\n",
      "Epoch 481/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5231 - accuracy: 0.8974\n",
      "Epoch 482/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5210 - accuracy: 0.8974\n",
      "Epoch 483/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5188 - accuracy: 0.8974\n",
      "Epoch 484/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5167 - accuracy: 0.8974\n",
      "Epoch 485/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5147 - accuracy: 0.8974\n",
      "Epoch 486/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5126 - accuracy: 0.8974\n",
      "Epoch 487/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5109 - accuracy: 0.8974\n",
      "Epoch 488/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5087 - accuracy: 0.8974\n",
      "Epoch 489/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5070 - accuracy: 0.8974\n",
      "Epoch 490/500\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.5050 - accuracy: 0.8974\n",
      "Epoch 491/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.5030 - accuracy: 0.8974\n",
      "Epoch 492/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.5009 - accuracy: 0.8974\n",
      "Epoch 493/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.4992 - accuracy: 0.8974\n",
      "Epoch 494/500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 2ms/step - loss: 0.4970 - accuracy: 0.8974\n",
      "Epoch 495/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.4952 - accuracy: 0.8974\n",
      "Epoch 496/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.4932 - accuracy: 0.8974\n",
      "Epoch 497/500\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.4914 - accuracy: 0.8974\n",
      "Epoch 498/500\n",
      "2/2 [==============================] - 0s 998us/step - loss: 0.4894 - accuracy: 0.8974\n",
      "Epoch 499/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.4876 - accuracy: 0.8974\n",
      "Epoch 500/500\n",
      "2/2 [==============================] - 0s 1ms/step - loss: 0.4857 - accuracy: 0.8974\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2545f3f2bb0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Dense(10, activation='linear', input_shape=[45]),\n",
    "  tf.keras.layers.Dense(45)\n",
    "])\n",
    "\n",
    "loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
    "model.compile(optimizer='adam',\n",
    "              loss=loss_fn,\n",
    "              metrics=['accuracy'])\n",
    "model.fit(X_train, Y_train, epochs=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((45, 10), (10,))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w, b = model.layers[0].get_weights()\n",
    "w.shape, b.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### word embedding of ith training target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word embedding of \"Wonderful\" = [[ 1.24432588 -1.99069145 -1.0718352   0.09638837  0.70506335 -0.69508408\n",
      "  -1.16809948  1.61583742  0.05783282 -0.37892926]]\n"
     ]
    }
   ],
   "source": [
    "i = 25\n",
    "print(f'word embedding of \"{training_data[i][1]}\" = {np.dot(X_train[i:(i+1)], w) + b}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get word to embedding mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['is', 'an', 'excellent', 'move', 'was'], 39)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = list(map(lambda x : x[1], training_data))\n",
    "words[:5], len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>1.279979</td>\n",
       "      <td>-0.168190</td>\n",
       "      <td>0.830259</td>\n",
       "      <td>-1.351963</td>\n",
       "      <td>1.281968</td>\n",
       "      <td>-0.693271</td>\n",
       "      <td>-1.392339</td>\n",
       "      <td>0.606229</td>\n",
       "      <td>1.036305</td>\n",
       "      <td>0.439766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>an</th>\n",
       "      <td>0.486860</td>\n",
       "      <td>-1.254338</td>\n",
       "      <td>-1.101827</td>\n",
       "      <td>0.904674</td>\n",
       "      <td>-1.031337</td>\n",
       "      <td>-0.531188</td>\n",
       "      <td>-1.630290</td>\n",
       "      <td>-0.570660</td>\n",
       "      <td>1.106059</td>\n",
       "      <td>0.412546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>excellent</th>\n",
       "      <td>0.943121</td>\n",
       "      <td>-1.230062</td>\n",
       "      <td>-1.163954</td>\n",
       "      <td>-0.355245</td>\n",
       "      <td>1.236271</td>\n",
       "      <td>-0.037581</td>\n",
       "      <td>-1.106240</td>\n",
       "      <td>0.614156</td>\n",
       "      <td>1.299751</td>\n",
       "      <td>1.332984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>move</th>\n",
       "      <td>1.262650</td>\n",
       "      <td>-1.626576</td>\n",
       "      <td>-1.098237</td>\n",
       "      <td>-1.685319</td>\n",
       "      <td>1.158950</td>\n",
       "      <td>1.234595</td>\n",
       "      <td>0.690524</td>\n",
       "      <td>-0.635415</td>\n",
       "      <td>-0.811159</td>\n",
       "      <td>-1.013804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>was</th>\n",
       "      <td>2.073212</td>\n",
       "      <td>-0.482738</td>\n",
       "      <td>0.414404</td>\n",
       "      <td>-0.393205</td>\n",
       "      <td>-1.195484</td>\n",
       "      <td>1.057598</td>\n",
       "      <td>-0.312190</td>\n",
       "      <td>1.046732</td>\n",
       "      <td>-0.351269</td>\n",
       "      <td>1.011717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fantastic</th>\n",
       "      <td>-0.125868</td>\n",
       "      <td>-0.987034</td>\n",
       "      <td>-0.892619</td>\n",
       "      <td>-1.352716</td>\n",
       "      <td>0.724098</td>\n",
       "      <td>-0.408413</td>\n",
       "      <td>-0.561697</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>-1.754664</td>\n",
       "      <td>-0.287357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I</th>\n",
       "      <td>1.495555</td>\n",
       "      <td>0.459882</td>\n",
       "      <td>-0.693218</td>\n",
       "      <td>0.799863</td>\n",
       "      <td>-0.342810</td>\n",
       "      <td>1.351781</td>\n",
       "      <td>0.193604</td>\n",
       "      <td>1.593190</td>\n",
       "      <td>-0.082925</td>\n",
       "      <td>1.054695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>-0.479119</td>\n",
       "      <td>-0.145885</td>\n",
       "      <td>0.817430</td>\n",
       "      <td>-1.168349</td>\n",
       "      <td>0.993559</td>\n",
       "      <td>-1.390241</td>\n",
       "      <td>-1.301019</td>\n",
       "      <td>1.848546</td>\n",
       "      <td>-1.644978</td>\n",
       "      <td>1.061836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>should</th>\n",
       "      <td>0.467065</td>\n",
       "      <td>0.853717</td>\n",
       "      <td>-1.323212</td>\n",
       "      <td>-0.346991</td>\n",
       "      <td>0.601265</td>\n",
       "      <td>1.220424</td>\n",
       "      <td>-0.572727</td>\n",
       "      <td>1.608496</td>\n",
       "      <td>1.524196</td>\n",
       "      <td>-0.734867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>watch</th>\n",
       "      <td>0.154833</td>\n",
       "      <td>0.239015</td>\n",
       "      <td>1.757576</td>\n",
       "      <td>-0.263729</td>\n",
       "      <td>1.068222</td>\n",
       "      <td>0.016080</td>\n",
       "      <td>-0.157643</td>\n",
       "      <td>1.234838</td>\n",
       "      <td>-1.576562</td>\n",
       "      <td>1.459731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>it</th>\n",
       "      <td>1.347788</td>\n",
       "      <td>-0.210955</td>\n",
       "      <td>-1.344289</td>\n",
       "      <td>0.757751</td>\n",
       "      <td>-0.100474</td>\n",
       "      <td>1.062381</td>\n",
       "      <td>-1.539497</td>\n",
       "      <td>0.315590</td>\n",
       "      <td>1.491016</td>\n",
       "      <td>-1.066123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>0.791544</td>\n",
       "      <td>-0.094089</td>\n",
       "      <td>1.307308</td>\n",
       "      <td>-1.445009</td>\n",
       "      <td>1.462330</td>\n",
       "      <td>-0.955036</td>\n",
       "      <td>-0.762629</td>\n",
       "      <td>1.795037</td>\n",
       "      <td>-0.592670</td>\n",
       "      <td>-0.036388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>directed</th>\n",
       "      <td>-0.639316</td>\n",
       "      <td>-1.395028</td>\n",
       "      <td>1.318872</td>\n",
       "      <td>1.217930</td>\n",
       "      <td>-0.931840</td>\n",
       "      <td>-0.877748</td>\n",
       "      <td>0.171016</td>\n",
       "      <td>1.409013</td>\n",
       "      <td>0.466272</td>\n",
       "      <td>1.472684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>and</th>\n",
       "      <td>1.424028</td>\n",
       "      <td>1.177709</td>\n",
       "      <td>0.963026</td>\n",
       "      <td>-1.311468</td>\n",
       "      <td>1.317188</td>\n",
       "      <td>-0.263086</td>\n",
       "      <td>0.766436</td>\n",
       "      <td>-0.785470</td>\n",
       "      <td>-1.394332</td>\n",
       "      <td>1.709904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>executed</th>\n",
       "      <td>-0.989480</td>\n",
       "      <td>-1.152912</td>\n",
       "      <td>0.530879</td>\n",
       "      <td>0.050701</td>\n",
       "      <td>-0.101084</td>\n",
       "      <td>-0.998683</td>\n",
       "      <td>-1.559932</td>\n",
       "      <td>1.590993</td>\n",
       "      <td>-0.751359</td>\n",
       "      <td>1.171728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I</th>\n",
       "      <td>0.982204</td>\n",
       "      <td>0.795800</td>\n",
       "      <td>-0.563434</td>\n",
       "      <td>0.252526</td>\n",
       "      <td>0.862688</td>\n",
       "      <td>0.983831</td>\n",
       "      <td>0.856547</td>\n",
       "      <td>0.826634</td>\n",
       "      <td>-0.498088</td>\n",
       "      <td>1.852580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>-0.479119</td>\n",
       "      <td>-0.145885</td>\n",
       "      <td>0.817430</td>\n",
       "      <td>-1.168349</td>\n",
       "      <td>0.993559</td>\n",
       "      <td>-1.390241</td>\n",
       "      <td>-1.301019</td>\n",
       "      <td>1.848546</td>\n",
       "      <td>-1.644978</td>\n",
       "      <td>1.061836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>1.852092</td>\n",
       "      <td>0.955814</td>\n",
       "      <td>0.551726</td>\n",
       "      <td>0.177806</td>\n",
       "      <td>-0.844049</td>\n",
       "      <td>1.581987</td>\n",
       "      <td>-0.833305</td>\n",
       "      <td>-0.086569</td>\n",
       "      <td>-0.763569</td>\n",
       "      <td>-0.544483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fantastic</th>\n",
       "      <td>0.387453</td>\n",
       "      <td>-1.093300</td>\n",
       "      <td>-0.833781</td>\n",
       "      <td>-0.803092</td>\n",
       "      <td>0.120173</td>\n",
       "      <td>-0.804376</td>\n",
       "      <td>-1.653741</td>\n",
       "      <td>1.185753</td>\n",
       "      <td>-1.506921</td>\n",
       "      <td>-1.245209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>watched</th>\n",
       "      <td>0.869159</td>\n",
       "      <td>0.888009</td>\n",
       "      <td>1.131007</td>\n",
       "      <td>-1.012071</td>\n",
       "      <td>-0.842516</td>\n",
       "      <td>-0.538187</td>\n",
       "      <td>-1.534824</td>\n",
       "      <td>-0.873609</td>\n",
       "      <td>0.727376</td>\n",
       "      <td>-1.073182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>such</th>\n",
       "      <td>0.687290</td>\n",
       "      <td>-1.604573</td>\n",
       "      <td>0.549139</td>\n",
       "      <td>0.491414</td>\n",
       "      <td>1.206981</td>\n",
       "      <td>0.344249</td>\n",
       "      <td>-1.660254</td>\n",
       "      <td>0.272007</td>\n",
       "      <td>-1.270126</td>\n",
       "      <td>-0.651424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>1.144849</td>\n",
       "      <td>0.722363</td>\n",
       "      <td>1.339614</td>\n",
       "      <td>-0.352596</td>\n",
       "      <td>-1.135759</td>\n",
       "      <td>0.844009</td>\n",
       "      <td>-1.447171</td>\n",
       "      <td>-0.957685</td>\n",
       "      <td>-0.811867</td>\n",
       "      <td>-0.764936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brillent</th>\n",
       "      <td>1.244326</td>\n",
       "      <td>-1.990691</td>\n",
       "      <td>-1.071835</td>\n",
       "      <td>0.096388</td>\n",
       "      <td>0.705063</td>\n",
       "      <td>-0.695084</td>\n",
       "      <td>-1.168099</td>\n",
       "      <td>1.615837</td>\n",
       "      <td>0.057833</td>\n",
       "      <td>-0.378929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>1.783988</td>\n",
       "      <td>-1.119847</td>\n",
       "      <td>0.562748</td>\n",
       "      <td>-0.541942</td>\n",
       "      <td>0.505410</td>\n",
       "      <td>-0.635859</td>\n",
       "      <td>-1.822202</td>\n",
       "      <td>1.366307</td>\n",
       "      <td>-0.256420</td>\n",
       "      <td>-0.279119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>1.364920</td>\n",
       "      <td>-0.349260</td>\n",
       "      <td>0.352901</td>\n",
       "      <td>0.954158</td>\n",
       "      <td>-0.794100</td>\n",
       "      <td>1.080916</td>\n",
       "      <td>-2.064640</td>\n",
       "      <td>-0.819796</td>\n",
       "      <td>-0.224315</td>\n",
       "      <td>-0.824494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wonderful</th>\n",
       "      <td>1.244326</td>\n",
       "      <td>-1.990691</td>\n",
       "      <td>-1.071835</td>\n",
       "      <td>0.096388</td>\n",
       "      <td>0.705063</td>\n",
       "      <td>-0.695084</td>\n",
       "      <td>-1.168099</td>\n",
       "      <td>1.615837</td>\n",
       "      <td>0.057833</td>\n",
       "      <td>-0.378929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>of</th>\n",
       "      <td>-0.571901</td>\n",
       "      <td>0.140055</td>\n",
       "      <td>-0.699897</td>\n",
       "      <td>-1.205471</td>\n",
       "      <td>-1.211351</td>\n",
       "      <td>1.092705</td>\n",
       "      <td>0.940259</td>\n",
       "      <td>1.767822</td>\n",
       "      <td>-1.455829</td>\n",
       "      <td>-0.881459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>was</th>\n",
       "      <td>1.514303</td>\n",
       "      <td>-0.569155</td>\n",
       "      <td>0.702545</td>\n",
       "      <td>-1.170012</td>\n",
       "      <td>-0.577843</td>\n",
       "      <td>0.716006</td>\n",
       "      <td>-1.468591</td>\n",
       "      <td>0.964453</td>\n",
       "      <td>0.271549</td>\n",
       "      <td>1.313006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>very</th>\n",
       "      <td>1.300840</td>\n",
       "      <td>-0.811262</td>\n",
       "      <td>-0.897182</td>\n",
       "      <td>-0.899629</td>\n",
       "      <td>1.218420</td>\n",
       "      <td>-0.637944</td>\n",
       "      <td>0.716570</td>\n",
       "      <td>0.250913</td>\n",
       "      <td>-1.497871</td>\n",
       "      <td>-1.426832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>did</th>\n",
       "      <td>-1.161269</td>\n",
       "      <td>-0.067313</td>\n",
       "      <td>-0.223932</td>\n",
       "      <td>-1.420252</td>\n",
       "      <td>0.446580</td>\n",
       "      <td>-1.186983</td>\n",
       "      <td>-1.652572</td>\n",
       "      <td>1.512532</td>\n",
       "      <td>-0.595576</td>\n",
       "      <td>0.839084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not</th>\n",
       "      <td>-0.103361</td>\n",
       "      <td>-0.768183</td>\n",
       "      <td>0.068858</td>\n",
       "      <td>0.256899</td>\n",
       "      <td>0.096875</td>\n",
       "      <td>1.419525</td>\n",
       "      <td>0.862799</td>\n",
       "      <td>1.654339</td>\n",
       "      <td>0.628388</td>\n",
       "      <td>1.630065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>-1.050563</td>\n",
       "      <td>0.070685</td>\n",
       "      <td>0.634740</td>\n",
       "      <td>-1.882337</td>\n",
       "      <td>0.698532</td>\n",
       "      <td>-0.691380</td>\n",
       "      <td>-0.499893</td>\n",
       "      <td>1.548982</td>\n",
       "      <td>-0.605183</td>\n",
       "      <td>1.051472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>the</th>\n",
       "      <td>0.926230</td>\n",
       "      <td>-0.954720</td>\n",
       "      <td>-1.551630</td>\n",
       "      <td>0.847681</td>\n",
       "      <td>0.792615</td>\n",
       "      <td>0.454574</td>\n",
       "      <td>-0.181653</td>\n",
       "      <td>1.814796</td>\n",
       "      <td>1.085338</td>\n",
       "      <td>1.359404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movie</th>\n",
       "      <td>1.262650</td>\n",
       "      <td>-1.626576</td>\n",
       "      <td>-1.098237</td>\n",
       "      <td>-1.685319</td>\n",
       "      <td>1.158950</td>\n",
       "      <td>1.234595</td>\n",
       "      <td>0.690524</td>\n",
       "      <td>-0.635415</td>\n",
       "      <td>-0.811159</td>\n",
       "      <td>-1.013804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>was</th>\n",
       "      <td>1.377285</td>\n",
       "      <td>-1.652971</td>\n",
       "      <td>-0.390271</td>\n",
       "      <td>-0.274751</td>\n",
       "      <td>-0.332159</td>\n",
       "      <td>0.343739</td>\n",
       "      <td>-1.180328</td>\n",
       "      <td>1.644106</td>\n",
       "      <td>0.539322</td>\n",
       "      <td>1.356817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>will</th>\n",
       "      <td>-1.161269</td>\n",
       "      <td>-0.067313</td>\n",
       "      <td>-0.223932</td>\n",
       "      <td>-1.420252</td>\n",
       "      <td>0.446580</td>\n",
       "      <td>-1.186983</td>\n",
       "      <td>-1.652572</td>\n",
       "      <td>1.512532</td>\n",
       "      <td>-0.595576</td>\n",
       "      <td>0.839084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not</th>\n",
       "      <td>-0.830435</td>\n",
       "      <td>-1.152848</td>\n",
       "      <td>1.420053</td>\n",
       "      <td>-1.009482</td>\n",
       "      <td>-0.655185</td>\n",
       "      <td>1.516797</td>\n",
       "      <td>0.803844</td>\n",
       "      <td>1.593383</td>\n",
       "      <td>0.719094</td>\n",
       "      <td>1.376007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acting</th>\n",
       "      <td>1.584816</td>\n",
       "      <td>-1.301690</td>\n",
       "      <td>-0.835288</td>\n",
       "      <td>-0.291000</td>\n",
       "      <td>0.168175</td>\n",
       "      <td>1.226218</td>\n",
       "      <td>-0.877729</td>\n",
       "      <td>-0.859732</td>\n",
       "      <td>0.714319</td>\n",
       "      <td>-0.858994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>1.223565</td>\n",
       "      <td>-0.195062</td>\n",
       "      <td>0.958674</td>\n",
       "      <td>-1.474705</td>\n",
       "      <td>1.321545</td>\n",
       "      <td>-0.575673</td>\n",
       "      <td>-1.466041</td>\n",
       "      <td>1.061264</td>\n",
       "      <td>0.814058</td>\n",
       "      <td>-0.571395</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1         2         3         4         5  \\\n",
       "is         1.279979 -0.168190  0.830259 -1.351963  1.281968 -0.693271   \n",
       "an         0.486860 -1.254338 -1.101827  0.904674 -1.031337 -0.531188   \n",
       "excellent  0.943121 -1.230062 -1.163954 -0.355245  1.236271 -0.037581   \n",
       "move       1.262650 -1.626576 -1.098237 -1.685319  1.158950  1.234595   \n",
       "was        2.073212 -0.482738  0.414404 -0.393205 -1.195484  1.057598   \n",
       "fantastic -0.125868 -0.987034 -0.892619 -1.352716  0.724098 -0.408413   \n",
       "I          1.495555  0.459882 -0.693218  0.799863 -0.342810  1.351781   \n",
       "like      -0.479119 -0.145885  0.817430 -1.168349  0.993559 -1.390241   \n",
       "should     0.467065  0.853717 -1.323212 -0.346991  0.601265  1.220424   \n",
       "watch      0.154833  0.239015  1.757576 -0.263729  1.068222  0.016080   \n",
       "it         1.347788 -0.210955 -1.344289  0.757751 -0.100474  1.062381   \n",
       "is         0.791544 -0.094089  1.307308 -1.445009  1.462330 -0.955036   \n",
       "directed  -0.639316 -1.395028  1.318872  1.217930 -0.931840 -0.877748   \n",
       "and        1.424028  1.177709  0.963026 -1.311468  1.317188 -0.263086   \n",
       "executed  -0.989480 -1.152912  0.530879  0.050701 -0.101084 -0.998683   \n",
       "I          0.982204  0.795800 -0.563434  0.252526  0.862688  0.983831   \n",
       "like      -0.479119 -0.145885  0.817430 -1.168349  0.993559 -1.390241   \n",
       "a          1.852092  0.955814  0.551726  0.177806 -0.844049  1.581987   \n",
       "fantastic  0.387453 -1.093300 -0.833781 -0.803092  0.120173 -0.804376   \n",
       "watched    0.869159  0.888009  1.131007 -1.012071 -0.842516 -0.538187   \n",
       "such       0.687290 -1.604573  0.549139  0.491414  1.206981  0.344249   \n",
       "a          1.144849  0.722363  1.339614 -0.352596 -1.135759  0.844009   \n",
       "brillent   1.244326 -1.990691 -1.071835  0.096388  0.705063 -0.695084   \n",
       "is         1.783988 -1.119847  0.562748 -0.541942  0.505410 -0.635859   \n",
       "a          1.364920 -0.349260  0.352901  0.954158 -0.794100  1.080916   \n",
       "Wonderful  1.244326 -1.990691 -1.071835  0.096388  0.705063 -0.695084   \n",
       "of        -0.571901  0.140055 -0.699897 -1.205471 -1.211351  1.092705   \n",
       "was        1.514303 -0.569155  0.702545 -1.170012 -0.577843  0.716006   \n",
       "very       1.300840 -0.811262 -0.897182 -0.899629  1.218420 -0.637944   \n",
       "did       -1.161269 -0.067313 -0.223932 -1.420252  0.446580 -1.186983   \n",
       "not       -0.103361 -0.768183  0.068858  0.256899  0.096875  1.419525   \n",
       "like      -1.050563  0.070685  0.634740 -1.882337  0.698532 -0.691380   \n",
       "the        0.926230 -0.954720 -1.551630  0.847681  0.792615  0.454574   \n",
       "movie      1.262650 -1.626576 -1.098237 -1.685319  1.158950  1.234595   \n",
       "was        1.377285 -1.652971 -0.390271 -0.274751 -0.332159  0.343739   \n",
       "will      -1.161269 -0.067313 -0.223932 -1.420252  0.446580 -1.186983   \n",
       "not       -0.830435 -1.152848  1.420053 -1.009482 -0.655185  1.516797   \n",
       "acting     1.584816 -1.301690 -0.835288 -0.291000  0.168175  1.226218   \n",
       "is         1.223565 -0.195062  0.958674 -1.474705  1.321545 -0.575673   \n",
       "\n",
       "                  6         7         8         9  \n",
       "is        -1.392339  0.606229  1.036305  0.439766  \n",
       "an        -1.630290 -0.570660  1.106059  0.412546  \n",
       "excellent -1.106240  0.614156  1.299751  1.332984  \n",
       "move       0.690524 -0.635415 -0.811159 -1.013804  \n",
       "was       -0.312190  1.046732 -0.351269  1.011717  \n",
       "fantastic -0.561697  0.590947 -1.754664 -0.287357  \n",
       "I          0.193604  1.593190 -0.082925  1.054695  \n",
       "like      -1.301019  1.848546 -1.644978  1.061836  \n",
       "should    -0.572727  1.608496  1.524196 -0.734867  \n",
       "watch     -0.157643  1.234838 -1.576562  1.459731  \n",
       "it        -1.539497  0.315590  1.491016 -1.066123  \n",
       "is        -0.762629  1.795037 -0.592670 -0.036388  \n",
       "directed   0.171016  1.409013  0.466272  1.472684  \n",
       "and        0.766436 -0.785470 -1.394332  1.709904  \n",
       "executed  -1.559932  1.590993 -0.751359  1.171728  \n",
       "I          0.856547  0.826634 -0.498088  1.852580  \n",
       "like      -1.301019  1.848546 -1.644978  1.061836  \n",
       "a         -0.833305 -0.086569 -0.763569 -0.544483  \n",
       "fantastic -1.653741  1.185753 -1.506921 -1.245209  \n",
       "watched   -1.534824 -0.873609  0.727376 -1.073182  \n",
       "such      -1.660254  0.272007 -1.270126 -0.651424  \n",
       "a         -1.447171 -0.957685 -0.811867 -0.764936  \n",
       "brillent  -1.168099  1.615837  0.057833 -0.378929  \n",
       "is        -1.822202  1.366307 -0.256420 -0.279119  \n",
       "a         -2.064640 -0.819796 -0.224315 -0.824494  \n",
       "Wonderful -1.168099  1.615837  0.057833 -0.378929  \n",
       "of         0.940259  1.767822 -1.455829 -0.881459  \n",
       "was       -1.468591  0.964453  0.271549  1.313006  \n",
       "very       0.716570  0.250913 -1.497871 -1.426832  \n",
       "did       -1.652572  1.512532 -0.595576  0.839084  \n",
       "not        0.862799  1.654339  0.628388  1.630065  \n",
       "like      -0.499893  1.548982 -0.605183  1.051472  \n",
       "the       -0.181653  1.814796  1.085338  1.359404  \n",
       "movie      0.690524 -0.635415 -0.811159 -1.013804  \n",
       "was       -1.180328  1.644106  0.539322  1.356817  \n",
       "will      -1.652572  1.512532 -0.595576  0.839084  \n",
       "not        0.803844  1.593383  0.719094  1.376007  \n",
       "acting    -0.877729 -0.859732  0.714319 -0.858994  \n",
       "is        -1.466041  1.061264  0.814058 -0.571395  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(np.dot(X_train, w) + b)\n",
    "df.index = words\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27, 10)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# average out representation of each word\n",
    "df = df.groupby(df.index).mean()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### visualize embeddings in tensorflow projector\n",
    "http://projector.tensorflow.org/\n",
    "\n",
    "upload metatdata.tsv and vectors.tsv to visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext tensorboard\n",
    "import os\n",
    "logdir = \"logs/visualize_embedding\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{logdir}/metadata.tsv', 'w') as f:\n",
    "    f.write('\\n'.join(df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(f'{logdir}/vectors.tsv', index=False, sep='\\t', header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the output was boring :(\n",
    "### Let's try with more data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "tfds.disable_progress_bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = tfds.load('imdb_reviews/plain_text', split='unsupervised')\n",
    "data = data.shuffle(1024)\n",
    "it = iter(data)\n",
    "\n",
    "reviews = []\n",
    "examples = 100\n",
    "for i in range(examples):\n",
    "    review = it.next()\n",
    "    reviews.append(review['text'].numpy().decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_a_sentence(sentence):\n",
    "    import re\n",
    "    sentence = re.sub(r'<br />', ' ', sentence)\n",
    "    sentence = re.sub(r'[, ]+', ' ', sentence).strip()\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sentences = []\n",
    "for review in reviews:\n",
    "    sentences.extend(list(filter(lambda line: line != '', map(lambda line: clean_a_sentence(line), review.split('.')))))\n",
    "sentences = [sentence.split(' ') for sentence in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['You',\n",
       " 'know',\n",
       " \"it's\",\n",
       " 'like',\n",
       " 'a',\n",
       " 'Spaghetti',\n",
       " 'Western',\n",
       " 'but',\n",
       " 'with',\n",
       " 'ray-guns']"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set([word for sen in sentences for word in sen])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5744"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2id = {word:i for i, word in enumerate(vocab)}\n",
    "id2word = list(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 3 # keep it odd\n",
    "training_data = [] # [[x1,x2],y1]\n",
    "for i, sentence in enumerate(sentences):\n",
    "    training_data.extend(create_training_data_from_a_sentence(sentence, window_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([[['You', \"it's\"], 'know'],\n",
       "  [['know', 'like'], \"it's\"],\n",
       "  [[\"it's\", 'a'], 'like'],\n",
       "  [['like', 'Spaghetti'], 'a']],\n",
       " 19993)"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data[:4], len(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  24, 4323],\n",
       "       [2765,  958],\n",
       "       [4323, 4316],\n",
       "       ...,\n",
       "       [4323, 4316],\n",
       "       [4610, 3148],\n",
       "       [4316, 3762]])"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = [[word2id[j] for j in i[0]] for i in training_data]\n",
    "train_X = np.array(train_X)\n",
    "train_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2765],\n",
       "       [4323],\n",
       "       [ 958],\n",
       "       ...,\n",
       "       [4610],\n",
       "       [4316],\n",
       "       [3148]])"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_Y = [[word2id[i[1]]] for i in training_data]\n",
    "train_Y = np.array(train_Y)\n",
    "train_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((19993, 2), (19993, 1))"
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape, train_Y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## this time using built in embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Embedding, Dense, GlobalAveragePooling1D\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.losses import SparseCategoricalCrossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 16\n",
    "model = Sequential([Embedding(input_dim=len(vocab), output_dim=embedding_dim),\n",
    "                   GlobalAveragePooling1D(),\n",
    "                   Dense(len(vocab))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(Adam(0.0005), loss=SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, None, 16)          91904     \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 5744)              97648     \n",
      "=================================================================\n",
      "Total params: 189,552\n",
      "Trainable params: 189,552\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I ran model fit multiple times with different epochs and learning rate\n",
    "# so don't think it started with 71% accuracy (instead it started with 5%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1532 - accuracy: 0.7141\n",
      "Epoch 2/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1483 - accuracy: 0.7151\n",
      "Epoch 3/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1475 - accuracy: 0.7145\n",
      "Epoch 4/10\n",
      "625/625 [==============================] - 3s 4ms/step - loss: 1.1468 - accuracy: 0.7150\n",
      "Epoch 5/10\n",
      "625/625 [==============================] - 3s 4ms/step - loss: 1.1461 - accuracy: 0.7154\n",
      "Epoch 6/10\n",
      "625/625 [==============================] - 3s 4ms/step - loss: 1.1456 - accuracy: 0.7148\n",
      "Epoch 7/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1450 - accuracy: 0.7149\n",
      "Epoch 8/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1445 - accuracy: 0.7153\n",
      "Epoch 9/10\n",
      "625/625 [==============================] - 2s 4ms/step - loss: 1.1440 - accuracy: 0.7156\n",
      "Epoch 10/10\n",
      "625/625 [==============================] - 3s 4ms/step - loss: 1.1435 - accuracy: 0.7170\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_X, train_Y, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(model.layers[1](model.layers[0](train_X)).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>know</th>\n",
       "      <td>-1.568521</td>\n",
       "      <td>2.159748</td>\n",
       "      <td>0.073384</td>\n",
       "      <td>0.697752</td>\n",
       "      <td>-2.365087</td>\n",
       "      <td>1.776173</td>\n",
       "      <td>-2.214857</td>\n",
       "      <td>-1.254740</td>\n",
       "      <td>-0.276653</td>\n",
       "      <td>0.482874</td>\n",
       "      <td>1.926827</td>\n",
       "      <td>-0.159665</td>\n",
       "      <td>0.721765</td>\n",
       "      <td>-1.239955</td>\n",
       "      <td>0.941057</td>\n",
       "      <td>-1.453552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>it's</th>\n",
       "      <td>1.219179</td>\n",
       "      <td>-1.624454</td>\n",
       "      <td>-0.584512</td>\n",
       "      <td>-0.441756</td>\n",
       "      <td>-1.670563</td>\n",
       "      <td>-0.200357</td>\n",
       "      <td>-1.627250</td>\n",
       "      <td>-0.691518</td>\n",
       "      <td>-0.184850</td>\n",
       "      <td>0.100584</td>\n",
       "      <td>0.852530</td>\n",
       "      <td>-0.432806</td>\n",
       "      <td>-1.578029</td>\n",
       "      <td>-0.864022</td>\n",
       "      <td>0.102814</td>\n",
       "      <td>0.621504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>-0.771871</td>\n",
       "      <td>0.139178</td>\n",
       "      <td>1.307506</td>\n",
       "      <td>-0.241945</td>\n",
       "      <td>0.452467</td>\n",
       "      <td>-0.584545</td>\n",
       "      <td>0.383954</td>\n",
       "      <td>-0.604088</td>\n",
       "      <td>-0.184459</td>\n",
       "      <td>-1.163455</td>\n",
       "      <td>0.934987</td>\n",
       "      <td>2.023859</td>\n",
       "      <td>0.721423</td>\n",
       "      <td>-0.588926</td>\n",
       "      <td>-0.190456</td>\n",
       "      <td>-1.564948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>-1.632095</td>\n",
       "      <td>-2.473989</td>\n",
       "      <td>-2.767762</td>\n",
       "      <td>0.265900</td>\n",
       "      <td>-2.337109</td>\n",
       "      <td>-0.610698</td>\n",
       "      <td>-0.703219</td>\n",
       "      <td>1.637693</td>\n",
       "      <td>-1.570144</td>\n",
       "      <td>-0.175825</td>\n",
       "      <td>-4.160265</td>\n",
       "      <td>2.142572</td>\n",
       "      <td>-3.034826</td>\n",
       "      <td>-0.165649</td>\n",
       "      <td>2.854442</td>\n",
       "      <td>0.478558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spaghetti</th>\n",
       "      <td>-3.633009</td>\n",
       "      <td>2.471024</td>\n",
       "      <td>-1.740682</td>\n",
       "      <td>-0.981225</td>\n",
       "      <td>0.150124</td>\n",
       "      <td>1.703184</td>\n",
       "      <td>1.680602</td>\n",
       "      <td>-2.388871</td>\n",
       "      <td>1.063939</td>\n",
       "      <td>1.032531</td>\n",
       "      <td>0.074336</td>\n",
       "      <td>-0.468216</td>\n",
       "      <td>1.887318</td>\n",
       "      <td>-2.759635</td>\n",
       "      <td>0.090123</td>\n",
       "      <td>-3.745395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>current</th>\n",
       "      <td>-0.176891</td>\n",
       "      <td>1.042812</td>\n",
       "      <td>1.305274</td>\n",
       "      <td>-0.822389</td>\n",
       "      <td>-1.419245</td>\n",
       "      <td>0.360981</td>\n",
       "      <td>1.518140</td>\n",
       "      <td>-0.964185</td>\n",
       "      <td>-0.819043</td>\n",
       "      <td>0.443630</td>\n",
       "      <td>0.728262</td>\n",
       "      <td>0.831522</td>\n",
       "      <td>0.604441</td>\n",
       "      <td>0.418223</td>\n",
       "      <td>-0.888275</td>\n",
       "      <td>-0.229456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>it's</th>\n",
       "      <td>1.698527</td>\n",
       "      <td>-3.353550</td>\n",
       "      <td>-2.800743</td>\n",
       "      <td>-0.441903</td>\n",
       "      <td>-3.601066</td>\n",
       "      <td>1.238906</td>\n",
       "      <td>-2.870667</td>\n",
       "      <td>-2.572163</td>\n",
       "      <td>-1.608111</td>\n",
       "      <td>1.117736</td>\n",
       "      <td>-1.851267</td>\n",
       "      <td>0.050666</td>\n",
       "      <td>-0.454642</td>\n",
       "      <td>0.484064</td>\n",
       "      <td>-0.713263</td>\n",
       "      <td>0.077774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not</th>\n",
       "      <td>-0.771871</td>\n",
       "      <td>0.139178</td>\n",
       "      <td>1.307506</td>\n",
       "      <td>-0.241945</td>\n",
       "      <td>0.452467</td>\n",
       "      <td>-0.584545</td>\n",
       "      <td>0.383954</td>\n",
       "      <td>-0.604088</td>\n",
       "      <td>-0.184459</td>\n",
       "      <td>-1.163455</td>\n",
       "      <td>0.934987</td>\n",
       "      <td>2.023859</td>\n",
       "      <td>0.721423</td>\n",
       "      <td>-0.588926</td>\n",
       "      <td>-0.190456</td>\n",
       "      <td>-1.564948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>-1.400604</td>\n",
       "      <td>-0.996924</td>\n",
       "      <td>-0.616170</td>\n",
       "      <td>-1.156898</td>\n",
       "      <td>-2.048256</td>\n",
       "      <td>1.007186</td>\n",
       "      <td>-0.675739</td>\n",
       "      <td>0.967947</td>\n",
       "      <td>-0.284731</td>\n",
       "      <td>1.763657</td>\n",
       "      <td>-1.291728</td>\n",
       "      <td>0.588641</td>\n",
       "      <td>-0.448475</td>\n",
       "      <td>-1.043121</td>\n",
       "      <td>1.432992</td>\n",
       "      <td>1.248252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total</th>\n",
       "      <td>-2.029602</td>\n",
       "      <td>-1.282246</td>\n",
       "      <td>4.441776</td>\n",
       "      <td>0.578677</td>\n",
       "      <td>3.847076</td>\n",
       "      <td>0.328053</td>\n",
       "      <td>4.449231</td>\n",
       "      <td>-2.305887</td>\n",
       "      <td>-1.618127</td>\n",
       "      <td>-3.811579</td>\n",
       "      <td>-1.577284</td>\n",
       "      <td>-0.018852</td>\n",
       "      <td>-1.309707</td>\n",
       "      <td>1.759614</td>\n",
       "      <td>-0.702199</td>\n",
       "      <td>-0.557159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19993 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1         2         3         4         5  \\\n",
       "know      -1.568521  2.159748  0.073384  0.697752 -2.365087  1.776173   \n",
       "it's       1.219179 -1.624454 -0.584512 -0.441756 -1.670563 -0.200357   \n",
       "like      -0.771871  0.139178  1.307506 -0.241945  0.452467 -0.584545   \n",
       "a         -1.632095 -2.473989 -2.767762  0.265900 -2.337109 -0.610698   \n",
       "Spaghetti -3.633009  2.471024 -1.740682 -0.981225  0.150124  1.703184   \n",
       "...             ...       ...       ...       ...       ...       ...   \n",
       "current   -0.176891  1.042812  1.305274 -0.822389 -1.419245  0.360981   \n",
       "it's       1.698527 -3.353550 -2.800743 -0.441903 -3.601066  1.238906   \n",
       "not       -0.771871  0.139178  1.307506 -0.241945  0.452467 -0.584545   \n",
       "a         -1.400604 -0.996924 -0.616170 -1.156898 -2.048256  1.007186   \n",
       "total     -2.029602 -1.282246  4.441776  0.578677  3.847076  0.328053   \n",
       "\n",
       "                  6         7         8         9        10        11  \\\n",
       "know      -2.214857 -1.254740 -0.276653  0.482874  1.926827 -0.159665   \n",
       "it's      -1.627250 -0.691518 -0.184850  0.100584  0.852530 -0.432806   \n",
       "like       0.383954 -0.604088 -0.184459 -1.163455  0.934987  2.023859   \n",
       "a         -0.703219  1.637693 -1.570144 -0.175825 -4.160265  2.142572   \n",
       "Spaghetti  1.680602 -2.388871  1.063939  1.032531  0.074336 -0.468216   \n",
       "...             ...       ...       ...       ...       ...       ...   \n",
       "current    1.518140 -0.964185 -0.819043  0.443630  0.728262  0.831522   \n",
       "it's      -2.870667 -2.572163 -1.608111  1.117736 -1.851267  0.050666   \n",
       "not        0.383954 -0.604088 -0.184459 -1.163455  0.934987  2.023859   \n",
       "a         -0.675739  0.967947 -0.284731  1.763657 -1.291728  0.588641   \n",
       "total      4.449231 -2.305887 -1.618127 -3.811579 -1.577284 -0.018852   \n",
       "\n",
       "                 12        13        14        15  \n",
       "know       0.721765 -1.239955  0.941057 -1.453552  \n",
       "it's      -1.578029 -0.864022  0.102814  0.621504  \n",
       "like       0.721423 -0.588926 -0.190456 -1.564948  \n",
       "a         -3.034826 -0.165649  2.854442  0.478558  \n",
       "Spaghetti  1.887318 -2.759635  0.090123 -3.745395  \n",
       "...             ...       ...       ...       ...  \n",
       "current    0.604441  0.418223 -0.888275 -0.229456  \n",
       "it's      -0.454642  0.484064 -0.713263  0.077774  \n",
       "not        0.721423 -0.588926 -0.190456 -1.564948  \n",
       "a         -0.448475 -1.043121  1.432992  1.248252  \n",
       "total     -1.309707  1.759614 -0.702199 -0.557159  \n",
       "\n",
       "[19993 rows x 16 columns]"
      ]
     },
     "execution_count": 402,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.index = list(map(lambda x : id2word[x], train_Y.squeeze()))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5126, 16)"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# average out representation of each word\n",
    "df = df.groupby(df.index).mean()\n",
    "df.shape # < vocab size which was expected - why < - first and last words of each sentence are not considered for Y var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>!</th>\n",
       "      <td>0.614773</td>\n",
       "      <td>2.184768</td>\n",
       "      <td>-1.875291</td>\n",
       "      <td>-2.392619</td>\n",
       "      <td>-1.983358</td>\n",
       "      <td>-0.466332</td>\n",
       "      <td>-1.477482</td>\n",
       "      <td>0.724128</td>\n",
       "      <td>1.692778</td>\n",
       "      <td>1.414513</td>\n",
       "      <td>1.626997</td>\n",
       "      <td>-0.187993</td>\n",
       "      <td>1.396734</td>\n",
       "      <td>0.893966</td>\n",
       "      <td>0.031648</td>\n",
       "      <td>0.464041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>!)</th>\n",
       "      <td>-1.242826</td>\n",
       "      <td>0.632045</td>\n",
       "      <td>-0.905323</td>\n",
       "      <td>-1.453357</td>\n",
       "      <td>-2.560370</td>\n",
       "      <td>-2.671518</td>\n",
       "      <td>-0.353624</td>\n",
       "      <td>1.379483</td>\n",
       "      <td>-0.828765</td>\n",
       "      <td>0.013821</td>\n",
       "      <td>1.080674</td>\n",
       "      <td>0.351833</td>\n",
       "      <td>0.339504</td>\n",
       "      <td>0.983073</td>\n",
       "      <td>0.977823</td>\n",
       "      <td>-0.570352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"</th>\n",
       "      <td>0.348271</td>\n",
       "      <td>1.808484</td>\n",
       "      <td>-2.669932</td>\n",
       "      <td>-0.228845</td>\n",
       "      <td>1.090854</td>\n",
       "      <td>-1.378595</td>\n",
       "      <td>-1.647741</td>\n",
       "      <td>-1.753770</td>\n",
       "      <td>-0.303282</td>\n",
       "      <td>0.836549</td>\n",
       "      <td>2.544893</td>\n",
       "      <td>-2.430078</td>\n",
       "      <td>1.735083</td>\n",
       "      <td>-0.380034</td>\n",
       "      <td>-1.728032</td>\n",
       "      <td>-1.318443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"Ai!</th>\n",
       "      <td>2.715514</td>\n",
       "      <td>-2.795877</td>\n",
       "      <td>-2.191293</td>\n",
       "      <td>0.957296</td>\n",
       "      <td>2.489571</td>\n",
       "      <td>4.871963</td>\n",
       "      <td>0.480067</td>\n",
       "      <td>4.701493</td>\n",
       "      <td>-0.834875</td>\n",
       "      <td>2.381193</td>\n",
       "      <td>-1.849827</td>\n",
       "      <td>1.158584</td>\n",
       "      <td>4.815081</td>\n",
       "      <td>0.151489</td>\n",
       "      <td>0.532469</td>\n",
       "      <td>4.825199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>\"American</th>\n",
       "      <td>1.888345</td>\n",
       "      <td>2.215180</td>\n",
       "      <td>0.214737</td>\n",
       "      <td>2.237525</td>\n",
       "      <td>3.116467</td>\n",
       "      <td>0.615493</td>\n",
       "      <td>2.369236</td>\n",
       "      <td>-1.580535</td>\n",
       "      <td>-1.097147</td>\n",
       "      <td>1.000498</td>\n",
       "      <td>2.292800</td>\n",
       "      <td>3.216928</td>\n",
       "      <td>0.400558</td>\n",
       "      <td>1.796580</td>\n",
       "      <td>-5.076876</td>\n",
       "      <td>2.271558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zesty</th>\n",
       "      <td>-1.665819</td>\n",
       "      <td>-0.860462</td>\n",
       "      <td>-1.023836</td>\n",
       "      <td>-1.570753</td>\n",
       "      <td>0.759116</td>\n",
       "      <td>-0.494996</td>\n",
       "      <td>1.400812</td>\n",
       "      <td>0.725497</td>\n",
       "      <td>0.829708</td>\n",
       "      <td>-1.468480</td>\n",
       "      <td>0.229226</td>\n",
       "      <td>1.561531</td>\n",
       "      <td>-0.804258</td>\n",
       "      <td>-0.340858</td>\n",
       "      <td>1.710068</td>\n",
       "      <td>0.819719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zombiefied</th>\n",
       "      <td>-2.177390</td>\n",
       "      <td>-1.137583</td>\n",
       "      <td>-3.040695</td>\n",
       "      <td>-0.101057</td>\n",
       "      <td>1.404633</td>\n",
       "      <td>3.883577</td>\n",
       "      <td>-1.636874</td>\n",
       "      <td>0.432281</td>\n",
       "      <td>-0.799712</td>\n",
       "      <td>1.516297</td>\n",
       "      <td>1.715312</td>\n",
       "      <td>2.050118</td>\n",
       "      <td>3.725192</td>\n",
       "      <td>-3.073760</td>\n",
       "      <td>2.624060</td>\n",
       "      <td>3.469141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zombies</th>\n",
       "      <td>3.182648</td>\n",
       "      <td>-2.354931</td>\n",
       "      <td>3.487257</td>\n",
       "      <td>-0.163048</td>\n",
       "      <td>2.256392</td>\n",
       "      <td>2.726834</td>\n",
       "      <td>2.055700</td>\n",
       "      <td>2.510818</td>\n",
       "      <td>-0.161862</td>\n",
       "      <td>2.594480</td>\n",
       "      <td>0.615106</td>\n",
       "      <td>-1.619133</td>\n",
       "      <td>-2.938797</td>\n",
       "      <td>0.928330</td>\n",
       "      <td>-4.164789</td>\n",
       "      <td>-1.487806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>~</th>\n",
       "      <td>-1.982251</td>\n",
       "      <td>0.917610</td>\n",
       "      <td>0.091327</td>\n",
       "      <td>-0.909476</td>\n",
       "      <td>2.679627</td>\n",
       "      <td>2.147624</td>\n",
       "      <td>3.226024</td>\n",
       "      <td>-0.264156</td>\n",
       "      <td>-1.890371</td>\n",
       "      <td>-2.483286</td>\n",
       "      <td>0.406137</td>\n",
       "      <td>-4.496862</td>\n",
       "      <td>2.493783</td>\n",
       "      <td>2.942935</td>\n",
       "      <td>1.256528</td>\n",
       "      <td>-2.970675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>2.095947</td>\n",
       "      <td>-3.538921</td>\n",
       "      <td>-1.772990</td>\n",
       "      <td>2.751573</td>\n",
       "      <td>1.635786</td>\n",
       "      <td>-1.302954</td>\n",
       "      <td>-1.159218</td>\n",
       "      <td>1.414440</td>\n",
       "      <td>1.368238</td>\n",
       "      <td>-0.817069</td>\n",
       "      <td>1.284453</td>\n",
       "      <td>-2.660940</td>\n",
       "      <td>-0.461940</td>\n",
       "      <td>-1.949020</td>\n",
       "      <td>-2.890440</td>\n",
       "      <td>3.045557</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5126 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1         2         3         4         5   \\\n",
       "!           0.614773  2.184768 -1.875291 -2.392619 -1.983358 -0.466332   \n",
       "!)         -1.242826  0.632045 -0.905323 -1.453357 -2.560370 -2.671518   \n",
       "\"           0.348271  1.808484 -2.669932 -0.228845  1.090854 -1.378595   \n",
       "\"Ai!        2.715514 -2.795877 -2.191293  0.957296  2.489571  4.871963   \n",
       "\"American   1.888345  2.215180  0.214737  2.237525  3.116467  0.615493   \n",
       "...              ...       ...       ...       ...       ...       ...   \n",
       "zesty      -1.665819 -0.860462 -1.023836 -1.570753  0.759116 -0.494996   \n",
       "zombiefied -2.177390 -1.137583 -3.040695 -0.101057  1.404633  3.883577   \n",
       "zombies     3.182648 -2.354931  3.487257 -0.163048  2.256392  2.726834   \n",
       "~          -1.982251  0.917610  0.091327 -0.909476  2.679627  2.147624   \n",
       "           2.095947 -3.538921 -1.772990  2.751573  1.635786 -1.302954   \n",
       "\n",
       "                  6         7         8         9         10        11  \\\n",
       "!          -1.477482  0.724128  1.692778  1.414513  1.626997 -0.187993   \n",
       "!)         -0.353624  1.379483 -0.828765  0.013821  1.080674  0.351833   \n",
       "\"          -1.647741 -1.753770 -0.303282  0.836549  2.544893 -2.430078   \n",
       "\"Ai!        0.480067  4.701493 -0.834875  2.381193 -1.849827  1.158584   \n",
       "\"American   2.369236 -1.580535 -1.097147  1.000498  2.292800  3.216928   \n",
       "...              ...       ...       ...       ...       ...       ...   \n",
       "zesty       1.400812  0.725497  0.829708 -1.468480  0.229226  1.561531   \n",
       "zombiefied -1.636874  0.432281 -0.799712  1.516297  1.715312  2.050118   \n",
       "zombies     2.055700  2.510818 -0.161862  2.594480  0.615106 -1.619133   \n",
       "~           3.226024 -0.264156 -1.890371 -2.483286  0.406137 -4.496862   \n",
       "          -1.159218  1.414440  1.368238 -0.817069  1.284453 -2.660940   \n",
       "\n",
       "                  12        13        14        15  \n",
       "!           1.396734  0.893966  0.031648  0.464041  \n",
       "!)          0.339504  0.983073  0.977823 -0.570352  \n",
       "\"           1.735083 -0.380034 -1.728032 -1.318443  \n",
       "\"Ai!        4.815081  0.151489  0.532469  4.825199  \n",
       "\"American   0.400558  1.796580 -5.076876  2.271558  \n",
       "...              ...       ...       ...       ...  \n",
       "zesty      -0.804258 -0.340858  1.710068  0.819719  \n",
       "zombiefied  3.725192 -3.073760  2.624060  3.469141  \n",
       "zombies    -2.938797  0.928330 -4.164789 -1.487806  \n",
       "~           2.493783  2.942935  1.256528 -2.970675  \n",
       "          -0.461940 -1.949020 -2.890440  3.045557  \n",
       "\n",
       "[5126 rows x 16 columns]"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame({'index':df.index.values}).to_csv(f'{logdir}/metadata.tsv', index=False, sep='\\t', header=False)\n",
    "df.to_csv(f'{logdir}/vectors.tsv', index=False, sep='\\t', header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DLVenv",
   "language": "python",
   "name": "dlvenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}