Next_Word_Prediction_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "colab": {
      "name": "Next Word Prediction-1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/scaperex/My_Projects/blob/master/Next_Word_Prediction_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LNUFrq7lqHSg"
      },
      "source": [
        "### Importing The Required Libraries:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f151CllrqHSj"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.layers import Embedding, LSTM, Dense\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "import pickle\n",
        "import numpy as np\n",
        "import os\n",
        "import string\n",
        "from tensorflow import keras\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, TensorBoard\n",
        "from gensim.parsing.preprocessing import remove_stopwords\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "94f4WitEtTbB"
      },
      "source": [
        "# Mount Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mSstXrpJqQ0r",
        "outputId": "185c6f71-fdae-41a9-f0fc-d6698f44d052",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZByTV-Al49I",
        "outputId": "d7ccac8d-d8c0-4260-ff62-b0c5d97e0419",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "with open('train_orig.txt', 'r') as f:\n",
        "    sentences = []\n",
        "    for line in f:\n",
        "        outline = line\n",
        "        line = line.strip()\n",
        "        try:\n",
        "            label, sentence = line.split('\\t')[0:2] # remove label and \\n\n",
        "            if label in ['1', '7']: \n",
        "                if len(sentence.split()) <=30:\n",
        "                    sentences.append(outline)\n",
        "        except:\n",
        "            print(line)\n",
        "        \n",
        "    print('num sentences:',len(sentences))\n",
        "with open('train_clean.txt', 'a') as f:\n",
        "    f.writelines(sentences)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num sentences: 19420\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1yEexmEcDkn"
      },
      "source": [
        "def preprocess_sentence(base_path = '/content/drive/My Drive/NewB-master', file_name='example.txt',news_id='0'):\n",
        "    \"\"\"\n",
        "    news_id  1 - New York Times {Liberal}, 7 - New York Post {Conservative}\n",
        "    \"\"\"\n",
        "\n",
        "    with open(os.path.join(base_path,file_name), 'r') as f:\n",
        "        sentences = []\n",
        "        for line in f:\n",
        "            line = line.strip()\n",
        "            label, sentence = line.split('\\t')[0:2] # remove label and \\n\n",
        "            if label == news_id: \n",
        "                if len(sentence.split()) <=30:\n",
        "                    sentences.append(remove_stopwords(sentence))\n",
        "        print('num sentences:',len(sentences))\n",
        "\n",
        "    # Tokenization\n",
        "    tokenizer = Tokenizer()\n",
        "    tokenizer.fit_on_texts(sentences)\n",
        "\n",
        "    # saving the tokenizer for predict function.\n",
        "    pickle.dump(tokenizer, open(f'tokenizer_{file_name.split(\".\")[0]}.pkl', 'wb'))\n",
        "\n",
        "    sequence_data = tokenizer.texts_to_sequences(sentences)\n",
        "\n",
        "    vocab_size = len(tokenizer.word_index) + 1\n",
        "    print('vocab_size: ',vocab_size)\n",
        "\n",
        "    # compute targets\n",
        "    targets = [sentence[1:]+[0] for sentence in sequence_data]\n",
        "\n",
        "    # Add padding\n",
        "    padded_targets = pad_sequences(targets, padding=\"post\")\n",
        "    padded_inputs = pad_sequences(sequence_data, padding=\"post\")\n",
        "\n",
        "    return  padded_inputs, padded_targets, vocab_size"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rkrsxL5FqHUr"
      },
      "source": [
        "### Creating the Model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mRsWCkwbqHUt"
      },
      "source": [
        "class nwp_model():\n",
        "    def __init__(self,vocab_size,model=None, embedding_dim=20):\n",
        "        if model:\n",
        "            self.model = model\n",
        "        else:\n",
        "            model = Sequential()\n",
        "            model.add(Embedding(vocab_size, output_dim=embedding_dim, mask_zero=True, input_length=26))\n",
        "            model.add(LSTM(80, return_sequences=True))\n",
        "            model.add(Dense(80, activation=\"relu\"))\n",
        "            model.add(Dense(vocab_size, activation=\"softmax\"))\n",
        "            self.model=model\n",
        "        print(model.summary())\n",
        "\n",
        "\n",
        "    def train(self, feature, target, num_epochs=50):\n",
        "        checkpoint = ModelCheckpoint(\"nextword1.h5\", monitor='loss', verbose=1, save_best_only=True, mode='auto')\n",
        "\n",
        "        # reduce_LR = ReduceLROnPlateau(monitor='loss', factor=0.2, patience=3, min_lr=0.0001, verbose = 1)\n",
        "\n",
        "        # logdir='logsnextword1'\n",
        "        # tensorboard_Visualization = TensorBoard(log_dir=logdir)\n",
        "        self.model.compile(optimizer=Adam(lr=0.001), loss='SparseCategoricalCrossentropy', metrics=['acc'])\n",
        "        self.model.fit(feature, target, epochs=num_epochs, batch_size=24, validation_split=0.1, callbacks=[checkpoint])#, reduce_LR, tensorboard_Visualization])\n",
        "\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCwDvVrSM-AV",
        "outputId": "8cef0c70-8473-4074-90a3-1be8cf37ac29",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "X1,y1, vocab_size1 = preprocess_sentence(base_path='', file_name='train_clean.txt',news_id='1')"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num sentences: 19420\n",
            "vocab_size:  18772\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4CvkaL6M_JC",
        "outputId": "1d109f08-00cb-44a3-cfaa-510e944f736b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "classifier = nwp_model(vocab_size=vocab_size1,embedding_dim=100)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 26, 100)           1877200   \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 26, 80)            57920     \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 26, 80)            6480      \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 26, 18772)         1520532   \n",
            "=================================================================\n",
            "Total params: 3,462,132\n",
            "Trainable params: 3,462,132\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PRyX-bJ0ZA61",
        "outputId": "b851ad95-8751-42c3-a4f2-6590d9735cdc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        }
      },
      "source": [
        "classifier.train(X1,y1, num_epochs=8)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 2.2978 - acc: 0.1433\n",
            "Epoch 00001: loss improved from inf to 2.29771, saving model to nextword1.h5\n",
            "729/729 [==============================] - 43s 58ms/step - loss: 2.2977 - acc: 0.1433 - val_loss: 2.1893 - val_acc: 0.1783\n",
            "Epoch 2/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 2.1332 - acc: 0.1758\n",
            "Epoch 00002: loss improved from 2.29771 to 2.13322, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 2.1332 - acc: 0.1758 - val_loss: 2.1826 - val_acc: 0.1787\n",
            "Epoch 3/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 2.0676 - acc: 0.1778\n",
            "Epoch 00003: loss improved from 2.13322 to 2.06775, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 2.0678 - acc: 0.1778 - val_loss: 2.1856 - val_acc: 0.1819\n",
            "Epoch 4/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 2.0044 - acc: 0.1814\n",
            "Epoch 00004: loss improved from 2.06775 to 2.00441, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 58ms/step - loss: 2.0044 - acc: 0.1814 - val_loss: 2.1930 - val_acc: 0.1826\n",
            "Epoch 5/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 1.9441 - acc: 0.1857\n",
            "Epoch 00005: loss improved from 2.00441 to 1.94415, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 1.9441 - acc: 0.1857 - val_loss: 2.2105 - val_acc: 0.1852\n",
            "Epoch 6/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 1.8869 - acc: 0.1910\n",
            "Epoch 00006: loss improved from 1.94415 to 1.88665, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 1.8867 - acc: 0.1910 - val_loss: 2.2363 - val_acc: 0.1854\n",
            "Epoch 7/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 1.8285 - acc: 0.1961\n",
            "Epoch 00007: loss improved from 1.88665 to 1.82826, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 1.8283 - acc: 0.1961 - val_loss: 2.2520 - val_acc: 0.1856\n",
            "Epoch 8/8\n",
            "728/729 [============================>.] - ETA: 0s - loss: 1.7663 - acc: 0.2021\n",
            "Epoch 00008: loss improved from 1.82826 to 1.76602, saving model to nextword1.h5\n",
            "729/729 [==============================] - 42s 57ms/step - loss: 1.7660 - acc: 0.2021 - val_loss: 2.2821 - val_acc: 0.1862\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pa1x-21vqHVB"
      },
      "source": [
        "### Plot The Model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ouU2ao7DqHVD"
      },
      "source": [
        "keras.utils.plot_model(model, to_file='model.png', show_layer_names=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWyx7TDRwFea"
      },
      "source": [
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir=\"./logsnextword1\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEEjeyhdu595"
      },
      "source": [
        "# Importing the Libraries\n",
        "\n",
        "from tensorflow.keras.models import load_model\n",
        "import numpy as np\n",
        "import pickle\n",
        "\n",
        "# Load the model and tokenizer\n",
        "\n",
        "\n",
        "def Predict_Next_Words(model, tokenizer, text, num_words_to_complete):\n",
        "    \"\"\"\n",
        "        In this function we are using the tokenizer and models trained\n",
        "        and we are creating the sequence of the text entered and then\n",
        "        using our model to predict and return the the predicted word.\n",
        "    \n",
        "    \"\"\"\n",
        "    res = \"\"\n",
        "    text_len = len(text)\n",
        "    for _ in range(num_words_to_complete):\n",
        "        sequence = tokenizer.texts_to_sequences([text])\n",
        "        padded_sequence = pad_sequences(sequence, padding=\"post\", maxlen=26)\n",
        "        output = model.predict(padded_sequence)\n",
        "        preds = np.argmax(output, axis=-1)\n",
        "        # print(preds)\n",
        "        predicted_word = tokenizer.sequences_to_texts(preds)[0].split()[text_len-1]\n",
        "        # print(predicted_word)\n",
        "        res += predicted_word + ' '\n",
        "        text += [predicted_word]\n",
        "        text_len+=1\n",
        "        # print(text)\n",
        "\n",
        "    print(res)"
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RzRssTdpu711",
        "outputId": "12bd7ed2-550f-4478-d2e1-62aa3fe46286",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "\"\"\"\n",
        "    We are testing our model and we will run the model\n",
        "    until the user decides to stop the script.\n",
        "    While the script is running we try and check if \n",
        "    the prediction can be made on the text. If no\n",
        "    prediction can be made we just continue.\n",
        "\n",
        "\"\"\"\n",
        "model = load_model('nextword111.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_clean11.pkl', 'rb'))\n",
        "\n",
        "num_words_to_complete = int(input(\"Enter number of words to complete:\"))\n",
        "\n",
        "while(True):\n",
        "\n",
        "    text = input(\"Enter beginning of sentence: \") \n",
        "    if text == \"x\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        text = text.split(\" \")\n",
        "        Predict_Next_Words(model, tokenizer, text, num_words_to_complete)\n",
        "        "
      ],
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter number of words to complete:5\n",
            "Enter beginning of sentence: trump\n",
            "trump trump trump trump trump \n",
            "Enter beginning of sentence: x\n",
            "Ending The Program.....\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WB0OFSoCwj3M",
        "outputId": "4ef96d2a-a2ef-4fec-8ca3-84a115e40e01",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "\"\"\"\n",
        "    We are testing our model and we will run the model\n",
        "    until the user decides to stop the script.\n",
        "    While the script is running we try and check if \n",
        "    the prediction can be made on the text. If no\n",
        "    prediction can be made we just continue.\n",
        "\n",
        "\"\"\"\n",
        "model = load_model('nextword177.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_clean7.pkl', 'rb'))\n",
        "\n",
        "num_words_to_complete = int(input(\"Enter number of words to complete:\"))\n",
        "\n",
        "while(True):\n",
        "\n",
        "    text = input(\"Enter beginning of sentence: \") \n",
        "    if text == \"x\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        text = text.split(\" \")\n",
        "        Predict_Next_Words(model, tokenizer, text, num_words_to_complete)\n",
        "        "
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter number of words to complete:5\n",
            "Enter beginning of sentence: hillary clinton\n",
            "trump probes disturbing new standard \n",
            "Enter beginning of sentence: trump\n",
            "said hes monitoring republican presidential \n",
            "Enter beginning of sentence: president trump\n",
            "rose garden nations plan military \n",
            "Enter beginning of sentence: clinton\n",
            "slim crowned clinton criticized donald \n",
            "Enter beginning of sentence: x\n",
            "Ending The Program.....\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJ9ou7BjPf4o",
        "outputId": "49044293-9291-43bf-fb5a-82bcf50ceff7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "X7,y7, vocab_size7 = preprocess_sentence(base_path='', file_name='train_clean.txt',news_id='7')"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "num sentences: 17502\n",
            "vocab_size:  17009\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PdG-ABJ-PkHb",
        "outputId": "7d837f6a-00f7-43dd-e3b3-35adb25ae464",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "classifier2 = nwp_model(embedding_dim=40,vocab_size=vocab_size7)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 26, 40)            680360    \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 26, 80)            38720     \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 26, 80)            6480      \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 26, 17009)         1377729   \n",
            "=================================================================\n",
            "Total params: 2,103,289\n",
            "Trainable params: 2,103,289\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1843k_5fPj-k",
        "outputId": "12951431-c61d-4b7a-c126-b79cf686c6a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "classifier2.train(X7,y7, num_epochs=20)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 2.9321 - acc: 0.1345\n",
            "Epoch 00001: loss improved from inf to 2.93213, saving model to nextword1.h5\n",
            "657/657 [==============================] - 33s 50ms/step - loss: 2.9321 - acc: 0.1345 - val_loss: 2.8289 - val_acc: 0.1747\n",
            "Epoch 2/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.7295 - acc: 0.1778\n",
            "Epoch 00002: loss improved from 2.93213 to 2.72927, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 2.7293 - acc: 0.1778 - val_loss: 2.8016 - val_acc: 0.1787\n",
            "Epoch 3/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 2.6529 - acc: 0.1818\n",
            "Epoch 00003: loss improved from 2.72927 to 2.65286, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 2.6529 - acc: 0.1818 - val_loss: 2.7807 - val_acc: 0.1839\n",
            "Epoch 4/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 2.5773 - acc: 0.1864\n",
            "Epoch 00004: loss improved from 2.65286 to 2.57733, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 2.5773 - acc: 0.1864 - val_loss: 2.7701 - val_acc: 0.1860\n",
            "Epoch 5/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.5017 - acc: 0.1927\n",
            "Epoch 00005: loss improved from 2.57733 to 2.50147, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 2.5015 - acc: 0.1928 - val_loss: 2.7636 - val_acc: 0.1916\n",
            "Epoch 6/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.4218 - acc: 0.1990\n",
            "Epoch 00006: loss improved from 2.50147 to 2.42158, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 47ms/step - loss: 2.4216 - acc: 0.1989 - val_loss: 2.7542 - val_acc: 0.1945\n",
            "Epoch 7/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.3414 - acc: 0.2050\n",
            "Epoch 00007: loss improved from 2.42158 to 2.34140, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 48ms/step - loss: 2.3414 - acc: 0.2051 - val_loss: 2.7525 - val_acc: 0.1974\n",
            "Epoch 8/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.2618 - acc: 0.2101\n",
            "Epoch 00008: loss improved from 2.34140 to 2.26186, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 48ms/step - loss: 2.2619 - acc: 0.2101 - val_loss: 2.7644 - val_acc: 0.1976\n",
            "Epoch 9/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.1788 - acc: 0.2162\n",
            "Epoch 00009: loss improved from 2.26186 to 2.17895, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 48ms/step - loss: 2.1789 - acc: 0.2162 - val_loss: 2.7700 - val_acc: 0.2015\n",
            "Epoch 10/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.0929 - acc: 0.2243\n",
            "Epoch 00010: loss improved from 2.17895 to 2.09274, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 48ms/step - loss: 2.0927 - acc: 0.2243 - val_loss: 2.7824 - val_acc: 0.2018\n",
            "Epoch 11/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 2.0053 - acc: 0.2350\n",
            "Epoch 00011: loss improved from 2.09274 to 2.00563, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 2.0056 - acc: 0.2350 - val_loss: 2.8016 - val_acc: 0.2035\n",
            "Epoch 12/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.9159 - acc: 0.2498\n",
            "Epoch 00012: loss improved from 2.00563 to 1.91587, saving model to nextword1.h5\n",
            "657/657 [==============================] - 31s 48ms/step - loss: 1.9159 - acc: 0.2498 - val_loss: 2.8214 - val_acc: 0.2046\n",
            "Epoch 13/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 1.8281 - acc: 0.2668\n",
            "Epoch 00013: loss improved from 1.91587 to 1.82815, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 1.8281 - acc: 0.2668 - val_loss: 2.8295 - val_acc: 0.2048\n",
            "Epoch 14/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 1.7453 - acc: 0.2849\n",
            "Epoch 00014: loss improved from 1.82815 to 1.74557, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 1.7456 - acc: 0.2849 - val_loss: 2.8490 - val_acc: 0.2086\n",
            "Epoch 15/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.6678 - acc: 0.3050\n",
            "Epoch 00015: loss improved from 1.74557 to 1.66779, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 1.6678 - acc: 0.3050 - val_loss: 2.8469 - val_acc: 0.2142\n",
            "Epoch 16/20\n",
            "656/657 [============================>.] - ETA: 0s - loss: 1.5964 - acc: 0.3266\n",
            "Epoch 00016: loss improved from 1.66779 to 1.59654, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 49ms/step - loss: 1.5965 - acc: 0.3266 - val_loss: 2.8604 - val_acc: 0.2146\n",
            "Epoch 17/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.5307 - acc: 0.3488\n",
            "Epoch 00017: loss improved from 1.59654 to 1.53069, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 1.5307 - acc: 0.3488 - val_loss: 2.8701 - val_acc: 0.2211\n",
            "Epoch 18/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.4720 - acc: 0.3675\n",
            "Epoch 00018: loss improved from 1.53069 to 1.47199, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 48ms/step - loss: 1.4720 - acc: 0.3675 - val_loss: 2.8807 - val_acc: 0.2244\n",
            "Epoch 19/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.4177 - acc: 0.3849\n",
            "Epoch 00019: loss improved from 1.47199 to 1.41767, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 49ms/step - loss: 1.4177 - acc: 0.3849 - val_loss: 2.8851 - val_acc: 0.2262\n",
            "Epoch 20/20\n",
            "657/657 [==============================] - ETA: 0s - loss: 1.3678 - acc: 0.4012\n",
            "Epoch 00020: loss improved from 1.41767 to 1.36781, saving model to nextword1.h5\n",
            "657/657 [==============================] - 32s 49ms/step - loss: 1.3678 - acc: 0.4012 - val_loss: 2.8867 - val_acc: 0.2313\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pb_onuB3xstK"
      },
      "source": [
        "model = load_model('nextword111.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_clean11.pkl', 'rb'))\n",
        "\n",
        "def Predict_Next_Word_options(model, tokenizer, text, num_words_to_complete):\n",
        "    \"\"\"\n",
        "        In this function we are using the tokenizer and models trained\n",
        "        and we are creating the sequence of the text entered and then\n",
        "        using our model to predict and return the the predicted word.\n",
        "    \n",
        "    \"\"\"\n",
        "    \n",
        "    sequence = tokenizer.texts_to_sequences([text])\n",
        "    padded_sequence = pad_sequences(sequence, padding=\"post\", maxlen=26)\n",
        "    output = model.predict(padded_sequence)[0,len(text)-1,:]\n",
        "    outs = np.argpartition(output, -num_words_to_complete,axis=-1)[-num_words_to_complete:]\n",
        "    predicted_word = tokenizer.sequences_to_texts([outs])[0].split()\n",
        "\n",
        "    print(predicted_word)"
      ],
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M7-b0BFZyFk5",
        "outputId": "7239a0ed-8e9d-4b16-d98b-bb3a3a1236d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "model = load_model('nextword111.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_clean11.pkl', 'rb'))\n",
        "num_words_to_complete = int(input(\"Enter number of words to complete:\"))\n",
        "\n",
        "while(True):\n",
        "\n",
        "    text = input(\"Enter beginning of sentence: \") \n",
        "    if text == \"x\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        text = text.split(\" \")\n",
        "        Predict_Next_Word_options(model, tokenizer, text, num_words_to_complete)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(18772,)\n",
            "(20,)\n",
            "['point', 'change', 'partner', 'past', 'issue', 'similar', 'seen', 'court', 'looking', 'leaders', 'trump', 'said', 'president', 'campaign', 'says', 'served', 'officials', 'record', 'got']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n_M23fn1BTBv"
      },
      "source": [
        "\n",
        "\n",
        "['offered', 'wrote', 'administration', 'appeared', 'won', 'organization', 'asked', 'mrs', 'says', 'republican', 'president', 'campaign', 'said', 'called', 'repeatedly', 'took', 'mr', 'told', 'realdonaldtrump']\n",
        "\n",
        "Enter beginning of sentence: clinton\n",
        "\n",
        "['administration', 'held', 'tried', 'think', 'asserted', 'says', 'melania', 'new', 'asked', 'ivanka', 'worked', 'campaign', 'president', 'said', 'like', 'donald', 'told', 'trump', 'later']\n",
        "\n",
        "Enter beginning of sentence: president trump\n",
        "\n",
        "['rally', 'going', 'won', 'presidency', 'win', 'took', 'campaign', 'supporters', 'said', 'told', 'organization', 'called', 'jr', 'says', 'administration', 'tower', 'wants', 'supporter', 'university']\n",
        "\n",
        "Enter beginning of sentence: president trump accused\n",
        "\n",
        "['presidential', 'immigration', 'comment', 'years', 'night', 'ms', 'year', 'percent', 'reporters', 'white', 'meeting', 'twitter', 'president', 'said', 'wednesday', 'republican', 'new', 'mr', 'times']\n",
        "\n",
        "Enter beginning of sentence: x\n",
        "Ending The Program....."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "acVHyEV8z-I2",
        "outputId": "ae6d307c-fae1-4a35-8222-abc323897298",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        }
      },
      "source": [
        "model = load_model('nextword177.h5')\n",
        "tokenizer = pickle.load(open('tokenizer_train_clean7.pkl', 'rb'))\n",
        "num_words_to_complete = int(input(\"Enter number of words to complete:\"))\n",
        "\n",
        "while(True):\n",
        "\n",
        "    text = input(\"Enter beginning of sentence: \") \n",
        "    if text == \"x\":\n",
        "        print(\"Ending The Program.....\")\n",
        "        break\n",
        "    \n",
        "    else:\n",
        "        text = text.split(\" \")\n",
        "        Predict_Next_Word_options(model, tokenizer, text, num_words_to_complete)"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter number of words to complete:20\n",
            "Enter beginning of sentence: trump\n",
            "WARNING:tensorflow:5 out of the last 12 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f71ae29dae8> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
            "(17009,)\n",
            "(20,)\n",
            "['won', 'supporters', 'wins', 'didnt', 'says', 'called', 'doesnt', 'jr', 'administration', 'tweeted', 'organization', 'campaign', 'told', 'added', 'went', 'later', 'took', 'said', 'wants']\n",
            "Enter beginning of sentence: clinton\n",
            "(17009,)\n",
            "(20,)\n",
            "['clobbering', 'easier', 'opened', '41', 'coolly', '43', 'gets', 'cleverly', 'pols', 'clinton', 'trump', 'cruz', 'underperforming', 'accused', 'regaining', 'said', 'undeserving', 'called', 'ex', 'slim']\n",
            "Enter beginning of sentence: president trump\n",
            "(17009,)\n",
            "(20,)\n",
            "['weighed', 'thundered', 'taking', 'sought', 'kept', 'learned', 'personally', 'oklahoma', 'dumping', 'instituted', 'careful', 'strengthened', 'trump', 'reportedly', 'says', 'said', 'rose', 'signed', 'ramp']\n",
            "Enter beginning of sentence: president trump accused\n",
            "(17009,)\n",
            "(20,)\n",
            "['people', 'said', 'attacking', 'intelligence', 'president', 'russian', 'politicians', 'ex', 'twitter', 'yellen', 'women', 'secret', 'irans', 'didnt', 'trump', 'won', 'white', 'hes', 'cruz', 'obama']\n",
            "Enter beginning of sentence: cruz\n",
            "(17009,)\n",
            "(20,)\n",
            "['president', 'media', 'spared', 'clinton', 'ex', 'later', 'demanded', 'poll', 'trump', 'said', 'let', 'appearing', 'plotted', 'came', 'whipped', 'donald', 'called', 'foe', 'gets', 'raised']\n",
            "Enter beginning of sentence: x\n",
            "Ending The Program.....\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}