Spelling Corrector/spell-check-using-encoder-decoder.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["# Model Creation"]},{"cell_type":"code","execution_count":29,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:52:59.488814Z","iopub.status.busy":"2022-04-10T01:52:59.488542Z","iopub.status.idle":"2022-04-10T01:52:59.836674Z","shell.execute_reply":"2022-04-10T01:52:59.835952Z","shell.execute_reply.started":"2022-04-10T01:52:59.488783Z"},"trusted":true},"outputs":[],"source":["from keras.models import Model,load_model\n","from keras.layers import Input, LSTM, Dense\n","from keras import metrics, backend as K\n","from tensorflow.keras.optimizers import Adam"]},{"cell_type":"code","execution_count":2,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:34:30.988407Z","iopub.status.busy":"2022-04-10T01:34:30.988141Z","iopub.status.idle":"2022-04-10T01:34:30.992811Z","shell.execute_reply":"2022-04-10T01:34:30.991672Z","shell.execute_reply.started":"2022-04-10T01:34:30.988376Z"},"trusted":true},"outputs":[],"source":["# For use with truncated metrics,\n","# take maxlen from the validation set.\n","# Hacky and hard-coded for now.\n","VAL_MAXLEN = 16"]},{"cell_type":"code","execution_count":3,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:34:34.059613Z","iopub.status.busy":"2022-04-10T01:34:34.059303Z","iopub.status.idle":"2022-04-10T01:34:34.064444Z","shell.execute_reply":"2022-04-10T01:34:34.063758Z","shell.execute_reply.started":"2022-04-10T01:34:34.059581Z"},"trusted":true},"outputs":[],"source":["def truncated_acc(y_true, y_pred):\n","    y_true = y_true[:, :VAL_MAXLEN, :]\n","    y_pred = y_pred[:, :VAL_MAXLEN, :]\n","    \n","    acc = metrics.categorical_accuracy(y_true, y_pred)\n","    return K.mean(acc, axis=-1)"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:34:49.410213Z","iopub.status.busy":"2022-04-10T01:34:49.409454Z","iopub.status.idle":"2022-04-10T01:34:49.415312Z","shell.execute_reply":"2022-04-10T01:34:49.414633Z","shell.execute_reply.started":"2022-04-10T01:34:49.410175Z"},"trusted":true},"outputs":[],"source":["def truncated_loss(y_true, y_pred):\n","    y_true = y_true[:, :VAL_MAXLEN, :]\n","    y_pred = y_pred[:, :VAL_MAXLEN, :]\n","    \n","    loss = K.categorical_crossentropy(\n","        target=y_true, output=y_pred, from_logits=False)\n","    return K.mean(loss, axis=-1)\n"]},{"cell_type":"code","execution_count":30,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:53:21.053415Z","iopub.status.busy":"2022-04-10T01:53:21.053157Z","iopub.status.idle":"2022-04-10T01:53:21.076625Z","shell.execute_reply":"2022-04-10T01:53:21.075688Z","shell.execute_reply.started":"2022-04-10T01:53:21.053386Z"},"trusted":true},"outputs":[],"source":["def seq2seq(hidden_size, nb_input_chars, nb_target_chars):\n","\n","    \n","    # Define the main model consisting of encoder and decoder.\n","    encoder_inputs = Input(shape=(None, nb_input_chars),\n","                           name='encoder_data')\n","    encoder_lstm = LSTM(hidden_size, recurrent_dropout=0.2,\n","                        return_sequences=True, return_state=False,\n","                        name='encoder_lstm_1')\n","    encoder_outputs = encoder_lstm(encoder_inputs)\n","    \n","    encoder_lstm = LSTM(hidden_size, recurrent_dropout=0.2,\n","                        return_sequences=False, return_state=True,\n","                        name='encoder_lstm_2')\n","    encoder_outputs, state_h, state_c = encoder_lstm(encoder_outputs)\n","    # We discard `encoder_outputs` and only keep the states.\n","    encoder_states = [state_h, state_c]\n","\n","    # Set up the decoder, using `encoder_states` as initial state.\n","    decoder_inputs = Input(shape=(None, nb_target_chars),\n","                           name='decoder_data')\n","    # We set up our decoder to return full output sequences,\n","    # and to return internal states as well. We don't use the return\n","    # states in the training model, but we will use them in inference.\n","    decoder_lstm = LSTM(hidden_size, dropout=0.2, return_sequences=True,\n","                        return_state=True, name='decoder_lstm')\n","    decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n","                                         initial_state=encoder_states)\n","    decoder_softmax = Dense(nb_target_chars, activation='softmax',\n","                            name='decoder_softmax')\n","    decoder_outputs = decoder_softmax(decoder_outputs)\n","\n","    # The main model will turn `encoder_input_data` & `decoder_input_data`\n","    # into `decoder_target_data`\n","    model = Model(inputs=[encoder_inputs, decoder_inputs],\n","                  outputs=decoder_outputs)\n","    \n","    adam = Adam(lr=0.001, decay=0.0)\n","    model.compile(optimizer=adam, loss='categorical_crossentropy',\n","                  metrics=['accuracy', truncated_acc, truncated_loss])\n","    \n","    # Define the encoder model separately.\n","    encoder_model = Model(inputs=encoder_inputs, outputs=encoder_states)\n","\n","    # Define the decoder model separately.\n","    decoder_state_input_h = Input(shape=(hidden_size,))\n","    decoder_state_input_c = Input(shape=(hidden_size,))\n","    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n","    decoder_outputs, state_h, state_c = decoder_lstm(\n","        decoder_inputs, initial_state=decoder_states_inputs)\n","    decoder_states = [state_h, state_c]\n","    decoder_outputs = decoder_softmax(decoder_outputs)\n","    decoder_model = Model(inputs=[decoder_inputs] + decoder_states_inputs,\n","                          outputs=[decoder_outputs] + decoder_states)\n","\n","    return model, encoder_model, decoder_model"]},{"cell_type":"markdown","metadata":{},"source":["Some Utils"]},{"cell_type":"code","execution_count":8,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:36:34.349613Z","iopub.status.busy":"2022-04-10T01:36:34.348902Z","iopub.status.idle":"2022-04-10T01:36:34.354021Z","shell.execute_reply":"2022-04-10T01:36:34.353308Z","shell.execute_reply.started":"2022-04-10T01:36:34.349574Z"},"trusted":true},"outputs":[],"source":["import re\n","import os\n","import unidecode\n","import numpy as np"]},{"cell_type":"code","execution_count":9,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:36:51.549863Z","iopub.status.busy":"2022-04-10T01:36:51.549560Z","iopub.status.idle":"2022-04-10T01:36:51.554511Z","shell.execute_reply":"2022-04-10T01:36:51.553844Z","shell.execute_reply.started":"2022-04-10T01:36:51.549830Z"},"trusted":true},"outputs":[],"source":["np.random.seed(1234)"]},{"cell_type":"code","execution_count":10,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:37:05.669517Z","iopub.status.busy":"2022-04-10T01:37:05.669213Z","iopub.status.idle":"2022-04-10T01:37:05.674054Z","shell.execute_reply":"2022-04-10T01:37:05.673315Z","shell.execute_reply.started":"2022-04-10T01:37:05.669483Z"},"trusted":true},"outputs":[],"source":["SOS = '\\t' # start of sequence.\n","EOS = '*' # end of sequence.\n","CHARS = list('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ')\n","REMOVE_CHARS = '[#$%\"\\+@<=>!&,-.?:;()*\\[\\]^_`{|}~/\\d\\t\\n\\r\\x0b\\x0c]'"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:37:49.071630Z","iopub.status.busy":"2022-04-10T01:37:49.071368Z","iopub.status.idle":"2022-04-10T01:37:49.083606Z","shell.execute_reply":"2022-04-10T01:37:49.082941Z","shell.execute_reply.started":"2022-04-10T01:37:49.071602Z"},"trusted":true},"outputs":[],"source":["class CharacterTable(object):\n","    \"\"\"Given a set of characters:\n","    + Encode them to a one-hot integer representation\n","    + Decode the one-hot integer representation to their character output\n","    + Decode a vector of probabilities to their character output\n","    \"\"\"\n","    def __init__(self, chars):\n","        \"\"\"Initialize character table.\n","        # Arguments\n","          chars: Characters that can appear in the input.\n","        \"\"\"\n","        self.chars = sorted(set(chars))\n","        self.char2index = dict((c, i) for i, c in enumerate(self.chars))\n","        self.index2char = dict((i, c) for i, c in enumerate(self.chars))\n","        self.size = len(self.chars)\n","    \n","    def encode(self, C, nb_rows):\n","        \"\"\"One-hot encode given string C.\n","        # Arguments\n","          C: string, to be encoded.\n","          nb_rows: Number of rows in the returned one-hot encoding. This is\n","          used to keep the # of rows for each data the same via padding.\n","        \"\"\"\n","        x = np.zeros((nb_rows, len(self.chars)), dtype=np.float32)\n","        for i, c in enumerate(C):\n","            x[i, self.char2index[c]] = 1.0\n","        return x\n","\n","    def decode(self, x, calc_argmax=True):\n","        \"\"\"Decode the given vector or 2D array to their character output.\n","        # Arguments\n","          x: A vector or 2D array of probabilities or one-hot encodings,\n","          or a vector of character indices (used with `calc_argmax=False`).\n","          calc_argmax: Whether to find the character index with maximum\n","          probability, defaults to `True`.\n","        \"\"\"\n","        if calc_argmax:\n","            indices = x.argmax(axis=-1)\n","        else:\n","            indices = x\n","        chars = ''.join(self.index2char[ind] for ind in indices)\n","        return indices, chars\n","\n","    def sample_multinomial(self, preds, temperature=1.0):\n","        \"\"\"Sample index and character output from `preds`,\n","        an array of softmax probabilities with shape (1, 1, nb_chars).\n","        \"\"\"\n","        # Reshaped to 1D array of shape (nb_chars,).\n","        preds = np.reshape(preds, len(self.chars)).astype(np.float64)\n","        preds = np.log(preds) / temperature\n","        exp_preds = np.exp(preds)\n","        preds = exp_preds / np.sum(exp_preds)\n","        probs = np.random.multinomial(1, preds, 1)\n","        index = np.argmax(probs)\n","        char  = self.index2char[index]\n","        return index, char"]},{"cell_type":"code","execution_count":12,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:38:09.068607Z","iopub.status.busy":"2022-04-10T01:38:09.068355Z","iopub.status.idle":"2022-04-10T01:38:09.074930Z","shell.execute_reply":"2022-04-10T01:38:09.074100Z","shell.execute_reply.started":"2022-04-10T01:38:09.068579Z"},"trusted":true},"outputs":[],"source":["def read_text(data_path, list_of_books):\n","    text = ''\n","    for book in list_of_books:\n","        file_path = os.path.join(data_path, book)\n","        strings = unidecode.unidecode(open(file_path).read())\n","        text += strings + ' '\n","    return text\n"]},{"cell_type":"code","execution_count":13,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:38:31.989950Z","iopub.status.busy":"2022-04-10T01:38:31.989630Z","iopub.status.idle":"2022-04-10T01:38:31.994770Z","shell.execute_reply":"2022-04-10T01:38:31.993729Z","shell.execute_reply.started":"2022-04-10T01:38:31.989914Z"},"trusted":true},"outputs":[],"source":["def tokenize(text):\n","    tokens = [re.sub(REMOVE_CHARS, '', token)\n","              for token in re.split(\"[-\\n ]\", text)]\n","    return tokens"]},{"cell_type":"code","execution_count":14,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:38:50.433697Z","iopub.status.busy":"2022-04-10T01:38:50.432943Z","iopub.status.idle":"2022-04-10T01:38:50.442497Z","shell.execute_reply":"2022-04-10T01:38:50.441659Z","shell.execute_reply.started":"2022-04-10T01:38:50.433653Z"},"trusted":true},"outputs":[],"source":["def add_speling_erors(token, error_rate):\n","    \"\"\"Simulate some artificial spelling mistakes.\"\"\"\n","    assert(0.0 <= error_rate < 1.0)\n","    if len(token) < 3:\n","        return token\n","    rand = np.random.rand()\n","    # Here are 4 different ways spelling mistakes can occur,\n","    # each of which has equal chance.\n","    prob = error_rate / 4.0\n","    if rand < prob:\n","        # Replace a character with a random character.\n","        random_char_index = np.random.randint(len(token))\n","        token = token[:random_char_index] + np.random.choice(CHARS) \\\n","                + token[random_char_index + 1:]\n","    elif prob < rand < prob * 2:\n","        # Delete a character.\n","        random_char_index = np.random.randint(len(token))\n","        token = token[:random_char_index] + token[random_char_index + 1:]\n","    elif prob * 2 < rand < prob * 3:\n","        # Add a random character.\n","        random_char_index = np.random.randint(len(token))\n","        token = token[:random_char_index] + np.random.choice(CHARS) \\\n","                + token[random_char_index:]\n","    elif prob * 3 < rand < prob * 4:\n","        # Transpose 2 characters.\n","        random_char_index = np.random.randint(len(token) - 1)\n","        token = token[:random_char_index]  + token[random_char_index + 1] \\\n","                + token[random_char_index] + token[random_char_index + 2:]\n","    else:\n","        # No spelling errors.\n","        pass\n","    return token"]},{"cell_type":"code","execution_count":17,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:39:52.350792Z","iopub.status.busy":"2022-04-10T01:39:52.350017Z","iopub.status.idle":"2022-04-10T01:39:52.359100Z","shell.execute_reply":"2022-04-10T01:39:52.358166Z","shell.execute_reply.started":"2022-04-10T01:39:52.350746Z"},"trusted":true},"outputs":[],"source":["def transform(tokens, maxlen, error_rate=0.3, shuffle=True):\n","    \"\"\"Transform tokens into model inputs and targets.\n","    All inputs and targets are padded to maxlen with EOS character.\n","    \"\"\"\n","    if shuffle:\n","        print('Shuffling data.')\n","        np.random.shuffle(tokens)\n","    encoder_tokens = []\n","    decoder_tokens = []\n","    target_tokens = []\n","    for token in tokens:\n","        encoder = add_speling_erors(token, error_rate=error_rate)\n","        encoder += EOS * (maxlen - len(encoder)) # Padded to maxlen.\n","        encoder_tokens.append(encoder)\n","    \n","        decoder = SOS + token\n","        decoder += EOS * (maxlen - len(decoder))\n","        decoder_tokens.append(decoder)\n","    \n","        target = decoder[1:]\n","        target += EOS * (maxlen - len(target))\n","        target_tokens.append(target)\n","        \n","        assert(len(encoder) == len(decoder) == len(target))\n","    return encoder_tokens, decoder_tokens, target_tokens"]},{"cell_type":"code","execution_count":15,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:39:43.729381Z","iopub.status.busy":"2022-04-10T01:39:43.729060Z","iopub.status.idle":"2022-04-10T01:39:43.736027Z","shell.execute_reply":"2022-04-10T01:39:43.735180Z","shell.execute_reply.started":"2022-04-10T01:39:43.729349Z"},"trusted":true},"outputs":[],"source":["def batch(tokens, maxlen, ctable, batch_size=128, reverse=False):\n","    \"\"\"Split data into chunks of `batch_size` examples.\"\"\"\n","    def generate(tokens, reverse):\n","        while(True): # This flag yields an infinite generator.\n","            for token in tokens:\n","                if reverse:\n","                    token = token[::-1]\n","                yield token\n","    \n","    token_iterator = generate(tokens, reverse)\n","    data_batch = np.zeros((batch_size, maxlen, ctable.size),\n","                          dtype=np.float32)\n","    while(True):\n","        for i in range(batch_size):\n","            token = next(token_iterator)\n","            data_batch[i] = ctable.encode(token, maxlen)\n","        yield data_batch"]},{"cell_type":"code","execution_count":16,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:39:45.869091Z","iopub.status.busy":"2022-04-10T01:39:45.868490Z","iopub.status.idle":"2022-04-10T01:39:45.874172Z","shell.execute_reply":"2022-04-10T01:39:45.873268Z","shell.execute_reply.started":"2022-04-10T01:39:45.869045Z"},"trusted":true},"outputs":[],"source":["def datagen(encoder_iter, decoder_iter, target_iter):\n","    \"\"\"Utility function to load data into required model format.\"\"\"\n","    inputs = zip(encoder_iter, decoder_iter)\n","    while(True):\n","        encoder_input, decoder_input = next(inputs)\n","        target = next(target_iter)\n","        yield ([encoder_input, decoder_input], target)"]},{"cell_type":"code","execution_count":18,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:40:42.794088Z","iopub.status.busy":"2022-04-10T01:40:42.793730Z","iopub.status.idle":"2022-04-10T01:40:42.808243Z","shell.execute_reply":"2022-04-10T01:40:42.807318Z","shell.execute_reply.started":"2022-04-10T01:40:42.794054Z"},"trusted":true},"outputs":[],"source":["def decode_sequences(inputs, targets, input_ctable, target_ctable,\n","                     maxlen, reverse, encoder_model, decoder_model,\n","                     nb_examples, sample_mode='argmax', random=True):\n","    input_tokens = []\n","    target_tokens = []\n","    \n","    if random:\n","        indices = np.random.randint(0, len(inputs), nb_examples)\n","    else:\n","        indices = range(nb_examples)\n","        \n","    for index in indices:\n","        input_tokens.append(inputs[index])\n","        target_tokens.append(targets[index])\n","    input_sequences = batch(input_tokens, maxlen, input_ctable,\n","                            nb_examples, reverse)\n","    input_sequences = next(input_sequences)\n","    \n","    # Procedure for inference mode (sampling):\n","    # 1) Encode input and retrieve initial decoder state.\n","    # 2) Run one step of decoder with this initial state\n","    #    and a start-of-sequence character as target.\n","    #    Output will be the next target character.\n","    # 3) Repeat with the current target character and current states.\n","\n","    # Encode the input as state vectors.    \n","    states_value = encoder_model.predict(input_sequences)\n","    \n","    # Create batch of empty target sequences of length 1 character.\n","    target_sequences = np.zeros((nb_examples, 1, target_ctable.size))\n","    # Populate the first element of target sequence\n","    # with the start-of-sequence character.\n","    target_sequences[:, 0, target_ctable.char2index[SOS]] = 1.0\n","\n","    # Sampling loop for a batch of sequences.\n","    # Exit condition: either hit max character limit\n","    # or encounter end-of-sequence character.\n","    decoded_tokens = [''] * nb_examples\n","    for _ in range(maxlen):\n","        # `char_probs` has shape\n","        # (nb_examples, 1, nb_target_chars)\n","        char_probs, h, c = decoder_model.predict(\n","            [target_sequences] + states_value)\n","\n","        # Reset the target sequences.\n","        target_sequences = np.zeros((nb_examples, 1, target_ctable.size))\n","\n","        # Sample next character using argmax or multinomial mode.\n","        sampled_chars = []\n","        for i in range(nb_examples):\n","            if sample_mode == 'argmax':\n","                next_index, next_char = target_ctable.decode(\n","                    char_probs[i], calc_argmax=True)\n","            elif sample_mode == 'multinomial':\n","                next_index, next_char = target_ctable.sample_multinomial(\n","                    char_probs[i], temperature=0.5)\n","            else:\n","                raise Exception(\n","                    \"`sample_mode` accepts `argmax` or `multinomial`.\")\n","            decoded_tokens[i] += next_char\n","            sampled_chars.append(next_char) \n","            # Update target sequence with index of next character.\n","            target_sequences[i, 0, next_index] = 1.0\n","\n","        stop_char = set(sampled_chars)\n","        if len(stop_char) == 1 and stop_char.pop() == EOS:\n","            break\n","            \n","        # Update states.\n","        states_value = [h, c]\n","    \n","    # Sampling finished.\n","    input_tokens   = [re.sub('[%s]' % EOS, '', token)\n","                      for token in input_tokens]\n","    target_tokens  = [re.sub('[%s]' % EOS, '', token)\n","                      for token in target_tokens]\n","    decoded_tokens = [re.sub('[%s]' % EOS, '', token)\n","                      for token in decoded_tokens]\n","    return input_tokens, target_tokens, decoded_tokens\n"]},{"cell_type":"code","execution_count":19,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:41:01.970259Z","iopub.status.busy":"2022-04-10T01:41:01.968982Z","iopub.status.idle":"2022-04-10T01:41:01.978256Z","shell.execute_reply":"2022-04-10T01:41:01.977443Z","shell.execute_reply.started":"2022-04-10T01:41:01.970221Z"},"trusted":true},"outputs":[],"source":["def restore_model(path_to_full_model, hidden_size):\n","    \"\"\"Restore model to construct the encoder and decoder.\"\"\"\n","    model = load_model(path_to_full_model, custom_objects={\n","        'truncated_acc': truncated_acc, 'truncated_loss': truncated_loss})\n","    \n","    encoder_inputs = model.input[0] # encoder_data\n","    encoder_lstm1 = model.get_layer('encoder_lstm_1')\n","    encoder_lstm2 = model.get_layer('encoder_lstm_2')\n","    \n","    encoder_outputs = encoder_lstm1(encoder_inputs)\n","    _, state_h, state_c = encoder_lstm2(encoder_outputs)\n","    encoder_states = [state_h, state_c]\n","    encoder_model = Model(inputs=encoder_inputs, outputs=encoder_states)\n","\n","    decoder_inputs = model.input[1] # decoder_data\n","    decoder_state_input_h = Input(shape=(hidden_size,))\n","    decoder_state_input_c = Input(shape=(hidden_size,))\n","    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n","    decoder_lstm = model.get_layer('decoder_lstm')\n","    decoder_outputs, state_h, state_c = decoder_lstm(\n","        decoder_inputs, initial_state=decoder_states_inputs)\n","    decoder_states = [state_h, state_c]\n","    decoder_softmax = model.get_layer('decoder_softmax')\n","    decoder_outputs = decoder_softmax(decoder_outputs)\n","    decoder_model = Model(inputs=[decoder_inputs] + decoder_states_inputs,\n","                          outputs=[decoder_outputs] + decoder_states)\n","    return encoder_model, decoder_model"]},{"cell_type":"code","execution_count":22,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:44:40.529381Z","iopub.status.busy":"2022-04-10T01:44:40.528753Z","iopub.status.idle":"2022-04-10T01:44:40.536217Z","shell.execute_reply":"2022-04-10T01:44:40.535194Z","shell.execute_reply.started":"2022-04-10T01:44:40.529339Z"},"trusted":true},"outputs":[],"source":["error_rate = 0.8\n","hidden_size = 512\n","nb_epochs = 100\n","train_batch_size = 128\n","val_batch_size = 256\n","sample_mode = 'argmax'\n","# Input sequences may optionally be reversed,\n","# shown to increase performance by introducing\n","# shorter term dependencies between source and target:\n","# \"Learning to Execute\"\n","# http://arxiv.org/abs/1410.4615\n","# \"Sequence to Sequence Learning with Neural Networks\"\n","# https://arxiv.org/abs/1409.3215\n","reverse = True\n","\n","data_path = '../input/spelling'\n","train_books = ['aspell.txt', 'big.txt',\n","               'birkbeck.txt', 'wikipedia.txt']\n","val_books = ['spell-testset1.txt']\n"]},{"cell_type":"code","execution_count":24,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:45:13.849045Z","iopub.status.busy":"2022-04-10T01:45:13.848423Z","iopub.status.idle":"2022-04-10T01:45:17.827675Z","shell.execute_reply":"2022-04-10T01:45:17.826940Z","shell.execute_reply.started":"2022-04-10T01:45:13.848990Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["['satifamtoin*********************************', 'svaagly*************************************', 'therouglhy**********************************', 'progroR*************************************', 'collects************************************', 'tenemeWt************************************', 'jranker*************************************', 'blotting************************************', 'voWye***************************************', 'ansiaIzaty**********************************']\n","['\\tsatifactoin********************************', '\\tsavagly************************************', '\\ttheroughly*********************************', '\\tprogrom************************************', '\\tcollects***********************************', '\\ttenement***********************************', '\\tranker*************************************', '\\tblotting***********************************', '\\tvolye**************************************', '\\tansiazaty**********************************']\n","['satifactoin*********************************', 'savagly*************************************', 'theroughly**********************************', 'progrom*************************************', 'collects************************************', 'tenement************************************', 'ranker**************************************', 'blotting************************************', 'volye***************************************', 'ansiazaty***********************************']\n","Size of training vocabulary = 71874\n","Number of unique input characters: 55\n","Number of unique target characters: 56\n","Max sequence length in the training set: 44\n"]}],"source":["# Prepare training data.\n","text  = read_text(data_path, train_books)\n","vocab = tokenize(text)\n","vocab = list(filter(None, set(vocab)))\n","    \n","# `maxlen` is the length of the longest word in the vocabulary\n","# plus two SOS and EOS characters.\n","maxlen = max([len(token) for token in vocab]) + 2\n","train_encoder, train_decoder, train_target = transform(\n","    vocab, maxlen, error_rate=error_rate, shuffle=False)\n","print(train_encoder[:10])\n","print(train_decoder[:10])\n","print(train_target[:10])\n","\n","input_chars = set(' '.join(train_encoder))\n","target_chars = set(' '.join(train_decoder))\n","nb_input_chars = len(input_chars)\n","nb_target_chars = len(target_chars)\n","\n","print('Size of training vocabulary =', len(vocab))\n","print('Number of unique input characters:', nb_input_chars)\n","print('Number of unique target characters:', nb_target_chars)\n","print('Max sequence length in the training set:', maxlen)\n"]},{"cell_type":"code","execution_count":25,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:46:09.871264Z","iopub.status.busy":"2022-04-10T01:46:09.870737Z","iopub.status.idle":"2022-04-10T01:46:09.899039Z","shell.execute_reply":"2022-04-10T01:46:09.898280Z","shell.execute_reply.started":"2022-04-10T01:46:09.871229Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["['cotnented***********************************', 'contnepted**********************************', 'contene*************************************', 'conended************************************', 'contentid***********************************', 'beginnin************************************', 'egining*************************************', 'preblem*************************************', 'prolam**************************************', 'porble**************************************']\n","['\\tcontented**********************************', '\\tcontenpted*********************************', '\\tcontende***********************************', '\\tcontended**********************************', '\\tcontentid**********************************', '\\tbeginning**********************************', '\\tbegining***********************************', '\\tproblem************************************', '\\tproblam************************************', '\\tproble*************************************']\n","['contented***********************************', 'contenpted**********************************', 'contende************************************', 'contended***********************************', 'contentid***********************************', 'beginning***********************************', 'begining************************************', 'problem*************************************', 'problam*************************************', 'proble**************************************']\n","Number of non-unique validation tokens = 411\n","Max sequence length in the validation set: 19\n"]}],"source":["# Prepare validation data.\n","text = read_text(data_path, val_books)\n","val_tokens = tokenize(text)\n","val_tokens = list(filter(None, val_tokens))\n","\n","val_maxlen = max([len(token) for token in val_tokens]) + 2\n","val_encoder, val_decoder, val_target = transform(\n","    val_tokens, maxlen, error_rate=error_rate, shuffle=False)\n","print(val_encoder[:10])\n","print(val_decoder[:10])\n","print(val_target[:10])\n","print('Number of non-unique validation tokens =', len(val_tokens))\n","print('Max sequence length in the validation set:', val_maxlen)"]},{"cell_type":"code","execution_count":26,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:46:34.489674Z","iopub.status.busy":"2022-04-10T01:46:34.489059Z","iopub.status.idle":"2022-04-10T01:46:34.496395Z","shell.execute_reply":"2022-04-10T01:46:34.495565Z","shell.execute_reply.started":"2022-04-10T01:46:34.489632Z"},"trusted":true},"outputs":[],"source":[" # Define training and evaluation configuration.\n","input_ctable  = CharacterTable(input_chars)\n","target_ctable = CharacterTable(target_chars)\n","\n","train_steps = len(vocab) // train_batch_size\n","val_steps = len(val_tokens) // val_batch_size\n"]},{"cell_type":"code","execution_count":31,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:53:27.768681Z","iopub.status.busy":"2022-04-10T01:53:27.768427Z","iopub.status.idle":"2022-04-10T01:53:28.751440Z","shell.execute_reply":"2022-04-10T01:53:28.750647Z","shell.execute_reply.started":"2022-04-10T01:53:27.768652Z"},"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.7/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n","  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"]},{"name":"stdout","output_type":"stream","text":["Model: \"model_1\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","encoder_data (InputLayer)       [(None, None, 55)]   0                                            \n","__________________________________________________________________________________________________\n","encoder_lstm_1 (LSTM)           (None, None, 512)    1163264     encoder_data[0][0]               \n","__________________________________________________________________________________________________\n","decoder_data (InputLayer)       [(None, None, 56)]   0                                            \n","__________________________________________________________________________________________________\n","encoder_lstm_2 (LSTM)           [(None, 512), (None, 2099200     encoder_lstm_1[0][0]             \n","__________________________________________________________________________________________________\n","decoder_lstm (LSTM)             [(None, None, 512),  1165312     decoder_data[0][0]               \n","                                                                 encoder_lstm_2[0][1]             \n","                                                                 encoder_lstm_2[0][2]             \n","__________________________________________________________________________________________________\n","decoder_softmax (Dense)         (None, None, 56)     28728       decoder_lstm[0][0]               \n","==================================================================================================\n","Total params: 4,456,504\n","Trainable params: 4,456,504\n","Non-trainable params: 0\n","__________________________________________________________________________________________________\n","None\n"]}],"source":["# Compile the model.\n","model, encoder_model, decoder_model = seq2seq(\n","    hidden_size, nb_input_chars, nb_target_chars)\n","print(model.summary())\n"]},{"cell_type":"code","execution_count":32,"metadata":{"execution":{"iopub.execute_input":"2022-04-10T01:54:55.540285Z","iopub.status.busy":"2022-04-10T01:54:55.539823Z","iopub.status.idle":"2022-04-10T07:19:00.484869Z","shell.execute_reply":"2022-04-10T07:19:00.484119Z","shell.execute_reply.started":"2022-04-10T01:54:55.540251Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Main Epoch 1/100\n","Shuffling data.\n"]},{"name":"stderr","output_type":"stream","text":["/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n","  warnings.warn('`Model.fit_generator` is deprecated and '\n","2022-04-10 01:54:57.284613: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n","2022-04-10 01:55:03.868153: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n"]},{"name":"stdout","output_type":"stream","text":["561/561 [==============================] - 192s 330ms/step - loss: 0.4673 - accuracy: 0.8697 - truncated_acc: 0.6450 - truncated_loss: 1.2510 - val_loss: 0.1629 - val_accuracy: 0.9524 - val_truncated_acc: 0.8694 - val_truncated_loss: 0.4478\n","-\n","Input tokens:   ['soVmone', 'evxtended', 'minutes', 'chzlenges', 'stumache']\n","Decoded tokens: ['soomen', 'extented', 'minters', 'chlenges', 'stumache']\n","Target tokens:  ['somone', 'extended', 'minutes', 'chalenges', 'stumache']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_1.h5\n","Main Epoch 2/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.1544 - accuracy: 0.9541 - truncated_acc: 0.8738 - truncated_loss: 0.4233 - val_loss: 0.0903 - val_accuracy: 0.9764 - val_truncated_acc: 0.9351 - val_truncated_loss: 0.2482\n","-\n","Input tokens:   ['centraly', 'opnisite', 'aDess', 'wotve', 'hierchky']\n","Decoded tokens: ['centrally', 'opisitie', 'aless', 'wotev', 'hirechy']\n","Target tokens:  ['centraly', 'oppisite', 'acess', 'wote', 'hierchy']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_2.h5\n","Main Epoch 3/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.1185 - accuracy: 0.9653 - truncated_acc: 0.9046 - truncated_loss: 0.3256 - val_loss: 0.0814 - val_accuracy: 0.9783 - val_truncated_acc: 0.9404 - val_truncated_loss: 0.2238\n","-\n","Input tokens:   ['pZretend', 'agh', 'possibl', 'supersed', 'neccesayr']\n","Decoded tokens: ['pretend', 'agh', 'possibl', 'supersed', 'neccesary']\n","Target tokens:  ['pretend', 'lagh', 'possible', 'supersede', 'neccesary']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_3.h5\n","Main Epoch 4/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.1053 - accuracy: 0.9694 - truncated_acc: 0.9160 - truncated_loss: 0.2894 - val_loss: 0.0749 - val_accuracy: 0.9809 - val_truncated_acc: 0.9475 - val_truncated_loss: 0.2061\n","-\n","Input tokens:   ['latiNt', 'parralell', 'hwether', 'magnifcient', 'neccesary']\n","Decoded tokens: ['latint', 'parrallel', 'whether', 'magnificent', 'neccesary']\n","Target tokens:  ['latist', 'parrallel', 'whether', 'magnificent', 'neccesary']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_4.h5\n","Main Epoch 5/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0969 - accuracy: 0.9721 - truncated_acc: 0.9235 - truncated_loss: 0.2663 - val_loss: 0.0732 - val_accuracy: 0.9820 - val_truncated_acc: 0.9504 - val_truncated_loss: 0.2014\n","-\n","Input tokens:   ['minutes', 'discriptin', 'wanred', 'complety', 'cetrans']\n","Decoded tokens: ['minutes', 'discriptin', 'wanred', 'complety', 'certans']\n","Target tokens:  ['minutes', 'discription', 'wanted', 'completly', 'certans']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_5.h5\n","Main Epoch 6/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0907 - accuracy: 0.9742 - truncated_acc: 0.9290 - truncated_loss: 0.2492 - val_loss: 0.0680 - val_accuracy: 0.9829 - val_truncated_acc: 0.9529 - val_truncated_loss: 0.1870\n","-\n","Input tokens:   ['trOangular', 'concidr', 'iew', 'dirzen', 'chzlenges']\n","Decoded tokens: ['trangular', 'concider', 'iew', 'dirzen', 'chllenges']\n","Target tokens:  ['triangular', 'concider', 'liew', 'dirven', 'chalenges']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_6.h5\n","Main Epoch 7/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0856 - accuracy: 0.9757 - truncated_acc: 0.9334 - truncated_loss: 0.2351 - val_loss: 0.0619 - val_accuracy: 0.9847 - val_truncated_acc: 0.9580 - val_truncated_loss: 0.1703\n","-\n","Input tokens:   ['apralell', 'reciet', 'often', 'pemetery', 'uFnique']\n","Decoded tokens: ['aprallel', 'reciet', 'often', 'pemetery', 'unique']\n","Target tokens:  ['paralell', 'reciet', 'often', 'cemetery', 'unique']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_7.h5\n","Main Epoch 8/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0813 - accuracy: 0.9771 - truncated_acc: 0.9370 - truncated_loss: 0.2235 - val_loss: 0.0606 - val_accuracy: 0.9844 - val_truncated_acc: 0.9570 - val_truncated_loss: 0.1667\n","-\n","Input tokens:   ['contentid', 'particulr', 'benefis', 'henifits', 'purSple']\n","Decoded tokens: ['contentid', 'particuler', 'benefis', 'henifits', 'purple']\n","Target tokens:  ['contentid', 'particular', 'benefits', 'benifits', 'purple']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_8.h5\n","Main Epoch 9/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0783 - accuracy: 0.9781 - truncated_acc: 0.9399 - truncated_loss: 0.2152 - val_loss: 0.0590 - val_accuracy: 0.9845 - val_truncated_acc: 0.9573 - val_truncated_loss: 0.1621\n","-\n","Input tokens:   ['promlbem', 'stomche', 'poition', 'careeJ', 'reoeipt']\n","Decoded tokens: ['promblem', 'stomche', 'poition', 'careed', 'reoepit']\n","Target tokens:  ['promblem', 'stomache', 'position', 'career', 'receipt']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_9.h5\n","Main Epoch 10/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0749 - accuracy: 0.9790 - truncated_acc: 0.9423 - truncated_loss: 0.2059 - val_loss: 0.0561 - val_accuracy: 0.9845 - val_truncated_acc: 0.9573 - val_truncated_loss: 0.1543\n","-\n","Input tokens:   ['initidls', 'byciKcle', 'vitsied', 'particlaur', 'variouAs']\n","Decoded tokens: ['initidls', 'bycicle', 'vistied', 'particalur', 'various']\n","Target tokens:  ['initials', 'bycicle', 'vistied', 'particulaur', 'various']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_10.h5\n","Main Epoch 11/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0726 - accuracy: 0.9797 - truncated_acc: 0.9442 - truncated_loss: 0.1995 - val_loss: 0.0561 - val_accuracy: 0.9848 - val_truncated_acc: 0.9583 - val_truncated_loss: 0.1544\n","-\n","Input tokens:   ['comTare', 'exrteamly', 'monitoring', 'deusicate', 'possibl']\n","Decoded tokens: ['compare', 'extreamly', 'monitoring', 'desuicate', 'possible']\n","Target tokens:  ['compare', 'extreamly', 'monitoring', 'desicate', 'possible']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_11.h5\n","Main Epoch 12/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0708 - accuracy: 0.9803 - truncated_acc: 0.9458 - truncated_loss: 0.1945 - val_loss: 0.0552 - val_accuracy: 0.9848 - val_truncated_acc: 0.9583 - val_truncated_loss: 0.1517\n","-\n","Input tokens:   ['defenition', 'olans', 'apralel', 'descide', 'biscuists']\n","Decoded tokens: ['defenition', 'colans', 'apralel', 'descide', 'biscuists']\n","Target tokens:  ['defenition', 'loans', 'paralel', 'descide', 'biscuits']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_12.h5\n","Main Epoch 13/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0685 - accuracy: 0.9809 - truncated_acc: 0.9475 - truncated_loss: 0.1882 - val_loss: 0.0508 - val_accuracy: 0.9856 - val_truncated_acc: 0.9604 - val_truncated_loss: 0.1398\n","-\n","Input tokens:   ['dessiccate', 'parraalell', 'discriptin', 'trOangular', 'scisor']\n","Decoded tokens: ['dessiccate', 'parralell', 'discripting', 'trangular', 'scisor']\n","Target tokens:  ['dessiccate', 'parrallell', 'discription', 'triangular', 'scisors']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_13.h5\n","Main Epoch 14/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0670 - accuracy: 0.9813 - truncated_acc: 0.9485 - truncated_loss: 0.1842 - val_loss: 0.0501 - val_accuracy: 0.9872 - val_truncated_acc: 0.9648 - val_truncated_loss: 0.1377\n","-\n","Input tokens:   ['lugY', 'altets', 'uFnique', 'conended', 'accommodaqtion']\n","Decoded tokens: ['lugh', 'altets', 'unique', 'contended', 'accommodation']\n","Target tokens:  ['lugh', 'latets', 'unique', 'contended', 'accommodation']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_14.h5\n","Main Epoch 15/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0653 - accuracy: 0.9817 - truncated_acc: 0.9498 - truncated_loss: 0.1795 - val_loss: 0.0502 - val_accuracy: 0.9865 - val_truncated_acc: 0.9629 - val_truncated_loss: 0.1381\n","-\n","Input tokens:   ['auxillary', 'lcoally', 'thher', 'hieKrarchal', 'beginnin']\n","Decoded tokens: ['auxillary', 'locally', 'ther', 'hierrachal', 'beginning']\n","Target tokens:  ['auxillary', 'locally', 'ther', 'hierarchal', 'beginning']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_15.h5\n","Main Epoch 16/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0636 - accuracy: 0.9822 - truncated_acc: 0.9512 - truncated_loss: 0.1748 - val_loss: 0.0481 - val_accuracy: 0.9867 - val_truncated_acc: 0.9634 - val_truncated_loss: 0.1322\n","-\n","Input tokens:   ['curtiEns', 'itnials', 'descide', 'rant', 'potere']\n","Decoded tokens: ['curtions', 'intials', 'descide', 'rant', 'potere']\n","Target tokens:  ['curtions', 'intials', 'descide', 'arnt', 'poetre']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_16.h5\n","Main Epoch 17/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0626 - accuracy: 0.9825 - truncated_acc: 0.9520 - truncated_loss: 0.1720 - val_loss: 0.0475 - val_accuracy: 0.9866 - val_truncated_acc: 0.9631 - val_truncated_loss: 0.1306\n","-\n","Input tokens:   ['valublA', 'edfinitions', 'lau', 'hieKrarchal', 'pZretend']\n","Decoded tokens: ['valubly', 'definitions', 'lau', 'hierarchal', 'pretend']\n","Target tokens:  ['valuble', 'definitions', 'lauf', 'hierarchal', 'pretend']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_17.h5\n","Main Epoch 18/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0612 - accuracy: 0.9828 - truncated_acc: 0.9528 - truncated_loss: 0.1681 - val_loss: 0.0453 - val_accuracy: 0.9869 - val_truncated_acc: 0.9639 - val_truncated_loss: 0.1246\n","-\n","Input tokens:   ['oppositO', 'unexcpted', 'cmoittee', 'litriture', 'centraly']\n","Decoded tokens: ['opposite', 'unexcpted', 'comittee', 'litriture', 'centraly']\n","Target tokens:  ['opposite', 'unexpcted', 'comittee', 'litriture', 'centraly']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_18.h5\n","Main Epoch 19/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0602 - accuracy: 0.9831 - truncated_acc: 0.9535 - truncated_loss: 0.1655 - val_loss: 0.0435 - val_accuracy: 0.9877 - val_truncated_acc: 0.9663 - val_truncated_loss: 0.1197\n","-\n","Input tokens:   ['iuce', 'liaiso', 'diagrammmtically', 'juTe', 'ails']\n","Decoded tokens: ['lice', 'liaison', 'diagrmmmatically', 'jure', 'ails']\n","Target tokens:  ['juce', 'liaison', 'diagrammatically', 'juse', 'fails']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_19.h5\n","Main Epoch 20/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0595 - accuracy: 0.9833 - truncated_acc: 0.9541 - truncated_loss: 0.1634 - val_loss: 0.0434 - val_accuracy: 0.9881 - val_truncated_acc: 0.9673 - val_truncated_loss: 0.1192\n","-\n","Input tokens:   ['arrcngeing', 'rermember', 'Pcourtens', 'Kminuscule', 'lpvels']\n","Decoded tokens: ['arrangeing', 'remember', 'courtens', 'minuscule', 'lavels']\n","Target tokens:  ['arrangeing', 'rermember', 'courtens', 'minuscule', 'levels']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_20.h5\n","Main Epoch 21/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0582 - accuracy: 0.9837 - truncated_acc: 0.9552 - truncated_loss: 0.1600 - val_loss: 0.0431 - val_accuracy: 0.9870 - val_truncated_acc: 0.9644 - val_truncated_loss: 0.1186\n","-\n","Input tokens:   ['exrteamly', 'Kminuscule', 'aQcount', 'dirzen', 'parlalel']\n","Decoded tokens: ['extreamly', 'minuscule', 'account', 'dirzen', 'parlalel']\n","Target tokens:  ['extreamly', 'minuscule', 'account', 'dirven', 'parallel']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_21.h5\n","Main Epoch 22/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0574 - accuracy: 0.9839 - truncated_acc: 0.9558 - truncated_loss: 0.1577 - val_loss: 0.0432 - val_accuracy: 0.9885 - val_truncated_acc: 0.9685 - val_truncated_loss: 0.1187\n","-\n","Input tokens:   ['Aelly', 'experince', 'biscitds', 'seperate', 'csissors']\n","Decoded tokens: ['Alelly', 'experince', 'biscits', 'seperate', 'scissors']\n","Target tokens:  ['relly', 'experiance', 'biscits', 'seperate', 'scissors']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_22.h5\n","Main Epoch 23/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0561 - accuracy: 0.9842 - truncated_acc: 0.9566 - truncated_loss: 0.1543 - val_loss: 0.0413 - val_accuracy: 0.9883 - val_truncated_acc: 0.9678 - val_truncated_loss: 0.1135\n","-\n","Input tokens:   ['inconvininet', 'unexpected', 'chaptdr', 'adrdess', 'cerain']\n","Decoded tokens: ['inconvinient', 'unexpected', 'chapter', 'address', 'cerain']\n","Target tokens:  ['inconvinient', 'unexpected', 'chapter', 'address', 'certain']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_23.h5\n","Main Epoch 24/100\n","Shuffling data.\n","561/561 [==============================] - 186s 332ms/step - loss: 0.0557 - accuracy: 0.9843 - truncated_acc: 0.9568 - truncated_loss: 0.1531 - val_loss: 0.0391 - val_accuracy: 0.9888 - val_truncated_acc: 0.9692 - val_truncated_loss: 0.1074\n","-\n","Input tokens:   ['latiset', 'decide', 'complety', 'unexspected', 'perBle']\n","Decoded tokens: ['latiset', 'decide', 'complety', 'unexspected', 'perse']\n","Target tokens:  ['latiest', 'decide', 'completly', 'unexspected', 'perple']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_24.h5\n","Main Epoch 25/100\n","Shuffling data.\n","561/561 [==============================] - 182s 325ms/step - loss: 0.0554 - accuracy: 0.9844 - truncated_acc: 0.9570 - truncated_loss: 0.1521 - val_loss: 0.0392 - val_accuracy: 0.9893 - val_truncated_acc: 0.9707 - val_truncated_loss: 0.1079\n","-\n","Input tokens:   ['occurreWce', 'often', 'diffreent', 'votign', 'familes']\n","Decoded tokens: ['occurrence', 'often', 'diffrent', 'voting', 'familes']\n","Target tokens:  ['occurrence', 'often', 'different', 'voting', 'families']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_25.h5\n","Main Epoch 26/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0543 - accuracy: 0.9847 - truncated_acc: 0.9579 - truncated_loss: 0.1492 - val_loss: 0.0376 - val_accuracy: 0.9891 - val_truncated_acc: 0.9700 - val_truncated_loss: 0.1033\n","-\n","Input tokens:   ['lpvels', 'wotve', 'olans', 'totaly', 'definitely']\n","Decoded tokens: ['lovels', 'wote', 'loans', 'totaly', 'definitely']\n","Target tokens:  ['levels', 'wote', 'loans', 'totally', 'definitely']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_26.h5\n","Main Epoch 27/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0540 - accuracy: 0.9848 - truncated_acc: 0.9582 - truncated_loss: 0.1484 - val_loss: 0.0391 - val_accuracy: 0.9886 - val_truncated_acc: 0.9688 - val_truncated_loss: 0.1075\n","-\n","Input tokens:   ['uant', 'prtiend', 'clearical', 'supersed', 'Pcourtens']\n","Decoded tokens: ['gant', 'prtiend', 'clearical', 'supersed', 'courtens']\n","Target tokens:  ['aunt', 'pritend', 'clearical', 'supersede', 'courtens']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_27.h5\n","Main Epoch 28/100\n","Shuffling data.\n","561/561 [==============================] - 187s 333ms/step - loss: 0.0529 - accuracy: 0.9850 - truncated_acc: 0.9588 - truncated_loss: 0.1454 - val_loss: 0.0386 - val_accuracy: 0.9887 - val_truncated_acc: 0.9690 - val_truncated_loss: 0.1061\n","-\n","Input tokens:   ['defenitions', 'agnifisent', 'scarPely', 'pemetery', 'initils']\n","Decoded tokens: ['defenitions', 'magnifisent', 'scarvely', 'pemetery', 'initils']\n","Target tokens:  ['defenitions', 'magnifisent', 'scarely', 'cemetery', 'initails']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_28.h5\n","Main Epoch 29/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0526 - accuracy: 0.9851 - truncated_acc: 0.9591 - truncated_loss: 0.1445 - val_loss: 0.0368 - val_accuracy: 0.9894 - val_truncated_acc: 0.9709 - val_truncated_loss: 0.1011\n","-\n","Input tokens:   ['reieve', 'neccesayr', 'initils', 'reLeber', 'magnifisant']\n","Decoded tokens: ['relieve', 'neccesary', 'initils', 'remeber', 'magnifisant']\n","Target tokens:  ['recieve', 'neccesary', 'initails', 'remeber', 'magnifiscant']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_29.h5\n","Main Epoch 30/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0522 - accuracy: 0.9852 - truncated_acc: 0.9593 - truncated_loss: 0.1434 - val_loss: 0.0372 - val_accuracy: 0.9893 - val_truncated_acc: 0.9705 - val_truncated_loss: 0.1022\n","-\n","Input tokens:   ['pome', 'biosquits', 'sources', 'reaCy', 'rHfresment']\n","Decoded tokens: ['pome', 'biosquits', 'sources', 'realy', 'refresment']\n","Target tokens:  ['poame', 'bisquits', 'sources', 'realy', 'refresment']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_30.h5\n","Main Epoch 31/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0513 - accuracy: 0.9856 - truncated_acc: 0.9603 - truncated_loss: 0.1411 - val_loss: 0.0360 - val_accuracy: 0.9898 - val_truncated_acc: 0.9719 - val_truncated_loss: 0.0991\n","-\n","Input tokens:   ['scarseQly', 'refreshwmant', 'addressable', 'ofmton', 'pronuncciation']\n","Decoded tokens: ['scarsely', 'refreshmant', 'addressable', 'fomton', 'pronunciation']\n","Target tokens:  ['scarsely', 'refreshmant', 'addressable', 'ofton', 'pronunciation']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_31.h5\n","Main Epoch 32/100\n","Shuffling data.\n","561/561 [==============================] - 184s 327ms/step - loss: 0.0507 - accuracy: 0.9856 - truncated_acc: 0.9605 - truncated_loss: 0.1394 - val_loss: 0.0359 - val_accuracy: 0.9899 - val_truncated_acc: 0.9722 - val_truncated_loss: 0.0986\n","-\n","Input tokens:   ['jucie', 'accommodaqtion', 'initils', 'poerty', 'access']\n","Decoded tokens: ['jucie', 'accommodation', 'initials', 'poerty', 'access']\n","Target tokens:  ['jucie', 'accommodation', 'initails', 'poertry', 'access']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_32.h5\n","Main Epoch 33/100\n","Shuffling data.\n","561/561 [==============================] - 184s 327ms/step - loss: 0.0507 - accuracy: 0.9856 - truncated_acc: 0.9605 - truncated_loss: 0.1393 - val_loss: 0.0364 - val_accuracy: 0.9909 - val_truncated_acc: 0.9749 - val_truncated_loss: 0.1001\n","-\n","Input tokens:   ['biscitds', 'chaphter', 'poarty', 'addressable', 'diffreent']\n","Decoded tokens: ['biscits', 'chaphter', 'poarty', 'addressable', 'different']\n","Target tokens:  ['biscits', 'chaphter', 'poartry', 'addressable', 'different']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_33.h5\n","Main Epoch 34/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0499 - accuracy: 0.9859 - truncated_acc: 0.9612 - truncated_loss: 0.1373 - val_loss: 0.0360 - val_accuracy: 0.9900 - val_truncated_acc: 0.9724 - val_truncated_loss: 0.0991\n","-\n","Input tokens:   ['occurreWce', 'berhaps', 'definitely', 'esiccate', 'moniteing']\n","Decoded tokens: ['occurrence', 'berhaps', 'definitely', 'esiccate', 'moniteing']\n","Target tokens:  ['occurrence', 'perhaps', 'definitely', 'desiccate', 'monitering']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_34.h5\n","Main Epoch 35/100\n","Shuffling data.\n","561/561 [==============================] - 184s 327ms/step - loss: 0.0490 - accuracy: 0.9862 - truncated_acc: 0.9620 - truncated_loss: 0.1347 - val_loss: 0.0366 - val_accuracy: 0.9900 - val_truncated_acc: 0.9724 - val_truncated_loss: 0.1006\n","-\n","Input tokens:   ['minutes', 'reaHlly', 'hievrarchy', 'altets', 'arranged']\n","Decoded tokens: ['minutes', 'really', 'hierarchy', 'altets', 'arranged']\n","Target tokens:  ['minutes', 'really', 'hierarchy', 'latets', 'arranged']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_35.h5\n","Main Epoch 36/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0495 - accuracy: 0.9860 - truncated_acc: 0.9615 - truncated_loss: 0.1360 - val_loss: 0.0361 - val_accuracy: 0.9896 - val_truncated_acc: 0.9714 - val_truncated_loss: 0.0991\n","-\n","Input tokens:   ['comTare', 'gZic', 'southern', 'Fember', 'embarass']\n","Decoded tokens: ['compare', 'gic', 'southern', 'Fember', 'embarass']\n","Target tokens:  ['compare', 'guic', 'southern', 'rember', 'embarrass']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_36.h5\n","Main Epoch 37/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0487 - accuracy: 0.9862 - truncated_acc: 0.9621 - truncated_loss: 0.1339 - val_loss: 0.0350 - val_accuracy: 0.9898 - val_truncated_acc: 0.9719 - val_truncated_loss: 0.0963\n","-\n","Input tokens:   ['rtansfred', 'further', 'plnaed', ' ycycle', 'plnaed']\n","Decoded tokens: ['transfred', 'further', 'planed', 'bycycle', 'planed']\n","Target tokens:  ['transfred', 'further', 'planed', 'bycycle', 'planed']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_37.h5\n","Main Epoch 38/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0479 - accuracy: 0.9864 - truncated_acc: 0.9627 - truncated_loss: 0.1318 - val_loss: 0.0346 - val_accuracy: 0.9896 - val_truncated_acc: 0.9714 - val_truncated_loss: 0.0950\n","-\n","Input tokens:   ['olans', 'prtned', 'miniscule', 'barrainged', 'potend']\n","Decoded tokens: ['loans', 'proned', 'miniscule', 'barrainged', 'potend']\n","Target tokens:  ['loans', 'prtend', 'miniscule', 'arrainged', 'protend']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_38.h5\n","Main Epoch 39/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0478 - accuracy: 0.9865 - truncated_acc: 0.9628 - truncated_loss: 0.1313 - val_loss: 0.0351 - val_accuracy: 0.9898 - val_truncated_acc: 0.9719 - val_truncated_loss: 0.0966\n","-\n","Input tokens:   ['valublA', 'expreances', 'rant', 'conended', 'articals']\n","Decoded tokens: ['valuble', 'experances', 'rant', 'contended', 'articals']\n","Target tokens:  ['valuble', 'experances', 'arnt', 'contended', 'articals']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_39.h5\n","Main Epoch 40/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0475 - accuracy: 0.9865 - truncated_acc: 0.9630 - truncated_loss: 0.1305 - val_loss: 0.0328 - val_accuracy: 0.9899 - val_truncated_acc: 0.9722 - val_truncated_loss: 0.0902\n","-\n","Input tokens:   ['speaOical', 'necepary', 'powetry', 'singjlar', 'lates']\n","Decoded tokens: ['spearical', 'necepary', 'powetry', 'singlar', 'lates']\n","Target tokens:  ['speaical', 'necesary', 'powetry', 'singular', 'lates']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_40.h5\n","Main Epoch 41/100\n","Shuffling data.\n","561/561 [==============================] - 183s 325ms/step - loss: 0.0471 - accuracy: 0.9866 - truncated_acc: 0.9633 - truncated_loss: 0.1294 - val_loss: 0.0341 - val_accuracy: 0.9904 - val_truncated_acc: 0.9736 - val_truncated_loss: 0.0938\n","-\n","Input tokens:   ['juTe', 'lau', 'scacrely', 'biscuists', 'qemind']\n","Decoded tokens: ['jure', 'lau', 'scarcely', 'biscuists', 'remind']\n","Target tokens:  ['juse', 'lauf', 'scarcely', 'biscuits', 'remind']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_41.h5\n","Main Epoch 42/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0470 - accuracy: 0.9867 - truncated_acc: 0.9634 - truncated_loss: 0.1293 - val_loss: 0.0328 - val_accuracy: 0.9901 - val_truncated_acc: 0.9729 - val_truncated_loss: 0.0903\n","-\n","Input tokens:   ['Nremined', 'futheT', 'scisor', 'potere', 'hievrarchy']\n","Decoded tokens: ['remined', 'futher', 'scisor', 'potere', 'hierarchy']\n","Target tokens:  ['remined', 'futher', 'scisors', 'poetre', 'hierarchy']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_42.h5\n","Main Epoch 43/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0465 - accuracy: 0.9868 - truncated_acc: 0.9636 - truncated_loss: 0.1279 - val_loss: 0.0340 - val_accuracy: 0.9894 - val_truncated_acc: 0.9709 - val_truncated_loss: 0.0935\n","-\n","Input tokens:   ['Bchalenges', 'promlbem', 'necepary', 'itnials', 'possalbe']\n","Decoded tokens: ['chalenges', 'promblem', 'necepary', 'intials', 'possable']\n","Target tokens:  ['chalenges', 'promblem', 'necesary', 'intials', 'possable']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_43.h5\n","Main Epoch 44/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0462 - accuracy: 0.9869 - truncated_acc: 0.9640 - truncated_loss: 0.1271 - val_loss: 0.0327 - val_accuracy: 0.9903 - val_truncated_acc: 0.9734 - val_truncated_loss: 0.0899\n","-\n","Input tokens:   ['receiMt', 'acoomdation', 'managemekt', 'dirzen', 'accssing']\n","Decoded tokens: ['receit', 'acomdation', 'management', 'dirzen', 'accessing']\n","Target tokens:  ['receit', 'acomodation', 'management', 'dirven', 'accessing']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_44.h5\n","Main Epoch 45/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0458 - accuracy: 0.9870 - truncated_acc: 0.9642 - truncated_loss: 0.1259 - val_loss: 0.0316 - val_accuracy: 0.9908 - val_truncated_acc: 0.9746 - val_truncated_loss: 0.0870\n","-\n","Input tokens:   ['bicycal', 'uant', 'aDess', 'hwether', 'exsacy']\n","Decoded tokens: ['bicycal', 'aunt', 'acess', 'whether', 'exsacy']\n","Target tokens:  ['bicycal', 'aunt', 'acess', 'whether', 'exstacy']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_45.h5\n","Main Epoch 46/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0455 - accuracy: 0.9871 - truncated_acc: 0.9645 - truncated_loss: 0.1252 - val_loss: 0.0327 - val_accuracy: 0.9905 - val_truncated_acc: 0.9739 - val_truncated_loss: 0.0900\n","-\n","Input tokens:   ['prtiend', 'fisYted', 'agh', 'defenition', 'bneefit']\n","Decoded tokens: ['prtiend', 'fisted', 'agh', 'defenition', 'benefit']\n","Target tokens:  ['pritend', 'fisited', 'lagh', 'defenition', 'benefit']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_46.h5\n","Main Epoch 47/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0456 - accuracy: 0.9870 - truncated_acc: 0.9643 - truncated_loss: 0.1254 - val_loss: 0.0334 - val_accuracy: 0.9902 - val_truncated_acc: 0.9731 - val_truncated_loss: 0.0918\n","-\n","Input tokens:   ['propln', 'latiset', 'unfortunately', 'socres', 'remmeber']\n","Decoded tokens: ['proplen', 'latiset', 'unfortunately', 'socres', 'remember']\n","Target tokens:  ['proplen', 'latiest', 'unfortunately', 'sorces', 'remember']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_47.h5\n","Main Epoch 48/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0451 - accuracy: 0.9872 - truncated_acc: 0.9647 - truncated_loss: 0.1240 - val_loss: 0.0331 - val_accuracy: 0.9905 - val_truncated_acc: 0.9739 - val_truncated_loss: 0.0909\n","-\n","Input tokens:   ['itnials', 'often', 'transporability', 'managemekt', 'csissors']\n","Decoded tokens: ['intials', 'often', 'transporability', 'management', 'scissors']\n","Target tokens:  ['intials', 'often', 'transportability', 'management', 'scissors']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_48.h5\n","Main Epoch 49/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0447 - accuracy: 0.9873 - truncated_acc: 0.9651 - truncated_loss: 0.1230 - val_loss: 0.0328 - val_accuracy: 0.9909 - val_truncated_acc: 0.9751 - val_truncated_loss: 0.0901\n","-\n","Input tokens:   ['bneefit', 'possibl', 'scarseQly', 'indepenyent', 'unexpeted']\n","Decoded tokens: ['benefit', 'possible', 'scarsely', 'independent', 'unexpeted']\n","Target tokens:  ['benefit', 'possible', 'scarsely', 'independent', 'unexpeted']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_49.h5\n","Main Epoch 50/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0445 - accuracy: 0.9873 - truncated_acc: 0.9652 - truncated_loss: 0.1223 - val_loss: 0.0309 - val_accuracy: 0.9903 - val_truncated_acc: 0.9734 - val_truncated_loss: 0.0850\n","-\n","Input tokens:   ['rant', 'pZretend', 'reLeber', 'quperceed', 'diffreent']\n","Decoded tokens: ['rant', 'pretend', 'remeber', 'superceed', 'different']\n","Target tokens:  ['arnt', 'pretend', 'remeber', 'superceed', 'different']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_50.h5\n","Main Epoch 51/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0444 - accuracy: 0.9873 - truncated_acc: 0.9651 - truncated_loss: 0.1221 - val_loss: 0.0313 - val_accuracy: 0.9906 - val_truncated_acc: 0.9741 - val_truncated_loss: 0.0862\n","-\n","Input tokens:   ['gallerOry', 'often', 'deusicate', 'esiccate', 'inconvininet']\n","Decoded tokens: ['gallerry', 'often', 'desuciate', 'esiccate', 'inconvinient']\n","Target tokens:  ['gallerry', 'offten', 'desicate', 'desiccate', 'inconvinient']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_51.h5\n","Main Epoch 52/100\n","Shuffling data.\n","561/561 [==============================] - 182s 324ms/step - loss: 0.0442 - accuracy: 0.9874 - truncated_acc: 0.9654 - truncated_loss: 0.1215 - val_loss: 0.0316 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0869\n","-\n","Input tokens:   ['jucie', 'unexcpted', 'olans', 'discriptin', 'desscate']\n","Decoded tokens: ['juice', 'unexcpeted', 'loans', 'discripting', 'dessicate']\n","Target tokens:  ['jucie', 'unexpcted', 'loans', 'discription', 'dessicate']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_52.h5\n","Main Epoch 53/100\n","Shuffling data.\n","561/561 [==============================] - 182s 324ms/step - loss: 0.0439 - accuracy: 0.9874 - truncated_acc: 0.9655 - truncated_loss: 0.1206 - val_loss: 0.0304 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0837\n","-\n","Input tokens:   ['poerty', 'pYertend', 'juice', 'poet', 'wonuted']\n","Decoded tokens: ['poerty', 'pertend', 'juice', 'poet', 'wounted']\n","Target tokens:  ['poertry', 'pertend', 'juice', 'poety', 'wonted']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_53.h5\n","Main Epoch 54/100\n","Shuffling data.\n","561/561 [==============================] - 181s 323ms/step - loss: 0.0437 - accuracy: 0.9875 - truncated_acc: 0.9656 - truncated_loss: 0.1201 - val_loss: 0.0305 - val_accuracy: 0.9907 - val_truncated_acc: 0.9744 - val_truncated_loss: 0.0840\n","-\n","Input tokens:   ['parraalell', 'cDhaper', 'rtansfred', 'lochaly', 'clerical']\n","Decoded tokens: ['parralall', 'chaper', 'transfred', 'lochaly', 'clerical']\n","Target tokens:  ['parrallell', 'chaper', 'transfred', 'localy', 'clerical']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_54.h5\n","Main Epoch 55/100\n","Shuffling data.\n","561/561 [==============================] - 181s 323ms/step - loss: 0.0434 - accuracy: 0.9876 - truncated_acc: 0.9659 - truncated_loss: 0.1192 - val_loss: 0.0302 - val_accuracy: 0.9911 - val_truncated_acc: 0.9756 - val_truncated_loss: 0.0830\n","-\n","Input tokens:   ['prtiend', 'poetrR', 'uant', 'potere', 'addressable']\n","Decoded tokens: ['pritend', 'poetry', 'aunt', 'potere', 'addressable']\n","Target tokens:  ['pritend', 'poetry', 'aunt', 'poetre', 'addressable']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_55.h5\n","Main Epoch 56/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0431 - accuracy: 0.9877 - truncated_acc: 0.9661 - truncated_loss: 0.1185 - val_loss: 0.0305 - val_accuracy: 0.9909 - val_truncated_acc: 0.9751 - val_truncated_loss: 0.0838\n","-\n","Input tokens:   ['between', 'built', 'astabishing', 'defenition', 'henifits']\n","Decoded tokens: ['between', 'built', 'astablishing', 'defenition', 'henifits']\n","Target tokens:  ['between', 'biult', 'astablishing', 'defenition', 'benifits']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_56.h5\n","Main Epoch 57/100\n","Shuffling data.\n","561/561 [==============================] - 182s 325ms/step - loss: 0.0425 - accuracy: 0.9879 - truncated_acc: 0.9667 - truncated_loss: 0.1169 - val_loss: 0.0297 - val_accuracy: 0.9913 - val_truncated_acc: 0.9761 - val_truncated_loss: 0.0817\n","-\n","Input tokens:   ['specivl', 'southern', 'baRicaly', 'oppossitte', 'leavls']\n","Decoded tokens: ['specivel', 'southern', 'banicaly', 'oppossitte', 'leavls']\n","Target tokens:  ['special', 'southern', 'basicaly', 'oppossitte', 'levals']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_57.h5\n","Main Epoch 58/100\n","Shuffling data.\n","561/561 [==============================] - 181s 322ms/step - loss: 0.0429 - accuracy: 0.9877 - truncated_acc: 0.9663 - truncated_loss: 0.1179 - val_loss: 0.0306 - val_accuracy: 0.9902 - val_truncated_acc: 0.9731 - val_truncated_loss: 0.0841\n","-\n","Input tokens:   ['Wfebuary', 'sPmetary', 'liaisioan', 'scacrely', 'cerain']\n","Decoded tokens: ['Webuary', 'symetary', 'liaisiona', 'scarcely', 'certain']\n","Target tokens:  ['febuary', 'semetary', 'liaision', 'scarcely', 'certain']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_58.h5\n","Main Epoch 59/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0424 - accuracy: 0.9878 - truncated_acc: 0.9666 - truncated_loss: 0.1166 - val_loss: 0.0304 - val_accuracy: 0.9906 - val_truncated_acc: 0.9741 - val_truncated_loss: 0.0836\n","-\n","Input tokens:   ['often', 'cmoittee', 'auxillary', 'ofmton', 'reoeipt']\n","Decoded tokens: ['often', 'comittee', 'auxillary', 'fomton', 'reoeipt']\n","Target tokens:  ['offten', 'comittee', 'auxillary', 'ofton', 'receipt']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_59.h5\n","Main Epoch 60/100\n","Shuffling data.\n","561/561 [==============================] - 181s 323ms/step - loss: 0.0425 - accuracy: 0.9879 - truncated_acc: 0.9667 - truncated_loss: 0.1169 - val_loss: 0.0311 - val_accuracy: 0.9902 - val_truncated_acc: 0.9731 - val_truncated_loss: 0.0855\n","-\n","Input tokens:   ['wrote', 'isssors', 'embarass', 'reaCy', 'oppossitte']\n","Decoded tokens: ['wrote', 'sissors', 'embarass', 'realy', 'oppossitte']\n","Target tokens:  ['wrote', 'sissors', 'embarrass', 'realy', 'oppossitte']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_60.h5\n","Main Epoch 61/100\n","Shuffling data.\n","561/561 [==============================] - 182s 324ms/step - loss: 0.0419 - accuracy: 0.9880 - truncated_acc: 0.9670 - truncated_loss: 0.1151 - val_loss: 0.0306 - val_accuracy: 0.9911 - val_truncated_acc: 0.9756 - val_truncated_loss: 0.0840\n","-\n","Input tokens:   ['ofmton', 'oppositO', 'sPmetary', 'glaery', 'occurreWce']\n","Decoded tokens: ['fomton', 'opposite', 'semetary', 'galery', 'occurrence']\n","Target tokens:  ['ofton', 'opposite', 'semetary', 'galery', 'occurrence']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_61.h5\n","Main Epoch 62/100\n","Shuffling data.\n","561/561 [==============================] - 183s 327ms/step - loss: 0.0416 - accuracy: 0.9881 - truncated_acc: 0.9673 - truncated_loss: 0.1143 - val_loss: 0.0303 - val_accuracy: 0.9903 - val_truncated_acc: 0.9734 - val_truncated_loss: 0.0833\n","-\n","Input tokens:   ['defenitions', 'diffreent', 'barrainged', 'accesing', 'eperance']\n","Decoded tokens: ['defenitions', 'diffrent', 'barrainged', 'accesing', 'eperance']\n","Target tokens:  ['defenitions', 'different', 'arrainged', 'accesing', 'experance']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_62.h5\n","Main Epoch 63/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0414 - accuracy: 0.9882 - truncated_acc: 0.9674 - truncated_loss: 0.1139 - val_loss: 0.0298 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0819\n","-\n","Input tokens:   ['wonuted', 'Oxperience', 'completegly', 'gallerOry', 'ujeful']\n","Decoded tokens: ['wounted', 'experience', 'completely', 'gallerry', 'jueful']\n","Target tokens:  ['wonted', 'experience', 'completely', 'gallerry', 'useful']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_63.h5\n","Main Epoch 64/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0418 - accuracy: 0.9879 - truncated_acc: 0.9669 - truncated_loss: 0.1150 - val_loss: 0.0294 - val_accuracy: 0.9905 - val_truncated_acc: 0.9739 - val_truncated_loss: 0.0808\n","-\n","Input tokens:   ['csissors', 'lcoally', 'accommodaqtion', 'diffren', 'hievrarchy']\n","Decoded tokens: ['scissors', 'locally', 'accommodation', 'differen', 'hierarchy']\n","Target tokens:  ['scissors', 'locally', 'accommodation', 'diffrent', 'hierarchy']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_64.h5\n","Main Epoch 65/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0413 - accuracy: 0.9882 - truncated_acc: 0.9675 - truncated_loss: 0.1135 - val_loss: 0.0297 - val_accuracy: 0.9909 - val_truncated_acc: 0.9749 - val_truncated_loss: 0.0817\n","-\n","Input tokens:   ['extremeKy', 'agnifisent', 'prtned', 'arrangemen', 'opnisite']\n","Decoded tokens: ['extremedy', 'magnifisent', 'prtend', 'arrangement', 'opinite']\n","Target tokens:  ['extremely', 'magnifisent', 'prtend', 'arrangement', 'oppisite']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_65.h5\n","Main Epoch 66/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0412 - accuracy: 0.9882 - truncated_acc: 0.9676 - truncated_loss: 0.1134 - val_loss: 0.0293 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0806\n","-\n","Input tokens:   ['bciycle', 'unexpeted', 'unfortunately', 'further', 'buQlt']\n","Decoded tokens: ['bicycle', 'unexpected', 'unfortunately', 'further', 'bult']\n","Target tokens:  ['bicycle', 'unexpeted', 'unfortunately', 'further', 'built']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_66.h5\n","Main Epoch 67/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0408 - accuracy: 0.9883 - truncated_acc: 0.9678 - truncated_loss: 0.1123 - val_loss: 0.0297 - val_accuracy: 0.9908 - val_truncated_acc: 0.9746 - val_truncated_loss: 0.0817\n","-\n","Input tokens:   ['receiv', 'biscuts', 'completegly', 'arrcngeing', 'voteinf']\n","Decoded tokens: ['receive', 'biscuts', 'completegly', 'arrangeing', 'voteing']\n","Target tokens:  ['receive', 'biscuts', 'completely', 'arrangeing', 'voteing']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_67.h5\n","Main Epoch 68/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0406 - accuracy: 0.9884 - truncated_acc: 0.9681 - truncated_loss: 0.1115 - val_loss: 0.0286 - val_accuracy: 0.9915 - val_truncated_acc: 0.9766 - val_truncated_loss: 0.0787\n","-\n","Input tokens:   ['vaCiant', 'wonuted', 'inHistals', 'emnt', 'extremeKy']\n","Decoded tokens: ['vaciant', 'wounted', 'inistals', 'ment', 'extremety']\n","Target tokens:  ['variant', 'wonted', 'inistals', 'ment', 'extremely']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_68.h5\n","Main Epoch 69/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0404 - accuracy: 0.9883 - truncated_acc: 0.9679 - truncated_loss: 0.1111 - val_loss: 0.0287 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0788\n","-\n","Input tokens:   ['emnt', 'stumache', 'vairaint', 'quperceed', 'necesFsary']\n","Decoded tokens: ['ment', 'stumache', 'vairiant', 'superceed', 'necessary']\n","Target tokens:  ['ment', 'stumache', 'vairiant', 'superceed', 'necessary']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_69.h5\n","Main Epoch 70/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0407 - accuracy: 0.9883 - truncated_acc: 0.9678 - truncated_loss: 0.1120 - val_loss: 0.0300 - val_accuracy: 0.9909 - val_truncated_acc: 0.9751 - val_truncated_loss: 0.0826\n","-\n","Input tokens:   ['oppeste', 'poarty', 'decide', 'undesrand', 'experince']\n","Decoded tokens: ['oppeste', 'poarty', 'decide', 'undersand', 'experience']\n","Target tokens:  ['oppesite', 'poartry', 'decide', 'undersand', 'experiance']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_70.h5\n","Main Epoch 71/100\n","Shuffling data.\n","561/561 [==============================] - 185s 331ms/step - loss: 0.0400 - accuracy: 0.9885 - truncated_acc: 0.9683 - truncated_loss: 0.1100 - val_loss: 0.0295 - val_accuracy: 0.9909 - val_truncated_acc: 0.9749 - val_truncated_loss: 0.0812\n","-\n","Input tokens:   ['lates', 'iew', 'scisor', 'ineRtials', 'ecstwsy']\n","Decoded tokens: ['lates', 'iew', 'scisor', 'inetials', 'ecstasy']\n","Target tokens:  ['lates', 'liew', 'scisors', 'inetials', 'ecstasy']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_71.h5\n","Main Epoch 72/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0399 - accuracy: 0.9886 - truncated_acc: 0.9685 - truncated_loss: 0.1098 - val_loss: 0.0289 - val_accuracy: 0.9916 - val_truncated_acc: 0.9768 - val_truncated_loss: 0.0793\n","-\n","Input tokens:   ['pslendid', 'baRicaly', 'incoFvient', 'lpvels', 'contene']\n","Decoded tokens: ['splendid', 'badicaly', 'inconvient', 'livels', 'contene']\n","Target tokens:  ['splendid', 'basicaly', 'inconvient', 'levels', 'contende']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_72.h5\n","Main Epoch 73/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0396 - accuracy: 0.9886 - truncated_acc: 0.9687 - truncated_loss: 0.1089 - val_loss: 0.0278 - val_accuracy: 0.9917 - val_truncated_acc: 0.9771 - val_truncated_loss: 0.0763\n","-\n","Input tokens:   ['ccurence', 'chaptdr', 'diagrammmtically', 'esiccate', 'olans']\n","Decoded tokens: ['occurence', 'chapter', 'diagrammatically', 'esiccate', 'olans']\n","Target tokens:  ['occurence', 'chapter', 'diagrammatically', 'desiccate', 'loans']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_73.h5\n","Main Epoch 74/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0395 - accuracy: 0.9886 - truncated_acc: 0.9688 - truncated_loss: 0.1087 - val_loss: 0.0279 - val_accuracy: 0.9918 - val_truncated_acc: 0.9775 - val_truncated_loss: 0.0767\n","-\n","Input tokens:   ['soVmone', 'cmoittee', 'dessiccate', 'voteinf', 'rmine']\n","Decoded tokens: ['somone', 'comittee', 'dessiccate', 'voteing', 'remine']\n","Target tokens:  ['somone', 'comittee', 'dessiccate', 'voteing', 'remine']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_74.h5\n","Main Epoch 75/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0400 - accuracy: 0.9885 - truncated_acc: 0.9684 - truncated_loss: 0.1100 - val_loss: 0.0277 - val_accuracy: 0.9918 - val_truncated_acc: 0.9775 - val_truncated_loss: 0.0761\n","-\n","Input tokens:   ['poarty', 'transportibiilty', 'initial', 'specivl', 'vistors']\n","Decoded tokens: ['poarty', 'transportibility', 'initial', 'specivl', 'vistors']\n","Target tokens:  ['poartry', 'transportibility', 'initial', 'special', 'vistors']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_75.h5\n","Main Epoch 76/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0393 - accuracy: 0.9886 - truncated_acc: 0.9688 - truncated_loss: 0.1080 - val_loss: 0.0280 - val_accuracy: 0.9915 - val_truncated_acc: 0.9766 - val_truncated_loss: 0.0769\n","-\n","Input tokens:   ['particulr', 'accssing', 'rermember', 'difinately', 'occurenec']\n","Decoded tokens: ['particular', 'accussing', 'rermember', 'difinately', 'occurence']\n","Target tokens:  ['particular', 'accessing', 'rermember', 'difinately', 'occurence']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_76.h5\n","Main Epoch 77/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0392 - accuracy: 0.9887 - truncated_acc: 0.9689 - truncated_loss: 0.1079 - val_loss: 0.0280 - val_accuracy: 0.9905 - val_truncated_acc: 0.9739 - val_truncated_loss: 0.0769\n","-\n","Input tokens:   ['prolam', 'necepary', 'Pcourtens', 'defenition', 'cerain']\n","Decoded tokens: ['prollam', 'necepary', 'courtens', 'defenition', 'certain']\n","Target tokens:  ['problam', 'necesary', 'courtens', 'defenition', 'certain']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_77.h5\n","Main Epoch 78/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0394 - accuracy: 0.9887 - truncated_acc: 0.9688 - truncated_loss: 0.1084 - val_loss: 0.0278 - val_accuracy: 0.9908 - val_truncated_acc: 0.9746 - val_truncated_loss: 0.0765\n","-\n","Input tokens:   ['Kminuscule', 'reafreshemnt', 'bicycal', 'vTluable', 'specivl']\n","Decoded tokens: ['minuscule', 'reafreshment', 'bicycal', 'valuable', 'specivl']\n","Target tokens:  ['minuscule', 'reafreshment', 'bicycal', 'valuable', 'special']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_78.h5\n","Main Epoch 79/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0394 - accuracy: 0.9886 - truncated_acc: 0.9687 - truncated_loss: 0.1083 - val_loss: 0.0277 - val_accuracy: 0.9917 - val_truncated_acc: 0.9771 - val_truncated_loss: 0.0763\n","-\n","Input tokens:   ['chzlenges', 'familes', 'magnifisant', 'decidde', 'votign']\n","Decoded tokens: ['chalenges', 'familes', 'magnifisant', 'decided', 'voting']\n","Target tokens:  ['chalenges', 'families', 'magnifiscant', 'decided', 'voting']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_79.h5\n","Main Epoch 80/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0390 - accuracy: 0.9888 - truncated_acc: 0.9692 - truncated_loss: 0.1072 - val_loss: 0.0276 - val_accuracy: 0.9918 - val_truncated_acc: 0.9775 - val_truncated_loss: 0.0759\n","-\n","Input tokens:   ['vTluable', 'particulr', 'refreshment', 'rmine', 'initils']\n","Decoded tokens: ['valuable', 'particular', 'refreshment', 'rimen', 'initials']\n","Target tokens:  ['valuable', 'particular', 'refreshment', 'remine', 'initails']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_80.h5\n","Main Epoch 81/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0387 - accuracy: 0.9889 - truncated_acc: 0.9694 - truncated_loss: 0.1065 - val_loss: 0.0278 - val_accuracy: 0.9916 - val_truncated_acc: 0.9768 - val_truncated_loss: 0.0765\n","-\n","Input tokens:   ['fisYted', 'vTluable', 'evxtended', 'diffreent', 'beginnin']\n","Decoded tokens: ['fisted', 'valuable', 'extended', 'different', 'beginning']\n","Target tokens:  ['fisited', 'valuable', 'extended', 'different', 'beginning']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_81.h5\n","Main Epoch 82/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0388 - accuracy: 0.9888 - truncated_acc: 0.9693 - truncated_loss: 0.1067 - val_loss: 0.0286 - val_accuracy: 0.9909 - val_truncated_acc: 0.9749 - val_truncated_loss: 0.0786\n","-\n","Input tokens:   ['lteval', ' ycycle', 'necassuary', 'definitely', 'establilshing']\n","Decoded tokens: ['letal', 'bycycle', 'necassurary', 'definitely', 'establishing']\n","Target tokens:  ['leval', 'bycycle', 'necassary', 'definitely', 'establishing']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_82.h5\n","Main Epoch 83/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0386 - accuracy: 0.9888 - truncated_acc: 0.9693 - truncated_loss: 0.1062 - val_loss: 0.0280 - val_accuracy: 0.9925 - val_truncated_acc: 0.9795 - val_truncated_loss: 0.0771\n","-\n","Input tokens:   ['pemetery', 'consider', 'juisye', 'stomche', 'mgnificnet']\n","Decoded tokens: ['pemetery', 'consider', 'juisey', 'stomche', 'mgnificent']\n","Target tokens:  ['cemetery', 'consider', 'juise', 'stomache', 'magnificnet']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_83.h5\n","Main Epoch 84/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0382 - accuracy: 0.9890 - truncated_acc: 0.9698 - truncated_loss: 0.1049 - val_loss: 0.0261 - val_accuracy: 0.9915 - val_truncated_acc: 0.9766 - val_truncated_loss: 0.0718\n","-\n","Input tokens:   ['pylanned', 'beniDfit', 'aEwful', 'reciet', 'desription']\n","Decoded tokens: ['planned', 'benifit', 'awful', 'reciet', 'desription']\n","Target tokens:  ['planned', 'benifit', 'awful', 'reciet', 'description']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_84.h5\n","Main Epoch 85/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0383 - accuracy: 0.9889 - truncated_acc: 0.9695 - truncated_loss: 0.1054 - val_loss: 0.0275 - val_accuracy: 0.9909 - val_truncated_acc: 0.9749 - val_truncated_loss: 0.0757\n","-\n","Input tokens:   ['neccesary', 'magnificFant', 'exrteamly', 'pronuncciation', 'rtansfred']\n","Decoded tokens: ['neccesary', 'magnificant', 'extrealy', 'pronunciation', 'transfred']\n","Target tokens:  ['neccesary', 'magnificant', 'extreamly', 'pronunciation', 'transfred']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_85.h5\n","Main Epoch 86/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0384 - accuracy: 0.9889 - truncated_acc: 0.9694 - truncated_loss: 0.1057 - val_loss: 0.0282 - val_accuracy: 0.9911 - val_truncated_acc: 0.9756 - val_truncated_loss: 0.0776\n","-\n","Input tokens:   ['sotmec', 'jucie', 'votign', 'latiset', 'caZe']\n","Decoded tokens: ['stomec', 'jucie', 'voting', 'latist', 'care']\n","Target tokens:  ['stomec', 'jucie', 'voting', 'latiest', 'cake']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_86.h5\n","Main Epoch 87/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0377 - accuracy: 0.9891 - truncated_acc: 0.9701 - truncated_loss: 0.1037 - val_loss: 0.0259 - val_accuracy: 0.9920 - val_truncated_acc: 0.9780 - val_truncated_loss: 0.0711\n","-\n","Input tokens:   ['Oxperience', 'Oxperience', 'articals', 'latiset', 'dessiccate']\n","Decoded tokens: ['Experience', 'Experience', 'articals', 'latiest', 'dessiccate']\n","Target tokens:  ['experience', 'experience', 'articals', 'latiest', 'dessiccate']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_87.h5\n","Main Epoch 88/100\n","Shuffling data.\n","561/561 [==============================] - 183s 326ms/step - loss: 0.0379 - accuracy: 0.9891 - truncated_acc: 0.9699 - truncated_loss: 0.1041 - val_loss: 0.0266 - val_accuracy: 0.9917 - val_truncated_acc: 0.9773 - val_truncated_loss: 0.0731\n","-\n","Input tokens:   ['remmeber', 'apralell', 'reafreshemnt', 'magificnet', 'vistors']\n","Decoded tokens: ['remember', 'paralell', 'reafreshment', 'magificent', 'vistors']\n","Target tokens:  ['remember', 'paralell', 'reafreshment', 'magificent', 'vistors']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_88.h5\n","Main Epoch 89/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0377 - accuracy: 0.9891 - truncated_acc: 0.9700 - truncated_loss: 0.1037 - val_loss: 0.0272 - val_accuracy: 0.9916 - val_truncated_acc: 0.9768 - val_truncated_loss: 0.0747\n","-\n","Input tokens:   ['cotnented', 'gallrYy', 'expreances', 'esiccate', 'scarcyl']\n","Decoded tokens: ['contented', 'gallry', 'expereances', 'esiccate', 'scarcly']\n","Target tokens:  ['contented', 'gallrey', 'experances', 'desiccate', 'scarcly']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_89.h5\n","Main Epoch 90/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0377 - accuracy: 0.9892 - truncated_acc: 0.9702 - truncated_loss: 0.1036 - val_loss: 0.0268 - val_accuracy: 0.9914 - val_truncated_acc: 0.9763 - val_truncated_loss: 0.0737\n","-\n","Input tokens:   ['useqfull', 'moniteing', 'conciderabye', 'falies', 'rant']\n","Decoded tokens: ['usefull', 'moniteing', 'conciderable', 'falies', 'rant']\n","Target tokens:  ['usefull', 'monitering', 'conciderable', 'failes', 'arnt']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_90.h5\n","Main Epoch 91/100\n","Shuffling data.\n","561/561 [==============================] - 185s 329ms/step - loss: 0.0378 - accuracy: 0.9891 - truncated_acc: 0.9699 - truncated_loss: 0.1040 - val_loss: 0.0271 - val_accuracy: 0.9915 - val_truncated_acc: 0.9766 - val_truncated_loss: 0.0746\n","-\n","Input tokens:   ['acoomdation', 'undesrand', 'superecde', 'exsacy', 'promlbem']\n","Decoded tokens: ['acomodation', 'undersrand', 'supercede', 'exsacy', 'promblem']\n","Target tokens:  ['acomodation', 'undersand', 'supercede', 'exstacy', 'promblem']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_91.h5\n","Main Epoch 92/100\n","Shuffling data.\n","561/561 [==============================] - 184s 329ms/step - loss: 0.0371 - accuracy: 0.9893 - truncated_acc: 0.9705 - truncated_loss: 0.1021 - val_loss: 0.0274 - val_accuracy: 0.9922 - val_truncated_acc: 0.9785 - val_truncated_loss: 0.0753\n","-\n","Input tokens:   ['refreshment', 'questponaire', 'pronuncciation', 'Ktanerdizing', 'acount']\n","Decoded tokens: ['refreshment', 'questionaire', 'pronunciation', 'tanerdizing', 'acount']\n","Target tokens:  ['refreshment', 'questionaire', 'pronunciation', 'stanerdizing', 'acount']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_92.h5\n","Main Epoch 93/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0374 - accuracy: 0.9892 - truncated_acc: 0.9702 - truncated_loss: 0.1028 - val_loss: 0.0267 - val_accuracy: 0.9919 - val_truncated_acc: 0.9778 - val_truncated_loss: 0.0734\n","-\n","Input tokens:   ['possalbe', 'parlalel', 'miniscule', 'hieracihal', 'receiMt']\n","Decoded tokens: ['possable', 'parallel', 'miniscule', 'hierachial', 'receit']\n","Target tokens:  ['possable', 'parallel', 'miniscule', 'hierachial', 'receit']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_93.h5\n","Main Epoch 94/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0374 - accuracy: 0.9892 - truncated_acc: 0.9702 - truncated_loss: 0.1027 - val_loss: 0.0260 - val_accuracy: 0.9921 - val_truncated_acc: 0.9783 - val_truncated_loss: 0.0716\n","-\n","Input tokens:   ['parralell', 'hieracihal', 'bciycle', 'unexpeted', 'magnificFant']\n","Decoded tokens: ['parralell', 'hierachial', 'bicycle', 'unexpected', 'magnificiant']\n","Target tokens:  ['parrallel', 'hierachial', 'bicycle', 'unexpeted', 'magnificant']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_94.h5\n","Main Epoch 95/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0367 - accuracy: 0.9893 - truncated_acc: 0.9706 - truncated_loss: 0.1009 - val_loss: 0.0263 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0724\n","-\n","Input tokens:   ['aEwful', 'ecntrally', 'diffren', 'socres', 'curtiEns']\n","Decoded tokens: ['aweful', 'centrally', 'differn', 'socres', 'curtins']\n","Target tokens:  ['awful', 'centrally', 'diffrent', 'sorces', 'curtions']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_95.h5\n","Main Epoch 96/100\n","Shuffling data.\n","561/561 [==============================] - 184s 328ms/step - loss: 0.0372 - accuracy: 0.9893 - truncated_acc: 0.9706 - truncated_loss: 0.1022 - val_loss: 0.0261 - val_accuracy: 0.9927 - val_truncated_acc: 0.9800 - val_truncated_loss: 0.0718\n","-\n","Input tokens:   ['scacrely', 'vTluable', 'chzlenges', 'cartains', 'prtiend']\n","Decoded tokens: ['scarcely', 'valuable', 'chalenges', 'cartains', 'pritend']\n","Target tokens:  ['scarcely', 'valuable', 'chalenges', 'cartains', 'pritend']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_96.h5\n","Main Epoch 97/100\n","Shuffling data.\n","561/561 [==============================] - 186s 330ms/step - loss: 0.0369 - accuracy: 0.9893 - truncated_acc: 0.9706 - truncated_loss: 0.1014 - val_loss: 0.0261 - val_accuracy: 0.9925 - val_truncated_acc: 0.9792 - val_truncated_loss: 0.0717\n","-\n","Input tokens:   ['ineRtials', 'cerain', 'evxtended', 'pkonounciation', 'monitoring']\n","Decoded tokens: ['inetials', 'certain', 'extended', 'pronounciation', 'monitoring']\n","Target tokens:  ['inetials', 'certain', 'extended', 'pronounciation', 'monitoring']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_97.h5\n","Main Epoch 98/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0368 - accuracy: 0.9894 - truncated_acc: 0.9707 - truncated_loss: 0.1012 - val_loss: 0.0265 - val_accuracy: 0.9917 - val_truncated_acc: 0.9771 - val_truncated_loss: 0.0728\n","-\n","Input tokens:   ['often', 'rzecipt', 'pslendid', 'spleneq', 'scarPely']\n","Decoded tokens: ['often', 'recipt', 'splendid', 'splened', 'scarely']\n","Target tokens:  ['offen', 'recipt', 'splendid', 'splened', 'scarely']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_98.h5\n","Main Epoch 99/100\n","Shuffling data.\n","561/561 [==============================] - 186s 331ms/step - loss: 0.0366 - accuracy: 0.9894 - truncated_acc: 0.9709 - truncated_loss: 0.1007 - val_loss: 0.0258 - val_accuracy: 0.9916 - val_truncated_acc: 0.9768 - val_truncated_loss: 0.0710\n","-\n","Input tokens:   ['preblem', 'moniteing', 'ecstwsy', 'further', 'lugY']\n","Decoded tokens: ['preblem', 'moniteing', 'ecstasy', 'further', 'lugh']\n","Target tokens:  ['problem', 'monitering', 'ecstasy', 'further', 'lugh']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_99.h5\n","Main Epoch 100/100\n","Shuffling data.\n","561/561 [==============================] - 185s 330ms/step - loss: 0.0369 - accuracy: 0.9893 - truncated_acc: 0.9707 - truncated_loss: 0.1014 - val_loss: 0.0261 - val_accuracy: 0.9910 - val_truncated_acc: 0.9753 - val_truncated_loss: 0.0717\n","-\n","Input tokens:   ['uhfortunatly', 'emnt', 'addressable', 'reYley', 'desscate']\n","Decoded tokens: ['unfortunatly', 'ment', 'addressable', 'reley', 'dessicate']\n","Target tokens:  ['unfortunatly', 'ment', 'addressable', 'relley', 'dessicate']\n","-\n","Saving full model to checkpoints/seq2seq_epoch_100.h5\n"]}],"source":["# Train and evaluate.\n","for epoch in range(nb_epochs):\n","    print('Main Epoch {:d}/{:d}'.format(epoch + 1, nb_epochs))\n","    \n","    train_encoder, train_decoder, train_target = transform(\n","            vocab, maxlen, error_rate=error_rate, shuffle=True)\n","        \n","    train_encoder_batch = batch(train_encoder, maxlen, input_ctable,\n","                                    train_batch_size, reverse)\n","    train_decoder_batch = batch(train_decoder, maxlen, target_ctable,\n","                                    train_batch_size)\n","    train_target_batch  = batch(train_target, maxlen, target_ctable,\n","                                    train_batch_size)    \n","\n","    val_encoder_batch = batch(val_encoder, maxlen, input_ctable,\n","                                  val_batch_size, reverse)\n","    val_decoder_batch = batch(val_decoder, maxlen, target_ctable,\n","                                  val_batch_size)\n","    val_target_batch  = batch(val_target, maxlen, target_ctable,\n","                                  val_batch_size)\n","    \n","    train_loader = datagen(train_encoder_batch,\n","                               train_decoder_batch, train_target_batch)\n","    val_loader = datagen(val_encoder_batch,\n","                             val_decoder_batch, val_target_batch)\n","    \n","    model.fit_generator(train_loader,\n","                        steps_per_epoch=train_steps,\n","                        epochs=1, verbose=1,\n","                        validation_data=val_loader,\n","                        validation_steps=val_steps)\n","\n","    # On epoch end - decode a batch of misspelled tokens from the\n","    # validation set to visualize speller performance.\n","    nb_tokens = 5\n","    input_tokens, target_tokens, decoded_tokens = decode_sequences(\n","        val_encoder, val_target, input_ctable, target_ctable,\n","        maxlen, reverse, encoder_model, decoder_model, nb_tokens,\n","        sample_mode=sample_mode, random=True)\n","        \n","    print('-')\n","    print('Input tokens:  ', input_tokens)\n","    print('Decoded tokens:', decoded_tokens)\n","    print('Target tokens: ', target_tokens)\n","    print('-')\n","        \n","    # Save the model at end of each epoch.\n","    model_file = '_'.join(['seq2seq', 'epoch', str(epoch + 1)]) + '.h5'\n","    save_dir = 'checkpoints'\n","    if not os.path.exists(save_dir):\n","        os.makedirs(save_dir)\n","    save_path = os.path.join(save_dir, model_file)\n","    print('Saving full model to {:s}'.format(save_path))\n","    model.save(save_path)"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}