IST597_HW3_Custom.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ZhaoQii/IST597/blob/main/IST597_HW3_Custom.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0XqMPv2q8ava",
        "outputId": "21ee6882-0424-431c-81d0-561fef61e480"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.8.0\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "print(tf.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i3z58S2Jx_Ay",
        "outputId": "a9239284-15dd-4fc7-83e5-80467f06bc71"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "tf.config.list_physical_devices('GPU')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A_8cYhDDIuCw"
      },
      "outputs": [],
      "source": [
        "#np.random.seed(1178)\n",
        "#tf.random.set_seed(1178)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eRVHzPqnZRrW",
        "outputId": "79a60393-5bae-4675-975c-9f086b3e5f09"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "40960/29515 [=========================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "26435584/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "16384/5148 [===============================================================================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n",
            "4431872/4422102 [==============================] - 0s 0us/step\n"
          ]
        }
      ],
      "source": [
        "from tensorflow.keras.datasets import mnist, fashion_mnist\n",
        "from sklearn import preprocessing\n",
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "(X_train_fs, y_train_fs), (X_test_fs, y_test_fs) = fashion_mnist.load_data()\n",
        "\n",
        "# first way to normalize data\n",
        "X_train = X_train.astype(float) / 255.\n",
        "X_test = X_test.astype(float) / 255.\n",
        "\n",
        "X_train = X_train.reshape([X_train.shape[0], -1])\n",
        "X_test = X_test.reshape([X_test.shape[0], -1])\n",
        "\n",
        "X_train_fs = X_train_fs.astype(float) / 255.\n",
        "X_test_fs = X_test_fs.astype(float) / 255.\n",
        "\n",
        "X_train_fs = X_train_fs.reshape([X_train_fs.shape[0], -1])\n",
        "X_test_fs = X_test_fs.reshape([X_test_fs.shape[0], -1])\n",
        "\n",
        "# one hot\n",
        "onehot = preprocessing.OneHotEncoder()\n",
        "\n",
        "y_train = y_train.reshape(-1, 1)\n",
        "y_test = y_test.reshape(-1, 1)\n",
        "\n",
        "onehot.fit(y_train)\n",
        "y_train = onehot.transform(y_train).toarray()\n",
        "onehot.fit(y_test)\n",
        "y_test = onehot.transform(y_test).toarray()\n",
        "\n",
        "y_train_fs = y_train_fs.reshape(-1, 1)\n",
        "y_test_fs = y_test_fs.reshape(-1, 1)\n",
        "\n",
        "onehot.fit(y_train_fs)\n",
        "y_train_fs = onehot.transform(y_train_fs).toarray()\n",
        "onehot.fit(y_test_fs)\n",
        "y_test_fs = onehot.transform(y_test_fs).toarray()\n",
        "\n",
        "# validation set\n",
        "X_train, X_val = X_train[:-10000], X_train[-10000:]\n",
        "y_train, y_val = y_train[:-10000], y_train[-10000:]\n",
        "\n",
        "X_train_fs, X_val_fs = X_train_fs[:-10000], X_train_fs[-10000:]\n",
        "y_train_fs, y_val_fs = y_train_fs[:-10000], y_train_fs[-10000:]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "70dqw3yaMTkK"
      },
      "source": [
        "## Define Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5ifDRhxKmQDY"
      },
      "outputs": [],
      "source": [
        "# Define class to build mlp model\n",
        "class MLP_QZ_Custom(object):\n",
        "  def __init__(self, size_data, size_input, size_hidden_1, size_hidden_2, size_output, device = None, regularizer = None,\n",
        "               L1_alpha = 0., L2_alpha = 0.,\n",
        "               alpha = 1e-2, beta_1 = 0.9, beta_2 = 0.999, beta_3 = 0.999987, e = 1e-8, e_2 = 1e-6):\n",
        "    \"\"\"\n",
        "    size_input: int, size of input layer\n",
        "    size_hidden: int, size of hidden layer\n",
        "    size_output: int, size of output layer\n",
        "    device: str or None, either 'cpu' or 'gpu' or None. If None, the device to be used will be decided automatically during Eager Execution\n",
        "    \"\"\"\n",
        "    self.size_data, self.size_input, self.size_hidden_1, self.size_hidden_2, self.size_output, self.device, self.regularizer, \\\n",
        "    self.L1_alpha, self.L2_alpha, self.alpha, self.beta_1, self.beta_2, self.beta_3, self.e, self.e_2 = \\\n",
        "    size_data, size_input, size_hidden_1, size_hidden_2, size_output, device, regularizer, L1_alpha, L2_alpha, alpha, beta_1, beta_2, beta_3, e, e_2\n",
        "    \n",
        "    # Xavier initialization\n",
        "    initializer = tf.initializers.GlorotUniform()\n",
        "\n",
        "    self.W0 = tf.Variable(initializer(shape = [self.size_data, self.size_input]))\n",
        "    #self.W0 = tf.Variable(tf.random.normal([self.size_data, self.size_input], stddev = 0.1))\n",
        "    self.b0 = tf.Variable(tf.zeros([1, self.size_input]))\n",
        "\n",
        "    self.W1 = tf.Variable(initializer(shape = [self.size_input, self.size_hidden_1]))\n",
        "    #self.W1 = tf.Variable(tf.random.normal([self.size_input, self.size_hidden_1], stddev = 0.1))\n",
        "    self.b1 = tf.Variable(tf.zeros([1, self.size_hidden_1]))\n",
        "\n",
        "    self.W2 = tf.Variable(initializer(shape = [self.size_hidden_1, self.size_hidden_2]))\n",
        "    #self.W2 = tf.Variable(tf.random.normal([self.size_hidden_1, self.size_hidden_2], stddev = 0.1))\n",
        "    self.b2 = tf.Variable(tf.zeros([1, self.size_hidden_2]))\n",
        "\n",
        "    self.W3 = tf.Variable(initializer(shape = [self.size_hidden_2, self.size_output]))\n",
        "    #self.W3 = tf.Variable(tf.random.normal([self.size_hidden_2, self.size_output], stddev = 0.1))\n",
        "    self.b3 = tf.Variable(tf.zeros([1, self.size_output]))\n",
        "    \n",
        "    # Define variables to be updated during backpropagation\n",
        "    self.variables = [self.W0, self.W1, self.W2, self.W3, self.b0, self.b1, self.b2, self.b3]\n",
        "    \n",
        "  def forward(self, X):\n",
        "    \"\"\"\n",
        "    forward pass\n",
        "    X: Tensor, inputs\n",
        "    \"\"\"\n",
        "    if self.device is not None:\n",
        "      with tf.device('gpu:0' if self.device=='gpu' else 'cpu'):\n",
        "        self.y = self.compute_output(X)\n",
        "    else:\n",
        "      self.y = self.compute_output(X)\n",
        "      \n",
        "    return self.y\n",
        "  \n",
        "  def loss(self, y_pred, y_true):\n",
        "    '''\n",
        "    y_pred - Tensor of shape (batch_size, size_output)\n",
        "    y_true - Tensor of shape (batch_size, size_output)\n",
        "    '''\n",
        "    y_true_tf = tf.cast(tf.reshape(y_true, (-1, self.size_output)), dtype=tf.float32)\n",
        "    y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "\n",
        "    #return tf.losses.categorical_crossentropy(y_true_tf, y_pred_tf, from_logits = False)\n",
        "    ce = tf.keras.losses.CategoricalCrossentropy(from_logits = True)\n",
        "\n",
        "    if self.regularizer is None:\n",
        "      return ce(y_true_tf, y_pred_tf)\n",
        "    elif self.regularizer == 'L1':\n",
        "      return (ce(y_true_tf, y_pred_tf) + self.L1_alpha * sum([tf.reduce_sum(tf.abs(var)) for var in self.variables]))\n",
        "    elif self.regularizer == 'L2':\n",
        "      return (ce(y_true_tf, y_pred_tf) + self.L2_alpha * sum([tf.nn.l2_loss(var) for var in self.variables]))\n",
        "    elif self.regularizer == 'L1 + L2':\n",
        "      return (ce(y_true_tf, y_pred_tf) + self.L1_alpha * sum([tf.reduce_sum(tf.abs(var)) for var in self.variables]) + self.L2_alpha * sum([tf.nn.l2_loss(var) for var in self.variables]))\n",
        "  \n",
        "  def backward(self, X_train, y_train, t, m, v, u):\n",
        "    \"\"\"\n",
        "    backward pass\n",
        "    \"\"\"\n",
        "\n",
        "    with tf.GradientTape() as tape:\n",
        "      predicted = self.forward(X_train)\n",
        "      current_loss = self.loss(predicted, y_train)\n",
        "    grads = tape.gradient(current_loss, self.variables)\n",
        "\n",
        "    # update\n",
        "    m = [self.beta_1 * a + (1 - self.beta_1) * b for a, b in zip(m, grads)]\n",
        "    v = [self.beta_2 * a + (1 - self.beta_2) * tf.math.square(b) for a, b in zip(v, grads)]\n",
        "    u = [self.beta_3 * a + (1 - self.beta_3) * tf.math.pow(b, 3) for a, b in zip(u, grads)]\n",
        "    #print(m[0][0])\n",
        "    m_hat = [i / (1 - self.beta_1 ** t) for i in m]\n",
        "    v_hat = [i / (1 - self.beta_2 ** t) for i in v]\n",
        "    u_hat = [i / (1 - self.beta_3 ** t) for i in u]\n",
        "\n",
        "    theta = [a - self.alpha * b / (np.sqrt(c)+ self.e * np.cbrt(d) + self.e_2) for a, b, c, d in zip(self.variables, m_hat, v_hat, u_hat)]\n",
        "    \n",
        "    # assign values to variables\n",
        "    for i in range(len(self.variables)):\n",
        "      self.variables[i].assign(theta[i])\n",
        "    return(m, v, u)\n",
        "        \n",
        "  def compute_output(self, X):\n",
        "    \"\"\"\n",
        "    Custom method to obtain output tensor during forward pass\n",
        "    \"\"\"\n",
        "    # Cast X to float32\n",
        "    X_tf = tf.cast(X, dtype=tf.float32)    \n",
        "    # Compute values in the first hidden layer\n",
        "    what_0 = tf.matmul(X_tf, self.W0) + self.b0\n",
        "    hhat_0 = tf.nn.relu(what_0)\n",
        "    # Compute values in the first hidden layer\n",
        "    what_1 = tf.matmul(hhat_0, self.W1) + self.b1\n",
        "    hhat_1 = tf.nn.relu(what_1)\n",
        "    # Compute values in the second hidden layer\n",
        "    what_2 = tf.matmul(hhat_1, self.W2) + self.b2\n",
        "    hhat_2 = tf.nn.relu(what_2)\n",
        "    # Compute output\n",
        "    output = tf.matmul(hhat_2, self.W3) + self.b3\n",
        "    #output = tf.nn.softmax(what_3)\n",
        "    return output"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2QilpF4VMRDC"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RPdjUX44UgEM"
      },
      "outputs": [],
      "source": [
        "size_data, size_input, size_hidden_1, size_hidden_2, size_output = 28 * 28, 512, 256, 128, 10 \n",
        "batch_size_train = 128\n",
        "batch_size_val = 16\n",
        "batch_size_test = 16"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W3vMk6gQKaRF"
      },
      "outputs": [],
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 5\n",
        "\n",
        "# Initialize model\n",
        "# , regularizer = 'L2', L1_alpha = 1 , regularizer = 'L2', L1_alpha = 1 , device='gpu' optimizer = 'Adam', \n",
        "mlp_on_cpu = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output, regularizer = 'L2', L2_alpha = 0.02)\n",
        "mlp_on_cpu_fs = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output, regularizer = 'L2', L2_alpha = 0.02)\n",
        "\n",
        "# column 0 for MINST column 1 for Fashion MNIST\n",
        "train_accuracy = np.zeros([NUM_EPOCHS, 2])\n",
        "val_accuracy = np.zeros([NUM_EPOCHS, 2])\n",
        "train_loss = np.zeros([NUM_EPOCHS, 2])\n",
        "\n",
        "time_start = time.time()\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "  # for MNIST\n",
        "  train_loss_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  train_acc_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  #val_loss_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  val_acc_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "\n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(100).batch(batch_size_train)\n",
        "\n",
        "  t = 0\n",
        "  m = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "  v = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "  u = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "\n",
        "  for inputs, outputs in train_ds:\n",
        "    t += 1\n",
        "    preds = mlp_on_cpu.forward(inputs)\n",
        "    train_loss_epoch = train_loss_epoch + mlp_on_cpu.loss(preds, outputs)\n",
        "    m, v, u = mlp_on_cpu.backward(inputs, outputs, t, m, v, u)\n",
        "\n",
        "  # training acc and loss\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  train_acc_epoch = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  train_accuracy[epoch, 0] = train_acc_epoch\n",
        "  train_loss_epoch = np.sum(train_loss_epoch) / X_train.shape[0]\n",
        "  train_loss[epoch, 0] = train_loss_epoch\n",
        "  print('\\nMNIST Train Accuracy: {:.4f}'.format(train_acc_epoch))\n",
        "  print('\\nMNIST Number of Epoch = {} - Average Cross Entropy:={}'.format(epoch + 1, train_loss_epoch))\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_val)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_val, 1))\n",
        "  val_acc_epoch = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  val_acc_epoch = val_acc_epoch.numpy()\n",
        "  val_accuracy[epoch, 0] = val_acc_epoch\n",
        "  print('\\nMNIST Validation Accuracy: {:.4f}'.format(val_acc_epoch))\n",
        "\n",
        "  # for FMNIST\n",
        "  #train_loss_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  train_acc_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  #val_loss_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "  val_acc_epoch = tf.zeros([1,1], dtype=tf.float32)\n",
        "\n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train_fs, y_train_fs)).shuffle(100).batch(batch_size_train)\n",
        "\n",
        "  t = 0\n",
        "  m = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "  v = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "  u = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "\n",
        "  for inputs, outputs in train_ds:\n",
        "    t += 1\n",
        "    preds = mlp_on_cpu_fs.forward(inputs)\n",
        "    train_loss_epoch = train_loss_epoch + mlp_on_cpu_fs.loss(preds, outputs)\n",
        "    m, v, u = mlp_on_cpu_fs.backward(inputs, outputs, t, m, v, u)\n",
        "\n",
        "  # training acc and loss\n",
        "  preds = mlp_on_cpu_fs.forward(X_train_fs)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train_fs, 1))\n",
        "  train_acc_epoch = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  train_accuracy[epoch, 1] = train_acc_epoch\n",
        "  train_loss_epoch = np.sum(train_loss_epoch) / X_train.shape[0]\n",
        "  train_loss[epoch, 1] = train_loss_epoch\n",
        "  print('\\nFashion MNIST Train Accuracy: {:.4f}'.format(train_acc_epoch))\n",
        "  print('\\nFashion MNIST Number of Epoch = {} - Average Cross Entropy:={}'.format(epoch + 1, train_loss_epoch))\n",
        "\n",
        "  preds = mlp_on_cpu_fs.forward(X_val_fs)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_val_fs, 1))\n",
        "  val_acc_epoch = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  val_acc_epoch = val_acc_epoch.numpy()\n",
        "  val_accuracy[epoch, 1] = val_acc_epoch\n",
        "  print('\\nFashion MNIST Validation Accuracy: {:.4f}'.format(val_acc_epoch))\n",
        "\n",
        "\n",
        "time_taken = time.time() - time_start\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "gk200weANlKo",
        "outputId": "e900c7c4-01a5-4f9c-8336-54882c2c9f65"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Epochs')"
            ]
          },
          "metadata": {},
          "execution_count": 26
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUdfrA8c+TDkloSaihCqLUACEq6snZDmx4CAqigli5O+t5np5X1KuWO8vZK1hOQNDfYTm9s6KiIirSUcQgkRZCSShJSPL8/phJsqTuht3MbvZ5v177cnbmO7NPRrJPvvOd5zuiqhhjjDH+ivE6AGOMMZHFEocxxpiAWOIwxhgTEEscxhhjAmKJwxhjTEAscRhjjAmIJQ7jGRH5j4hMDXZbL4lIroicHILjvicil7rLU0Tkv/60bcLn9BCRPSIS29RYTctnicMExP1SqXxViMh+n/dTAjmWqo5V1VnBbhuOROQmEVlYx/p0ESkVkUH+HktVn1fVU4MU10GJTlW/V9UUVS0PxvFrfJaKSN9gH9c0P0scJiDul0qKqqYA3wNn+qx7vrKdiMR5F2VYeg4YJSK9a6yfBCxX1RUexGRMk1jiMEEhIqNFJE9Efi0iW4CnRaS9iLwqIvkistNdzvTZx/fyyzQR+VBE7nbbficiY5vYtreILBSRIhF5S0QeFJHn6onbnxj/KCIfucf7r4ik+2y/UEQ2iEiBiNxS3/lR1TzgHeDCGpsuAp5pLI4aMU8TkQ993p8iImtEZLeIPACIz7bDROQdN77tIvK8iLRztz0L9ABecXuMN4pIL7dnEOe26SoiC0Rkh4isE5HLfI59q4jMFZFn3HOzUkSy6zsH9RGRtu4x8t1z+VsRiXG39RWR992fbbuIzHHXi4jcIyLbRKRQRJYH0mszh8YShwmmzkAHoCdwOc6/r6fd9z2A/cADDex/FLAWSAfuBJ4UEWlC238Bi4E04FZqf1n78ifG84GLgY5AAnADgIgMAB52j9/V/bw6v+xds3xjEZH+QJYbb6DnqvIY6cBLwG9xzsW3wLG+TYC/uvEdCXTHOSeo6oUc3Gu8s46PmA3kuftPAP4iIif6bD/LbdMOWOBPzHX4J9AW6AOcgJNML3a3/RH4L9Ae59z+011/KvAj4HB333OBgiZ8tmkKVbWXvZr0AnKBk93l0UApkNRA+yxgp8/794BL3eVpwDqfba0BBToH0hbnS7cMaO2z/TngOT9/prpi/K3P+58Bb7jLvwdm+2xLds/ByfUcuzVQCIxy3/8Z+HcTz9WH7vJFwCc+7QTni/7Seo57NvBlXf8P3fe93HMZh5NkyoFUn+1/BWa6y7cCb/lsGwDsb+DcKtC3xrpY95wN8Fl3BfCeu/wM8BiQWWO/E4GvgaOBGK9/F6LtZT0OE0z5qlpc+UZEWovIo+7lh0JgIdBO6r9jZ0vlgqrucxdTAmzbFdjhsw5gY30B+xnjFp/lfT4xdfU9tqrupYG/et2YXgQucntHU3C+GJtyrirVjEF934tIJxGZLSI/uMd9Dqdn4o/Kc1nks24D0M3nfc1zkySBjW+lA/Hucev6jBtxkuFi91LYdABVfQend/MgsE1EHhORNgF8rjkEljhMMNWcavmXQH/gKFVtg3NpAXyuwYfAZqCDiLT2Wde9gfaHEuNm32O7n5nWyD6zcC6rnAKkAq8cYhw1YxAO/nn/gvP/ZbB73AtqHLOh6bE34ZzLVJ91PYAfGokpENuBAziX6Gp9hqpuUdXLVLUrTk/kIXHvzFLV+1V1BE5P53DgV0GMyzTAEocJpVSca/W7RKQD8IdQf6CqbgCWALeKSIKIHAOcGaIY5wFniMhxIpIA3E7jv1MfALtwLr/MVtXSQ4zjNWCgiIx3/9K/GueSXaVUYA+wW0S6UfvLdSvO2EItqroRWAT8VUSSRGQIcAlOr6WpEtxjJYlIkrtuLvBnEUkVkZ7A9ZWfISITfW4S2ImT6CpEZKSIHCUi8cBeoBioOIS4TAAscZhQuhdohfNX5SfAG830uVOAY3AuG/0JmAOU1NO2yTGq6krg5ziD25txvtjyGtlHcS5P9XT/e0hxqOp2YCLwN5yftx/wkU+T24DhwG6cJPNSjUP8FfitiOwSkRvq+IjJOOMem4CXgT+o6lv+xFaPlTgJsvJ1MXAVzpf/euBDnPP5lNt+JPCpiOzBGXy/RlXXA22Ax3HO+Qacn/2uQ4jLBEDcgSZjWiz3Fs41qhryHo8x0cB6HKbFcS9jHCYiMSIyBhgH/J/XcRnTUlh1r2mJOuNckknDuXQ0Q1W/9DYkY1oOu1RljDEmIHapyhhjTECi4lJVenq69urVy+swjDEmonz++efbVTWj5vqoSBy9evViyZIlXodhjDERRUQ21LXeLlUZY4wJiCUOY4wxAbHEYYwxJiBRMcZhTEtx4MAB8vLyKC4ubryxMX5KSkoiMzOT+Ph4v9pb4jAmguTl5ZGamkqvXr2o/xlXxvhPVSkoKCAvL4/evWs+2bhudqnKmAhSXFxMWlqaJQ0TNCJCWlpaQL1YSxzGRBhLGibYAv03ZYmjPqrwxbOw5nWvIzHGmLBiiaM+FWXw2ROw4BdQtNXraIzxXEFBAVlZWWRlZdG5c2e6detW9b60tLTBfZcsWcLVV1/d6GeMGjUqKLG+9957iAhPPPFE1bqlS5ciItx9990ATJs2jW7dulFS4jyqZfv27VTOMJGbm8ugQYMA2LdvH1OmTGHw4MEMGjSI4447jg0bNjT5XLQENjhen9h4GP8YPPoj+PfPYcqLYJcITBRLS0tj6dKlANx6662kpKRwww3Vz34qKysjLq7ur5Ts7Gyys7Mb/YxFixYFJ1hg0KBBzJ07l0svvRSAF154gaFDhx7UJjY2lqeeeooZM2bUe5z77ruPTp06sXz5cgDWrl1L586dGzwXLZ31OBqS0R9O/ROs+x8sedLraIwJO9OmTePKK6/kqKOO4sYbb2Tx4sUcc8wxDBs2jFGjRrF27VrA6QGcccYZgPNFO336dEaPHk2fPn24//77q46XkpJS1X706NFMmDCBI444gilTplA5k/frr7/OEUccwYgRI7j66qurjltTz549KS4uZuvWragqb7zxBmPHjj2ozbXXXss999xDWVlZvT/j5s2b6datW9X7/v37k5iY2ISz1XJYj6MxIy+Fr9+AN38LvU+A9H5eR2QMALe9spJVmwqDeswBXdvwhzMHBrRPXl4eixYtIjY2lsLCQj744APi4uJ46623+M1vfsP8+fNr7bNmzRreffddioqK6N+/PzNmzKhVQ/Dll1+ycuVKunbtyrHHHstHH31EdnY2V1xxBQsXLqR3795Mnjy5wdgmTJjAiy++yLBhwxg+fHitL/wePXpw3HHH8eyzz3LmmXU/mn769OmceuqpzJs3j5NOOompU6fSr190fw9Yj6MxIjDuQYhvBS9dBuUHvI7ImLAyceJEYmNjAdi9ezcTJ05k0KBBXHfddaxcubLOfU4//XQSExNJT0+nY8eObN1aexwxJyeHzMxMYmJiyMrKIjc3lzVr1tCnT5+qeoPGEse5557Liy++yAsvvFBv25tvvpm77rqLioqKOrdnZWWxfv16fvWrX7Fjxw5GjhzJ6tWrG/zcls56HP5I7Qxn3gdzL4T374ATf+t1RMYE3DMIleTk5Krl3/3ud/z4xz/m5ZdfJjc3l9GjR9e5j+9f/rGxsXVeKvKnTWM6d+5MfHw8//vf/7jvvvvqHEPp168fWVlZzJ07t97jpKSkMH78eMaPH09MTAyvv/46Rx55ZMDxtBTW4/DXgLMgawp88Hf4/lOvozEmLO3evbtqPGDmzJlBP37//v1Zv349ubm5AMyZM6fRfW6//XbuuOOOql5RXW655Zaqu61q+uijj9i5cycApaWlrFq1ip49ewYefAtiiSMQY/4GbTPh5cuhpMjraIwJOzfeeCM333wzw4YNa1IPoTGtWrXioYceYsyYMYwYMYLU1FTatm3b4D6jRo3i7LPPbrDNwIEDGT58eJ3bvv32W0444QQGDx7MsGHDyM7O5pxzzmnyz9ASRMUzx7OzszVoD3La8DHMPM3pfYx7IDjHNMZPq1evjupLJAB79uwhJSUFVeXnP/85/fr147rrrvM6rIhX178tEflcVWvdR209jkD1PAaOuw6+fBZWv+p1NMZEnccff5ysrCwGDhzI7t27ueKKK7wOKerY4HhTnHATrHsLXrkaMkdCaievIzImalx33XXWw/CY9TiaIi4Bxj8OpXudqvIouNxnjDGVLHE0VUZ/OOWPVlVujIk6ljgORc5lcNhJTlX59m+8jsYYY5pFSBOHiIwRkbUisk5Ebqpje6KIzHG3fyoivdz1aSLyrojsEZEHauzzhoh8JSIrReQREan/5uxQs6pyY0wUClnicL/QHwTGAgOAySIyoEazS4CdqtoXuAe4w11fDPwOqGu6yXNVdSgwCMgAJoYgfP+16eJUlW/60qkqN6aFirRp1du2bVsV38knnxzQ/r7Tqtf0+9//nrfeeisoMUbq1O+hvKsqB1inqusBRGQ2MA5Y5dNmHHCruzwPeEBERFX3Ah+KSN+aB1XVylnd4oAEwPuRad+q8r6nQI+jvI7ImKCLtGnVjz/+eF59Nfi3zN9+++1BO1akTv0eyktV3YCNPu/z3HV1tlHVMmA3kNbYgUXkTWAbUISTcOpqc7mILBGRJfn5+YFHHyirKjdRKJynVa+pvthWrlxJTk4OWVlZDBkyhG++ccYry8vLueyyyxg4cCCnnnoq+/fvr/qZ581zvnbefvtthg0bxuDBg5k+fXpVz6BXr1784Q9/YPjw4QwePJg1a9bUGVOkTv0ekXUcqvoTEUkCngdOBP5XR5vHgMfAqRwPeVBJbeCnjzlV5W/cbFXlJvT+cxNsWR7cY3YeDGP/FtAu4Tqt+gcffEBWVhbgzOB71VVX1RnbI488wjXXXMOUKVMoLS2lvLycrVu38s033/DCCy/w+OOPc+655zJ//nwuuOCCquMXFxczbdo03n77bQ4//HAuuugiHn74Ya699loA0tPT+eKLL3jooYe4++67D7ok5SsSp34PZY/jB6C7z/tMd12dbUQkDmgLFPhzcFUtBv6Nc7krPFhVuYlC4Tqt+vHHH8/SpUtZunQpt9xyS72xHXPMMfzlL3/hjjvuYMOGDbRq1QqA3r17VyWeESNGVE2sWGnt2rX07t2bww8/HICpU6eycOHCqu3jx4+vd19fkTj1eyh7HJ8B/USkN06CmAScX6PNAmAq8DEwAXhHG5g8S0RSgFRV3ewmmtOBD0IRfJNZVblpLgH2DEIlnKdV91VfbOeffz5HHXUUr732GqeddhqPPvooffr0qfX5lZeq/FW5f2OxR+LU7yHrcbhjFr8A3gRWA3NVdaWI3C4iZ7nNngTSRGQdcD1QdcuuiOQC/wCmiUiee0dWMrBARJYBS3HGOR4J1c/QJFZVbqJYOE6r3lhs69evp0+fPlx99dWMGzeOZcuW+R1Lbm4u69atA+DZZ5/lhBNO8DseX5E29XtI6zhU9XVVPVxVD1PVP7vrfq+qC9zlYlWdqKp9VTWn8g4sd1svVe2gqimqmqmqq1R1q6qOVNUhqjpIVa9yE1R4sapyE6XCcVr1xmKbO3cugwYNIisrixUrVnDRRRf5dbykpCSefvppJk6cyODBg4mJieHKK69s0s8VaVO/27TqoaIKz50DGxbBlR/Ys8pNUNi06jateqjYtOrhoKqqPMmqyo0JIptW3XuWOEKpTRc4836rKjcmiK677jqWLl3KqlWreP7552ndurXXIUUdSxyhZs8qN0EWDZeXTfMK9N+UJY7mYFXlJkiSkpIoKCiw5GGCRlUpKCggKSnJ730isnI84lhVuQmSzMxM8vLyaJZpdEzUSEpKIjMz0+/2ljiaS89j4Nhr4cN/wOFj4Ej/5tcxxld8fHxV1bQxXrFLVc1p9M3QZahTVV5Ue4oFY4yJBJY4mpNVlRtjWgBLHM3NqsqNMRHOEocX7FnlxpgIZonDC1ZVboyJYJY4vHLQs8rv9DoaY4zxmyUOLw0Y51aV321V5caYiGGJw2tWVW6MiTCWOLxWWVW+63unqtwYY8KcJY5wUFlVbs8qN8ZEAEsc4cKqyo0xEcISR7iwqnJjTISwxBFOrKrcGBMBLHGEG6sqN8aEOUsc4caqyo0xYc4SRziyqnJjTBizxBGuBoyDoedbVbkxJuxY4ghnY++wqnJjTNixxBHOktrATx+1qnJjTFixxBHueo6yqnJjTFixxBEJrKrcGBNGLHFEAt+q8gW/sKpyY4ynQpo4RGSMiKwVkXUiclMd2xNFZI67/VMR6eWuTxORd0Vkj4g84NO+tYi8JiJrRGSliPwtlPGHlYz+cMrt8M1/rarcGOOpkCUOEYkFHgTGAgOAySIyoEazS4CdqtoXuAe4w11fDPwOuKGOQ9+tqkcAw4BjRWRsKOIPSyOtqtwY471Q9jhygHWqul5VS4HZwLgabcYBs9zlecBJIiKquldVP8RJIFVUdZ+qvusulwJfAJkh/BnCS0yMVZUbYzwXysTRDdjo8z7PXVdnG1UtA3YDaf4cXETaAWcCbx9ypJHEqsqNMR6LyMFxEYkDXgDuV9X19bS5XESWiMiS/Pz85g0w1Hyryjcu9joaY0yUCWXi+AHo7vM+011XZxs3GbQFCvw49mPAN6p6b30NVPUxVc1W1eyMjIyAAo8IlVXlL11mVeXGmGYVysTxGdBPRHqLSAIwCVhQo80CYKq7PAF4R7Xhe01F5E84CebaIMcbWSqryndusKpyY0yzClnicMcsfgG8CawG5qrqShG5XUTOcps9CaSJyDrgeqDqll0RyQX+AUwTkTwRGSAimcAtOHdpfSEiS0Xk0lD9DGGv5yg47jqrKjfGNCtp5A/8FiE7O1uXLFnidRihUVYKT54Mu/NgxseQ2snriIwxLYSIfK6q2TXXR+TguPFhVeXGmGZmiaMlOKiq/CmvozHGtHCWOFqKkZfBYSfCm7dYVbkxJqQscbQUMTEw7iGrKjfGhJwljpakTRc4416rKjfGhJQljpZm4NlWVW6MCSlLHC1RVVW5PavcGBN8ljhaoqqq8lyrKjfGBJ0ljpaq5yg4zp5VbowJPkscLdno30DnIfascmNMUFniaMniEuCcJ6yq3BgTVJY4WjqrKjfGBJkljmhgVeXGmCCyxBENrKrcGBNEljiihVWVG2OCxBJHNLGqcmNMEFjiiDYHVZXv8ToaY0wEssQRbXyryt+0qnJjTOAscUSjyqryL56xqnJjTMAscUQrqyo3xjSRJY5oZc8qN8Y0kSWOaNbxCKsqN8YEzBJHtLOqcmNMgCxxRDurKjfGBMgSh7GqcmNMQCxxGMfAs2HoZKsqN8Y0yhKHqTb2TqsqN8Y0yhKHqWZV5cYYP1jiMAfzrSpf85rX0RhjwpAlDlNbZVX5gqusqtwYU4tfiUNEkkUkxl0+XETOEpF4P/YbIyJrRWSdiNxUx/ZEEZnjbv9URHq569NE5F0R2SMiD9TY588islFE7CJ8qFhVuTGmAf72OBYCSSLSDfgvcCEws6EdRCQWeBAYCwwAJovIgBrNLgF2qmpf4B7gDnd9MfA74IY6Dv0KkONn3IdEVdFo/dLseAScfJtVlRtjavE3cYiq7gPGAw+p6kRgYCP75ADrVHW9qpYCs4FxNdqMA2a5y/OAk0REVHWvqn6Ik0AOoqqfqOpmP+NusgPlFfxy7lc89N63of6o8JVzuVNV/t/fwvZ1XkdjjAkTficOETkGmAJUjpjGNrJPN2Cjz/s8d12dbVS1DNgNpPkZU4NE5HIRWSIiS/Lz8wPeP1aEClXuenMtLy7Z2PgOLVFlVXlcolWVG2Oq+Js4rgVuBl5W1ZUi0gd4N3RhHTpVfUxVs1U1OyMjI+D9Y2KEOycM5fh+6dz00nLeXbstBFFGgKqq8i+sqtwYA/iZOFT1fVU9S1XvcAfJt6vq1Y3s9gPQ3ed9pruuzjYiEge0BQr8irwZJMTF8PAFIziicyo/e+4Lvtq4y+uQvGFV5cYYH/7eVfUvEWkjIsnACmCViPyqkd0+A/qJSG8RSQAmAQtqtFkATHWXJwDvaJiNRqckxvH0xSNJT01g+szPyN2+1+uQvDH2DmhjVeXGGP8vVQ1Q1ULgbOA/QG+cO6vq5Y5Z/AJ4E1gNzHUvc90uIme5zZ4E0kRkHXA9UHXLrojkAv8ApolIXuUdWSJyp4jkAa3d9bf6+TM0WcfUJGZdnIMCFz21mPyiklB/ZPhJagvjrarcGOPcLdV4I5GVQBbwL+ABVX1fRL5S1aGhDjAYsrOzdcmSJYd8nC+/38n5j39K344pzL78aJIT44IQXYR561b48B6Y9C844nSvozHGhJCIfK6q2TXX+9vjeBTIBZKBhSLSEygMXniRYViP9jw4ZRirNhcy4/kvOFBe4XVIzc+qyo2Jev4Ojt+vqt1U9TR1bAB+HOLYwtKJR3Tirz8dzMKv8/n1/GXRVyBoVeXGRD1/B8fbisg/KusiROTvOL2PqHTuyO5cf8rhvPTFD9z55lqvw2l+VlVuTFTz91LVU0ARcK77KgSeDlVQkeCqE/ty/lE9ePi9b5m1KNfrcJpfzuXQ58dWVW5MFPI3cRymqn9wpw9Zr6q3AX1CGVi4ExH+OG4QpwzoxK2vrOT15SGfBSW8xMTA2Q9bVbkxUcjfxLFfRI6rfCMixwL7QxNS5IiNEf45eRjDe7Tn2jlL+XR92NQuNg/fqvKFd3kdjTGmmfibOK4EHhSRXLe+4gHgipBFFUGS4mN5cmo23du34tJnlrB2S5HXITWvyqryhXdZVbkxUcLfu6oqazaGAENUdRhwYkgjiyDtWicwa3oOrRNimfrUYjbtirLOmFWVGxNVAnoCoKoWuhXk4FR6G1dm+9bMvDiHvSVlTH1qMbv3RdE1f9+q8hcmwZrXoazU66iMMSFyKI+OlaBF0UIc2aUNj140gg0F+7jsmSUUHyj3OqTm03MUnHYXbFsNsyfD3/vDazdA3hKr9TCmhTmUxGHfBnUYdVg6fz93KItzd3Dt7KWUV0TRacq5DH65BibPgT6j4ctn4YmT4IFsZ0r2nbkeB2iMCYYG56oSkSLqThACtFLViJisKVhzVQXiyQ+/44+vruKiY3py21kDEYnCDlrxbli1AJbNgdwPnHU9joEh5zmD6q3aexufMaZB9c1V5dckh5HOi8QB8JfXV/PYwvX86if9+fmP+zb754eVXRth+Vz4ag5sXwuxCXD4GBg6Cfqe4kxlYowJK/UljojoMUSqm8YcwbbCYu56cy2d2iQxYUSm1yF5p113OP6XcNz1sHmpk0BWzIPVC6BVBxg03umJZI6EaOydGRNBrMcRYqVlFUyf+Rkfry/gyanZjO7f0ZM4wlJ5GXz7DiybDWteg7Ji6NDHSSBDznWWjTGesUtVHiUOgKLiA5z36CfkFuzlhcuOZmj3dp7FEraKC53ex1ezIfdDQKH7Ue54yE+hdQevIzQm6lji8DBxAGwrKmb8Q4vYX1rO/Bmj6JUetZMLN253Hix/0bmclb/aGQ/pd6ozHtLvVGd+LGNMyFni8DhxAKzP38M5Dy+iTat45s8YRXqKfQE2SBW2LHMSyPIXYe82SGrnjodMgu45Nh5iTAhZ4giDxAHO42cnP/4J/TqmRu/jZ5uivAzWv+eMh6x+Fcr2Q/ve1eMhaYd5HaExLY4ljjBJHABvr97K5c9+zrF903lyajbxsYdShxmFSopg9SvOeMh3CwF17sYach4MOsfGQ4wJEkscYZQ4AGYv/p6bXlrO+OHd+PvEodFZIBgMu39wLmMtmwPbVkFMvDsecp5TJ2LjIcY0mdVxhJlJOT3YWljCPW99Tac2Sfx6zBFehxSZ2naD466FY6+BLcudBLL8RVj7mjP54sCfOuMhPY628RBjgsQSh4euPqkvWwqLefi9b+ncJompo3p5HVLkEoEuQ5zXKbe74yFzYNlc+HwmtOvpXMoaOsnGQ4w5RJY4POQ8fnYg+UUl3PrKSjJSEzltcBevw4p8MbHQ9yTnVbIH1rzqjId8cDcsvBO6ZTsJZOB4SE7zOlpjIo6NcYSB/aXlTHniE1ZsKuTZ6Tkc1ce+zEKicHP1eMjWFRAT54yHDHHHQ+KTvI7QmLBig+NhnDgAdu4tZcIji9hWVMK8K0fRv3Oq1yG1bFtWOLf2LnsR9myBxLYwcJw7HnIMxNidbsZY4gjzxAGQt3Mf5zy8CEF46Wej6NquldchtXwV5fDd+06R4epX4MBeaNvDqQ0ZOgnS+3kdoTGescQRAYkDYPXmQs595GM6t01i3pWjaNs63uuQokfpXqe4cNlsZ3BdK6DrcCeBDDoHktO9jtCYZmWJI0ISB8Cib7cz7anPyOrejmcuySEpPtbrkKJP0Zbq+bK2LnfGQ/qe7IyH9B8L8dYbNC2fJY4IShwAr3y1iate+JIxAzvz4JThxMZYDYJntq50b+19EYo2QWIbGDDO6Yn0GGXjIabFqi9xhPRfvIiMEZG1IrJORG6qY3uiiMxxt38qIr3c9Wki8q6I7BGRB2rsM0JElrv73C8ttOT6zKFd+d0ZA3hj5RZue2Ul0ZDgw1angU5tyHUr4KJ/wxFnwMqXYebpcN8QeOs2yF/rdZTGNJuQJQ4RiQUeBMYCA4DJIjKgRrNLgJ2q2he4B7jDXV8M/A64oY5DPwxcBvRzX2OCH314uOS43lz+oz488/EGHnrvW6/DMTGx0Gc0/PRhuOFrGP8EZPSHj+6FB3Pg0RPgk4dhT77XkRoTUqHsceQA61R1vaqWArOBcTXajANmucvzgJNERFR1r6p+iJNAqohIF6CNqn6izp/gzwBnh/Bn8NxNY47g7Kyu3PXmWuZ9nud1OKZSQjIMmQgXzIfr18BP/uIMpr9xE/y9Pzw/EZbPgwP7vY7UmKALZeV4N2Cjz/s84Kj62qhqmYjsBtKA7Q0c0/fbM89dV4uIXA5cDtCjR49AYw8bMTHCnROGsn1PKb+ev4z0lAR7/Gy4Se0Ex/zceW1b7VSpL38R5l8CCW+9CaoAABRkSURBVKnueMh50PM4Gw8xLUKL/Vesqo+paraqZmdkZHgdziFJiIvh4QuG079TKj97/gu+2rjL65BMfToeCafcBteugIsWwICzYNX/wawz4d7B8NatsG2N11Eac0hCmTh+ALr7vM9019XZRkTigLZAQSPHzGzkmC1SalI8M6ePpENyAtNnfkbu9r1eh2QaEhMDfU6Asx+CG76Bc550kspH98NDR8GjP4KPH4RdG50nHRoTQUKZOD4D+olIbxFJACYBC2q0WQBMdZcnAO9oA7cPqepmoFBEjnbvproI+HfwQw9PHVOTeGZ6DhWqTH16Mdv3lHgdkvFHQmsYPAEumAe/XANj/gYIvPkbuHcQ3H24Mybyzp9hzetQuMmSiQlrIa3jEJHTgHuBWOApVf2ziNwOLFHVBSKSBDwLDAN2AJNUdb27by7QBkgAdgGnquoqEckGZgKtgP8AVzWUbCAy6zgaUvn42cM7pfLCZfb42Yi1bY0z3cmmpbDpS9i+1hlgB0jpBF2yoOsw6JrlLLexmZNN87ICwBaUOMAeP9sile51Hka1aSlsrkwmX/skk85OEuk6zE0qWZDa2duYTYtmiaOFJQ6wx89Ghapk8qVPz+RrwP29Te1SnUQqE0pqJ09DNi2HPTq2BfJ9/GznNkncaI+fbXkSkp3H3vY4unpdyZ7qZFLZM/n6DaqTSdfqy1uVl7pS7BZuEzyWOCJc5eNnH3rvWzrZ42ejQ2IK9DzGeVUqKTq4Z7J5Kaz9D1XJpE232j2TlMi+Td14xxJHhKv5+NmOqYmMtcfPRp/EVOg5ynlVKimCzct8eiZLYe1r1dvbZNbumdjU8cYPNsbRQvg+fva5S44ip3cHr0My4ai4ELYsO7hnUrCuentlMqnqmQyz57JHMRscb+GJA6ofP5tfVMK8GaM4vJM9ftb4oXh37Z7JDp9JNdt2r9EzGQat7Q+TaGCJIwoSBziPnx3/0CJiY4T5M+zxs6aJ9u9yeyZLqxPKjvXV29v2qNEzybJk0gJZ4oiSxAGwalMh5z36MV3aJfHiFfb4WRMk+3fB5q+q7+TatBR2fle9vV0PnxqTYdBlqCWTCGeJI4oSB8CidduZ+vRihvVozzPT7fGzJkT273SSiW/PZGdu9fZ2PasH3iuTSav2noVrAmOJI8oSB8CCrzZx9QtfMnZQZx443x4/a5rJvh21eya7NlRvb9/r4Op3SyZhywoAo9BZQ7uyrbCYP722mttfWcmtZw206nITeq07wGE/dl6V9u2oHnjf9CX88Lnz+N1K7XsfPC9Xl6HQql3zx278Yomjhbv0+D5sLSzm8Q++o1PbJH42uq/XIZlo1LoDHHai86q0b8fBd3LlLYGVL1Vv79Cnds8kqW3zx25qscQRBW4eeyTbikq48421dEpN4pwRmY3vZEyote4AfU9yXpX2FsBmnxqTjYthxfzq7SmdIDYBYuJqvGIP7X1sY8cL5JjxhxBXXEQ8JdISRxSIiRHumjCU7XtK+PX8ZaTZ42dNuEpOg74nO69Ke7e7ieRL2PU9VJRDRZnPq4735aXOBJH1bW/ovZZ79/MDIAEmtUbeT5wJcYnBjdAGx6NHUfEBznv0E3IL9jL78qMZkmnXkI2pRdUnmRzwM/k00Ka8vmP4m9QOBNi+xmdc+g7EJTTpVNhdVZY4ANhWWMz4hxdRfKCc+TNG0TMt2euQjDFhqr7EEf4X00xQdWyTxKzpOZRXKBc9ZY+fNcYEzhJHFDosI4Unp41ka2Ex02d+xt6SMq9DMsZEEEscUWp4j/Y8MHk4K37Yzc+e/4ID5RVeh2SMiRCWOKLYyQM68ZefDub9r/O5af5yomG8yxhz6Ox23Cg3KacHWwqLufetb+jcNpFf/cQeP2uMaZglDsM1J/Vja2ExD777LZ3bJHHhMb28DskYE8YscRj38bODyC8q5fcLVpKRmsiYQfb4WWNM3WyMwwAQFxvDPycPY1j3dlw9eymLv9vhdUjGmDBlicNUaZUQy5NTR5LZvhWXzvqMr7cWeR2SMSYMWeIwB2mfnMCsi3NIjI9l6lOL2bRrv9chGWPCjCUOU0v3Dq2ZefFIiorLmPb0YnbvO+B1SMaYMGKJw9RpYNe2PHbhCL7bvpfLnl1C8QGvZww1xoQLSxymXqP6pvP3c7NY/N0OrpuzlPIKKxA0xljiMI04a2hXfnv6kfxnxRZuf2WlVZcbY0KbOERkjIisFZF1InJTHdsTRWSOu/1TEenls+1md/1aEfmJz/prRGSFiKwUkWtDGb9xXHp8Hy49rjezPt7Aw+9/63U4xhiPhSxxiEgs8CAwFhgATBaRATWaXQLsVNW+wD3AHe6+A4BJwEBgDPCQiMSKyCDgMiAHGAqcISL2EO1m8JvTjuSsoV258421zP88z+twjDEeCmWPIwdYp6rrVbUUmA2Mq9FmHDDLXZ4HnCQi4q6fraolqvodsM493pHAp6q6T1XLgPeB8SH8GYwrJka4a+IQRh2Wxq/nL+O9tdu8DskY45FQJo5uwEaf93nuujrbuIlgN5DWwL4rgONFJE1EWgOnAd3r+nARuVxElojIkvz8/CD8OCYxLpZHLxxBv06p/Oz5L1iWt8vrkIwxHoiowXFVXY1zOeu/wBvAUqDO+0RV9TFVzVbV7IyMjGaMsmVLTYpn1sUjad86gekzP2NDwV6vQzLGNLNQJo4fOLg3kOmuq7ONiMQBbYGChvZV1SdVdYSq/gjYCXwdkuhNvTq2SeKZS3Ios8fPGhOVQpk4PgP6iUhvEUnAGexeUKPNAmCquzwBeEed+z0XAJPcu656A/2AxQAi0tH9bw+c8Y1/hfBnMPU4LCOFJ6fa42eNiUYhSxzumMUvgDeB1cBcVV0pIreLyFlusyeBNBFZB1wP3OTuuxKYC6zCuST1c1WtvCQ1X0RWAa+46+1Cu0dG9GzPP+3xs8ZEHYmGgq7s7GxdsmSJ12G0WC8s/p6bX1rOOcMzuXviEJwb44wxkU5EPlfV7Jrr7UFO5pBNzunBlt3F3Pf2N+zYW8LgzHb06NC66tUxNZGYGEsmxrQUljhMUFx7cj8OlFfw76WbeP/rfHyntUqMi6G7TyLp3qE1PTu0pkdaa7q3b02rhFjvAjfGBMwuVZmgKy2rYNOu/Xy/Yx8bduxj4459fF+wj+93OK89NQbSM1IT6eEmk6oEk1bdW7FLX8Z4wy5VmWaTEBdDr/RkeqUn19qmquzcd6AqiWzcsY8NBXv5fsc+Pv1uBy8v/QHfv2WS4mPo3r66p9KjQ2t6plW/T4q33ooxzc0Sh2lWIkKH5AQ6JCeQ1b1dre0lZeVs2lXMhoK9Tk+l6rWfT9YXsLf04HrPjm5vpbKH4vvKsN6KMSFhicOElcS4WHqnJ9O7nt7Kjr2lNXorzvIn3xbw8pe1eyt1jav06NCazPbWWzGmqSxxmIghIqSlJJKWksiwHu1rbS8pKydv5/6qpOI7rrLo2wL21eitdGqTSM8OyT7jKq3cRJNMekqC9VaMqYclDtNiJMbFclhGCodlpNTapqoUuL0V356Kk1S2M/+L4oPat4qPre6ppB18R1hm+1bWWzFRzRKHiQoiQnpKIukpiQyvo7dSfMDprfiOq2wocJLMR+u2s9/nmesi0LlN0kG3GPuOs6QlW2/FtGyWOIwBkuJj6dsxhb4d6+6tbN9TObayl+8Lqi+HffjNdrYUHtxbaZ0QW7tepUPl2EorEuOst2IimyUOYxohImSkJpKRmsiInvX1VtyeSkF17cqGgr188E0+xQcqfI4FXXx6KxmpicTFxhAfI85/Y4W4GCG21roY4mKlxnIMcTHuf9318bFy0PHiYoX4mMrtYj0hExSWOIw5RE5vJZW+HVNrbVNV8veUHDRQX9lbef/rfHbuK+VAefMV4cbGSD3JxjfBVCeghhNT9XKsz3Hq27/q+PUktvqSYmxMdXytEmJtfCkMWOIwJoREhI6pSXRMTSK7V4c626gq5RVKWYVyoLyCsnLlQIXzX9/lA+UVlFUo5RUVHKixray8ggMVzn/r2qes3N2nar2zfMDdt/Kzyyu0RrsK9h8op6y4xv41jnNwfKFNhKlJcWSkJtIxNZGM1CQyUhLp2CaRjBSnV1i53L51gs2RFiKWOIzxmEjlX++0iL+mVetOTHUnsIqDEmaZT1L03b8yKe4rLSe/qKTqteKH3WwrLK5VGApO7yo9JYGOqUnOpcbKBOO7nOJss/nSAmOJwxgTVCJCQpyQ0IxPpt5bUsb2PSVs80kq24qKq5a3Fhaz4ofdbN9TQl0dotREpxeT7o5ldUz1TTBJVb2ZDskJxFovxhKHMSbyJSfGkZwYR8+02jMO+CqvUHbuK2VbYQn5e2onmG1FJazeVMjCohKK6niqZWyMkJacUHWzRK0E49ObaZ3Qcr9eW+5PZowxNTiXr5x6nsbsr7wstqe4Kqn4Jpj8ohLWbC5i+54SyuroxiQnxPokmKSq5YzUgy+XpSUnRlwvxhKHMcbUoVVCrFPUmda6wXYVbi8mf0+J05Mp8u3NlJBfVMyaLYUs/KaEouLavZgYgQ7JPr2XBnozyQmxYXFLtSUOY4w5BDEx1XOoHdG54bbFB8oP7r3sKSG/sLgq0eQXlfD11iLyi+ruxbSKjz3oDrKDEoxPzyYtOYG42NCNMVniMMaYZpIUH0t3d1aBhlRUKLv3H/BJMMW1ejPfbNvDom8L2L3/QK39RSAtOYH0lETmzxhFcmJwv+otcRhjTJiJiRHaJyfQPjmB/p1rF5b6Kik7+BZl395MwZ4SWofgVmNLHMYYE8ES42LJbO88Y6a5NN+N1sYYY1oESxzGGGMCYonDGGNMQCxxGGOMCYglDmOMMQGxxGGMMSYgljiMMcYExBKHMcaYgIhq8z220isikg9saOLu6cD2IIYTLBZXYCyuwFhcgWmpcfVU1YyaK6MicRwKEVmiqtlex1GTxRUYiyswFldgoi0uu1RljDEmIJY4jDHGBMQSR+Me8zqAelhcgbG4AmNxBSaq4rIxDmOMMQGxHocxxpiAWOIwxhgTEEscLhEZIyJrRWSdiNxUx/ZEEZnjbv9URHqFSVzTRCRfRJa6r0ubIaanRGSbiKyoZ7uIyP1uzMtEZHioY/IzrtEistvnXP2+meLqLiLvisgqEVkpItfU0abZz5mfcTX7ORORJBFZLCJfuXHdVkebZv999DOuZv999PnsWBH5UkRerWNbcM+Xqkb9C4gFvgX6AAnAV8CAGm1+BjziLk8C5oRJXNOAB5r5fP0IGA6sqGf7acB/AAGOBj4Nk7hGA6968O+rCzDcXU4Fvq7j/2OznzM/42r2c+aegxR3OR74FDi6Rhsvfh/9iavZfx99Pvt64F91/f8K9vmyHocjB1inqutVtRSYDYyr0WYcMMtdngecJCISBnE1O1VdCOxooMk44Bl1fAK0E5EuYRCXJ1R1s6p+4S4XAauBbjWaNfs58zOuZueegz3u23j3VfMunmb/ffQzLk+ISCZwOvBEPU2Cer4scTi6ARt93udR+xeoqo2qlgG7gbQwiAvgHPfyxjwR6R7imPzhb9xeOMa91PAfERnY3B/uXiIYhvPXqi9Pz1kDcYEH58y97LIU2Ab8T1XrPV/N+PvoT1zgze/jvcCNQEU924N6vixxRL5XgF6qOgT4H9V/VZjavsCZe2co8E/g/5rzw0UkBZgPXKuqhc352Q1pJC5PzpmqlqtqFpAJ5IjIoOb43Mb4EVez/z6KyBnANlX9PNSfVckSh+MHwPcvg0x3XZ1tRCQOaAsUeB2Xqhaoaon79glgRIhj8oc/57PZqWph5aUGVX0diBeR9Ob4bBGJx/lyfl5VX6qjiSfnrLG4vDxn7mfuAt4FxtTY5MXvY6NxefT7eCxwlojk4lzOPlFEnqvRJqjnyxKH4zOgn4j0FpEEnMGjBTXaLACmussTgHfUHWnyMq4a18HPwrlO7bUFwEXunUJHA7tVdbPXQYlI58rruiKSg/PvP+RfNu5nPgmsVtV/1NOs2c+ZP3F5cc5EJENE2rnLrYBTgDU1mjX776M/cXnx+6iqN6tqpqr2wvmOeEdVL6jRLKjnK66pO7YkqlomIr8A3sS5k+kpVV0pIrcDS1R1Ac4v2LMisg5nAHZSmMR1tYicBZS5cU0LdVwi8gLO3TbpIpIH/AFnoBBVfQR4HecuoXXAPuDiUMfkZ1wTgBkiUgbsByY1Q/IH5y/CC4Hl7vVxgN8APXxi8+Kc+ROXF+esCzBLRGJxEtVcVX3V699HP+Nq9t/H+oTyfNmUI8YYYwJil6qMMcYExBKHMcaYgFjiMMYYExBLHMYYYwJiicMYY0xALHEY00QiUu4zC+pSqWP24kM4di+pZ5ZfY7xmdRzGNN1+d/oJY6KK9TiMCTIRyRWRO0Vkufv8hr7u+l4i8o47Ad7bItLDXd9JRF52JxL8SkRGuYeKFZHHxXn2w3/damVE5GpxnqGxTERme/RjmihmicOYpmtV41LVeT7bdqvqYOABnJlLwZkkcJY7Ad7zwP3u+vuB992JBIcDK931/YAHVXUgsAs4x11/EzDMPc6VofrhjKmPVY4b00QiskdVU+pYnwucqKrr3UkEt6hqmohsB7qo6gF3/WZVTReRfCDTZ3K8ymnO/6eq/dz3vwbiVfVPIvIGsAdnptr/83lGhDHNwnocxoSG1rMciBKf5XKqxyRPBx7E6Z185s52akyzscRhTGic5/Pfj93lRVRPLjcF+MBdfhuYAVUPCmpb30FFJAborqrvAr/GmR67Vq/HmFCyv1SMabpWPrPKAryhqpW35LYXkWU4vYbJ7rqrgKdF5FdAPtUz4F4DPCYil+D0LGYA9U2pHgs85yYXAe53nw1hTLOxMQ5jgswd48hW1e1ex2JMKNilKmOMMQGxHocxxpiAWI/DGGNMQCxxGGOMCYglDmOMMQGxxGGMMSYgljiMMcYE5P8B6tRH2VoAcVgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(train_loss[:, 0], label = 'Training MNIST')\n",
        "#plt.plot(val_loss[:, 0], label = 'Validation MNIST')\n",
        "plt.plot(train_loss[:, 1], label = 'Training Fashion MNIST')\n",
        "#plt.plot(val_loss[:, 1], label = 'Validation Fashion MNIST')\n",
        "plt.title('Training and Validation Loss')\n",
        "plt.legend()\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('Epochs')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "bHtwdHP5lrzD",
        "outputId": "7f158736-e7a0-4f7e-a345-0beaf25464c3"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Epochs')"
            ]
          },
          "metadata": {},
          "execution_count": 18
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eXxV1dX//165mWcyQhJmmWcMg4I4W+cRq9QB5zq11ta22tHHDvbX+jytttpWrbNfrWK1ap1bqYiCgEwhAiIiJIHMZLoZbu5dvz/OSbgJAQLkchNY79frvO4+++y97zrnJvtz9l57EFXFMAzDMLpLRLgNMAzDMPoWJhyGYRjGfmHCYRiGYewXJhyGYRjGfmHCYRiGYewXJhyGYRjGfmHCYRw0IvKmiMzv6bThRES2iMgpISh3oYhc54YvE5F3upP2AL5nkIjUi4jnQG01jD1hwnGE4lYqbUdARBqDzi/bn7JU9QxVfbKn0/ZGROROEfmgi/gMEWkRkfHdLUtVn1XV03rIrg5Cp6pbVTVRVf09UX4X3ycisllECkNRvtG7MeE4QnErlURVTQS2AucExT3blk5EIsNnZa/kGeBYERnaKf5SYK2qFoTBpnAwB8gChonItEP5xfY3GX5MOIwOiMgJIlIkIj8UkR3A4yLST0ReF5FyEal2w3lBeYK7X64SkQ9F5D437ZcicsYBph0qIh+ISJ2IvCciD4rIM3uwuzs2/kJEFrvlvSMiGUHXrxCRr0SkUkR+vKfno6pFwH+AKzpduhJ4al92dLL5KhH5MOj8VBFZLyI1IvInQIKuDReR/7j2VYjIsyKS6l57GhgEvOa2GH8gIkNERNsqWRHJEZFXRaRKRDaJyPVBZd8tIi+IyFPus1knIvl7egYu84F/Am+44eD7Gici77rfVSoiP3LjPSLyIxH5wv2eFSIysLOtbtrOfyeLReT3IlIJ3L235+HmGSgi/3B/h0oR+ZOIRLs2TQhKlyUiXhHJ3Mf9GkGYcBhd0R9IAwYDN+D8nTzung8CGoE/7SX/DGADkAH8FvibiMgBpP1/wCdAOnA3u1fWwXTHxm8AV+O8KUcDdwCIyFjgz275Oe73dVnZuzwZbIuIjAImu/bu77NqKyMD+AfwE5xn8QUwKzgJcK9r3xhgIM4zQVWvoGOr8bddfMXzQJGbfy7waxE5Kej6uW6aVODVvdksIvFuGc+6x6UiEu1eSwLeA95yv+so4N9u1u8C84AzgWTgGsC71wezixnAZiAb+NXenoc4fp3Xga+AIUAu8Lyqtrj3eHlQufOAf6tqeTftMABU1Y4j/AC2AKe44ROAFiB2L+knA9VB5wuB69zwVcCmoGvxgAL99yctTqXbCsQHXX8GeKab99SVjT8JOr8ZeMsN/wynYmm7luA+g1P2UHY8UAsc657/CvjnAT6rD93wlcCSoHSCU9Fft4dyzwdWdvUbuudD3GcZiVOp+oGkoOv3Ak+44buB94KujQUa9/JsLwfK3bJjgRrgAvfavGC7OuXbAJzXRXy7rXt5Tlv38Xu3Pw/gmDb7ukg3A0dkxT1fDnw9nP9/ffGwFofRFeWq2tR2IiLxIvJXtyunFvgASJU9j9jZ0RZQ1bY3ysT9TJsDVAXFAWzbk8HdtHFHUNgbZFNOcNmq2gBU7um7XJteBK50W0eXAU/thx1d0dkGDT4XkWwReV5Eit1yn8FpmXSHtmdZFxT3Fc6beBudn02s7NmXMB94QVVb3b+Tl9jVXTUQp7XUFXu7ti86/Pb7eB4Dga9UtbVzIaq6FOf+ThCR0TgtolcP0KYjFhMOoys6L5n8PWAUMENVk3EcoxDUBx8CtgNpbrdIGwP3kv5gbNweXLb7nen7yPMk8HXgVCAJeO0g7ehsg9Dxfn+N87tMcMu9vFOZe1vmugTnWSYFxQ0Civdh0264/pqTgMtFZIc4frC5wJlud9s2YNgesm8DhncR3+B+Bv/W/Tul6Xx/e3se24BBexG+J930VwALgl+SjO5hwmF0hyScvvqdIpIG/DzUX6iqX+F0I9ztOjWPAc4JkY0LgLNFZLbbV38P+/7fWATsBB5mV//5wdjxL2CciFzoVnjfpmPlmQTUAzUikgt8v1P+UvZQYavqNuAj4F4RiRWRicC1OG/p+8sVwEYccZzsHiNxutXm4fgWBojId0QkRkSSRGSGm/dR4BciMkIcJopIujr+hWIcMfKIyDV0LTDB7O15fIIjxL8RkQT3noP9Rc8AF+CIx1MH8AyOeEw4jO7wByAOqACW4Dg+DwWX4fRXVwK/BP4ONO8h7QHbqKrrgFtwnNvbgWqcinBveRSn0hlMx8rngOxQ1QrgYuA3OPc7AlgclOR/gKk4/oR/4TjSg7kX+ImI7BSRO7r4ink4voQS4GXg56r6Xnds68R84CFV3RF8AH8B5rvdYafiiPwO4HPgRDfv/wEvAO/g+Ij+hvOsAK7HqfwrgXE4Qrc39vg81Jm7cg5ON9RWnN/ykqDr24BPcVosi/b/ERhtDiLD6PWIyN+B9aoa8haPcXgjIo8BJar6k3Db0hcx4TB6LeJMLKsCvgROA14BjlHVlWE1zOjTiMgQYBUwRVW/DK81fZOQdlWJyGMiUiYiXc6mdfs5HxBnQtIaEZkadG2+iHzuHvOD4o8WkbVungf2Mj/A6Pv0xxmWWQ88ANxkomEcDCLyC6AA+J2JxoET0haHiMzB+ad/SlV3W8NHRM4EvoUzIWgGcL+qznCdisuBfJx+yBXA0apaLSKf4DgOl+LMWn1AVd8M2U0YhmEYHQhpi0NVP8DpatgT5+GIiqrqEpzx7gOArwHvqmqVqlYD7wKnu9eSVXVJkHPy/FDeg2EYhtGRcC8WlkvHiT1Fbtze4ou6iN8rGRkZOmTIkIO11TAM44hixYoVFaq62zpe4RaOkCEiN+Css8SgQYNYvnx5mC0yDMPoW4jIV13Fh3seRzEdZ8fmuXF7i8/rIn43VPVhVc1X1fzMTFv40jAMo6cIt3C8irvej4jMBGpUdTvwNnCaOEtU98MZivm2e61WRGa6o6muxFna2TAMwzhEhLSrSkSew1ltNUNEinCWX4gCUNW/4IyKOhPYhLPw2NXutSp32Nwyt6h7VLXNyX4z8ATOjNM33cMwDMM4RBwREwDz8/PVfByGYRj7h4isUNXdNvUKd1eVYRiG0ccw4TAMwzD2CxMOwzAMY784bOdxGIZhHIn4A8qXFQ2sK6lhXUktt58ykrjofW1AuX+YcBiGYfRRWloDbCyto7CklgJXKApLamn0+QGI9kRwwZRcxgxI7tHvNeEwDMPoA3hbWvlse53Tkih2hGJjaR0+vzMyNiHaw7icFC6ZNpDxuSmMy0nmqKxEojw975Ew4TAMw+hl1Hh97V1NbS2JzeX1BNzZE/3ioxifm8K1s4cxLieZ8bkpDE6LJyLi0OwyYcJhGEZI8AcUnz9Aiz+ArzWAzx907g/ga9VdYfdoaYtrDYpz87XFtZ8H5Qk+j4+OJC0huv1IT4imn/uZlhBNanw0nkNUwXaHstomRyCKdwlFUXVj+/UBKbGMy0nhrAkD2lsSA1JiCedWRCYchmHsN/XNrawrrqHArfAKimuobGjB17pLGAIhmlscHRlBtCeCKI8Q5YkgyhNBdKRz7omIwNvSSlV9C3XNrV3mF4HUuKgO4pKWEENaQhRpCTG7CU1aQjSxUQfvXFZViqobOwjEupJayuua29MMzUhg0sBULpsxmHE5yYzLSSY9Meagv7unMeEwDGOv1DT6XJGooaDYEYovKxtoW3QiOzmG8TkpzBiWRrTHQ1SkuBV72yFuxb6rko8OqvSduE7nngiiguLahMITId1+025u9bPT66OyvoWqhhaqvC1U1TfvCje0UFnfwpcVDaz4qpqqhpY9il18tKdD62X3lswu4UlLiCYh2uOObNrVklhXUkNtkyNmnghhRFYic0Zktnc1jRmQRFJsVE/8ZCHHhMMwjHaqG1ooKKlhbbHjgF1bXMPWKm/79dzUOMblJHPBlFyn2yQ3mayk2DBavGdiIj1kJ3vITu6efYGAUtvko7LBFZoujkpXbD4vraeyoZkmX6AbdkQwekAyZ0/KYXyO09U0qn9Sj7RiwoUJh2EcoVTUN7sC4QhFQXEtxTt39a0PTItjQq4zSmeC27feG7tNeoqICCE13vGBDO/mTgyNLX4qG5p3E5jaRh+D0xMYn5vC8MwEIkMwsimcmHAYxmGOqlJW18zaorbuJkckdtQ2tacZmpHA1MH9uPKYwa5IpJAS3ze6TcJJXLSHvOh48vrFh9uUQ4oJh2EcRqgqJTVN7Q7rguIa1hbXUlHvOGBFYHhmIscMT2/vWx+Xk9xn+taN3oEJh2H0QVSVaq+P0tomvqxocAXCccJWNbQAuxywx4/MZEJumwM2mYQY+7c3Dg77CzKMXoSqUt/cSmltM2W1TZTWNbGjppnS2ibK6poorXXDtc20+Hc5ZiMjhFH9kzh1TDbjg0SiLztgjd6LCYdhHCKafH7KapsprWuitLaJHTVNlNU5QtAmBjtqm/C2+HfLmxQTSVZyDNnJsUwbkkZWcgz9k2PJTo5lYL94RvZPJCbSRMI4NJhwGMZB4vMHqKhvprS22RWDJlcMmjsIQk2jb7e80ZERrgDEMCYnmRNGZdE/xRGIrCQnPjs51rqXjF5FqPccPx24H/AAj6rqbzpdHww8BmQCVcDlqlokIicCvw9KOhq4VFVfEZEngOOBGvfaVaq6KpT3YRyZqCq1ja3sqG1iR20TpTVNHcJt3UiVDc103oHZEyFkJcWQlRzL4PR4pg9No39KLFlJjhBku2KREhcV1qUjDONACJlwiIgHeBA4FSgClonIq6paGJTsPuApVX1SRE4C7gWuUNX3gcluOWnAJuCdoHzfV9UFobLdOPzx+QOU1TkthLZuo1JXFILDXU3wSkuIbq/4x+ekdBCC7ORYspJjSE+I6VXrIRlGTxLKFsd0YJOqbgYQkeeB84Bg4RgLfNcNvw+80kU5c4E3VdXbxTXD6ICqUtfcSmlNE9trOrYUdglD162EaE8EWckxDEiJZXxuCqeMyaZ/iiMK/VNi6e+KgvkSjD5BUw3UlkD6UeDp2eHWoRSOXGBb0HkRMKNTmtXAhTjdWRcASSKSrqqVQWkuBf6vU75ficjPgH8Dd6pqc6friMgNwA0AgwYNOpj7MHoR/oBSWFJL8U4vO2qa2OH6EYJbCV05l1Pjo9qdyeMGpJDtCkGbP6F/cixpCdHWbbQ3VKG1CZrroKkWmmuCwnXgbwm3hQ6xyZCQBYlZkJAJcf2cCSyHE22iUFMMtUFHTbETX1sCLXVO2ltXQMZRPfr14fa43QH8SUSuAj4AioH2/3oRGQBMAN4OynMXsAOIBh4Gfgjc07lgVX3YvU5+fn6I1uk0DgWqysptO3ltdQn/WrOdsqDVRKM8QlaS0yIYM6Cjc7m/21LITo61YakBv1O5N9dBc+2uyr651qmE2sLtQtAWrukYH9jdwd/riYh0BCQh0xWTLEjI2BVOzHTjMp34iDD/rTTVdiEERUFCESQK7QgkZkNKLmSOhOEnQnIuJOc499TDhFI4ioGBQed5blw7qlqC0+JARBKBi1R1Z1CSrwMvq6ovKM92N9gsIo/jiI9xmKGqfLa9jtfWlPDa6hKKqhuJjozgxFGZnDlhAMMzE+mfEktafPQh27ym19BUCzVFbsVSBA3lHSv/DqLghneraLpAIiAm2Xljj3GP5ByIGRUUn+SGU4LCbrynN6xjpc6zqC9znkvbZ0MZ1LufZeudzy5bSALx6a6oZHRsubR9Bocj9/Oem+s6tRJK3N+yZNd5c+3uNiVmO79FZ1FIyXM+kwb0eHfU3gilcCwDRojIUBzBuBT4RnACEckAqlQ1gNOSeKxTGfPc+OA8A1R1uzh9CucDBSGy3wgDX5TX89pqRyy+KG/AEyHMPiqD208Zyanjskk+3JfG8DV1fNusKXLeNtvDxV1ULEBkXFCFn+SEk/p3EoKkvVf+UfGHR5dOcg5kjdl7GlXnObaJSbvAlHcUneIVTrilvutyYlNcMenUcolPA29lp1ZDV79dkChkjIBhJzjh5FznSMmFxP4QGd0DD6bnCJlwqGqriNyK083kAR5T1XUicg+wXFVfBU4A7hURxemquqUtv4gMwWmx/LdT0c+KSCYgwCrgxlDdg3FoKKr28vqa7by2uoR1JbWIwIyhaVwzeyhnjB9AWkLv+qc5YAJ+qNvhVibbdr151hTtEoWG8t3zxWc4FUjaMBg6xwkn5zpvmyl5TmXVyyqWXo+IU+nHpnSv/7/FG9Rq6dSCaSh3wmWfQf1/oamt00SclklyLqQPh2HH9wlR6A6inYeWHIbk5+fr8uXLw22GEURZXRNvrNnOq6tL+HSr8482eWAq50zK4eyJA7q9h0KvQRW8VW7rwG0hdAi7b53ayXEfneRUICl5u8QgWBSScyAqLjz3ZBwYrS2OeMSm9klRCEZEVqhqfuf4cDvHjSOI6oYW3lq3g9dWl7BkcyUBhTEDkvnB6aM4Z2IOA9NCuDS1qtOn3drk/GP7m6HVPTqEW9xwU1C4LY2bvy3cUh/kayiG1saO3+mJ3iUCg2e5YpALyXm7xCI2JXT3bISHyGinpXEYY8JhhJT65lbeLdzBa6u388HGcloDytCMBG49aQTnTBzAiOyk7hUU8ENZIWxdAttXQ0tDF5V+cOXeEiQETT03VFQiHCdwZAxEJzgtgv4TYOTpu7cW4jMg4vDawMcwwITDCAFNPj/vry/jtTUl/PuzMppbA+SmxnHtcUM5Z2IO43KS9z1foqUBipbDtqWOWBQt2+VYTMh0uwFinLf6yFiIjgdPPycuMmZX5R6cJjK6U3yMExcZ66aJ6RTuXE4MeOxfxjDsv8DoEVpaA3y4qZzXVm/nnXU7aGjxk5EYw7zpgzhn0gCmDOy392GzdTscgdi6BLYtge1rXH+AQNZYmDAXBh0DA2dA6qDDY/SPYfRRTDiMA8YfUJZuruS1NSW8WbCDnV4fKXFRnDMph3Mn5TBjWHrX6zUFAlCxoaNQVG9xrkXGQe7RMPt2GDQT8qZBXOohvS/DMPaOCYexX6gqn251Z3Gv3U55XTMJ0R5OHZvNuZNzmH1UJtGRnfr1fY1QshK2fgxblzrdT21DFhMynVbEtOsdoeg/sc+PRDGMwx0TDmOfBALK6qKdvFWwg9fXbKd4pzOL++TRWZwzKYcTR2URFx20TENDxa6WxNYlULJq11IVGSNh7LkwcKYjFGnDrNvJMPoYJhxGl7T6A3yypYq3C3bw9rpSdtQ2ERkhHDcig++dNpJTx2aTFBvlDHOt3NRRKCo3OYV4oiFnChxzsyMUA2dAQnp4b8wwjIPGhMNop8nn56MvKnirYAfvFpZS7fURGxXB8SMz+eH4UZw0KpuU6IAzHHbFP1yxWAreCqeAuH6OQEy53HFkD5gMUX1sIp9hGPvEhOMIp765lYUbynirYAcLN5RT39xKUkwkJ4/J4vTx/ZmTK8SXfgrbXoOVnzhr9/jd1Wn7DYURp8GgGY5QpI+weQuGcQRgwnEEUt3QwnuflfL2uh188HkFLa0BMhKjOXdiNhfk1jJFPiey5GX4z1Ko2uxkioiCARNh2nWOb2LgDEjKDu+NGIYRFkw4jhBKa5t4Z90O3lq3gyWbq/AHlJEpAX42poIT478kp24NsmE5rHWX347PcMRh6nznM2eyrZlkGAZgwnFYs6WigbddsVi5tZqhsoOvpWzlB3lfMcr3GTHVG5HPFRDIHgcTL3ZEYuB0pxvKRjsZhtEFJhyHEarKhtI63irYwftrtxBTtoajIz7nzvjNTEzaSJxvJzQBmuxMrJt0kSMSuUc7+zEYhmF0AxOOPk4goKwq2slHK1ZTsX4RgxvWcmLE59wa8RWRMe4S3slHwcCzHJEYOAMyRpkT2zCMA8aEow/S2tJE4aeLKV67kKjtyxjn38CtUuVci44lkDOVyCHnOyKRN83mThiG0aOYcPQFGqtp+eJDStb+l8C2peR4P2MiPiYClZFZeAfMoHHMHOKGHUNk9vhDuvewYRhHHiYcvRz/tuU0P3Eh8f4actTDZzKMJWnnkzJyNqOnnUx6+kCsPWEYxqEkpMIhIqcD9+PsOf6oqv6m0/XBwGNAJlAFXK6qRe41P7DWTbpVVc9144cCzwPpwArgClXtoV16ehlf/Af/s9+gojWRV4f8gUkzT2HGyNzdFxE0DMM4hISsBhIRD/AgcAYwFpgnImM7JbsPeEpVJwL3APcGXWtU1cnucW5Q/P8H/F5VjwKqgWtDdQ9hpeAf+J+5mC9aM3lh0t+49ZqrOW7sQBMNwzDCTihroenAJlXd7LYIngfO65RmLPAfN/x+F9c7IM62cScBC9yoJ4Hze8zi3sKyR9EF1/Cpfzh/GfpHbr9gTrgtMgzDaCeUwpELbAs6L3LjglkNXOiGLwCSRKStyz5WRJaLyBIRaROHdGCnqrbupUwAROQGN//y8vLyg72XQ4Mq/Pe38K/vsVCn8NvMX/Oby47rejMkwzCMMBHufo87gONFZCVwPFAMuJMPGKyq+cA3gD+IyPD9KVhVH1bVfFXNz8zM7FGjQ0IgAG/+EN7/Fa/L8fxP/F08dNVxHfe5MAzD6AWE0jleDAwMOs9z49pR1RLcFoeIJAIXqepO91qx+7lZRBYCU4CXgFQRiXRbHbuV2SdpbYF/3gxrX2RB1Ln80ncZC64+lsykmHBbZhiGsRuhbHEsA0aIyFARiQYuBV4NTiAiGSLSZsNdOCOsEJF+IhLTlgaYBRSqquL4Qua6eeYD/wzhPYSelgZ4fh6sfZHnkq7iR955/PXKaRyVlRhuywzDMLokZMLhtghuBd4GPgNeUNV1InKPiLSNkjoB2CAiG4Fs4Fdu/BhguYisxhGK36hqoXvth8B3RWQTjs/jb6G6h5DjrYKnzke/+A/P97+Du8pP43cXT2LGMJuZYRhG70Wcl/jDm/z8fF2+fHm4zehIbQk8fSFUfcE/h9/DbWsG8f2vjeKWE48Kt2WGYRgAiMgK19fcAZs5Hg4qv4CnzofGKt7Pf4jb/hvPpdMGcvMJ++X/NwzDCAvhHlV15FGyCv52Gvga+PTEp7luUQJzRmbyi/PHI7b/hWEYfQATjkPJl4vgibMhKo5NZ7/ElW/5GJGVyIPfmEKUx34KwzD6BlZbHSo+ex2euQhScim7+FUuf6WKxJhIHr96GkmxtpqtYRh9BxOOQ8GnT8MLV0D/CdR941Xmv1RMfXMrj101jQEpto+3YRh9CxOOUPPhH+DVW2HYCfguf5lbXv6KjaV1PHjZVMbm2HathmH0PWxUVahQhXd/Ch/9EcZfhJ7/Z376zw18sLGc31w4geNH9oFlUAzDMLrAhCMU+FvhtW/Dqmdh2vVwxm956L+beX7ZNm45cTiXTh8UbgsNwzAOGBOOnsbXCAuugQ1vwAl3wfE/5J+rS/jd2xs4b3IOd5w2KtwWGoZhHBQmHD1JUw08Nw+++gjOvA+mX8/SzZV8/8U1TB+axm/nTrS5GoZh9HlMOHqK+jJ45kIo+wwuehQmzGVTWT03PL2CvLQ4Hr7iaGIibYl0wzD6PiYcPUHVl/D0BVBfCt/4Oxx1CuV1zVz9xCdEeYQnr55Oanx0uK00DMPoEUw4DpYdBU5Lo7UZrnwVBk6jscXPdU8tp7yumedvOIaBafHhttIwDKPHMOE4GL76GJ67BKIS4Jq3IGsM/oBy2/MrWVO0k79efjSTB6aG20rDMIwexSYAHigb33a6pxIy4dq3IWsMAL/8VyHvFJbys7PHctq4/mE20jAMo+cx4TgQVv/dGT2VORKufgtSnXkZj334JY8v3sI1s4Zy9ayhYTbSMAwjNJhw7C8fPwQv3wBDZsH81yHRmQH+9rod/OJfhXxtXDY/PmtMmI00DMMIHebj6C6q8J9fwqL7YMw5cOGjEBULwKptO7nt+ZVMykvlD5dMwRNhczUMwzh8CWmLQ0ROF5ENIrJJRO7s4vpgEfm3iKwRkYUikufGTxaRj0VknXvtkqA8T4jIlyKyyj0mh/IeAAj44fXbHdGYeiVc/GS7aGyt9HLtE8vITIrh0fn5xEXbXA3DMA5vQtbiEBEP8CBwKlAELBORV1W1MCjZfcBTqvqkiJwE3AtcAXiBK1X1cxHJAVaIyNuqutPN931VXRAq2zvQ2gz/uB4K/wmzvwsn/wzc2d87vS1c9cQntAaUJ66eTkZizCExyTAMI5yEssUxHdikqptVtQV4HjivU5qxwH/c8Ptt11V1o6p+7oZLgDLg0C8n21wHz17siMZpv4JTft4uGs2tfm54egVFVY08fMXRDM9MPOTmGYZhhINQCkcusC3ovMiNC2Y1cKEbvgBIEpH04AQiMh2IBr4Iiv6V24X1exHp8jVfRG4QkeUisry8vHz/rVeF578BWz6E8/8Cx97afikQUL7/4ho++bKK3108kRnD0vdSkGEYxuFFuEdV3QEcLyIrgeOBYsDfdlFEBgBPA1erasCNvgsYDUwD0oAfdlWwqj6sqvmqmp+ZeQCNFRGY83249FmYPK/Dpf99dwOvri7h+18bxXmTO2uhYRjG4U0oR1UVAwODzvPcuHbcbqgLAUQkEbiozY8hIsnAv4Afq+qSoDzb3WCziDyOIz6hYeic3aKe+2QrD77/BfOmD+TmE4aH7KsNwzB6K6FscSwDRojIUBGJBi4FXg1OICIZItJmw13AY258NPAyjuN8Qac8A9xPAc4HCkJ4Dx3478ZyfvJKAXNGZnLPeeNtiXTDMI5IQiYcqtoK3Aq8DXwGvKCq60TkHhE51012ArBBRDYC2cCv3PivA3OAq7oYdvusiKwF1gIZwC9DdQ/BFJbUcvMzKxiZncRDl00lyhPuXj7DMIzwIKoabhtCTn5+vi5fvvyA82+vaeSCBz8C4JVbZtE/JbanTDMMw+i1iMgKVc3vHG8zx/dBXZOPqx9fRn1zKy/eeIyJhmEYRzwmHHvB5w9w87Of8nlZPY9fNY0xA5LDbZJhGEbY2WdHvYicE+TAPmJQVX76SgGLPq/g1xeMZ87IQ3RoG5MAACAASURBVD//0DAMozfSHUG4BPhcRH4rIqNDbVBvQUQYMyCZb590FJdMGxRucwzDMHoN++yqUtXL3TkV84AnRESBx4HnVLUu1AaGk/nHDgm3CYZhGL2ObnVBqWotsABnvakBOMuDfCoi3wqhbYZhGEYvpDs+jnNF5GVgIRAFTFfVM4BJwPdCa55hGIbR2+jOqKqLgN+r6gfBkarqFZFrQ2OWYRiG0VvpjnDcDbStD4WIxAHZqrpFVf8dKsMMwzCM3kl3fBwvAoGgc78bZxiGYRyBdEc4It2NmABww9GhM8kwDMPozXRHOMqDFiVERM4DKkJnkmEYhtGb6Y6P40acFWn/BAjOrn5XhtQqwzAMo9fSnQmAXwAz3Y2WUNX6kFtlGIZh9Fq6tcihiJwFjANi2zYvUtV7QmiXYRiG0UvpzgTAv+CsV/UtnK6qi4HBIbbLMAzD6KV0xzl+rKpeCVSr6v8AxwAjQ2uWYRiG0VvpjnA0uZ9eEckBfDjrVRmGYRhHIN0RjtdEJBX4HfApsAX4f90pXEROF5ENIrJJRO7s4vpgEfm3iKwRkYUikhd0bb6IfO4e84PijxaRtW6ZD0ib08UwDMM4JOxVONwNnP6tqjtV9SUc38ZoVf3ZvgoWEQ/wIHAGMBaYJyJjOyW7D3hKVScC9wD3unnTgJ8DM4DpwM9FpJ+b58/A9cAI9zi9OzdqGIZh9Ax7HVWlqgEReRCY4p43A83dLHs6sElVNwOIyPPAeUBhUJqxwHfd8PvAK274a8C7qlrl5n0XOF1EFgLJqrrEjX8KOB94s5s2GYZhhJQybxmvbHoFX8BHjCem4xEZQ6wnlmhPNLGeWGIiY4iJ6BQfGUt0RDS9uTOlO8Nx/y0iFwH/UFXdj7JzcSYLtlGE04IIZjVwIXA/zh4fSSKSvoe8ue5R1EX8bojIDcANAIMG2Q5+hmGElqqmKh5b+xjPb3ieZn9336/3TJvYdBCUNsEJEqJgYeoqzSmDTyElJqUH7nAX3RGOb+K0ClpFpAlnSK6qanIPfP8dwJ9E5CrgA6AYZxHFg0ZVHwYeBsjPz98fwTMMw+g2tS21PLnuSZ4pfIYmfxNnDzubGyfdSG5iLi3+Fpr9zc7R2rwr7G+myd/kxAV2v9aWvsnfRIu/ZddnaxPN/ma8rV6qm6u7TNcaaO1g39TsqYdeOFQ16QDLLgYGBp3nuXHBZZfgtDhwZ6ZfpKo7RaQYOKFT3oVu/rxO8R3KNAzDOBR4fV6e/exZHl/3OHUtdXxtyNe4edLNDEsd1p4mNjKW2MjYQ2qXP+DvIEDpcek9/h37FA4RmdNVfOeNnbpgGTBCRIbiVO6XAt/oVHYGUKWqAeAu4DH30tvAr4Mc4qcBd6lqlYjUishMYCnOmll/3Nc9GIZh9BRNrU28sOEF/lbwN6qaqjgh7wRumXILo9NGh9s0ADwRHuIj4omPig/Zd3Snq+r7QeFYHKf3CuCkvWVS1VYRuRVHBDzAY6q6TkTuAZar6qs4rYp7RURxuqpucfNWicgvcMQH4J42RzlwM/AEEIfjFDfHuGEYIcfn9/Hyppf565q/UuYtY+aAmdw65VYmZU4Kt2mHHNk/fzeIyEDgD6p6UWhM6nny8/N1+fLl4TbDMIw+iD/g5/XNr/Pn1X+muL6YyZmT+fbUbzOt/7RwmxZyRGSFquZ3ju/WIoedKALGHLxJhmEYvZeABnjnq3d4aNVDfFnzJWPSxvDjk3/M7NzZvXqo7KGgOz6OPwJtzZIIYDLODHLDMIzDDlXlv0X/5U8r/8SG6g0MTxnO70/4PScPOvmIF4w2utPiCO7jaQWeU9XFIbLHMAwjLKgqS7Yv4U8r/8SaijUMTBrIvcfdyxlDzsAT4Qm3eb2K7gjHAqBJVf3gLCUiIvGq6g2taYZhGIeGlWUreeDTB1heupz+Cf25+5i7Ofeoc4mKiAq3ab2Sbs0cB04B2nb+iwPeAY4NlVGGYRiHgnWV6/jjyj+yuHgx6bHp3Dn9Ti4eeTHRnuhwm9ar6Y5wxAZvF6uq9SISugHChmEYIebz6s95aNVDvLf1PVJiUrj96NuZN3oecZFx4TatT9Ad4WgQkamq+ik4y5oDjaE1yzAMo+fZWruVB1c9yJtfvkl8VDw3T7qZK8ZeQWJ0YrhN61N0Rzi+A7woIiU461T1x9lK1jAMo0+wvX47f13zV17Z9ApREVFcPf5qrh53NamxqeE2rU/SnbWqlonIaGCUG7VBVX2hNcswDOPgqWis4JE1j/DixhcBuHT0pVw34Toy4jLCbFnfpjvzOG4BnlXVAve8n4jMU9WHQm6dYRjGAbCzaSePrXuM5z57Dl/Ax/lHnc+Nk26kf0L/cJt2WNCdrqrrVfXBthNVrRaR6wETDsMwehV1LXU8Xfg0TxU+hdfn5axhZ3HTpJsYlGx78vQk3REOj4hI2yZO7pawNlbNMIywUtdSx8bqjayvWs+Gqg2sr1rPpp2b8AV8nDr4VG6ZfAvDU4eH28zDku4Ix1vA30Xkr+75N7EVaQ3DOESoKqXeUtZXre8gEkX1uzYDTYtNY3TaaC4fczmnDz2dseljw2jx4U93hOOHOFuw3uier8EZWWUYRphp8bdQ1VRFelz6YTHL2Rfw8WXNl+3isKFqA+ur11PTXAOAIAxOHsy4jHFcNPIiRvUbxei00WTEZdg6UoeQ7oyqCojIUmA48HUgA3gp1IYZhrFnNtds5qWNL/HqF6+ys3kngtAvth/Z8dlkxmeSGZdJVnxW+5EZl0lmfCZpsWlESES4zQegvqWeDdUbuuxqAmfP7ZH9RnLq4FMZ3W80o9JGMbLfyJBuUNQTqCotmzahfj8R8fFExMUhcfFExMchEb3j2R8sexQOERkJzHOPCuDvAKp64qExzTCMYJpam3j3q3dZsHEBn5Z9SqREctKgk5jWfxpVTVWUecsobyyn3FtOQUUBVU1Vu5URKZFkxGeQFZfVLjDZCdntwpIVl0VWQhZJUUk99ga/X11NYy9ndL/RjE4bzaDkQURGHMjOD+Gj4ZNPKH/gARqXr+jyusTGtotJRHwcEh9PRFx8h7iI+HgkLo6I+IQu4trSxwXliUeiD63beW+/ynpgEXC2qm4CEJHbD4lVhmG0s6l6Ey997rQualtqGZQ0iNuPvp1zh5+71/kIvoCPysZKR1C85ZR6SylvLG8/31KzhU92fEJdS91ueWM9sXttubSFO7/9+wI+ttRs6SgSnbqaBiUPYmz6WC4ccSGj00YfFl1N3pUrKX/gAbwfLyEyM5Psu+4kcsAAAl4v2thIwOsl4G0k0NhIwNuwW5xv504CjV4nvRtHINB9AyIj20WkXUzi44iIi2fA3T8nKje3R+93b8JxIc4+4e+LyFvA8zgzxw3DCDGNrY3trYuVZSuJjIjklEGnMHfkXKb1n9at7qaoiCj6J/Tf59yFxtZGKrwVlDWWUeYtaxeWtvPCykIWbltIk79pt7xJUUmOwMRnUtdSx6bqTbQEWgCnq2lE6ghOGXRKu0CM6DeChKiEA3sovZDGtQWU//EBGj5YhCc9naw7f0i/Sy8lIjb2oMpVVbS52RGaBi/a6HVFxxUbr5dAYydRcuOCxcdfXQ0h6B7b59axIpIAnIfTZXUS8BTwsqq+s8/CRU4H7sfZc/xRVf1Np+uDgCeBVDfNnar6hohcRse9zicCU1V1lYgsBAawa72s01S1bG922NaxRl9hY/VGFmxcwOtfvE6dr44hyUOYO3Iu5ww/h7TYtLDZparU++q7bLmUecsoaywjPjKeMWljGJXmOKwHJw/uc11N3aVpwwbKH/gj9f/+N56UFNKuu5a0yy4jIr53+1/2lz1tHbtfe46LSD/gYuASVT15H2k9wEbgVJztZpcB81S1MCjNw8BKVf2ziIwF3lDVIZ3KmQC8oqrD3fOFwB2q2m0lMOEwejNen5e3t7zNgs8XsKZ8DVERUZw6+FTmjpxLfnZ+n+7COdxo3rSJ8j89SN1bbxGRlETa1VeRduWVeBIPz0USe2TPcVWtBh52j30xHdikqptdA57HabkUBqVRINkNpwAlXZQzD6ebzDAOKzZUbeDFjS/yr83/ot5Xz9CUoXw///ucM/wc+sX2C7d5RhAtW7ZQ/uBD1L7+OhFxcaTf+E3Sr74aT0pKuE0LC6FsR+YC24LOi4AZndLcDbwjIt8CEnA2jOrMJTiCE8zjIuLHGRb8S+2i2SQiN+DMP2HQIFtuwOgdeH1e3tryFgs2LmBtxVqiI6I5bchpzB05l6lZU6110ctoKSqi4qE/U/PPfyJRUaRdczXp111HZL8jW9jD3QE5D3hCVf9XRI4BnhaR8aoaABCRGYC3bYFFl8tUtVhEknCE4wocv0sHVLW9ZZSfn9/9/jjDCAGFlYW8tPEl/vXlv2jwNTA8ZTh3Tr+Ts4edTUrMkfnW2pvxbd9OxV/+ys6XXkIiIuh32TfIuP56IjMzw21aryCUwlEMDAw6z3PjgrkWOB1AVT8WkVicCYZtzu5LgeeCM6hqsftZJyL/D6dLbDfhMIxw0+Br4I0v32DBxgUUVhYS44nha0O+xsUjL2ZS5iRrXfRCfGVlVD78CDv//ncUSL14Lhnf/CZR/W2xjGBCKRzLgBEiMhRHMC4FvtEpzVbgZOAJERkDxALlACISgTNT/bi2xCISCaSqaoWIRAFnA++F8B4MY79QVQorC3lx44u88eUbNLY2MqLfCO6afhdnDTvLWhe9lNaqKiofeZTq555DfT5SLjifjBtvIjqvZ+c/HC6ETDhUtVVEbgXexhlq+5iqrhORe4Dlqvoq8D3gEXdioQJXBfkr5gDb2pzrLjHA265oeHBE45FQ3YNhdJf6lvr21sVnVZ8R64nl9KGnM3fkXCZmTLTWRS/Fv3MnlY89TtUzz6BNTaScczYZN99M9ODB4TatV7Nfw3H7KjYc1wgFqsrairUs2LiAt7a8RWNrI6P6jeLikRdz5rAzSYpOCreJxh7w19VR9eRTVD3xBIH6epLPPIOMW24hZrgtwx5MjwzHNYwjnTJvGWvL17K2Yi2LihexsXojcZFxnDn0TOaOnMu49HHWuujFBBoaqHrmWSofe4xATQ2Jp5xM5re+ReyoUfvObLRjwmEYe6DB10BhZSFrytdQUFHAmoo1lHmdcRuREZGMTR/LT2f+lDOHnklidHgmgAVaWtDGxiN2PkF3CTQ2Uv3c81Q++ij+qioSjp9D5re+Tdz4ceE2rU9iwmEYQGuglU07N7G2Ym17i2JzzWYCzshwBiYNJD87nwkZE5iQOYHRaaOJ8cQccjtVlZbNm2n48EPqP1yMd9kytKmJqJwcYseN23WMH3fEzzUACDQ3s/OFF6l4+K/4yytIOPZYMr51K/FTpoTbtD6NCYdxxKGqbG/YzpqKNRSUF7C2Yi2FlYXti/ilxqQyPmM8pw0+jfEZ45mQMYHU2NSw2euvqaHh449pWLyY+g8X07p9OwDRQ4eSOncuUf2zaSospHHdOurefbc9X7uYjB/vCsrYI0ZMtKWFnf94mYq//IXWHTuIz88n8//+j/hp08Jt2mGBCYdx2FPbUktBRQEFFQXtrYnKpkoAoiOiGZ0+mrkj5zI+YzwTMyaSl5QXVj+FtrbSuHYtDR8upuHDD2lcuxYCASKSkkiYOZOEG28kYdasLoeK+mtraSospGndOprWretaTNqF5PATE21tpeafr1Lx0EP4iouJmzSJnF//ivhjjjHfUw9io6qMwwqf38fG6o1Oa6KigDXla9hSu6X9+tCUoU53k3uM7DeSKE/4t1z1lZRQ/+GHjlgsWUKgthYiIoidMJ7EWbNJmD2buIkTkMj9f9fbTUwK1uHburX9elRubsdurl4sJv76BlrLymgtK6W1tBRfaRmtpaW0ljlhX1ER/qoqYseNI/Pb3yJhzhwTjIOgR1bH7auYcByeqCrb6rY5fgn3WF+5vn0/iLTYNCZmTGRCpiMS4zLGkRydvI9SDw0BrxfvsmXUL15Mw4eLadnsTFeKzM4mYfYsEmfPJuGYY/CkhqaLLFhMGgsKaFpX2LWYjB9P7LixxI0bFzJbANTvp7WiwhWBMnylpbS6ouAr2xUONDTsljciMZHI7GyisrOIzMom6ZSTSTz5ZBOMHsCEw4Sjz+MP+FlbsZaPSz5mdcVqCioK2neWi4uMY0zamHbn9YSMCQxIGNBrKg9VpXnjRtep/SGNy1egPh8SE0P8tGntYhE9fHjYbPbX1OwSk3XruhaTIH9Jd8XEX1/vCEJbC6Fsd0ForajYfce7yEgiMzOJzMokKiubyOxsIrOziMrOJjLLDWdlEZFw+GwM1dsw4TDh6JPUNNewuHgxHxR/wOLixexs3okgDE8dzsTMie1+ieGpw3vdpkGtVVU0LP7IEYuPFuMvrwAgZsQIEmbNImH2bOLzjz7o3eJCyW5iUrAO37Zdi15H5eW1d3FFxMU5XUhlZbu6kEpLCXi9u5UbkZzc3kKIzM52xCG7Ley0HjxpaYjHcyhv1+iECYcJR59AVdlYvZFFxYv4oOgDVpevJqABUmNSmZ07mzl5czg259heueaTtrTgXbWqXSyaCgtBFU9KCgmzjiVh1mwSZs8iKjt7t7w+n4+ioiKamnbfnrW3oYEA6vOhLS3Op88Hfv+uBB4PEhHhfHo87rkHPBG7zntJS9BwiI2NJS8vj6iojv4+mzlu9Fq8Pi9Lty/lg+IPWFS0iFJvKQBj0sZw3YTrmJM3h/Hp4/FE9L63z5atW9ud2t4lS5y3a4+HuMmTHefs7NnEjh27zzfnoqIikpKSGDJkSJ+sVLW11QmYKPQ5VJXKykqKiooYOnRot/KYcBhhYVvttnahWLZjGS2BFuIj4zkm5xhuzruZ2bmzyYrPCreZqCqB2lp8JSX4iovxFRfTUlyMr6SE5g0b27ttonJzST7nHBJmzyJh5kw8Sfu3TlVTU1OfFQ3ggEZ7Gb0DESE9PZ3y8vJu57Ff2zgk+Pw+VpSt4IMiRyzahsgOSR7CJaMvYU7eHKZmTSXaE31I7VJV/Dt3BgmD+xkkFIH6+g55JD6e6NxcYkaNJG3+fBJnzyJq8OCDrvT7qmgYfZ/9/dsz4TBCRrm3nEXFi1hUtIiPt39Mg6+BqIgopvWfxqWjL+W43OMYlBzabX1VFX919S5BaDuChaGT8zYiIYGovDyicnKInzaNqNxconJziMpxPj2pqVbJG0c0JhxGj+EP+CmoLGBRkePY/qzqMwCy4rM4Y+gZzMmdw4wBM4iPiu+x71RV/JWVXXYltbUgtLGxQ56I5GRHDAYNIv6YmUTn5rrikEtUTg4RyclHnDBUVlZy8sknA7Bjxw48Hg+Z7japn3zyCdHRe24JLl++nKeeeooHHnhgr99x7LHH8tFHHx20rQsXLuTEE0/kkUce4brrrgNg1apVTJkyhd/97nfccccdXHXVVbz77rts3ryZmJgYKioqyM/PZ8uWLWzZsoWzzz6bgoICvF4v119/PWvWrEFVSU1N5dlnn+W88847oGdxpGDCYRwUNc01fFTyEYuKFvFh8YdUN1cTIRFMypzEbVNv47jc4xjZb2SPVMS+0jK8nyzF++mn+Ip2tRy000gkT0oKUbm5xAwdRuKs2Y4g5O0Shv31PxwJpKens2rVKgDuvvtuEhMTueOOO9qvt7a2ErkHP0Z+fj75+bsNvNmNnhCNNsaPH88LL7zQLhzPPfcckyZN6pDG4/Hw2GOPcdNNN+2xnPvvv5/s7GzWrl0LwIYNG+jfv/9en4VhwmHsJ6rK5zs/b29VrC5fjV/9pMSkOMNlc53hsj2xKGBrdTXeT5bhXbqEhiVL22dXRyQlET14MDEjRpB4/PG7upJyc4nKycWT2LcnhP3Pa+soLKnt0TLH5iTz83P2bwnxq666itjYWFauXMmsWbO49NJLue2222hqaiIuLo7HH3+cUaNGsXDhQu677z5ef/117r77brZu3crmzZvZunUr3/nOd/j2t78NQGJiIvX19SxcuJC7776bjIwMCgoKOProo3nmmWcQEd544w2++93vkpCQwKxZs9i8eTOvv/76brYNHjyY2tpaSktLycrK4q233uLMM8/skOY73/kOv//977n++uv3eI/bt29ncNBuf6NsX45uYcJh7JEGXwPb6raxrW4bRXVFbKndwsclH7O9wVmddXTaaK4Zfw1z8uYwIWPCQQ+X9dfX4122DO+SpTQsXUrz+vUARMTHEzctn9S5c0mYOYOYUaNsYtghoqioiI8++giPx0NtbS2LFi0iMjKS9957jx/96Ee89NJLu+VZv34977//PnV1dYwaNYqbbrppt/kBK1euZN26deTk5DBr1iwWL15Mfn4+3/zmN/nggw8YOnQo8+bN26ttc+fO5cUXX2TKlClMnTqVmJiOy9wPGjSI2bNn8/TTT3POOed0WcY111zDaaedxoIFCzj55JOZP38+I0aM2M+ndOQRUuEQkdOB+3H2B39UVX/T6fog4Ekg1U1zp6q+ISJDgM+ADW7SJap6o5vnaOAJIA54A7hNj4RZjCEgoAHKveUU1Rd1EIiiOue8urm6Q/rUmFSmZk3lhok3cFzucWQn7D6Rbb++v7GRxpUraViylIalS2gqWAd+PxIdTdzUqWR+5zbiZ8wgbvx4JCr8CxEeKva3ZRBKLr74YjyuSNfU1DB//nw+//xzRASfz9dlnrPOOouYmBhiYmLIysqitLSUvLy8DmmmT5/eHjd58mS2bNlCYmIiw4YNa59LMG/ePB5++OE92vb1r3+dSy65hPXr1zNv3rwuu8LuuusuzjvvPM4666wuy5g8eTKbN2/mnXfe4b333mPatGl8/PHHjBkzZt8P5wgmZMIhIh7gQeBUoAhYJiKvqmphULKfAC+o6p9FZCyOEAxxr32hqpO7KPrPwPXAUjf96cCbobmLvk+Lv4Wi+l1iEPxZVF9Es7+5PW2ERDAgYQB5iXmcNOgkBiYNZGDSQPKS8shLyjvoBQK1pcVZLnzJErxLltK4apUz6zgykrgJE0i/4XoSZswkbspkImIO/SZJxu4kBK0D9dOf/pQTTzyRl19+mS1btnDCCSd0mSf4zd/j8dDaNjlwP9Psi/79+xMVFcW7777L/fff36VwjBgxgsmTJ/PCCy/ssZzExEQuvPBCLrzwQiIiInjjjTdMOPZBKFsc04FNqroZQESeB84DgoVDgbbaKAUo2VuBIjIASFbVJe75U8D5HMHCoarUttR2aDG0h+uLKG0oRdnVIIuLjCMvKY/ByYOZlTurgzjkJOT06BLj6vfTVFiId+lSGpYsxbtihTPCSYTYsWPpd+UVJMycSfzUqbZQXR+gpqaG3FxnD5Annniix8sfNWoUmzdvZsuWLQwZMoS///3v+8xzzz33UFZW1t4q6oof//jHe2xxLF68mLFjx9KvXz9aWlooLCzcoyAauwilcOQC24LOi4AZndLcDbwjIt8CEoBTgq4NFZGVQC3wE1Vd5JZZ1KnM3XezAUTkBuAGcPo6+zL+gJ9Sb2m7IHRuOdT56jqkz4jLIC8xj2nZ09pFoe0zPTY9ZENNNRCg+fNN7c5s77JlBOoc22JGHEXqRReRMHMG8dOm2R7ZfZAf/OAHzJ8/n1/+8pd7rIgPhri4OB566CFOP/10EhISmNaN3fqOPfbYfaYZN24cU6dO5dNPP93t2hdffMFNN93krBAQCHDWWWdx0UUXHZD9RxIhW+RQROYCp6vqde75FcAMVb01KM13XRv+V0SOAf4GjAeigERVrXR9Gq8A44CRwG9U9RQ3/3HAD1X17L3Z0pcXOaxorODat69lc83m9rjIiEhyE3OdLqTEvI5dSol5PTpPYm+oKr6vvmr3UXiXfoK/qgqAqEGDSJgxg/iZM0iYPp1Idxy80TWfffaZdY8A9fX1JCYmoqrccsstjBgxgttvvz3cZh0RdPU3GI5FDouBgUHneW5cMNfi+ChQ1Y9FJBbIUNUyoNmNXyEiX+CIRrFbzt7KPGxo8DVw83s3s71hO3dNv4vhqcMZmDSQ7PjssC345yspoWHpJ3iXLKFh6VJad+wAnA2IEo+bTfyMmSTMmE5UbpcNQcPYK4888ghPPvkkLS0tTJkyhW9+85vhNsnoglAKxzJghIgMxancLwW+0SnNVuBk4AkRGQPEAuUikglUqapfRIYBI4DNqlolIrUiMhPHOX4l8McQ3kPY8Pl9fOf977CxeiN/POmPHJd33CG3Qf1+mjdtonHlShpXrsS7clX7xj6efv2c1sSMmSTMnNEjazUZxu23324tjD5AyIRDVVtF5FbgbZyhto+p6joRuQdYrqqvAt8DHhGR23Ec5VepqorIHOAeEfEBAeBGVa1yi76ZXcNx3+QwdIwHNMCPF/+YJduX8MtZvzxkouGvraVx9RpHKFatpHH1mvatOj0ZGcRNnkTa5ZcRP2MmMSOOcvZcMAzjiCOk8zhU9Q2cIbPBcT8LChcCs7rI9xKw+8wi59pyHD/IYYmqct/y+3jzyze5beptnHfUeSH7npYtW2hcuYrGVatoXLmS5k2bQBUiIogZNYqU884lbsoU4iZPJiovz1oUhmEANnO81/Hkuid5uvBpLhtzGdeOv7bHyg00NtK4di2Nq1a3dz35d+4EnEX/4iZPIvnMM4ibPJnYCRP7/LIdhmGEDhOOXsRrX7zG/674X04fcjo/mPaDA37DV1Vat2/Hu3Jlu1A0rV8P7iSr6GHDSDzpJOKmTCZ+yhSihw2zbifDMLqN1Ra9hA+LP+Rni3/GjP4z/v/27j2+xitt+PjvqiIhmiK0SDRxqDiVSMqQ0cq0HYqH4dGS6lQwitE6Vav6Zto4pJ86vK12nCbqkFfNu2vopHji0YphPNW3EkQEbWlEBHVqkUjYSaz3j72zJXKQRLYdcn0/Hx8792Hl2rfItde673UttYMJjAAAGJFJREFUIn8byQNS9n8aY7WSfeAAv0RHkz55Csd6hXDsd89w+o1pXFq/ngfq1KHh6NF4L1tK62930zL2v2j6fiT1X3iB2q30XoWCkJAQtm7dWmjbwoULS60s26tXL/Ifc+/bty+X7D3YgiIiIliwYEGp3zsmJobDh2/OC3733XfZtm1becIv1o4dOxARPv30U8e2xMRERMQRU1hYGM2aNeP6dVsFhQsXLuDr6wtAamoqHTrYRsWzsrIYPnw4HTt2pEOHDvz2t7/lxIkTdO7cmc6dO/Poo4/SrFkzx9dWq/WO46/KtMdRBSRfSGbqjqm0qt+KhSELb7sKXu7Fi477Eln7E7mWnIyx/+DXbNqUOkFBtnsTAZ1xa9NGl/VUtxUaGorFYqF3796ObRaLhXnz5pXp/NjY2NsfVIKYmBj69+9Pu3btANts8Mqi5dedQ3+juNiJKyf487Y/08CtAUufXYpHLY8ix1jTT3H1f3YVeSSWmjVxb9eO+qGhN29iP+L6dbrVHdryNvx8sHLbfLQjPP9BibuHDBlCeHg4VquVWrVqkZqayunTp+nZsyfjx48nPj6e7OxshgwZwsyZM4uc7+vrS0JCAl5eXkRGRhIdHU3jxo3x8fEhMDAQsM3RiIqKwmq10qpVK9asWUNiYiIbN25k586dzJkzhw0bNjB79mz69+/PkCFDiIuLY9q0aeTm5vLkk0+ydOlSateuja+vLyNGjGDTpk3k5OTwj3/8A39//yJxafl159AxChe6kH2BsV/bJjj97bm/4eXuVeSYrH37SBkwgJ8jZpL5zW7c2jxO4zff5LG//502CfH4fm7hkben81Dv32vSUBXWoEEDunbtypYttqfbLRYLL774IiJCZGQkCQkJJCUlsXPnTpKSkkpsZ+/evVgsFhITE4mNjSU+Pt6xb/DgwcTHx3PgwAHatm3LihUr6NGjBwMGDGD+/PkkJibSsmVLx/HXrl0jLCyMzz//nIMHD5Kbm8vSpUsd+728vNi3bx/jx48vdTgsv/z67t27b1t+vSSjRo1i7ty5dO/enfDwcI4ePVryxawGtMfhIpnWTMZvG88v135hZe+VPPbQY0WOydq/n5N/GkPNRx7Be/Eiavn56SOx1UEpPQNnyh+uGjhwIBaLhRUrVgCwbt06oqKiyM3N5cyZMxw+fJgnnnii2DZ27drFoEGDqFPHVvZmwIABjn3JycmEh4dz6dIlMjMzCw2LFeeHH37Az8+Pxx9/HIARI0awePFiJk+eDNgSEUBgYCBffPFFie1o+fXKpz0OF7DmWZn8r8kc+/UYH/X6iA5eRaelZCclcXLMqzzYqBHNV6+mdosWmjSUUw0cOJC4uDj27dtHVlYWgYGBHD9+nAULFhAXF0dSUhL9+vXj2i1L9ZZVWFgYixYt4uDBg7z33nsVbidffs/hdmXZC5Zfz19X/VblKb++ZMkSXn755Tu6r3Ov08Rxl90wN3jnf97hu5+/Y1bwLIKbFZn/SHbyIdL+NIYa9evTPHq1DkGpu8LDw4OQkBBGjRrlWH3vypUr1K1bF09PT86ePesYyirJU089RUxMDNnZ2WRkZLBp0ybHvoyMDJo0aUJOTg5r1651bK9Xrx4ZGRlF2mrTpg2pqakcO3YMgDVr1vD0009X6L3NmjWLuXPn3rb8eklDXt988w2//mpb2Cy//HrBex7VjQ5V3UXGGObFz2Nr6lbeCHyD/2hZdDnLa0eOkDZ6NDXq1eOx6NXUfPRRF0SqqqvQ0FAGDRqExWIBoFOnTgQEBODv74+Pjw/BwUU/6BTUpUsXhg4dSqdOnWjcuHGh0uizZ8+mW7duNGrUiG7dujmSxbBhwxgzZgyffPIJ69evdxzv5ubGqlWreOGFFxw3x8eNG1eh96Xl1yuX08qqVyVVpaz6pwc/5eN9H/PHdn/kzaA3iww9XfvhR9JGjEDc3XlszRpqeWuF2epCy6orVytPWXUdqrpLYo7F8PG+j+nr15dpQdOKJI3rR4+SNnIkUrs2j0Wv1qShlKqyNHHcBf9O/zcRuyP4TZPfMCd4TpFZ4ddTUjgxchRSo4YtadzjKxYqpe5vmjicLOl8EtN2TuPx+o+zMGRhkTW9rx8/TtqIMACaR0dTy17uQCmlqiq9Oe5Exy8fZ0LcBLzcvVjy7BLq1ixccdZ64gRpI8IweXk89n+iqd3Cz0WRKqVU2WnicJJzWecY9/U4HpAH+NuzRWeFW9PTORE2EpOTQ/Po1dRu1cpFkSqlVPlo4nCCDGsG47eN59L1S6zssxKfh3wK7c85dYq0V0ZgsrJoHr0aN/vMWKWUuhfoPY5Kdj3vOhO3TyTlcgofhXxE+4btC+3POXOGEyPCyMvMxGflCtyKKcym1N128eLFCpcIT0hIYOLEibf9HmWZS1EWO3bswNPT0xHfs88+W67zC5ZLv5WWdC8bp/Y4RKQP8DG2Ncc/NcZ8cMv+5kA08LD9mLeNMbEi8hzwAVALsAJvGmO228/ZATQBsu3N/N4Yc86Z76Os8m7kMWPXDBLOJjC351x6NC38HyXn7Flb0rh8meYrV+Levn0JLSl1dzVs2LDUEuG5ubk8WEJ5/qCgIIKCijzqX0RxNaIqqmfPnmzevLnS2sunJd3LxmmJQ0RqAIuB54B0IF5ENtrXGc8XDqwzxiwVkXbY1if3BS4A/2GMOS0iHYCtQMGJDcPta49XGcYYPtjzAV+f+Jo3g96kb4vCpZtzzp0jbUQYeRcv0nzlCtw73rfLpqs7NHfPXL7/5ftKbdO/gT/Tu04v1zlhYWG4ubmxf/9+goODGTZsGJMmTeLatWu4u7uzatUq2rRpw44dO1iwYAGbN28mIiKCtLQ0UlJSSEtLY/LkyY7eiIeHB5mZmezYsYOIiAi8vLxITk4mMDCQzz77DBEhNjaWqVOnUrduXYKDg0lJSSlTgtizZ0+xsR06dIiRI0ditVq5ceMGGzZsoGbNmuTl5TFmzBh2795Ns2bN+PLLL3F3dycsLExLupeBM4equgLHjDEpxhgrYAEG3nKMAR6yv/YETgMYY/YbY07btx8C3EWkNlXY8oPLsfxgIax9GK+0f6XQvtwLF0gLG0nuuXP4LF+O+y2fOpSqqtLT09m9ezcffvgh/v7+7Nq1i/379zNr1izeeeedYs/5/vvv2bp1K3v27GHmzJnk5OQUOWb//v0sXLiQw4cPk5KSwjfffMO1a9cYO3YsW7ZsYe/evZw/f77EuHbt2uUYmomMjCwxtmXLljFp0iQSExNJSEjA29sbgKNHjzJhwgQOHTrEww8/zIYNGwq1ryXdS+fMoapmwMkCX6cD3W45JgL4SkReB+oCxQ1W/iewzxhzvcC2VSKSB2wA5phi6qaIyKvAq2D7x3Gmfx79J3/d/1f6t+jPlMAphfbl/vILaSNHknPmDM2XR1GnS4BTY1H3vvL2DJzphRdecBQGvHz5MiNGjODo0aOISLEJAaBfv37Url2b2rVr07hxY86ePev4hZ2va9eujm2dO3cmNTUVDw8PWrRogZ+f7bH00NBQoqKiiv0etw5VnTx5stjYunfvTmRkJOnp6QwePJjWrVsD4OfnR+fOnQFbWfbU1NRC7WtJ99K5+uZ4KLDaGOMN9AXWiNycVi0i7YG5wNgC5ww3xnQEetr//LG4ho0xUcaYIGNMUKNGjZz2Bnae3MnMb2cS3DSYWcGzCs0Kz/31V9LCRmI9mY7PsmXUKcM4sFJVSd26N+ce/eUvfyEkJITk5GQ2bdpUYln0gp+qSyp5XpZjyqOk2F566SU2btyIu7s7ffv2Zfv27ZXy/at7SXdnJo5TQMHnUL3t2woaDawDMMZ8C7gBXgAi4g38E3jFGPNT/gnGmFP2vzOAv2MbEnOJxHOJTNs5Df8G/nzY60NqPnBzVnjepUukjRqN9cQJfJYuoW43l4WpVKW4fPkyzZrZbjWuXr260ttv06YNKSkpjk//n3/++R3HlpKSQosWLZg4cSIDBw4sdfXCW2PRku4lc2biiAdai4ifiNQChgEbbzkmDXgGQETaYksc50XkYeC/sD1l9U3+wSLyoIjkJ5aaQH8g2YnvoUQpl1J4bftrNK7TmMXPLKZOzTqOfXlXrpA2+k9Yjx3De9Ei6nbv7ooQlapUb731FjNmzCAgIOCOewjFcXd3Z8mSJfTp04fAwEDq1auHp6fnHcW2bt06OnToQOfOnUlOTuaVV14ppZWbCpZ079ixIw888MAdlXT/wx/+UOox+SXdi/PTTz/x9NNP07FjRwICAggKCnJ5SXenllUXkb7AQmyP2q40xkSKyCwgwRiz0f4k1XLAA9uN8reMMV+JSDgwAyh4F+j3wFXg30BNe5vbgKnGmLzS4qjssupnr57l5S0vk5OXw5q+a/Cpd7NjlZeRQdroP3HtyBF8Fv0Vjwp+SlHVi5ZVt8nMzMTDwwNjDBMmTKB169ZMmTLl9ieqO1aesupOncdhjInF9ohtwW3vFnh9GCiyMowxZg4wp4RmAyszxvK6fP0y47aNI8OawareqwonjcyrnBzzKteOHMH74481aShVTsuXLyc6Ohqr1UpAQABjx469/UnqrtOSI+VwLfcaE7dPJPVKKsueXUbbhjez842rVzk5dizZBw/SbOFH1PtdiAsjVereNGXKFO1h3AM0cZRR3o083t71NvvP7WfeU/Po1uTmk8U3srI4OW482YmJNPvfC3jouedcGKlSSjmXqx/HvScYY3j/u/eJS4tjetfp9PHr49h3Izubk3+eQNbevTSdN5eH+vQppSWllLr3aY+jDJYlLWPdj+sY1WEUw9sOd2y/cf066RNeI+u772g69wM8S5jAo5RS9xPtcdzG+h/XsyRxCQNaDmByl8mO7TesVtJfe52r335Lk/ffx3PAABdGqZRSd48mjlJsT9vO7P83m57NehLRIwIRAcBYrZyaOImru3bRZPYsHh5U+jPaSlV1ISEhbN26tdC2hQsXllq1tVevXuQ/5t63b18uXbpU5JiIiIhSazkBxMTEcPjwzdqnlVnaXMuvO6f8ug5VlcAYw/of19O+YXsWPL3AMSvc5OSQPnUqmTt28GhEBA8PGeLiSJW6c6GhoVgsFnr37u3YZrFYmDdvXpnOv5MSGDExMfTv35927doBlVvaXMuvO6f8uiaOEogIC0MWkp2b7ZgVbnJyOPXGNDK3xfHIX8KpP2yoi6NU96Of33+f60cqt6x67bb+PFpCNVuwVXENDw/HarVSq1YtUlNTOX36ND179mT8+PHEx8eTnZ3NkCFDmDlzZpHzfX19SUhIwMvLi8jISKKjo2ncuDE+Pj4EBtqmXi1fvpyoqCisViutWrVizZo1JCYmsnHjRnbu3MmcOXPYsGEDs2fPrrTS5rfS8uuVQ4eqSlGrRi08a9tKHpjcXE5Pn07GV1/xyDszaDB8+G3OVure0aBBA7p27cqWLVsAW2/jxRdfRESIjIwkISGBpKQkdu7cWWq9p71792KxWEhMTCQ2Npb4+HjHvsGDBxMfH8+BAwdo27YtK1asoEePHgwYMID58+eTmJhIy5YtHcdXRmlzLb/uHNrjKAOTl8fpt2dwJXYLjd96iwZlrHejVEWU1jNwpvzhqoEDB2KxWFixYgVgq/cUFRVFbm4uZ86c4fDhwzzxxBPFtrFr1y4GDRpEnTq2XvqAAg+NJCcnEx4ezqVLl8jMzCw0LFacyihtruXXnVN+XXsct2Hy8jjzzv/iyubNNHpjKg1HjXR1SEo5xcCBA4mLi2Pfvn1kZWURGBjI8ePHWbBgAXFxcSQlJdGvX78Sy6nfTlhYGIsWLeLgwYO89957FW4nX1lLmxek5dcrhyaOUpgbNzjz7rtc/vJLGk2aiFcp44tK3es8PDwICQlh1KhRhIaGAnDlyhXq1q2Lp6cnZ8+edQxlleSpp54iJiaG7OxsMjIy2LRpk2NfRkYGTZo0IScnh7Vr1zq216tXj4yMjCJtVWZp83xafr1yaOIogTGGnyNmcnnDF3hNmIBXKU8zKHW/CA0N5cCBA47E0alTJwICAvD39+ell14iOLhITdJCunTpwtChQ+nUqRPPP/88Tz75pGPf7Nmz6datG8HBwYVuEg8bNoz58+cTEBDATz85lt6p1NLm+bT8euVwaln1qqKiZdV/iY4m99dfaTRpkmMOh1LOoGXVlatVmbLq97oGI0a4OgSllKpydKhKKaVUuWjiUKqKqA7DxqpqKu/PnlMTh4j0EZEfROSYiLxdzP7mIvIvEdkvIkn2pWbz982wn/eDiPQua5tK3Yvc3Ny4ePGiJg911xljuHjxIm5ubmU+x2n3OESkBrAYeA5IB+JFZKN9udh84cA6Y8xS+/rjsYCv/fUwoD3QFNgmIo/bz7ldm0rdc7y9vUlPT+f8+fOuDkVVQ25ubo7Z8mXhzJvjXYFjxpgUABGxAAOBgr/kDfCQ/bUncNr+eiBgMcZcB46LyDF7e5ShTaXuOTVr1sTPz8/VYShVJs4cqmoGnCzwdbp9W0ERwMsiko6tt/H6bc4tS5tKKaWcyNU3x0OB1cYYb6AvsEZEKiUmEXlVRBJEJEG7/0opVXmcmThOAT4Fvva2bytoNLAOwBjzLeAGeJVyblnaxN5elDEmyBgT1KhRozt4G0oppQpy2sxxEXkQ+BF4Btsv93jgJWPMoQLHbAE+N8asFpG2QBy2oad2wN+x3ddoat/eGpDbtVlCLOeBExV8K17AhQqeez/S63GTXovC9HoUdj9cj8eMMUU+eTvt5rgxJldEXgO2AjWAlcaYQyIyC0gwxmwE3gCWi8gUbDfKw4wtkx0SkXXYbnrnAhOMMXkAxbVZhlgq3OUQkYTiptxXV3o9btJrUZhej8Lu5+tRLWpV3Yn7+R+/IvR63KTXojC9HoXdz9fD1TfHlVJK3WM0cdxelKsDqGL0etyk16IwvR6F3bfXQ4eqlFJKlYv2OJRSSpWLJg6llFLloomjFFqJ10ZEfOxVjA+LyCERmeTqmKoCEalhr+y82dWxuJqIPCwi60XkexE5IiLdXR2Tq4jIFPv/k2QR+b8iUvays/cITRwlKFDd93lsExJD7VV7q6Nc4A1jTDvgN8CEanwtCpoEHHF1EFXEx8B/G2P8gU5U0+siIs2AiUCQMaYDtvlmw1wbVeXTxFEyR3VfY4wVyK/EW+0YY84YY/bZX2dg+6VQrYtLiog30A/41NWxuJqIeAJPASsAjDFWY8wl10blUg8C7vbqGXW4WfX7vqGJo2RaibcYIuILBADfuTYSl1sIvAXccHUgVYAfcB5YZR+6+1RE6ro6KFcwxpwCFgBpwBngsjHmK9dGVfk0cagyExEPYAMw2RhzxdXxuIqI9AfOGWP2ujqWKuJBoAuw1BgTAFwFquU9QRGpj21kwg9bnb26IvKya6OqfJo4SlbmSrzVgYjUxJY01hpjvnB1PC4WDAwQkVRsQ5i/E5HPXBuSS6UD6caY/F7oemyJpDp6FjhujDlvjMkBvgB6uDimSqeJo2TxQGsR8RORWthucG10cUwuISKCbfz6iDHmQ1fH42rGmBnGGG9jjC+2n4vtxpj77lNlWRljfgZOikgb+6ZnqL6rcqYBvxGROvb/N89wHz4o4MylY+9pJVX3dXFYrhIM/BE4KCKJ9m3vGGNiXRiTqlpeB9baP2SlACNdHI9LGGO+E5H1wD5sTyPu5z4sPaIlR5RSSpWLDlUppZQqF00cSimlykUTh1JKqXLRxKGUUqpcNHEopZQqF00cSlWQiOSJSGKBP5U2W1pEfEUkubLaU6oy6TwOpSou2xjT2dVBKHW3aY9DqUomIqkiMk9EDorIHhFpZd/uKyLbRSRJROJEpLl9+yMi8k8ROWD/k1+iooaILLev7fCViLjbj59oXxslSUQsLnqbqhrTxKFUxbnfMlQ1tMC+y8aYjsAibJV0Af4KRBtjngDWAp/Yt38C7DTGdMJW4ym/QkFrYLExpj1wCfhP+/a3gQB7O+Oc9eaUKonOHFeqgkQk0xjjUcz2VOB3xpgUe3HIn40xDUXkAtDEGJNj337GGOMlIucBb2PM9QJt+AJfG2Na27+eDtQ0xswRkf8GMoEYIMYYk+nkt6pUIdrjUMo5TAmvy+N6gdd53Lwn2Q/b6pRdgHj7gkFK3TWaOJRyjqEF/v7W/no3N5cRHQ7ssr+OA8aDYx1zz5IaFZEHAB9jzL+A6YAnUKTXo5Qz6ScVpSrOvUC1YLCtuZ3/SG59EUnC1msItW97HdsqeW9iWzEvv4LsJCBKREZj61mMx7Z6XHFqAJ/Zk4sAn1TzZVqVC+g9DqUqmf0eR5Ax5oKrY1HKGXSoSimlVLloj0MppVS5aI9DKaVUuWjiUEopVS6aOJRSSpWLJg6llFLloolDKaVUufx/9uzEypF3FswAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(train_accuracy[:, 0], label = 'Training MNIST')\n",
        "plt.plot(val_accuracy[:, 0], label = 'Validation MNIST')\n",
        "plt.plot(train_accuracy[:, 1], label = 'Training Fashion MNIST')\n",
        "plt.plot(val_accuracy[:, 1], label = 'Validation Fashion MNIST')\n",
        "plt.title('Training and Validation Accuracy')\n",
        "plt.legend()\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epochs')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HOxLZCfk3O9O"
      },
      "source": [
        "## Bias vs. Variance"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# storage\n",
        "num_model = 2 # 0 for no regularization 1 for regularization\n",
        "num_rep = 10\n",
        "num_dataset = 2  # 0 for MNIST 1 for FMNIST\n",
        "\n",
        "test_accuracy = np.zeros([num_rep, num_model, num_dataset])\n",
        "used_time = np.zeros([num_rep, num_model])"
      ],
      "metadata": {
        "id": "iEmbNMG3AGyq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for n in range(num_rep):\n",
        "  print(n)\n",
        "\n",
        "  # Set number of epochs\n",
        "  NUM_EPOCHS = 10\n",
        "\n",
        "  # Initialize model without regularization\n",
        "  # , regularizer = 'L2', L1_alpha = 1 , regularizer = 'L2', L1_alpha = 1 , device='gpu' optimizer = 'Adam', \n",
        "  mlp_on_cpu = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output) #, regularizer = 'L2', L2_alpha = 0.1,\n",
        "  mlp_on_cpu_fs = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output) #, lr = 1e-2, device = 'gpu', optimizer = alg) #, regularizer = 'L2', L2_alpha = 0.1,\n",
        "  \n",
        "  time_start = time.time()\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    t = 0\n",
        "    m = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    v = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    u = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    # MNIST\n",
        "    train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(100).batch(batch_size_train)\n",
        "    for inputs, outputs in train_ds:\n",
        "      t += 1\n",
        "      m, v, u = mlp_on_cpu.backward(inputs, outputs, t, m, v, u)\n",
        "\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    t = 0\n",
        "    m = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    v = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    u = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    # FMNIST\n",
        "    train_ds_fs = tf.data.Dataset.from_tensor_slices((X_train_fs, y_train_fs)).shuffle(100).batch(batch_size_train)\n",
        "    for inputs, outputs in train_ds_fs:\n",
        "      t += 1\n",
        "      m, v, u = mlp_on_cpu_fs.backward(inputs, outputs, t, m, v, u)\n",
        "  time_taken = time.time() - time_start\n",
        "\n",
        "  # record time\n",
        "  used_time[n, 0] = time_taken\n",
        "\n",
        "  # test sample\n",
        "  # for MNIST\n",
        "  preds = mlp_on_cpu.forward(X_test)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_test, 1))\n",
        "  test_accuracy[n, 0, 0] = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  # for Fashion MNIST\n",
        "  preds_fs = mlp_on_cpu_fs.forward(X_test_fs)\n",
        "  preds_fs = tf.nn.softmax(preds_fs)\n",
        "  correct_preds_fs = tf.equal(tf.argmax(preds_fs, 1), tf.argmax(y_test_fs, 1))\n",
        "  test_accuracy[n, 0, 1] = tf.reduce_mean(tf.cast(correct_preds_fs, 'float'))\n",
        "\n",
        "\n",
        "  # Initialize model with regularization\n",
        "  # , regularizer = 'L2', L1_alpha = 1 , regularizer = 'L2', L1_alpha = 1 , device='gpu' optimizer = 'Adam', \n",
        "  mlp_on_cpu = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output, regularizer = 'L2', L2_alpha = 0.02)\n",
        "  mlp_on_cpu_fs = MLP_QZ_Custom(size_data, size_input, size_hidden_1, size_hidden_2, size_output, regularizer = 'L2', L2_alpha = 0.02)\n",
        "  \n",
        "  time_start = time.time()\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    t = 0\n",
        "    m = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    v = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    u = [tf.zeros_like(var) for var in mlp_on_cpu.variables]\n",
        "    # MNIST\n",
        "    train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(100).batch(batch_size_train)\n",
        "    for inputs, outputs in train_ds:\n",
        "      t += 1\n",
        "      m, v, u = mlp_on_cpu.backward(inputs, outputs, t, m, v, u)\n",
        "\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "\n",
        "    t = 0\n",
        "    m = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    v = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    u = [tf.zeros_like(var) for var in mlp_on_cpu_fs.variables]\n",
        "    # FMNIST\n",
        "    train_ds_fs = tf.data.Dataset.from_tensor_slices((X_train_fs, y_train_fs)).shuffle(100).batch(batch_size_train)\n",
        "    for inputs, outputs in train_ds_fs:\n",
        "      t += 1\n",
        "      m, v, u = mlp_on_cpu_fs.backward(inputs, outputs, t, m, v, u)\n",
        "  time_taken = time.time() - time_start\n",
        "\n",
        "  # record time\n",
        "  used_time[n, 1] = time_taken\n",
        "    \n",
        "  # test sample\n",
        "  # for MNIST\n",
        "  preds = mlp_on_cpu.forward(X_test)\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(y_test, 1))\n",
        "  test_accuracy[n, 1, 0] = tf.reduce_mean(tf.cast(correct_preds, 'float'))\n",
        "  # for Fashion MNIST\n",
        "  preds_fs = mlp_on_cpu_fs.forward(X_test_fs)\n",
        "  preds_fs = tf.nn.softmax(preds_fs)\n",
        "  correct_preds_fs = tf.equal(tf.argmax(preds_fs, 1), tf.argmax(y_test_fs, 1))\n",
        "  test_accuracy[n, 1, 1] = tf.reduce_mean(tf.cast(correct_preds_fs, 'float'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BjVEU8CtAM-l",
        "outputId": "0cd6f9d9-267b-46b9-f38e-4b3a0411ef7b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "4\n",
            "5\n",
            "6\n",
            "7\n",
            "8\n",
            "9\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_accuracy[:, 1, :]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iaQJwA5CGDpv",
        "outputId": "89532f8c-81f7-4105-8781-104d25bccb16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.86290002, 0.76550001],\n",
              "       [0.85729998, 0.78200001],\n",
              "       [0.83499998, 0.74629998],\n",
              "       [0.87159997, 0.7834    ],\n",
              "       [0.83530003, 0.73729998],\n",
              "       [0.82740003, 0.74940002],\n",
              "       [0.84539998, 0.764     ],\n",
              "       [0.88849998, 0.77179998],\n",
              "       [0.83789998, 0.77249998],\n",
              "       [0.8642    , 0.73250002]])"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "used_time[:, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H_pfPBCV5oI8",
        "outputId": "d1f170ae-aad8-464c-8120-082ca6bacfbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([667.33868337, 676.989254  , 696.44490123, 654.94622135,\n",
              "       657.28889894, 665.99384689, 675.46364832, 693.78219604,\n",
              "       651.33903599, 748.68760395])"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "en4kY94i6HZt"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "IST597_HW3_Custom.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPwQc4EBiSZcCDm7EEp091f",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}