Model_Evaluation_and_Selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c43adbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for array computations and loading data\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "# for building linear regression models and preparing data\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' \n",
    "# for building and training neural networks\n",
    "import tensorflow as tf\n",
    "\n",
    "# custom functions\n",
    "import utils\n",
    "\n",
    "# reduce display precision on numpy arrays\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# suppress warnings\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "tf.autograph.set_verbosity(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01c9801d",
   "metadata": {},
   "source": [
    "### Load and vizualize the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "038a6727",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the shape of the inputs x is: (50, 1)\n",
      "the shape of the targets y is: (50, 1)\n"
     ]
    }
   ],
   "source": [
    "data = np.array([[1651.,    432.65],\n",
    " [1691.82,  454.94],\n",
    " [1732.63,  471.53],\n",
    " [1773.45,  482.51],\n",
    " [1814.27,  468.36],\n",
    " [1855.08,  482.15],\n",
    " [1895.9,   540.02],\n",
    " [1936.71,  534.58],\n",
    " [1977.53,  558.35],\n",
    " [2018.35,  566.42],\n",
    " [2059.16,  581.4 ],\n",
    " [2099.98,  596.46],\n",
    " [2140.8 ,  596.71],\n",
    " [2181.61,  619.45],\n",
    " [2222.43,  616.58],\n",
    " [2263.24,  653.16],\n",
    " [2304.06,  666.52],\n",
    " [2344.88,  670.59],\n",
    " [2385.69,  669.02],\n",
    " [2426.51,  678.91],\n",
    " [2467.33,  707.44],\n",
    " [2508.14,  710.76],\n",
    " [2548.96,  745.19],\n",
    " [2589.78,  729.85],\n",
    " [2630.59,  743.8 ],\n",
    " [2671.41,  738.2 ],\n",
    " [2712.22,  772.95],\n",
    " [2753.04,  772.22],\n",
    " [2793.86,  784.21],\n",
    " [2834.67,  776.43],\n",
    " [2875.49,  804.78],\n",
    " [2916.31,  833.27],\n",
    " [2957.12,  825.69],\n",
    " [2997.94,  821.05],\n",
    " [3038.76,  833.82],\n",
    " [3079.57,  833.06],\n",
    " [3120.39,  825.7 ],\n",
    " [3161.2,   843.58],\n",
    " [3202.02,  869.4 ],\n",
    " [3242.84,  851.5 ],\n",
    " [3283.65,  863.18],\n",
    " [3324.47,  853.01],\n",
    " [3365.29,  877.16],\n",
    " [3406.1,   863.74],\n",
    " [3446.92,  874.67],\n",
    " [3487.73,  877.74],\n",
    " [3528.55,  874.11],\n",
    " [3569.37,  882.8 ],\n",
    " [3610.18,  910.83],\n",
    " [3651.,    897.42]]\n",
    ")\n",
    "\n",
    "# Split the inputs and outputs into separate arrays\n",
    "x = data[:,0]\n",
    "y = data[:,1]\n",
    "\n",
    "# Convert 1-D arrays into 2-D because the commands later will require it\n",
    "x = np.expand_dims(x, axis=1)\n",
    "y = np.expand_dims(y, axis=1)\n",
    "\n",
    "print(f\"the shape of the inputs x is: {x.shape}\")\n",
    "print(f\"the shape of the targets y is: {y.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9e9cde2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGzCAYAAADNKAZOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6y0lEQVR4nO3deXhU1eH/8U8CWQkzAUISggEBU7ZCodhiFMNXSYmIWiW1QvkJVgoqoEUUBVuVqpXFqt8vtIIrtBW1UqoiVRBBiUtE5YGCSzEgEhUCRsgMMWQj5/cHnSlDtjswmeXm/XqePDX3nrlzTsbrfHruWaKMMUYAAAA2Ex3qCgAAALQEQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4QppYvX66oqCh98cUXoa4KAEQkQg4ASz755BPNnTs3rEPXu+++q7lz56qsrCzUVbEk0uoLRBpCDhCmrr76ah09elTdu3cPdVUkHQ85v/vd78I+5Pzud7+LmNAQafUFIg0hBwhTbdq0UXx8vKKiokJdlVbNGKOjR4+GuhoATgEhBwhTDY3JOfPMM3XJJZfo7bff1o9//GPFx8erZ8+e+stf/tLgawsKCnTdddepU6dOcjgcmjBhgg4fPuxTNioqSnPnzq33/meeeaauueYa7/WuvPJKSdIFF1ygqKgoRUVF6c0332yw7n/4wx8UFRWlvXv31js3Z84cxcbGeutRVFSk/Px8paenKz4+XmeccYbGjh0rl8tl8S913Ny5czVr1ixJUo8ePbx19Pz9li1bpgsvvFCpqamKi4tTv379tGTJkgbbfckll2jdunU6++yzlZCQoEcffVSStHfvXl122WVq166dUlNTdfPNN2vdunUN/i02b96siy66SE6nU4mJiRo+fLjeeecdy/UFcPrahroCAPyza9cu/exnP9OkSZM0ceJEPfXUU7rmmms0ZMgQ9e/f36fs9OnTlZycrLlz52rnzp1asmSJ9u7dqzfffNOvHqKcnBzddNNNWrRoke644w717dtXkrz/e7Kf//znuu222/T88897v8g9nn/+eY0cOVIdOnRQdXW18vLyVFVVpRtvvFHp6en6+uuvtWbNGpWVlcnpdFqu45gxY/TZZ5/p2Wef1cMPP6yUlBRJUufOnSVJS5YsUf/+/XXZZZepbdu2evnllzV16lTV1dVp2rRpPtfauXOnxo0bp+uuu06TJ09W79699d133+nCCy/U/v379etf/1rp6el65pln9MYbb9Sry8aNGzVq1CgNGTJEd999t6Kjo70h66233tKPf/zjZusLIAAMgLC0bNkyI8ns2bPHe6x79+5GkikoKPAeO3jwoImLizO33HJLvdcOGTLEVFdXe48vXLjQSDIvvfSS95gkc/fdd9d7/+7du5uJEyd6f1+5cqWRZN544w1L9c/OzjZDhgzxOfb+++8bSeYvf/mLMcaYrVu3Gklm5cqVlq7ZnAceeKDe38yjoqKi3rG8vDzTs2dPn2Oev/HatWt9jj/44INGknnxxRe9x44ePWr69Onj83epq6szWVlZJi8vz9TV1fm8f48ePcxPfvITS/UFcPp4XAVEmH79+un888/3/t65c2f17t1bn3/+eb2yU6ZMUUxMjPf3G264QW3bttUrr7zS4vW86qqrtGXLFu3evdt77G9/+5vi4uL005/+VJK8PTXr1q1TRUVFi9YnISHB+88ul0ulpaUaPny4Pv/883qPxnr06KG8vDyfY2vXrlXXrl112WWXeY/Fx8dr8uTJPuW2bdumoqIi/eIXv9C3336r0tJSlZaW6rvvvtOIESNUUFCgurq6FmghgJMRcoAI061bt3rHOnToUG+sjSRlZWX5/J6UlKQuXboEZdzHlVdeqejoaP3tb3+TdHwA78qVKzVq1Cg5HA5Jx8PEzJkz9cQTTyglJUV5eXn605/+5Pd4HCveeecd5ebmql27dkpOTlbnzp11xx13SFKDIedke/fuVa9eveo95jvrrLN8fi8qKpIkTZw4UZ07d/b5eeKJJ1RVVdUi7QNQH2NygAjTpk2bBo8bYwL6PseOHTut12dkZOj888/X888/rzvuuEPvvfeeiouLtWDBAp9yDz74oK655hq99NJLeu2113TTTTdp3rx5eu+993TGGWecVh08du/erREjRqhPnz566KGHlJmZqdjYWL3yyit6+OGH6/WsnNjr4y/PtR544AENGjSowTJJSUmnfH0A1hFyABsrKirSBRdc4P29vLxc+/fv18UXX+w91qFDh3rrtFRXV2v//v0+x05lKvtVV12lqVOnaufOnfrb3/6mxMREXXrppfXKDRgwQAMGDNBvf/tbvfvuuzrvvPO0dOlS3XfffX69X2N1fPnll1VVVaXVq1f79IQ1NGi4Md27d9cnn3wiY4zP++zatcunXK9evSRJDodDubm5p1RfAIHB4yrAxh577DHV1NR4f1+yZIlqa2s1atQo77FevXqpoKCg3utO7slp166dJPm1cF1+fr7atGmjZ599VitXrtQll1zivY4kud1u1dbW+rxmwIABio6OVlVVlfdYcXGx/v3vfzf7fo3V0dP7dWJvl8vl0rJlyyy3JS8vT19//bVWr17tPVZZWanHH3/cp9yQIUPUq1cv/eEPf1B5eXm963zzzTfN1hdAYNCTA9hYdXW1RowYoZ///OfauXOnHnnkEQ0bNsxn8OyvfvUrXX/99crPz9dPfvIT/etf/9K6deu8U5o9Bg0apDZt2mjBggVyuVyKi4vzrjvTmNTUVF1wwQV66KGHdOTIEV111VU+5zdu3Kjp06fryiuv1Pe+9z3V1tbqr3/9q9q0aaP8/HxvuQkTJmjTpk3NPpIbMmSIJOk3v/mNxo4dq5iYGF166aUaOXKkYmNjdemll+q6665TeXm5Hn/8caWmptbrsWrMddddpz/+8Y8aN26cfv3rX6tLly5asWKF4uPjJf23VyY6OlpPPPGERo0apf79++uXv/ylunbtqq+//lpvvPGGHA6HXn755Sbre2IQBHAaQjq3C0CjGptCPnr06Hplhw8fboYPH17vtZs2bTJTpkwxHTp0MElJSWb8+PHm22+/9XntsWPHzO23325SUlJMYmKiycvLM7t27ao3hdwYYx5//HHTs2dP06ZNG8vTyR9//HEjybRv394cPXrU59znn39urr32WtOrVy8THx9vOnbsaC644ALz+uuv12uf1f9c3XvvvaZr164mOjra5++3evVqM3DgQBMfH2/OPPNMs2DBAvPUU09Z/ht76jt69GiTkJBgOnfubG655RazatUqI8m89957PmW3bt1qxowZYzp16mTi4uJM9+7dzc9//nOzYcMGS/UFcPqijAnwaEUAIbd8+XL98pe/1AcffKCzzz471NWxtf/93//VzTffrK+++kpdu3YNdXUAnIAxOQBg0cl7WFVWVurRRx9VVlYWAQcIQ4zJAQCLxowZo27dumnQoEFyuVx6+umn9e9//1srVqwIddUANICQAwAW5eXl6YknntCKFSt07Ngx9evXT88991y9AdUAwgNjcgAAgC0xJgcAANgSIQcAANhSRI7Jqaur0759+9S+fXuWRQcAIEIYY3TkyBFlZGQoOrrl+1kiMuTs27dPmZmZoa4GAAA4BV9++WXANuBtSkSGnPbt20s6/kdyOBwhrg0AALDC7XYrMzPT+z3e0iIy5HgeUTkcDkIOAAARJlhDTRh4DAAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbImQAwAAbCkiVzwGAAD+cVVUq7S8Wu7KGjkSYpTSLlbOxNhQV6tFEXIAALC5fWVHdfuq7XqrqNR7LCcrRfPzByojOSGENWtZPK4CAMDGXBXV9QKOJBUUlWr2qu1yVVSHqGYtj5ADAICNlZZX1ws4HgVFpSotJ+QAAIAI5K6safL8kWbORzJCDgAANuaIj2nyfPtmzkcyQg4AADaWkhSrnKyUBs/lZKUoJcm+M6wIOQAA2JgzMVbz8wfWCzo5WSlakD/Q1tPI/Q45R44c0YwZM9S9e3clJCTo3HPP1QcffOA9b4zRXXfdpS5duighIUG5ubkqKiryucahQ4c0fvx4ORwOJScna9KkSSovLz/91gAAgHoykhO0eNxgbZg5XC9OPVcbZg7X4nGD1cXG08elUwg5v/rVr7R+/Xr99a9/1Y4dOzRy5Ejl5ubq66+/liQtXLhQixYt0tKlS7V582a1a9dOeXl5qqys9F5j/Pjx+vjjj7V+/XqtWbNGBQUFmjJlSuBaBQAAfDgTY9UrNUmDunVQr9QkW/fgeEQZY4zVwkePHlX79u310ksvafTo0d7jQ4YM0ahRo3TvvfcqIyNDt9xyi2699VZJksvlUlpampYvX66xY8fq008/Vb9+/fTBBx/o7LPPliStXbtWF198sb766itlZGQ0Ww+32y2n0ymXyyWHw+FvmwEAQCNacmXkYH9/+7XicW1trY4dO6b4+Hif4wkJCXr77be1Z88elZSUKDc313vO6XRq6NChKiws1NixY1VYWKjk5GRvwJGk3NxcRUdHa/PmzbriiivqvW9VVZWqqqq8v7vdbn+qDQAALLDbysh+Pa5q3769srOzde+992rfvn06duyYnn76aRUWFmr//v0qKSmRJKWlpfm8Li0tzXuupKREqampPufbtm2rjh07esucbN68eXI6nd6fzMxMf6oNAACaYceVkf0ek/PXv/5Vxhh17dpVcXFxWrRokcaNG6fo6JabqDVnzhy5XC7vz5dfftli7wUAQGtkx5WR/U4mvXr10qZNm1ReXq4vv/xS77//vmpqatSzZ0+lp6dLkg4cOODzmgMHDnjPpaen6+DBgz7na2trdejQIW+Zk8XFxcnhcPj8AACAwLHjysin3P3Srl07denSRYcPH9a6dev005/+VD169FB6ero2bNjgLed2u7V582ZlZ2dLkrKzs1VWVqYtW7Z4y2zcuFF1dXUaOnToaTQFAACcKjuujOzXwGNJWrdunYwx6t27t3bt2qVZs2apT58++uUvf6moqCjNmDFD9913n7KystSjRw/deeedysjI0OWXXy5J6tu3ry666CJNnjxZS5cuVU1NjaZPn66xY8damlkFAAACz7MyckEDj6widWVkv3tyXC6Xpk2bpj59+mjChAkaNmyY1q1bp5iY4wnvtttu04033qgpU6boRz/6kcrLy7V27VqfGVkrVqxQnz59NGLECF188cUaNmyYHnvsscC1CgAA+MWOKyP7tU5OuGCdHAAAWoZnnZwjlTVqHx+jlKRWsk4OAACwN2di4EJNqBFyAACIcC25SnEkI+QAABDB7LZKcSC13Ap+AACgRdlxleJAIuQAABCh7LhKcSDxuAoAgP8I9tiW030/O65SHEiEHAAAFPyxLYF4PzuuUhxIPK4CALR6LTG2xVVRrd0Hy7W1+LB2f1Pucw1/3q+p63hWKW5IpK5SHEj05AAAWj0rY1v8eYzUXC+N1fdr7jqeVYpnr9rusx1DJK9SHEiEHABAqxfIsS3N9dIsHjfY0vtZuY4zMVYZyQlaPG5wi61SHMkIOQCAVi+QY1us9NJYeT9/epfstEpxIDEmBwAQEk2NNQm2QI5tsdJLY+X9mDl1+gg5AICg21d2VNOf3aoRD23SFY+8qxEPbtKNz27VvrKjIalPIHfgttJLY+X9mDl1+nhcBQAIKqtjTYItUGNbPL00BQ08ajqxV6i597N6HTSOnhwAQFCF8yq9zsRY9UpN0qBuHdQrNemUwpY/vUJNvV8ge5daK3pyAABB1RJjTYK5UrGV9wpUrxAzp04PIQcAEFSBHmsSzJWK/XmvQM14YubUqeNxFQAgqAI5kymYu3Cz43fkIeQAAIIqkGNNgjm+J5zHEqFhPK4CAARdoMaaBHMtGdatiTyEHABASARirEkw15Jh3ZrIw+MqAEDECuYu3Oz4HXkIOQCAiOXP+J7T3UaCdWsiT5QxxoS6Ev5yu91yOp1yuVxyOByhrg4AoAVZWZfGU6ax8T2BnGbe3HuhccH+/ibkAADCViDCiauiWtOf3drgzKicrJSQbSPRGgX7+5vHVQCAsBSodWmY+t16EXIAAGEpUOGEqd+tFyEHABCWAhVOmPrdehFyAABhKVDhhKnfrRchBwAQcKc7XVsKXDhh6nfrxewqAEBABXK69r6yo5q9arsKTrrWgvyB6sLU74jDFHILCDkAEJ5aYro24cQ+gv39zd5VAICAsTIjyt+AEog9rtA6MSYHABAwTNdGOCHkAAAChunaCCeEHABAwDBdG+GEMTkAECJWNp4M9vudbp0807UbmxHF2BoEEyEHAEIgkNOsA/V+gapTRnKCFo8bzIwohBxTyAEgyIK9K7aV95PETt1ocexCDgA2F+xdsa28Hzt1w454XAUAQRbsadZW3q+5Ln2mfiMSEXIAIMj8mWYdiMHJgZjWHeg6AcFAyAGAIPNMsy5oZPyLZ5p1oAYCW32/YNYJCAbG5ABAkFnZFdtVUV0vTEjHx8fMXrXdr129rbxfsOsEBAM9OQAQAs1Nsw70HlBWpnUHu05ASyPkAECINLXxpD+Dk62OkbGy0WWg6gSEA0IOAIQhq4OFgzlGhn2pEGkYkwMAYcjKHlDBHiPDvlSINIQcAAhDVgYCB3sBPyt1AsIJj6sAIEw1NxA4FGNk2JcKkYSQAwBhrKmBwKEaI2NlADMQDnhcBQARijEyQNMIOQAQoRgjAzSNx1UAEMEYIwM0jpADABGOMTJAw3hcBQAAbImeHABoAVa3WgDQcgg5ABBgwdxqAUDjeFwFAAEU7K0WADSOkAMAARTsrRYANM6vkHPs2DHdeeed6tGjhxISEtSrVy/de++9MsZ4yxhjdNddd6lLly5KSEhQbm6uioqKfK5z6NAhjR8/Xg6HQ8nJyZo0aZLKy8sD0yIACKFQbLUAoGF+hZwFCxZoyZIl+uMf/6hPP/1UCxYs0MKFC7V48WJvmYULF2rRokVaunSpNm/erHbt2ikvL0+VlZXeMuPHj9fHH3+s9evXa82aNSooKNCUKVMC1yoACJFQbbUAoL4oc2I3TDMuueQSpaWl6cknn/Qey8/PV0JCgp5++mkZY5SRkaFbbrlFt956qyTJ5XIpLS1Ny5cv19ixY/Xpp5+qX79++uCDD3T22WdLktauXauLL75YX331lTIyMuq9b1VVlaqqqry/u91uZWZmyuVyyeFwnHLjASDQXBXVuvHZrSpo4JFVTlaKFo8bzCwrtFput1tOpzNo399+9eSce+652rBhgz777DNJ0r/+9S+9/fbbGjVqlCRpz549KikpUW5urvc1TqdTQ4cOVWFhoSSpsLBQycnJ3oAjSbm5uYqOjtbmzZsbfN958+bJ6XR6fzIzM/1rJQAECVstAOHDrynks2fPltvtVp8+fdSmTRsdO3ZMv//97zV+/HhJUklJiSQpLS3N53VpaWnecyUlJUpNTfWtRNu26tixo7fMyebMmaOZM2d6f/f05ABAOGKrBSA8+BVynn/+ea1YsULPPPOM+vfvr23btmnGjBnKyMjQxIkTW6qOiouLU1xcXItdHwACja0WgNDzK+TMmjVLs2fP1tixYyVJAwYM0N69ezVv3jxNnDhR6enpkqQDBw6oS5cu3tcdOHBAgwYNkiSlp6fr4MGDPtetra3VoUOHvK8HAAA4XX6NyamoqFB0tO9L2rRpo7q6OklSjx49lJ6erg0bNnjPu91ubd68WdnZ2ZKk7OxslZWVacuWLd4yGzduVF1dnYYOHXrKDQEAADiRXz05l156qX7/+9+rW7du6t+/v7Zu3aqHHnpI1157rSQpKipKM2bM0H333aesrCz16NFDd955pzIyMnT55ZdLkvr27auLLrpIkydP1tKlS1VTU6Pp06dr7NixDc6sAgAAOBV+hZzFixfrzjvv1NSpU3Xw4EFlZGTouuuu01133eUtc9ttt+m7777TlClTVFZWpmHDhmnt2rWKj4/3llmxYoWmT5+uESNGKDo6Wvn5+Vq0aFHgWgUAAFo9v9bJCRfBnmcPoHVg53CgZQX7+5tdyAFA7BwO2BEbdAJo9dg5HLAnQg6AVo+dwwF7IuQAaPXYORywJ0IOgFaPncMBeyLkAGj1UpJi622o6ZGTlaKUJN8ZVq6Kau0+WK6txYe1+5tyxuwAYYrZVQBaPc/O4bNXbVfBSbOrTt45nFlYQORgnRwAES9Q69t4rtPYzuGuimpNf3Zrg4OUc7JStHjcYNbVAZrAOjkA4AerPStWglBzO4dbmYVFyAHCByEHQMRqbn0bT89KoB4xMQsLiCwMPAYQsaz0rARyoT9mYQGRhZADIGJZ6VkJ5EJ//s7CAhBahBwAEctKz0ogHzF5ZmGdHHQamoUFIPQYkwMgYnl6Vgoame1kpWfF30dMGckJWjxucJOzsACEB3pyAEQsKz0rLfGIyZkYq16pSRrUrYN6pSYRcIAwxTo5ACJec+vb7Cs72uhCf11YwA8IGtbJAQA/Nbe+DY+YgNaJkAOgVWguCAGwH8bkAAAAWyLkAAAAWyLkAAAAWyLkAAAAWyLkAAAAWyLkAAAAWyLkAAAAWyLkAAAAWyLkAAAAW2LFYwB+8ewT5a6skSMhRintWEkYQHgi5ACwbF/ZUd2+arveOmmjy/n5A5Xh50aXhCUALY2QA8ASV0V1vYAjSQVFpZq9arsWjxtsOaQEMiwBQGMYkwPAktLy6noBx6OgqFSl5dWWrtNcWHJVWLsOADSHkAPAEndlTZPnjzRz3iNQYQkAmsPjKgCWOOJjmjzfvpnzHv6GJcbuADhVhBwAlqQkxSonK0UFDfTC5GSlKCXJWvDwJywxdgfA6eBxFQBLnImxmp8/UDlZKT7Hc7JStCB/oOXeFU9YasiJYYmxOwBOFz05ACzLSE7Q4nGDVVperSOVNWofH6OUJP8eH3nC0uxV2316hU4OS1bG7vDYCkBTCDkA/OJMPP0xMVbCUqAGOgNovQg5AEKiubAUqIHOAFovxuQACDhXRbV2HyzX1uLD2v1N+SmNn7E6dgcAGkNPDoCACtSMKKtjdwCgMVHGGBPqSvjL7XbL6XTK5XLJ4XCEujoA/sNVUa3pz25tcMBwTlaKX1s/nHjN0xnoDCB8BPv7m54cAAHTEjOiAjHQGUDrxJgcAAHDjCgA4YSQAyBgmBEFIJwQcgAEDDOiAIQTQg6AgAnU1g8AEAgMPAYQUIHY+gEAAoGQAyDgmBEFIBzwuAoAANgSIQcAANgSIQcAANgSY3IAeHm2UHBX1siREKOUdoytARC5CDkAJAVuY00ACBc8rgJaCVdFtXYfLNfW4sPa/U25XBXVPudODjjS8f2mZq/a7lMWACIFPTlAK9BcL01LbKwJAKFGTw5gc1Z6adhYE4AdEXIAm7PSS8PGmgDsiJAD2JyVXho21gRgR4QcwOas9NKwsSYAO/Ir5Jx55pmKioqq9zNt2jRJUmVlpaZNm6ZOnTopKSlJ+fn5OnDggM81iouLNXr0aCUmJio1NVWzZs1SbW1t4FoEwIfVXhrPxpobZg7Xi1PP1YaZw7V43GB1Yfo4gAjlV8j54IMPtH//fu/P+vXrJUlXXnmlJOnmm2/Wyy+/rJUrV2rTpk3at2+fxowZ4339sWPHNHr0aFVXV+vdd9/Vn//8Zy1fvlx33XVXAJsE4ET+9NI4E2PVKzVJg7p1UK/UJHpwAES0KGOMOdUXz5gxQ2vWrFFRUZHcbrc6d+6sZ555Rj/72c8kSf/+97/Vt29fFRYW6pxzztGrr76qSy65RPv27VNaWpokaenSpbr99tv1zTffKDbW2n9Q3W63nE6nXC6XHA7HqVYfaFU8qxkfqaxR+/gYpSSxmjGA4Ar29/cpj8mprq7W008/rWuvvVZRUVHasmWLampqlJub6y3Tp08fdevWTYWFhZKkwsJCDRgwwBtwJCkvL09ut1sff/xxo+9VVVUlt9vt8wPAP/TSAGhtTjnkvPjiiyorK9M111wjSSopKVFsbKySk5N9yqWlpamkpMRb5sSA4znvOdeYefPmyel0en8yMzNPtdoAAKCVOOWQ8+STT2rUqFHKyMgIZH0aNGfOHLlcLu/Pl19+2eLvCQAAItspbeuwd+9evf766/rHP/7hPZaenq7q6mqVlZX59OYcOHBA6enp3jLvv/++z7U8s688ZRoSFxenuLi4U6kqAABopU6pJ2fZsmVKTU3V6NGjvceGDBmimJgYbdiwwXts586dKi4uVnZ2tiQpOztbO3bs0MGDB71l1q9fL4fDoX79+p1qGwAAAOrxuyenrq5Oy5Yt08SJE9W27X9f7nQ6NWnSJM2cOVMdO3aUw+HQjTfeqOzsbJ1zzjmSpJEjR6pfv366+uqrtXDhQpWUlOi3v/2tpk2bRk8NAAAIKL9Dzuuvv67i4mJde+219c49/PDDio6OVn5+vqqqqpSXl6dHHnnEe75NmzZas2aNbrjhBmVnZ6tdu3aaOHGi7rnnntNrBQAAwElOa52cUGGdHAAAIk/ErJMDAAAQzgg5AADAlgg5AADAlk5pnRwA4cOzJ5W7skaOhBiltGNPKgCQCDlARNtXdlS3r9qut4pKvcdyslI0P3+gMpITQlgzAAg9HlcBEcpVUV0v4EhSQVGpZq/aLldFdYhqBgDhgZADRKjS8up6AcejoKhUpeWEHACtGyEHiFDuypomzx9p5jwA2B0hB4hQjviYJs+3b+Y8ANgdIQcIEVdFtXYfLNfW4sPa/U2532NoUpJilZOV0uC5nKwUpSQxwwpA68bsKiAErM6Kamp6uDMxVvPzB2r2qu0qOOk6C/IHMo0cQKvH3lVAkLkqqjX92a0NDhrOyUrR4nGD5UyM9TsIHamsUfv4GKUksU4OgPAU7O9venKAFtBUD4zVWVFNTQ/3BCHpeI8OoQYA6iPkAAHWXA+M1VlRzQUhgg0ANI2Bx0AAWVmgz8qsKKaHA8DpI+QAAWTlUZSVWVFMDweA00fIAQLISg+MZ1bUyUHnxFlRTA8HgNPHmBwggKz2wGQkJ2jxuMGNzopiejgAnD5CDhBAnh6Ygkamh5/YA9PcrKjmghAAoGk8rgICyMqjKH+v1ys1SYO6dVCv1CQCDgD4gZ4cIMDogQGA8EDIAVoAC/QBQOgRcgA/NbWaMQAgfBByAD9Y3U8KABB6DDwGLLKymjEAIHwQcgCLrG6sCQAID4QcwCL2kwKAyELIASxiPykAiCyEHMAi9pMCgMhCyAEsCvRqxgCAlsUUcsAPrGYMAJGDkAP4idWMASAy8LgKAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEiEHAADYEruQo1VwVVSrtLxa7soaORJilNKOncQBwO4IObC9fWVHdfuq7XqrqNR7LCcrRfPzByojOSGENQMAtCQeV8HWXBXV9QKOJBUUlWr2qu1yVVT7lN19sFxbiw9r9zflPucAAJGHnhzYWml5db2A41FQVKrS8mo5E2Pp7QEAG6InB7bmrqxp8vyRyhq/ensAAJGDkANbc8THNHm+fXyMpd4eAEDkIeTA1lKSYpWTldLguZysFKUkxVrq7QEARB5CDmzNmRir+fkD6wWdnKwULcgfKGdirKXeHgBA5GHgMWwvIzlBi8cNVml5tY5U1qh9fIxSkv67To6nt6eggUdWnt4eAEDkoScHrYIzMVa9UpM0qFsH9UpN8lkI0EpvDwAg8tCTA6j53h4AQOQh5AD/4Uwk1ACAnfC4CgAA2BIhBwAA2JLfIefrr7/W//t//0+dOnVSQkKCBgwYoA8//NB73hiju+66S126dFFCQoJyc3NVVFTkc41Dhw5p/PjxcjgcSk5O1qRJk1ReXn76rQEAAPgPv0LO4cOHdd555ykmJkavvvqqPvnkEz344IPq0KGDt8zChQu1aNEiLV26VJs3b1a7du2Ul5enyspKb5nx48fr448/1vr167VmzRoVFBRoypQpgWsVAABo9aKMMcZq4dmzZ+udd97RW2+91eB5Y4wyMjJ0yy236NZbb5UkuVwupaWlafny5Ro7dqw+/fRT9evXTx988IHOPvtsSdLatWt18cUX66uvvlJGRkaz9XC73XI6nXK5XHI4HFarjwjkqqhWaXm13JU1ciTEKKVd/cHBVsoAAEIv2N/ffs2uWr16tfLy8nTllVdq06ZN6tq1q6ZOnarJkydLkvbs2aOSkhLl5uZ6X+N0OjV06FAVFhZq7NixKiwsVHJysjfgSFJubq6io6O1efNmXXHFFfXet6qqSlVVVd7f3W633w1F5LGyMzi7hwMAGuPX46rPP/9cS5YsUVZWltatW6cbbrhBN910k/785z9LkkpKSiRJaWlpPq9LS0vznispKVFqaqrP+bZt26pjx47eMiebN2+enE6n9yczM9OfaiMCWdkZnN3DAQBN8Svk1NXV6Yc//KHuv/9+DR48WFOmTNHkyZO1dOnSlqqfJGnOnDlyuVzeny+//LJF3w+hZ2VncHYPBwA0xa+Q06VLF/Xr18/nWN++fVVcXCxJSk9PlyQdOHDAp8yBAwe859LT03Xw4EGf87W1tTp06JC3zMni4uLkcDh8fmBvVnYGZ/dwAEBT/Ao55513nnbu3Olz7LPPPlP37t0lST169FB6ero2bNjgPe92u7V582ZlZ2dLkrKzs1VWVqYtW7Z4y2zcuFF1dXUaOnToKTcE9mJlZ3B2DwcANMWvkHPzzTfrvffe0/33369du3bpmWee0WOPPaZp06ZJkqKiojRjxgzdd999Wr16tXbs2KEJEyYoIyNDl19+uaTjPT8XXXSRJk+erPfff1/vvPOOpk+frrFjx1qaWYXWwbMzeEM8O4NbKQMAaL38mkIuSWvWrNGcOXNUVFSkHj16aObMmd7ZVdLxaeR33323HnvsMZWVlWnYsGF65JFH9L3vfc9b5tChQ5o+fbpefvllRUdHKz8/X4sWLVJSUpKlOjCFvHXYV3ZUs1dtV8FJM6cW5A9UlxNmVzVXBgAQHoL9/e13yAkHhJzWw7MGTlM7g1spAwAIvbBeJwcINis7g7N7OACgIWzQCQAAbImeHIQM2zEAAFoSIQchwXYMAICWxuMqBB3bMQAAgoGQg6BjOwYAQDAQchB0bMcAAAgGQg6Cju0YAADBQMhB0LEdAwAgGAg5CDpnYqzm5w+sF3Q82zEwjRwAEAhMIUdIZCQnaPG4wWzHAABoMYQchAzbMQAAWhIhBwHHSsYAgHBAyEFAsZIxACBcMPAYAcNKxgCAcELIQcCwkjEAIJwQchAwrGQMAAgnhBwEDCsZAwDCCSEHfnFVVGv3wXJtLT6s3d+U+4yzYSVjAEA4YXYVLGtu5pRnJePZq7ar4KQyrGQMAAi2KGOMCXUl/OV2u+V0OuVyueRwOEJdnVbBVVGt6c9ubXBgcU5WihaPG+wNMZ51cljJGABwomB/f9OTA0uszJzyBBlWMgYAhAPG5MASZk4BACINIQeWMHMKABBpCDmwhJlTAIBIQ8iBJZ6ZUycHHWZOAQDCFQOPYVlGcoIWjxvMzCkAQEQg5MAvzJwCAEQKHlcBAABbIuQAAABbIuQAAABbYkxOK+HZasFdWSNHQoxS2jG2BgBgb4ScVqC5jTUBALAjHlfZnKuiul7AkY7vNzV71Xa5KqpDVDMAAFoWIcfmrGysCQCAHfG4yub82ViTcTsAADsh5Nic1Y01GbcDALAbHlfZnJWNNRm3AwCwI0KOzVnZWJNxOwAAO+JxVSvQ3Maa/ozbAQAgUhByWommNta0Om4HAIBIwuMqWBq3AwBApCHkwNK4HQAAIg2PqyCp+XE7AABEGkIOvJoatwMAQKThcRUAALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlQg4AALAlv0LO3LlzFRUV5fPTp08f7/nKykpNmzZNnTp1UlJSkvLz83XgwAGfaxQXF2v06NFKTExUamqqZs2apdra2sC0BgAA4D/a+vuC/v376/XXX//vBdr+9xI333yz/vnPf2rlypVyOp2aPn26xowZo3feeUeSdOzYMY0ePVrp6el69913tX//fk2YMEExMTG6//77A9AcAACA4/wOOW3btlV6enq94y6XS08++aSeeeYZXXjhhZKkZcuWqW/fvnrvvfd0zjnn6LXXXtMnn3yi119/XWlpaRo0aJDuvfde3X777Zo7d65iY2NPv0UAAAA6hTE5RUVFysjIUM+ePTV+/HgVFxdLkrZs2aKamhrl5uZ6y/bp00fdunVTYWGhJKmwsFADBgxQWlqat0xeXp7cbrc+/vjjRt+zqqpKbrfb5wcAAKApfoWcoUOHavny5Vq7dq2WLFmiPXv26Pzzz9eRI0dUUlKi2NhYJScn+7wmLS1NJSUlkqSSkhKfgOM57znXmHnz5snpdHp/MjMz/ak2AABohfx6XDVq1CjvPw8cOFBDhw5V9+7d9fzzzyshISHglfOYM2eOZs6c6f3d7XYTdAAAQJNOawp5cnKyvve972nXrl1KT09XdXW1ysrKfMocOHDAO4YnPT293mwrz+8NjfPxiIuLk8Ph8PkBAABoymmFnPLycu3evVtdunTRkCFDFBMTow0bNnjP79y5U8XFxcrOzpYkZWdna8eOHTp48KC3zPr16+VwONSvX7/TqQoAAIAPvx5X3Xrrrbr00kvVvXt37du3T3fffbfatGmjcePGyel0atKkSZo5c6Y6duwoh8OhG2+8UdnZ2TrnnHMkSSNHjlS/fv109dVXa+HChSopKdFvf/tbTZs2TXFxcS3SQAAA0Dr5FXK++uorjRs3Tt9++606d+6sYcOG6b333lPnzp0lSQ8//LCio6OVn5+vqqoq5eXl6ZFHHvG+vk2bNlqzZo1uuOEGZWdnq127dpo4caLuueeewLYKAAC0elHGGBPqSvjL7XbL6XTK5XIxPgcAgAgR7O9v9q4CAAC25PeKxwgMV0W1Ssur5a6skSMhRintYuVMZMVnAAAChZATAvvKjur2Vdv1VlGp91hOVorm5w9URrL/6w0RmAAAqI+QE2Suiup6AUeSCopKNXvVdi0eN9ivgBLowAQAgF0wJifISsur6wUcj4KiUpWWV1u+VnOByVVh/VoAANgNISfI3JU1TZ4/0sz5EwUyMAEAYDeEnCBzxMc0eb59M+dPFMjABACA3RBygiwlKVY5WSkNnsvJSlFKkvXxOIEMTAAA2A0hJ8icibGanz+wXtDJyUrRgvyBfg06DmRgAgDAbljxuAVYmdLtKXOkskbt42OUknRq0773lR3V7FXbVXDS7KoF+QPVhdlVAIAwEuzvb6aQB5jVKd3OxMCsZZORnKDF4wYHJDABAGAnhJwACvQaOFYX+QtUYAIAwE4IOQFkZUq31TDCIn8AAJweBh4HUKCmdLPIHwAAp4+QE0CBmtLNIn8AAJw+Qk4ABWpKN4v8AQBw+gg5ARSoNXBY5A8AgNPHwOMAC8SUbk+PUEEDj6xY5A8AAGvoyWkBzsRY9UpN0qBuHdQrNcnv6d2BXBUZAIDWip6cMMUifwAAnB5CThhjkT8AAE4dIcdPVlchBgAAoUXI8QOrEAMAEDkYeGwRqxADABBZCDkWsQoxAACRhZBjEasQAwAQWQg5FrEKMQAAkYWQY1Gg9qUCAADBQcixiFWIAQCILEwh9wOrEAMAEDkIOX5iFWIAACIDj6sAAIAtEXIAAIAtEXIAAIAtEXIAAIAtEXIAAIAtMbvqBK6KapWWV8tdWSNHQoxS2jGTCgCASEXI+Y99ZUfr7TKek5Wi+fkDlZGcEMKaAQCAU8HjKh3vwTk54EjHdxefvWq7XBXsMA4AQKQh5EgqLa+uF3A8CopKVVpOyAEAINIQciS5K2uaPH+kmfMAACD8EHIkOeJjmjzfvpnzAAAg/BByJKUkxdbbXdwjJytFKUnMsAIAINIQcnR80835+QPrBZ2crBQtyB/INHIAACIQU8j/IyM5QYvHDVZpebWOVNaofXyMUpJYJwcAgEhFyDmBM5FQAwCAXfC4CgAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2BIhBwAA2FJEbutgjJEkud3uENcEAABY5fne9nyPt7SIDDlHjhyRJGVmZoa4JgAAwF/ffvutnE5ni79PlAlWnAqguro67du3T+3bt1dUVFSoq2OJ2+1WZmamvvzySzkcjlBXJ+Ds3j7J/m2kfZHP7m20e/sk+7fR5XKpW7duOnz4sJKTk1v8/SKyJyc6OlpnnHFGqKtxShwOhy3/xfWwe/sk+7eR9kU+u7fR7u2T7N/G6OjgDAlm4DEAALAlQg4AALAlQk6QxMXF6e6771ZcXFyoq9Ii7N4+yf5tpH2Rz+5ttHv7JPu3Mdjti8iBxwAAAM2hJwcAANgSIQcAANgSIQcAANgSIQcAANgSIQcAANgSIccPBQUFuvTSS5WRkaGoqCi9+OKLPuejoqIa/HnggQe8Zc4888x65+fPn+9zne3bt+v8889XfHy8MjMztXDhwmA0T/PmzdOPfvQjtW/fXqmpqbr88su1c+dOnzKVlZWaNm2aOnXqpKSkJOXn5+vAgQM+ZYqLizV69GglJiYqNTVVs2bNUm1trU+ZN998Uz/84Q8VFxens846S8uXL2/p5jXbvkOHDunGG29U7969lZCQoG7duummm26Sy+XyuU5Dn/Fzzz0X8vZJ1j7D//mf/6lX/+uvv96nTKR+hl988UWj9+HKlSu95cL1M1yyZIkGDhzoXe02Oztbr776qvd8JN9/Hk210Q73YHOfYSTff1LT7QvL+8/AsldeecX85je/Mf/4xz+MJPPCCy/4nN+/f7/Pz1NPPWWioqLM7t27vWW6d+9u7rnnHp9y5eXl3vMul8ukpaWZ8ePHm48++sg8++yzJiEhwTz66KMt3r68vDyzbNky89FHH5lt27aZiy++2HTr1s2nftdff73JzMw0GzZsMB9++KE555xzzLnnnus9X1tba77//e+b3Nxcs3XrVvPKK6+YlJQUM2fOHG+Zzz//3CQmJpqZM2eaTz75xCxevNi0adPGrF27NqTt27FjhxkzZoxZvXq12bVrl9mwYYPJysoy+fn5PteRZJYtW+bzGR49ejTk7bPSRmOMGT58uJk8ebJP/V0ul/d8JH+GtbW19e7D3/3udyYpKckcOXLEe51w/QxXr15t/vnPf5rPPvvM7Ny509xxxx0mJibGfPTRR8aYyL7/rLTRDvdgc59hJN9/zbUvHO8/Qs4paijknOynP/2pufDCC32Ode/e3Tz88MONvuaRRx4xHTp0MFVVVd5jt99+u+ndu/fpVPeUHDx40EgymzZtMsYYU1ZWZmJiYszKlSu9ZT799FMjyRQWFhpjjgfB6OhoU1JS4i2zZMkS43A4vG267bbbTP/+/X3e66qrrjJ5eXkt3SQfJ7evIc8//7yJjY01NTU13mPNffbh0j5jGm7j8OHDza9//etGX2O3z3DQoEHm2muv9TkWSZ9hhw4dzBNPPGG7++9EnjY2JNLvQWN822en+8+jqc8v1Pcfj6tayIEDB/TPf/5TkyZNqndu/vz56tSpkwYPHqwHHnjApxuysLBQOTk5io2N9R7Ly8vTzp07dfjw4aDU3cPTRdyxY0dJ0pYtW1RTU6Pc3FxvmT59+qhbt24qLCyUdLz+AwYMUFpamrdMXl6e3G63Pv74Y2+ZE6/hKeO5RrCc3L7GyjgcDrVt67uX7bRp05SSkqIf//jHeuqpp2ROWFMzXNonNd7GFStWKCUlRd///vc1Z84cVVRUeM/Z6TPcsmWLtm3b1uB9GO6f4bFjx/Tcc8/pu+++U3Z2tu3uP6l+GxsSyfdgY+2zy/3X3OcXDvdfRO5CHgn+/Oc/q3379hozZozP8Ztuukk//OEP1bFjR7377ruaM2eO9u/fr4ceekiSVFJSoh49evi8xvMve0lJiTp06BCU+tfV1WnGjBk677zz9P3vf9/7/rGxsUpOTq5Xv5KSEm+ZE29Oz3nPuabKuN1uHT16VAkJCS3RJB8Nte9kpaWluvfeezVlyhSf4/fcc48uvPBCJSYm6rXXXtPUqVNVXl6um266SVJ4tE9qvI2/+MUv1L17d2VkZGj79u26/fbbtXPnTv3jH/9osv6ec02VCbfP8Mknn1Tfvn117rnn+hwP589wx44dys7OVmVlpZKSkvTCCy+oX79+2rZtm23uv8baeLJIvQebap8d7j+rn1843H+EnBby1FNPafz48YqPj/c5PnPmTO8/Dxw4ULGxsbruuus0b968sNqrZNq0afroo4/09ttvh7oqLaK59rndbo0ePVr9+vXT3Llzfc7deeed3n8ePHiwvvvuOz3wwAPeGzRcNNbGE78wBgwYoC5dumjEiBHavXu3evXqFexqnrLmPsOjR4/qmWee8fm8PML5M+zdu7e2bdsml8ulv//975o4caI2bdoU6moFVGNtPPGLMpLvwabaZ4f7z8rnFy73H4+rWsBbb72lnTt36le/+lWzZYcOHara2lp98cUXkqT09PR6syU8v6enpwe8rg2ZPn261qxZozfeeENnnHGG93h6erqqq6tVVlZWr36eulmpf2NlHA5HUP5fZGPt8zhy5IguuugitW/fXi+88IJiYmKavN7QoUP11VdfqaqqSlLo2yc138YTDR06VJK0a9cuSfb4DCXp73//uyoqKjRhwoRmrxdOn2FsbKzOOussDRkyRPPmzdMPfvAD/d///Z9t7j+p8TZ6RPo92Fz7Tq67FFn3n5X2hcv9R8hpAU8++aSGDBmiH/zgB82W3bZtm6Kjo5WamipJys7OVkFBgWpqarxl1q9fr969e7f4oypjjKZPn64XXnhBGzdurPfYbMiQIYqJidGGDRu8x3bu3Kni4mLv89js7Gzt2LFDBw8e9Km/w+Hwpvzs7Gyfa3jKNPZMPlCaa590/P89jhw5UrGxsVq9enW9nriGbNu2TR06dPD2xIWqfZK1Np5s27ZtkqQuXbpIivzP0OPJJ5/UZZddps6dOzd73XD6DE9WV1enqqqqiL//muJpoxT592BDTmzfySLp/mtMQ+0Lm/vP76HKrdiRI0fM1q1bzdatW40k89BDD5mtW7eavXv3esu4XC6TmJholixZUu/17777rnn44YfNtm3bzO7du83TTz9tOnfubCZMmOAtU1ZWZtLS0szVV19tPvroI/Pcc8+ZxMTEoEwhv+GGG4zT6TRvvvmmz9S+iooKb5nrr7/edOvWzWzcuNF8+OGHJjs722RnZ3vPe6Y/jhw50mzbts2sXbvWdO7cucHpj7NmzTKffvqp+dOf/hSU6Y/Ntc/lcpmhQ4eaAQMGmF27dvmUqa2tNcYcnz75+OOPmx07dpiioiLzyCOPmMTERHPXXXeFvH1W2rhr1y5zzz33mA8//NDs2bPHvPTSS6Znz54mJyfHe41I/gw9ioqKTFRUlHn11VfrXSOcP8PZs2ebTZs2mT179pjt27eb2bNnm6ioKPPaa68ZYyL7/rPSRjvcg021L9Lvv+ba5xFO9x8hxw9vvPGGkVTvZ+LEid4yjz76qElISDBlZWX1Xr9lyxYzdOhQ43Q6TXx8vOnbt6+5//77TWVlpU+5f/3rX2bYsGEmLi7OdO3a1cyfP7+lm2aMMQ22Tf9Zz8Dj6NGjZurUqaZDhw4mMTHRXHHFFWb//v0+1/niiy/MqFGjTEJCgklJSTG33HKLz/RPY47/LQcNGmRiY2NNz549fd4jVO1r7POVZPbs2WOMMebVV181gwYNMklJSaZdu3bmBz/4gVm6dKk5duxYyNtnpY3FxcUmJyfHdOzY0cTFxZmzzjrLzJo1y2edDmMi9zP0mDNnjsnMzKz3uRgT3p/htddea7p3725iY2NN586dzYgRI3y+PCL5/vNoqo12uAebal+k33/Ntc8jnO6/KGNOmLcFAABgE4zJAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtkTIAQAAtvT/AT2/S38fjRkBAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.scatterplot(x=x[:, 0], y=y[:, 0])\n",
    "ax.set_title(\"input vs. target\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14696a9d",
   "metadata": {},
   "source": [
    "### Split the dataset into training, cross validation, and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b2fd3bc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the shape of the training set (input) is: (30, 1)\n",
      "the shape of the training set (target) is: (30, 1)\n",
      "\n",
      "the shape of the cross validation set (input) is: (10, 1)\n",
      "the shape of the cross validation set (target) is: (10, 1)\n",
      "\n",
      "the shape of the test set (input) is: (10, 1)\n",
      "the shape of the test set (target) is: (10, 1)\n"
     ]
    }
   ],
   "source": [
    "# Get 60% of the dataset as the training set. Put the remaining 40% in temporary variables: x_ and y_.\n",
    "x_train, x_, y_train, y_ = train_test_split(x, y, test_size=0.40, random_state=1)\n",
    "\n",
    "# Split the 40% subset above into two: one half for cross validation and the other for the test set\n",
    "x_cv, x_test, y_cv, y_test = train_test_split(x_, y_, test_size=0.50, random_state=1)\n",
    "\n",
    "# Delete temporary variables\n",
    "del x_, y_\n",
    "\n",
    "print(f\"the shape of the training set (input) is: {x_train.shape}\")\n",
    "print(f\"the shape of the training set (target) is: {y_train.shape}\\n\")\n",
    "print(f\"the shape of the cross validation set (input) is: {x_cv.shape}\")\n",
    "print(f\"the shape of the cross validation set (target) is: {y_cv.shape}\\n\")\n",
    "print(f\"the shape of the test set (input) is: {x_test.shape}\")\n",
    "print(f\"the shape of the test set (target) is: {y_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a8d48df3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5MklEQVR4nO3df3xU1Z3/8fdkgBh+TBCFJDATgq1VqLT+2o2xTgs1D8HSLu6QugpVt1ppFWqiLVW+VuuPKhZ3NbFbpe36g0eV9IHp2FpaqIhG05pSZaUisggWTAhJcMVkQAVhcr5/XGfM5NfcIZPJzM3r+XjkEXPvyZ17nE7z9txzPsdljDECAABwmKzBvgEAAICBQMgBAACORMgBAACORMgBAACORMgBAACORMgBAACORMgBAACORMgBAACONGywb+BYdHR0aO/evRozZoxcLtdg3w4AALDBGKMDBw5o4sSJysoa+HGWjAw5e/fulc/nG+zbAAAAx6CxsVFer3fAXycjQ86YMWMkWf+SPB7PIN8NAACwIxQKyefzRf+OD7SMDDmRR1Qej4eQAwBAhknVVBMmHgMAAEci5AAAAEci5AAAAEci5AAAAEci5AAAAEci5AAAAEci5AAAAEci5AAAAEfKyGKAAAAgMeGwVFcnNTdLBQWS3y+53YN9VwOLkAMAgMMFg1J5ubRnzyfHvF6pqkoKBAbvvgYaj6sAAHCwYFAqK4sNOJLU1GQdDwYH575SgZADAIBDhcPWCI4x3c9FjlVUWO2ciJADAIBD1dV1H8HpzBipsdFq50SEHAAAHKq5ObntMg0hBwAAhyooSG67TEPIAQDAofx+axWVy9XzeZdL8vmsdk5EyAEAwKHcbmuZuNQ96ER+rqx0br2chEPOgQMHVFFRocmTJysnJ0fnnnuuXn755eh5Y4xuvfVWFRQUKCcnR6WlpdqxY0fMNfbv368FCxbI4/Fo7Nixuuqqq3Tw4MH+9wYAAMQIBKSaGmnSpNjjXq91nDo5nXzrW9/S+vXr9atf/UpbtmzRBRdcoNLSUjU1NUmSli9frgceeEArVqzQxo0bNWrUKM2aNUuHDh2KXmPBggXaunWr1q9frzVr1ujFF1/UwoULk9crAAAQFQhIu3dLzz8vrVplfd+1q+eAEw5LtbVSdbX1PaOXl5sEfPDBB8btdps1a9bEHD/zzDPNzTffbDo6Okx+fr659957o+fa2tpMdna2qa6uNsYY88YbbxhJ5uWXX462Wbt2rXG5XKapqcnWfbS3txtJpr29PZHbBwAAffjNb4zxeo2xFpdbX16vdTwZUv33O6GRnKNHjyocDuu4446LOZ6Tk6M///nP2rVrl1paWlRaWho9l5ubq+LiYtXX10uS6uvrNXbsWJ199tnRNqWlpcrKytLGjRt7fN3Dhw8rFArFfAEAgORxYmXkhELOmDFjVFJSojvvvFN79+5VOBzW448/rvr6ejU3N6ulpUWSlJeXF/N7eXl50XMtLS2aMGFCzPlhw4Zp3Lhx0TZdLVu2TLm5udEvn8+XyG0DAIA+OLUycsJzcn71q1/JGKNJkyYpOztbDzzwgC699FJlZQ3cQq2lS5eqvb09+tXY2DhgrwUAwFDj1MrICSeTT33qU3rhhRd08OBBNTY26m9/+5uOHDmik046Sfn5+ZKk1tbWmN9pbW2NnsvPz9e+fftizh89elT79++PtukqOztbHo8n5gsAACSHUysjH/Pwy6hRo1RQUKD33ntPf/rTnzR37lxNmTJF+fn52rBhQ7RdKBTSxo0bVVJSIkkqKSlRW1ubNm3aFG3z3HPPqaOjQ8XFxf3oCgAAOBZOrYzsMqanJ3C9+9Of/iRjjE455RTt3LlTS5Ys0XHHHae6ujoNHz5cP/nJT3TPPfdo5cqVmjJlim655Ra99tpreuONN6ITli+88EK1trZqxYoVOnLkiL75zW/q7LPP1qpVq2zdQygUUm5urtrb2xnVAQCgn8JhqajImmTcUypwuay6Ort29a9wYKr/fg9L9Bfa29u1dOlS7dmzR+PGjdO8efN01113afjw4ZKkH/zgB3r//fe1cOFCtbW16bzzztO6detiVmQ98cQTWrx4sc4//3xlZWVp3rx5euCBB5LXKwAAYFukMnJZmRVoOgedTK6MnPBITjpgJAcAgOQLBq1VVp0nIft8VsBJRmXktB/JAQAAaSYctpY+NTdbE2f8/mMadgkEpLlzk3KptEDIAQAgk/U0/OL1Ws+fjmH4xe2WZsxI3u0NJnYhBwAgUzmxTHESEXIAAMhETi1TnEQ8rgIA4GPhjrDqGurUfKBZBWMK5C/0y501gBNS+jOXJpEyxU55/pQgQg4AAJKC24IqX1euPaFPgoPX41XV7CoFpiZhaVG3F+znXBqnlilOIh5XAQCGvOC2oMpWl8UEHElqCjWpbHWZgtuOYW5LOCzV1krV1db3zo+NEphLE+4Iq3Z3raq3VKt2d63CHR9fx6llipOIOjkAgCEt3BFWUVVRt4AT4ZJLXo9Xu8p32X901dcozdy5Vnnh3h41dSovHHzzd72PLn1mbmrKFCdRqv9+M5IDABjS6hrqeg04kmRk1BhqVF2DzS24443S3HWXrbk0weBdfY8uvfk7KzRJn5QljsjkMsVJRMgBAAxpzQfszVmx1c7Oiicb2xiFXVL5m1Uy6n6dyLGKdRUKXzRXqqmRJk2KbeT1WseTUaY4gzHxGAAwKJJUpLffCsbYm7Niq52dFU/vvhv/MpOlPUf3936ZTqNLM5xWpjiJCDkAgJRLcpHefvEX+uX1eNUUaupx5CQyJ8df6I9/MbsrmcaNk957r9e5NM2Tj5fUe8iJvlxkdMlJZYqTiMdVAICUSrcive4st6pmW3NbXIqd2xL5uXJ2pb1Jx3ZXMpWXf/wCPc+lKbiy3NZl7I5CDVWEHABAyqRrkd7A1IBqLq7RJE/s3Bavx6uai2vs18nx+60hqa7hJcLlsrb1vvnmPufS+L9xs7web7fQFb2MXPJ5fPZGl4YwlpADAFKmtlaaOTN+u+efT+zpS7Lm99iqeBzvxSJDVVJsmosEn84Tgvu4VqR2j6SYx2iR4JNQ+EoTqf77zZwcAEDKDESR3mTO73FnuTWjaEb/XiwQsIJMT+0qK2Nvqo+5NJHRpZ7q5FTOrsy4gDMYGMkBAKRMskdyIoMmXf+S9TRo0m+JvliShpdSvp/WAEr1329CDgAgZcLh5BXpjVzLRuHg/q+mTumLORcVjwEAjuV2J69IbyKbcPdbSl8MyULIAQCkVGTKSn+L9KZ0E252/M5ITDwGAKRcMor0pnQTbnb8zkjMyQEAZKSE5/f0ZyJwMicTDWHMyQEAoJNwR1i1u2tVvaVatbtrFe6wKgUmNL8nGLRCysyZ0vz51veiIvvllZM5mQgpQ8gBAKSt4LagiqqKNHPlTM0PztfMlTNVVFWk4DYrnNia35OsfSSSNZkIKcPjKgBAWopU/O26aWZPFX97fRI1EEu/02X79AxEnRwbCDkA4GzhjrCKqopiKv12FtkZfFf5rr4L4w3UPhI4JszJAQAMeXUNdb0GHMnay6kx1Ki6hjh1aVj6PaQRcgAAaaf5gL3QEbcdS7+HNOrkAACSr5/zVgrG2Asdcdv5/dacm3hLv/1+2/eGzMFIDgAgufq7XFuSv9Avr8cbnWTclUsu+Tw++QvjhBOWfg9phBwAQPIkabm2O8utqtlWOOkadCI/V86utLcbN0u/hyxWVwEAkmMAlmsHtwVVvq48ZhKyz+NT5ezK6PLxhO6Ppd+DiiXkNhByACANDdBy7XBHWHUNdWo+0KyCMQXyF/rtjeAg7aT67zcTjwEAyTFAy7XdWW7NKJqR+P1gyGNODgAgOViujTTDSA4ADJKUP4axMSelX9NWWK6NNEPIAYBB0NOEWq/Hq6rZVYlPqLX1gkGpvDx2UrDXay2v/nh1kY0mfYss1y4rswJN56DDcm0MAiYeA0CKJbLxZHJe8ONl3V3/7z4SPGpqFFQgXhP7K617Sks+nxVwWK49pLG6ygZCDoBMlbSNJ22/YPxl3eFJhSrSLu3Z00vhvWPYqJvl2ugJq6sAwMES2XgyKSuK6up6DziSZIzq9hRpTy+VhT9uosZG61K2V3673ezqjUFHyAGAFEp048l+D4jYWK7dLHurnSKXom4NMgUhBwBSKJGNJ/s9EViytVy7QPaCV0HBIEyYBvqBOjkAkEJ2N558Z5M/GVtAfbKsu+vmlNEXdMnv3S2v1/TVRD6f9H/jrQnTXR+3NYWaVLa6TMFt9jfgBFKBkAMAKWRn48n7LqjUDRXuHkvNRI5VVFiPsuK/YPxduN1V96mqytVXE/3n/WFd/0x5txVhkqLHKtZVKNxh56aA1CDkAECKBaYGVHNxjSZ5YnfF9nq8qrm4Rie+E4g3Vzg6EViSlXZqa6Xqaut71/RjYxfueE3Gn2V/wjSQLpiTAwCDIDA1oLmnzO1xAm/1ZnvXaG6W/Qp+gYA0d26fs5j7alK9JbEJ00A6IOQAwCDpbeNJ21tA7XhRuq2HCn6RiTtdK/jZWNbdW5NEJkwD6YLHVQCQZmzMFZbPa+T/xWU97xGV8MQdG/dkc8K0v5B9qZA+CDkAkGZszBVW5dVb5W5q6P0i3Sbu9POebEyYrpxdSb0cpBVCDgCkobhzhU/eYu9CNooB2r6nOBOmqZODdMOcHABIU33OFa61O3EnuXNk+powDaQbNugEgEwU2XizqanneTnHtKsmMLBS/febx1UAkIlsTdypJOBgSCPkAECmslHkDxjKmJMDAJnMRpE/YKgi5ADAQAiHUxc8bBT5A4YiQg4AJJvdrRYADCjm5ABAMgWD1pYKXXfYjGy1EAwOzn0BQxAhBwCSJRy2RnBStNUCgL4lFHLC4bBuueUWTZkyRTk5OfrUpz6lO++8U51L7RhjdOutt6qgoEA5OTkqLS3Vjh07Yq6zf/9+LViwQB6PR2PHjtVVV12lgwcPJqdHADBY6uq6j+B0luStFgD0LaGQ85Of/EQPPfSQ/uu//kvbtm3TT37yEy1fvlw//elPo22WL1+uBx54QCtWrNDGjRs1atQozZo1S4cOHYq2WbBggbZu3ar169drzZo1evHFF7Vw4cLk9QoABoPdLRSSuNUCgN4lVPH4q1/9qvLy8vTwww9Hj82bN085OTl6/PHHZYzRxIkT9b3vfU/f//73JUnt7e3Ky8vTY489pksuuUTbtm3TtGnT9PLLL+vss8+WJK1bt05f+cpXtGfPHk2cODHufVDxGEBaqq2VZs6M3+7551kNhSEprSsen3vuudqwYYPefPNNSdLf//53/fnPf9aFF14oSdq1a5daWlpUWloa/Z3c3FwVFxervr5eklRfX6+xY8dGA44klZaWKisrSxs3buzxdQ8fPqxQKBTzBQBpx++3VlF1rUAc4XJJPp/VDsCAS2gJ+U033aRQKKRTTz1Vbrdb4XBYd911lxYsWCBJamlpkSTl5eXF/F5eXl70XEtLiyZMmBB7E8OGady4cdE2XS1btky33357IrcKAKkX2WqhrMwKNJ0HytlqAUi5hEZyVq9erSeeeEKrVq3S//zP/2jlypX6j//4D61cuXKg7k+StHTpUrW3t0e/GhsbB/T1AOCYsdUCkDYSGslZsmSJbrrpJl1yySWSpOnTp+vtt9/WsmXLdMUVVyg/P1+S1NraqoKCgujvtba26vTTT5ck5efna9++fTHXPXr0qPbv3x/9/a6ys7OVnZ2dyK0CwOBhqwUgLSQ0kvPBBx8oKyv2V9xutzo6OiRJU6ZMUX5+vjZs2BA9HwqFtHHjRpWUlEiSSkpK1NbWpk2bNkXbPPfcc+ro6FBxcfExdwQA0kpkq4VLL7W+E3CAlEtoJOdrX/ua7rrrLhUWFuqzn/2sXn31Vd1333268sorJUkul0sVFRX68Y9/rJNPPllTpkzRLbfcookTJ+qiiy6SJE2dOlWzZ8/W1VdfrRUrVujIkSNavHixLrnkElsrqwAAAOxIKOT89Kc/1S233KJrr71W+/bt08SJE/Xtb39bt956a7TND37wA73//vtauHCh2tradN5552ndunU67rjjom2eeOIJLV68WOeff76ysrI0b948PfDAA8nrFQAAGPISqpOTLqiTA2BApHLncGAISvXfb3YhBwCJncMBB2KDTgBg53DAkQg5AIY2dg4HHIuQA2BoY+dwwLEIOQCGNnYOBxyLiccAhrZO1dnttmMRFpAZGMkBMLQluHN4MCgVFUkzZ0rz51vfi4qYmwykI0IOgIwXDku1tVJ1tfU9oTnCkZ3Dpe5Bp8vO4SzCAjILIQdARrM7shLuCKt2d62qt1Srdnetwh2dkpCNncNZhAVkHioeA8hYkZGVrv8vFhmA+TifKLgtqPJ15doT+mQIxuvxqmp2lQJTOxX662OyTW2tFaDief55az9OAN1R8RgAbIg3suJyWSMrHacEdXFNmYxiGzaFmlS2ukw1F9d8EnQiO4f3gEVYQObhcRWAjGSrvM2esK59urxbwJEUPVaxriL20VUvjmERFoBBRsgBkJFsjZhMrtM7H/WehIyMGkONqmuIX+gvwUVYANIAIQdARrI1YjLa3rOj5gPx2yWwCAtAmiDkAMhIdkZWxufYe3ZUMMZeOxuLsACkEUIOgIxkZ2TlZzf65fV45VLPScgll3wen/yF9p8xBQLS7t3WKqpVq6zvu3YRcIB0RMgBkLHijax8fZ5bVbOtJNQ16ER+rpxdKXdWYs+YIouwLr3U+s4jKiA9UScHQMaLt5dUT3VyfB6fKmdXxtbJATCgUv33m5ADYEgId4RV11Cn5gPNKhhTIH+hP+ERHAD9QzFAABgA7iy3ZhTNGOzbAJBCzMkBAACORMgBAACORMgBAACORMgBAACORMgBAACORMgBAACORMgBAACORJ0cAAmhqB6ATEHIAWBbT9sjeD1eVc2uSnx7hHh7MQBAP/G4CoAtwW1Bla0uiwk4ktQUalLZ6jIFtwUTuFhQKiqSZs6U5s+3vhcVWccBIEkIOQDiCneEVb6uXEbdt7qLHKtYV6FwRzj+xYJBqaxM2hMbltTUZB0n6ABIEkIOgLjqGuq6jeB0ZmTUGGpUXUNd3xcKh6XycqmnfYEjxyoqrHYA0E+EHABxNR9oTk67urruIzidGSM1NlrtIsJhqbZWqq62vhOAANjExGMAcRWMKUhOu2Z7YSnaLhi0Rn46ByOvV6qqkgIJTnQGMOQwkgMgLn+hX16PVy65ejzvkks+j0/+Qn/fFyqwF5ZUUMDcHQD9RsgBEJc7y62q2VWS1C3oRH6unF0Zv16O32+NxLh6DktyuSSfTzr3XObuAOg3Qg4AWwJTA6q5uEaTPJNijns9XtVcXGOvTo7bbT1qkroHncjPlZXSSy8lPncHALpgTg4A2wJTA5p7ytz+VTwOBKSamp7n2lRWWuerq+1dy+4cHwBDEiEHQELcWW7NKJrRd6N41YwDAWnu3N7bJDJ3BwB6QcgBkFx2V0S53dKMGT1fIzJ3p6mp53k5Lpd13h9nojOAIY05OQCSJ1krouzO3WGvKwB9IOQASI5kVzOOzN2ZFDvRWV6vdZw6OQDi4HEVgORIpJpxb4+puoo3dwcA+kDIAZAciVYztquvuTsA0AceVwFIDlZEAUgzhBwAyWG3mjErogCkCCEHQHKwIgpAmiHkAEgeVkQBSCNMPAaQXKyIApAmCDkAko8VUQDSAI+rAACAIzGSAyAq3BHu3w7jAJBGCDkAJEnBbUGVryvXntAnVYu9Hq+qZlcpMJUJwwAyD4+rgKEiHJZqa6Xqaut7pz2kgtuCKltdFhNwJKkp1KSy1WUKbrO5sSYApBFCDjAUBINSUZE0c6Y0f771vahICgYV7girfF25jLpvrBk5VrGuQuEOmxtrAkCaIOQAThcMSmVl3TfPbGqSyspU9/hd3UZwOjMyagw1qq6hboBvFACSi5ADOFk4LJWXWzuAd/XxseZHqmxdqvlAghtrAsAgI+QATlZX130EpzNjVPD2fluXKhjDxpoAMgshB3Cy5vijL/63Je+wcXKp5401XXLJ5/HJX8jGmgAyS0Ihp6ioSC6Xq9vXokWLJEmHDh3SokWLdMIJJ2j06NGaN2+eWltbY67R0NCgOXPmaOTIkZowYYKWLFmio0ePJq9HAD5REH/0xW2kqs+US1K3oBP5uXJ2JfVyAGSchELOyy+/rObm5ujX+vXrJUlf//rXJUnXX3+9fv/73+vJJ5/UCy+8oL179yrQaUO+cDisOXPm6KOPPtJLL72klStX6rHHHtOtt96axC4BiPL7rc0xu+4KHuFyST6fAoGbVXNxjSZ5YjfW9Hq8qrm4hjo5ADKSy5ieZiTaU1FRoTVr1mjHjh0KhUIaP368Vq1apbKyMknS//7v/2rq1Kmqr6/XOeeco7Vr1+qrX/2q9u7dq7y8PEnSihUrdOONN+qdd97RiBEjbL1uKBRSbm6u2tvb5fF4jvX2gaEhsrpKip2AHAk+nXYHp+IxgIGU6r/fxzwn56OPPtLjjz+uK6+8Ui6XS5s2bdKRI0dUWloabXPqqaeqsLBQ9fX1kqT6+npNnz49GnAkadasWQqFQtq6dWs/ugGgV4GAFWQmxY7SyOuNCTiS5M5ya0bRDF06/VLNKJpBwAGQ0Y55W4ff/va3amtr07//+79LklpaWjRixAiNHTs2pl1eXp5aWlqibToHnMj5yLneHD58WIcPH47+HAqFjvW2gaEpEJDmzrVWWzU3W3N1/H5rt3AAcKhjDjkPP/ywLrzwQk2cODGZ99OjZcuW6fbbbx/w1wEcze2WZswY7LsAgJQ5psdVb7/9tp599ll961vfih7Lz8/XRx99pLa2tpi2ra2tys/Pj7bputoq8nOkTU+WLl2q9vb26FdjY+Ox3DYAABhCjinkPProo5owYYLmzJkTPXbWWWdp+PDh2rBhQ/TY9u3b1dDQoJKSEklSSUmJtmzZon379kXbrF+/Xh6PR9OmTev19bKzs+XxeGK+AAAA+pLw46qOjg49+uijuuKKKzRs2Ce/npubq6uuuko33HCDxo0bJ4/Ho+9+97sqKSnROeecI0m64IILNG3aNF122WVavny5Wlpa9MMf/lCLFi1SdnZ28noFAACGvIRDzrPPPquGhgZdeeWV3c7df//9ysrK0rx583T48GHNmjVLDz74YPS82+3WmjVrdM0116ikpESjRo3SFVdcoTvuuKN/vQAAAOiiX3VyBgt1cgAAyDwZUycHAAAgnRFyAACAIxFyAACAIx1zMUAA6SEcppAxAPSEkANksGBQKi+X9uz55JjXK1VVxWxJBQBDEo+rgAwV2Vy8c8CRpKYm63gwODj3BQDpgpADZKBw2BrB6akARORYRYXVDgCGKkIOkIHq6rqP4HRmjNTYaLUDgKGKkANkoObm5LYDACdi4jEwWPqxLKqgwN5L2G0HAE5EyAEGg91lUb0EIb/fat7U1PO8HJfLOu/3D3xXACBd8bgKSDW7y6KCQamoSJo5U5o/3/peVCQFg3K7rTwkWYGms8jPlZXUywEwtBFygIEQDku1tVJ1tfU9sszJ7rKompq4QSgQsJpNmhTbxOu1jlMnB8BQxy7kQLL19Shq3DhrRCae8eOld97p+VzkWdSuXZLbTcVjABkj1X+/mZMDJFPkUVTX/3aIjMCUl9u7Tm8BR4pdHz5jhtxuacaMY75jAHAsHlcByWLnUdQTTyTv9VgfDgB9IuQAyWKnQt8770gnnth9tnCEy2U9qrKD9eEA0CdCDpAsdkdWvvEN63tvy6J+9jNrzk1fQcjnY304AMRByAGSxe7Iyty5fS+L+vrXWR8OAEnA6iogWcJhq45NvAp9H6+KirssqqdVWj6fFXBYHw4gA6X67zchB0imyOoqKTboREZgEi1gw/pwAA7CEnIgk0Uq9PVUJ+dYRmBYHw4Ax4yQAyQq3uhKIGDNu2EEBgAGFSEHSITdjTUZgQGAQcfqKsAuuxtrAgDSAiEHsMPuxpqRjTgBAIOOkAPYYaeacWQ/KQBAWiDkAHbYrWbMflIAkDYIOYAddqsZs58UAKQNQg5gh9/PflIAkGEIOYAdbjf7SQFAhiHkAHZFqhn3trEm+0kBQFqhGCCQCKoZA0DGIOQAiaKaMQBkBB5XAQAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAARyLkAAAAR2LvKgwJ4Y6w6hrq1HygWQVjCuQv9MudxaaaAOBkhBw4XnBbUOXryrUntCd6zOvxqmp2lQJTA4N4ZwCAgcTjKjhacFtQZavLYgKOJDWFmlS2ukzBbcFPDobDUm2tVF1tfQ+HU3qvAIDkIuTAscIdYZWvK5eR6XYucqxiXYXCHWEpGJSKiqSZM6X5863vRUXWcQBARiLkwLHqGuq6jeB0ZmTUGGpU3eN3SWVl0p4ubZuarOMEHQDISIQcOFbzgWZ77R6ukkz30Z7osYoKHl0BQAYi5MCxCsYU2GvXsL/3k8ZIjY1SXV2S7goAkCqEHDiWv9Avr8crl1w9nnfJJd+wE+R/28bFmu2NCgEA0gchB47lznKranaVJHULOpGfKz9zndw9PKnqpsDeqBAAIH0QcuBogakB1Vxco0meSTHHvR6vai6uUSBws+T1Sq6eR3vkckk+n+T3p+BuAQDJ5DKmpxmX6S0UCik3N1ft7e3yeDyDfTvIAH1WPA4GrVVUUuwE5EjwqamRAhQNBID+SvXfbyoeY0hwZ7k1o2hGzycDASvIlJfHLiP3eqXKSgIOAGQoQg4gWUFm7lxrFVVzszUHx++X3OxvBQCZipADRLjd0owZg30XAIAkYeIxAABwpIRDTlNTk77xjW/ohBNOUE5OjqZPn65XXnklet4Yo1tvvVUFBQXKyclRaWmpduzYEXON/fv3a8GCBfJ4PBo7dqyuuuoqHTx4sP+9AQAA+FhCIee9997TF77wBQ0fPlxr167VG2+8of/8z//U8ccfH22zfPlyPfDAA1qxYoU2btyoUaNGadasWTp06FC0zYIFC7R161atX79ea9as0YsvvqiFCxcmr1dwDjs7g7N7OACgJyYBN954oznvvPN6Pd/R0WHy8/PNvffeGz3W1tZmsrOzTXV1tTHGmDfeeMNIMi+//HK0zdq1a43L5TJNTU227qO9vd1IMu3t7YncPjLNb35jjNdrjLWw2/ryeq3jibQBAKSFVP/9Tmgk5+mnn9bZZ5+tr3/965owYYLOOOMM/fKXv4ye37Vrl1paWlRaWho9lpubq+LiYtXX10uS6uvrNXbsWJ199tnRNqWlpcrKytLGjRt7fN3Dhw8rFArFfMHhIrVr+toZ3E4bAMCQlVDI+cc//qGHHnpIJ598sv70pz/pmmuu0XXXXaeVK1dKklpaWiRJeXl5Mb+Xl5cXPdfS0qIJEybEnB82bJjGjRsXbdPVsmXLlJubG/3y+XyJ3DYyTThs1azpa2fw8vL4bdg9HACGtIRCTkdHh84880zdfffdOuOMM7Rw4UJdffXVWrFixUDdnyRp6dKlam9vj341NjYO6OthkNXVdR+d6cwY63y8NuweDgBDWkIhp6CgQNOmTYs5NnXqVDU0NEiS8vPzJUmtra0xbVpbW6Pn8vPztW/fvpjzR48e1f79+6NtusrOzpbH44n5goMlc8dvdg8HgCEroZDzhS98Qdu3b4859uabb2ry5MmSpClTpig/P18bNmyIng+FQtq4caNKSkokSSUlJWpra9OmTZuibZ577jl1dHSouLj4mDsCB0nmjt/sHg4AQ1ZCIef666/XX//6V919993auXOnVq1apV/84hdatGiRJMnlcqmiokI//vGP9fTTT2vLli26/PLLNXHiRF100UWSrJGf2bNn6+qrr9bf/vY3/eUvf9HixYt1ySWXaOLEiUnvIDKQ3x9/Z3Cvl93DAQB9S3Q51u9//3tz2mmnmezsbHPqqaeaX/ziFzHnOzo6zC233GLy8vJMdna2Of/888327dtj2rz77rvm0ksvNaNHjzYej8d885vfNAcOHLB9DywhHwJ+8xtjXC7rq/Py8Mix3/zGXhsAQNpI9d9vlzE9LU9Jb6neqh2DJBjsvjO4zxe7M7idNgCAtJDqv9+EHKS3cDj+zuB22gAABl2q/36zCzkGj51wYmdncHYPBwD0gJCDwdHTYyavV6qq4jETACApEt6FHOg3tmMAAKQAIQepZWfLBrZjAAAkASEHqWVnywa2YwAAJAEhB6lld5sFtmMAAPQTIQepZXebBbZjAAD0EyEHqWVnywa2YwAAJAEhB6nldlvLxKXuQSfyc2UlxfwAAP1GyEHqBQJSTY00aVLsca/XOk6dHABAElAMEMlnp5JxICDNnct2DACAAUPIQXIlUsmY7RgAAAOIx1VIHioZAwDSCCEHyUElYwBAmiHkIDmoZAwASDOEHCQHlYwBAGmGicdITG8rp6hkDABIM4Qc2NfXyqm5c61/bmrqeV6Oy2Wdp5IxACBFeFwFe+KtnPrd76hkDABIK4QcxGd35dTcuVQyBgCkDR5XIb5EVk5RyRgAkCYIOYgv0ZVTVDIGAKQBHlchPlZOAQAyECEH8fn91ryarhOKI1wuyedj5RQAIK0QchCf283KKQBAxiHkwJ5AgJVTAICMwsRj2MfKKQBABiHkIDGsnAIAZAgeVwEAAEdiJGeo6G1jTQAAHIqQMxT0tbEmE4YBAA7F4yqni7exZjA4OPcFAMAAI+Q4md2NNcPhlN4WAACpQMhxskQ21pSssFNbK1VXW98JPwCADMacHCdLZGNN5u0AAByGkRwns7th5o4dzNsBADgOIcfJ7Gys6fVKv/wl83YAAI5DyHEyOxtrXn11YvN2AADIEIQcp4u3sebJJ9u7jt35PQAApAkmHg8FfW2sWVtr7xp25/cAAJAmCDlDRW8ba0bm7TQ19TwvJzJvx+8f8FsEACCZeFw11NmZt1NZyT5XAICMQ8hB/Hk71MkBAGQgHlfB0te8HQAAMhAhB5/obd4OAAAZiMdVAADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkQg5AADAkRIKObfddptcLlfM16mnnho9f+jQIS1atEgnnHCCRo8erXnz5qm1tTXmGg0NDZozZ45GjhypCRMmaMmSJTp69GhyegMAAPCxYYn+wmc/+1k9++yzn1xg2CeXuP766/WHP/xBTz75pHJzc7V48WIFAgH95S9/kSSFw2HNmTNH+fn5eumll9Tc3KzLL79cw4cP1913352E7gAAAFgSDjnDhg1Tfn5+t+Pt7e16+OGHtWrVKn35y1+WJD366KOaOnWq/vrXv+qcc87RM888ozfeeEPPPvus8vLydPrpp+vOO+/UjTfeqNtuu00jRozof48AAAB0DHNyduzYoYkTJ+qkk07SggUL1NDQIEnatGmTjhw5otLS0mjbU089VYWFhaqvr5ck1dfXa/r06crLy4u2mTVrlkKhkLZu3drrax4+fFihUCjmCwAAoC8JhZzi4mI99thjWrdunR566CHt2rVLfr9fBw4cUEtLi0aMGKGxY8fG/E5eXp5aWlokSS0tLTEBJ3I+cq43y5YtU25ubvTL5/MlctsAAGAISuhx1YUXXhj958997nMqLi7W5MmTtXr1auXk5CT95iKWLl2qG264IfpzKBQi6AAAgD71awn52LFj9ZnPfEY7d+5Ufn6+PvroI7W1tcW0aW1tjc7hyc/P77baKvJzT/N8IrKzs+XxeGK+AAAA+tKvkHPw4EG99dZbKigo0FlnnaXhw4drw4YN0fPbt29XQ0ODSkpKJEklJSXasmWL9u3bF22zfv16eTweTZs2rT+3AgAAECOhx1Xf//739bWvfU2TJ0/W3r179aMf/Uhut1uXXnqpcnNzddVVV+mGG27QuHHj5PF49N3vflclJSU655xzJEkXXHCBpk2bpssuu0zLly9XS0uLfvjDH2rRokXKzs4ekA4CAIChKaGQs2fPHl166aV69913NX78eJ133nn661//qvHjx0uS7r//fmVlZWnevHk6fPiwZs2apQcffDD6+263W2vWrNE111yjkpISjRo1SldccYXuuOOO5PYKAAAMeS5jjBnsm0hUKBRSbm6u2tvbmZ8DAECGSPXf74SLASI5wh1h1TXUqflAswrGFMhf6Jc7yz3YtwUAgGMQcgZBcFtQ5evKtSe0J3rM6/GqanaVAlMDiV8wHJbq6qTmZqmgQPL7JTeBCQAwtLELeYoFtwVVtrosJuBIUlOoSWWryxTcFkzwgkGpqEiaOVOaP9/6XlRkHQcAYAgj5KRQuCOs8nXlMuo+DSpyrGJdhcIdYXsXDAalsjJpT2xgUlOTdZygAwAYwgg5KVTXUNdtBKczI6PGUKPqGuriXywclsrLpZ7mjUeOVVRY7QAAGIIIOSnUfKA5ee3q6rqP4HRmjNTYaLUDAGAIIuSkUMGYguS1a7YXmGy3AwDAYQg5KeQv9Mvr8colV4/nXXLJ5/HJX+iPf7ECe4HJdjsAAByGkDMAwmGptlaqrra+R6bFuLPcqppdJUndgk7k58rZlfbq5fj9ktcruXoOTHK5JJ/PagcAwBBEyEmyeCu6A1MDqrm4RpM8k2J+z+vxqubiGvt1ctxuqcoKTN2CTuTnykrq5QAAhiy2dUiiyIrurv9GI5mjpkYKfJxhbFU8tlPkLxi0Vll1noTs81kBJ3AMhQUBABggqf77TchJknDYGrHpbcGTy2U9Xdq1y+bgSk/hxeu1Rm+6hhcqHgMAMgB7V2WoRFZ0z5gR52K9DQlFivx1HhKSrEAT96IAAAwtzMlJkqSt6KbIHwAASUHISZKkreimyB8AAElByEmSpK3opsgfAABJQchJkqSt6KbIHwAASUHISaJAwJoTPCm2BI683u5zhXtFkT8AAJKC1VVJFghIc+f2Y0V3ZEiorMwKNJ0nIFPkDwAA2wg5A6DfK7ojQ0I91cmhyB8AALYQchKVqsJ7/R4SAgBgaCPkJCKRKsTJQJE/AACOGROP7YpUIe5awyZShTiyAycAAEgLhBw7qEIMAEDGIeTYQRViAAAyDiHHDqoQAwCQcQg5dlCFGACAjEPIsYMqxAAAZBxCjh1J25gKAACkCiHHrqRsTAUAAFKFYoCJoAoxAAAZg5CTKKoQAwCQEXhcBQAAHImQAwAAHImQAwAAHIk5OZ2Fw0wqBgDAIQg5EcGgtQln5z2qvF6rPg7LwwEAyDg8rpKsgFNW1n0TzqYm63gwODj3BQAAjhkhJxy2RnCM6X4ucqyiwmoHAAAyBiGnrq77CE5nxkiNjVY7AACQMQg5zc3JbQcAANICIaegILntAABAWiDk+P3WKqquu4tHuFySz2e1AwAAGYOQ43Zby8Sl7kEn8nNlJfVyAADIMIQcyaqDU1MjTZoUe9zrtY5TJwcAgIxDMcCIQECaO5eKxwAAOAQhpzO3W5oxY7DvAgAAJAGPqwAAgCMRcgAAgCMRcgAAgCMRcgAAgCMRcgAAgCMRcgAAgCMRcgAAgCMRcgAAgCMRcgAAgCNlZMVjY4wkKRQKDfKdAAAAuyJ/tyN/xwdaRoacAwcOSJJ8Pt8g3wkAAEjUu+++q9zc3AF/HZdJVZxKoo6ODu3du1djxoyRy+Ua7NuxJRQKyefzqbGxUR6PZ7BvJ+mc3j/J+X2kf5nP6X10ev8k5/exvb1dhYWFeu+99zR27NgBf72MHMnJysqS1+sd7Ns4Jh6Px5H/w41wev8k5/eR/mU+p/fR6f2TnN/HrKzUTAlm4jEAAHAkQg4AAHAkQk6KZGdn60c/+pGys7MH+1YGhNP7Jzm/j/Qv8zm9j07vn+T8Pqa6fxk58RgAACAeRnIAAIAjEXIAAIAjEXIAAIAjEXIAAIAjEXIS8OKLL+prX/uaJk6cKJfLpd/+9rcx510uV49f9957b7RNUVFRt/P33HNPzHVee+01+f1+HXfccfL5fFq+fHkquqdly5bpn/7pnzRmzBhNmDBBF110kbZv3x7T5tChQ1q0aJFOOOEEjR49WvPmzVNra2tMm4aGBs2ZM0cjR47UhAkTtGTJEh09ejSmTW1trc4880xlZ2fr05/+tB577LGB7l7c/u3fv1/f/e53dcoppygnJ0eFhYW67rrr1N7eHnOdnt7jX//614PeP8neezhjxoxu9/+d73wnpk2mvoe7d+/u9XP45JNPRtul63v40EMP6XOf+1y0EFxJSYnWrl0bPZ/Jn7+IvvrohM9gvPcwkz9/Ut/9S8vPn4Ftf/zjH83NN99sgsGgkWSeeuqpmPPNzc0xX4888ohxuVzmrbfeiraZPHmyueOOO2LaHTx4MHq+vb3d5OXlmQULFpjXX3/dVFdXm5ycHPPzn/98wPs3a9Ys8+ijj5rXX3/dbN682XzlK18xhYWFMff3ne98x/h8PrNhwwbzyiuvmHPOOcece+650fNHjx41p512miktLTWvvvqq+eMf/2hOPPFEs3Tp0mibf/zjH2bkyJHmhhtuMG+88Yb56U9/atxut1m3bt2g9m/Lli0mEAiYp59+2uzcudNs2LDBnHzyyWbevHkx15FkHn300Zj38MMPPxz0/tnpozHGfOlLXzJXX311zP23t7dHz2fye3j06NFun8Pbb7/djB492hw4cCB6nXR9D59++mnzhz/8wbz55ptm+/bt5v/9v/9nhg8fbl5//XVjTGZ//uz00QmfwXjvYSZ//uL1Lx0/f4ScY9RTyOlq7ty55stf/nLMscmTJ5v777+/19958MEHzfHHH28OHz4cPXbjjTeaU045pT+3e0z27dtnJJkXXnjBGGNMW1ubGT58uHnyySejbbZt22Ykmfr6emOMFQSzsrJMS0tLtM1DDz1kPB5PtE8/+MEPzGc/+9mY1/q3f/s3M2vWrIHuUoyu/evJ6tWrzYgRI8yRI0eix+K99+nSP2N67uOXvvQlU15e3uvvOO09PP30082VV14ZcyyT3sPjjz/e/Pd//7fjPn+dRfrYk0z/DBoT2z8nff4i+nr/Bvvzx+OqAdLa2qo//OEPuuqqq7qdu+eee3TCCSfojDPO0L333hszDFlfX68vfvGLGjFiRPTYrFmztH37dr333nspufeIyBDxuHHjJEmbNm3SkSNHVFpaGm1z6qmnqrCwUPX19ZKs+58+fbry8vKibWbNmqVQKKStW7dG23S+RqRN5Bqp0rV/vbXxeDwaNix2m7dFixbpxBNP1D//8z/rkUcekelUbipd+if13scnnnhCJ554ok477TQtXbpUH3zwQfSck97DTZs2afPmzT1+DtP9PQyHw/r1r3+t999/XyUlJY77/End+9iTTP4M9tY/p3z+4r1/6fD5y8gNOjPBypUrNWbMGAUCgZjj1113nc4880yNGzdOL730kpYuXarm5mbdd999kqSWlhZNmTIl5nci/2NvaWnR8ccfn5L77+joUEVFhb7whS/otNNOi77+iBEjuu0cm5eXp5aWlmibzh/OyPnIub7ahEIhffjhh8rJyRmILsXoqX9d/d///Z/uvPNOLVy4MOb4HXfcoS9/+csaOXKknnnmGV177bU6ePCgrrvuOknp0T+p9z7Onz9fkydP1sSJE/Xaa6/pxhtv1Pbt2xUMBvu8/8i5vtqk23v48MMPa+rUqTr33HNjjqfze7hlyxaVlJTo0KFDGj16tJ566ilNmzZNmzdvdsznr7c+dpWpn8G++ueEz5/d9y8dPn+EnAHyyCOPaMGCBTruuONijt9www3Rf/7c5z6nESNG6Nvf/raWLVuWVmW8Fy1apNdff11//vOfB/tWBkS8/oVCIc2ZM0fTpk3TbbfdFnPulltuif7zGWecoffff1/33ntv9AOaLnrrY+c/GNOnT1dBQYHOP/98vfXWW/rUpz6V6ts8ZvHeww8//FCrVq2Keb8i0vk9POWUU7R582a1t7erpqZGV1xxhV544YXBvq2k6q2Pnf9QZvJnsK/+OeHzZ+f9S5fPH4+rBkBdXZ22b9+ub33rW3HbFhcX6+jRo9q9e7ckKT8/v9tqicjP+fn5Sb/XnixevFhr1qzR888/L6/XGz2en5+vjz76SG1tbd3uL3Jvdu6/tzYejycl/xXZW/8iDhw4oNmzZ2vMmDF66qmnNHz48D6vV1xcrD179ujw4cOSBr9/Uvw+dlZcXCxJ2rlzpyRnvIeSVFNTow8++ECXX3553Oul03s4YsQIffrTn9ZZZ52lZcuW6fOf/7yqqqoc8/mTeu9jRKZ/BuP1r+u9S5n1+bPTv3T5/BFyBsDDDz+ss846S5///Ofjtt28ebOysrI0YcIESVJJSYlefPFFHTlyJNpm/fr1OuWUUwb8UZUxRosXL9ZTTz2l5557rttjs7POOkvDhw/Xhg0bose2b9+uhoaG6PPYkpISbdmyRfv27Yu5f4/HE035JSUlMdeItOntmXyyxOufZP3X4wUXXKARI0bo6aef7jYS15PNmzfr+OOPj47EDVb/JHt97Grz5s2SpIKCAkmZ/x5GPPzww/qXf/kXjR8/Pu510+k97Kqjo0OHDx/O+M9fXyJ9lDL/M9iTzv3rKpM+f73pqX9p8/lLeKryEHbgwAHz6quvmldffdVIMvfdd5959dVXzdtvvx1t097ebkaOHGkeeuihbr//0ksvmfvvv99s3rzZvPXWW+bxxx8348ePN5dffnm0TVtbm8nLyzOXXXaZef31182vf/1rM3LkyJQsIb/mmmtMbm6uqa2tjVna98EHH0TbfOc73zGFhYXmueeeM6+88oopKSkxJSUl0fOR5Y8XXHCB2bx5s1m3bp0ZP358j8sflyxZYrZt22Z+9rOfpWT5Y7z+tbe3m+LiYjN9+nSzc+fOmDZHjx41xljLJ3/5y1+aLVu2mB07dpgHH3zQjBw50tx6662D3j87fdy5c6e54447zCuvvGJ27dplfve735mTTjrJfPGLX4xeI5Pfw4gdO3YYl8tl1q5d2+0a6fwe3nTTTeaFF14wu3btMq+99pq56aabjMvlMs8884wxJrM/f3b66ITPYF/9y/TPX7z+RaTT54+Qk4Dnn3/eSOr2dcUVV0Tb/PznPzc5OTmmra2t2+9v2rTJFBcXm9zcXHPccceZqVOnmrvvvtscOnQopt3f//53c95555ns7GwzadIkc8899wx014wxpse+6eN6BhEffvihufbaa83xxx9vRo4caf71X//VNDc3x1xn9+7d5sILLzQ5OTnmxBNPNN/73vdiln8aY/27PP30082IESPMSSedFPMag9W/3t5fSWbXrl3GGGPWrl1rTj/9dDN69GgzatQo8/nPf96sWLHChMPhQe+fnT42NDSYL37xi2bcuHEmOzvbfPrTnzZLliyJqdNhTOa+hxFLly41Pp+v2/tiTHq/h1deeaWZPHmyGTFihBk/frw5//zzY/54ZPLnL6KvPjrhM9hX/zL98xevfxHp9PlzGdNp3RYAAIBDMCcHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA4EiEHAAA40v8HuCJ4nXbyXFQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x=x_train[:, 0], y=y_train[:, 0], c='red')\n",
    "plt.scatter(x=x_test[:, 0], y=y_test[:, 0], c='blue')\n",
    "plt.scatter(x=x_cv[:, 0], y=y_cv[:, 0], c='green')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51b21c53",
   "metadata": {},
   "source": [
    "## Linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a12c805",
   "metadata": {},
   "source": [
    "### Fit a linear model\n",
    "**Feature scaling**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2a9cb6e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computed mean of the training set: 2504.06\n",
      "Computed standard deviation of the training set: 574.85\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGeCAYAAACKDztsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAu8ElEQVR4nO3de3TU9Z3/8ddkgIBIJouSTEIit1IuivXCGgdNxSVrUOoJJ2AXpBaVQpeCS7ygsqdovbLSbkVcK9uuKx4VdpVOtaUrLmDRqDFailsEikCzEkImdMHMgMpt8vn9Mb8ZMiSB+SZz+2aej3Pm0Hy/n5n5zNeczKuf7+fz/jiMMUYAAAA2kpXqDgAAAFhFgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALbTI9Ud6IyWlhbt379f/fr1k8PhSHV3AABADIwxOnz4sAoLC5WV1cUxFGNRIBAwCxYsMBdccIHp3bu38Xg85sMPP4ycb2lpMYsXLzZut9v07t3bTJgwwXz66adRr3Hw4EFz8803m379+hmXy2Vuv/12c/jw4Zj7UF9fbyTx4MGDBw8ePGz4qK+vtxo/2rA8AvO9731Pn3zyiV588UUVFhbqpZdeUllZmbZv366BAwdq6dKlWr58uV544QUNGTJEixcvVnl5ubZv367evXtLkmbMmKHGxkatX79eJ06c0G233aY5c+Zo1apVMfWhX79+kqT6+nrl5ORY/QgAACAFAoGAiouLI9/jXeEwJvbNHL/66iv169dPr7/+uiZNmhQ5fvnll+v666/XI488osLCQt1999265557JEl+v1/5+flauXKlpk2bph07dmj06NH66KOPNHbsWEnSunXrdMMNN2jfvn0qLCw8az8CgYBcLpf8fj8BBgAAm4jn97elG1AnT55UMBiMjKSE9enTR++++67q6urk8/lUVlYWOedyuVRSUqKamhpJUk1NjXJzcyPhRZLKysqUlZWl2tradt/32LFjCgQCUQ8AAJC5LAWYfv36yePx6JFHHtH+/fsVDAb10ksvqaamRo2NjfL5fJKk/Pz8qOfl5+dHzvl8PuXl5UWd79Gjh/r37x9pc7olS5bI5XJFHsXFxVa6DQAAuhnLU4BffPFFGWM0cOBAZWdna/ny5Zo+fXrXZxOfwaJFi+T3+yOP+vr6hL0XAABIf5ZTx7Bhw/T222/ryJEjqq+v14cffqgTJ05o6NChcrvdkqSmpqao5zQ1NUXOud1uHThwIOr8yZMndejQoUib02VnZysnJyfqAQAAMlenh0369u2rgoICff7553rzzTdVUVGhIUOGyO12a+PGjZF2gUBAtbW18ng8kiSPx6Pm5mZt3rw50uatt95SS0uLSkpKuvBRAABAprC8jPrNN9+UMUYjRozQ7t27tXDhQo0cOVK33XabHA6Hqqqq9Oijj2r48OGRZdSFhYWaPHmyJGnUqFGaOHGiZs+erRUrVujEiROaP3++pk2bFtMKJAAAAMsBxu/3a9GiRdq3b5/69++vKVOm6LHHHlPPnj0lSffee6+++OILzZkzR83Nzbr66qu1bt26qJVLL7/8subPn68JEyYoKytLU6ZM0fLly+P3qQAAQLdmqQ5MuqAODAAA9hPP729b7oUEAAASJBiUqqulxkapoEAqLZWczlT3qg0CDAAACPF6pQULpH37Th0rKpKeekqqrExdv9qRuOItAADAPrxeaerU6PAiSQ0NoeNeb2r61QECDAAAmS4YDI28tDctNnysqirULk0QYAAAsLNgUNq0SVq9OvRvZ0JGdXXbkZfWjJHq60Pt0gRzYAAAsKt4zVlpbIxvuyRgBAYAADuK55yVgoL4tksC6sAAAJCuOlrSHAxKgwd3fNvH4QiNxNTVxbYEOvx6DQ3tz4Ox+nodiOf3NyMwAACkI683FCquvVa6+ebQv4MHh47He86K0xm67SSFwkpr4Z+XLUurejAEGAAA0s3Zbg+9/npsr2NlzkplpbRmjTRwYPTxoqLQ8TSrA8MkXgAA0snZljQ7HNLLL8f2WlbnrFRWShUVVOIFAAAWxXJ76C9/kc4/Xzp48MxzVkpLrb+/0ymNH2/9eUnGLSQAANJJrLd9vvOd0L82mbMSbwQYAADSSay3fSoqbDVnJd64hQQAQDopLQ2FkLMtaQ7PTbHJnJV4I8AAAJBsHdV3kU4taZ46NRRWWoeY9m4P2WTOSrxxCwkAgGQ6U32XMJstaU4FKvECAJAs4foup3/1hkdWTg8nZxqpsaF4fn8TYAAASIZ4l/+3IbYSAADAbuJd/j/DEWAAAEiGWOu7WCn/n8EIMAAAJEOs9V2slv/PUAQYAACSIVzf5fTKuWEOh1Rc3Lny/xmIAAMAQCyCQWnTJmn16tC/waC154fru0gZW/4/nggwAACcTSy1W2JBfZe4YRk1AABnYrV2Syy6WX2XWFEHhgADAEgGarfEFXVgAABIBmq3pC0CDAAAHaF2S9oiwAAA0BFqt6StHqnuAAAAUdJpgmu4dktDQ9tJvNKpOTDUbkk6RmAAAOkjXsuV44XaLWmLAAMASA/h5cqnT5ptaAgdT1WIoXZLWmIZNQAg9eywXDmdbm3ZVDy/v5kDAwBIPSvLlcePT1q3ojidqXtvtMEtJABA6rFcGRYRYAAAqcdyZVjELSQAQMeSNe+D5cqwiBEYAED7krmkmeXKsIgAAwBoKxVLmlmuDAtYRg0AiJbqJc0sV+62WEYNAEicVC9pZrkyYsAtJABANJY0wwYIMACAaCxphg0QYAAA0cJLmk9fDRTmcEjFxSxpRkoRYAAA0VjSDBsgwAAA2mJJM9Icq5AAIBPFslS5slKqqGBJM9ISAQYAMo3XKy1YEL1UuqgodNvo9JEVljQjTXELCQAySSoq7AIJQIABgEwRDIZGXtorwB4+VlUVagekOQIMAGQKKxV2gTRHgAGATEGFXXQjBBgAyBRU2EU3QoABgExBhV10IwQYAMgUVNhFN0KAAYBMQoVddBMUsgOATEOFXXQDBBgAyERU2IXNEWAAINVi2ZcIQBQCDACkkpV9iQBEMIkXAFKFfYmATiPAAEAqsC8R0CUEGABIBfYlArrEUoAJBoNavHixhgwZoj59+mjYsGF65JFHZFr9PwhjjB544AEVFBSoT58+Kisr065du6Je59ChQ5oxY4ZycnKUm5urWbNm6ciRI/H5RABgB+xLBHSJpQDzxBNP6Nlnn9W//Mu/aMeOHXriiSe0dOlSPf3005E2S5cu1fLly7VixQrV1taqb9++Ki8v19GjRyNtZsyYoW3btmn9+vVau3at3nnnHc2ZMyd+nwoA0h37EgFd4jCmvRuw7fvWt76l/Px8Pffcc5FjU6ZMUZ8+ffTSSy/JGKPCwkLdfffduueeeyRJfr9f+fn5WrlypaZNm6YdO3Zo9OjR+uijjzR27FhJ0rp163TDDTdo3759KiwsbPO+x44d07FjxyI/BwIBFRcXy+/3Kycnp9MfHgBSJhiUBg8OTdht78+wwxFajVRXx5JqdBuBQEAulysu39+WRmDGjRunjRs36tNPP5Uk/c///I/effddXX/99ZKkuro6+Xw+lZWVRZ7jcrlUUlKimpoaSVJNTY1yc3Mj4UWSysrKlJWVpdra2nbfd8mSJXK5XJFHcXGxtU8JAOmGfYmALrEUYO6//35NmzZNI0eOVM+ePXXppZeqqqpKM2bMkCT5fD5JUn5+ftTz8vPzI+d8Pp/y8vKizvfo0UP9+/ePtDndokWL5Pf7I4/6+nor3QaA9MS+RECnWSpk98orr+jll1/WqlWrdOGFF+rjjz9WVVWVCgsLNXPmzET1UdnZ2crOzk7Y6wNAyrAvEdAplgLMwoULI6MwkjRmzBh99tlnWrJkiWbOnCm32y1JampqUkGriWdNTU265JJLJElut1sHDhyIet2TJ0/q0KFDkecDQEZhXyLAMku3kL788ktlZUU/xel0qqWlRZI0ZMgQud1ubdy4MXI+EAiotrZWHo9HkuTxeNTc3KzNmzdH2rz11ltqaWlRSUlJpz8IAADIHJZGYG688UY99thjuuCCC3ThhRdqy5Yt+ulPf6rbb79dkuRwOFRVVaVHH31Uw4cP15AhQ7R48WIVFhZq8uTJkqRRo0Zp4sSJmj17tlasWKETJ05o/vz5mjZtWrsrkAAAAE5nKcA8/fTTWrx4sX7wgx/owIEDKiws1Pe//3098MADkTb33nuvvvjiC82ZM0fNzc26+uqrtW7dOvXu3TvS5uWXX9b8+fM1YcIEZWVlacqUKVq+fHn8PhUAAOjWLNWBSRfxXEcOAJ0SDDLxFrAont/flkZgAAAK7RK9YEH0XkZFRaG6Lix9BpKCzRwBwAqvV5o6te1GjA0NoeNeb2r6BWQYAgwAxCoYDI28tHfnPXysqirUDkBCEWAAIFbV1W1HXlozRqqvD7UDkFAEGACIVWNjfNsB6DQCDADEqlWF8bi0A9BpBBgAiFVpaWi10em7R4c5HFJxcagdgIQiwABArJzO0FJpqW2ICf+8bBn1YIAkIMAAgBWVldKaNdLAgdHHi4pCx6kDAyQFhewAwKrKSqmigkq8QAoRYACgM5xOafz4VPcCyFjcQgIAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALbTI9UdAIAzCgal6mqpsVEqKJBKSyWnM9W9ApBiBBgA6cvrlRYskPbtO3WsqEh66impsjJ1/QKQctxCApCevF5p6tTo8CJJDQ2h415vavoFIC0QYACkn2AwNPJiTNtz4WNVVaF2ADISAQZA+qmubjvy0poxUn19qJ0UCjKbNkmrV4f+JdgA3R5zYACkn8bG2NsxTwbISIzAAEg/BQWxtdu1i3kyQIYiwABIP6WloVEUh6P98w5H6PwvfsE8GSBDEWAApB+nM3QLSGobYsI/z55tbZ4MgG6FAAMgPVVWSmvWSAMHRh8vKgodHz48tteJdT4NAFthEi+A9FVZKVVUtF+Jd9Om2F4j1vk0AGyFAAMgNWLdIsDplMaPb3s8PE+moaH9eTDheTKlpXHvOoDU4xYSgOTzeqXBg6Vrr5Vuvjn07+DB1lYNxTJPZtky9k0CuikCDIDkiucWAWebJ0MdGKDbchjT3thregsEAnK5XPL7/crJyUl1dwDEKhgMjbR0tHoofNunrs7ayAk7VgO2EM/vb+bAAEgeK1sEtDfvpSMdzZMB0G1xCwlA8ljZIgAAzoAAAyB5Yl3SzNJnAGdBgAGQPLFsEVBczNJnAGdFgAGQPCx9BhAnBBgAycXSZwBxwCokAMl3pi0CACAGBBgAqcHSZwBdwC0kAABgOwQYAABgOwQYAABgOwQYAABgO0ziBRDChogAbIQAA0DyeqUFC6I3WiwqChWdoy4LgDTELSQg03m90tSpbXeJbmgIHfd6U9MvADgDAgyQyYLB0MiLMW3PhY9VVYXaAUAaIcAAmay6uu3IS2vGSPX1oXYAkEYIMEAma2yMbzsASBICDJDJCgri2w4AksRSgBk8eLAcDkebx7x58yRJR48e1bx583Teeefp3HPP1ZQpU9TU1BT1Gnv37tWkSZN0zjnnKC8vTwsXLtTJkyfj94kAxK60NLTayOFo/7zDIRUXh9oBQBqxFGA++ugjNTY2Rh7r16+XJN10002SpDvvvFO/+c1v9Oqrr+rtt9/W/v37VdlqCWYwGNSkSZN0/Phxvf/++3rhhRe0cuVKPfDAA3H8SABi5nSGlkpLbUNM+Odly6gHAyDtOIxpb/lBbKqqqrR27Vrt2rVLgUBAAwYM0KpVqzR16lRJ0p/+9CeNGjVKNTU1uvLKK/XGG2/oW9/6lvbv36/8/HxJ0ooVK3TffffpL3/5i3r16hXT+wYCAblcLvn9fuXk5HS2+wDC2qsDU1wcCi/UgQEQJ/H8/u70HJjjx4/rpZde0u233y6Hw6HNmzfrxIkTKisri7QZOXKkLrjgAtXU1EiSampqNGbMmEh4kaTy8nIFAgFt27atw/c6duyYAoFA1ANAHFVWSv/7v9LvfietWhX6t66O8AIgbXW6Eu9rr72m5uZm3XrrrZIkn8+nXr16KTc3N6pdfn6+fD5fpE3r8BI+Hz7XkSVLluihhx7qbFcBxMLplMaPT3UvACAmnR6Bee6553T99dersLAwnv1p16JFi+T3+yOP+vr6hL8nAABIX50agfnss8+0YcMGeVuVGHe73Tp+/Liam5ujRmGamprkdrsjbT788MOo1wqvUgq3aU92drays7M701UAANANdWoE5vnnn1deXp4mTZoUOXb55ZerZ8+e2rhxY+TYzp07tXfvXnk8HkmSx+PR1q1bdeDAgUib9evXKycnR6NHj+7sZwAAABnG8ghMS0uLnn/+ec2cOVM9epx6usvl0qxZs3TXXXepf//+ysnJ0R133CGPx6Mrr7xSknTddddp9OjRuuWWW7R06VL5fD798Ic/1Lx58xhhAQAAMbMcYDZs2KC9e/fq9ttvb3PuySefVFZWlqZMmaJjx46pvLxcP/vZzyLnnU6n1q5dq7lz58rj8ahv376aOXOmHn744a59CgAAkFG6VAcmVagDAwCA/aRFHRgAAIBUIcAAAADbIcAAAADbIcAAAADbIcAAAADbIcAAAADbIcAAAADbIcAAAADbIcAAAADbIcAAAADbsbwXEoAkCAal6mqpsVEqKJBKSyWnM9W9AoC0QYAB0o3XKy1YIO3bd+pYUZH01FNSZeWpY4QcABmMW0hAOvF6palTo8OLJDU0hI57vafaDR4sXXutdPPNoX8HDz51HgC6OUZggGQ606hJMBgaeWlvg3hjJIdDqqqSWlqkb3+7bbtwyFmzJnqkBgC6IUZggGQ526hJdXXbkZfWjJHq66Uf/KDjkCOFQk4wGOfOA0B6IcAAyRDLraHGxthe6y9/6fhcOORUV3e+rwBgAwQYINHOdmtICo2a5OXF7z1jDUMAYFMEGCDRYr01JIVWGzkc7bdzOKQBA2J7z4ICa30EAJshwACJFutoyIEDoaXSUtsQE/75mWfOHnKKi0OTgwGgGyPAAIkW62hIQUFo9dCaNdLAgdHniopCx2+66ewhZ9ky6sEA6PYcxrR3Yz69BQIBuVwu+f1+5eTkpLo7wJkFg6HVRg0N7c+DcThCAaWuLnpJ9ZmK1LVX7K64OBReWEINIE3F8/ubAAMkQ3gVkhQdYsKjJp2p3UIlXgA2E8/vbwrZAckQvjXU3hYBnR01cTql8ePj1UMAsBUCDBAvZxsRqayUKioYNQGAOCDAAPEQ6waMjJoAQFywCgnoqlg3YAQAxA0BBuiKWKvssjcRAMQVAQboilir7LI3EQDEFQEG6IpYq+yyNxEAxBUBBugKK1V2AQBxQ4ABuqK0lL2JACAFCDBAVzid7E0EAClAgAG66mwbMLI3EQDEHYXsgHigyi4AJBUBBogXquwCQNJwCwkAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANgOAQYAANhOj1R3AEh7waBUXS01NkoFBVJpqeR0prpXAJDRCDDAmXi90oIF0r59p44VFUlPPSVVVqauXwCQ4biFBHTE65WmTo0OL5LU0BA67vWmpl8AAAIM0K5gMDTyYkzbc+FjVVWhdgCApCPAAO2prm478tKaMVJ9fagdACDpCDBAexob49sOABBXBBigPQUF8W0HAIgrAgzQntLS0Gojh6P98w6HVFwcagcASDoCDNAepzO0VFpqG2LCPy9bRj0YAEgRAgzQkcpKac0aaeDA6ONFRaHj1IEBgJShkB1wJpWVUkUFlXgBIM0QYICzcTql8eNT3QsAQCvcQgIAALZDgAEAALZjOcA0NDToO9/5js477zz16dNHY8aM0e9///vIeWOMHnjgARUUFKhPnz4qKyvTrl27ol7j0KFDmjFjhnJycpSbm6tZs2bpyJEjXf80AAAgI1gKMJ9//rmuuuoq9ezZU2+88Ya2b9+uf/7nf9Zf/dVfRdosXbpUy5cv14oVK1RbW6u+ffuqvLxcR48ejbSZMWOGtm3bpvXr12vt2rV65513NGfOnPh9KnRfwaC0aZO0enXo3472Ioq1HQDAnowF9913n7n66qs7PN/S0mLcbrf58Y9/HDnW3NxssrOzzerVq40xxmzfvt1IMh999FGkzRtvvGEcDodpaGiIqR9+v99IMn6/30r3YXe//KUxRUXGhHYiCj2KikLHO9MOAJBU8fz+tjQC8+tf/1pjx47VTTfdpLy8PF166aX6xS9+ETlfV1cnn8+nsrKyyDGXy6WSkhLV1NRIkmpqapSbm6uxY8dG2pSVlSkrK0u1tbXtvu+xY8cUCASiHsgwXq80dWrbDRYbGkLHvV5r7QAAtmYpwPz5z3/Ws88+q+HDh+vNN9/U3Llz9Q//8A964YUXJEk+n0+SlJ+fH/W8/Pz8yDmfz6e8vLyo8z169FD//v0jbU63ZMkSuVyuyKO4uNhKt2F3waC0YEFoLOV04WNVVdLx47G143YSANiepQDT0tKiyy67TI8//rguvfRSzZkzR7Nnz9aKFSsS1T9J0qJFi+T3+yOP+vr6hL4f0kx1ddsRldaMkerrpZ/9LLZ21dXx7yMAIKksBZiCggKNHj066tioUaO0d+9eSZLb7ZYkNTU1RbVpamqKnHO73Tpw4EDU+ZMnT+rQoUORNqfLzs5WTk5O1AMZpLExtnZ79sT39QAAactSgLnqqqu0c+fOqGOffvqpBg0aJEkaMmSI3G63Nm7cGDkfCARUW1srj8cjSfJ4PGpubtbmzZsjbd566y21tLSopKSk0x8E3VhBQWzthg2L7+sBANKWpQBz55136oMPPtDjjz+u3bt3a9WqVfr5z3+uefPmSZIcDoeqqqr06KOP6te//rW2bt2q7373uyosLNTkyZMlhUZsJk6cqNmzZ+vDDz/Ue++9p/nz52vatGkqLCyM+wdEN1BaGtpA8fRdocMcDqm4WPrBD2JrV1qauL4CAJLD6rKl3/zmN+aiiy4y2dnZZuTIkebnP/951PmWlhazePFik5+fb7Kzs82ECRPMzp07o9ocPHjQTJ8+3Zx77rkmJyfH3Hbbbebw4cMx94Fl1Bnol780xuEIPVovjw4fCy+RjrUdACDp4vn97TCmvSUb6S0QCMjlcsnv9zMfJpN4vaFVRq0n6hYXS8uWhXaNttoOAJBU8fz+JsDAXoLB0CqixsbQXJbS0tBu0Z1tBwBImnh+f/eIU5+ArosldDid0vjxZ3+tWNsBAGyJAIP00N5tn6Ii6amnuO0DAGjD8m7UQNxR/h8AYBEBBqkV6zYBlP8HALRCgEFqxbpNAOX/AQCtEGCQWrGW9af8PwCgFQIMUivWsv6U/wcAtEKAQWrFuk0A5f8BAK0QYJBaTmdoqbTUNsSEf162jCJ0AIAoBBikXmWltGaNNHBg9PGiotBx6sAAAE5DITukh8pKqaKC8v8AgJgQYJA+KP8PAIgRAQaJx8aKAIA4I8AgsdjjCACQAEziReKwxxEAIEEIMEgM9jgCACQQAQaJwR5HAIAEIsAgMdjjCACQQEziReedaXURexwBABKIERh0jtcrDR4sXXutdPPNoX8HDz41MZc9jgAACUSAgXWxrC5ijyMAQAIRYGCNldVF7HEEAEgQ5sDAGiuri8aPZ48jAEBCEGBgTWdWF7HHEQAgzriFBGtYXQQASAMEGFjD6iIAQBogwMAaVhcBANIAAQbWsboIAJBiTOJF57C6CACQQgQYdB6riwAAKcItJAAAYDuMwGSaM23ACACATRBgMonXG9oGoHUl3aKi0KoiJt4CAGyEW0iZIpYNGAEAsAkCTCawsgEjAAA2QIDJBFY2YAwLBqVNm6TVq0P/Em4AAGmEOTCZwOoGjMyVAQCkOUZgMoGVDRiZKwMAsAECTCaIdQPGceOYKwMAsAUCTCaIdQPG99+3PlcGAIAUIMBkilg2YLQ6VwYAgBRhEm8mOdsGjFbmygAAkEIEmExzpg0Yw3NlGhranwfjcITOl5YmtIsAAJwNt5BwSqxzZdg7CQCQYgQYRItlrgwAACnGLSS0dba5MgAApBgBBu0701wZAABSjFtIAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdggwAADAdiwFmB/96EdyOBxRj5EjR0bOHz16VPPmzdN5552nc889V1OmTFFTU1PUa+zdu1eTJk3SOeeco7y8PC1cuFAnT56Mz6cBAAAZoYfVJ1x44YXasGHDqRfoceol7rzzTv32t7/Vq6++KpfLpfnz56uyslLvvfeeJCkYDGrSpElyu916//331djYqO9+97vq2bOnHn/88Th8HAAAkAksB5gePXrI7Xa3Oe73+/Xcc89p1apV+pu/+RtJ0vPPP69Ro0bpgw8+0JVXXqn//u//1vbt27Vhwwbl5+frkksu0SOPPKL77rtPP/rRj9SrV6923/PYsWM6duxY5OdAIGC12wAAoBuxPAdm165dKiws1NChQzVjxgzt3btXkrR582adOHFCZWVlkbYjR47UBRdcoJqaGklSTU2NxowZo/z8/Eib8vJyBQIBbdu2rcP3XLJkiVwuV+RRXFxstdsAAKAbsRRgSkpKtHLlSq1bt07PPvus6urqVFpaqsOHD8vn86lXr17Kzc2Nek5+fr58Pp8kyefzRYWX8PnwuY4sWrRIfr8/8qivr7fSbQAA0M1YuoV0/fXXR/73xRdfrJKSEg0aNEivvPKK+vTpE/fOhWVnZys7Ozthrw8AAOylS8uoc3Nz9fWvf127d++W2+3W8ePH1dzcHNWmqakpMmfG7Xa3WZUU/rm9eTUAAADt6VKAOXLkiPbs2aOCggJdfvnl6tmzpzZu3Bg5v3PnTu3du1cej0eS5PF4tHXrVh04cCDSZv369crJydHo0aO70hUAAJBBLN1Cuueee3TjjTdq0KBB2r9/vx588EE5nU5Nnz5dLpdLs2bN0l133aX+/fsrJydHd9xxhzwej6688kpJ0nXXXafRo0frlltu0dKlS+Xz+fTDH/5Q8+bN4xYRAACImaUAs2/fPk2fPl0HDx7UgAEDdPXVV+uDDz7QgAEDJElPPvmksrKyNGXKFB07dkzl5eX62c9+Fnm+0+nU2rVrNXfuXHk8HvXt21czZ87Uww8/HN9PBQAAujWHMcakuhNWBQIBuVwu+f1+5eTkpLo7AAAgBvH8/mYvJAAAYDuWK/EijQWDUnW11NgoFRRIpaWS05nqXgEAEHcEmO7C65UWLJD27Tt1rKhIeuopqbIydf0CACABuIXUHXi90tSp0eFFkhoaQse93tT0CwCABCHA2F0wGBp5aW8udvhYVVWoHQAA3QQBxu6qq9uOvLRmjFRfH2oHAEA3QYCxu8bG+LYDAMAGCDB2V1AQ33YAANgAAcbuSktDq40cjvbPOxxScXGoHQAA3QQBxu6cztBSaaltiAn/vGwZ9WAAAN0KAaY7qKyU1qyRBg6MPl5UFDpOHRgAQDdDITs7iKXCbmWlVFFBJV4AQEYgwKQ7KxV2nU5p/Pikdg8AgFTgFlI6o8IuAADtIsCkKyrsAgDQIQJMuqLCLgAAHSLApCsq7AIA0CECTLqiwi4AAB0iwKQrKuwCANAhAky6osIuAAAdIsCkMyrsAgDQLgrZWRVLVdx4osIuAABtEGCssFIVN56osAsAQBRuIcWKqrgAAKQNAkwsqIoLAEBaIcDEgqq4AACkFQJMLKiKCwBAWiHAxIKquAAApBUCTCyoigsAQFohwMSCqrgAAKQVAkysqIoLAEDaoJCdFVTFBQAgLRBgrKIqLgAAKcctJAAAYDsEGAAAYDsEGAAAYDsEGAAAYDtM4m0tGGSFEQAANkCACfN6QztOt960sagoVMCOGi8AAKQVbiFJofAydWrbHacbGkLHvd7U9AsAALSLABMMhkZejGl7LnysqirUDgAApAUCTHV125GX1oyR6utD7QAAQFogwDQ2xrcdAABIOAJMQUF82wEAgIQjwJSWhlYbORztn3c4pOLiUDsAAJAWCDBOZ2iptNQ2xIR/XraMejAAAKQRAowUqvOyZo00cGD08aKi0HHqwAAAkFYoZBdWWSlVVFCJFwAAGyDAtOZ0SuPHp7oXAADgLLiFBAAAbIcAAwAAbIcAAwAAbIcAAwAAbIcAAwAAbIcAAwAAbIcAAwAAbIcAAwAAbIcAAwAAbMeWlXiNMZKkQCCQ4p4AAIBYhb+3w9/jXWHLAHP48GFJUnFxcYp7AgAArDp8+LBcLleXXsNh4hGDkqylpUX79+9Xv3795HA4uvx6gUBAxcXFqq+vV05OThx6aG9cj2hcj1O4FtG4HqdwLaJxPU5pfS369eunw4cPq7CwUFlZXZvFYssRmKysLBUVFcX9dXNycjL+F601rkc0rscpXItoXI9TuBbRuB6nhK9FV0dewpjECwAAbIcAAwAAbIcAIyk7O1sPPvigsrOzU92VtMD1iMb1OIVrEY3rcQrXIhrX45REXQtbTuIFAACZjREYAABgOwQYAABgOwQYAABgOwQYAABgOwQYAABgOxkbYB577DGNGzdO55xzjnJzc2N6zq233iqHwxH1mDhxYmI7miSduR7GGD3wwAMqKChQnz59VFZWpl27diW2o0lw6NAhzZgxQzk5OcrNzdWsWbN05MiRMz5n/PjxbX43/v7v/z5JPY6vZ555RoMHD1bv3r1VUlKiDz/88IztX331VY0cOVK9e/fWmDFj9F//9V9J6mlyWLkeK1eubPN70Lt37yT2NnHeeecd3XjjjSosLJTD4dBrr7121uds2rRJl112mbKzs/W1r31NK1euTHg/k8Hqtdi0aVOb3wuHwyGfz5ecDifQkiVL9Nd//dfq16+f8vLyNHnyZO3cufOsz4vH342MDTDHjx/XTTfdpLlz51p63sSJE9XY2Bh5rF69OkE9TK7OXI+lS5dq+fLlWrFihWpra9W3b1+Vl5fr6NGjCexp4s2YMUPbtm3T+vXrtXbtWr3zzjuaM2fOWZ83e/bsqN+NpUuXJqG38fWf//mfuuuuu/Tggw/qD3/4g77xjW+ovLxcBw4caLf9+++/r+nTp2vWrFnasmWLJk+erMmTJ+uTTz5Jcs8Tw+r1kELl0lv/Hnz22WdJ7HHifPHFF/rGN76hZ555Jqb2dXV1mjRpkq699lp9/PHHqqqq0ve+9z29+eabCe5p4lm9FmE7d+6M+t3Iy8tLUA+T5+2339a8efP0wQcfaP369Tpx4oSuu+46ffHFFx0+J25/N0yGe/75543L5Yqp7cyZM01FRUVC+5NqsV6PlpYW43a7zY9//OPIsebmZpOdnW1Wr16dwB4m1vbt240k89FHH0WOvfHGG8bhcJiGhoYOn3fNNdeYBQsWJKGHiXXFFVeYefPmRX4OBoOmsLDQLFmypN323/72t82kSZOijpWUlJjvf//7Ce1nsli9Hlb+ntiZJPOrX/3qjG3uvfdec+GFF0Yd+7u/+ztTXl6ewJ4lXyzX4ne/+52RZD7//POk9CmVDhw4YCSZt99+u8M28fq7kbEjMJ21adMm5eXlacSIEZo7d64OHjyY6i6lRF1dnXw+n8rKyiLHXC6XSkpKVFNTk8KedU1NTY1yc3M1duzYyLGysjJlZWWptrb2jM99+eWXdf755+uiiy7SokWL9OWXXya6u3F1/Phxbd68Oeq/aVZWlsrKyjr8b1pTUxPVXpLKy8tt/TsQ1pnrIUlHjhzRoEGDVFxcrIqKCm3bti0Z3U073fl3o7MuueQSFRQU6G//9m/13nvvpbo7CeH3+yVJ/fv377BNvH43bLkbdapMnDhRlZWVGjJkiPbs2aN//Md/1PXXX6+amho5nc5Udy+pwvdu8/Pzo47n5+fb+r6uz+drM6zbo0cP9e/f/4yf6+abb9agQYNUWFioP/7xj7rvvvu0c+dOeb3eRHc5bv7v//5PwWCw3f+mf/rTn9p9js/n63a/A2GduR4jRozQv//7v+viiy+W3+/XT37yE40bN07btm1TUVFRMrqdNjr63QgEAvrqq6/Up0+fFPUs+QoKCrRixQqNHTtWx44d07/9279p/Pjxqq2t1WWXXZbq7sVNS0uLqqqqdNVVV+miiy7qsF28/m50qwBz//3364knnjhjmx07dmjkyJGdev1p06ZF/veYMWN08cUXa9iwYdq0aZMmTJjQqddMpERfDzuJ9Vp0Vus5MmPGjFFBQYEmTJigPXv2aNiwYZ1+XdiLx+ORx+OJ/Dxu3DiNGjVK//qv/6pHHnkkhT1DKo0YMUIjRoyI/Dxu3Djt2bNHTz75pF588cUU9iy+5s2bp08++UTvvvtuUt6vWwWYu+++W7feeusZ2wwdOjRu7zd06FCdf/752r17d1oGmEReD7fbLUlqampSQUFB5HhTU5MuueSSTr1mIsV6Ldxud5sJmidPntShQ4cinzkWJSUlkqTdu3fbJsCcf/75cjqdampqijre1NTU4Wd3u92W2ttJZ67H6Xr27KlLL71Uu3fvTkQX01pHvxs5OTkZNfrSkSuuuCJpX/TJMH/+/Miih7ONNsbr70a3CjADBgzQgAEDkvZ++/bt08GDB6O+wNNJIq/HkCFD5Ha7tXHjxkhgCQQCqq2ttbyyKxlivRYej0fNzc3avHmzLr/8cknSW2+9pZaWlkgoicXHH38sSWn7u9GeXr166fLLL9fGjRs1efJkSaEh4Y0bN2r+/PntPsfj8Wjjxo2qqqqKHFu/fn3UKIRddeZ6nC4YDGrr1q264YYbEtjT9OTxeNosje0uvxvx8PHHH9vq70NHjDG644479Ktf/UqbNm3SkCFDzvqcuP3d6Mws4+7gs88+M1u2bDEPPfSQOffcc82WLVvMli1bzOHDhyNtRowYYbxerzHGmMOHD5t77rnH1NTUmLq6OrNhwwZz2WWXmeHDh5ujR4+m6mPEjdXrYYwx//RP/2Ryc3PN66+/bv74xz+aiooKM2TIEPPVV1+l4iPEzcSJE82ll15qamtrzbvvvmuGDx9upk+fHjm/b98+M2LECFNbW2uMMWb37t3m4YcfNr///e9NXV2def31183QoUPNN7/5zVR9hE77j//4D5OdnW1Wrlxptm/fbubMmWNyc3ONz+czxhhzyy23mPvvvz/S/r333jM9evQwP/nJT8yOHTvMgw8+aHr27Gm2bt2aqo8QV1avx0MPPWTefPNNs2fPHrN582Yzbdo007t3b7Nt27ZUfYS4OXz4cOTvgiTz05/+1GzZssV89tlnxhhj7r//fnPLLbdE2v/5z38255xzjlm4cKHZsWOHeeaZZ4zT6TTr1q1L1UeIG6vX4sknnzSvvfaa2bVrl9m6datZsGCBycrKMhs2bEjVR4ibuXPnGpfLZTZt2mQaGxsjjy+//DLSJlF/NzI2wMycOdNIavP43e9+F2kjyTz//PPGGGO+/PJLc91115kBAwaYnj17mkGDBpnZs2dH/pDZndXrYUxoKfXixYtNfn6+yc7ONhMmTDA7d+5Mfufj7ODBg2b69Onm3HPPNTk5Oea2226LCnJ1dXVR12bv3r3mm9/8punfv7/Jzs42X/va18zChQuN3+9P0SfomqefftpccMEFplevXuaKK64wH3zwQeTcNddcY2bOnBnV/pVXXjFf//rXTa9evcyFF15ofvvb3ya5x4ll5XpUVVVF2ubn55sbbrjB/OEPf0hBr+MvvBT49Ef488+cOdNcc801bZ5zySWXmF69epmhQ4dG/f2wM6vX4oknnjDDhg0zvXv3Nv379zfjx483b731Vmo6H2ftXYfTvysS9XfD8f87AAAAYBvUgQEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALZDgAEAALbz/wA6ynuNoxuTdQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize the class\n",
    "scaler_linear = StandardScaler()\n",
    "\n",
    "# Compute the mean and standard deviation of the training set then transform it\n",
    "X_train_scaled = scaler_linear.fit_transform(x_train)\n",
    "\n",
    "print(f\"Computed mean of the training set: {scaler_linear.mean_.squeeze():.2f}\")\n",
    "print(f\"Computed standard deviation of the training set: {scaler_linear.scale_.squeeze():.2f}\")\n",
    "\n",
    "# Plot the results\n",
    "plt.scatter(x=X_train_scaled[:, 0], y=y_train[:, 0], c='red')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a58531a9",
   "metadata": {},
   "source": [
    "**Train the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "30b67fd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize the class\n",
    "linear_model = LinearRegression()\n",
    "\n",
    "# Train the model\n",
    "linear_model.fit(X_train_scaled, y_train )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6712983",
   "metadata": {},
   "source": [
    "**Evaluate the Model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "cb8d9598",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training MSE (using sklearn function): 406.18142643101237\n",
      "training MSE (for-loop implementation): 406.1814264310124\n"
     ]
    }
   ],
   "source": [
    "# Feed the scaled training set and get the predictions\n",
    "yhat = linear_model.predict(X_train_scaled)\n",
    "\n",
    "# Use scikit-learn's utility function and divide by 2\n",
    "print(f\"training MSE (using sklearn function): {mean_squared_error(y_train, yhat) / 2}\")\n",
    "\n",
    "# for-loop implementation\n",
    "total_squared_error = 0\n",
    "\n",
    "for i in range(len(yhat)):\n",
    "    squared_error_i  = (yhat[i] - y_train[i])**2\n",
    "    total_squared_error += squared_error_i                                              \n",
    "\n",
    "mse = total_squared_error / (2*len(yhat))\n",
    "\n",
    "print(f\"training MSE (for-loop implementation): {mse.squeeze()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f53d3cbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean used to scale the CV set: 2504.06\n",
      "Standard deviation used to scale the CV set: 574.85\n",
      "Cross validation MSE(meen squared error): 551.7633686764407\n"
     ]
    }
   ],
   "source": [
    "# Scale the cross validation set using the mean and standard deviation of the training set\n",
    "X_cv_scaled = scaler_linear.transform(x_cv)\n",
    "\n",
    "print(f\"Mean used to scale the CV set: {scaler_linear.mean_.squeeze():.2f}\")\n",
    "print(f\"Standard deviation used to scale the CV set: {scaler_linear.scale_.squeeze():.2f}\")\n",
    "\n",
    "# Feed the scaled cross validation set\n",
    "yhat = linear_model.predict(X_cv_scaled)\n",
    "\n",
    "# Use scikit-learn's utility function and divide by 2\n",
    "print(f\"Cross validation MSE(meen squared error): {mean_squared_error(y_cv, yhat) / 2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7478201",
   "metadata": {},
   "source": [
    "### Adding Polynomial Features\n",
    "**Adding Polynomial Features**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7fd51d6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3.32e+03 1.11e+07]\n",
      " [2.34e+03 5.50e+06]\n",
      " [3.49e+03 1.22e+07]\n",
      " [2.63e+03 6.92e+06]\n",
      " [2.59e+03 6.71e+06]]\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the class to make polynomial features\n",
    "poly = PolynomialFeatures(degree=2, include_bias=False)\n",
    "\n",
    "# Compute the number of features and transform the training set\n",
    "X_train_mapped = poly.fit_transform(x_train)\n",
    "\n",
    "# Preview the first 5 elements of the new training set. Left column is `x` and right column is `x^2`\n",
    "# Note: The `e+<number>` in the output denotes how many places the decimal point should \n",
    "# be moved. For example, `3.24e+03` is equal to `3240`\n",
    "print(X_train_mapped[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "34d7c263",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.43  1.47]\n",
      " [-0.28 -0.36]\n",
      " [ 1.71  1.84]\n",
      " [ 0.22  0.11]\n",
      " [ 0.15  0.04]]\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the class\n",
    "scaler_poly = StandardScaler()\n",
    "\n",
    "# Compute the mean and standard deviation of the training set then transform it\n",
    "X_train_mapped_scaled = scaler_poly.fit_transform(X_train_mapped)\n",
    "\n",
    "# Preview the first 5 elements of the scaled training set.\n",
    "print(X_train_mapped_scaled[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "570abc97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MSE: 49.115763074720604\n",
      "Cross validation MSE: 87.69466438435839\n"
     ]
    }
   ],
   "source": [
    "# Initialize the class\n",
    "model = LinearRegression()\n",
    "\n",
    "# Train the model\n",
    "model.fit(X_train_mapped_scaled, y_train )\n",
    "\n",
    "# Compute the training MSE\n",
    "yhat = model.predict(X_train_mapped_scaled)\n",
    "print(f\"Training MSE: {mean_squared_error(y_train, yhat) / 2}\")\n",
    "\n",
    "# Add the polynomial features to the cross validation set\n",
    "X_cv_mapped = poly.transform(x_cv)\n",
    "\n",
    "# Scale the cross validation set using the mean and standard deviation of the training set\n",
    "X_cv_mapped_scaled = scaler_poly.transform(X_cv_mapped)\n",
    "\n",
    "# Compute the cross validation MSE\n",
    "yhat = model.predict(X_cv_mapped_scaled)\n",
    "print(f\"Cross validation MSE: {mean_squared_error(y_cv, yhat) / 2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "3cf0e9a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjw0lEQVR4nO3de3BU5cHH8d8mISFAdjEM2U2axEarhQiogMKK9KKRSCMjTWiLLyJVpp0ygRJQBmm5eA/SqRdaAXEcYAZTRlrQQgc0IsYL4SKUDhdFbdMmmmzClLILdBJCsu8f6a4sBGVz22fPfj8zO5hzTpLnmQj7zdnznLX5/X6/AAAADBIX6QEAAABciEABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJyESA+gI1pbW1VbW6uUlBTZbLZIDwcAAFwGv9+vU6dOKSMjQ3FxX32OJCoDpba2VllZWZEeBgAA6ICamhplZmZ+5TFRGSgpKSmS2iZot9sjPBoAAHA5fD6fsrKygs/jXyUqAyXwso7dbidQAACIMpdzeQYXyQIAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACME5U3ausuLa1+7a06oYZTjUpL6a2bc1IVH8d7/QAA0NMIlP/ZfrhOj245qjpvY3BbuqO3lkzI1Z1D0iM4MgAAYg8v8agtTmasPxASJ5Lk8TZqxvoD2n64LkIjAwAgNsV8oLS0+vXolqPyt7MvsO3RLUfV0treEQAAoDvEfKDsrTpx0ZmT8/kl1XkbtbfqRM8NCgCAGBfzgdJw6tJx0pHjAABA58V8oKSl9O7S4wAAQOfFfKDcnJOqdEdvXWoxsU1tq3luzkntyWEBABDTYj5Q4uNsWjIhV5IuipTAx0sm5HI/FAAAelDMB4ok3TkkXSvvHS6XI/RlHJejt1beO5z7oAAA0MO4Udv/3DkkXXfkuriTLAAABiBQzhMfZ5P76gGRHgYAADGPl3gAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYJ6xAeeSRR2Sz2UIegwYNCu5vbGxUcXGxBgwYoH79+qmoqEj19fUhX6O6uloFBQXq06eP0tLSNG/ePJ07d65rZgMAACwhIdxPuO666/TWW299+QUSvvwSc+bM0V/+8hdt3LhRDodDM2fOVGFhoT744ANJUktLiwoKCuRyubRr1y7V1dXpvvvuU69evfTUU091wXQAAIAVhB0oCQkJcrlcF233er16+eWXVVZWpttuu02StGbNGg0ePFi7d+/W6NGj9eabb+ro0aN666235HQ6dcMNN+jxxx/X/Pnz9cgjjygxMbHzMwIAAFEv7GtQPv30U2VkZOiqq67SlClTVF1dLUnav3+/mpublZeXFzx20KBBys7OVmVlpSSpsrJSQ4cOldPpDB6Tn58vn8+nI0eOXPJ7NjU1yefzhTwAAIB1hRUoo0aN0tq1a7V9+3atXLlSVVVVGjt2rE6dOiWPx6PExET1798/5HOcTqc8Ho8kyePxhMRJYH9g36WUlpbK4XAEH1lZWeEMGwAARJmwXuIZP3588L+HDRumUaNG6corr9Srr76q5OTkLh9cwIIFCzR37tzgxz6fj0gBAMDCOrXMuH///rr22mv12WefyeVy6ezZszp58mTIMfX19cFrVlwu10WregIft3ddS0BSUpLsdnvIAwAAWFenAuX06dP6+9//rvT0dI0YMUK9evXSjh07gvuPHTum6upqud1uSZLb7dahQ4fU0NAQPKa8vFx2u125ubmdGQoAALCQsF7ieeihhzRhwgRdeeWVqq2t1ZIlSxQfH6977rlHDodD06dP19y5c5Wamiq73a5Zs2bJ7XZr9OjRkqRx48YpNzdXU6dO1bJly+TxeLRw4UIVFxcrKSmpWyYIAACiT1iB8vnnn+uee+7Rv//9bw0cOFC33nqrdu/erYEDB0qSnn32WcXFxamoqEhNTU3Kz8/XihUrgp8fHx+vrVu3asaMGXK73erbt6+mTZumxx57rGtnBQAAoprN7/f7Iz2IcPl8PjkcDnm9Xq5HAQAgSoTz/M178QAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgJkR6AUVpapPfek+rqpPR0aexYKT4+0qMCACDmECgBmzZJs2dLn3/+5bbMTOn556XCwsiNCwCAGMRLPFJbnEyaFBonkvTFF23bN22KzLgAAIhRBEpLS9uZE7//4n2BbSUlbccBAIAe0alAWbp0qWw2m0pKSoLbGhsbVVxcrAEDBqhfv34qKipSfX19yOdVV1eroKBAffr0UVpamubNm6dz5851Zigd9957F585OZ/fL9XUtB0HAAB6RIcDZd++fXrxxRc1bNiwkO1z5szRli1btHHjRlVUVKi2tlaF513D0dLSooKCAp09e1a7du3SunXrtHbtWi1evLjjs+iMurquPQ4AAHRahwLl9OnTmjJlil566SVdccUVwe1er1cvv/yynnnmGd12220aMWKE1qxZo127dmn37t2SpDfffFNHjx7V+vXrdcMNN2j8+PF6/PHH9cILL+js2bNdM6twpKd37XEAAKDTOhQoxcXFKigoUF5eXsj2/fv3q7m5OWT7oEGDlJ2drcrKSklSZWWlhg4dKqfTGTwmPz9fPp9PR44c6chwOmfs2LbVOjZb+/ttNikrq+04AADQI8JeZrxhwwYdOHBA+/btu2ifx+NRYmKi+vfvH7Ld6XTK4/EEjzk/TgL7A/va09TUpKampuDHPp8v3GFfWnx821LiSZPaYuT8i2UD0fLcc9wPBQCAHhTWGZSamhrNnj1br7zyinr37t1dY7pIaWmpHA5H8JGVldW136CwUPrjH6VvfCN0e2Zm23bugwIAQI8KK1D279+vhoYGDR8+XAkJCUpISFBFRYWWL1+uhIQEOZ1OnT17VidPngz5vPr6erlcLkmSy+W6aFVP4OPAMRdasGCBvF5v8FFTUxPOsC9PYaH0z39KO3dKZWVtf1ZVEScAAERAWC/x3H777Tp06FDItvvvv1+DBg3S/PnzlZWVpV69emnHjh0qKiqSJB07dkzV1dVyu92SJLfbrSeffFINDQ1KS0uTJJWXl8tutys3N7fd75uUlKSkpKSwJxe2+Hjpe9/r/u8DAAC+UliBkpKSoiFDhoRs69u3rwYMGBDcPn36dM2dO1epqamy2+2aNWuW3G63Ro8eLUkaN26ccnNzNXXqVC1btkwej0cLFy5UcXFxz0QIAAAwXpe/F8+zzz6ruLg4FRUVqampSfn5+VqxYkVwf3x8vLZu3aoZM2bI7Xarb9++mjZtmh577LGuHgoAAIhSNr+/vXu8m83n88nhcMjr9cput0d6OAAA4DKE8/zNe/EAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAME5YgbJy5UoNGzZMdrtddrtdbrdb27ZtC+5vbGxUcXGxBgwYoH79+qmoqEj19fUhX6O6uloFBQXq06eP0tLSNG/ePJ07d65rZgMAACwhrEDJzMzU0qVLtX//fn344Ye67bbbdPfdd+vIkSOSpDlz5mjLli3auHGjKioqVFtbq8LCwuDnt7S0qKCgQGfPntWuXbu0bt06rV27VosXL+7aWQEAgKhm8/v9/s58gdTUVP3mN7/RpEmTNHDgQJWVlWnSpEmSpI8//liDBw9WZWWlRo8erW3btumuu+5SbW2tnE6nJGnVqlWaP3++jh8/rsTExMv6nj6fTw6HQ16vV3a7vTPDBwAAPSSc5+8OX4PS0tKiDRs26MyZM3K73dq/f7+am5uVl5cXPGbQoEHKzs5WZWWlJKmyslJDhw4Nxokk5efny+fzBc/CtKepqUk+ny/kAQAArCvsQDl06JD69eunpKQk/eIXv9DmzZuVm5srj8ejxMRE9e/fP+R4p9Mpj8cjSfJ4PCFxEtgf2HcppaWlcjgcwUdWVla4wwYAAFEk7ED59re/rYMHD2rPnj2aMWOGpk2bpqNHj3bH2IIWLFggr9cbfNTU1HTr9wMAAJGVEO4nJCYm6lvf+pYkacSIEdq3b5+ef/55/eQnP9HZs2d18uTJkLMo9fX1crlckiSXy6W9e/eGfL3AKp/AMe1JSkpSUlJSuEMFAABRqtP3QWltbVVTU5NGjBihXr16aceOHcF9x44dU3V1tdxutyTJ7Xbr0KFDamhoCB5TXl4uu92u3Nzczg4FAABYRFhnUBYsWKDx48crOztbp06dUllZmd555x298cYbcjgcmj59uubOnavU1FTZ7XbNmjVLbrdbo0ePliSNGzdOubm5mjp1qpYtWyaPx6OFCxequLiYMyQAACAorEBpaGjQfffdp7q6OjkcDg0bNkxvvPGG7rjjDknSs88+q7i4OBUVFampqUn5+flasWJF8PPj4+O1detWzZgxQ263W3379tW0adP02GOPde2sAABAVOv0fVAigfugAAAQfXrkPigAAADdhUABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYJK1BKS0t10003KSUlRWlpaZo4caKOHTsWckxjY6OKi4s1YMAA9evXT0VFRaqvrw85prq6WgUFBerTp4/S0tI0b948nTt3rvOzAQAAlhBWoFRUVKi4uFi7d+9WeXm5mpubNW7cOJ05cyZ4zJw5c7RlyxZt3LhRFRUVqq2tVWFhYXB/S0uLCgoKdPbsWe3atUvr1q3T2rVrtXjx4q6bFQAAiGo2v9/v7+gnHz9+XGlpaaqoqNB3vvMdeb1eDRw4UGVlZZo0aZIk6eOPP9bgwYNVWVmp0aNHa9u2bbrrrrtUW1srp9MpSVq1apXmz5+v48ePKzEx8Wu/r8/nk8PhkNfrld1u7+jwAQBADwrn+btT16B4vV5JUmpqqiRp//79am5uVl5eXvCYQYMGKTs7W5WVlZKkyspKDR06NBgnkpSfny+fz6cjR460+32amprk8/lCHgAAwLo6HCitra0qKSnRmDFjNGTIEEmSx+NRYmKi+vfvH3Ks0+mUx+MJHnN+nAT2B/a1p7S0VA6HI/jIysrq6LABAEAU6HCgFBcX6/Dhw9qwYUNXjqddCxYskNfrDT5qamq6/XsCAIDISejIJ82cOVNbt27Vu+++q8zMzOB2l8uls2fP6uTJkyFnUerr6+VyuYLH7N27N+TrBVb5BI65UFJSkpKSkjoyVAAAEIXCOoPi9/s1c+ZMbd68WW+//bZycnJC9o8YMUK9evXSjh07gtuOHTum6upqud1uSZLb7dahQ4fU0NAQPKa8vFx2u125ubmdmQsAALCIsM6gFBcXq6ysTK+//rpSUlKC14w4HA4lJyfL4XBo+vTpmjt3rlJTU2W32zVr1iy53W6NHj1akjRu3Djl5uZq6tSpWrZsmTwejxYuXKji4mLOkgAAAElhLjO22Wztbl+zZo1++tOfSmq7UduDDz6oP/zhD2pqalJ+fr5WrFgR8vLNv/71L82YMUPvvPOO+vbtq2nTpmnp0qVKSLi8XmKZMQAA0Sec5+9O3QclUggUAACiT4/dBwUAAKA7ECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADBOQqQHAAAAzNHS6tfeqhNqONWotJTeujknVfFxth4fB4ECAAAkSdsP1+nRLUdV520Mbkt39NaSCbm6c0h6j46Fl3gAAIC2H67TjPUHQuJEkjzeRs1Yf0DbD9f16HgIFAAAYlxLq1+Pbjkqfzv7Atse3XJULa3tHdE9CBQAAGLc3qoTF505OZ9fUp23UXurTvTYmAgUAABiXMOpS8dJR47rCgQKAAAxLi2ld5ce1xUIFAAAYtzNOalKd/TWpRYT29S2mufmnNQeGxOBAgBAJ7W0+lX593/r9YNfqPLv/+7Ri0m7QnycTUsm5ErSRZES+HjJhNwevR8K90EBAKATTLp3SGfcOSRdK+8dftFcXBGai83v90dX5kny+XxyOBzyer2y2+2RHg4AIEYF7h1y4RNp4DzDynuHR1WkSN17J9lwnr85gwIAQAd83b1DbGq7d8gdua6I3Cq+o+LjbHJfPSDSw+AaFAAAOsLEe4dYCYECAEAHmHjvECshUAAA6AAT7x1iJWEHyrvvvqsJEyYoIyNDNptNr732Wsh+v9+vxYsXKz09XcnJycrLy9Onn34acsyJEyc0ZcoU2e129e/fX9OnT9fp06c7NREAuBzRvhw0gHlEnon3DrGSsC+SPXPmjK6//no98MADKiwsvGj/smXLtHz5cq1bt045OTlatGiR8vPzdfToUfXu3VaRU6ZMUV1dncrLy9Xc3Kz7779fP//5z1VWVtb5GQHAJVhlOSjzMEPg3iEz1h+QTQq5WDZS9w6xkk4tM7bZbNq8ebMmTpwoqe3sSUZGhh588EE99NBDkiSv1yun06m1a9dq8uTJ+uijj5Sbm6t9+/Zp5MiRkqTt27frBz/4gT7//HNlZGR87fdlmTGAcFllOSjzME+0h1ZPitgy46qqKnk8HuXl5QW3ORwOjRo1SpWVlZo8ebIqKyvVv3//YJxIUl5enuLi4rRnzx798Ic/vOjrNjU1qampKfixz+frymEDsDirLAdlHma6c0i67sh1ddu9Q2JVl14k6/F4JElOpzNku9PpDO7zeDxKS0sL2Z+QkKDU1NTgMRcqLS2Vw+EIPrKysrpy2AAszirLQZmHuQL3Drn7hm/IffUA4qQLRMUqngULFsjr9QYfNTU1kR4SgChileWgzAOxpEsDxeVySZLq6+tDttfX1wf3uVwuNTQ0hOw/d+6cTpw4ETzmQklJSbLb7SEPALhcVlkOyjwQS7o0UHJycuRyubRjx47gNp/Ppz179sjtdkuS3G63Tp48qf379wePefvtt9Xa2qpRo0Z15XBiVjQv2wO6g1WWgzIPxJKwL5I9ffq0Pvvss+DHVVVVOnjwoFJTU5Wdna2SkhI98cQTuuaaa4LLjDMyMoIrfQYPHqw777xTP/vZz7Rq1So1Nzdr5syZmjx58mWt4MFX42py4GJWWQ7KPBBLwl5m/M477+j73//+RdunTZumtWvXyu/3a8mSJVq9erVOnjypW2+9VStWrNC1114bPPbEiROaOXOmtmzZori4OBUVFWn58uXq16/fZY2BZcbts9KyPaA7WCXgmQeiVTjP3526D0qkECgXa2n169an377klfE2SS5Hb70//zZ+K0FM6863ku9JzAPRKGL3QUHkhLNsz4S30QYixZS3ku8s5gGrI1AsworL9vjNCgBiF4FiEVZbtmel16atElpWmQeA6ECgWERg2Z7H29ju7aMD16BEw7K9S13s6/E2asb6A1F1sa9VQssq8wAQPaLiTrL4eoFle5IuurdANC3b+7r36JDa3qMjGu7tEgitC68NCoTW9sN1ERpZeKwyDwDRhUCxkDuHpGvlvcPlcoS+jONy9I6asw5WeY8Oq4SWVeYBIPrwEo/FRPu7alrlYl+rrKqyyjwARB8CxYKiedmeVS72tUpoWWUeAKIPL/HAKFZ5jw6rhJZV5gEg+hAoMIpVLva1SmhZZR4Aog+BAuNY4WJfq4SWVeYBIPrwXjwwlhVuDGaV+4dYZR4AIos3CwQMYoXQkqwzDwCRw5sFAgaJ5lVV57PKPABEB65BAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHG7VZUUuL9N57Ul2dlJ4ujR0rxcdHelQAAFw2AsVqNm2SZs+WPv/8y22ZmdLzz0uFhZEbV0dYJbSYBwCEjZd4rGTTJmnSpNA4kaQvvmjbvmlTZMbVEZs2Sd/8pvT970v/939tf37zm9E1B4l5AEAH8WaBVtHS0vaEcWGcBNhsbWdSqqrM/603EFoX/q9p+98b0/3xj9FxNoh5AEAI3s04Fr3zTttvtV9n507pe9/r7tF0nFVCi3kAwEXCef7mJR6rqKvr2uMi5b33Lv1kKLX9Fl9T03acyZiHuVpa2oL+D39o+7OlJdIjAtAOLpK1ivT0rj0uUqwSWszDTFxEbh6rzANdjkCxirFj2/6h/eKLi68VkL48FT92bM+PLRxWCS3mYZ5LXUsTuIg8mq6lsUpoWWUekrVCy5S5+KOQ1+v1S/J7vd5ID8Usf/qT32+ztT3a/hluewS2/elPkR7h1zt3zu/PzLx4DufPJSur7TiTMQ+zBObR3hyiaR5+/5d/z9ubQ7T8Pff7rTMPv79trBf+/5WZGV1zCOjmuYTz/E2gWE17/3NlZUXXXxQrhJbfzzxMsnPnpePk/MfOnZEe6VezSmhZZR5+v/VCq5vnEs7zNxfJWk1hofTPf7at1ikra/uzqiq6TpcWFradbv/GN0K3Z2ZG12l45mEOq1xLY5WLlq0yj5aWtpeo2ntZPbCtpCQ6LsQ2cC5cg2JF8fFmLyW+HIWF0t13m/E6aGcwDzNY5Voaq4SWVeYRTmiZ/m+ygXMhUGAuK4SWxDxMwEXkZrHKPKwSWpKRc+ElHgDWFx/ftjJE+vIOuAGBj597zvwzQoHQunAOATablJVlfmhZZR5WCS3JyLkQKABigxWupbFKaFllHlYJLcnIuRAoAGIHF5GbwwrzsEpoSUbOhffiAYBoZMrNtDrLCvNo74ZzWVltT+jREFrn6+a58GaBAAD0JCuEVkA3ziWc529W8QAA0FnRvMrtQobMhWtQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHGi8k6ygbvz+3y+CI8EAABcrsDz9uW8y05UBsqpU6ckSVlZWREeCQAACNepU6fkcDi+8piofLPA1tZW1dbWKiUlRbYL3xYaktoqNSsrSzU1NbyhogH4eZiFn4dZ+HmYp7t+Jn6/X6dOnVJGRobi4r76KpOoPIMSFxenzMzMSA8jKtjtdv7CG4Sfh1n4eZiFn4d5uuNn8nVnTgK4SBYAABiHQAEAAMYhUCwqKSlJS5YsUVJSUqSHAvHzMA0/D7Pw8zCPCT+TqLxIFgAAWBtnUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQLKS0tFQ33XSTUlJSlJaWpokTJ+rYsWORHhb+Z+nSpbLZbCopKYn0UGLaF198oXvvvVcDBgxQcnKyhg4dqg8//DDSw4pJLS0tWrRokXJycpScnKyrr75ajz/++GW9Tws6791339WECROUkZEhm82m1157LWS/3+/X4sWLlZ6eruTkZOXl5enTTz/tsfERKBZSUVGh4uJi7d69W+Xl5Wpubta4ceN05syZSA8t5u3bt08vvviihg0bFumhxLT//Oc/GjNmjHr16qVt27bp6NGj+u1vf6srrrgi0kOLSU8//bRWrlyp3//+9/roo4/09NNPa9myZfrd734X6aHFhDNnzuj666/XCy+80O7+ZcuWafny5Vq1apX27Nmjvn37Kj8/X42NjT0yPpYZW9jx48eVlpamiooKfec734n0cGLW6dOnNXz4cK1YsUJPPPGEbrjhBj333HORHlZMevjhh/XBBx/ovffei/RQIOmuu+6S0+nUyy+/HNxWVFSk5ORkrV+/PoIjiz02m02bN2/WxIkTJbWdPcnIyNCDDz6ohx56SJLk9XrldDq1du1aTZ48udvHxBkUC/N6vZKk1NTUCI8kthUXF6ugoEB5eXmRHkrM+/Of/6yRI0fqRz/6kdLS0nTjjTfqpZdeivSwYtYtt9yiHTt26JNPPpEk/e1vf9P777+v8ePHR3hkqKqqksfjCfl3y+FwaNSoUaqsrOyRMUTlmwXi67W2tqqkpERjxozRkCFDIj2cmLVhwwYdOHBA+/bti/RQIOkf//iHVq5cqblz5+pXv/qV9u3bp1/+8pdKTEzUtGnTIj28mPPwww/L5/Np0KBBio+PV0tLi5588klNmTIl0kOLeR6PR5LkdDpDtjudzuC+7kagWFRxcbEOHz6s999/P9JDiVk1NTWaPXu2ysvL1bt370gPB2oL95EjR+qpp56SJN144406fPiwVq1aRaBEwKuvvqpXXnlFZWVluu6663Tw4EGVlJQoIyODnwd4iceKZs6cqa1bt2rnzp3KzMyM9HBi1v79+9XQ0KDhw4crISFBCQkJqqio0PLly5WQkKCWlpZIDzHmpKenKzc3N2Tb4MGDVV1dHaERxbZ58+bp4Ycf1uTJkzV06FBNnTpVc+bMUWlpaaSHFvNcLpckqb6+PmR7fX19cF93I1AsxO/3a+bMmdq8ebPefvtt5eTkRHpIMe3222/XoUOHdPDgweBj5MiRmjJlig4ePKj4+PhIDzHmjBkz5qKl95988omuvPLKCI0otv33v/9VXFzo01B8fLxaW1sjNCIE5OTkyOVyaceOHcFtPp9Pe/bskdvt7pEx8BKPhRQXF6usrEyvv/66UlJSgq8TOhwOJScnR3h0sSclJeWi63/69u2rAQMGcF1QhMyZM0e33HKLnnrqKf34xz/W3r17tXr1aq1evTrSQ4tJEyZM0JNPPqns7Gxdd911+utf/6pnnnlGDzzwQKSHFhNOnz6tzz77LPhxVVWVDh48qNTUVGVnZ6ukpERPPPGErrnmGuXk5GjRokXKyMgIrvTpdn5YhqR2H2vWrIn00PA/3/3ud/2zZ8+O9DBi2pYtW/xDhgzxJyUl+QcNGuRfvXp1pIcUs3w+n3/27Nn+7Oxsf+/evf1XXXWV/9e//rW/qakp0kOLCTt37mz3OWPatGl+v9/vb21t9S9atMjvdDr9SUlJ/ttvv91/7NixHhsf90EBAADG4RoUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcf4ftlgPlE5JCzEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize lists to save the errors, models, and feature transforms\n",
    "train_mses = []\n",
    "cv_mses = []\n",
    "models = []\n",
    "polys = []\n",
    "scalers = []\n",
    "\n",
    "# Loop over 10 times. Each adding one more degree of polynomial higher than the last.\n",
    "for degree in range(1,11):\n",
    "    \n",
    "    # Add polynomial features to the training set\n",
    "    poly = PolynomialFeatures(degree, include_bias=False)\n",
    "    X_train_mapped = poly.fit_transform(x_train)\n",
    "    polys.append(poly)\n",
    "    \n",
    "    # Scale the training set\n",
    "    scaler_poly = StandardScaler()\n",
    "    X_train_mapped_scaled = scaler_poly.fit_transform(X_train_mapped)\n",
    "    scalers.append(scaler_poly)\n",
    "    \n",
    "    # Create and train the model\n",
    "    model = LinearRegression()\n",
    "    model.fit(X_train_mapped_scaled, y_train )\n",
    "    models.append(model)\n",
    "    \n",
    "    # Compute the training MSE\n",
    "    yhat = model.predict(X_train_mapped_scaled)\n",
    "    train_mse = mean_squared_error(y_train, yhat) / 2\n",
    "    train_mses.append(train_mse)\n",
    "    \n",
    "    # Add polynomial features and scale the cross validation set\n",
    "    X_cv_mapped = poly.transform(x_cv)\n",
    "    X_cv_mapped_scaled = scaler_poly.transform(X_cv_mapped)\n",
    "    \n",
    "    # Compute the cross validation MSE\n",
    "    yhat = model.predict(X_cv_mapped_scaled)\n",
    "    cv_mse = mean_squared_error(y_cv, yhat) / 2\n",
    "    cv_mses.append(cv_mse)\n",
    "    \n",
    "# Plot the results\n",
    "degrees=range(1,11)\n",
    "# Plot the results\n",
    "plt.scatter(x=degrees, y=train_mses, color='red')\n",
    "plt.scatter(x=degrees, y=cv_mses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82ed498a",
   "metadata": {},
   "source": [
    "**Choosing the best model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c5161239",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lowest CV MSE is found in the model with degree=4\n"
     ]
    }
   ],
   "source": [
    "# Get the model with the lowest CV MSE (add 1 because list indices start at 0)\n",
    "# This also corresponds to the degree of the polynomial added\n",
    "degree = np.argmin(cv_mses) + 1\n",
    "print(f\"Lowest CV MSE is found in the model with degree={degree}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d881c2c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MSE: 47.15\n",
      "Cross Validation MSE: 79.42\n",
      "Test MSE: 104.64\n"
     ]
    }
   ],
   "source": [
    "# Add polynomial features to the test set\n",
    "X_test_mapped = polys[degree-1].transform(x_test)\n",
    "\n",
    "# Scale the test set\n",
    "X_test_mapped_scaled = scalers[degree-1].transform(X_test_mapped)\n",
    "\n",
    "# Compute the test MSE\n",
    "yhat = models[degree-1].predict(X_test_mapped_scaled)\n",
    "test_mse = mean_squared_error(y_test, yhat) / 2\n",
    "\n",
    "print(f\"Training MSE: {train_mses[degree-1]:.2f}\")\n",
    "print(f\"Cross Validation MSE: {cv_mses[degree-1]:.2f}\")\n",
    "print(f\"Test MSE: {test_mse:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dff9c485",
   "metadata": {},
   "source": [
    "## Neural Networks\n",
    "### Prepare the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "3d4a3919",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add polynomial features\n",
    "degree = 1\n",
    "poly = PolynomialFeatures(degree, include_bias=False)\n",
    "X_train_mapped = poly.fit_transform(x_train)\n",
    "X_cv_mapped = poly.transform(x_cv)\n",
    "X_test_mapped = poly.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "28b01598",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the features using the z-score\n",
    "scaler = StandardScaler()\n",
    "X_train_mapped_scaled = scaler.fit_transform(X_train_mapped)\n",
    "X_cv_mapped_scaled = scaler.transform(X_cv_mapped)\n",
    "X_test_mapped_scaled = scaler.transform(X_test_mapped)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14f8de20",
   "metadata": {},
   "source": [
    "### Build and train the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "f7c30690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model_arch_1...\n",
      "Done!\n",
      "\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 63ms/step\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 50ms/step\n",
      "Training model_arch_2...\n",
      "Done!\n",
      "\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 62ms/step\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 71ms/step\n",
      "Training model_arch_3...\n",
      "Done!\n",
      "\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 88ms/step\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 99ms/step\n",
      "RESULTS:\n",
      "Model 1: Training MSE: 406.18, CV MSE: 571.75\n",
      "Model 2: Training MSE: 406.18, CV MSE: 551.89\n",
      "Model 3: Training MSE: 406.18, CV MSE: 295399.77\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "# Initialize lists that will contain the errors for each model\n",
    "nn_train_mses = []\n",
    "nn_cv_mses = []\n",
    "\n",
    "# Build the models\n",
    "nn_models = [\n",
    "    Sequential(\n",
    "    [               \n",
    "        Dense(units=25, activation='relu', name='L1'),\n",
    "        Dense(units=15, activation='relu', name='L2'),\n",
    "        Dense(units=1, activation='relu', name='L3')\n",
    "    ], name = \"model_arch_1\" \n",
    "),\n",
    "    Sequential(\n",
    "    [               \n",
    "        Dense(units=20, activation='relu', name='L1'),\n",
    "        Dense(units=12, activation='relu', name='L2'),\n",
    "        Dense(units=12, activation='relu', name='L3'),\n",
    "        Dense(units=20, activation='relu', name='L4'),\n",
    "        Dense(units=1, activation='relu', name='L5')\n",
    "    ], name = \"model_arch_2\" \n",
    "), \n",
    "        Sequential(\n",
    "    [               \n",
    "        Dense(units=32, activation='relu', name='L1'),\n",
    "        Dense(units=16, activation='relu', name='L2'),\n",
    "        Dense(units=8, activation='relu', name='L3'),\n",
    "        Dense(units=4, activation='relu', name='L4'),\n",
    "        Dense(units=12, activation='relu', name='L5'),\n",
    "        Dense(units=1, activation='relu', name='L6')\n",
    "    ], name = \"model_arch_3\" \n",
    ")\n",
    "]\n",
    "\n",
    "# Loop over the the models\n",
    "for model in nn_models:\n",
    "    \n",
    "    # Setup the loss and optimizer\n",
    "    model.compile(\n",
    "    loss='mse',\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),\n",
    "    )\n",
    "\n",
    "    print(f\"Training {model.name}...\")\n",
    "    \n",
    "    # Train the model\n",
    "    model.fit(\n",
    "        X_train_mapped_scaled, y_train,\n",
    "        epochs=300,\n",
    "        verbose=0\n",
    "    )\n",
    "    \n",
    "    print(\"Done!\\n\")\n",
    "\n",
    "    \n",
    "    # Record the training MSEs\n",
    "    yhat = model.predict(X_train_mapped_scaled)\n",
    "    nn_train_mses.append(train_mse)\n",
    "    \n",
    "    # Record the cross validation MSEs \n",
    "    yhat = model.predict(X_cv_mapped_scaled)\n",
    "    cv_mse = mean_squared_error(y_cv, yhat) / 2\n",
    "    nn_cv_mses.append(cv_mse)\n",
    "\n",
    "    \n",
    "# print results\n",
    "print(\"RESULTS:\")\n",
    "for model_num in range(len(nn_train_mses)):\n",
    "    print(\n",
    "        f\"Model {model_num+1}: Training MSE: {nn_train_mses[model_num]:.2f}, \" +\n",
    "        f\"CV MSE: {nn_cv_mses[model_num]:.2f}\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "165c5409",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 21ms/step\n",
      "Selected Model: 2\n",
      "Training MSE: 406.18\n",
      "Cross Validation MSE: 551.89\n",
      "Test MSE: 511.44\n"
     ]
    }
   ],
   "source": [
    "# Select the model with the lowest CV MSE\n",
    "model_num = 2\n",
    "\n",
    "# Compute the test MSE\n",
    "yhat = nn_models[model_num-1].predict(X_test_mapped_scaled)\n",
    "test_mse = mean_squared_error(y_test, yhat) / 2\n",
    "\n",
    "print(f\"Selected Model: {model_num}\")\n",
    "print(f\"Training MSE: {nn_train_mses[model_num-1]:.2f}\")\n",
    "print(f\"Cross Validation MSE: {nn_cv_mses[model_num-1]:.2f}\")\n",
    "print(f\"Test MSE: {test_mse:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84e9a8c1",
   "metadata": {},
   "source": [
    "### Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "74fc96c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the shape of the inputs x is: (200, 2)\n",
      "the shape of the targets y is: (200, 1)\n"
     ]
    }
   ],
   "source": [
    "data = np.array([[1.00e+03, 1.46e+03, 0.00e+00],\n",
    "       [1.05e+03, 1.01e+03, 0.00e+00],\n",
    "       [1.09e+03, 8.54e+02, 0.00e+00],\n",
    "       [1.14e+03, 2.56e+03, 0.00e+00],\n",
    "       [1.18e+03, 4.62e+03, 0.00e+00],\n",
    "       [1.23e+03, 4.97e+03, 0.00e+00],\n",
    "       [1.27e+03, 2.39e+03, 0.00e+00],\n",
    "       [1.32e+03, 1.01e+02, 0.00e+00],\n",
    "       [1.36e+03, 7.29e+02, 0.00e+00],\n",
    "       [1.41e+03, 4.22e+03, 0.00e+00],\n",
    "       [1.45e+03, 4.30e+03, 0.00e+00],\n",
    "       [1.50e+03, 4.52e+02, 0.00e+00],\n",
    "       [1.54e+03, 2.76e+02, 0.00e+00],\n",
    "       [1.59e+03, 2.24e+03, 0.00e+00],\n",
    "       [1.63e+03, 2.76e+03, 0.00e+00],\n",
    "       [1.68e+03, 2.96e+03, 0.00e+00],\n",
    "       [1.72e+03, 3.99e+03, 0.00e+00],\n",
    "       [1.77e+03, 8.79e+02, 0.00e+00],\n",
    "       [1.81e+03, 3.42e+03, 0.00e+00],\n",
    "       [1.86e+03, 1.48e+03, 0.00e+00],\n",
    "       [1.90e+03, 1.28e+03, 0.00e+00],\n",
    "       [1.95e+03, 4.02e+02, 0.00e+00],\n",
    "       [1.99e+03, 1.11e+03, 0.00e+00],\n",
    "       [2.04e+03, 2.36e+03, 0.00e+00],\n",
    "       [2.09e+03, 7.79e+02, 1.00e+00],\n",
    "       [2.13e+03, 4.07e+03, 0.00e+00],\n",
    "       [2.18e+03, 9.55e+02, 0.00e+00],\n",
    "       [2.22e+03, 7.04e+02, 1.00e+00],\n",
    "       [2.27e+03, 4.85e+03, 0.00e+00],\n",
    "       [2.31e+03, 6.78e+02, 0.00e+00],\n",
    "       [2.36e+03, 1.18e+03, 1.00e+00],\n",
    "       [2.40e+03, 4.15e+03, 0.00e+00],\n",
    "       [2.45e+03, 4.87e+03, 0.00e+00],\n",
    "       [2.49e+03, 4.45e+03, 0.00e+00],\n",
    "       [2.54e+03, 4.42e+03, 0.00e+00],\n",
    "       [2.58e+03, 2.44e+03, 0.00e+00],\n",
    "       [2.63e+03, 4.37e+03, 0.00e+00],\n",
    "       [2.67e+03, 1.83e+03, 1.00e+00],\n",
    "       [2.72e+03, 1.73e+03, 0.00e+00],\n",
    "       [2.76e+03, 4.32e+03, 0.00e+00],\n",
    "       [2.81e+03, 2.71e+03, 0.00e+00],\n",
    "       [2.85e+03, 2.69e+03, 0.00e+00],\n",
    "       [2.90e+03, 4.75e+03, 0.00e+00],\n",
    "       [2.94e+03, 3.52e+02, 1.00e+00],\n",
    "       [2.99e+03, 1.41e+03, 0.00e+00],\n",
    "       [3.04e+03, 4.77e+02, 0.00e+00],\n",
    "       [3.08e+03, 2.86e+03, 0.00e+00],\n",
    "       [3.13e+03, 9.80e+02, 0.00e+00],\n",
    "       [3.17e+03, 4.65e+03, 0.00e+00],\n",
    "       [3.22e+03, 3.12e+03, 0.00e+00],\n",
    "       [3.26e+03, 2.46e+03, 0.00e+00],\n",
    "       [3.31e+03, 3.09e+03, 0.00e+00],\n",
    "       [3.35e+03, 2.99e+03, 0.00e+00],\n",
    "       [3.40e+03, 1.33e+03, 1.00e+00],\n",
    "       [3.44e+03, 8.29e+02, 1.00e+00],\n",
    "       [3.49e+03, 4.50e+03, 0.00e+00],\n",
    "       [3.53e+03, 4.55e+03, 0.00e+00],\n",
    "       [3.58e+03, 2.66e+03, 1.00e+00],\n",
    "       [3.62e+03, 5.00e+03, 0.00e+00],\n",
    "       [3.67e+03, 3.47e+03, 0.00e+00],\n",
    "       [3.71e+03, 2.91e+03, 0.00e+00],\n",
    "       [3.76e+03, 1.68e+03, 0.00e+00],\n",
    "       [3.80e+03, 1.96e+03, 0.00e+00],\n",
    "       [3.85e+03, 1.06e+03, 0.00e+00],\n",
    "       [3.89e+03, 4.27e+02, 0.00e+00],\n",
    "       [3.94e+03, 1.26e+02, 1.00e+00],\n",
    "       [3.98e+03, 3.19e+03, 0.00e+00],\n",
    "       [4.03e+03, 2.64e+03, 1.00e+00],\n",
    "       [4.08e+03, 1.21e+03, 1.00e+00],\n",
    "       [4.12e+03, 1.66e+03, 1.00e+00],\n",
    "       [4.17e+03, 1.36e+03, 0.00e+00],\n",
    "       [4.21e+03, 2.11e+03, 0.00e+00],\n",
    "       [4.26e+03, 4.60e+03, 0.00e+00],\n",
    "       [4.30e+03, 3.97e+03, 0.00e+00],\n",
    "       [4.35e+03, 4.17e+03, 0.00e+00],\n",
    "       [4.39e+03, 2.84e+03, 1.00e+00],\n",
    "       [4.44e+03, 3.02e+02, 1.00e+00],\n",
    "       [4.48e+03, 2.94e+03, 0.00e+00],\n",
    "       [4.53e+03, 2.34e+03, 1.00e+00],\n",
    "       [4.57e+03, 3.02e+03, 0.00e+00],\n",
    "       [4.62e+03, 3.87e+03, 0.00e+00],\n",
    "       [4.66e+03, 2.26e+03, 0.00e+00],\n",
    "       [4.71e+03, 2.04e+03, 0.00e+00],\n",
    "       [4.75e+03, 3.07e+03, 1.00e+00],\n",
    "       [4.80e+03, 4.80e+03, 0.00e+00],\n",
    "       [4.84e+03, 3.27e+02, 1.00e+00],\n",
    "       [4.89e+03, 2.06e+03, 0.00e+00],\n",
    "       [4.93e+03, 3.32e+03, 1.00e+00],\n",
    "       [4.98e+03, 4.70e+03, 0.00e+00],\n",
    "       [5.03e+03, 1.13e+03, 0.00e+00],\n",
    "       [5.07e+03, 2.49e+03, 1.00e+00],\n",
    "       [5.12e+03, 9.05e+02, 1.00e+00],\n",
    "       [5.16e+03, 4.05e+03, 0.00e+00],\n",
    "       [5.21e+03, 4.67e+03, 0.00e+00],\n",
    "       [5.25e+03, 3.84e+03, 0.00e+00],\n",
    "       [5.30e+03, 2.59e+03, 1.00e+00],\n",
    "       [5.34e+03, 4.90e+03, 0.00e+00],\n",
    "       [5.39e+03, 4.95e+03, 0.00e+00],\n",
    "       [5.43e+03, 3.72e+03, 1.00e+00],\n",
    "       [5.48e+03, 4.35e+03, 0.00e+00],\n",
    "       [5.52e+03, 1.88e+03, 1.00e+00],\n",
    "       [5.57e+03, 5.28e+02, 1.00e+00],\n",
    "       [5.61e+03, 2.29e+03, 1.00e+00],\n",
    "       [5.66e+03, 3.82e+03, 1.00e+00],\n",
    "       [5.70e+03, 5.03e+01, 1.00e+00],\n",
    "       [5.75e+03, 1.76e+03, 1.00e+00],\n",
    "       [5.79e+03, 2.14e+03, 1.00e+00],\n",
    "       [5.84e+03, 3.77e+03, 1.00e+00],\n",
    "       [5.88e+03, 1.51e+02, 1.00e+00],\n",
    "       [5.93e+03, 2.81e+03, 0.00e+00],\n",
    "       [5.97e+03, 0.00e+00, 1.00e+00],\n",
    "       [6.02e+03, 3.89e+03, 0.00e+00],\n",
    "       [6.07e+03, 1.93e+03, 1.00e+00],\n",
    "       [6.11e+03, 1.63e+03, 1.00e+00],\n",
    "       [6.16e+03, 1.38e+03, 1.00e+00],\n",
    "       [6.20e+03, 4.20e+03, 1.00e+00],\n",
    "       [6.25e+03, 2.21e+03, 1.00e+00],\n",
    "       [6.29e+03, 3.27e+03, 1.00e+00],\n",
    "       [6.34e+03, 1.16e+03, 1.00e+00],\n",
    "       [6.38e+03, 1.56e+03, 1.00e+00],\n",
    "       [6.43e+03, 1.86e+03, 1.00e+00],\n",
    "       [6.47e+03, 2.31e+03, 1.00e+00],\n",
    "       [6.52e+03, 3.69e+03, 0.00e+00],\n",
    "       [6.56e+03, 4.02e+03, 1.00e+00],\n",
    "       [6.61e+03, 3.59e+03, 1.00e+00],\n",
    "       [6.65e+03, 2.19e+03, 1.00e+00],\n",
    "       [6.70e+03, 4.52e+03, 0.00e+00],\n",
    "       [6.74e+03, 3.64e+03, 0.00e+00],\n",
    "       [6.79e+03, 4.12e+03, 0.00e+00],\n",
    "       [6.83e+03, 2.51e+02, 1.00e+00],\n",
    "       [6.88e+03, 8.04e+02, 1.00e+00],\n",
    "       [6.92e+03, 2.09e+03, 1.00e+00],\n",
    "       [6.97e+03, 4.57e+03, 1.00e+00],\n",
    "       [7.02e+03, 2.51e+03, 1.00e+00],\n",
    "       [7.06e+03, 3.14e+03, 1.00e+00],\n",
    "       [7.11e+03, 5.78e+02, 1.00e+00],\n",
    "       [7.15e+03, 3.17e+03, 1.00e+00],\n",
    "       [7.20e+03, 2.26e+02, 1.00e+00],\n",
    "       [7.24e+03, 4.27e+03, 1.00e+00],\n",
    "       [7.29e+03, 2.61e+03, 1.00e+00],\n",
    "       [7.33e+03, 3.79e+03, 1.00e+00],\n",
    "       [7.38e+03, 3.39e+03, 1.00e+00],\n",
    "       [7.42e+03, 2.79e+03, 1.00e+00],\n",
    "       [7.47e+03, 4.72e+03, 1.00e+00],\n",
    "       [7.51e+03, 1.61e+03, 1.00e+00],\n",
    "       [7.56e+03, 3.77e+02, 1.00e+00],\n",
    "       [7.60e+03, 1.03e+03, 1.00e+00],\n",
    "       [7.65e+03, 4.10e+03, 1.00e+00],\n",
    "       [7.69e+03, 2.74e+03, 1.00e+00],\n",
    "       [7.74e+03, 2.01e+03, 1.00e+00],\n",
    "       [7.78e+03, 1.31e+03, 1.00e+00],\n",
    "       [7.83e+03, 6.53e+02, 1.00e+00],\n",
    "       [7.87e+03, 1.91e+03, 1.00e+00],\n",
    "       [7.92e+03, 1.08e+03, 1.00e+00],\n",
    "       [7.96e+03, 6.03e+02, 1.00e+00],\n",
    "       [8.01e+03, 7.54e+01, 1.00e+00],\n",
    "       [8.06e+03, 4.25e+03, 1.00e+00],\n",
    "       [8.10e+03, 1.23e+03, 1.00e+00],\n",
    "       [8.15e+03, 3.74e+03, 1.00e+00],\n",
    "       [8.19e+03, 3.29e+03, 1.00e+00],\n",
    "       [8.24e+03, 4.77e+03, 1.00e+00],\n",
    "       [8.28e+03, 7.54e+02, 1.00e+00],\n",
    "       [8.33e+03, 3.04e+03, 1.00e+00],\n",
    "       [8.37e+03, 2.89e+03, 1.00e+00],\n",
    "       [8.42e+03, 4.40e+03, 1.00e+00],\n",
    "       [8.46e+03, 2.01e+02, 1.00e+00],\n",
    "       [8.51e+03, 1.51e+03, 1.00e+00],\n",
    "       [8.55e+03, 3.22e+03, 1.00e+00],\n",
    "       [8.60e+03, 2.51e+01, 1.00e+00],\n",
    "       [8.64e+03, 1.43e+03, 1.00e+00],\n",
    "       [8.69e+03, 5.53e+02, 1.00e+00],\n",
    "       [8.73e+03, 1.53e+03, 1.00e+00],\n",
    "       [8.78e+03, 1.58e+03, 1.00e+00],\n",
    "       [8.82e+03, 1.76e+02, 1.00e+00],\n",
    "       [8.87e+03, 4.92e+03, 1.00e+00],\n",
    "       [8.91e+03, 3.54e+03, 1.00e+00],\n",
    "       [8.96e+03, 2.16e+03, 1.00e+00],\n",
    "       [9.01e+03, 2.41e+03, 1.00e+00],\n",
    "       [9.05e+03, 1.71e+03, 1.00e+00],\n",
    "       [9.10e+03, 1.26e+03, 1.00e+00],\n",
    "       [9.14e+03, 3.57e+03, 1.00e+00],\n",
    "       [9.19e+03, 3.94e+03, 1.00e+00],\n",
    "       [9.23e+03, 3.92e+03, 1.00e+00],\n",
    "       [9.28e+03, 3.49e+03, 1.00e+00],\n",
    "       [9.32e+03, 3.67e+03, 1.00e+00],\n",
    "       [9.37e+03, 2.54e+03, 1.00e+00],\n",
    "       [9.41e+03, 5.03e+02, 1.00e+00],\n",
    "       [9.46e+03, 4.47e+03, 1.00e+00],\n",
    "       [9.50e+03, 6.28e+02, 1.00e+00],\n",
    "       [9.55e+03, 3.37e+03, 1.00e+00],\n",
    "       [9.59e+03, 1.78e+03, 1.00e+00],\n",
    "       [9.64e+03, 3.24e+03, 1.00e+00],\n",
    "       [9.68e+03, 3.62e+03, 1.00e+00],\n",
    "       [9.73e+03, 4.82e+03, 1.00e+00],\n",
    "       [9.77e+03, 1.98e+03, 1.00e+00],\n",
    "       [9.82e+03, 3.34e+03, 1.00e+00],\n",
    "       [9.86e+03, 3.44e+03, 1.00e+00],\n",
    "       [9.91e+03, 1.81e+03, 1.00e+00],\n",
    "       [9.95e+03, 3.52e+03, 1.00e+00],\n",
    "       [1.00e+04, 9.30e+02, 1.00e+00]])\n",
    "\n",
    "# Split the inputs and outputs into separate arrays\n",
    "x_bc = data[:,:-1]\n",
    "y_bc = data[:,-1]\n",
    "\n",
    "# Convert y into 2-D because the commands later will require it (x is already 2-D)\n",
    "y_bc = np.expand_dims(y_bc, axis=1)\n",
    "\n",
    "print(f\"the shape of the inputs x is: {x_bc.shape}\")\n",
    "print(f\"the shape of the targets y is: {y_bc.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c6e10cb",
   "metadata": {},
   "source": [
    "### Split and prepare the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "8a47dd56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the shape of the training set (input) is: (120, 2)\n",
      "the shape of the training set (target) is: (120, 1)\n",
      "\n",
      "the shape of the cross validation set (input) is: (40, 2)\n",
      "the shape of the cross validation set (target) is: (40, 1)\n",
      "\n",
      "the shape of the test set (input) is: (40, 2)\n",
      "the shape of the test set (target) is: (40, 1)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Get 60% of the dataset as the training set. Put the remaining 40% in temporary variables.\n",
    "x_bc_train, x_, y_bc_train, y_ = train_test_split(x_bc, y_bc, test_size=0.40, random_state=1)\n",
    "\n",
    "# Split the 40% subset above into two: one half for cross validation and the other for the test set\n",
    "x_bc_cv, x_bc_test, y_bc_cv, y_bc_test = train_test_split(x_, y_, test_size=0.50, random_state=1)\n",
    "\n",
    "# Delete temporary variables\n",
    "del x_, y_\n",
    "\n",
    "print(f\"the shape of the training set (input) is: {x_bc_train.shape}\")\n",
    "print(f\"the shape of the training set (target) is: {y_bc_train.shape}\\n\")\n",
    "print(f\"the shape of the cross validation set (input) is: {x_bc_cv.shape}\")\n",
    "print(f\"the shape of the cross validation set (target) is: {y_bc_cv.shape}\\n\")\n",
    "print(f\"the shape of the test set (input) is: {x_bc_test.shape}\")\n",
    "print(f\"the shape of the test set (target) is: {y_bc_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "0ba726a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the features\n",
    "\n",
    "# Initialize the class\n",
    "scaler_linear = StandardScaler()\n",
    "\n",
    "# Compute the mean and standard deviation of the training set then transform it\n",
    "x_bc_train_scaled = scaler_linear.fit_transform(x_bc_train)\n",
    "x_bc_cv_scaled = scaler_linear.transform(x_bc_cv)\n",
    "x_bc_test_scaled = scaler_linear.transform(x_bc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39b183a8",
   "metadata": {},
   "source": [
    "### Evaluating the error for classification models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "9723b61b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "probabilities: [0.2 0.6 0.7 0.3 0.8]\n",
      "predictions with threshold=0.5: [0 1 1 0 1]\n",
      "targets: [1 1 1 1 1]\n",
      "fraction of misclassified data (for-loop): 0.4\n",
      "fraction of misclassified data (with np.mean()): 0.4\n"
     ]
    }
   ],
   "source": [
    "# Sample model output\n",
    "probabilities = np.array([0.2, 0.6, 0.7, 0.3, 0.8])\n",
    "\n",
    "# Apply a threshold to the model output. If greater than 0.5, set to 1. Else 0.\n",
    "predictions = np.where(probabilities >= 0.5, 1, 0)\n",
    "\n",
    "# Ground truth labels\n",
    "ground_truth = np.array([1, 1, 1, 1, 1])\n",
    "\n",
    "# Initialize counter for misclassified data\n",
    "misclassified = 0\n",
    "\n",
    "# Get number of predictions\n",
    "num_predictions = len(predictions)\n",
    "\n",
    "# Loop over each prediction\n",
    "for i in range(num_predictions):\n",
    "    \n",
    "    # Check if it matches the ground truth\n",
    "    if predictions[i] != ground_truth[i]:\n",
    "        \n",
    "        # Add one to the counter if the prediction is wrong\n",
    "        misclassified += 1\n",
    "\n",
    "# Compute the fraction of the data that the model misclassified\n",
    "fraction_error = misclassified/num_predictions\n",
    "\n",
    "print(f\"probabilities: {probabilities}\")\n",
    "print(f\"predictions with threshold=0.5: {predictions}\")\n",
    "print(f\"targets: {ground_truth}\")\n",
    "print(f\"fraction of misclassified data (for-loop): {fraction_error}\")\n",
    "print(f\"fraction of misclassified data (with np.mean()): {np.mean(predictions != ground_truth)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "d5c305e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model_arch_1...\n",
      "Done!\n",
      "\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 15ms/step\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step \n",
      "Training model_arch_2...\n",
      "Done!\n",
      "\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 28ms/step\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step \n",
      "Training model_arch_3...\n",
      "Done!\n",
      "\n",
      "\u001b[1m4/4\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 25ms/step\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step \n",
      "Model 1: Training Set Classification Error: 0.06667, CV Set Classification Error: 0.15000\n",
      "Model 2: Training Set Classification Error: 0.10833, CV Set Classification Error: 0.17500\n",
      "Model 3: Training Set Classification Error: 0.07500, CV Set Classification Error: 0.10000\n"
     ]
    }
   ],
   "source": [
    "# Initialize lists that will contain the errors for each model\n",
    "nn_train_error = []\n",
    "nn_cv_error = []\n",
    "\n",
    "# Build the models\n",
    "models_bc = nn_models = [\n",
    "    Sequential(\n",
    "    [               \n",
    "        Dense(units=25, activation='relu', name='L1'),\n",
    "        Dense(units=15, activation='relu', name='L2'),\n",
    "        Dense(units=1, activation='linear', name='L3')\n",
    "    ], name = \"model_arch_1\" \n",
    "),\n",
    "    Sequential(\n",
    "    [               \n",
    "        Dense(units=20, activation='relu', name='L1'),\n",
    "        Dense(units=12, activation='relu', name='L2'),\n",
    "        Dense(units=12, activation='relu', name='L3'),\n",
    "        Dense(units=20, activation='relu', name='L4'),\n",
    "        Dense(units=1, activation='linear', name='L5')\n",
    "    ], name = \"model_arch_2\" \n",
    "), \n",
    "        Sequential(\n",
    "    [               \n",
    "        Dense(units=32, activation='relu', name='L1'),\n",
    "        Dense(units=16, activation='relu', name='L2'),\n",
    "        Dense(units=8, activation='relu', name='L3'),\n",
    "        Dense(units=4, activation='relu', name='L4'),\n",
    "        Dense(units=12, activation='relu', name='L5'),\n",
    "        Dense(units=1, activation='linear', name='L6')\n",
    "    ], name = \"model_arch_3\" \n",
    ")\n",
    "]\n",
    "\n",
    "\n",
    "# Loop over each model\n",
    "for model in models_bc:\n",
    "    \n",
    "    # Setup the loss and optimizer\n",
    "    model.compile(\n",
    "    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),\n",
    "    )\n",
    "\n",
    "    print(f\"Training {model.name}...\")\n",
    "\n",
    "    # Train the model\n",
    "    model.fit(\n",
    "        x_bc_train_scaled, y_bc_train,\n",
    "        epochs=200,\n",
    "        verbose=0\n",
    "    )\n",
    "    \n",
    "    print(\"Done!\\n\")\n",
    "    \n",
    "    # Set the threshold for classification\n",
    "    threshold = 0.5\n",
    "    \n",
    "    # Record the fraction of misclassified examples for the training set\n",
    "    yhat = model.predict(x_bc_train_scaled)\n",
    "    yhat = tf.math.sigmoid(yhat)\n",
    "    yhat = np.where(yhat >= threshold, 1, 0)\n",
    "    train_error = np.mean(yhat != y_bc_train)\n",
    "    nn_train_error.append(train_error)\n",
    "\n",
    "    # Record the fraction of misclassified examples for the cross validation set\n",
    "    yhat = model.predict(x_bc_cv_scaled)\n",
    "    yhat = tf.math.sigmoid(yhat)\n",
    "    yhat = np.where(yhat >= threshold, 1, 0)\n",
    "    cv_error = np.mean(yhat != y_bc_cv)\n",
    "    nn_cv_error.append(cv_error)\n",
    "\n",
    "# Print the result\n",
    "for model_num in range(len(nn_train_error)):\n",
    "    print(\n",
    "        f\"Model {model_num+1}: Training Set Classification Error: {nn_train_error[model_num]:.5f}, \" +\n",
    "        f\"CV Set Classification Error: {nn_cv_error[model_num]:.5f}\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "952872d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 3ms/step \n",
      "Selected Model: 2\n",
      "Training Set Classification Error: 0.1083\n",
      "CV Set Classification Error: 0.1750\n",
      "Test Set Classification Error: 0.2000\n"
     ]
    }
   ],
   "source": [
    "# Select the model with the lowest error\n",
    "model_num = 2\n",
    "\n",
    "# Compute the test error\n",
    "yhat = models_bc[model_num-1].predict(x_bc_test_scaled)\n",
    "yhat = tf.math.sigmoid(yhat)\n",
    "yhat = np.where(yhat >= threshold, 1, 0)\n",
    "nn_test_error = np.mean(yhat != y_bc_test)\n",
    "\n",
    "print(f\"Selected Model: {model_num}\")\n",
    "print(f\"Training Set Classification Error: {nn_train_error[model_num-1]:.4f}\")\n",
    "print(f\"CV Set Classification Error: {nn_cv_error[model_num-1]:.4f}\")\n",
    "print(f\"Test Set Classification Error: {nn_test_error:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fba6f0d0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}