ppo-example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6369b095",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:34:13.496222Z",
     "iopub.status.busy": "2023-07-20T10:34:13.495427Z",
     "iopub.status.idle": "2023-07-20T10:37:04.855921Z",
     "shell.execute_reply": "2023-07-20T10:37:04.854677Z"
    },
    "papermill": {
     "duration": 171.369124,
     "end_time": "2023-07-20T10:37:04.858462",
     "exception": false,
     "start_time": "2023-07-20T10:34:13.489338",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gymnasium[accept-rom-license,atari] in /opt/conda/lib/python3.10/site-packages (0.26.3)\r\n",
      "Requirement already satisfied: numpy>=1.18.0 in /opt/conda/lib/python3.10/site-packages (from gymnasium[accept-rom-license,atari]) (1.23.5)\r\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from gymnasium[accept-rom-license,atari]) (2.2.1)\r\n",
      "Requirement already satisfied: gymnasium-notices>=0.0.1 in /opt/conda/lib/python3.10/site-packages (from gymnasium[accept-rom-license,atari]) (0.0.1)\r\n",
      "Collecting autorom[accept-rom-license]~=0.4.2 (from gymnasium[accept-rom-license,atari])\r\n",
      "  Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\r\n",
      "Collecting ale-py~=0.8.0 (from gymnasium[accept-rom-license,atari])\r\n",
      "  Downloading ale_py-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: importlib-resources in /opt/conda/lib/python3.10/site-packages (from ale-py~=0.8.0->gymnasium[accept-rom-license,atari]) (5.12.0)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from ale-py~=0.8.0->gymnasium[accept-rom-license,atari]) (4.5.0)\r\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.10/site-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (8.1.3)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (2.28.2)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (4.64.1)\r\n",
      "Collecting AutoROM.accept-rom-license (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari])\r\n",
      "  Downloading AutoROM.accept-rom-license-0.6.1.tar.gz (434 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m434.7/434.7 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (2.1.1)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (3.4)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari]) (2023.5.7)\r\n",
      "Building wheels for collected packages: AutoROM.accept-rom-license\r\n",
      "  Building wheel for AutoROM.accept-rom-license (pyproject.toml) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.6.1-py3-none-any.whl size=446660 sha256=a9ecfa43384fde45d66d3af569cb7f921160d45235e07b2f23ceec06e965d665\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/6b/1b/ef/a43ff1a2f1736d5711faa1ba4c1f61be1131b8899e6a057811\r\n",
      "Successfully built AutoROM.accept-rom-license\r\n",
      "Installing collected packages: ale-py, AutoROM.accept-rom-license, autorom\r\n",
      "Successfully installed AutoROM.accept-rom-license-0.6.1 ale-py-0.8.1 autorom-0.4.2\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0mCollecting atari-py\r\n",
      "  Downloading atari-py-0.2.9.tar.gz (540 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m540.6/540.6 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from atari-py) (1.23.5)\r\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from atari-py) (1.16.0)\r\n",
      "Building wheels for collected packages: atari-py\r\n",
      "  Building wheel for atari-py (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \bdone\r\n",
      "\u001b[?25h  Created wheel for atari-py: filename=atari_py-0.2.9-cp310-cp310-linux_x86_64.whl size=2868439 sha256=55e6d4893f1c8e9b3038a4038bc8c4cab1a0416710b5c50991560b63d6a60ae3\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/75/6f/04/1f3bf5255580101e16ff487564354dddcdd23ec3b43b775b7a\r\n",
      "Successfully built atari-py\r\n",
      "Installing collected packages: atari-py\r\n",
      "Successfully installed atari-py-0.2.9\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0mCollecting shimmy\r\n",
      "  Downloading Shimmy-1.1.0-py3-none-any.whl (37 kB)\r\n",
      "Requirement already satisfied: numpy>=1.18.0 in /opt/conda/lib/python3.10/site-packages (from shimmy) (1.23.5)\r\n",
      "Collecting gymnasium>=0.27.0 (from shimmy)\r\n",
      "  Downloading gymnasium-0.29.0-py3-none-any.whl (953 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m953.8/953.8 kB\u001b[0m \u001b[31m23.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: cloudpickle>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from gymnasium>=0.27.0->shimmy) (2.2.1)\r\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in /opt/conda/lib/python3.10/site-packages (from gymnasium>=0.27.0->shimmy) (4.5.0)\r\n",
      "Collecting farama-notifications>=0.0.1 (from gymnasium>=0.27.0->shimmy)\r\n",
      "  Using cached Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\r\n",
      "Installing collected packages: farama-notifications, gymnasium, shimmy\r\n",
      "  Attempting uninstall: gymnasium\r\n",
      "    Found existing installation: Gymnasium 0.26.3\r\n",
      "    Uninstalling Gymnasium-0.26.3:\r\n",
      "      Successfully uninstalled Gymnasium-0.26.3\r\n",
      "Successfully installed farama-notifications-0.0.4 gymnasium-0.29.0 shimmy-1.1.0\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0mCollecting moviepy\r\n",
      "  Downloading moviepy-1.0.3.tar.gz (388 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.3/388.3 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25hCollecting decorator<5.0,>=4.0.2 (from moviepy)\r\n",
      "  Downloading decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)\r\n",
      "Requirement already satisfied: tqdm<5.0,>=4.11.2 in /opt/conda/lib/python3.10/site-packages (from moviepy) (4.64.1)\r\n",
      "Requirement already satisfied: requests<3.0,>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from moviepy) (2.28.2)\r\n",
      "Collecting proglog<=1.0.0 (from moviepy)\r\n",
      "  Downloading proglog-0.1.10-py3-none-any.whl (6.1 kB)\r\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.10/site-packages (from moviepy) (1.23.5)\r\n",
      "Requirement already satisfied: imageio<3.0,>=2.5 in /opt/conda/lib/python3.10/site-packages (from moviepy) (2.28.1)\r\n",
      "Collecting imageio_ffmpeg>=0.2.0 (from moviepy)\r\n",
      "  Downloading imageio_ffmpeg-0.4.8-py3-none-manylinux2010_x86_64.whl (26.9 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.9/26.9 MB\u001b[0m \u001b[31m45.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: pillow>=8.3.2 in /opt/conda/lib/python3.10/site-packages (from imageio<3.0,>=2.5->moviepy) (9.5.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3.0,>=2.8.1->moviepy) (2.1.1)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.4)\r\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3.0,>=2.8.1->moviepy) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3.0,>=2.8.1->moviepy) (2023.5.7)\r\n",
      "Building wheels for collected packages: moviepy\r\n",
      "  Building wheel for moviepy (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \bdone\r\n",
      "\u001b[?25h  Created wheel for moviepy: filename=moviepy-1.0.3-py3-none-any.whl size=110744 sha256=74a33b55c307932d30f46673093bd0f6cacbd9e14415bc350af269b5b14df623\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/96/32/2d/e10123bd88fbfc02fed53cc18c80a171d3c87479ed845fa7c1\r\n",
      "Successfully built moviepy\r\n",
      "Installing collected packages: proglog, imageio_ffmpeg, decorator, moviepy\r\n",
      "  Attempting uninstall: decorator\r\n",
      "    Found existing installation: decorator 5.1.1\r\n",
      "    Uninstalling decorator-5.1.1:\r\n",
      "      Successfully uninstalled decorator-5.1.1\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "jupyterlab-lsp 4.1.0 requires jupyter-lsp>=2.0.0, but you have jupyter-lsp 1.5.1 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0mSuccessfully installed decorator-4.4.2 imageio_ffmpeg-0.4.8 moviepy-1.0.3 proglog-0.1.10\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install \"gymnasium[atari, accept-rom-license]\"\n",
    "!pip install atari-py -U\n",
    "!pip install shimmy\n",
    "!pip install moviepy\n",
    "#!apt-get install -y xvfb x11-utils\n",
    "#!pip install pyvirtualdisplay==0.2.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "18c98cd0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:04.903319Z",
     "iopub.status.busy": "2023-07-20T10:37:04.902992Z",
     "iopub.status.idle": "2023-07-20T10:37:09.184445Z",
     "shell.execute_reply": "2023-07-20T10:37:09.183424Z"
    },
    "papermill": {
     "duration": 4.306919,
     "end_time": "2023-07-20T10:37:09.187299",
     "exception": false,
     "start_time": "2023-07-20T10:37:04.880380",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "import ale_py\n",
    "import shimmy\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm.notebook import tqdm\n",
    "from torch.distributions import Categorical\n",
    "from gym.wrappers.monitoring.video_recorder import VideoRecorder\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "import ale_py\n",
    "import os\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "33aaa49a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.232597Z",
     "iopub.status.busy": "2023-07-20T10:37:09.230948Z",
     "iopub.status.idle": "2023-07-20T10:37:09.237381Z",
     "shell.execute_reply": "2023-07-20T10:37:09.236568Z"
    },
    "papermill": {
     "duration": 0.030368,
     "end_time": "2023-07-20T10:37:09.239264",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.208896",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def out_size(input_size, kernel_size = 3, padding = 1, stride = 2):\n",
    "    return (input_size - kernel_size + 2 * padding) // stride + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "03649b04",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.283714Z",
     "iopub.status.busy": "2023-07-20T10:37:09.283048Z",
     "iopub.status.idle": "2023-07-20T10:37:09.295924Z",
     "shell.execute_reply": "2023-07-20T10:37:09.295102Z"
    },
    "papermill": {
     "duration": 0.037192,
     "end_time": "2023-07-20T10:37:09.297876",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.260684",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Policy(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(Policy, self).__init__()\n",
    "        self.inp = nn.Linear(input_dim, 32)\n",
    "        self.out = nn.Linear(32, output_dim)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.softmax = nn.Softmax()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.inp(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.out(x)\n",
    "        x = self.softmax(x)\n",
    "        return x\n",
    "    \n",
    "class CNNPolicy(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(CNNPolicy, self).__init__()\n",
    "        self.inp = torch.nn.Conv2d(input_dim[2], 3, 3, stride=1, padding=1)\n",
    "        self.conv = nn.Sequential(\n",
    "            torch.nn.Conv2d(3, 3, 3, stride=1, padding=1),\n",
    "            torch.nn.Conv2d(3, 3, 3, stride=2, padding=1),\n",
    "            torch.nn.Conv2d(3, 1, 3, stride=2, padding=1),\n",
    "        )\n",
    "        self.flatten = nn.Flatten()\n",
    "        s = (out_size(input_dim[0]), out_size(input_dim[1]))\n",
    "        s2 = (out_size(s[0]), out_size(s[1]))\n",
    "        self.out = nn.Linear(s2[0] * s2[1], output_dim)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.softmax = nn.Softmax()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.transpose(1, -1)\n",
    "        x = self.inp(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.conv(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.flatten(x)\n",
    "        x = self.out(x)\n",
    "        x = self.softmax(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b041098",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.356336Z",
     "iopub.status.busy": "2023-07-20T10:37:09.355315Z",
     "iopub.status.idle": "2023-07-20T10:37:09.384434Z",
     "shell.execute_reply": "2023-07-20T10:37:09.383509Z"
    },
    "papermill": {
     "duration": 0.055668,
     "end_time": "2023-07-20T10:37:09.386532",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.330864",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNNPolicy(\n",
       "  (inp): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (conv): Sequential(\n",
       "    (0): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): Conv2d(3, 3, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "    (2): Conv2d(3, 1, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "  )\n",
       "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
       "  (out): Linear(in_features=2520, out_features=3, bias=True)\n",
       "  (relu): ReLU()\n",
       "  (softmax): Softmax(dim=None)\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CNNPolicy((250, 160, 3), 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4d82d2c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.431512Z",
     "iopub.status.busy": "2023-07-20T10:37:09.430122Z",
     "iopub.status.idle": "2023-07-20T10:37:09.441235Z",
     "shell.execute_reply": "2023-07-20T10:37:09.440423Z"
    },
    "papermill": {
     "duration": 0.035294,
     "end_time": "2023-07-20T10:37:09.443190",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.407896",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Value(nn.Module):\n",
    "    def __init__(self, input_dim):\n",
    "        super(Value, self).__init__()\n",
    "        self.inp = nn.Linear(input_dim, 32)\n",
    "        self.out = nn.Linear(32, 1)\n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.inp(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.out(x)\n",
    "        return x\n",
    "    \n",
    "class CNNValue(nn.Module):\n",
    "    def __init__(self, input_dim):\n",
    "        super(CNNValue, self).__init__()\n",
    "        self.inp = torch.nn.Conv2d(input_dim[2], 3, 3, stride=1, padding=1)\n",
    "        self.conv = torch.nn.Conv2d(3, 1, 3, stride=1, padding=1)\n",
    "        self.flatten = nn.Flatten()\n",
    "        self.out = nn.Linear(input_dim[0] * input_dim[1], 1)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.softmax = nn.Softmax()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.transpose(1, -1)\n",
    "        x = self.inp(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.conv(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.flatten(x)\n",
    "        x = self.out(x)\n",
    "        x = self.softmax(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e685e3d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.487973Z",
     "iopub.status.busy": "2023-07-20T10:37:09.487218Z",
     "iopub.status.idle": "2023-07-20T10:37:09.516074Z",
     "shell.execute_reply": "2023-07-20T10:37:09.515267Z"
    },
    "papermill": {
     "duration": 0.053899,
     "end_time": "2023-07-20T10:37:09.518324",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.464425",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class PPO:\n",
    "    def __init__(self, policy, value, env, num_epochs=20, max_steps=1000, epsilon=0.2, optimization_steps=4, gamma=0.99, lr=0.001, device = 'cpu'):\n",
    "        self.policy = policy\n",
    "        self.value = value\n",
    "        self.env = env\n",
    "        self.num_epochs = num_epochs\n",
    "        self.max_steps = max_steps\n",
    "        self.epsilon = epsilon\n",
    "        self.optimization_steps = optimization_steps\n",
    "        self.gamma = gamma\n",
    "        self.lr = lr\n",
    "        self.device = device\n",
    "        self.total_rewards = []\n",
    "        \n",
    "        \n",
    "    def calculate_returns(self, rewards, gamma):\n",
    "        returns = []\n",
    "        discounted_reward = 0\n",
    "        for r in reversed(rewards):\n",
    "            discounted_reward = r + gamma * discounted_reward\n",
    "            returns.append(discounted_reward)\n",
    "        return returns[::-1]\n",
    "    \n",
    "    def calculate_advantages(self, returns, values):\n",
    "        return torch.Tensor(returns) - torch.Tensor(values)\n",
    "        \n",
    "    def train(self, schedule = [[100, 0.001]], show_rewards = False):\n",
    "        self.total_rewards = []\n",
    "        policy_optimizer = optim.Adam(self.policy.parameters(), lr = self.lr)\n",
    "        value_optimizer = optim.Adam(self.value.parameters(), lr = self.lr)\n",
    "        \n",
    "        mse_loss = nn.MSELoss()\n",
    "        for num_epochs, lr in schedule:\n",
    "            self.lr = lr\n",
    "            self.num_epochs = num_epochs\n",
    "            for g in policy_optimizer.param_groups:\n",
    "                g['lr'] = self.lr\n",
    "            for g in value_optimizer.param_groups:\n",
    "                g['lr'] = self.lr\n",
    "            for epoch in tqdm(range(self.num_epochs), desc = f\"Training [lr = {self.lr}]\", unit = \" epochs\"):\n",
    "                states = []\n",
    "                actions = []\n",
    "                rewards = []\n",
    "                old_logprobs = []\n",
    "                values = []\n",
    "                advantages = []\n",
    "                returns = []\n",
    "                total_rewards = 0\n",
    "\n",
    "                state, _ = env.reset()\n",
    "\n",
    "                for step in range(self.max_steps):\n",
    "                    state = torch.FloatTensor(state).unsqueeze(0)\n",
    "                    action_probs = self.policy(state.to(self.device)).cpu()\n",
    "                    dist = Categorical(action_probs)\n",
    "                    action = dist.sample()\n",
    "                    logprob = dist.log_prob(action)\n",
    "                    value_estimate = self.value(state.to(self.device)).cpu()\n",
    "\n",
    "                    next_state, reward, terminated, truncated, info = env.step(action.item())\n",
    "\n",
    "                    states.append(torch.Tensor(state).squeeze(0))\n",
    "                    actions.append(action)\n",
    "                    rewards.append(reward)\n",
    "                    old_logprobs.append(logprob)\n",
    "                    values.append(value_estimate)\n",
    "\n",
    "                    total_rewards += reward\n",
    "                    state = next_state\n",
    "\n",
    "                    if terminated or truncated:\n",
    "                        break\n",
    "\n",
    "                returns = self.calculate_returns(rewards, self.gamma)\n",
    "                advantages = self.calculate_advantages(returns, values)\n",
    "\n",
    "                states = torch.stack(states).detach()\n",
    "                actions = torch.Tensor(actions).detach()\n",
    "                old_logprobs = torch.Tensor(old_logprobs).detach()\n",
    "                returns = torch.Tensor(returns).detach()\n",
    "                advantages = torch.Tensor(advantages).detach()\n",
    "\n",
    "                for _ in range(self.optimization_steps):\n",
    "                    action_probs = self.policy(states.to(self.device)).cpu()\n",
    "                    dist = Categorical(action_probs)\n",
    "                    logprob = dist.log_prob(actions)\n",
    "\n",
    "                    ratio = torch.exp(logprob - old_logprobs)\n",
    "\n",
    "                    surr1 = ratio * advantages\n",
    "                    surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * advantages\n",
    "\n",
    "                    policy_loss = -torch.min(surr1, surr2)\n",
    "\n",
    "                    policy_optimizer.zero_grad()\n",
    "                    policy_loss.mean().backward()\n",
    "                    policy_optimizer.step()\n",
    "\n",
    "                    value_estimate = self.value(states.to(self.device)).cpu()\n",
    "                    value_loss = mse_loss(value_estimate, returns)\n",
    "\n",
    "                    value_optimizer.zero_grad()\n",
    "                    value_loss.mean().backward()\n",
    "                    value_optimizer.step()\n",
    "\n",
    "                self.total_rewards.append(total_rewards)\n",
    "                if(show_rewards):\n",
    "                    print('Epoch [{}/{}], Total Reward: {:.2f}'.format(epoch + 1, self.num_epochs, total_rewards))\n",
    "    def plot(self):\n",
    "        plt.plot(pd.DataFrame({'rewards': self.total_rewards}).rolling(10).mean(), alpha = 0.8, linewidth = 4)\n",
    "        plt.title(\"Reward over training\")\n",
    "        plt.xlabel(\"Epoch\")\n",
    "        plt.ylabel(\"Reward\")\n",
    "        plt.grid()\n",
    "        plt.show()\n",
    "    def video(self, env):\n",
    "        recorder = VideoRecorder(env, 'video.mp4')\n",
    "        state, _ = env.reset()\n",
    "        for step in range(self.max_steps):\n",
    "            state = torch.FloatTensor(state).unsqueeze(0)\n",
    "            action_probs = self.policy(state.to(self.device)).cpu()\n",
    "            dist = Categorical(action_probs)\n",
    "            action = dist.sample()\n",
    "            logprob = dist.log_prob(action)\n",
    "            value_estimate = self.value(state.to(self.device)).cpu()\n",
    "\n",
    "            state, reward, terminated, truncated, info = env.step(action.item())\n",
    "            recorder.capture_frame()\n",
    "            if terminated or truncated:\n",
    "                break\n",
    "        recorder.close()\n",
    "        \n",
    "    def save(self, path = \"./\"):\n",
    "        torch.save(self.policy.state_dict(), os.path.join(path, \"policy.pt\"))\n",
    "        torch.save(self.value.state_dict(), os.path.join(path, \"value.pt\"))\n",
    "    def load(self, path = \"./\"):\n",
    "        self.policy.load_state_dict(torch.load(os.path.join(path, \"policy.pt\")))\n",
    "        self.value.load_state_dict(torch.load(os.path.join(path, \"value.pt\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a717921",
   "metadata": {
    "papermill": {
     "duration": 0.036605,
     "end_time": "2023-07-20T10:37:09.591714",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.555109",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b2a36f09",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:09.649730Z",
     "iopub.status.busy": "2023-07-20T10:37:09.649375Z",
     "iopub.status.idle": "2023-07-20T10:37:15.776870Z",
     "shell.execute_reply": "2023-07-20T10:37:15.775982Z"
    },
    "papermill": {
     "duration": 6.152476,
     "end_time": "2023-07-20T10:37:15.778983",
     "exception": false,
     "start_time": "2023-07-20T10:37:09.626507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53ca1e2a856d46098886fcabf976d940",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training [lr = 0.01]:   0%|          | 0/20 [00:00<?, ? epochs/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56d6c82d0e684fe7891122d3d25e6655",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training [lr = 0.001]:   0%|          | 0/100 [00:00<?, ? epochs/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABuz0lEQVR4nO3deXhTVf4/8PdNmqZN94U23SgtlH0VFASUIrQiIs6gouKCy7gMLuAyKKOO1e8IIzrIb2CU0VF0VNxmQBwXpCCLisi+l70tpSstbdM1SZPz+6PTQMhNm7Zps71fz8Pz0HPvTT45FPrhnM85RxJCCBARERF5KYWrAyAiIiLqSkx2iIiIyKsx2SEiIiKvxmSHiIiIvBqTHSIiIvJqTHaIiIjIqzHZISIiIq/GZIeIiIi8GpMdIiIi8mpMdojcyPvvvw9Jkiy//Pz8EBcXh9tuuw0nTpxwdXhO06tXL9xzzz2uDsNpVq1ahaVLl3bZ63emv+655x706tXLqfEQeRo/VwdARLZWrlyJ/v37o7GxET///DNeeeUVbNq0CUePHkVERISrw6NLrFq1CocOHcK8efO65PXXrFmD0NDQDj37wgsvYO7cuU6OiMizMNkhckODBw/GqFGjAADp6ekwmUx48cUX8eWXX+Lee+91cXRtq6+vh0ajcXUYTtXQ0IDAwMBOv47JZEJTUxPUarXDz4wYMaLD79e7d+8OP0vkLTiNReQBWhKf0tJSq/Zdu3Zh+vTpiIyMREBAAEaMGIHPP//ccl2n08HPzw+vvfaapa28vBwKhQJhYWFoamqytD/++OPo0aMHWs4Gzs7Oxo033ojExEQEBASgT58+eOihh1BeXm4VQ1ZWFiRJwp49e3DzzTcjIiLC8gPWaDRi/vz50Gq10Gg0GD9+PHbs2OHw5z5//jzmzJmDhIQE+Pv7IzU1Fc899xz0er3lnhEjRuCqq66yedZkMiEhIQEzZsywtBkMBvz5z39G//79oVar0aNHD9x77704d+6c1bO9evXCtGnTsHr1aowYMQIBAQF46aWXZGNMT0/HN998g/z8fKspSADIy8uDJElYvHgx/vznPyMlJQVqtRqbNm1CY2MjnnrqKQwfPhxhYWGIjIzElVdeibVr19q8x6XTWJs3b4YkSfjkk0/w3HPPIT4+HqGhoZg8eTKOHTtm9azcNJYkSXj00Ufx4YcfYsCAAdBoNBg2bBi+/vprm/deu3Ythg4dCrVajdTUVPy///f/LH/mRJ6CIztEHiA3NxcA0LdvX0vbpk2bMGXKFIwePRorVqxAWFgYPv30U9x6662or6/HPffcg9DQUFx++eXYsGED/vCHPwAANm7cCLVajZqaGuzYsQNjx44FAGzYsAHXXHON5YfYqVOncOWVV+J3v/sdwsLCkJeXhyVLlmD8+PE4ePAgVCqVVYwzZszAbbfdhocffhh1dXUAgAceeAD/+te/8PTTTyMjIwOHDh3CjBkzUFNT0+ZnbmxsxMSJE3Hq1Cm89NJLGDp0KH788UcsWrQI+/btwzfffAMAuPfeezF37lycOHECaWlplufXr1+PoqIiy0iY2WzGjTfeiB9//BHz58/H2LFjkZ+fjxdffBHp6enYtWuX1cjNnj17kJOTg+effx4pKSkICgqSjfPNN9/Egw8+iFOnTmHNmjWy9/ztb39D37598frrryM0NBRpaWnQ6/U4f/48nn76aSQkJMBgMGDDhg2YMWMGVq5cibvvvrvNPvrjH/+IcePG4Z///Cd0Oh2eeeYZ3HDDDcjJyYFSqWz12W+++QY7d+7Eyy+/jODgYCxevBi//e1vcezYMaSmpgIA1q1bhxkzZuDqq6/GZ599hqamJrz++us2STeR2xNE5DZWrlwpAIjt27cLo9EoampqxLp164RWqxVXX321MBqNlnv79+8vRowYYdUmhBDTpk0TcXFxwmQyCSGEeP7550VgYKBobGwUQgjxu9/9TkyZMkUMHTpUvPTSS0IIIQoLCwUA8fbbb8vGZTabhdFoFPn5+QKAWLt2reXaiy++KACIP/3pT1bP5OTkCADiiSeesGr/+OOPBQAxe/bsVvtixYoVAoD4/PPPrdpfffVVAUCsX79eCCFEeXm58Pf3F3/84x+t7ps5c6aIjY219M8nn3wiAIj//Oc/Vvft3LlTABBvvvmmpS05OVkolUpx7NixVmNscf3114vk5GSb9tzcXAFA9O7dWxgMhlZfo6mpSRiNRnH//feLESNGWF1LTk626q9NmzYJAGLq1KlW933++ecCgPjll18sbbNnz7aJDYCIjY0VOp3O0lZSUiIUCoVYtGiRpe3yyy8XSUlJQq/XW9pqampEVFSU4I8P8iScxiJyQ2PGjIFKpUJISAimTJmCiIgIrF27Fn5+zYOxJ0+exNGjR3HHHXcAAJqamiy/pk6diuLiYst0xqRJk9DQ0IBt27YBaB7BycjIwOTJk5GdnW1pA4DJkydbYigrK8PDDz+MpKQk+Pn5QaVSITk5GQCQk5NjE/NNN91k9fWmTZsAwBJji5kzZ1o+R2t++OEHBAUF4eabb7Zqb5nO2bhxIwAgKioKN9xwAz744AOYzWYAQGVlJdauXYu7777b8l5ff/01wsPDccMNN1j11/Dhw6HVarF582ar9xk6dKjVSFpnTJ8+3WYkDAC++OILjBs3DsHBwZY+fvfdd2X7197rXhozAOTn57f57MSJExESEmL5OjY2FjExMZZn6+rqsGvXLvzmN7+Bv7+/5b7g4GDccMMNDsVH5C6Y7BC5oX/961/YuXMnfvjhBzz00EPIycnB7bffbrneMo3w9NNPQ6VSWf2aM2cOAFhqa8aOHQuNRoMNGzbg5MmTyMvLsyQ7v/76K2pra7FhwwakpqYiJSUFQPOUT2ZmJlavXo358+dj48aN2LFjB7Zv3w6guVj3UnFxcVZfV1RUAAC0Wq1Vu5+fH6Kiotrsg4qKCmi1WpvakJiYGPj5+VleHwDuu+8+FBYWWpK3Tz75BHq93qrOpbS0FFVVVfD397fps5KSEptapEs/T2fIvdbq1asxc+ZMJCQk4KOPPsIvv/yCnTt34r777kNjY6NDr3tpP7YUPcv9+bT1bMvzLc9WVlZCCIHY2Fib++TaiNwZa3aI3NCAAQMsRckTJ06EyWTCP//5T/z73//GzTffjOjoaADAggULrApwL9avXz8AgL+/P8aPH48NGzYgMTERWq0WQ4YMsdRlbN68GRs3bsS0adMszx46dAj79+/H+++/j9mzZ1vaT548aTfmS5OSlh+mJSUlSEhIsLQ3NTVZJSr2REVF4ddff4UQwuq1y8rK0NTUZOkDALj22msRHx+PlStX4tprr8XKlSsxevRoDBw40HJPdHQ0oqKisG7dOtn3u3iUQ+7zdIbca3300UdISUnBZ599ZnX94uJrV4qIiIAkSbL1OSUlJS6IiKjjOLJD5AEWL16MiIgI/OlPf4LZbEa/fv2QlpaG/fv3Y9SoUbK/Lv7hPXnyZOzevRv/+c9/LFNVQUFBGDNmDJYtW4aioiKrKayWH76XLo/+xz/+4XDM6enpAICPP/7Yqv3zzz+3WgVmz6RJk1BbW4svv/zSqv1f//qX5XoLpVKJu+66C19++SV+/PFH7Nq1C/fdd5/Vc9OmTUNFRQVMJpNsf7Ukhx1x8YiIoyRJgr+/v1WiU1JSIrsayxWCgoIwatQofPnllzAYDJb22tpa2VVbRO6MIztEHiAiIgILFizA/PnzsWrVKtx55534xz/+geuuuw7XXnst7rnnHiQkJOD8+fPIycnBnj178MUXX1ienzRpEkwmEzZu3IgPPvjA0j558mS8+OKLkCQJ11xzjaW9f//+6N27N5599lkIIRAZGYn//ve/lmkiRwwYMAB33nknli5dCpVKhcmTJ+PQoUOWFUltufvuu/H3v/8ds2fPRl5eHoYMGYKffvoJCxcuxNSpU62SM6B5KuvVV1/FrFmzEBgYiFtvvdXq+m233YaPP/4YU6dOxdy5c3HFFVdApVLh7Nmz2LRpE2688Ub89re/dfjzXWzIkCFYvXo13nrrLYwcORIKhcIyMmdPy9L2OXPm4Oabb0ZBQQH+7//+D3FxcW6zW/bLL7+M66+/Htdeey3mzp0Lk8mE1157DcHBwTh//ryrwyNynIsLpInoIi2rsXbu3GlzraGhQfTs2VOkpaWJpqYmIYQQ+/fvFzNnzhQxMTFCpVIJrVYrrrnmGrFixQqrZ81ms4iOjhYARGFhoaX9559/FgDEZZddZvN+R44cERkZGSIkJERERESIW265RZw5c0YAEC+++KLlvpbVWOfOnbN5Db1eL5566ikRExMjAgICxJgxY8Qvv/xis7rInoqKCvHwww+LuLg44efnJ5KTk8WCBQssK8suNXbsWAFA3HHHHbLXjUajeP3118WwYcNEQECACA4OFv379xcPPfSQOHHihOW+5ORkcf3117cZX4vz58+Lm2++WYSHhwtJkiwrlVpWY7322muyz/3lL38RvXr1Emq1WgwYMEC88847lv68mL3VWF988YXVfS3vt3LlSkubvdVYjzzyiE08cn8ua9asEUOGDBH+/v6iZ8+e4i9/+Yt4/PHHRURERBu9QuQ+JCH+t4MYERFRG4xGI4YPH46EhASsX7/e1eEQOYTTWEREZNf999+PjIwMxMXFoaSkBCtWrEBOTg7+3//7f64OjchhTHaIiMiumpoaPP300zh37hxUKhUuu+wyfPvttzY1U0TujNNYRERE5NW49JyIiIi8GpMdIiIi8mpMdoiIiMirsUAZzecAFRUVISQkxKlbxBMREVHXEUKgpqYG8fHxUCjsj98w2QFQVFSEpKQkV4dBREREHVBQUIDExES715ns4MIBgAUFBQ5tY+/tjEYj1q9fj8zMTKhUKleH47HYj87Bfuw89qFzsB+dw5n9qNPpkJSUZHOQ76WY7ODCoYehoaFMdtD8jajRaBAaGsq/0J3AfnQO9mPnsQ+dg/3oHF3Rj22VoLi0QHnr1q244YYbEB8fD0mSrE43NhqNeOaZZzBkyBAEBQUhPj4ed999N4qKiqxeQ6/X47HHHkN0dDSCgoIwffp0nD17tps/CREREbkrlyY7dXV1GDZsGJYvX25zrb6+Hnv27MELL7yAPXv2YPXq1Th+/DimT59udd+8efOwZs0afPrpp/jpp59QW1uLadOmwWQyddfHICIiIjfm0mms6667Dtddd53stbCwMGRnZ1u1LVu2DFdccQXOnDmDnj17orq6Gu+++y4+/PBDy9blH330EZKSkrBhwwZce+21Xf4ZiIiIyL15VM1OdXU1JElCeHg4AGD37t0wGo3IzMy03BMfH4/Bgwdj27ZtdpMdvV4PvV5v+Vqn0wFonjozGo1d9wE8REsfsC86h/3oHOzHzmMfOgf70Tmc2Y+OvobHJDuNjY149tlnMWvWLEsRcUlJCfz9/REREWF1b2xsLEpKSuy+1qJFi/DSSy/ZtK9fvx4ajca5gXuwS0fWqGPYj87Bfuw89qFzsB+dwxn9WF9f79B9HpHsGI1G3HbbbTCbzXjzzTfbvF8I0Wpl9oIFC/Dkk09avm5ZupaZmcnVWGju7+zsbGRkZHDFQSewH52D/dh57EPnYD86hzP7sWVmpi1un+wYjUbMnDkTubm5+OGHH6ySEa1WC4PBgMrKSqvRnbKyMowdO9bua6rVaqjVapt2lUrFb+CLsD+cg/3oHOzHzmMfOgf70Tmc0Y+OPu/WZ2O1JDonTpzAhg0bEBUVZXV95MiRUKlUVkNhxcXFOHToUKvJDhEREfkOl47s1NbW4uTJk5avc3NzsW/fPkRGRiI+Ph4333wz9uzZg6+//homk8lShxMZGQl/f3+EhYXh/vvvx1NPPYWoqChERkbi6aefxpAhQyyrs4iIiMi3uTTZ2bVrFyZOnGj5uqWOZvbs2cjKysJXX30FABg+fLjVc5s2bUJ6ejoA4I033oCfnx9mzpyJhoYGTJo0Ce+//z6USmW3fAYiIiJyby5NdtLT0yGEsHu9tWstAgICsGzZMixbtsyZoREREVEnmMwClfUGRGr8oVC0fpxDV3P7AmUiIiLyPCW6Rjz84W4oFRKig9WIDVUjJiQA1/SLavthJ2OyQ0RERE5XpmsE0DzCU6prRKmuEUA1hie2fkJ5V3Dr1VhERETkmcpq9LLtPUJst37pakx2iIiIyOlaRnYuFcNkh4iIiLzBOZmRHX8/BUIDur+ChskOEREROV2pzjbZiQ1Vt3qcU1dhskNEREROV1ZjO40VExLggkiY7BAREZGTNZnMOF9nsGl3RXEywGSHiIiInKyizgCzzL7ArihOBpjsEBERkZOV2lmJFRvKaSwiIiLyAmUyxckAp7GIiIjIS9jbUJAjO0REROQV5FZi+SklhAeqXBANkx0iIiJyMrk9dmJC1C47/ZzJDhERETnVOZmRHVfV6wBMdoiIiMiJzGaBc7W2e+zEumhDQYDJDhERETlRRZ0BZplNdmJCObJDREREXsDeHjuuOioCYLJDRERETiR32jnAmh0iIiLyEnLLzgHX7bEDMNkhIiIiJ5LbPVmhkBAV5O+CaP73/i57ZyIiIvI6pXLLzoP9XbbHDsBkh4iIiJxIrmanhwuLkwEmO0REROQkZrOQPRcr1oXLzgEmO0REROQkVQ1GNJlk9tjhyA4RERF5A/t77HBkh4iIiLyAvT12XLl7MsBkh4iIiJzE3siOK/fYAZjsEBERkZPIFScrJLh0jx2AyQ4RERE5idw0VmSQP/yUrk03mOwQERGRU8hNY7l6JRbAZIeIiIicQAj33GMHYLJDRERETlDdYIShyWzT7srTzlsw2SEiIqJOs7/snNNYRERE5AVKZU47B1y/oSDAZIeIiIicwO7uyRzZISIiIm9w6lytTZtCAnoEc2SHiIiIvEBOsc6mLTkqCP5+rk81XB8BERERebRzNXqU1xps2vvHhbggGltMdoiIiKhT5EZ1AGCANrSbI5HHZIeIiIg65WiJfLIzMJ7JDhEREXmBnOIam7Zwjcotlp0DTHaIiIioExqNJpyWWYk1MC4UkiS5ICJbTHaIiIiow06U1sIsbNsHxLnHFBbAZIeIiIg6wV5xsrusxAKY7BAREVEnHJFJdlRKCanRwS6IRh6THSIiIuoQs1ngWIltcXJaTIhbbCbYwn0iISIiIo9ytrIBtfomm3Z3msICmOwQERFRB+XY2V/HnYqTASY7RERE1EHuvnNyCyY7RERE1CFyyU58eADCNCoXRGMfkx0iIiJqt+oGI4qqGm3a+7vZqA7AZIeIiIg64Ki9KSw3q9cBmOwQERFRB9ir1xnIZIeIiIi8wUmZ87A0/kokRgS6IJrWMdkhIiKidhFC4PS5Opv2PjHBUCjc4/DPi7k02dm6dStuuOEGxMfHQ5IkfPnll1bXhRDIyspCfHw8AgMDkZ6ejsOHD1vdo9fr8dhjjyE6OhpBQUGYPn06zp49242fgoiIyLeU1xpQ02i7mWBqD/c5IuJiLk126urqMGzYMCxfvlz2+uLFi7FkyRIsX74cO3fuhFarRUZGBmpqLmxNPW/ePKxZswaffvopfvrpJ9TW1mLatGkwmUzd9TGIiIh8ymmZKSwASO0R1M2ROMbPlW9+3XXX4brrrpO9JoTA0qVL8dxzz2HGjBkAgA8++ACxsbFYtWoVHnroIVRXV+Pdd9/Fhx9+iMmTJwMAPvroIyQlJWHDhg249tpru+2zEBER+YpTMlNYANDbjQ7/vJhLk53W5ObmoqSkBJmZmZY2tVqNCRMmYNu2bXjooYewe/duGI1Gq3vi4+MxePBgbNu2zW6yo9frodfrLV/rdM0V5UajEUajsYs+kedo6QP2ReewH52D/dh57EPnYD9ecKJUByGEVZu/nwKxwX5t9o8z+9HR13DbZKekpAQAEBsba9UeGxuL/Px8yz3+/v6IiIiwuafleTmLFi3CSy+9ZNO+fv16aDSazobuNbKzs10dgldgPzoH+7Hz2IfOwX4Efs1RoMZoXYgcpxFYt+47h1/DGf1YX1/v0H1um+y0kCTrzhRC2LRdqq17FixYgCeffNLytU6nQ1JSEjIzMxEa6n77A3Q3o9GI7OxsZGRkQKVyry2/PQn70TnYj53HPnQO9mMzXYMR7+TvRtgl7VcPisXUq1PafN6Z/dgyM9MWt012tFotgObRm7i4OEt7WVmZZbRHq9XCYDCgsrLSanSnrKwMY8eOtfvaarUaarXapl2lUvn0N/Cl2B/OwX50DvZj57EPncPX+/FMca3sgEKf2NB29Ysz+tHR5912n52UlBRotVqrYS6DwYAtW7ZYEpmRI0dCpVJZ3VNcXIxDhw61muwQERFRx8jtrwMAfWLccyUW4OKRndraWpw8edLydW5uLvbt24fIyEj07NkT8+bNw8KFC5GWloa0tDQsXLgQGo0Gs2bNAgCEhYXh/vvvx1NPPYWoqChERkbi6aefxpAhQyyrs4iIiMh5TsksO1coJPSMZLIja9euXZg4caLl65Y6mtmzZ+P999/H/Pnz0dDQgDlz5qCyshKjR4/G+vXrERISYnnmjTfegJ+fH2bOnImGhgZMmjQJ77//PpRKZbd/HiIiIk+WX1GHnXmVCFQpcVXfaIQG2E4TyY3s9IzUwN/PbSeLXJvspKen2yxdu5gkScjKykJWVpbdewICArBs2TIsW7asCyIkIiLyDQfOViHrq8Mwmpp/Lq/ZW4g3bh2GkIsSngaDCUXVDTbPpka776gO4MY1O0RERNQ99E0mLMk+bkl0AKBU14hVv56xui+3vA5yYxS9Y9xzM8EWTHaIiIh83Nf7i1FRa7Bp/+FoGRqNF45fOl1u55gIjuwQERGRu6rVN+Hfu+UP0K43mLDtVLnl61Nl8iux3PVMrBZMdoiIiHzYf3afRa3e9gTzFusOXTiRQG4lVlxYADT+brttHwAmO0RERD6rolaPr/YXtXpPTnENzlTUw2gy48x52+MZUnu4d70OwGSHiIjIZ326swCGJnOb931/uARnztfDZLatTu7t5lNYgBsfF0FERERd52xlPdYftn9o9sV+OFqGuPAA2Wsc2SEiIiK39PGvZyAzUAOFwvbcq1p9E975MVf2dTxhZIfJDhERkY8pr9Vj28lym3alQsK8yWmyz5hlMqPIIH+Ea/ydHp+zMdkhIiLyMRuOlMqO6kwdokV63x7oGalx6HVGJkc4ObKuwWSHiIjIh5jNAuuPlNq0KyRgxmWJkCQJmYNi23ydxIhA3H1lcleE6HRMdoiIiHzI3oJKnKvR27SPTI5EdLAaAHBN/xiolLa1Oy0m9uuBJTOHe8QUFsBkh4iIyKd8f9h2VAcArr1oNCckQIX0fjE29wSoFHgiIw1PZvZDoL+yy2J0Ni49JyIi8hHn6wz4Nfe8TXtUsD9G9Yq0art3XC/kldfhRFnzrsn9tCF4IqMvEsIDuyVWZ2KyQ0RE5CM25JTKrqqaPCAWykuWnIcEqPD6LcOQW1EHhSShV5QGkmR/asudMdkhIiLyAWazkN1EUJKAzIHyBckKhYTeHrBpYFtYs0NEROQDDhRWo1RnW5h8Wc8IxITK747sLZjsEBER+YCLTy+/mCPLzD0dkx0iIiIvZ2gy49fcCpv2cI0KV1xSmOyNmOwQERF5uVJdI5pM8oXJfkrvTwW8/xMSERH5uOoGo2x7qgcc4ukMTHaIiIi8XFW9fLIT4SE7IHcWkx0iIiIvV9VgkG0PC1R1cySuwWSHiIjIy9mbxgrTMNkhIiIiLyA3jaWQgGB/39hbmMkOERGRl9PJjOyEBqqgUHjm8Q/txWSHiIjIy8mN7PhKcTLAZIeIiMjryRUo+0pxMsBkh4iIyOvJjeyE+0hxMsBkh4iIyKsZmsyoN5hs2jmyQ0RERF5B12hn2TmTHSIiIvIG9nZPDmeBMhEREXmDaju7J/tSzY5v7CZEREROI4RAXkU9AKBXlAaS5Bt7tXgquyM7PjSNxWSHiIgcVqtvwv/99wiOFOsAAIMTQvHiDYMQoFK6ODKyx16yw5odIiIiGZ/uOGNJdADgUKEOH23Pd2FE1BZ752KFMtkhIiKyZjYLbD52zqb9+8MlqNM3uSAickSVTLITqFL61Ggckx0iInJITolOdpSg0WjGhpxSF0REjpA7F8tXTjtvwWSHiIgc8uvp83avfXuwGGaz6MZoyFGV9barsXypOBlgskNERA76NbfC7rWiqkbsLajsxmjIUXKjcb5UnAww2SEiIgcUnK9HUVVjq/f8d39xN0VDjhJC+Py5WACXnhMRkQN+zbU/hdViz5lKFFU1ID48sBsi8nxFVQ344WgZAGDKYC2ig9VOf486gwkmmelFjuwQEZFPazKZcbREh6KqBkvbr6ftT2G1EKK5dofadqK0Bo9/shef7SzAZzsL8PCHu1Fc3dD2g+1kb9m5Lx0VATDZISKiixRWNeDBD3fjD18cwEMf7sb/23AC52r0OFZa49Dz2UdK0Wi0PWGbrP1791nom8yWr/VNZnz4i/P3K6qskz8qwtdGdjiNRUREAJrrO/76/TGcq9Fb2jbklGL/2SoImYVWGn8l6g3WiU29wYTNx8owZXCcpW1/QRU+2p6PynoDhieF4/7xqQj09509XuQcLtLZtO3Kq4TRZIZK6bxxCLll54Dv1exwZIeIiAA0/wA+UVZr035x8nOxxyelybZ/faAY4n/ZUWFVA175JgdHS2pQqtPj+8OleHPzSecF7YGq6g2y00sNRhOOyCRBnXove8lOIKexiIjIB/33QJHD96ZEB2Fcn2ikxQbbXMuvqEdOcfO01793nUXDJdNaP54oR4PBd6e6Wg5RlbMzr+1C8PbguVjNmOwQERHKa/XYfqrtIuQWo1MjAQDThsbJXv/2YDGq6g3YfLzM5prJLFDUBcW4niK/os7utV15zt2rSG4ESSEBIQG+VcXCZIeIiPDdwWK0ZwPk0SlRAIDxfXogWG37g/PnU+X4dGcBmkzyL1pY6cvJjv2RncKqhnatyqqsM2B/QRVyiptXz9XqmyxTiEDzlNmlQgNVUCik9gXt4XwrtSMiIhuGJjO+P+z42VZRwf7o3SMIAODvp8DkgbH4cm+h1T1NJoFvDthfhn7xsnZf01qyAwA78yoxfVjbexX9Z/dZvL8tz6ZdpZQwNDEcv0/vzd2T/4cjO0REPu7nk+WyPxR7RmogNwAwvk80JOnChSmDte1+T19NdsxmgTPn7U9jAcAuB+p2fjxxTjbRAQCjSWB3fiVeXHsYFTJLz31tJRbAZIeIyOf9d798YfLDE3rjqcx+UCkvJDbhGhVuGZVkdV9CeCCGJ4W36z3P+miyc65Wj0ajudV7DhZWt1rAfb7OgDc3nWrzvQqrGlBSbXvEhy+O7HAai4jIhx0vrZFdbt4zSoPBCaGQJAl9YoLx04lyqPwkZAzUytboXDdEi30FVQ6/b1FVA4QQViNEviCvvPVRHaB5CnD/2SqMSY2yuSaEwLIfTqBW39ThGHxt2TnAZIeIyKd9bWdU54ahcZZEJD48EDMvT5K9r8XolChEBfujolZ+x95L1elN0DU0IczHplTaqtdpsSvvvGyys/5IaadXbPlanwOcxiIi8lmGJjN+Ollu0x6kViK9X0y7XkupkJA5sH21O4U+OJWV30a9ToudeZVWq6oAoFTXiHd/zJW9f2RyBPprQxx67XAfnMZy62SnqakJzz//PFJSUhAYGIjU1FS8/PLLMJsvzHcKIZCVlYX4+HgEBgYiPT0dhw8fdmHURESeoaS6EUaZpeGTB8QiQNX+4xwyB8XKFjTb44vJTmsbCl7sfJ0Bpy+a8jKbBZZuOG6zQSPQXEf1REZfZAyMdei1fbFmx62TnVdffRUrVqzA8uXLkZOTg8WLF+O1117DsmXLLPcsXrwYS5YswfLly7Fz505otVpkZGSgpsaxQ+uIiHyVvY39hiSEdej1ooPVslMv9hIgX1uRZTSZcVZmf6HQQPmKkotXZf10shyHCuWPknjsmjSEBapwWXKEQ3H42onngJsnO7/88gtuvPFGXH/99ejVqxduvvlmZGZmYteuXQCaR3WWLl2K5557DjNmzMDgwYPxwQcfoL6+HqtWrXJx9ERE7s3e5nXx4W3v8WLP7Vf0hNrP+kfLzMuTZA/+9LVkp7iqEWaZnRunDNLCT2mbEW45fg5ms4AQAl/uK7S5DjSPwl2R0rybdXSwGj0jNW3G4YtLz926QHn8+PFYsWIFjh8/jr59+2L//v346aefsHTpUgBAbm4uSkpKkJmZaXlGrVZjwoQJ2LZtGx566CHZ19Xr9dDrLxxsp9M1Z8tGoxFGo/w5Ir6kpQ/YF53DfnQO9mPn2evDsxV1NnUhkgREBio73N8JYf54dkpfrN1fjMo6A8b2jsRNw+Ow83QFTp6zrlcpOF/nUX+unf1ePFlWbdPfANA7WoNBcSHYV1Bt1X6moh7bTpYhLFCF4yW2sxXRwf6458pEq3iGJYa2ehwFAGj8XPv3yZl/px19DUnI9bybEELgj3/8I1599VUolUqYTCa88sorWLBgAQBg27ZtGDduHAoLCxEfH2957sEHH0R+fj6+//572dfNysrCSy+9ZNO+atUqaDRtZ8VERN7g89MKnKm1HlEIUQk8NKD1fWA64pszCuRUWb+XUgLmDja1q87Hk/1YIuHXMtsJlQf6m1BYJ+HbAttr2kCBUH/geLVtJ01OMGN4lPWP8Lwa4N+59uutVIrmPvcW9fX1mDVrFqqrqxEaGmr3Prce2fnss8/w0UcfYdWqVRg0aBD27duHefPmIT4+HrNnz7bcd+k+DW3t3bBgwQI8+eSTlq91Oh2SkpKQmZnZamf5CqPRiOzsbGRkZECl8r3hTmdhPzoH+7Hz7PXh2o/2Ikypt7p3SEIopk4d6PQYdDvPomjXWZv2K64egZgQtdPfryt09ntx33fHEKa3XjYeoFLg1umXwywETnyyH2U11n8eDf/7FXZJGVWwWoknb7vMppDc0GTGlpW7YGiST1hjQtSYOnVEu2N3Jmf+nW6ZmWmLWyc7f/jDH/Dss8/itttuAwAMGTIE+fn5WLRoEWbPng2ttnmZY0lJCeLiLpy8W1ZWhthY+1XparUaarXtXy6VSsV/TC/C/nAO9qNzsB877+I+NJrMqKgz2PzHMCFC0yX93DM6WPY/oWW1RiREBjv9/bpSR78XCyobbfqgV3Qw1OrmguGbRiZhxZa2d0YGgCmD4xCiCZCJDRiaGI7d+fJ78UQEqd3m75Ez/k47+rxbFyjX19dDobAOUalUWpaep6SkQKvVIjs723LdYDBgy5YtGDt2bLfGSkTkScpq9LKnnMeFdbw4uTUJ4bY/mAHfWX7eaDShVGd7dEPyRQXFkwfGOFQ8rFBImDYs3u711lZl+WJxMuDmyc4NN9yAV155Bd988w3y8vKwZs0aLFmyBL/97W8BNE9fzZs3DwsXLsSaNWtw6NAh3HPPPdBoNJg1a5aLoycicl/FdpKMuDD5pKSz7K3w8pUVWWfOy++vkxwdZPm92k+J6a0kMS3G94lCdLD9qb+RrSQ7vrjHDuDm01jLli3DCy+8gDlz5qCsrAzx8fF46KGH8Kc//clyz/z589HQ0IA5c+agsrISo0ePxvr16xES4thOkkREvqhI5oBIAIjrxLLz1mj8/RCuUaGq3nr1TFGVfBzext4xEcmXLBWfOiQOX+w+2+pBoDcOT2j1veLDAhCs9pM9P8tXR3bcOtkJCQnB0qVLLUvN5UiShKysLGRlZXVbXEREnq7Ezh47XTWyAzSfjn5psuMr01j2loMnR1knO0FqP0wbGocvZIq5AaC/NgR9Y1v/z7wkSegdE4T9lyxlBwCT3NylD3DraSwiIuoaciMq4RpVh46JcJTcVFaZrhFGk/OXurubU+dsT5YP16hkdzOePiweKplNBoG2R3VaTB4gv0inT4xnFYM7C5MdIiIfJLd7cnwXFSe3SJBJdsyi+Ywub/br6QrZox7s7XYcrvFH5iDbQ1VjQtS4srftcRxyxqQ2n0J/MZVSwoiejh0p4W2Y7BAR+RiTWaBUp7dpj7OzYspZfLFIubrBiOWbTspeGxhvf1+3O8ckW01xKRUSnsjoC6WDOzAGqJSYN7mvJeEJVCnxREZfBKvdunqly/jmpyYi8mHltXrZ2g1XjOwA3lu3I4TAm5tP2tQpAc3Jx7UyozctgtV+WDRjCHbknsf5OgPGpEYhyYFzry42PCkcb981Cudq9YgK8u/SKUp3x2SHiMjH2BtJ0XZhcXLL60sScOkhRd46srPl+DlsO1khe+2Bq1NbXT4OACEBKkyyU3vjKH8/hd0k05cw2SEi8jH2amS6ciUW0PyDNyZEbTOF5okjO2U1jXj3x1wcK61BXFgA5qT3sRp5Ka/V290N+fJekZg8IKa7QiWwZoeIyOfY22Onq0d2APmprEIP22tHCIHXvz+GbacqUFFrwKFCHZ76Yr8laTObBd7IPo46ve1eOSEBfnjsmj6tnt9Izsdkh4jIx8jtnhwS4IeQgK7fcE6uSLmyzoB6g+0GeO4qr6IeOcU1Vm0NBhMWfZuDRqMJH/+ajwNnbfe4AYA56X0QEWS73Jy6FpMdIiIfUywzstNVZ2JdKiFC/n2Ol9ruQ+OuTp+T3yAwv6IeL3x5CJ/b2RDw6r7RGJ8W3ZWhkR1MdoiIfIjZLGT32Onqep0W9ja1O3i2qlve3xny7JxzBQBHS2pk26OD/fHwhN5dFRK1gckOEZEPOV9vgNFku+y8q/fYaZEWE4JAmSXQ9qZ93FGenXOu7FEqJCyYOqBbpglJHpMdIiIfYm8lVlfvsdNCqZBkN9M7XlrT6uGX7kIIIK9cfhrLnt9dldLmeVbUtZjsEBH5EFftsXOxYUlhNm1mARwpdv/RndomoFZmlZU9V/eNxvVD4rowInIEkx0iIh8iV5wMdN/IDgAMSQiXbZc7pdvdlLdjlXxSZCAenZjGZeZugMkOEZEPKZIpTg5UKREa2H17zKZGB8me0XTAA4qUyxrkE5dLP0+gSokF1w1AoL/vHtHgTpjsEBH5kGKZDfziwgO6dfRBoZAwJNF2Kut0eR1qGm3PkXIn5xpt+0mpkPDP2aMwfVg8ekZqcEVKJF6/ZVi7z7KiruNwKj9jxgyHX3T16tUdCoaIiLqOEEK2QLm79ti52JCEMPxyyvrcKCGAQ4U6XNk7qtvjcdS5RgCXHGmVFKlBkNoPD1yd6pKYqG0Oj+yEhYVZfoWGhmLjxo3YtWuX5fru3buxceNGhIXZZutEROR6pTV6NBhti2vju2nZ+cWGJYbLth8srOrWONrD0GTGeZmRnZQojuC4O4dHdlauXGn5/TPPPIOZM2dixYoVUCqb5yNNJhPmzJmD0FDbJYVEROR6n+6U39nXFSM7SZGBCNeoUFVvPW2134332zlb2QDbHYqAXtFB3R4LtU+Hanbee+89PP3005ZEBwCUSiWefPJJvPfee04LjoiInKOkHthyvNymXZLkl4J3NUmSMCTB9n3PVNSjqt7Q7fE4IrdCfn8dJjvur0PJTlNTE3Jycmzac3JyYDabOx0UERE5jxACm4vl/7mf1D8WMSHdP40FAENlipQB4GChe47u5NvZOTklismOu+vQWsN7770X9913H06ePIkxY8YAALZv346//OUvuPfee50aIBERdc6OvEqcrZNwaUml2k+BO8f0dE1QAIbYqds5cLYaV6X16N5gHJAvcyZWuEbFU8w9QIeSnddffx1arRZvvPEGiouLAQBxcXGYP38+nnrqKacGSEREHWc0mfHBL2dkr824LBFRwWrZa90hPiwAUcH+qKi1nrZyx/12hBDIK7dNdpJZnOwR2p3sNDU14eOPP8bdd9+N+fPnQ6fTAQALk4nIZ5TpGnG4SIdDhdU4VFSN8loDUqOD8Ng1aejpZj/8vjtUIrtrckSQP2ZcluCCiC6QJAlDE8Ox6WiZVXtRVSNOn6tFag/5E9JdobLeCF1jk017L05heYR21+z4+fnh97//PfR6PYDmJIeJDhH5ghOlNXjq8/24/4NdWJJ9HOuPlKKoqhGGJjOOltTgj2sO4nyd+xTXNhpN+HSH/KjOXWOSESBz+nh3GypTpAwAb20+BbNZbu2Ta+TaOfwztQeTHU/QoQLl0aNHY+/evc6OhYjIbTUaTXj56yM4Xlpj957qBiOWZB9zmx/Su/MrUSMzGpESHYRJ/WNcEJGty3tFQqW03bvmaEkN1h8pdUFE8uyddM6RHc/QoZqdOXPm4KmnnsLZs2cxcuRIBAVZ/2EPHTrUKcEREbmLX05V2OwJI2d/QTXW7C3ETSMTuyGq1u3MOy/bft/4FCgU7nE4ZZhGhd9elojPdxbYXPtgWx7GpEYiXOP6AuA8mWXnCoWExAj3mrYkeR1Kdm699VYAwOOPP25pkyQJQghIkgSTyXaHTiIiT3bqXK3D9/5rez6GJoYhLTakCyNqndkssDu/0qY9JkSNYXaWfLvKzFGJ2HLsHEp11rVFtfomrPw5D09k9HVRZBfITWMlRgTC349HTHqCDiU7ubm5zo6DiMit2dtjRaGQbKatzGaBxd8fw99uG+GyU69PnauVHYka1SuyWw/9dITaT4nfp6ci66sjNtd+OFqGyQNiZQ8O7S6GJjMKKm1Pi+f+Op6jQylpcnJyq7+IiLyN3B4rgxPC8MpvBkNuRqikuhErtpzqhsjk7cyzHdUBgMt7RXRzJI4ZmRyJcX2iZa+9teWkS+ugDhdVy74/d072HB0a2Wlx5MgRnDlzBgaD9eqD6dOndyooIiJ3oms0olJmlVWvKA0GJ4ThllFJ+Eym5uSHo2W49fIkxId3/9lTu2Tqdfz9FC4dIWnL765KwZ78SpvDSgvON2Df2Spc1tP5iVpL+YU9+iaT3aQ1JZr1Op6iQ8nO6dOn8dvf/hYHDx601OoAsHzDsGaHiLzJGTtTWMn/m8a4/Yqe2F9QhaMltiu19pyp7PZkp7LOgBNltjVGQxJCofZz/XJze6KD1bhjTE/880fbUonDhdVOTXY2HSvDpzvOoLrBiKvSeuCBq1Jl628++fUMiqps9ykK9FdiULz7Jo5krUPTWHPnzkVKSgpKS0uh0Whw+PBhbN26FaNGjcLmzZudHCIRkWvJrcQBLuyeq1RIePrafrL3FMrUenS1XTKFyQAwsmd49wbSAdOGxiNQZv+fHJlEsqN+PlmOJeuPo6iqEXV6E9YdKsHyTSdt7jtZVoM1ewtlX+OO0T3dYp8ickyHkp1ffvkFL7/8Mnr06AGFQgGFQoHx48dj0aJFViu0iIi8gb3i5IuPCogNDUC4RmVzT2GVC5IdO0vORyW7Z73OxZQKCX21tjsnnyitgckJdTuVdQb8XSax2XS0DIcuOoC0yWTG3zaehNxbpsUE4Yah8Z2OhbpPh5Idk8mE4ODmb8bo6GgUFRUBaC5cPnbsmPOiIyJyA/kyIzsxIWpo/K0rARIjbKeruntkx2gyY++ZKpv26ACBHiGuOwerPfprbXflbzSa7Y6wOUoIgWU/nJTdaBEA/rH1tCWhWr2nUHa5uUICHpnY2232KSLHdKhmZ/DgwThw4ABSU1MxevRoLF68GP7+/nj77beRmprq7BiJiFxGCCE7siN3BlZihAaHCnVWbedq9dA3mbqtVuZwkc6mwBcAUkPcY1dnRwyIk9+f6GhxDXp34rysDTlldjdaBJp3SV53qAQhAX74+Nd82XvGxJiRHMnCZE/ToWTn+eefR11dc8b75z//GdOmTcNVV12FqKgofPbZZ04NkIjIlcprDag32CYPcscEJMgUIgsBFFc1dtsyZXtTWKmhnpPs9LWzGeOxEh2uHxrXodcs1TXina2n27zvg2150DeZZKevekYGYnSQ/WSJ3FeHkp1rr73W8vvU1FQcOXIE58+fR0REhNttVkVE1BlnzstPnciN7CTITGMBzXU73ZXsyI1cBKuViPegwYiQABWSIgNRcN56CrCjRcpms8DSDSdkR7wuZe8eSQIeSU/Fyd3yBcvk3jpUs5OdnY36euth3chI99uVk4ios/LK5YuTHR3ZAbqvbqeoqkF2mfTwpHDZjQ/dWb9Y27qdkupGVNW3/1T5bacqrIqPO+LWy5PsjjiR++tQsnPTTTchIiICY8eOxYIFC/D999+jttbxc2OIiDyFXHGyQpJPbGJDA6CUySrOVsonTM62I1d+imVkcni3vL8z2a3b6cDoztYT52TbJw+IxZW9o9p8fsZlCZh1Rc92vy+5jw4lO5WVldi8eTOmT5+OvXv34pZbbkFkZCTGjBmDZ5991tkxEhG5jNwxEQl2DoBUKiTEhwfYtJ/tpuXnv5yqsGlTSMBlSeHd8v7OJLciCwCOFutk21sjl2xGBfvjgatTcP/4FKiU9oe9Zl6ehHvG9uLMhYfrULKjVCpx5ZVX4tlnn8W6deuwbds2zJo1C7t378Zrr73m7BiJiFzCbBYokEl2kls5AFJuxKewssGy03xXqao3IKfENhEYGB+K0EDb/X/cXWJEIILUtivYjpW2b2THbBYoqbad2huaEAaNvx9iQwMw47JE2WfvGN0Td41JZqLjBTqU7OTk5GDFihW47bbbEBcXh2uuuQY6nQ5//etfsWfPHmfHSETkEkXVDTCabJOU1pYeyx0NUW8wobrB9gRyZ/o19zzk8qkxqW1P07gjhUKSHd05XlqLJpPZ4dc5X2+Q/TPUhl34c7plVCIGxl14L4UE3DO2F27j1JXX6NBqrEGDBqFHjx6YN28eXnjhBQwaNMjZcRERuVxbZ2LJSYyQT4TOVjYgXOPvlLjkyE1hAZ6b7ABAP20Idl9y9IWhyYzc8jqkOVgsXCxTsA0AcWEXphvVfkq8/JtB+OVUBSrrDRiWGI7UTuznQ+6nQ8nO448/jq1btyIrKwtffvkl0tPTkZ6ejquuusqyszIRkafLc+CYiEvZW5F1trIBgxO65uDIekMT9p+tsmlP7RGE2NAAGI1dO6rUVfpr7RcpO5zsVMvXS2nDrGur1H5KpPeLaV+A5DE6lOwsXboUAFBVVYUff/wRW7ZswZ/+9CccPHgQw4cPx/bt250ZIxFRtyjVNWJDTimq6o0YkRQuezyBv58C2lDbIuQW9vbaKerCIuVdeZVokpmqudKDR3WA5pEdSYLN9FxOsQ43DHPsbKpimXodwHpkh7xfh5KdFmazGU1NTTAYDNDr9TAajcjLy3NSaEREzmU2Cxwu0sFPKSElOshyarUQAt8fLsU7P56Goam5HmTdoRLZ10iO1LR6LlJYoArBaj/U6q3PX+rKA0G3n5afwnJkWbU70/j7oWekxua4jmPtWH4ul+wEqpQI88Cibeq4DiU7c+fOxebNm3H48GFERkbi6quvxoMPPoj09HQMHjzY2TESEXVanb4JC1YftBzuGOivxDX9Y5Derwe+3FuEn0+WO/Q6cjsnXyoxItBmP5iu2mvH0GTGrrxKm3ZtWAB6esEZTgPiQm2SnbIaPc7XGRAZ1HYNVInMNJY2LIArrHxMh5KdwsJCPPDAA0xuiMhjfLLjjNUp1g0GE745UIxvDhS363Xkdk6+VIJMslOi06PJZIafskOLYO06cLZK9oiDK1OjvOIHer/YENlRtqPFOoztE93qs0IIFMmM7HAKy/d0KNn597//7ew4iIi6jBACW084NnLTFkdGduSKlM1mgRJdo93VWh3lrVNYLfrb2Un5iAPJjq6xCQ0yh7gy2fE9Hf4vxocffohx48YhPj4e+fn5AJoLl9euXeu04IiInKHgfAMq69p/ppKc1vbYaWH3QFCZM7IaDCbkFOtwtETn0MaDZrPAocJq/Gf3WbyRfRxbj9smceEaFfp5yTlOCeGBCA20/X/5kaK2d1KW20wQsN5jh3xDh0Z23nrrLfzpT3/CvHnz8Morr8Bkas6cw8PDsXTpUtx4441ODZKIqDP2FtjWtNgTrPZDxsBYfH+4BPWXjAoMiAtBVLC6zddIDLe/187o//2+usGItfsK8fX+Yss01OCEMPzfjYPsTnUZmsxY+G2Ozd4zlxqTGtVqEbUnkSQJA+NCsf209blfp87VosFgQqC/7S7LLYrsLDuXO9KDvFuHRnaWLVuGd955B8899xyUygvfaKNGjcLBgwedFhwRkTPsK6hy6L7BCaFYNmsE7hufgrfvGoWpQ+IsZ2AlRgTisWvSHHodbViA7CnjhVXNI0zv/pSL+9/fiS92nbWqtzlUWI2v9hfZfd0tx8+1megAwJjUSIfi9BSD4m33JzKLto+OsD+yw2TH13RoZCc3NxcjRoywaVer1airs92XgojIVZpMZhwqrLZpT+0RhIyBsdiVV4lGownp/Xogc6DWMiISplHh9+m9cf/4FFQ1GNAjWO1wwa+/nwI9QgJQqrP+YZt9pBQ/nSiXLShusfnYObtnNe3Kkz/V/GJhgSoMTQx3KE5PMTBe/lDQI0U6DG/lkNNimeX+fkoJ0UFtj86Rd+lQspOSkoJ9+/YhOTnZqv27777DgAEDnBIYEZEzHC2pQaPR9iylUckRmDY0HtOGtr45nb+fAjEh7R8JSIwItEl2ALSa6ABAbnkdKmr1stNlbY1kKCTg4Qm9oXLyii9XS40OgtpPAX2T9Z/j4SLbJPZicnvsaEMDvGaKjxzXoWTnD3/4Ax555BE0NjZCCIEdO3bgk08+wcKFC/Huu+86O0Yiog6TO0YBAEb0jOjS902MCHRoyknOnjNVyBgYa9VWXqtHRa1tkXW4RoX+2hD0jArCpP4xsgeRejo/pQL9tCE4cNY6uTlWUtPqcv4SmWSTU1i+qUPp/7333osXX3wR8+fPR319PWbNmoUVK1Zg2bJluOqqq5waYGFhIe68805ERUVBo9Fg+PDh2L17t+W6EAJZWVmIj49HYGAg0tPTcfjwYafGQESea9+ZKps2tZ8Cfbt4tZK9M7IcITddddzOrsH3juuF564fiLvGJHtlotNCrm5H32TG6Yv2ThJCwGxuXtHWYDChqt72TDAuO/dNHR7rfOCBB5Cfn4+ysjKUlJRgx44d2Lt3L/r06eO04CorKzFu3DioVCp89913OHLkCP76178iPDzccs/ixYuxZMkSLF++HDt37oRWq0VGRgZqahzfTpyIvFOdvgnHZaZ+BieEWQqPu4q95ecXG9s7SnakYW9BFZpM1lM29qaw+mnl61m8jb26ncNF1TCZBd7afAp3vvsrfvevXfjmQLHdA0DjuOzcJ7Xrb3tVVRXuuOMO9OjRA/Hx8fjb3/6GyMhI/P3vf0efPn2wfft2vPfee04L7tVXX0VSUhJWrlyJK664Ar169cKkSZPQu3dvAM1Z/NKlS/Hcc89hxowZGDx4MD744APU19dj1apVTouDiDzTwcJqmGW2rmmtqNVZWts8UOOvxDNT+mPB1AEYK7P5X4PBZLMDs1zSFqz2Q7yPjFT0iw2RXeF2uFCHFVtO4duDxdA1NOFcjR4rtpzCf/fL74zNkR3f1K6anT/+8Y/YunUrZs+ejXXr1uGJJ57AunXr0NjYiG+//RYTJkxwanBfffUVrr32Wtxyyy3YsmULEhISMGfOHDzwwAMAmleFlZSUIDMz0/KMWq3GhAkTsG3bNjz00EOyr6vX66HX6y1f63TNm1MZjUYYjbbDnr6mpQ/YF53DfnSOzvTj7rwK2Y36BscFd/mfS4i/hP7aYOQUWycp/bTBeGJSH8SGBsBoNGJYQgj+s9s2xh2ny9EvpjlhMpkFjpfU2HyWPj00aGpqsnn2Ut7wvegnASlRGpw8Z73id8+ZSsvhrRfLPiJ/kGuPIL8O94M39KM7cGY/OvoaknBky87/SU5OxrvvvovJkyfj9OnT6NOnDx5//HEsXbq0o3G2KiCgOQN/8sknccstt2DHjh2YN28e/vGPf+Duu+/Gtm3bMG7cOBQWFiI+/sKKigcffBD5+fn4/vvvZV83KysLL730kk37qlWroNF4/sF5RNTs3WMKVOqthwM0fgK/H2BGdxwbdV4PfJWvQHmjBLUSGBFlxpWxAsqL3ttkBv5+RAnDJT+vewQIzO7b3FjWAPzrhO3meWNjzRgb6/A/4R5vU5GE3eUdn36UAMwbbIKXLVbzaS11w9XV1QgNtT+l266RnaKiIgwcOBAAkJqaioCAAPzud7/rXKStMJvNGDVqFBYuXAgAGDFiBA4fPoy33noLd999t+W+S/e+EEK0uh/GggUL8OSTT1q+1ul0SEpKQmZmZqud5SuMRiOys7ORkZEBlUrl6nA8FvvROTraj+W1eryTvxeXzlpcnRaN6yc7r7awLXcIgeqGJgSqFFCr5Hf7Pex3DL/mWq/cMgAYffUIRAWr8f3hUoSV5do8N2NSf1zWM7zNGLzlezHy9Hmc/P54h5/vEeKPG6Zd1uHnvaUfXc2Z/dgyM9OWdiU7ZrPZKjClUomgoLZPAO6ouLg4S3LVYsCAAfjPf/4DANBqtQCAkpISxMXFWe4pKytDbKz1ss2LqdVqqNW2e1ioVCp+A1+E/eEc7EfncKQft50qx9bj5SiraURFrUH2Pz2XJUd2+59HD3//Vq9fnhKNHXlVNu0HimqROSgYp8obZD/LgITwdn0WT/9eHJIU0amT3BMiNE75/J7ej+7CGf3o6PPtSnaEELjnnnssiUJjYyMefvhhm4Rn9erV7XlZu8aNG4djx45ZtR0/ftyymWFKSgq0Wi2ys7MtOzobDAZs2bIFr776qlNiICLP8NX+Iryz9XSb9w13YCSku43qJb/nz+78SmQO0soWJ8eFBSA0wLd+4IZr/JEQHohCmZ2RHcGVWL6rXcnO7Nmzrb6+8847nRrMpZ544gmMHTsWCxcuxMyZM7Fjxw68/fbbePvttwE0T1/NmzcPCxcuRFpaGtLS0rBw4UJoNBrMmjWrS2MjIvdRWNWAlT/bTvNcKjEiENEOHOTZ3aKD1UiO0iC/ot6qfW9BFaobjCiorLd5pr/WO041b69B8aEdTna0oVyJ5avaleysXLmyq+KQdfnll2PNmjVYsGABXn75ZaSkpGDp0qW44447LPfMnz8fDQ0NmDNnDiorKzF69GisX78eISG++Q8Bka8RQuCtzSfRZGq7UPeKFPc9IHNkcoRNstNgMOG/+4sgt4ykr48mOwPjQ7H+SGmHnuWyc9/VoeMiutO0adMwbdo0u9clSUJWVhaysrK6LygichtbT5Rjf0HrZyQBQGxoAH4zPKEbIuqYUcmRWL2n0Kb9q33yp6D36+IdoN2V3E7KAKBQSHjgqhT8Y4v9qUweFeG73D7ZISKyp1bfhH/+KP/DrXePIPSMCkJYoApJEYG4sncUQty4xmVAXAgCVUqbg0LlDg5VKSX0iu66xSHuLDZULVu3c+OweEwdHIev9xfbneZizY7v4m4DROSxPtqeL3v+UWSQPxbOGIInM/ri/vEpyBykdetEB2g+7NLRabbePYK97mRzR0mShPuvSoHqos2K+mtDcPsVPaFQSJg2LE72uXCNCoH+8kv/yftxZIeIPNKJ0hp8e1D+SIAHrkqFxt/z/nm7fXRP/JpbgUaj7Y7AF+vno/U6LS7vFYm/3T4C+wuqERaowuUpEVD7NScyk/rH4l/b8m1GxOI5quPTfPO/BkTk8f75Y65s4e7I5AiM62N73pQnSAgPxLPX9Zc9A+piXX1iuydIjNDg+qFxGJ8WbUl0ACDQXyk7unNV3+juDI/cDJMdIvI4xdUNOFJsu3OqSinhoQmpndp4ztVGJkdizsTWd3j29ZGdttx+RU9clRZtSRozB8Yic6DWtUGRS3neOC8R+bwduedl228ZleQVRajXDtKirEaPz3cW2FwL16gQE+J+ewW5E5VSgflT+kPXaITaT2E18kO+iSM7RORxdubZJjuSBEwdLF+c6onuHN0TE/vH2LRPHhDr0SNX3Sk0QMVEhwBwZIeIukh+RR3WHy5Frb4JE/r1wGU95Y9EaK86fRMOFtpOYfWLDUGYxr1XXLWHJEl47Jo+CA9UYe2+QpgFMD4tGjeNTHR1aEQeh8kOETldbnkdFqw+gDp984qYTcfK8OyU/hjbp/NFonvOVMJstq1MvtyNd0fuKJVSgfvGp2DW6J5oMgsEq/lPNlFH8G8OEXXIgbNV2HCkFGqVEtOHxSMpUgMAMJsFlv1wwpLoAIAQwFtbTuGy5AgEqDo3rbDTTr3OaC9Mdlp0ts+IfB1rdoio3Xblncdzaw5h07FzWHeoBPM+24cDZ6sAAD8cLcOJ0lqbZ6rqjfj6gPy+OI4ymwV25VfatMeGqtHzf8kWEdGlmOwQUbsIIfDOJUc0GJrMWPTtUZw6V4sPfsmz++x/dp9FvaGpw++dU6JDTaPt85f3imTRLhHZxWSHiNrlaEkNiqoabdpr9U14+ov9ssc3XHyPvYMtHWFvybk7n2ZORK7HZIeI2uWHo2V2rzWZZLY0vsSavYWoabyQEMkVG9sjt+Q8UKXE4AT5k7CJiAAWKBNRO+ibTNhy/FynXqPeYMKKLaegVCiwJ78S1Q3G5pPJIwORGKFBz0gNhiaGITnK+lTv4upGFJy3Pc16RHK4zx6KSUSOYbJDRA779fR5NBhMbd/4PwoJkBu42Xq83Orr6gYjqguNOHTR/jn9tSG4bogWVySHQwhg+2nfW4VFRM7BZIeIHLYxp7Rd908fnoCC8/XYLbOCqi1HS2pwtKQGgSoFzp9XIuDMGZsiZEkCRvZkskNErePYLxE5pLxWj30FVTbt4RoVlDLHdIdrVLj9iiTcOaZnp9633mCC3s5gkrftmkxEXYPJDhE5ZPOxc7JTUtOHxWNOem+rNkkCHp7QGxp/P/SJCcGVvaO6JCauwiIiR3Aai4jaJITAD0dtp7AkCZjYPwbRwWpEBvnju0MlMJkFpgzWYkzqhQRn9the2F9QhfpL6n2igv0xPCkcVfVGnDlfj3M1eodjClQpZQ/KJCK6FJMdIrKrusGIgvP1yCnWya6EGp4UjuhgNQBgVK9IjOolP9KSEB6IV347GP/ZU4jaxiakRAdhXJ9opMUEQ3HRFFitvglbj5/DtweLkV9RbzeuqGB/PHhVquW9iYhaw2SHiGycrzNgSfYx7C+obvW+SQNiHX7NPjEheGZK/1bvCVb7YeqQOFw3WIujJTXYdKwMZ8/XoUhU4aoR8UjpEYykCA16RQdxuTkROYzJDhFZMZrMyPrqMHLL61q9L9BfiTGpXVMzI0kSBsSFYkBcKIxGI779Ng9Tx/SESsViZCJqP/7XiIisfLrjTJuJDgBcnRYNtR9P4yYi98dkh4gsjpbo8O/dZ9u8L9BfiVtGJXVDREREncdpLCICADQaTXgj+7js8vIWEUH+GBQfiptHJiI2NKD7giMi6gQmO0QEAPhgW57saeZKhYQF1/XHwPhQhASwZoaIPA+THSLCvoIqfH2gWPbarCt6YnRq12wKSETUHVizQ+TjGo0mLNt4QvZa39gQ3DQysZsjIiJyLiY7RD7u0x1nUCazc7G/nwJPZKTJnntFRORJmOwQ+bD8ijqs2Vcke2322F5IjNB0c0RERM7HZIfIR5nNAm9uOgWzzPKrAXEhmDYkzgVRERE5H5MdIh+18WgZjhTrbNoVCgmPTOxjdWYVEZEn42osIi90rkaP/+4vQlmNHsMSwzBlsBaSdCF50TUasfLnXNlnfzM8HslRQd0VKhFRl2OyQ+Rl6vRNeP7Lg5Y9c34+WY6TZbV49Jo+loRn5U95qGlssnm2R4gat1/Rs1vjJSLqapzGIvIyX+0vstkccP2RUmw6VgYA2Hr8HDbklMo++9DVqQhQ8bwrIvIuHNkh8iKGJjO+PSi/OeCKzacRrFZh+Q8nZa+PTonk5oFE5JWY7BB5kR9PnENVvVH2WoPRhP/7+ojstQCVAg9OSO3K0IiIXIbTWEReQgiBtXb2zGnLwxN6IyaEB3sSkXdiskPkJQ4V6pBbXtfu564dFItJA2K7ICIiIvfAZIfIS6zdV9juZ3r3CMKDV/fugmiIiNwHkx0iL1BU1YAdeedt2oPVflAp5TcHDFIrsWDqAPj78Z8BIvJu/FeOyAt8faAIwvbUB0wbFod7xqXIPvNkRj/EhrJOh4i8H1djEXm4On0TNhwps2n3U0qYOjgO4RoVzlTU4fvDzXvrSBJw37gUXJES2d2hEhG5BJMdIg+3IacUDUaTTfvVaT0QEeQPAHhkYh9M7B+DgvP16BsbgtQewd0dJhGRyzDZIfJgQghkH5HfDfnG4fGW30uShEHxYRgUH9ZdoRERuQ3W7BB5sJNltcivqLdpH5wQytEbIqL/YbJD5MHW2xnVyRyo7eZIiIjcF5MdIg/VaDRhy/FzNu2B/kpc2ZtnXBERtWCyQ+ShfjlVgQaDbWHyhL49eHI5EdFFmOwQeajsHPkprIyBPPqBiOhiTHaIPFBxdQMOnq22ae8ZpUFaDAuTiYguxmSHqBsUVzdgw5FSHC6yTVA6YkOO7SaCAJA5MBaSJH88BBGRr+I+O0QOMDSZYRai3bUwTWbgvZ/z8M2hUstxDlMGa/HIxD7tjqHBYEJeRR0KztfL7q2jVEhI7xvT7tclIvJ2THaILmFoMmNn3nkcK6lBQWU9Cs7Xo6xGDyGAYUlheDqzH8I1/m2+TlFVA1adUkDvX2I12rLuUAnG9YnG8KRwh+IxmwXe35aHbw8WQ99ktnvf6JRIhGlUDr0mEZEv8ahprEWLFkGSJMybN8/SJoRAVlYW4uPjERgYiPT0dBw+fNh1QZJH0zUa8fQX+/GX745izd5C7MqrRKlObxmV2V9QjdfXH4OQO3Xzf0xmgY05pXjq3wdR1iA/pfT94RKHY8rOKcWavYWtJjoAkDmIhclERHI8JtnZuXMn3n77bQwdOtSqffHixViyZAmWL1+OnTt3QqvVIiMjAzU1NS6KlDyVEAJ//+EkcsvrWr1vf0E19pyptGozmszYnX8eyzaewN3v/YqlG06g0Wg/Odl+ugK6RqNDcX1zoLjNe6KC/TEiKcKh1yMi8jUekezU1tbijjvuwDvvvIOIiAv/oAshsHTpUjz33HOYMWMGBg8ejA8++AD19fVYtWqVCyMmT7T5+DlsO1Xh0L0fbMuH2dw8urP3TCUe/NcuZH11BOuPlELX0NTm800mgc3HbDcEvFRVvaHN5AsA7h2XAoWChclERHI8Itl55JFHcP3112Py5MlW7bm5uSgpKUFmZqalTa1WY8KECdi2bVt3h0kepNFogtF0YeSlvFaPf2w55fDzueV12HLiHPacqUTWf4+gvNbQ7hiyj5S2Oh0GAPtllpe3CAnww7CkMCy4rj8m9O3R7vcnIvIVbl+g/Omnn2LPnj3YuXOnzbWSkua6h9hY61qF2NhY5Ofn231NvV4PvV5v+Vqn0wEAjEYjjEbHpha8WUsfeGNfNBhMeGvLafx8qgKSJOGynuG4dmAMvjlYgtpG2xEZlVKCNjQABZUNNtfe/ykXdYYmmEzy01UtiYwQAkFqJer01rsd556rxdGiKvRpZV+c3XkVsgnRkluGICU6yPK1N/5ZtfDm78fuwj50DvajczizHx19DbdOdgoKCjB37lysX78eAQEBdu+7dF8RIUSre40sWrQIL730kk37+vXrodFoOh6wl8nOznZ1CE5lMgP/yVPgTO2F740NlVXYsD/P7jM3JJvRJ1jgvSIFqg3W31PVDmyZE6wSmBBRBaMZWFdmO5D65tofMTlBfnRHCGDDUQVqjNbvG+YvcOTXLcjxsVkrb/t+dAX2oXOwH53DGf1YX1/v0H1unezs3r0bZWVlGDlypKXNZDJh69atWL58OY4dOwageYQnLi7Ock9ZWZnNaM/FFixYgCeffNLytU6nQ1JSEjIzMxEaGtoFn8SzGI1GZGdnIyMjAyqVdyxlFkLgbz+cQrWyHGFhjj1zVVoUnpicBgAIP1mOJdknHXouLFCFK1MjMapnKIoP/4prMzNgggJ7P9htU7RcplRiUuZIqP1sE6HCqgYozuzHpeFmDIjB9empjn0IL+CN34/djX3oHOxH53BmP7bMzLTFrZOdSZMm4eDBg1Zt9957L/r3749nnnkGqamp0Gq1yM7OxogRIwAABoMBW7Zswauvvmr3ddVqNdRqtU27SqXiN/BFvKk/Ptqejy0nKhzeXTgyyB9zJqZZPn96Py3W7i/B6XOtFwtP7B+DeZPSoFBIMBqN+PZIcz9qVCqk94vB94etNwOsN5ix60w1xvWJRr3BhNAAP0uMh4rPycY7sleU1/y5tIc3fT+6CvvQOdiPzuGMfnT0ebdOdkJCQjB48GCrtqCgIERFRVna582bh4ULFyItLQ1paWlYuHAhNBoNZs2a5YqQyY0IIVBea8CPJ87hs50F7Xr28Ul9EBJw4S+RQiFh9theeHGt/T2chiWF4bFr+thdFTV5YKxNsgMAb2Qfx1/XHwcAaPyVuGNMMqYPi8e+M1U290oSMDTJwaEpIiIC4ObJjiPmz5+PhoYGzJkzB5WVlRg9ejTWr1+PkJAQV4dGLmA2C/z3QBF+PFGOMxX1aDCa2n7oElMGazEyOdKmfURSOIYmhuGAzAqp5CgNFlw3ACql/QWO/WJD0DNSgzPnreeYzReV7NQbTHhn62kE+ClwoND2fXr3CEZoAP9HSUTUHh6X7GzevNnqa0mSkJWVhaysLJfEQ+7l/W15WLO3sM37FAoJf5o2AE0mge8OlWD/2SoAwKT+MXjoavl6GEmScO+4XnjqiwOWPXaA5g39sqYPQpC69b9OkiRh8sAYvPdTXpvxvbn5FExm28LlYYkc1SEiai+PS3aI7DE0mfHdobZ3GwaAxyb2sYzejE6Ngsks0GQ2Q+3X+kGffWJC8FRGX7z3cy4qag3orw3BExl9ER1sWwMmZ2K/GLx/0YaE9sglOgAwvCd3SSYiai8mO+Q1CirrWz2iocXtV/TE5IHWq/WUCglKhWMnml/dtwfG94mG0YHk6FLhGn/8Zng8Vu9pe/TpUiqlhIFxXC1IRNReTHbIa7S1UipApcBNlyXi1suTOv1eCoUEtYPJ0aVmX9kLPULU2J1fCSGA0AA/FFU34lhJ6+e5DYoPg7/MEnUiImodkx3yGrnltbLtj09KQ0p0EJIiA9s9EtMVFAoJ04bGY9rQeEtbVb0Bv/9oD2r19s/VGp4U3g3RERF5H/43kbyG3IGZEUH+yBgYiz4xwW6R6NgTrvHHg3YKo1sM7xnePcEQEXkZJjvkFYQQstNYqRedH+Xu0vv1wMhk+QLk0EA/pER5zmchInInTHbIK5TV6FFvsN1TJ8WDkh1JkjBnYm8EqmxHoEYkRdjdrJCIiFrHZIe8gr3iZE9KdgAgJiQAv0/vjYvzGj+lhNtH93RdUEREHo4FyuQV5Op1ACC1h2clO0Dz+VqhgSr8eOIchABuHB6PhPBAV4dFROSxmOyQV5BbiaX2UyA+zDOThJHJEXbrd4iIqH04jUVeQW5kJzkqiHUuRETEZIc8X62+CaU6vU27J05hERGR8zHZIY+XZ69ex8OKk4mIqGsw2SGPd9pOspPCkR0iIgKTHfICuTLLziUJ6MVN+IiICEx2yAucllmJFRcWgACZzfmIiMj3MNkhj9ZkMuPM+Xqb9tQewS6IhoiI3BGTHfJoZysb0GQSNu2etnMyERF1HSY75NHs7Zzcm8XJRET0P0x2yKOdOmdbrwOwOJmIiC5gskMeTW5kJyxQhcggfxdEQ0RE7ohnY5FbKaluxBe7ClBeq0d8eCDG9o7GoPhQ2WMftp+uwLGSGpv2lOggSBKPiSAiomZMdsht1Omb8PyXBy1HP+w5U4WvDxQjLFCFMamR6BMTjNAAFUICVPj2UDF+OlEu+zo8JoKIiC7GZIfcxjcHi2XPuKpuMOL7w6X4/nCpQ6/TLzbE2aEREZEHY80OuQWzWWD94ZJOv058eABG9Yp0QkREROQtOLJDbuFAYbXsqE57XNYzHHMn94W/H3N4IiK6gMkOuYV1hzo+qhOs9sMDV6dgYr8YFiYTEZENJjvU7RqNJpjMAkHq5m+/qnoDtp+usLkvXKPCmNQobD9dgap6o+xrje0Thd9P6I1wDZeaExGRPCY71G2EEPhwez6+3FsIo0lgVK8IPDKxD7YcOweT2fbIh4yBsbj7yl6Yk94bVfVGVDdc+KVvMiMlOgh9YngGFhERtY7JDnWbTcfK8MWus5avd+VVYv6/D8DexFPmQC0AQJIkRAT5I4IbBRIRUQcw2aFuYTYLfL7zrE37uRr5ouRhSWHQhgV0dVhEROQDuGyFusXegkoUVjU4fP+UQXFdGA0REfkSJjvULb7cW+TwvWGBKoxO5V45RETkHEx2qMudqajHvoIqh++fNCAGKiW/NYmIyDn4E6ULlVQ3tuuHvLdau6+wXfdnDtJ2USREROSLmOx0kQaDCf/3zRG8uPYQ1u4rhBC2S6u9QVlNI745UIz1h0twpqLe5nNW1xux6ViZzXP+fgpkDIy1aR/bOwoJ4YFdFi8REfkersbqAmazwBsbjuNMRT0A4J8/5uL0uTo8MrGPVx1lcPBsNV786hCMpgsJTmigHwbFh2FgXCh6Rmmwv6DK6nqLa/rHYE56b/SNDcbHv55BdYMRV6ZGYc7EPt35EYiIyAcw2ekCn+0qwC+nrHcE/uFoGQoq6/Hc1AGICla7KDLnqWk0YvH3R20SGV1DE345VWHz+S914/B4SJKEKYPjMGUwV14REVHX8Z5hBjdxrKQGq349I3vtRGktnvh8P46W6Lo5Kud776c8u0c4tGVkcgQSIzROjoiIiEgekx0n6xsbjLuuTIa98ygr6wxYsPogtp0s797AnGhfQRU25JR2+Pkbh8c7MRoiIqLWMdlxMkmSMHNUEp6/fiACVUrZe5pMAq+vPwZdY8dGRlyp0WjC8h9OdPj5npEaDE8Kd15AREREbWCy00WuSInEX2cOQ3y4/JEHRpPAgYLqbo6q8z7ano9Sne0RD4EqJW4YFoc+McFQ2BnVkiTgd1elQLI37EVERNQFWKDchZIiNfjrzOF4/ftj2J1faXO9qNrx4xNczWgyY8ORUny1X34n5LvHJmPa0ObpqUajCYVVDThzvh5nz9ejsKoR/koJmYO0GJwQ1p1hExERMdnpasFqPzx3/QDc/NY2mC9ZgV1S3eiaoNqh0WjCxpwy/GfPWbuHdg6IC8HUi1ZUBaiU6N0jGL17BHdXmERERHYx2ekGKqUCPUICUKqzTm6K3SzZEULgcJEOO06XY2ueAl+v2oeyGr1NknYxP6WEx65Jg8Le3BUREZGLMdnpJnFhtslOiZtNY/3rl3z8e/dZCCFQrZMQJjW2WV9z++U9kRTJZeREROS+WKDcTbRhtoXKFXUGGJrMLojG1s8ny/Hv3Wfb9czE/jGYcVlCF0VERETkHBzZ6SZxMsmOEECprtHlIyNlukYsa8dy8l7RQbh1VBLG9YniyioiInJ7THa6idzIDgCUuDjZMZmb9/yp05vavLdPTDBuvTwJV/SKZI0OERF5DCY73SQuTP4kb1cXKX+2swA5xTWy1/r0CEJKjxD0jApEf20o+mtDOJJDREQeh8lON9GG2hnZcWGR8qHCany2U/4cr4HhAq/dPAQqlaqboyIiInIuFih3k0B/JcI1tomDq0Z29hVU4ZVvcmSXlceGqjEpwT0Kp4mIiDqLyU43khvd6e6NBYUQWLuvEC+uPYRafZPNdYVCwlOT06CWP9aLiIjI43AaqxvFhQXgaIl1fUyprhFms+iygt8GgwlnK+tR3WCErtGI3fmV2Hrc/onrd41JRlpsMDp+1CcREZF7YbLTjbQyRcpGk0BFnQE9QtROfS8hBD7ZUYDPdxXA1NoWyBcZmRyBGSMSYDLZjvgQERF5Kk5jdSO5vXYA509lCSHw/rY8fLLjjMOJzlVp0Xj2uv5cUk5ERF7HrZOdRYsW4fLLL0dISAhiYmLwm9/8BseOHbO6RwiBrKwsxMfHIzAwEOnp6Th8+LCLIm6dvb12ip28IuvzXQVYvafQoXslCbjrymT84dp+CFCxUIeIiLyPWyc7W7ZswSOPPILt27cjOzsbTU1NyMzMRF1dneWexYsXY8mSJVi+fDl27twJrVaLjIwM1NTI7x3jSnZHdnQdG9mp1TdhY04p1u4rxC+nKlBY1YC1+wrx0Xb55eSXClQp8fz1AzFzVBL3zyEiIq/l1jU769ats/p65cqViImJwe7du3H11VdDCIGlS5fiueeew4wZMwAAH3zwAWJjY7Fq1So89NBDrgjbrrBAFQJVSjQYrXcr7sjy88KqBry49hBKdfp2PytJwKD4UMxJ7+PyoyqIiIi6mlsnO5eqrq4GAERGRgIAcnNzUVJSgszMTMs9arUaEyZMwLZt2+wmO3q9Hnr9hSRBp9MBAIxGI4xGY1eFDwCICfFHXkW9VVtRZX273rfBYMKf/3vY4VofSQLuG5uMftoQhAb4ITxQBfX/pqzk3relrav7wtuxH52D/dh57EPnYD86hzP70dHXkIQQjlWwupgQAjfeeCMqKyvx448/AgC2bduGcePGobCwEPHx8ZZ7H3zwQeTn5+P777+Xfa2srCy89NJLNu2rVq2CRtO1Ix1r8xU4UW09ZaRWCjw2yLFN/IQAvi1QIKfK8WmnaxPNGBLpEX/MREREDquvr8esWbNQXV2N0NBQu/d5zMjOo48+igMHDuCnn36yuXZpvYkQotUalAULFuDJJ5+0fK3T6ZCUlITMzMxWO8sZzv2Sj7J9xTbtV10zCiEBbf9xfHeoBEVn8hAW5tj73TcuGTcMjWtXjEajEdnZ2cjIyOBxEZ3AfnQO9mPnsQ+dg/3oHM7sx5aZmbZ4RLLz2GOP4auvvsLWrVuRmJhoaddqtQCAkpISxMVd+IFeVlaG2NhYu6+nVquhVtvua6NSqbr8GzgxMkg2Eauob0JkiPxhoS2Ol9bgvW1nHC4mvmtMMmaMTOpQnED39IcvYD86B/ux89iHzsF+dA5n9KOjz7t1siOEwGOPPYY1a9Zg8+bNSElJsbqekpICrVaL7OxsjBgxAgBgMBiwZcsWvPrqq64IuU1yGwsCzUXKabEhdp/TNRrxl++Oyu6bE6RW4qGre+NcjR5nK+vhp1TgqrRojOgZ4bS4iYiIPJVbJzuPPPIIVq1ahbVr1yIkJAQlJSUAgLCwMAQGBkKSJMybNw8LFy5EWloa0tLSsHDhQmg0GsyaNcvF0cvryMaCZrPAkvXHca5GfuXVkxn9cEVKpFPiIyIi8jZuney89dZbAID09HSr9pUrV+Kee+4BAMyfPx8NDQ2YM2cOKisrMXr0aKxfvx4hIfZHSVwpOlgNhUKC+ZIRmtaWn3+2qwC78ytlr90yKpGJDhERUSvcOtlxZKGYJEnIyspCVlZW1wfkBEqFhNgQtU1yU6KT30V575lKfLJDfpPAIYlhuGN0stNjJCIi8iZuvYOyt5KbypIb2TlXo8fr649BLueLCPLH/Gv7QcmzrIiIiFrFZMcF5IqUK2oN0Ddd2FlZ12jE4nVHoWuwPYFcIQHPTOmHcI1/l8ZJRETkDdx6Gstb2StS/u/+YlzRKxIbj5bi24PFaDTKbzR477gUDIp3cKMdIiIiH8dkxwXsnX7+wbY8fLAtr9Vnx/aOwo3D41u9h4iIiC7gNJYL2BvZaUt8eADmTk7jCeVERETtwGTHBZIiNIgNbV/CE6z2w4LrBkDjz8E4IiKi9mCy4wIKhYSHJ6QiSK106P4rUiKx5NZh6BUd1MWREREReR8OE7jIqF6RePOOkdh0tAw/nyrHidJaq+uSBFzZOwq3jkpCao9gF0VJRETk+ZjsuFBkkD9uGpmIm0YmoqymEb+cqsCpslqEa/wxeUAsekZpXB0iERGRx2Oy4yZiQgJw4/AEV4dBRETkdVizQ0RERF6NyQ4RERF5NSY7RERE5NWY7BAREZFXY7JDREREXo3JDhEREXk1JjtERETk1ZjsEBERkVdjskNERERejckOEREReTUmO0REROTVmOwQERGRV+NBoACEEAAAnU7n4kjcg9FoRH19PXQ6HVQqlavD8VjsR+dgP3Ye+9A52I/O4cx+bPm53fJz3B4mOwBqamoAAElJSS6OhIiIiNqrpqYGYWFhdq9Loq10yAeYzWYUFRUhJCQEkiS5OhyX0+l0SEpKQkFBAUJDQ10djsdiPzoH+7Hz2IfOwX50Dmf2oxACNTU1iI+Ph0JhvzKHIzsAFAoFEhMTXR2G2wkNDeVfaCdgPzoH+7Hz2IfOwX50Dmf1Y2sjOi1YoExERERejckOEREReTUmO2RDrVbjxRdfhFqtdnUoHo396Bzsx85jHzoH+9E5XNGPLFAmIiIir8aRHSIiIvJqTHaIiIjIqzHZISIiIq/GZIeIiIi8GpMdH7Vo0SJcfvnlCAkJQUxMDH7zm9/g2LFjVvcIIZCVlYX4+HgEBgYiPT0dhw8fdlHE7m/RokWQJAnz5s2ztLEPHVdYWIg777wTUVFR0Gg0GD58OHbv3m25zr5sW1NTE55//nmkpKQgMDAQqampePnll2E2my33sB9tbd26FTfccAPi4+MhSRK+/PJLq+uO9Jler8djjz2G6OhoBAUFYfr06Th79mw3fgrXa60fjUYjnnnmGQwZMgRBQUGIj4/H3XffjaKiIqvX6Kp+ZLLjo7Zs2YJHHnkE27dvR3Z2NpqampCZmYm6ujrLPYsXL8aSJUuwfPly7Ny5E1qtFhkZGZazxOiCnTt34u2338bQoUOt2tmHjqmsrMS4ceOgUqnw3Xff4ciRI/jrX/+K8PBwyz3sy7a9+uqrWLFiBZYvX46cnBwsXrwYr732GpYtW2a5h/1oq66uDsOGDcPy5ctlrzvSZ/PmzcOaNWvw6aef4qeffkJtbS2mTZsGk8nUXR/D5Vrrx/r6euzZswcvvPAC9uzZg9WrV+P48eOYPn261X1d1o+CSAhRVlYmAIgtW7YIIYQwm81Cq9WKv/zlL5Z7GhsbRVhYmFixYoWrwnRLNTU1Ii0tTWRnZ4sJEyaIuXPnCiHYh+3xzDPPiPHjx9u9zr50zPXXXy/uu+8+q7YZM2aIO++8UwjBfnQEALFmzRrL1470WVVVlVCpVOLTTz+13FNYWCgUCoVYt25dt8XuTi7tRzk7duwQAER+fr4Qomv7kSM7BACorq4GAERGRgIAcnNzUVJSgszMTMs9arUaEyZMwLZt21wSo7t65JFHcP3112Py5MlW7exDx3311VcYNWoUbrnlFsTExGDEiBF45513LNfZl44ZP348Nm7ciOPHjwMA9u/fj59++glTp04FwH7sCEf6bPfu3TAajVb3xMfHY/DgwezXVlRXV0OSJMsIblf2Iw8CJQgh8OSTT2L8+PEYPHgwAKCkpAQAEBsba3VvbGws8vPzuz1Gd/Xpp59iz5492Llzp8019qHjTp8+jbfeegtPPvkk/vjHP2LHjh14/PHHoVarcffdd7MvHfTMM8+guroa/fv3h1KphMlkwiuvvILbb78dAL8nO8KRPispKYG/vz8iIiJs7ml5nqw1Njbi2WefxaxZsyyHgXZlPzLZITz66KM4cOAAfvrpJ5trkiRZfS2EsGnzVQUFBZg7dy7Wr1+PgIAAu/exD9tmNpsxatQoLFy4EAAwYsQIHD58GG+99Rbuvvtuy33sy9Z99tln+Oijj7Bq1SoMGjQI+/btw7x58xAfH4/Zs2db7mM/tl9H+oz9Ks9oNOK2226D2WzGm2++2eb9zuhHTmP5uMceewxfffUVNm3ahMTEREu7VqsFAJtsuqyszOZ/OL5q9+7dKCsrw8iRI+Hn5wc/Pz9s2bIFf/vb3+Dn52fpJ/Zh2+Li4jBw4ECrtgEDBuDMmTMA+P3oqD/84Q949tlncdttt2HIkCG466678MQTT2DRokUA2I8d4UifabVaGAwGVFZW2r2HmhmNRsycORO5ubnIzs62jOoAXduPTHZ8lBACjz76KFavXo0ffvgBKSkpVtdTUlKg1WqRnZ1taTMYDNiyZQvGjh3b3eG6pUmTJuHgwYPYt2+f5deoUaNwxx13YN++fUhNTWUfOmjcuHE2Wx8cP34cycnJAPj96Kj6+nooFNb/rCuVSsvSc/Zj+znSZyNHjoRKpbK6p7i4GIcOHWK/XqQl0Tlx4gQ2bNiAqKgoq+td2o+dKm8mj/X73/9ehIWFic2bN4vi4mLLr/r6ess9f/nLX0RYWJhYvXq1OHjwoLj99ttFXFyc0Ol0LozcvV28GksI9qGjduzYIfz8/MQrr7wiTpw4IT7++GOh0WjERx99ZLmHfdm22bNni4SEBPH111+L3NxcsXr1ahEdHS3mz59vuYf9aKumpkbs3btX7N27VwAQS5YsEXv37rWsEnKkzx5++GGRmJgoNmzYIPbs2SOuueYaMWzYMNHU1OSqj9XtWutHo9Eopk+fLhITE8W+ffusfu7o9XrLa3RVPzLZ8VEAZH+tXLnSco/ZbBYvvvii0Gq1Qq1Wi6uvvlocPHjQdUF7gEuTHfah4/773/+KwYMHC7VaLfr37y/efvttq+vsy7bpdDoxd+5c0bNnTxEQECBSU1PFc889Z/XDhP1oa9OmTbL/Hs6ePVsI4VifNTQ0iEcffVRERkaKwMBAMW3aNHHmzBkXfBrXaa0fc3Nz7f7c2bRpk+U1uqofJSGE6NzYEBEREZH7Ys0OEREReTUmO0REROTVmOwQERGRV2OyQ0RERF6NyQ4RERF5NSY7RERE5NWY7BAREZFXY7JDRCRDkiR8+eWXrg6DiJyAyQ4RuZ177rkHkiTZ/JoyZYqrQyMiD+Tn6gCIiORMmTIFK1eutGpTq9UuioaIPBlHdojILanVami1WqtfERERAJqnmN566y1cd911CAwMREpKCr744gur5w8ePIhrrrkGgYGBiIqKwoMPPoja2lqre9577z0MGjQIarUacXFxePTRR62ul5eX47e//S00Gg3S0tLw1Vdfde2HJqIuwWSHiDzSCy+8gJtuugn79+/HnXfeidtvvx05OTkAgPr6ekyZMgURERHYuXMnvvjiC2zYsMEqmXnrrbfwyCOP4MEHH8TBgwfx1VdfoU+fPlbv8dJLL2HmzJk4cOAApk6dijvuuAPnz5/v1s9JRE7Q6aNEiYicbPbs2UKpVIqgoCCrXy+//LIQQggA4uGHH7Z6ZvTo0eL3v/+9EEKIt99+W0RERIja2lrL9W+++UYoFApRUlIihBAiPj5ePPfcc3ZjACCef/55y9e1tbVCkiTx3XffOe1zElH3YM0OEbmliRMn4q233rJqi4yMtPz+yiuvtLp25ZVXYt++fQCAnJwcDBs2DEFBQZbr48aNg9lsxrFjxyBJEoqKijBp0qRWYxg6dKjl90FBQQgJCUFZWVlHPxIRuQiTHSJyS0FBQTbTSm2RJAkAIISw/F7unsDAQIdeT6VS2TxrNpvbFRMRuR5rdojII23fvt3m6/79+wMABg4ciH379qGurs5y/eeff4ZCoUDfvn0REhKCXr16YePGjd0aMxG5Bkd2iMgt6fV6lJSUWLX5+fkhOjoaAPDFF19g1KhRGD9+PD7++GPs2LED7777LgDgjjvuwIsvvojZs2cjKysL586dw2OPPYa77roLsbGxAICsrCw8/PDDiImJwXXXXYeamhr8/PPPeOyxx7r3gxJRl2OyQ0Ruad26dYiLi7Nq69evH44ePQqgeaXUp59+ijlz5kCr1eLjjz/GwIEDAQAajQbff/895s6di8svvxwajQY33XQTlixZYnmt2bNno7GxEW+88QaefvppREdH4+abb+6+D0hE3UYSQghXB0FE1B6SJGHNmjX4zW9+4+pQiMgDsGaHiIiIvBqTHSIiIvJqrNkhIo/D2Xciag+O7BAREZFXY7JDREREXo3JDhEREXk1JjtERETk1ZjsEBERkVdjskNERERejckOEREReTUmO0REROTVmOwQERGRV/v/QFgtumSSnw4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "policyNN = Policy(4, 2)\n",
    "valueNN = Value(4)\n",
    "ppo = PPO(policyNN, valueNN, env)\n",
    "schedule = [\n",
    "    (20, 0.01),\n",
    "    (100, 0.001),\n",
    "]\n",
    "\n",
    "ppo.train(schedule)\n",
    "ppo.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "28375bc2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-07-20T10:37:15.826291Z",
     "iopub.status.busy": "2023-07-20T10:37:15.824627Z",
     "iopub.status.idle": "2023-07-20T10:37:15.830423Z",
     "shell.execute_reply": "2023-07-20T10:37:15.829589Z"
    },
    "papermill": {
     "duration": 0.030881,
     "end_time": "2023-07-20T10:37:15.832463",
     "exception": false,
     "start_time": "2023-07-20T10:37:15.801582",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "# env = gym.make('ALE/AirRaid-v5')\n",
    "# policyNN = CNNPolicy(env.observation_space.shape, 6)\n",
    "# policyNN.to(device)\n",
    "# valueNN = CNNValue(env.observation_space.shape)\n",
    "# valueNN.to(device)\n",
    "# ppo = PPO(policyNN, valueNN, env, device = device)\n",
    "# schedule = [\n",
    "#     (20, 0.001),\n",
    "#     (80, 0.0005),\n",
    "# ]\n",
    "# ppo.train(schedule = schedule, show_rewards = True)\n",
    "# ppo.plot()\n",
    "# ppo.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a783ee33",
   "metadata": {
    "papermill": {
     "duration": 0.021983,
     "end_time": "2023-07-20T10:37:15.876683",
     "exception": false,
     "start_time": "2023-07-20T10:37:15.854700",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 195.045068,
   "end_time": "2023-07-20T10:37:17.522446",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-07-20T10:34:02.477378",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0dd53ab5e7f04c93bdc6cfc79ba60aee": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "13b17e5f33c54229b86472af8ba8596f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fc0e5a5b24474496bd3374f24d9c38a9",
       "placeholder": "​",
       "style": "IPY_MODEL_93944946bb8a444a980ed458c678f8ea",
       "value": "Training [lr = 0.01]: 100%"
      }
     },
     "1813c016c1cf4db7b89588d2be3a4d25": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2d36264f0770456e8640283eed9ff9ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d1dc5e0a323643a5b3cfd0992fd3cf9b",
       "placeholder": "​",
       "style": "IPY_MODEL_c0d6368865a14273a4e9a77670a22b82",
       "value": " 100/100 [00:05&lt;00:00, 14.63 epochs/s]"
      }
     },
     "2d3bb1ed99284b4295558273ec73930c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "3b0f039b5a0447ffb9fd03a2b0455e40": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "53ca1e2a856d46098886fcabf976d940": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_13b17e5f33c54229b86472af8ba8596f",
        "IPY_MODEL_75757f42fc0a44ddb57bea0535c94036",
        "IPY_MODEL_7cd7e8759196409283fa0589021ff9ad"
       ],
       "layout": "IPY_MODEL_e02dc33cd8c64983934a5d47b94de07f"
      }
     },
     "55a38fa8be594086b509a728668755aa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "56d6c82d0e684fe7891122d3d25e6655": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_b4510091daec4c6eb7cdcfae9a49cf3f",
        "IPY_MODEL_aafd2b51690d498d825fb3c8385d3be3",
        "IPY_MODEL_2d36264f0770456e8640283eed9ff9ab"
       ],
       "layout": "IPY_MODEL_0dd53ab5e7f04c93bdc6cfc79ba60aee"
      }
     },
     "6d0479ce52d142c099660d5330f87d09": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "75757f42fc0a44ddb57bea0535c94036": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3b0f039b5a0447ffb9fd03a2b0455e40",
       "max": 20.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_1813c016c1cf4db7b89588d2be3a4d25",
       "value": 20.0
      }
     },
     "7cd7e8759196409283fa0589021ff9ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a2b2c4af1d8f4f1884752e4924164862",
       "placeholder": "​",
       "style": "IPY_MODEL_2d3bb1ed99284b4295558273ec73930c",
       "value": " 20/20 [00:00&lt;00:00, 35.25 epochs/s]"
      }
     },
     "93944946bb8a444a980ed458c678f8ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a2b2c4af1d8f4f1884752e4924164862": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "aafd2b51690d498d825fb3c8385d3be3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6d0479ce52d142c099660d5330f87d09",
       "max": 100.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_55a38fa8be594086b509a728668755aa",
       "value": 100.0
      }
     },
     "b4510091daec4c6eb7cdcfae9a49cf3f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fcc1cc671b7b4a1e8d04f88cfcc872b9",
       "placeholder": "​",
       "style": "IPY_MODEL_c64f46beac3c475dba316e11b6f84c2e",
       "value": "Training [lr = 0.001]: 100%"
      }
     },
     "c0d6368865a14273a4e9a77670a22b82": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c64f46beac3c475dba316e11b6f84c2e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "d1dc5e0a323643a5b3cfd0992fd3cf9b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e02dc33cd8c64983934a5d47b94de07f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fc0e5a5b24474496bd3374f24d9c38a9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fcc1cc671b7b4a1e8d04f88cfcc872b9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}