rl_cma_maze.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyO1cnkk5m9wg9Li/71MBSwY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Cesare-Caputo/RL-stuff/blob/main/rl_cma_maze.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#setup"
      ],
      "metadata": {
        "id": "tfbUjCXLKnwP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "import random\n",
        "import scipy.integrate as scp\n",
        "import copy\n",
        "import numpy.random as rnd\n",
        "import time\n",
        "import gym"
      ],
      "metadata": {
        "id": "lu2Rlm8VKpUy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install box2d-py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aWib9X94FgF9",
        "outputId": "00f35b9d-3cb9-4cc0-f549-1b89efdd80d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting box2d-py\n",
            "  Downloading box2d_py-2.3.8-cp37-cp37m-manylinux1_x86_64.whl (448 kB)\n",
            "\u001b[?25l\r\u001b[K     |▊                               | 10 kB 27.0 MB/s eta 0:00:01\r\u001b[K     |█▌                              | 20 kB 28.4 MB/s eta 0:00:01\r\u001b[K     |██▏                             | 30 kB 28.3 MB/s eta 0:00:01\r\u001b[K     |███                             | 40 kB 16.4 MB/s eta 0:00:01\r\u001b[K     |███▋                            | 51 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |████▍                           | 61 kB 14.9 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 71 kB 13.5 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 81 kB 14.7 MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 92 kB 15.9 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 102 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████                        | 112 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 122 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 133 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 143 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 153 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 163 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 174 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 184 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 194 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 204 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 215 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 225 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 235 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 245 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 256 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 266 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 276 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 286 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 296 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 307 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 317 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 327 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 337 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 348 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 358 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 368 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 378 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 389 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 399 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 409 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 419 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 430 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 440 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 448 kB 14.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: box2d-py\n",
            "Successfully installed box2d-py-2.3.8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "env = gym.make('LunarLander-v2')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "nZOgvlbLFUhO",
        "outputId": "1f876807-17a7-4e10-c08f-ba8aba9110d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-14d1038c6885>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgym\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'LunarLander-v2'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Making new env: %s'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m         \u001b[0;31m# We used to have people override _reset/_step rather than\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;31m# reset/step. Set _gym_disable_underscore_compat = True on\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mentry_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0m_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m             \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mentry_point\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m             \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0m_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name)\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mmod_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\":\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mmod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0mfn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: module 'gym.envs.box2d' has no attribute 'LunarLander'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#torchnet"
      ],
      "metadata": {
        "id": "bgHVZiB0K6gL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch.autograd import Variable\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "\n",
        "class BaseModel:\n",
        "    def get_weight(self):\n",
        "        weight = []\n",
        "        for param in self.parameters():\n",
        "            weight.append(param.data.numpy().flatten())\n",
        "        weight = np.concatenate(weight, 0)\n",
        "        return weight\n",
        "\n",
        "    def set_weight(self, solution):\n",
        "        offset = 0\n",
        "        for param in self.parameters():\n",
        "            param_shape = param.data.numpy().shape\n",
        "            param_size = np.prod(param_shape)\n",
        "            src_param = solution[offset: offset + param_size]\n",
        "            if len(param_shape) > 1:\n",
        "                src_param = src_param.reshape(param_shape)\n",
        "            param.data = torch.FloatTensor(src_param)\n",
        "            offset += param_size\n",
        "        assert offset == len(solution)\n",
        "\n",
        "class StandardFCNet(nn.Module, BaseModel):\n",
        "    def __init__(self, state_dim, action_dim, hidden_size):\n",
        "        super(StandardFCNet, self).__init__()\n",
        "        self.fc1 = nn.Linear(state_dim, hidden_size)\n",
        "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
        "        self.fc3 = nn.Linear(hidden_size, action_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = Variable(torch.FloatTensor(x))\n",
        "        x = torch.tanh(self.fc1(x))\n",
        "        x = torch.tanh(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x"
      ],
      "metadata": {
        "id": "IaRoMjK8K76W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#cma-es"
      ],
      "metadata": {
        "id": "rV9bV9_3LEfa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install cma"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K0-epfy1LO6v",
        "outputId": "968e90f0-e2df-4530-ddaa-da7222871335"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting cma\n",
            "  Downloading cma-3.2.2-py2.py3-none-any.whl (249 kB)\n",
            "\u001b[K     |████████████████████████████████| 249 kB 4.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from cma) (1.21.6)\n",
            "Installing collected packages: cma\n",
            "Successfully installed cma-3.2.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "import torch\n",
        "import numpy as np\n",
        "import multiprocessing as mp\n",
        "import os\n",
        "import pickle\n",
        "import sys\n",
        "import time\n",
        "import logging\n",
        "import cma\n",
        "import argparse\n",
        "\n",
        "def _makedir(name):\n",
        "    if not os.path.exists(name):\n",
        "        os.makedirs(name)\n",
        "\n",
        "def get_logger():\n",
        "    _makedir('log')\n",
        "    _makedir('data')\n",
        "    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s: %(message)s')\n",
        "    logger = logging.getLogger('MAIN')\n",
        "    logger.setLevel(logging.DEBUG)\n",
        "    return logger\n",
        "\n",
        "class Task:\n",
        "\n",
        "    def __init__(self, envname, hidden_size, max_steps, target, pop_size, reps, test_reps, weight_decay, noise_std, sigma):\n",
        "\n",
        "        self.task = envname\n",
        "        self.env_fn = lambda: gym.make(self.task)\n",
        "        self.repetitions = reps\n",
        "        self.test_repetitions = test_reps\n",
        "        env = self.env_fn()\n",
        "        #self.env = env\n",
        "        self.action_dim = env.action_space.n\n",
        "        self.state_dim = env.observation_space.shape[0]\n",
        "        self.reward_to_fitness = lambda r: r\n",
        "\n",
        "        self.max_steps = max_steps\n",
        "        self.pop_size = pop_size\n",
        "\n",
        "        self.num_workers = mp.cpu_count()\n",
        "\n",
        "        self.action_clip = lambda a: np.clip(a, -1, 1)\n",
        "        self.target = target\n",
        "\n",
        "        self.hidden_size = hidden_size\n",
        "        self.model_fn = lambda: StandardFCNet(self.state_dim, self.action_dim, self.hidden_size)\n",
        "        model = self.model_fn()\n",
        "        self.initial_weight = model.get_weight()\n",
        "        self.weight_decay = weight_decay\n",
        "        self.action_noise_std = noise_std\n",
        "        self.sigma = sigma\n",
        "        self.tag = 'CMA-%d' % (hidden_size)\n",
        "\n",
        "class BaseModel:\n",
        "    def get_weight(self):\n",
        "        weight = []\n",
        "        for param in self.parameters():\n",
        "            weight.append(param.data.numpy().flatten())\n",
        "        weight = np.concatenate(weight, 0)\n",
        "        return weight\n",
        "\n",
        "    def set_weight(self, solution):\n",
        "        offset = 0\n",
        "        for param in self.parameters():\n",
        "            param_shape = param.data.numpy().shape\n",
        "            param_size = np.prod(param_shape)\n",
        "            src_param = solution[offset: offset + param_size]\n",
        "            if len(param_shape) > 1:\n",
        "                src_param = src_param.reshape(param_shape)\n",
        "            param.data = torch.FloatTensor(src_param)\n",
        "            offset += param_size\n",
        "        assert offset == len(solution)\n",
        "\n",
        "class Normalizer:\n",
        "    def __init__(self, filter_mean=True):\n",
        "        self.m = 0\n",
        "        self.v = 0\n",
        "        self.n = 0.\n",
        "        self.filter_mean = filter_mean\n",
        "\n",
        "    def state_dict(self):\n",
        "        return {'m': self.m,\n",
        "                'v': self.v,\n",
        "                'n': self.n}\n",
        "\n",
        "    def load_state_dict(self, saved):\n",
        "        self.m = saved['m']\n",
        "        self.v = saved['v']\n",
        "        self.n = saved['n']\n",
        "\n",
        "    def __call__(self, o):\n",
        "        self.m = self.m * (self.n / (self.n + 1)) + o * 1 / (1 + self.n)\n",
        "        self.v = self.v * (self.n / (self.n + 1)) + (o - self.m) ** 2 * 1 / (1 + self.n)\n",
        "        self.std = (self.v + 1e-6) ** .5  # std\n",
        "        self.n += 1\n",
        "        if self.filter_mean:\n",
        "            o_ = (o - self.m) / self.std\n",
        "        else:\n",
        "            o_ = o / self.std\n",
        "        return o_\n",
        "\n",
        "class StaticNormalizer:\n",
        "    def __init__(self, o_size):\n",
        "        self.offline_stats = SharedStats(o_size)\n",
        "        self.online_stats = SharedStats(o_size)\n",
        "\n",
        "    def __call__(self, o_):\n",
        "        o = torch.FloatTensor([o_] if np.isscalar(o_) else o_)\n",
        "        self.online_stats.feed(o)\n",
        "        if self.offline_stats.n[0] == 0:\n",
        "            return o_\n",
        "        std = (self.offline_stats.v + 1e-6) ** .5\n",
        "        o = (o - self.offline_stats.m) / std\n",
        "        o = o.numpy()\n",
        "        if np.isscalar(o_):\n",
        "            o = np.asscalar(o)\n",
        "        else:\n",
        "            o = o.reshape(o_.shape)\n",
        "        return o\n",
        "\n",
        "class SharedStats:\n",
        "    def __init__(self, o_size):\n",
        "        self.m = torch.zeros(o_size)\n",
        "        self.v = torch.zeros(o_size)\n",
        "        self.n = torch.zeros(1)\n",
        "        self.m.share_memory_()\n",
        "        self.v.share_memory_()\n",
        "        self.n.share_memory_()\n",
        "\n",
        "    def feed(self, o):\n",
        "        n = self.n[0]\n",
        "        new_m = self.m * (n / (n + 1)) + o / (n + 1)\n",
        "        self.v.copy_(self.v * (n / (n + 1)) + (o - self.m) * (o - new_m) / (n + 1))\n",
        "        self.m.copy_(new_m)\n",
        "        self.n.add_(1)\n",
        "\n",
        "    def zero(self):\n",
        "        self.m.zero_()\n",
        "        self.v.zero_()\n",
        "        self.n.zero_()\n",
        "\n",
        "    def load(self, stats):\n",
        "        self.m.copy_(stats.m)\n",
        "        self.v.copy_(stats.v)\n",
        "        self.n.copy_(stats.n)\n",
        "\n",
        "    def merge(self, B):\n",
        "        A = self\n",
        "        n_A = self.n[0]\n",
        "        n_B = B.n[0]\n",
        "        n = n_A + n_B\n",
        "        delta = B.m - A.m\n",
        "        m = A.m + delta * n_B / n\n",
        "        v = A.v * n_A + B.v * n_B + delta * delta * n_A * n_B / n\n",
        "        v /= n\n",
        "        self.m.copy_(m)\n",
        "        self.v.copy_(v)\n",
        "        self.n.add_(B.n)\n",
        "\n",
        "    def state_dict(self):\n",
        "        return {'m': self.m.numpy(),\n",
        "                'v': self.v.numpy(),\n",
        "                'n': self.n.numpy()}\n",
        "\n",
        "    def load_state_dict(self, saved):\n",
        "        self.m = torch.FloatTensor(saved['m'])\n",
        "        self.v = torch.FloatTensor(saved['v'])\n",
        "        self.n = torch.FloatTensor(saved['n'])\n",
        "\n",
        "def fitness_shift(x):\n",
        "    x = np.asarray(x).flatten()\n",
        "    ranks = np.empty(len(x))\n",
        "    ranks[x.argsort()] = np.arange(len(x))\n",
        "    ranks /= (len(x) - 1)\n",
        "    ranks -= .5\n",
        "    return ranks\n",
        "\n",
        "class Worker(mp.Process):\n",
        "    def __init__(self, id, task_q, result_q, stop):\n",
        "        mp.Process.__init__(self)\n",
        "        self.id = id\n",
        "        self.task_q = task_q\n",
        "        self.result_q = result_q\n",
        "        self.stop = stop\n",
        "\n",
        "    def run(self):\n",
        "        np.random.seed()\n",
        "        while not self.stop.value:\n",
        "            if self.task_q.empty():\n",
        "                continue\n",
        "            id, solution = self.task_q.get()\n",
        "            fitness, steps = self.evalfun(solution)\n",
        "            self.result_q.put([id, fitness, steps])\n",
        "\n",
        "class Evaluator:\n",
        "\n",
        "    def __init__(self, config, state_normalizer):\n",
        "        self.model = config.model_fn()\n",
        "        self.repetitions = config.repetitions\n",
        "        self.env = config.env_fn()\n",
        "        self.state_normalizer = state_normalizer\n",
        "        self.config = config\n",
        "\n",
        "    def eval(self, solution):\n",
        "        self.model.set_weight(solution)\n",
        "        rewards = []\n",
        "        steps = []\n",
        "        for i in range(self.repetitions):\n",
        "            reward, step = self.single_run()\n",
        "            rewards.append(reward)\n",
        "            steps.append(step)\n",
        "        return -np.mean(rewards), np.sum(steps)\n",
        "\n",
        "    def single_run(self):\n",
        "        state = self.env.reset()\n",
        "        total_reward = 0\n",
        "        steps = 0\n",
        "        while True:\n",
        "            state = self.state_normalizer(state)\n",
        "            action = self.model(np.stack([state])).data.numpy().flatten()\n",
        "            action += np.random.randn(len(action)) * self.config.action_noise_std\n",
        "            action = self.config.action_clip(action)\n",
        "            state, reward, done, info = self.env.step(action)\n",
        "            steps += 1\n",
        "            total_reward += reward\n",
        "            if done:\n",
        "                return total_reward, steps\n",
        "\n",
        "\n",
        "class CMAWorker(Worker):\n",
        "\n",
        "    def __init__(self, id, state_normalizer, task_q, result_q, stop, config):\n",
        "        Worker.__init__(self, id, task_q, result_q, stop)\n",
        "        self.evalfun = Evaluator(config, state_normalizer).eval\n",
        "\n",
        "def train(config, logger):\n",
        "    task_queue = mp.SimpleQueue()\n",
        "    result_queue = mp.SimpleQueue()\n",
        "    stop = mp.Value('i', False)\n",
        "    stats = SharedStats(config.state_dim)\n",
        "    normalizers = [StaticNormalizer(config.state_dim) for _ in range(config.num_workers)]\n",
        "    for normalizer in normalizers:\n",
        "        normalizer.offline_stats.load(stats)\n",
        "\n",
        "    workers = [CMAWorker(id, normalizers[id], task_queue, result_queue, stop, config) for id in range(config.num_workers)]\n",
        "    for w in workers: w.start()\n",
        "\n",
        "    opt = cma.CMAOptions()\n",
        "    opt['tolfun'] = -config.target\n",
        "    opt['popsize'] = config.pop_size\n",
        "    opt['verb_disp'] = 0\n",
        "    opt['verb_log'] = 0\n",
        "    opt['maxiter'] = sys.maxsize\n",
        "    es = cma.CMAEvolutionStrategy(config.initial_weight, config.sigma, opt)\n",
        "\n",
        "    total_steps = 0\n",
        "    initial_time = time.time()\n",
        "    training_rewards = []\n",
        "    training_steps = []\n",
        "    training_timestamps = []\n",
        "    test_mean, test_std = test(config, config.initial_weight, stats)\n",
        "    logger.info('total steps %8d, %+4.0f(%+4.0f)' % (total_steps, test_mean, test_std))\n",
        "    training_rewards.append(test_mean)\n",
        "    training_steps.append(0)\n",
        "    training_timestamps.append(0)\n",
        "    while True:\n",
        "        solutions = es.ask()\n",
        "        for id, solution in enumerate(solutions):\n",
        "            task_queue.put((id, solution))\n",
        "        while not task_queue.empty():\n",
        "            continue\n",
        "        result = []\n",
        "        while len(result) < len(solutions):\n",
        "            if result_queue.empty():\n",
        "                continue\n",
        "            result.append(result_queue.get())\n",
        "        result = sorted(result, key=lambda x: x[0])\n",
        "        total_steps += np.sum([r[2] for r in result])\n",
        "        cost = [r[1] for r in result]\n",
        "        best_solution = solutions[np.argmin(cost)]\n",
        "        elapsed_time = time.time() - initial_time\n",
        "        test_mean, test_std = test(config, best_solution, stats)\n",
        "        best = -np.min(cost)\n",
        "        logger.info('total steps = %8d    test = %+4.0f (%4.0f)    best = %+4.0f (%+4.0f)    elapased time = %4.0f sec' %\n",
        "            (total_steps, test_mean, test_std, best, config.target, elapsed_time))\n",
        "        training_rewards.append(test_mean)\n",
        "        training_steps.append(total_steps)\n",
        "        training_timestamps.append(elapsed_time)\n",
        "        #with open('data/%s-best_solution_%s.bin' % (TAG, config.task), 'wb') as f: # XXX gets stuck\n",
        "        #    pickle.dump(solutions[np.argmin(result)], f)\n",
        "        if best > config.target:\n",
        "            logger.info('Best score of %f exceeds target %f' % (best, config.target))\n",
        "            break\n",
        "        if config.max_steps and total_steps > config.max_steps:\n",
        "            logger.info('Maximum number of steps exceeded')\n",
        "            stop.value = True\n",
        "            break\n",
        "\n",
        "        cost = fitness_shift(cost)\n",
        "        es.tell(solutions, cost)\n",
        "        # es.disp()\n",
        "        for normalizer in normalizers:\n",
        "            stats.merge(normalizer.online_stats)\n",
        "            normalizer.online_stats.zero()\n",
        "        for normalizer in normalizers:\n",
        "            normalizer.offline_stats.load(stats)\n",
        "\n",
        "    stop.value = True\n",
        "    for w in workers: w.join()\n",
        "    return [training_rewards, training_steps, training_timestamps]\n",
        "\n",
        "def test(config, solution, stats):\n",
        "    normalizer = StaticNormalizer(config.state_dim)\n",
        "    normalizer.offline_stats.load_state_dict(stats.state_dict())\n",
        "    evaluator = Evaluator(config, normalizer)\n",
        "    evaluator.model.set_weight(solution)\n",
        "    rewards = []\n",
        "    for i in range(config.test_repetitions):\n",
        "        reward, _ = evaluator.single_run()\n",
        "        rewards.append(reward)\n",
        "    return np.mean(rewards), np.std(rewards) / config.repetitions\n",
        "\n",
        "def multi_runs(task, logger, runs=1):\n",
        "    if not os.path.exists('log'):\n",
        "        os.makedirs('log')\n",
        "    fh = logging.FileHandler('log/%s-%s.txt' % (task.tag, task.task))\n",
        "    fh.setLevel(logging.DEBUG)\n",
        "    logger.addHandler(fh)\n",
        "\n",
        "    stats = []\n",
        "    for run in range(runs):\n",
        "        logger.info('Run %3d/%3d' % (run+1, runs))\n",
        "        stats.append(train(task, logger))\n",
        "        with open('data/%s-stats-%s.bin' % (task.tag, task.task), 'wb') as f:\n",
        "            pickle.dump(stats, f)"
      ],
      "metadata": {
        "id": "7pDJn8wwLH39"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#mazelab"
      ],
      "metadata": {
        "id": "WKmDrwGCKQwi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "### including all in one code cell\n",
        "import gym\n",
        "from gym.spaces import Box\n",
        "from gym.spaces import Discrete\n",
        "\n",
        "from dataclasses import dataclass\n",
        "from dataclasses import field\n",
        "@dataclass\n",
        "class Object:\n",
        "    r\"\"\"Defines an object with some of its properties.\n",
        "\n",
        "    An object can be an obstacle, free space or food etc. It can also have properties like impassable, positions.\n",
        "\n",
        "    \"\"\"\n",
        "    name: str\n",
        "    value: int\n",
        "    rgb: tuple\n",
        "    impassable: bool\n",
        "    positions: list = field(default_factory=list)\n",
        "\n",
        "\n",
        "from abc import ABC\n",
        "from abc import abstractmethod\n",
        "\n",
        "from collections import namedtuple\n",
        "import numpy as np\n",
        "\n",
        "#from .object import Object\n",
        "\n",
        "\n",
        "class BaseMaze(ABC):\n",
        "    def __init__(self, **kwargs):\n",
        "        objects = self.make_objects()\n",
        "        assert all([isinstance(obj, Object) for obj in objects])\n",
        "        self.objects = namedtuple('Objects', map(lambda x: x.name, objects), defaults=objects)()\n",
        "\n",
        "        for key, value in kwargs.items():\n",
        "            setattr(self, key, value)\n",
        "\n",
        "    @property\n",
        "    @abstractmethod\n",
        "    def size(self):\n",
        "        r\"\"\"Returns a pair of (height, width). \"\"\"\n",
        "        pass\n",
        "\n",
        "    @abstractmethod\n",
        "    def make_objects(self):\n",
        "        r\"\"\"Returns a list of defined objects. \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _convert(self, x, name):\n",
        "        for obj in self.objects:\n",
        "            pos = np.asarray(obj.positions)\n",
        "            x[pos[:, 0], pos[:, 1]] = getattr(obj, name, None)\n",
        "        return x\n",
        "\n",
        "    def to_name(self):\n",
        "        x = np.empty(self.size, dtype=object)\n",
        "        return self._convert(x, 'name')\n",
        "\n",
        "    def to_value(self):\n",
        "        x = np.empty(self.size, dtype=int)\n",
        "        return self._convert(x, 'value')\n",
        "\n",
        "    def to_rgb(self):\n",
        "        x = np.empty((*self.size, 3), dtype=np.uint8)\n",
        "        return self._convert(x, 'rgb')\n",
        "\n",
        "    def to_impassable(self):\n",
        "        x = np.empty(self.size, dtype=bool)\n",
        "        return self._convert(x, 'impassable')\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}{self.size}'\n",
        "\n",
        "\n",
        "from collections import namedtuple\n",
        "\n",
        "\n",
        "VonNeumannMotion = namedtuple('VonNeumannMotion',\n",
        "                              ['north', 'south', 'west', 'east'],\n",
        "                              defaults=[[-1, 0], [1, 0], [0, -1], [0, 1]])\n",
        "\n",
        "\n",
        "MooreMotion = namedtuple('MooreMotion',\n",
        "                         ['north', 'south', 'west', 'east',\n",
        "                          'northwest', 'northeast', 'southwest', 'southeast'],\n",
        "                         defaults=[[-1, 0], [1, 0], [0, -1], [0, 1],\n",
        "                                   [-1, -1], [-1, 1], [1, -1], [1, 1]])\n",
        "\n",
        "from abc import ABC\n",
        "from abc import abstractmethod\n",
        "\n",
        "import numpy as np\n",
        "import gym\n",
        "from gym.utils import seeding\n",
        "from PIL import Image\n",
        "\n",
        "\n",
        "class BaseEnv(gym.Env, ABC):\n",
        "    metadata = {'render.modes': ['human', 'rgb_array'],\n",
        "                'video.frames_per_second' : 3}\n",
        "    reward_range = (-float('inf'), float('inf'))\n",
        "\n",
        "    def __init__(self):\n",
        "        self.viewer = None\n",
        "        self.seed()\n",
        "\n",
        "    @abstractmethod\n",
        "    def step(self, action):\n",
        "        pass\n",
        "\n",
        "    def seed(self, seed=None):\n",
        "        self.np_random, seed = seeding.np_random(seed)\n",
        "        return [seed]\n",
        "\n",
        "    @abstractmethod\n",
        "    def reset(self):\n",
        "        pass\n",
        "\n",
        "    @abstractmethod\n",
        "    def get_image(self):\n",
        "        pass\n",
        "\n",
        "    def render(self, mode='human', max_width=500):\n",
        "        img = self.get_image()\n",
        "        img = np.asarray(img).astype(np.uint8)\n",
        "        img_height, img_width = img.shape[:2]\n",
        "        ratio = max_width/img_width\n",
        "        img = Image.fromarray(img).resize([int(ratio*img_width), int(ratio*img_height)])\n",
        "        img = np.asarray(img)\n",
        "        if mode == 'rgb_array':\n",
        "            return img\n",
        "        elif mode == 'human':\n",
        "            from gym.envs.classic_control.rendering import SimpleImageViewer\n",
        "            if self.viewer is None:\n",
        "                self.viewer = SimpleImageViewer()\n",
        "            self.viewer.imshow(img)\n",
        "\n",
        "            return self.viewer.isopen\n",
        "\n",
        "    def close(self):\n",
        "        if self.viewer is not None:\n",
        "            self.viewer.close()\n",
        "            self.viewer = None\n",
        "\n",
        "from dataclasses import dataclass\n",
        "\n",
        "\n",
        "@dataclass\n",
        "# class DeepMindColor:\n",
        "#     obstacle = (160, 160, 160)\n",
        "#     free = (224, 224, 224)\n",
        "#     agent = (51, 153, 255)\n",
        "#     goal = (51, 255, 51)\n",
        "#     button = (102, 0, 204)\n",
        "#     interruption = (255, 0, 255)\n",
        "#     box = (0, 102, 102)\n",
        "#     lava = (255, 0, 0)\n",
        "#     water = (0, 0, 255)\n",
        "\n",
        "## this one is better for smaller maze and david silver style\n",
        "class color:\n",
        "    obstacle = (0, 0, 0)\n",
        "    free = (255, 255, 255)\n",
        "    agent = (255, 51, 51)\n",
        "    goal = (51, 255, 51)\n",
        "    button = (102, 0, 204)\n",
        "    interruption = (255, 0, 255)\n",
        "    box = (0, 102, 102)\n",
        "    lava = (255, 0, 0)\n",
        "    water = (0, 0, 255)\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def random_maze(width=81, height=51, complexity=.75, density=.75):\n",
        "    r\"\"\"Generate a random maze array.\n",
        "\n",
        "    It only contains two kind of objects, obstacle and free space. The numerical value for obstacle\n",
        "    is ``1`` and for free space is ``0``.\n",
        "\n",
        "    Code from https://en.wikipedia.org/wiki/Maze_generation_algorithm\n",
        "    \"\"\"\n",
        "    # Only odd shapes\n",
        "    shape = ((height // 2) * 2 + 1, (width // 2) * 2 + 1)\n",
        "    # Adjust complexity and density relative to maze size\n",
        "    complexity = int(complexity * (5 * (shape[0] + shape[1])))\n",
        "    density    = int(density * ((shape[0] // 2) * (shape[1] // 2)))\n",
        "    # Build actual maze\n",
        "    Z = np.zeros(shape, dtype=bool)\n",
        "    # Fill borders\n",
        "    Z[0, :] = Z[-1, :] = 1\n",
        "    Z[:, 0] = Z[:, -1] = 1\n",
        "    # Make aisles\n",
        "    for i in range(density):\n",
        "        x, y = np.random.randint(0, shape[1]//2 + 1) * 2, np.random.randint(0, shape[0]//2 + 1) * 2\n",
        "        Z[y, x] = 1\n",
        "        for j in range(complexity):\n",
        "            neighbours = []\n",
        "            if x > 1:             neighbours.append((y, x - 2))\n",
        "            if x < shape[1] - 2:  neighbours.append((y, x + 2))\n",
        "            if y > 1:             neighbours.append((y - 2, x))\n",
        "            if y < shape[0] - 2:  neighbours.append((y + 2, x))\n",
        "            if len(neighbours):\n",
        "                y_,x_ = neighbours[np.random.randint(0, len(neighbours))]\n",
        "                if Z[y_, x_] == 0:\n",
        "                    Z[y_, x_] = 1\n",
        "                    Z[y_ + (y - y_) // 2, x_ + (x - x_) // 2] = 1\n",
        "                    x, y = x_, y_\n",
        "\n",
        "    return Z.astype(int)\n",
        "\n",
        "\n",
        "\n",
        "class Maze(BaseMaze):\n",
        "    @property\n",
        "    def size(self):\n",
        "        return x.shape\n",
        "\n",
        "    def make_objects(self):\n",
        "        free = Object('free', 0, color.free, False, np.stack(np.where(x == 0), axis=1))\n",
        "        obstacle = Object('obstacle', 1, color.obstacle, True, np.stack(np.where(x == 1), axis=1))\n",
        "        agent = Object('agent', 2, color.agent, False, [])\n",
        "        goal = Object('goal', 3, color.goal, False, [])\n",
        "        return free, obstacle, agent, goal\n",
        "\n",
        "\n",
        "class Env(BaseEnv):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "\n",
        "        self.maze = Maze()\n",
        "        self.motions = VonNeumannMotion()\n",
        "\n",
        "        self.observation_space = Box(low=0, high=len(self.maze.objects), shape=self.maze.size, dtype=np.uint8)\n",
        "        self.action_space = Discrete(len(self.motions))\n",
        "\n",
        "    def step(self, action):\n",
        "        motion = self.motions[action]\n",
        "        current_position = self.maze.objects.agent.positions[0]\n",
        "        new_position = [current_position[0] + motion[0], current_position[1] + motion[1]]\n",
        "        valid = self._is_valid(new_position)\n",
        "        if valid:\n",
        "            self.maze.objects.agent.positions = [new_position]\n",
        "\n",
        "        if self._is_goal(new_position):\n",
        "            reward = +1\n",
        "            done = True\n",
        "        elif not valid:\n",
        "            reward = -1\n",
        "            done = False\n",
        "        else:\n",
        "            reward = -0.01\n",
        "            done = False\n",
        "        return self.maze.to_value(), reward, done, {}\n",
        "\n",
        "    def reset(self):\n",
        "        self.maze.objects.agent.positions = start_idx\n",
        "        self.maze.objects.goal.positions = goal_idx\n",
        "        return self.maze.to_value()\n",
        "\n",
        "    def _is_valid(self, position):\n",
        "        nonnegative = position[0] >= 0 and position[1] >= 0\n",
        "        within_edge = position[0] < self.maze.size[0] and position[1] < self.maze.size[1]\n",
        "        passable = not self.maze.to_impassable()[position[0]][position[1]]\n",
        "        return nonnegative and within_edge and passable\n",
        "\n",
        "    def _is_goal(self, position):\n",
        "        out = False\n",
        "        for pos in self.maze.objects.goal.positions:\n",
        "            if position[0] == pos[0] and position[1] == pos[1]:\n",
        "                out = True\n",
        "                break\n",
        "        return out\n",
        "\n",
        "    def get_image(self):\n",
        "        return self.maze.to_rgb()"
      ],
      "metadata": {
        "id": "DwRBtnczTLXj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#load prev maze"
      ],
      "metadata": {
        "id": "xoOseGRvKbCg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.load(\"stupid-maze.npy\")\n",
        "print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zfx_3v8pKgbF",
        "outputId": "59fce649-9ca2-4084-86a1-4f5f07e14af0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[1 1 1 ... 1 1 1]\n",
            " [1 0 0 ... 0 0 1]\n",
            " [1 0 0 ... 0 0 1]\n",
            " ...\n",
            " [1 0 0 ... 0 0 1]\n",
            " [1 0 0 ... 0 0 1]\n",
            " [1 1 1 ... 1 1 1]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "start_idx = [[4, 1]]\n",
        "goal_idx = [[49, 48]]\n",
        "env_id = 'RandomMaze50by50-v3'\n",
        "\n",
        "\n",
        "gym.envs.register(id=env_id, entry_point=Env, max_episode_steps=1000)\n"
      ],
      "metadata": {
        "id": "eD5fX666Ki_P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "env = gym.make(env_id)\n",
        "obs = env.reset()\n",
        "screen = env.render(mode = 'rgb_array', max_width = 51)\n",
        "plt.imshow(screen)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "5TgoCMrrK0Aa",
        "outputId": "7f9bcd8b-ca03-4bb1-b8b8-083dc72107c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f300bd9dd90>"
            ]
          },
          "metadata": {},
          "execution_count": 18
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD6CAYAAABuxZF5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAN2UlEQVR4nO3dXawc5X3H8e8vNpSUpAJDalkYalBQIi4SIlk0UbigVFSURoELhIhSyZVQfdNKRKmUQCu1TaRKyU1eLqpWVkHxRRugSVojbqhLHLW9AcxbA7gOTgQKlsFKAQVu0hr+vdhxOT4csnP27Mvseb4faXV25uyZ+e/s/s4zzzMzu6kqJLXnPYsuQNJiGH6pUYZfapThlxpl+KVGGX6pURsKf5LrkxxNcizJHdMqStLsZdLj/Em2AD8CrgNeBB4FPlNVz/6Sv/GkAmnGqip9HreRlv8q4FhV/aSq/ge4B7hxA8uTNEcbCf9FwE9XTL/YzTtDkr1JDic5vIF1SZqyrbNeQVXtA/aBu/3SkGwk/MeBi1dM7+zm9eZ1BdLGJb26+O+wkd3+R4HLk1ya5GzgVuD+DSxP0hxN3PJX1akkfww8CGwB7q6qZ6ZWmaSZmvhQ30QrW9Xnd7df2rjVu/3zONQnaYnNfLRfm8Mkg0ru2Q2bLb/UKMMvNcrwS42yz6+J2J9ffrb8UqMMv9Qowy81arF9/t273znvsFf+rsdax99X98cnvfBDm5stv9Qowy81yvBLjfKqviU3SZ9/ku0+jWVoNryqT9K6GH6pUYZfapThlxrlhT0CPBGoRbb8UqMMv9Qowy81yj5/A/qckONJO+2x5ZcaZfilRhl+qVH2+QduGsffvShHa7Hllxpl+KVGGX6pUfb5l4z9dU2LLb/UKMMvNcrwS40aG/4kdyc5meTpFfO2JTmY5Lnu5/mzLVPStPVp+b8FXL9q3h3AQ1V1OfBQN60lkWTsbaj61L4sz2XRxoa/qv4NeGXV7BuB/d39/cBNU65L0oxNeqhve1Wd6O6/BGx/twcm2QvsnXA9kmZkw8f5q6pWfxnHqt/vA/bBO7+0Q9LiTBr+l5PsqKoTSXYAJ6dZlN42jZN6+ixjWfrGm+m5LNqkh/ruB/Z09/cAB6ZTjqR5GftdfUm+DVwDXAi8DPwF8M/AfcAlwAvALVW1elBwrWX5XX0DtZku+91Mz6WPSb+rzy/qFLC5ArOZnksfk4bfC3sasJn6wJvpuSyap/dKjTL8UqMMv9Qowy81ygG/Bi3z6Pcy1z40tvxSowy/1CjDLzXKPv/ATXJSi/3i5bHIsxFt+aVGGX6pUYZfapR9/oHrccn1hpehNtnyS40y/FKjDL/UqE3Z52/tk1xWm8YHXkyyzWbxQRtDfu3Gvc8meR+O24bT3B62/FKjDL/UKMMvNcrwS43alAN+LRnygNiQaxuKaZzENSlbfqlRhl9qlOGXGmWfX734TTmbjy2/1CjDLzXK8EuNss+vXjxmv37T2Gaz3O62/FKjDL/UqLHhT3JxkkNJnk3yTJLbu/nbkhxM8lz38/zZlytpWtLj3OIdwI6qejzJ+4HHgJuAPwBeqaqvJLkDOL+qvjhmWWesbFb9mdY/zEPzsYhzH9Z6L6/xfu9V2NiWv6pOVNXj3f3XgSPARcCNwP7uYfsZ/UOQtCTW1edPsgv4GPAwsL2qTnS/egnYPtXKJM1U70N9Sd4HfBf4XFX9fOWuRlXV6l36FX+3F9i70UIlTdfYPj9AkrOAB4AHq+pr3byjwDVVdaIbF/hBVX1ozHLs82vT2PR9/oyWfBdw5HTwO/cDe7r7e4ADfVY4D1V1xk2ahdXvs3ncpqnPaP/VwL8DPwTe6mb/KaN+/33AJcALwC1V9cqYZc2l5ZdaMmnL32u3f1oMvzR9M9vtl7Q5GX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUYZfapThlxpl+KVGGX6pUWPDn+ScJI8keSrJM0m+1M2/NMnDSY4luTfJ2bMvV9K0pKp++QOSAOdW1RtJzgL+A7gd+Dzwvaq6J8nfAk9V1d+MWdYZKxu3bsFo86/PMm/XSZ7vOEPeHquf7yS1rrGMXhtxbMtfI290k2d1twKuBb7Tzd8P3NS3WEmL16vPn2RLkieBk8BB4MfAa1V1qnvIi8BF7/K3e5McTnJ4GgVLmo5e4a+qN6vqSmAncBXw4b4rqKp9VbW7qnZPWKOkGVjXaH9VvQYcAj4BnJdka/erncDxKdcmaYb6jPZ/IMl53f33AtcBRxj9E7i5e9ge4MCsitTbquqM22a3+vlOctPa+oz2f4TRgN4WRv8s7quqLye5DLgH2AY8Afx+Vf1izLIc7V+ncaPBa42OL/N2ncbo9zJZ5Gj/2PBPk+FfP8O/vM+lj0Ef6pO0OW0d/xBpcfqc9DPUvYOh78XY8kuNMvxSowy/1Cj7/AM3rp84tH7kRm225zNktvxSowy/1CjDLzXKPv8C9Tk7r7Uz/DQ/tvxSowy/1CjDLzXK8EuNMvxSowy/1CjDLzXK8EuN8iSfJTOLb7TRcIx7fad5Apctv9Qowy81yvBLjbLPv2S8aGdzWeTracsvNcrwS40y/FKj7PMPzCyO4w/lyyOGUseiDO0LSGz5pUYZfqlRhl9qlOGXGuWA3wK1NuDVmj6v7yIv1LLllxpl+KVG9Q5/ki1JnkjyQDd9aZKHkxxLcm+Ss2dXpqRpW0/LfztwZMX0V4GvV9UHgVeB29a78iRjb5q+WW13X7vl0iv8SXYCvwf8XTcd4FrgO91D9gM3zaJASbPRt+X/BvAF4K1u+gLgtao61U2/CFy01h8m2ZvkcJLDG6pU0lSNDX+STwEnq+qxSVZQVfuqandV7Z7k7yXNRp/j/J8EPp3kBuAc4NeAbwLnJdnatf47gePrXflax0HtK07f0I83t2zQH+ZRVXdW1c6q2gXcCny/qj4LHAJu7h62BzgwsyolTd1GjvN/Efh8kmOMxgDumk5JkuYh89ztSHLGyvrs9nsK7HxMY7uPW4av7WyssV179eE8w09q1OAv7JnkG0zWO3hlCzQbQx1EnKSuzbiXassvNcrwS40y/FKjBtfnn0W/adn6YoswjW20LNvZk55GbPmlRhl+qVGGX2rU4Pr80hBNMgYw9PMAbPmlRhl+qVGGX2qUfX5pDUPrn8+CLb/UKMMvNcrwS40y/FKjmhjwm+QDQbQY6/0osD6m8ZFks1rPItnyS40y/FKjDL/UqCb6/NPoN2oY5vVBHC184Ictv9Qowy81yvBLjWqizz/OtPpuszjOO4t+5TIdj57F819UX73Peuf52tjyS40y/FKjDL/UqE3Z5x/Xb1qmPu80al2m49HzeG0mOYY/SV3TeC672X3G9GEOb3iZp9nyS40y/FKjeu32J3keeB14EzhVVbuTbAPuBXYBzwO3VNWrsylT0rStp+X/raq6sqpOd0LuAB6qqsuBh7ppSUsiPQc/ngd2V9XPVsw7ClxTVSeS7AB+UFUfGrOc5Rlpk5ZUVfUa4e3b8hfwL0keS7K3m7e9qk50918Ctq+zRkkL1PdQ39VVdTzJrwMHk/zXyl9WVb1bq979s9i71u8kLU6v3f4z/iD5S+AN4A9xt18anL67/WNb/iTnAu+pqte7+78DfBm4H9gDfKX7eaDH+n4GvABc2N1fBstS67LUCctT67LUCW/X+ht9/2Bsy5/kMuCfusmtwD9U1V8luQC4D7iEUaBvqapXeq00ObziqMGgLUuty1InLE+ty1InTFbr2Ja/qn4CfHSN+f8N/PZ6ViZpODzDT2rUosK/b0HrncSy1LosdcLy1LosdcIEta57tF/S5uBuv9Qowy81aq7hT3J9kqNJjiUZ1IVASe5OcjLJ0yvmbUtyMMlz3c/zF1njaUkuTnIoybNJnklyezd/UPUmOSfJI0me6ur8Ujf/0iQPd++De5Ocvcg6V0qyJckTSR7opgdZa5Lnk/wwyZNJDnfz1vX6zy38SbYAfw38LnAF8JkkV8xr/T18C7h+1byhXrl4CviTqroC+DjwR922HFq9vwCuraqPAlcC1yf5OPBV4OtV9UHgVeC2Bda42u3AkRXTQ651Y1faVtVcbsAngAdXTN8J3Dmv9fescRfw9Irpo8CO7v4O4Oiia3yXug8A1w25XuBXgceB32R0JtrWtd4XC65xZxeaa4EHgAy41ueBC1fNW9frP8/d/ouAn66YfrGbN2SDv3IxyS7gY8DDDLDebjf6SeAkcBD4MfBaVZ3qHjKk98E3gC8Ab3XTFzDcWjd8pe2m/ADPWah69ysXFyXJ+4DvAp+rqp+v/NDJodRbVW8CVyY5j9Fp4h9ecElrSvIp4GRVPZbkmkXX08PEV9qeNs+W/zhw8Yrpnd28IXu5u2KR7ufJBdfz/5KcxSj4f19V3+tmD7beqnoNOMRo1/m8JKcbnqG8Dz4JfLr74Jp7GO36f5Nh1kpVHe9+nmT0T/Uq1vn6zzP8jwKXd6OnZwO3MroycMhOX7kI/a9cnLmMmvi7gCNV9bUVvxpUvUk+0LX4JHkvo3GJI4z+CdzcPWzhdQJU1Z1VtbOqdjF6b36/qj7LAGtNcm6S95++z+hK26dZ7+s/50GKG4AfMer3/dmiB01W1fZt4ATwv4z6drcx6vM9BDwH/CuwbdF1drVezajP95/Ak93thqHVC3wEeKKr82ngz7v5lwGPAMeAfwR+ZdHbdFXd1wAPDLXWrqanutszp7O03tff03ulRnmGn9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjfo/9sAmKLyLZWUAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# default hyperparameters\n",
        "nhid = 8\n",
        "target = -np.inf\n",
        "max_steps = int(2e3)\n",
        "pop_size = 64\n",
        "reps = 10\n",
        "test_reps = 10\n",
        "weight_decay = .005\n",
        "noise_std = 0\n",
        "sigma = 1"
      ],
      "metadata": {
        "id": "_w5siiv3MWzW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "env.action_space.n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ygD02XHlNcpU",
        "outputId": "3e58d7ce-15be-4e1c-d23d-c3917693ca1b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "task = Task('RandomMaze50by50-v1', nhid, max_steps, target, pop_size, reps, test_reps,\n",
        "        weight_decay, noise_std, sigma)\n",
        "\n",
        "logger = get_logger()\n",
        "\n",
        "p = mp.Process(target=multi_runs, args=(task,logger))\n",
        "\n",
        "p.start()\n",
        "p.join()"
      ],
      "metadata": {
        "id": "jk-LXmUsL3ot"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##sb approach\n"
      ],
      "metadata": {
        "id": "1BIi_R__YwW2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!apt install swig\n",
        "!pip install stable-baselines3[extra]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "L4Emka1FZFio",
        "outputId": "10317eb6-8f7c-478c-ec10-4db3fa48fdb9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  swig3.0\n",
            "Suggested packages:\n",
            "  swig-doc swig-examples swig3.0-examples swig3.0-doc\n",
            "The following NEW packages will be installed:\n",
            "  swig swig3.0\n",
            "0 upgraded, 2 newly installed, 0 to remove and 41 not upgraded.\n",
            "Need to get 1,100 kB of archives.\n",
            "After this operation, 5,822 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 swig3.0 amd64 3.0.12-1 [1,094 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 swig amd64 3.0.12-1 [6,460 B]\n",
            "Fetched 1,100 kB in 2s (492 kB/s)\n",
            "Selecting previously unselected package swig3.0.\n",
            "(Reading database ... 155514 files and directories currently installed.)\n",
            "Preparing to unpack .../swig3.0_3.0.12-1_amd64.deb ...\n",
            "Unpacking swig3.0 (3.0.12-1) ...\n",
            "Selecting previously unselected package swig.\n",
            "Preparing to unpack .../swig_3.0.12-1_amd64.deb ...\n",
            "Unpacking swig (3.0.12-1) ...\n",
            "Setting up swig3.0 (3.0.12-1) ...\n",
            "Setting up swig (3.0.12-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Collecting stable-baselines3[extra]\n",
            "  Downloading stable_baselines3-1.5.0-py3-none-any.whl (177 kB)\n",
            "\u001b[K     |████████████████████████████████| 177 kB 4.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.21.6)\n",
            "Collecting gym==0.21\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 46.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (3.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.5)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.0)\n",
            "Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.11.0+cu113)\n",
            "Collecting ale-py~=0.7.4\n",
            "  Downloading ale_py-0.7.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 33.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (2.8.0)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.1.2.30)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (5.4.8)\n",
            "Collecting autorom[accept-rom-license]~=0.4.2\n",
            "  Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: importlib_metadata>=4.8.1 in /usr/local/lib/python3.7/dist-packages (from gym==0.21->stable-baselines3[extra]) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from ale-py~=0.7.4->stable-baselines3[extra]) (5.7.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.23.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.64.0)\n",
            "Collecting AutoROM.accept-rom-license\n",
            "  Downloading AutoROM.accept-rom-license-0.4.2.tar.gz (9.8 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib_metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (4.2.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib_metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.8.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.17.3)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.3.6)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.37.1)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.44.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (57.4.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->stable-baselines3[extra]) (1.15.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (1.4.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (3.0.8)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3[extra]) (2022.1)\n",
            "Building wheels for collected packages: gym, AutoROM.accept-rom-license\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gym: filename=gym-0.21.0-py3-none-any.whl size=1616824 sha256=0a103495e119548a7ae8ed6a3823ec2ee7cbe34775f06a18972069e9aafade28\n",
            "  Stored in directory: /root/.cache/pip/wheels/76/ee/9c/36bfe3e079df99acf5ae57f4e3464ff2771b34447d6d2f2148\n",
            "  Building wheel for AutoROM.accept-rom-license (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.4.2-py3-none-any.whl size=441027 sha256=dbd7cc98d790861b050e4088821ab91109426087f4aaf887a016ae19d987a190\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/67/2e/6147e7912fe37f5408b80d07527dab807c1d25f5c403a9538a\n",
            "Successfully built gym AutoROM.accept-rom-license\n",
            "Installing collected packages: gym, AutoROM.accept-rom-license, autorom, stable-baselines3, ale-py\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.17.3\n",
            "    Uninstalling gym-0.17.3:\n",
            "      Successfully uninstalled gym-0.17.3\n",
            "Successfully installed AutoROM.accept-rom-license-0.4.2 ale-py-0.7.5 autorom-0.4.2 gym-0.21.0 stable-baselines3-1.5.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gym"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gym\n",
        "import numpy as np\n",
        "import cma\n",
        "from collections import OrderedDict\n",
        "from stable_baselines3 import A2C\n",
        "\n",
        "\n",
        "def flatten(params):\n",
        "    \"\"\"\n",
        "    :param params: (dict)\n",
        "    :return: (np.ndarray)\n",
        "    \"\"\"\n",
        "    params_ = []\n",
        "    for key in params.keys():\n",
        "        params_.append(params[key].flatten())\n",
        "    return np.concatenate(params_)\n",
        "\n",
        "def to_dict(flat_vec, params):\n",
        "    \"\"\"\n",
        "    :param flat_vec: (np.ndarray)\n",
        "    :param params: (OrderedDict)\n",
        "    :return: (OrderedDict)\n",
        "    \"\"\"\n",
        "    params_ = OrderedDict()\n",
        "    start_idx = 0\n",
        "    for key in params.keys():\n",
        "        n_elem = params[key].size\n",
        "        params_[key] = flat_vec[start_idx:start_idx + n_elem].reshape(params[key].shape)\n",
        "        start_idx += n_elem\n",
        "    return params_\n",
        "\n",
        "def filter_policy_params(params):\n",
        "    \"\"\"\n",
        "    Include only variables with \"/pi/\" (policy) or \"/shared\" (shared layers)\n",
        "    in their name: Only these ones affect the action.\n",
        "    :param params: (OrderedDict)\n",
        "    :return: (OrderedDict)\n",
        "    \"\"\"\n",
        "    return OrderedDict((key, value) for key, value in params.items()\n",
        "                        if (\"/pi/\" in key or \"/shared\" in key))\n",
        "\n",
        "\n",
        "def evaluate(env, model):\n",
        "    \"\"\"\n",
        "    Return mean fitness (negative sum of episodic rewards)\n",
        "    for given model.\n",
        "    :param env: (gym.Env)\n",
        "    :param model: (RL Model)\n",
        "    :return: (float)\n",
        "    \"\"\"\n",
        "    episode_rewards = []\n",
        "    for _ in range(10):\n",
        "        reward_sum = 0\n",
        "        done = False\n",
        "        obs = env.reset()\n",
        "        while not done:\n",
        "            action, _states = model.predict(obs)\n",
        "            obs, reward, done, info = env.step(action)\n",
        "            reward_sum += reward\n",
        "        episode_rewards.append(reward_sum)\n",
        "    return - np.mean(episode_rewards)\n",
        "\n"
      ],
      "metadata": {
        "id": "VcU2J6P5Y7Vn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3 import TD3\n",
        "from stable_baselines3.common.monitor import Monitor\n",
        "from stable_baselines3.common.results_plotter import load_results, ts2xy\n",
        "from stable_baselines3.common.noise import NormalActionNoise\n",
        "from stable_baselines3.common.callbacks import BaseCallback"
      ],
      "metadata": {
        "id": "e4aPDzL62VtO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SaveOnBestTrainingRewardCallback(BaseCallback):\n",
        "    \"\"\"\n",
        "    Callback for saving a model (the check is done every ``check_freq`` steps)\n",
        "    based on the training reward (in practice, we recommend using ``EvalCallback``).\n",
        "\n",
        "    :param check_freq: (int)\n",
        "    :param log_dir: (str) Path to the folder where the model will be saved.\n",
        "      It must contains the file created by the ``Monitor`` wrapper.\n",
        "    :param verbose: (int)\n",
        "    \"\"\"\n",
        "    def __init__(self, check_freq: int, log_dir: str, verbose=1):\n",
        "        super(SaveOnBestTrainingRewardCallback, self).__init__(verbose)\n",
        "        self.check_freq = check_freq\n",
        "        self.log_dir = log_dir\n",
        "        self.save_path = os.path.join(log_dir, 'best_model')\n",
        "        self.best_mean_reward = -np.inf\n",
        "\n",
        "    def _init_callback(self) -> None:\n",
        "        # Create folder if needed\n",
        "        if self.save_path is not None:\n",
        "            os.makedirs(self.save_path, exist_ok=True)\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        if self.n_calls % self.check_freq == 0:\n",
        "\n",
        "          # Retrieve training reward\n",
        "          x, y = ts2xy(load_results(self.log_dir), 'timesteps')\n",
        "          if len(x) > 0:\n",
        "              # Mean training reward over the last 100 episodes\n",
        "              mean_reward = np.mean(y[-100:])\n",
        "              if self.verbose > 0:\n",
        "                print(f\"Num timesteps: {self.num_timesteps}\")\n",
        "                print(f\"Best mean reward: {self.best_mean_reward:.2f} - Last mean reward per episode: {mean_reward:.2f}\")\n",
        "\n",
        "              # New best model, you could save the agent here\n",
        "              if mean_reward > self.best_mean_reward:\n",
        "                  self.best_mean_reward = mean_reward\n",
        "                  # Example for saving best model\n",
        "                  if self.verbose > 0:\n",
        "                    print(f\"Saving new best model to {self.save_path}.zip\")\n",
        "                  self.model.save(self.save_path)\n",
        "\n",
        "        return True"
      ],
      "metadata": {
        "id": "eLcwwZis2cSm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create log dir\n",
        "log_dir = \"/tmp/gym/\"\n",
        "os.makedirs(log_dir, exist_ok=True)\n"
      ],
      "metadata": {
        "id": "pcr9ueV52gtF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tb_dir =\"./50mazev1\""
      ],
      "metadata": {
        "id": "x-Ig8i6l62v2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Logs will be saved in log_dir/monitor.csv\n",
        "env = Monitor(env, log_dir)"
      ],
      "metadata": {
        "id": "dwbF-r-72pPu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "callback = SaveOnBestTrainingRewardCallback(check_freq=1000, log_dir=log_dir)"
      ],
      "metadata": {
        "id": "PEjcGZEv2uVA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create RL model with a small policy network\n",
        "model1 = A2C('MlpPolicy', env, ent_coef=0.0, learning_rate=0.001, policy_kwargs={'net_arch': [8]},  verbose=1, tensorboard_log=\"/tmp/50mazev0/\")\n",
        "\n",
        "# Use RL actor-critic policy gradient updates to\n",
        "# find good initial parameters\n",
        "model1.learn(total_timesteps=100000, callback=callback)\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "hUrdv53WZNQe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create RL model with a small policy network\n",
        "from stable_baselines3 import PPO\n",
        "model2 = PPO('MlpPolicy', env,  learning_rate=0.001, policy_kwargs={'net_arch': [8]},  verbose=1, tensorboard_log=\"/tmp/50mazev0/\")\n",
        "\n",
        "# Use RL actor-critic policy gradient updates to\n",
        "# find good initial parameters\n",
        "model2.learn(total_timesteps=100000, callback=callback)\n"
      ],
      "metadata": {
        "id": "ZYtubkEq_H9Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# find good initial parameters\n",
        "model2.learn(total_timesteps=100000, callback=callback)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aq3rjjbhH5E2",
        "outputId": "c7206fab-fab0-401d-e0de-038974a51b53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logging to /tmp/50mazev0/PPO_2\n",
            "Num timesteps: 648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -102.18\n",
            "Num timesteps: 1648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.19\n",
            "---------------------------------\n",
            "| rollout/           |          |\n",
            "|    ep_len_mean     | 1e+03    |\n",
            "|    ep_rew_mean     | -116     |\n",
            "| time/              |          |\n",
            "|    fps             | 518      |\n",
            "|    iterations      | 1        |\n",
            "|    time_elapsed    | 3        |\n",
            "|    total_timesteps | 2048     |\n",
            "---------------------------------\n",
            "Num timesteps: 2648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.60\n",
            "Num timesteps: 3648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.55\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 1e+03       |\n",
            "|    ep_rew_mean          | -131        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 460         |\n",
            "|    iterations           | 2           |\n",
            "|    time_elapsed         | 8           |\n",
            "|    total_timesteps      | 4096        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012944706 |\n",
            "|    clip_fraction        | 0.124       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.62        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.35        |\n",
            "|    n_updates            | 500         |\n",
            "|    policy_gradient_loss | -0.0122     |\n",
            "|    value_loss           | 4.16        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 4648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.46\n",
            "Num timesteps: 5648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.56\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 1e+03       |\n",
            "|    ep_rew_mean          | -127        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 518         |\n",
            "|    iterations           | 3           |\n",
            "|    time_elapsed         | 11          |\n",
            "|    total_timesteps      | 6144        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019240374 |\n",
            "|    clip_fraction        | 0.207       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.645       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.26        |\n",
            "|    n_updates            | 510         |\n",
            "|    policy_gradient_loss | -0.0223     |\n",
            "|    value_loss           | 3.97        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 6648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.60\n",
            "Num timesteps: 7648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.38\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 1e+03       |\n",
            "|    ep_rew_mean          | -128        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 550         |\n",
            "|    iterations           | 4           |\n",
            "|    time_elapsed         | 14          |\n",
            "|    total_timesteps      | 8192        |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015314907 |\n",
            "|    clip_fraction        | 0.109       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.811       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.02        |\n",
            "|    n_updates            | 520         |\n",
            "|    policy_gradient_loss | -0.0145     |\n",
            "|    value_loss           | 3.11        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 8648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.44\n",
            "Num timesteps: 9648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.89\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 990         |\n",
            "|    ep_rew_mean          | -134        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 577         |\n",
            "|    iterations           | 5           |\n",
            "|    time_elapsed         | 17          |\n",
            "|    total_timesteps      | 10240       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017339528 |\n",
            "|    clip_fraction        | 0.171       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.733       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 2           |\n",
            "|    n_updates            | 530         |\n",
            "|    policy_gradient_loss | -0.019      |\n",
            "|    value_loss           | 3.4         |\n",
            "-----------------------------------------\n",
            "Num timesteps: 10648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -101.37\n",
            "Num timesteps: 11648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.75\n",
            "------------------------------------------\n",
            "| rollout/                |              |\n",
            "|    ep_len_mean          | 992          |\n",
            "|    ep_rew_mean          | -124         |\n",
            "| time/                   |              |\n",
            "|    fps                  | 594          |\n",
            "|    iterations           | 6            |\n",
            "|    time_elapsed         | 20           |\n",
            "|    total_timesteps      | 12288        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0069120536 |\n",
            "|    clip_fraction        | 0.0279       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -1.38        |\n",
            "|    explained_variance   | 0.631        |\n",
            "|    learning_rate        | 0.001        |\n",
            "|    loss                 | 2            |\n",
            "|    n_updates            | 540          |\n",
            "|    policy_gradient_loss | -0.00605     |\n",
            "|    value_loss           | 7.17         |\n",
            "------------------------------------------\n",
            "Num timesteps: 12648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.94\n",
            "Num timesteps: 13648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.56\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 993        |\n",
            "|    ep_rew_mean          | -124       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 609        |\n",
            "|    iterations           | 7          |\n",
            "|    time_elapsed         | 23         |\n",
            "|    total_timesteps      | 14336      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01622931 |\n",
            "|    clip_fraction        | 0.225      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.37      |\n",
            "|    explained_variance   | 0.708      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 1.13       |\n",
            "|    n_updates            | 550        |\n",
            "|    policy_gradient_loss | -0.0215    |\n",
            "|    value_loss           | 3.29       |\n",
            "----------------------------------------\n",
            "Num timesteps: 14648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -101.19\n",
            "Num timesteps: 15648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.76\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 994         |\n",
            "|    ep_rew_mean          | -116        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 620         |\n",
            "|    iterations           | 8           |\n",
            "|    time_elapsed         | 26          |\n",
            "|    total_timesteps      | 16384       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013314471 |\n",
            "|    clip_fraction        | 0.123       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.783       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.947       |\n",
            "|    n_updates            | 560         |\n",
            "|    policy_gradient_loss | -0.0119     |\n",
            "|    value_loss           | 3.97        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 16648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.44\n",
            "Num timesteps: 17648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.98\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 994         |\n",
            "|    ep_rew_mean          | -116        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 625         |\n",
            "|    iterations           | 9           |\n",
            "|    time_elapsed         | 29          |\n",
            "|    total_timesteps      | 18432       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.011808393 |\n",
            "|    clip_fraction        | 0.24        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.35       |\n",
            "|    explained_variance   | 0.385       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.586       |\n",
            "|    n_updates            | 570         |\n",
            "|    policy_gradient_loss | -0.018      |\n",
            "|    value_loss           | 1.52        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 18648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.24\n",
            "Num timesteps: 19648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.83\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 993         |\n",
            "|    ep_rew_mean          | -114        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 629         |\n",
            "|    iterations           | 10          |\n",
            "|    time_elapsed         | 32          |\n",
            "|    total_timesteps      | 20480       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015338158 |\n",
            "|    clip_fraction        | 0.186       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.788       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.41        |\n",
            "|    n_updates            | 580         |\n",
            "|    policy_gradient_loss | -0.0219     |\n",
            "|    value_loss           | 3.29        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 20648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.32\n",
            "Num timesteps: 21648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.36\n",
            "------------------------------------------\n",
            "| rollout/                |              |\n",
            "|    ep_len_mean          | 993          |\n",
            "|    ep_rew_mean          | -112         |\n",
            "| time/                   |              |\n",
            "|    fps                  | 633          |\n",
            "|    iterations           | 11           |\n",
            "|    time_elapsed         | 35           |\n",
            "|    total_timesteps      | 22528        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0064544305 |\n",
            "|    clip_fraction        | 0.0325       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -1.37        |\n",
            "|    explained_variance   | 0.745        |\n",
            "|    learning_rate        | 0.001        |\n",
            "|    loss                 | 2.38         |\n",
            "|    n_updates            | 590          |\n",
            "|    policy_gradient_loss | -0.0047      |\n",
            "|    value_loss           | 3.79         |\n",
            "------------------------------------------\n",
            "Num timesteps: 22648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.75\n",
            "Num timesteps: 23648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.36\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 994         |\n",
            "|    ep_rew_mean          | -108        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 638         |\n",
            "|    iterations           | 12          |\n",
            "|    time_elapsed         | 38          |\n",
            "|    total_timesteps      | 24576       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.013369663 |\n",
            "|    clip_fraction        | 0.0452      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.803       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.08        |\n",
            "|    n_updates            | 600         |\n",
            "|    policy_gradient_loss | -0.00591    |\n",
            "|    value_loss           | 3.34        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 24648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.74\n",
            "Num timesteps: 25648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.97\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 994         |\n",
            "|    ep_rew_mean          | -107        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 644         |\n",
            "|    iterations           | 13          |\n",
            "|    time_elapsed         | 41          |\n",
            "|    total_timesteps      | 26624       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016837519 |\n",
            "|    clip_fraction        | 0.299       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.705       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.724       |\n",
            "|    n_updates            | 610         |\n",
            "|    policy_gradient_loss | -0.0293     |\n",
            "|    value_loss           | 2.02        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 26648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.39\n",
            "Num timesteps: 27648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -97.43\n",
            "Num timesteps: 28648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -95.73\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 995        |\n",
            "|    ep_rew_mean          | -106       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 648        |\n",
            "|    iterations           | 14         |\n",
            "|    time_elapsed         | 44         |\n",
            "|    total_timesteps      | 28672      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01866873 |\n",
            "|    clip_fraction        | 0.244      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.85       |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 1.78       |\n",
            "|    n_updates            | 620        |\n",
            "|    policy_gradient_loss | -0.0262    |\n",
            "|    value_loss           | 2.9        |\n",
            "----------------------------------------\n",
            "Num timesteps: 29648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -95.60\n",
            "Num timesteps: 30648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -96.59\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 995        |\n",
            "|    ep_rew_mean          | -106       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 652        |\n",
            "|    iterations           | 15         |\n",
            "|    time_elapsed         | 47         |\n",
            "|    total_timesteps      | 30720      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01967856 |\n",
            "|    clip_fraction        | 0.19       |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.678      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 2.21       |\n",
            "|    n_updates            | 630        |\n",
            "|    policy_gradient_loss | -0.019     |\n",
            "|    value_loss           | 3.69       |\n",
            "----------------------------------------\n",
            "Num timesteps: 31648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -96.36\n",
            "Num timesteps: 32648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -95.15\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 995         |\n",
            "|    ep_rew_mean          | -104        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 656         |\n",
            "|    iterations           | 16          |\n",
            "|    time_elapsed         | 49          |\n",
            "|    total_timesteps      | 32768       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014809424 |\n",
            "|    clip_fraction        | 0.191       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.78        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.58        |\n",
            "|    n_updates            | 640         |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    value_loss           | 4.03        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 33648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.09\n",
            "Num timesteps: 34648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -95.03\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 996         |\n",
            "|    ep_rew_mean          | -104        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 659         |\n",
            "|    iterations           | 17          |\n",
            "|    time_elapsed         | 52          |\n",
            "|    total_timesteps      | 34816       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014319622 |\n",
            "|    clip_fraction        | 0.134       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.831       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.55        |\n",
            "|    n_updates            | 650         |\n",
            "|    policy_gradient_loss | -0.02       |\n",
            "|    value_loss           | 1.84        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 35648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.99\n",
            "Num timesteps: 36648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.59\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 996         |\n",
            "|    ep_rew_mean          | -105        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 660         |\n",
            "|    iterations           | 18          |\n",
            "|    time_elapsed         | 55          |\n",
            "|    total_timesteps      | 36864       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015859632 |\n",
            "|    clip_fraction        | 0.279       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.38       |\n",
            "|    explained_variance   | 0.71        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.07        |\n",
            "|    n_updates            | 660         |\n",
            "|    policy_gradient_loss | -0.0219     |\n",
            "|    value_loss           | 3.91        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 37648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.97\n",
            "Num timesteps: 38648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.76\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 996         |\n",
            "|    ep_rew_mean          | -106        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 663         |\n",
            "|    iterations           | 19          |\n",
            "|    time_elapsed         | 58          |\n",
            "|    total_timesteps      | 38912       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017242748 |\n",
            "|    clip_fraction        | 0.194       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.77        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 2.08        |\n",
            "|    n_updates            | 670         |\n",
            "|    policy_gradient_loss | -0.0154     |\n",
            "|    value_loss           | 4.35        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 39648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.88\n",
            "Num timesteps: 40648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.29\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 996         |\n",
            "|    ep_rew_mean          | -105        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 666         |\n",
            "|    iterations           | 20          |\n",
            "|    time_elapsed         | 61          |\n",
            "|    total_timesteps      | 40960       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017397156 |\n",
            "|    clip_fraction        | 0.293       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.83        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.12        |\n",
            "|    n_updates            | 680         |\n",
            "|    policy_gradient_loss | -0.0304     |\n",
            "|    value_loss           | 3.85        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 41648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.82\n",
            "Num timesteps: 42648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.67\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 997        |\n",
            "|    ep_rew_mean          | -107       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 668        |\n",
            "|    iterations           | 21         |\n",
            "|    time_elapsed         | 64         |\n",
            "|    total_timesteps      | 43008      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01612708 |\n",
            "|    clip_fraction        | 0.152      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.838      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 0.822      |\n",
            "|    n_updates            | 690        |\n",
            "|    policy_gradient_loss | -0.0109    |\n",
            "|    value_loss           | 2.19       |\n",
            "----------------------------------------\n",
            "Num timesteps: 43648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -95.28\n",
            "Num timesteps: 44648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.45\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 997        |\n",
            "|    ep_rew_mean          | -105       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 669        |\n",
            "|    iterations           | 22         |\n",
            "|    time_elapsed         | 67         |\n",
            "|    total_timesteps      | 45056      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.02037263 |\n",
            "|    clip_fraction        | 0.428      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.37      |\n",
            "|    explained_variance   | 0.556      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 1.7        |\n",
            "|    n_updates            | 700        |\n",
            "|    policy_gradient_loss | -0.0425    |\n",
            "|    value_loss           | 4.92       |\n",
            "----------------------------------------\n",
            "Num timesteps: 45648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.09\n",
            "Num timesteps: 46648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -92.57\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -103        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 670         |\n",
            "|    iterations           | 23          |\n",
            "|    time_elapsed         | 70          |\n",
            "|    total_timesteps      | 47104       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014025909 |\n",
            "|    clip_fraction        | 0.0629      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.35       |\n",
            "|    explained_variance   | 0.586       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.49        |\n",
            "|    n_updates            | 710         |\n",
            "|    policy_gradient_loss | -0.00509    |\n",
            "|    value_loss           | 3.66        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 47648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -93.06\n",
            "Num timesteps: 48648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -94.34\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -107        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 670         |\n",
            "|    iterations           | 24          |\n",
            "|    time_elapsed         | 73          |\n",
            "|    total_timesteps      | 49152       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015294399 |\n",
            "|    clip_fraction        | 0.233       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.33       |\n",
            "|    explained_variance   | 0.525       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.08        |\n",
            "|    n_updates            | 720         |\n",
            "|    policy_gradient_loss | -0.0134     |\n",
            "|    value_loss           | 1.92        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 49648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -96.19\n",
            "Num timesteps: 50648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -96.42\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -108        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 670         |\n",
            "|    iterations           | 25          |\n",
            "|    time_elapsed         | 76          |\n",
            "|    total_timesteps      | 51200       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018358938 |\n",
            "|    clip_fraction        | 0.32        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.35       |\n",
            "|    explained_variance   | 0.717       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.11        |\n",
            "|    n_updates            | 730         |\n",
            "|    policy_gradient_loss | -0.0275     |\n",
            "|    value_loss           | 3.56        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 51648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -96.90\n",
            "Num timesteps: 52648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -97.08\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -109        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 670         |\n",
            "|    iterations           | 26          |\n",
            "|    time_elapsed         | 79          |\n",
            "|    total_timesteps      | 53248       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.008248855 |\n",
            "|    clip_fraction        | 0.0894      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.35       |\n",
            "|    explained_variance   | 0.772       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.35        |\n",
            "|    n_updates            | 740         |\n",
            "|    policy_gradient_loss | -0.00945    |\n",
            "|    value_loss           | 3.84        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 53648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.08\n",
            "Num timesteps: 54648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -97.78\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -108        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 671         |\n",
            "|    iterations           | 27          |\n",
            "|    time_elapsed         | 82          |\n",
            "|    total_timesteps      | 55296       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.015110781 |\n",
            "|    clip_fraction        | 0.132       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.762       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 2.74        |\n",
            "|    n_updates            | 750         |\n",
            "|    policy_gradient_loss | -0.0119     |\n",
            "|    value_loss           | 5.92        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 55648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -97.81\n",
            "Num timesteps: 56648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.34\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 997         |\n",
            "|    ep_rew_mean          | -109        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 669         |\n",
            "|    iterations           | 28          |\n",
            "|    time_elapsed         | 85          |\n",
            "|    total_timesteps      | 57344       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017066907 |\n",
            "|    clip_fraction        | 0.126       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.598       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.57        |\n",
            "|    n_updates            | 760         |\n",
            "|    policy_gradient_loss | -0.0149     |\n",
            "|    value_loss           | 3.58        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 57648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.28\n",
            "Num timesteps: 58648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.94\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -109        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 669         |\n",
            "|    iterations           | 29          |\n",
            "|    time_elapsed         | 88          |\n",
            "|    total_timesteps      | 59392       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017459905 |\n",
            "|    clip_fraction        | 0.394       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.579       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.45        |\n",
            "|    n_updates            | 770         |\n",
            "|    policy_gradient_loss | -0.04       |\n",
            "|    value_loss           | 3.39        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 59648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.47\n",
            "Num timesteps: 60648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -98.81\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 998        |\n",
            "|    ep_rew_mean          | -109       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 668        |\n",
            "|    iterations           | 30         |\n",
            "|    time_elapsed         | 91         |\n",
            "|    total_timesteps      | 61440      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01779477 |\n",
            "|    clip_fraction        | 0.326      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.537      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 0.726      |\n",
            "|    n_updates            | 780        |\n",
            "|    policy_gradient_loss | -0.0303    |\n",
            "|    value_loss           | 1.74       |\n",
            "----------------------------------------\n",
            "Num timesteps: 61648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.49\n",
            "Num timesteps: 62648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -99.35\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 998        |\n",
            "|    ep_rew_mean          | -110       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 666        |\n",
            "|    iterations           | 31         |\n",
            "|    time_elapsed         | 95         |\n",
            "|    total_timesteps      | 63488      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01758914 |\n",
            "|    clip_fraction        | 0.17       |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.738      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 1.58       |\n",
            "|    n_updates            | 790        |\n",
            "|    policy_gradient_loss | -0.0204    |\n",
            "|    value_loss           | 3.38       |\n",
            "----------------------------------------\n",
            "Num timesteps: 63648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.16\n",
            "Num timesteps: 64648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -100.87\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -112        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 666         |\n",
            "|    iterations           | 32          |\n",
            "|    time_elapsed         | 98          |\n",
            "|    total_timesteps      | 65536       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017758168 |\n",
            "|    clip_fraction        | 0.153       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.794       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.98        |\n",
            "|    n_updates            | 800         |\n",
            "|    policy_gradient_loss | -0.00965    |\n",
            "|    value_loss           | 4.12        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 65648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -101.76\n",
            "Num timesteps: 66648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -102.00\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 998        |\n",
            "|    ep_rew_mean          | -111       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 666        |\n",
            "|    iterations           | 33         |\n",
            "|    time_elapsed         | 101        |\n",
            "|    total_timesteps      | 67584      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01698809 |\n",
            "|    clip_fraction        | 0.279      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.37      |\n",
            "|    explained_variance   | 0.786      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 2.25       |\n",
            "|    n_updates            | 810        |\n",
            "|    policy_gradient_loss | -0.0274    |\n",
            "|    value_loss           | 4.39       |\n",
            "----------------------------------------\n",
            "Num timesteps: 67648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -101.40\n",
            "Num timesteps: 68648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -103.25\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -114        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 663         |\n",
            "|    iterations           | 34          |\n",
            "|    time_elapsed         | 104         |\n",
            "|    total_timesteps      | 69632       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017778508 |\n",
            "|    clip_fraction        | 0.252       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.845       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.803       |\n",
            "|    n_updates            | 820         |\n",
            "|    policy_gradient_loss | -0.0147     |\n",
            "|    value_loss           | 3.22        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 69648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -103.70\n",
            "Num timesteps: 70648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -104.06\n",
            "Num timesteps: 71648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -103.39\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -113        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 659         |\n",
            "|    iterations           | 35          |\n",
            "|    time_elapsed         | 108         |\n",
            "|    total_timesteps      | 71680       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016211322 |\n",
            "|    clip_fraction        | 0.344       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.808       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 3.05        |\n",
            "|    n_updates            | 830         |\n",
            "|    policy_gradient_loss | -0.0338     |\n",
            "|    value_loss           | 6.36        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 72648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -104.54\n",
            "Num timesteps: 73648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -104.67\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -114        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 655         |\n",
            "|    iterations           | 36          |\n",
            "|    time_elapsed         | 112         |\n",
            "|    total_timesteps      | 73728       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017687947 |\n",
            "|    clip_fraction        | 0.205       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.842       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.499       |\n",
            "|    n_updates            | 840         |\n",
            "|    policy_gradient_loss | -0.0126     |\n",
            "|    value_loss           | 1.94        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 74648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.88\n",
            "Num timesteps: 75648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.96\n",
            "------------------------------------------\n",
            "| rollout/                |              |\n",
            "|    ep_len_mean          | 998          |\n",
            "|    ep_rew_mean          | -115         |\n",
            "| time/                   |              |\n",
            "|    fps                  | 653          |\n",
            "|    iterations           | 37           |\n",
            "|    time_elapsed         | 115          |\n",
            "|    total_timesteps      | 75776        |\n",
            "| train/                  |              |\n",
            "|    approx_kl            | 0.0134376045 |\n",
            "|    clip_fraction        | 0.0729       |\n",
            "|    clip_range           | 0.2          |\n",
            "|    entropy_loss         | -1.37        |\n",
            "|    explained_variance   | 0.768        |\n",
            "|    learning_rate        | 0.001        |\n",
            "|    loss                 | 1.77         |\n",
            "|    n_updates            | 850          |\n",
            "|    policy_gradient_loss | -0.0105      |\n",
            "|    value_loss           | 5.42         |\n",
            "------------------------------------------\n",
            "Num timesteps: 76648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.94\n",
            "Num timesteps: 77648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.38\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -115        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 651         |\n",
            "|    iterations           | 38          |\n",
            "|    time_elapsed         | 119         |\n",
            "|    total_timesteps      | 77824       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019515056 |\n",
            "|    clip_fraction        | 0.262       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.675       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.494       |\n",
            "|    n_updates            | 860         |\n",
            "|    policy_gradient_loss | -0.0139     |\n",
            "|    value_loss           | 2.61        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 78648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.46\n",
            "Num timesteps: 79648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.24\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -114        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 650         |\n",
            "|    iterations           | 39          |\n",
            "|    time_elapsed         | 122         |\n",
            "|    total_timesteps      | 79872       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016130228 |\n",
            "|    clip_fraction        | 0.31        |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.38       |\n",
            "|    explained_variance   | 0.614       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.45        |\n",
            "|    n_updates            | 870         |\n",
            "|    policy_gradient_loss | -0.0271     |\n",
            "|    value_loss           | 3.15        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 80648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.89\n",
            "Num timesteps: 81648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.45\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -112        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 649         |\n",
            "|    iterations           | 40          |\n",
            "|    time_elapsed         | 126         |\n",
            "|    total_timesteps      | 81920       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.007399334 |\n",
            "|    clip_fraction        | 0.0115      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.37       |\n",
            "|    explained_variance   | 0.824       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.905       |\n",
            "|    n_updates            | 880         |\n",
            "|    policy_gradient_loss | -0.00238    |\n",
            "|    value_loss           | 2.53        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 82648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.37\n",
            "Num timesteps: 83648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.83\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 998        |\n",
            "|    ep_rew_mean          | -112       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 649        |\n",
            "|    iterations           | 41         |\n",
            "|    time_elapsed         | 129        |\n",
            "|    total_timesteps      | 83968      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01398185 |\n",
            "|    clip_fraction        | 0.308      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.35      |\n",
            "|    explained_variance   | 0.629      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 0.75       |\n",
            "|    n_updates            | 890        |\n",
            "|    policy_gradient_loss | -0.0196    |\n",
            "|    value_loss           | 1.44       |\n",
            "----------------------------------------\n",
            "Num timesteps: 84648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -105.82\n",
            "Num timesteps: 85648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.15\n",
            "----------------------------------------\n",
            "| rollout/                |            |\n",
            "|    ep_len_mean          | 998        |\n",
            "|    ep_rew_mean          | -111       |\n",
            "| time/                   |            |\n",
            "|    fps                  | 648        |\n",
            "|    iterations           | 42         |\n",
            "|    time_elapsed         | 132        |\n",
            "|    total_timesteps      | 86016      |\n",
            "| train/                  |            |\n",
            "|    approx_kl            | 0.01925268 |\n",
            "|    clip_fraction        | 0.394      |\n",
            "|    clip_range           | 0.2        |\n",
            "|    entropy_loss         | -1.36      |\n",
            "|    explained_variance   | 0.737      |\n",
            "|    learning_rate        | 0.001      |\n",
            "|    loss                 | 1.04       |\n",
            "|    n_updates            | 900        |\n",
            "|    policy_gradient_loss | -0.0331    |\n",
            "|    value_loss           | 2.28       |\n",
            "----------------------------------------\n",
            "Num timesteps: 86648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.03\n",
            "Num timesteps: 87648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.47\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -111        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 648         |\n",
            "|    iterations           | 43          |\n",
            "|    time_elapsed         | 135         |\n",
            "|    total_timesteps      | 88064       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.017388377 |\n",
            "|    clip_fraction        | 0.368       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.676       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.469       |\n",
            "|    n_updates            | 910         |\n",
            "|    policy_gradient_loss | -0.034      |\n",
            "|    value_loss           | 1.67        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 88648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.55\n",
            "Num timesteps: 89648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.53\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -111        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 648         |\n",
            "|    iterations           | 44          |\n",
            "|    time_elapsed         | 139         |\n",
            "|    total_timesteps      | 90112       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.020832378 |\n",
            "|    clip_fraction        | 0.423       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.643       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.49        |\n",
            "|    n_updates            | 920         |\n",
            "|    policy_gradient_loss | -0.0366     |\n",
            "|    value_loss           | 3.15        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 90648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -107.65\n",
            "Num timesteps: 91648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -107.21\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -109        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 648         |\n",
            "|    iterations           | 45          |\n",
            "|    time_elapsed         | 142         |\n",
            "|    total_timesteps      | 92160       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.018913835 |\n",
            "|    clip_fraction        | 0.361       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.676       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.53        |\n",
            "|    n_updates            | 930         |\n",
            "|    policy_gradient_loss | -0.0337     |\n",
            "|    value_loss           | 3.3         |\n",
            "-----------------------------------------\n",
            "Num timesteps: 92648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -106.57\n",
            "Num timesteps: 93648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -107.41\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -110        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 648         |\n",
            "|    iterations           | 46          |\n",
            "|    time_elapsed         | 145         |\n",
            "|    total_timesteps      | 94208       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.012848281 |\n",
            "|    clip_fraction        | 0.0757      |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.453       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 0.49        |\n",
            "|    n_updates            | 940         |\n",
            "|    policy_gradient_loss | -0.00416    |\n",
            "|    value_loss           | 1.81        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 94648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -108.09\n",
            "Num timesteps: 95648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -107.67\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -109        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 648         |\n",
            "|    iterations           | 47          |\n",
            "|    time_elapsed         | 148         |\n",
            "|    total_timesteps      | 96256       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.019080725 |\n",
            "|    clip_fraction        | 0.332       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.733       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.15        |\n",
            "|    n_updates            | 950         |\n",
            "|    policy_gradient_loss | -0.0327     |\n",
            "|    value_loss           | 3.31        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 96648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -108.01\n",
            "Num timesteps: 97648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -108.70\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 998         |\n",
            "|    ep_rew_mean          | -110        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 647         |\n",
            "|    iterations           | 48          |\n",
            "|    time_elapsed         | 151         |\n",
            "|    total_timesteps      | 98304       |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.016355377 |\n",
            "|    clip_fraction        | 0.122       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.35       |\n",
            "|    explained_variance   | 0.46        |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.22        |\n",
            "|    n_updates            | 960         |\n",
            "|    policy_gradient_loss | -0.00743    |\n",
            "|    value_loss           | 2.39        |\n",
            "-----------------------------------------\n",
            "Num timesteps: 98648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -109.92\n",
            "Num timesteps: 99648\n",
            "Best mean reward: -67.60 - Last mean reward per episode: -110.21\n",
            "-----------------------------------------\n",
            "| rollout/                |             |\n",
            "|    ep_len_mean          | 999         |\n",
            "|    ep_rew_mean          | -110        |\n",
            "| time/                   |             |\n",
            "|    fps                  | 647         |\n",
            "|    iterations           | 49          |\n",
            "|    time_elapsed         | 154         |\n",
            "|    total_timesteps      | 100352      |\n",
            "| train/                  |             |\n",
            "|    approx_kl            | 0.014357025 |\n",
            "|    clip_fraction        | 0.111       |\n",
            "|    clip_range           | 0.2         |\n",
            "|    entropy_loss         | -1.36       |\n",
            "|    explained_variance   | 0.672       |\n",
            "|    learning_rate        | 0.001       |\n",
            "|    loss                 | 1.32        |\n",
            "|    n_updates            | 970         |\n",
            "|    policy_gradient_loss | -0.0102     |\n",
            "|    value_loss           | 3.22        |\n",
            "-----------------------------------------\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.ppo.ppo.PPO at 0x7f2f9cee1310>"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /tmp/50mazev0/"
      ],
      "metadata": {
        "id": "sObkkTqU7aWY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def moving_average(values, window):\n",
        "    \"\"\"\n",
        "    Smooth values by doing a moving average\n",
        "    :param values: (numpy array)\n",
        "    :param window: (int)\n",
        "    :return: (numpy array)\n",
        "    \"\"\"\n",
        "    weights = np.repeat(1.0, window) / window\n",
        "    return np.convolve(values, weights, 'valid')\n",
        "\n",
        "\n",
        "def plot_results(log_folder, title='Learning Curve'):\n",
        "    \"\"\"\n",
        "    plot the results\n",
        "\n",
        "    :param log_folder: (str) the save location of the results to plot\n",
        "    :param title: (str) the title of the task to plot\n",
        "    \"\"\"\n",
        "    x, y = ts2xy(load_results(log_folder), 'timesteps')\n",
        "    y = moving_average(y, window=50)\n",
        "    # Truncate x\n",
        "    x = x[len(x) - len(y):]\n",
        "\n",
        "    fig = plt.figure(title)\n",
        "    plt.plot(x, y)\n",
        "    plt.xlabel('Number of Timesteps')\n",
        "    plt.ylabel('Rewards')\n",
        "    plt.title(title + \" Smoothed\")\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "jfopFO0B4ox5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_results(log_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "UnobnOoA0hDf",
        "outputId": "987928d3-2841-4db7-fb08-c388fe704ba5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hc5ZX48e9R771axb1XbGPTSzCYEjoESDa0JKSQhJDfJoHdZDfZhCSksrBJgBBISKND6GCqaQZs427Lkots9WL1Ls35/XGv5LEtyZKl0Yyk83meeTzz3jsz52pgzrxdVBVjjDHmWAX5OwBjjDGjmyUSY4wxQ2KJxBhjzJBYIjHGGDMklkiMMcYMiSUSY4wxQ2KJxIwZInKqiOT5Ow5zkIjsFZEVw/RafxaRnwzHa5nhZYnEDIvh/MI4Vqr6jqrO9NXri8hKEVktIg0iUikib4vIRb56v0HEFSYivxaRIhFpdD+Lu/wQh33Rj1OWSMyoISLBfnzvK4DHgYeBbCAd+C/gwmN4LRGR4fx/73ZgKbAMiAXOANYP4+sb0y9LJManRCRIRG4TkV0iUi0ij4lIktfxx0WkTETq3F/7c72O/VlE/iAiL4pIE3Cm+2v730Vkk/ucR0Ukwj3/DBEp8np+n+e6x78rIqUiUiIiXxQRFZFpvVyDAL8BfqyqD6hqnap6VPVtVf2Se84PReRvXs+Z5L5eiPv4LRG5Q0TeA5qB74jI2sPe51YReda9Hy4ivxKRfSJSLiL3ikhkH3/m44GnVbVEHXtV9eHD/g7fcf8OTSLyJxFJF5GX3NrVayKS6HX+RSKyVURq3bhnex2b7ZbVuudc5JbfBHwO+K5bK3rOK75F/XwGnxaRDe7rvS8iC7yOHSci690YHwUiMIFJVe1mtyHfgL3Ail7KbwHW4PyKDwfuA/7pdfxGnF/R4cBdwAavY38G6oCTcX70RLjv8xEwAUgCtgNfcc8/Ayg6LKa+zj0XKAPmAlHA3wAFpvVyDbPcY5P7uf4fAn/zejzJfU6I+/gtYJ/7fiFAPNAATPd6zsfA1e793wLPunHHAs8BP+vjvb/vvvbXgPmA9PLZrMGpRWUBFTg1luPcv+kbwH+7584AmoCzgVDgu0ABEOY+LgD+w338KfcaZnp9Xj/p5b37+gyOc2NZDgQD17nnh7uvXwjc6r7vFUDH4a9vt8C4WY3E+NpXgP9U1SJVbcP5wr2i+5e6qj6oqg1exxaKSLzX8/+lqu+pUwNodcvuVufX9wGcL9hF/bx/X+d+BnhIVbeqarP73n1Jdv8tHehF9+HP7vt1qmod8C/gGgARmY6TsJ51a0A3Abeq6gFVbQB+Clzdx+v+DLgTp0awFigWkesOO+ceVS1X1WLgHeBDVf3E/Zs+jfOlDnAV8IKqrlLVDuBXQCRwEnACEAP8XFXbVfUN4Pnua+hHX5/BTcB9qvqhqnap6l+ANvd9TsBJIHepaoeqPoGTaE0AskRifG0i8LTbdFGL84u0C0gXkWAR+bnb7FWP82sUIMXr+ft7ec0yr/vNOF9ufenr3AmHvXZv79Ot2v03s59zBuLw9/gHB7+EPws84ya1VJxa0jqvv9vLbvkR3C/h36nqyUACcAfwoHeTFFDudb+ll8fef5dCr9f2uHFnucf2u2XdCt1j/enrM5gI/L/ua3SvM8d9nwlAsap6rypbiAlIlkiMr+0HzlPVBK9bhPvL+LPAxcAKnKaeSe5zxOv5vlqeuhSnua1bTj/n5uFcx+X9nNOE8+XfLaOXcw6/llVAqogswkko/3DLq3C+3Od6/c3iVbW/hOm8gWqLqv4OqAHmHO38XpTgfMEDPf1DOUCxeyznsIECue4xGPxntR+447D/NqJU9Z84n0+W+/7e72UCkCUSM5xCRSTC6xYC3AvcISITAUQkVUQuds+PxWnKqMb5Ev7pCMb6GHCD23kcBfygrxPdX8XfBn4gIjeISJw7iOAUEbnfPW0DcJqI5LpNc7cfLQC36ehx4Jc4/Qer3HIP8EfgtyKSBiAiWSKysrfXEZFvuQMNIkUkxG3WigU+GdBf4lCPAReIyFkiEgr8P5zP6H3gQ5waxXdFJFREzsAZtfaI+9xyYMog3uuPwFdEZLk4okXkAhGJBT4AOoFvuu91Gc6oNBOALJGY4fQizi/p7tsPgf/F6TR+VUQacDp9l7vnP4zTXFEMbHOPjQhVfQm4G3gTpwO5+73b+jj/CZz+gxtxfpmXAz/B6edAVVcBjwKbgHU4fQcD8Q+cGtnjqtrpVf697rjcZr/XgL7myDQDv8ZpQqoCbgYuV9XdA4yhh6rmAf8G3OO+1oXAhW6fSLv7+Dz32O+Ba1V1h/v0PwFz3GaqZwbwXmuBLwH/h1ODKgCud4+1A5e5jw/g/O2fGuz1mJEhhzZBGjM+uf0JW4Dww77QjTFHYTUSM26JyKXufI1EnFFPz1kSMWbwLJGY8ezLOPMYduGMJPuqf8MxZnSypi1jjDFDEnA1EnepiWJ32YQNInK+17HbRaRARPL6GsFijDFmZIX4O4A+/FZVf+VdICJzcGb2zsWZrPSaiMxQ1a7+XiglJUUnTZrks0CNMWasWbduXZWq9joBtjeBmkh6czHwiLuUxh4RKcAZV/5Bf0+aNGkSa9eu7e8UY4wxXkRkUKsIBFzTluvr7mqhD3qtSprFoUtMFNHH0gwicpOIrBWRtZWVlb6O1RhjxjW/JBJ32eotvdwuBv4ATMVZ2K0UZ6LVoKjq/aq6VFWXpqYOuHZmjDHmGPilaUtVB7STnoj8kYMzhIs5dD2kbA6u8WOMMcZPAq5pS0S8V1i9FGe2MTjLbFztTiCbDEzH2efAGGOMHwViZ/sv3NVQFWdZ8S8DqOpWEXkMZ02mTuDmo43YMsYY43sBl0hU9fP9HLsDZ68FY4wxASLgmraMMcaMLpZIzFGpKo98tI+apnZ/h2KMCUCWSMxRFVQ0cttTmznlzjfo8tjabMaYQ1kiMUdV2ejs9dTU3sXfPyzkk301bCup93NUxphAEXCd7Wbk1TV3sLWkjjkT4kiICjvi+AG3SSs5OoxfvpxHS0cXsREhvHrr6aTGho90uMaYAGM1knGusLqJU+58g88+8CEX3P0u+eUNFFY3cf7/vsOHu6upa+7oSSR3Xb2IpvZOpqXF0NTexXUPfsR//2sLrR02CtuY8cwSyTj31PpiGts7+dWVC2nr9PDdJzfxbkEV20rruer+NZz489epanQSyYlTknnpltN48qsncccl86hv7eAvHxSyeqetZ2bMeGaJZBwqrG6ivL4VVeW5jSUsn5zEFUuyuXxJFluL69lcVEd0WDAAze1dlNW1kBAVSkhwEDMzYokOD+HKpTm8+e9nEBMewpt5lkiMGc+sj2Sc6ejycMW9H9Da0cU1y3LZXdXEF0+dAsD8rHjauzy8tKWMORPiuOr4XP798Y1sLaknOfrIvpPQ4CBOnZ7CW3kVqCoiMtKXY4wJAFYjGWfeyquksqGNmPAQ7l+9m4y4CM6fnwE4iQSgrqWDWRlxTEiIACCvrIHk6N471c+cmUZpXSs7yhpG5gKMMQHHaiTjRJdHeSe/kr+uKSQlJpy3v3MmTW2dJESF9tQkcpOiiIsIob61k1mZsWQlRALQ6VGSeqmRAJw6IwWANburmZ0ZNzIXY4wJKFYjGSee21jC9Q99zOqdlVyyaAJhIUEkRocd0hwlIsxzayWzMuLIiI/oOZYU03siyYiLIDEqlJ3lViMxZryyRDJOvLi5lPS4cO66ahHfOGt6n+ctykkgJEiYmRFLeEhwzzyR3vpIwEk+09Nj2Vne6JO4jTGBzxLJONDS3sXq/EpWzs3gkuOyiI8M7fPcL58+lce/ciIx4U6rZ3fzVl+JBGBmeiw7yxpQteVTjBmPLJGMA2/sqKC1w8PKuRlHPTc+MpTjchN7HncnkqSYvmewz0iPoaGtk7L61qEHa4wZdSyRjHGrd1by3Sc2kp0YybLJSYN+fvfIrf5qJDPSYwFndJcxZvyxRDLGlNe38ptX83h5SykdXR6+9+QmshIjeeIrJxEaPPiPe0J3jWQAiSTf+kmMGZds+O8Y8+yGEu5+owCAE6YkUVrXyn9fuPiQEViDsWJ2OjtKG5iaGtPnOYnRYWTERfDy1jJuPGUywUE2MdGY8SQgayQi8g0R2SEiW0XkF17lt4tIgYjkichKf8YYqMrrW4kMDebChRNYs/sAqbHhnDU7/ZhfLycpijuvWEBYSP//qfz7ypmsK6zh/9wkZowZPwKuRiIiZwIXAwtVtU1E0tzyOcDVwFxgAvCaiMxQVVt61kt5QxvpceH88MI5fLznAJ8/ceIxNWkN1uWLs3g3v5L/fX0nJ05NPqb+GGPM6BSINZKvAj9X1TYAVa1wyy8GHlHVNlXdAxQAy/wUY8CqqG8lLTaC5Jhw3rvtU9x85rQReV8R4SeXzicnKYpvPfIJtc22La8x40UgJpIZwKki8qGIvC0ix7vlWcB+r/OK3LIjiMhNIrJWRNZWVo6vlWkrGtpIi3OG6o50X0VMeAj3XHMclY1tfPeJTTavxJhxwi+JREReE5EtvdwuxmluSwJOAL4DPCaDXFZWVe9X1aWqujQ1NdUHVxC4umsk/rIgO4HvnTuLV7eV8+T6Yr/FYYwZOX7pI1HVFX0dE5GvAk+p83P2IxHxAClAMZDjdWq2W2ZcjW2dNLV39dRI/OXGkyfz/KZSfv7Sds6Zm05cRN8z6Y0xo18gNm09A5wJICIzgDCgCngWuFpEwkVkMjAd+MhvUQagCndmebqfE0lQkPDji+dR3dTOb1ft7PM8a/oyZmwIxETyIDBFRLYAjwDXqWMr8BiwDXgZuNlGbB2qoqENwK9NW93mZ8dzzbJcHv6gkB1l9Ucc/8XLO/jMfR/4ITJjzHALuESiqu2q+m+qOk9VF6vqG17H7lDVqao6U1Vf8mecgag8QGok3b5zzkxiI0L40bPbjqh9bNhfy7rCGlo77LeAMaNdwCUSc+wq3RpJagDUSMCZ8X7rihl8sLuatw7b172ktgWPQkGFLatizGhniWQMKa9vJTwkiLiIwJlnes2yXCYlR/HjF7ZRVufUmFSVEve+bdFrzOhniWSMKK1r4dVt5eQkRTHI0dI+FRYSxE8vnU9ZXSufvuddqhrbqG5qp73TA0BeL/0nxpjRxRLJGPG1v6+nurGdn18239+hHOGkaSn87YvLqWps46XNpZTUtvQcsxqJMaOfJZIxoLWji437a7nx5EksnRSYa1wdl5PAlJRoXtla3pNIZmXE2l7vxowBlkjGgPzyRjwKszPj/B1Kn0SEc+ZmsGZ3NdtLneRx5qw0yuvbegYJGGNGJ0skY8B2t59hVgAnEoBz5qbT6VEe+XgfEaFBXDA/E4A3d1Qc5ZnGmEBmiWQM2FHaQGRoMLlJUf4OpV+LshOYlRFLeX0bExIimTshjqyESF7ZWubv0IwxQ2CJZAzYUVbPjIzYgN+ZMChIuPXsGQBkJUS6zV3pvFNQRVNbp5+jM8YcK0sko1hxbQs3/2M9m4vqmJ0R6+9wBuScOemsmJ3GqdNTAFg5N4P2Tg8vbC71c2TGmGMVODPXzKD988N9vLDJ+QKenx3v52gGRkR44Lrjex4vm5TEnMw4fvdmAZcelzUiuzkaY4aX/V87ir20pZQTpiTx/DdO4TNLc47+hAAUFCR8++wZFFY38/jaIn+HY4w5BpZIRqn88gZ2VTZx/vxM5mXFj+pf8mfNTmPZ5CR+9tJ2Sutajv4EY0xAGb3fPuNcd5/CyrkZfo5k6ESEX1y+gM4u5Y4Xtvs7HGPMIFkiGYW6PMrja4s4eVoy6XGBsdLvUE1KiebiRRN4J78Kj8c2vDJmNLFEMgq9lVdBcW0L/7Z8or9DGVaLcxOpa+lgd1WTv0MxxgyCJZJRprqxjd+s2klabDgr5qT7O5xhtXhiAgCf7KvxcyTGmMGwRDKKtHZ0ccW9H1BQ0ciPL5k3qjvYezMlJYa4iBDW76v1dyjGmEEIuG8iEXlURDa4t70issHr2O0iUiAieSKy0p9x+sPja/ezp6qJez+/ZEx0sh8uKEhYlJtoNRJjRpmAm5Coqld13xeRXwN17v05wNXAXGAC8JqIzFDVMbvp95/e3cN7BVUA5CRG8srWcpZMTOSMGal+jsx3luQmctfrO6ltbichKszf4RhjBiDgaiTdxNnm7zPAP92ii4FHVLVNVfcABcAyf8Xnawea2vnpi9vZWd5AaV0rT60vpryhlW+fPSOgdkAcbidPS0YV3t9V7e9QjDEDFHA1Ei+nAuWqmu8+zgLWeB0vcsuOICI3ATcB5Obm+jJGn1m1rYwuj3Lvvy1hXlY8qkp7l4fwkGB/h+ZTC3MSiA0P4Z38Ss53l5k3xgQ2v9RIROQ1EdnSy+1ir9Ou4WBtZFBU9X5VXaqqS1NTR2cz0EtbyshJcpZaB2fS3lhPIgChwUGcODWZ1TurULX5JMaMBn5JJKq6QlXn9XL7F4CIhACXAY96Pa0Y8F5QKtstG3MaWjt4r6CK8+dljulmrL6cOj2F4toWPn3Pu6y3jndjAl6g9pGsAHaoqvcqfs8CV4tIuIhMBqYDH/klOh/bVlJPR5dywtRkf4fiFxctzOLaEydSXt/Gj57dajUTYwJcoCaSqzmsWUtVtwKPAduAl4Gbx+qIrR1lzp7mszMCe+tcX4mPCuV/Lp7Ht8+ewcaiOt7Jr/J3SMaYfgRkIlHV61X13l7K71DVqao6U1Vf8kdsI2FHWQPxkaGkx4X7OxS/unxJFhPiI7jz5R10dnn8HY4xpg8BmUjGux1l9czKiB2X/SPewkOC+c8L5rC1pJ6/fFDo73CMMX2wRBJgPB5lZ1kDs0bJ1rm+dv78DE6fkcpdq5xJisaYwGOJJMAU1bTQ1N7FrMzx2T9yOBHh9vNn0dDWyQPv7PF3OMaYXlgiCQD55Q20tDvjBraV1gMw02okPWZlxHHBgkweem8PB5qsVmJMoLFE4meNbZ1ccM+7PPDObsBZQj00WJhjNZJD3LpiOs0dXdy/ere/QzHGHMYSiZ9tL62nvdPD5uI6ANYW1jAvK56I0LE/i30wpqXFcvHCCfzl/b2U1bX6OxxjjBdLJH621U0geeUNtHZ0sbmojuMnJfk5qsB0y4oZKMrlf3ifHWX1/g7HGOOyROJnW0ucL8R9B5r5aM8B2rs8LJmY6OeoAtPklGge//JJdHo8XHXfGjbutw2wjAkElkj8bGtJPWEhQajCPz7cB2CJpB/zs+N54isnERcZwhf+spbqxjZ/h2TMuGeJxE+6PMq+6mZ2ljdw9mxn7/WXt5axKCeBlJjxPaP9aHKSovjjtUupb+ng9qc221pcxviZJRI/+eM7uzntl2/S6VFWzssgItT5KL61YrqfIxsdZmXE8d1zZ/LqtnLufdtGchnjT4G8sdWY5fEo//hwH7Mz47hgfgbnzEnnHzkJeDxw+hjeRne4feGUyWwqquMXr+ygub2TdwuqOGlqMt9ZOcvfoRkzrlgi8YOP9h5g34FmfnvVQi49LhuAB647niBh3K+vNRgiwi+uWEBHl4d73igAnC2KLZEYM7IskfjB0+uLiQ0P4dy5B7eSjQm3j+JYRIQG8/vPLea17RW8v6uKh97bS21zOwlRYf4OzZhxw/pI/GB7WT2LchOIDLNJh8NBRDh7TnrPoIWNRXV+jsiY8cUSiR8UVjeTmxTl7zDGnHnZ8Yhg80uMGWGWSEZYXXMHdS0dTEy2RDLc4iJCmZoaw6YiSyTGjCRLJCOs8EATALlJ0X6OZGxalJPAx3traO0Yk7swGxOQAi6RiMgiEVkjIhtEZK2ILHPLRUTuFpECEdkkIov9HeuxKKxuBrAaiY9cuSSbupYOHl+739+hGDNuBFwiAX4B/EhVFwH/5T4GOA+Y7t5uAv7gn/CGZt8BJ5FYH4lvLJucxOLcBO5bvZv2Ttvn3ZiREIiJRIHuzTjigRL3/sXAw+pYAySISGZvLxDICqubSIkJJ9qG+/qEiPCNs6ZTVNPCnS/v8Hc4xowLgZhIvgX8UkT2A78CbnfLswDv9ooit+wIInKT2yy2trKy0qfBDlZhdbM1a/nYmTPTuP6kSfzp3T28vr3c3+EYM+b5JZGIyGsisqWX28XAV4FbVTUHuBX402BfX1XvV9Wlqro0NTVwlhxRVfZUNTHRmrV87j/On82M9Bj+619baW7v9Hc4xoxpfkkkqrpCVef1cvsXcB3wlHvq48Ay934xkOP1Mtlu2Yj6+j/W87s3C47pubsqm6hoaGOxLRPvc2EhQfz00vkU17bw+zd3+TscY8a0QGzaKgFOd+9/Csh37z8LXOuO3joBqFPV0pEMrLC6iec3lfL+rqpjev6bOyoAOGNm4NSSxrKlk5I4f34GD3+wl6Y2q5UY4yuBmEi+BPxaRDYCP8UZoQXwIrAbKAD+CHzN14G0d3po6zw4H+HZDU6/f1VD+zG93pt5FcxIjyE70Zq2RsoXT51CfWunDQc2xocCLpGo6ruqukRVF6rqclVd55arqt6sqlNVdb6qrvV1LF/52zpu/PPHAHR0eXhmg9OSVtnPrnwt7V08/UnRIQkIoKqxjY/3HuDMWWm+C9gcYXFuIksnJvKHt3dR39rh73CMGZMCLpEEiuLaFt7Mq+C9gmo27K/lqvs+YFdlE9PSYqhpbqez68g5CqrK7U9t4tZHN3LP6wf7Udo6u7jlkU8QES5fnD2Sl2GAH3x6DpUNbfzsxe3+DsWYMckSSR+eXl+EKojA5x/4kM3FddxzzXFcd9IkVJ19Lw73xLointlQwoT4CO5bvYtdlY08ua6Ixf+zivcKqvnJJfOYkR7rh6sZ3xbmJHDjyZP550f72e9OCDVmrGjv9FBU49//rgeUSETkFhGJczu6/yQi60XkHF8H5y+qypPri1k+OYmTpibT0NbJt1bM4MKFE0iNcfa5qGg4tHmrsa2TO1/OY8nERJ65+WQiQ4O57clN/M/z25iWFsPfvrCczyzN6e3tzAi47qRJADy/aUTHZxjjU10e5YsPr+VTv3qbfdX+SyYDrZHcqKr1wDlAIvB54Oc+i8rPmtu7WDIxkc8uz+Wbn5rOtSdO5MunTQEgNTYccPo8uqkqv3l1J1WNbfzg03NIi4vgO+fO4uO9NdS1dHDHpfM5ZXqKX67FOHKSojguN4FnN5Yc/WRjRom7X89n9c5KulS55438oz/BRwa6Tkf3/q/nA39V1a0yhveEjQ4P4VdXLux5vHxKcs/9lJjuRNJOXlkDd7+RT31LB+/kV/G55bksykkA4LPLclm1rZzcpEjmZcWP7AWYXl20cAI/em4buyobmZoa4+9wjBmyV7aWcdLUZGZlxPGXD/bS3NHFf5w/m6yEyBGNY6CJZJ2IvApMBm4XkVhgXK6I151IXtlaxn88vZnI0GBiwkP48mlT+N65B/cKDw4S/nLD8bYHewA5flISAAUVlkjM6NflUXZXNXHajFS+dsZUalvaeWFTKZGhwYf8EB4JA00kXwAWAbtVtVlEkoEbfBdW4IoODyEqLJhV28qJDgvm9f93ek9yOZwlkcCS7PZvVfUzfNuY0WL/gWbaOz1MS4shISqM33xmEUEivLKljJ9cMo+I0JHbyrvfPhIRWezu+7HILZriPp7IwJPQmNOdOE6eltJnEjGBJzna+ayqG49tQqkxgSS/ohGA6WkHa9cXLpxAQ1snb+WN7GK1R0sGv3b/jQCWAJtw+ksWAGuBE30XWuBKjQ1n34FmTrelTkaVsJAg4iJCqLYaiRkD8isaAJjmlUhOnppMSkwYtz66gb+uSeCvNy4nKMj3LSP91khU9UxVPRMoBZa4K+ouAY7DDwsmBooUt4nk9BmWSEablJhwqnqZA2TMaFNQ3khmfASxEaE9ZSHBQTx0/TKuOj6H9NiIEUkiMPDmqZmqurn7gapuEZHZPoop4J0yLQVVbM2sUSglJpyqBquRmNEvv6LxkNpIt/nZ8czPHtmRogOdR7JZRB4QkTPc2x9xmrnGpc+fOIn7r13q7zDMMUiOCaPaaiRmlGtq62RneQMzA2SljIEmkuuBrcAt7m0b43TUlhndkmPCrI/EjHqrtpXT1ulh5bwMf4cCDKBpS0SCgZfcvpLf+j4kY3wnJSacmuYOOro8hAbbUnNm9Mgra6CoppmzZqfzzIZishIiWZIbGJvkHTWRqGqXiHhEJF5V60YiKGN8Jdkdrl3T1E5aXISfozFmYO57exc/f3kHqvDg9Ut5J7+KL506ZcQ6049moJ3tjTj9JKuApu5CVf2mT6IyxkdSorsnJVoiMaPH3z/cx3E5CRRUNHLTw+sIFuGaZYGzCOxAE8lTHNxH3ZhRK8VddLO6yfpJzOhQ09TOvgPNXLMsl9NnePjtazv55lnTmJgc7e/QegwokajqX3wdiDEjITnalkkxo8vGoloAFmbHs3hiIhOTozhvfmB0sncb6H4k00XkCRHZJiK7u2++CEhEForIByKyWUSeE5E4r2O3i0iBiOSJyEpfvL8Z27r7SCrduSSF1U185/GNtHZ09fc0Y/xmU1EdIjAvO56I0GAuOS6L8JCRW0drIAY6bOUh4A9AJ3Am8DDwNx/F9ABwm6rOB54GvgMgInOAq4G5wLnA790RZcYMWFxECDlJkbyTXwXAL1/J4/F1RawvrPFzZMb0blNRLVNSoonzmsEeaAaaSCJV9XVAVLVQVX8IXOCjmGYAq937q4DL3fsXA4+oapuq7gEKgGU+isGMUSLCpYuyeK+gijW7q3lhs7Nj4rbS+p5zPtxdzfUPfURFQyu3P7WJj/ce8Fe4Zhwrrm3hm//8hHfyq1iYneDvcPo10M72NhEJAvJF5Os462z5akOHrThJ4xngSqB7aEIWsMbrvCK37AgichNwE0Bubq6PwjSj1aWLs7n7jQJueOhjwoKDiAgNZntpQ8/xZzYU81ZeJWf/ZjV1LR00tXX17GVizEjIK2vgmj+uoa2ji5VzM7jp9Cn+DqlfA00ktwBRwDeBH+M0b113rG8qIq8BvfUW/SdwI3C3iPwAeBYY9HoWqno/cD/A0qVL9VjjNGPT5JRoTp2eQkY26XMAACAASURBVFFNC3dcOo97397Ndq8aydq9NcRFhFDX0gGAR+0/ITOy7nt7Fx2dHp79ximjYhO2gSaSA6raiDOfZMhLo6jqiqOccg6AiMzgYBNaMQdrJwDZjOMViM3Q/OWGZYg4TV1v76zkoV3VdHR5aGrrJL+ike+snMkF8zP57hObqLBFHs0Iamjt4MUtpVy2OHtUJBEYeB/JgyKyS0QeEZGbRWS+rwISkTT33yDg+8C97qFngatFJFxEJgPTgY98FYcZ24KCpGcHyzmZcbR3eVhfWMNHe5z+kCUTE5mUEk1qXHjPCC9jRsLzm0pp7fBw5ZJsf4cyYAOdR3K6iIQBxwNnAC+ISIyq+qLh+BoRudm9/xTOiDFUdauIPIazYGQncLOq2phNM2RzMp0R5lfdv4bQYCEkSHo6N9Niw3mrvtWf4Zlx5u28SrITI1mUE9gd7N4GlEhE5BTgVPeWADwPvOOLgFT1f4H/7ePYHcAdvnhfM35NS4vht1ctpKapg1e2lpGVGElkmDOyPC02gqb2LpraOokOH7e7S5sRtKeqiVkZsT015tFgoP9nvAWsA34GvKiqtqGDGTNEhEuPc5oRbjxl8iHH0mIPTmC0RGJ8zeNR9lQ3cdqMFH+HMigD7SNJAf4HZ4/2l0XkNRH5se/CMiYwpLqJxDrczUgorm2hvdPDlFHSyd5toH0kte6SKDk4o6VOAgJ3mqUxwyQtrjuRWD+J8b09Vc7i6pNTAmdBxoEYaB/JbmAH8C7OUik3WPOWGQ/SYp2l5ivqrUZifK87kUxJHYOJBJimqh6fRmJMAEqMCiU0WKxpy4yIPVVNxISHkOouLjpaDLSPZJqIvC4iWwBEZIGIfN+HcRkTEESE1Jhw9h1oshWCjc/tqmxkckr0qBqxBQNPJH8Ebgc6AFR1E85KvMaMeRMSInlxcxkX3vOuv0MxY1hbZxdbiuuYnj66Otph4IkkSlUPn0XeOdzBGBOI7rxiAefMSWdXZSNdHlt3y/jGy1vKqGnu4NLjel2LNqANNJFUichUQAFE5Aqg1GdRGRNApqbGcNLUZDwKtc02xsQMr4qGVr792Abufj2ficlRnDx1dM0hgYF3tt+Ms5ruLBEpBvYAn/NZVMYEmO6dFasa23vuGzMcnt1QwlPrnfVn//vCOQQFja7+ERj4PJLdwAoRicapxTTj9JEU+jA2YwJGips8qhvbgFj/BmPGlPd3VTMlJZpHbjqhZwLsaNNv05aIxLn7pP+fiJyNk0Cuw9md8DMjEaAxgSAlJgyAykYbBmyGT2eXh4/2HODEqcmkxUWMutFa3Y5WI/krUAN8AHwJZ+MpAS5V1Q0+js2YgHGwRmJ9JGb4bC6uo7GtkxOnJvs7lCE5WiKZoqrzAUTkAZwO9lxVtfUizLgSHxlKSJBQZTUSM4zeya8C4IQpozuRHG3UVkf3HXfvjyJLImY8CgoSkqLDrEZihk2XR3n04/2cMCWpp8Y7Wh2tRrJQRLo3sxYg0n0sgKpqnE+jMyaApMSEW43EDJvXtpdTXNvCDz4929+hDFm/iURVg0cqEGMCXXJMGFVNViMxQ1dR38qvXsljQnwEK2an+zucIbOdeowZoNSYcHZXNg3b6+2ubOTu1/PpUrjnmuOG7XVNYHtjRzk/eGYrNc3t/PHapYQED3ReeODyyxWIyJUislVEPCKy9LBjt4tIgYjkichKr/Jz3bICEblt5KM2411yTBjVTW2oDs8yKbc8soFnNpTw3MYSOrtsce3x4LmNJdz457VEhgXzzy+dwMnTRt8s9t74KxVuAS4DVnsXisgcnImOc4Fzgd+LSLCIBAO/A84D5gDXuOcaM2JSYsJp7fDQ1D70VYDrWjrYUlJHcnRYz2MzthVWN3H7U5tZMjGRF795KgtzEvwd0rDxSyJR1e2qmtfLoYuBR1S1TVX34Ex8XObeClR1t7uh1iPuucaMmO6RNZXDsDfJusIDqMI5czMAqGm2RDKWtXV28fV/fEJwkHD3NccRFjL6m7O8BdrVZAH7vR4XuWV9lfdKRG4SkbUisraystIngZrxJyPe2S2xrM4ZAb+7spEtxXXH9Fof7akhNFg4Y2YqYItBjnW/e3MXm4vr+MUVC8hKiPR3OMPOZ4lERF4TkS293Hxek1DV+1V1qaouTU1N9fXbmXGiJ5HUtwDw389u5duPHdsCDx/tqWZBdgKZ7mvWWo1kTHt1axknT0tmpVsDHWt8NmpLVVccw9OKgRyvx9luGf2UGzMiur/0S90ayfbSehrbOlHVQa2R1NjWyebiOr546hQSo5w+kpoh1Ejqmjt4ZWsZVy7NHrVrNY1ldc0d5JU3cOv8Gf4OxWcCrWnrWeBqEQkXkcnAdOAj4GNguohMFpEwnA75Z/0YpxmHosJCiI8MpayulcqGNqoa22nt8Ay6f+Pd/Co6upTTZ6QSHxUKDK1G8vAHe/nuk5vYXTV8Q5PN8Fm3z+kPO35Skr9D8Rl/Df+9VESKgBOBF0TkFQBV3Qo8BmwDXgZuVtUuVe0Evg68AmwHHnPPNWZEZcZHUFrXSl5ZQ09ZcU3LoF7jrbwKYiNCWDIxkdjwEIKDhNqWY6+RvL+rGoA9wzjHxQyfj/c6/WGLxtAorcP5ZUKiqj4NPN3HsTuAO3opfxF40cehGdOvjPgIyupa2VFW31NWXNvC/Oz4AT1fVXkzr4LTpqcS6k5ES4gMPeZRW60dXazbVwPA3mpLJIHo4z0HmJcVT2TY2F0oJNCatowJaN01kh1lDUS5XwzFtU6NZFdlIy1HmWOyo6yB8vo2Tp95cBBIQlToMY/a+mRfLe2dzmTGPda0FXA8HmVrST0Ls8dubQQskRgzKBlxkVQ1trG5qI4lExOJDA2mpLaFfdXNrPztah58b0+/z9+4vxaA5ZMPtpcnRoVR0zT4GkleWQP3rd5FkMDU1GirkQSg4toWWjq6mJE+tnfVtERizCB0j9zKK29gcW4iWYmRlNS2cO/qXXR69JC+k95sKq4jLiKE3KSonrKEqFBqBzmzvaKhlSvufZ/3C6r54qlTWJCdwN6q5sFfkBkUVeWtvIojap6qyi9e3sHLW8oOKS+oaARgenrMiMXoD5ZIjBmE7rkkkaHBXHviRCYkRLK5uI4n1hYBR++n2FxUx/zs+EOG6SZEhQ26aesnz2+nrcPDS986lf84fzaTkqMprm2htWPoy7eYvv3tw31c/9DH/G1NIZUNbeyrdpL31pJ6fv/WLr7yt3X88pUddHmc9dh2ljs/LKanWSIxxriyEp1ZyZ9bnktyTDhZCREU1bQQFhLEWbPS2FPV1Oeijm2dXewoq2d+1qHt5YlRoYOaR1JQ0cCzG0v48ulTmJrqfEFNSnFqOIXVA6+VWOIZnF2Vjfz4+W0AvLGjgq//Yz2f+9MaVJXnN5USEiRcdlwWv3tzF1/7+zo8HiW/opHU2HAS3PlCY5UlEmMGYUpKNPdccxy3nu1MLpuYHA3AL69YwMnTUmho7eRAH3uW5JU10NGlLDhshFdCVBitHZ4Bf6k/ub6Y4CDh2hMn9ZTNy4pHBH75St6AVhLu6PJw7l2rufv1/AG9p4HVOytp7/Rw0cIJfLT3AB/uOcD+Ay0UVDTy/KYSTp6Wwq8/s5DvnTuLV7aW8+T6IvIrGsd8bQQskRgzKCLChQsnEB3ujJz/3PJcHv/KiZw3P5PJKU5SObx5q761gx8/v40v/mUtAPOzDk8kA5+U2OVRnl5fzOkzUkmNPbg969TUGP7norm8tr2cv64pPOrr7KpspKG1k/fcOSjHQlW5+/V89h8YH30ze6qaiI0I4bPLc+nyKEFu6+SdL++gqKaFCxZkIiJ8+bQpLM5N4KcvbmdnWYMlEmNM/2IjQntmLE9yE4n35lct7V1c8rv3ePC9PSzOTeSOS+eR49XRDpDkNntUNx19VeH3d1VRVt/KZYuPXLP08ydOIiUmjJ3ljUccu/ftXby69WBH8LYSZx7M1uI6mts7j/q+3bo82rPk/f4DLfxm1U6e/mR8rFa0p6qJKSnRLJmYSEJUKCtmpzM9LYbXtleQlRDJhQsmABAUJNx5+QISo8No6egaU8vF98V2SDRmmGQnRhIcJOytbmL/gWbuei2f9i4Puyub+PMNx3PGzLRen9edWPZVNzN3Qv8TG59aX0xcREif27OmxUZQUd96RPl9b+9iXlZ8z7L13Ymk06Ns2F/LSVMHtsHS/at3c/fr+Tx0w/F0uE1oRTXjo0ayu7KJ4yclEhocxJNfPYmkqDB+92YB+RWNfP+C2YdMOJyeHsvr3z6dopoWJozB1X4PZzUSY4ZJaHAQOYmRbCup59ev5vHk+iKe21jCVUtz+kwiQE+T2K7KI2sS3hrbOnl5SxmfXjiBiNDeZ0lnxEdQdlgiaevsoqa5g3yvmsq20nomp0QjAuv21gz0EtlWWk9LRxc3PPQxa3Y7zWL7DwxuiZjqxjY8nuHZZXKktHZ0UVzbwhR3cMPU1BgSo8O46fQp/PrKhZw778hVfUWEnKQogoPG/kKalkiMGUYXLZzAm3mV/GtjCdefNIkHr1/KDy+a2+9zosNDyIiLOOp+8E+vL6Klo4vLe2nW6pYeF055/aFNZN0bcZXVt1LX0oGqsq20nhOmJDEjLbZniZWBKKxuIjcpipaOLv7x4T4AimoHXiOpbmzjpJ+/wXObSgb8nEDQ3e/VnfS7pcVGcPkSW3XZEokxw+jrn5rOgux4QoOD+OoZU/nUrPQBrbE0JTWaXV5LnKgqVY0HE8K6wgP85IXtHD8pkcW5iX2+TlpsBNVNbT3NTgAVXjs6FlQ0UFLXSm1zB3My45iYHEVp7ZFNYX0prG7m1OkpZCVE9qwPVlrbOuA95zcV1dHW6WFz0bFtCOYv3Un+8ERiHJZIjBlGYSFB/PULy3nhG6eQHhcx4OdNTY1hd2VjzxyUn7ywnePveI2fvbSdprZObnlkA5nxEdz3+aX9/vrNiI9A9dDtgL37THaWN/LmjgoAlk5KIjkmfECd/ODsq1HX0sHE5ChOmprcU97p0SOa0/qytcRJIHsHMd/FHzq7PDz68T46ujxUN7axals5YImkL9bZbswwi48MJT4ydFDPmZIaTUNrJ1WN7awrrOFP7+5hZnos9729mxc2lVJU08Jfv7CMpOj+J7alxzlDgsvqW3s6ebtrJEHizLTeVlLP1NRoZmXEkhITxoGmdjweJaiPtvw/vLWLfQeauXBBJgC5SdGkx0Xw+LoipqXFUFDRSFFNC9mJUb0+39u2UqeTvzDA1wV7K6+S7z25maiwEP707h427K9l2eSknmHf5lD2VzEmAHR34u6ubOQ3q/KYmR7Lc984hb9/WMiPntvGqdNTOHX60beN7q4FeddCyutbCQkSZmbE8l5BFfkVjXzzU9MREZKjw/Ao1LZ09Jmk7nx5BwBv7HB+lU9KiSI5OpzwkCDOnpPek0gGYqs7WqzwQHO/ycvfuhPecxtL2LC/lm+eNZ1bV0z3c1SBy5q2jAkA09xJa398Zzc7yxu57qRJhIUEccPJk3n0phO4++rjBvQ63YnkmU9KehJARX0bKTHhLJucxM7yRlThwoXOnIfkGKcGU93Ye/NWQ+vBSZLdnfi5SVGkxobzznfP5JazpiPCgCYldnR5KKxuJj0unPZOD6UDbA7zh+7h0a+6TVorZqeN+w71/lgiMSYAZCVE8ukFmby2vYLI0GAuXJjZc2z5lGQSj9Kk1S0pKozQYOHlrWX84a1d1Ld2UNHQRnpcOD+4YA4v3XIqT3zlxJ7Eley+blVj78u65Lur19585lQAUmPDiQpzGjLS4iKICA0mIy5iQEvYdw8AmJURB0BhAO+fsr2snu68ER8ZetT5PeOdJRJjAsTPLpvPrIxYPrs8l9iIwfWxdAsKEtJiD3by55U1UF7fSmpsBEFBwuzMOJZ67R3eUyPpo8M931299solOSzIjmdmL/tqHD8piXfyq446cqujyxlI0J3E9gRYP0lZXSsvbi6lqKaZwupmzpnjTPo8aWryuJgLMhR+6SMRkSuBHwKzgWWqutYtTwaeAI4H/qyqX/d6zhLgz0Akzpa7t2hfy6waMwrFRoTy4jdPZagtKBOTnU7v4toWdpTWU9nQxuKJvQ8ZTo5xl2fpo0ays7yR8JAgcpKi+PMNy3pd2fj8+Rk8u7GEj/Yc4KRpfc+Q715aPSshkvCQoEGtVDwSfrtqJ4+u3c/M9FjCgoP4/gVz6PLA1cty/R1awPNXZ/sW4DLgvsPKW4EfAPPcm7c/AF8CPsRJJOcCL/k2TGNG1nB0Pt919SKCRDjr12+zqaiO6qZ20mN7H4qcGBWGSN99JLsrG5mSGkNwkPTZGX/6jDQiQ4P53lObCBbhsa+ceEitqFt3jSXMTUz7AiiRdHmU17Y7/SF55Q3cePJkcpKieOC6pX6ObHTwS9OWqm5X1bxeyptU9V2chNJDRDKBOFVd49ZCHgYuGZlojRld0mIjSIkJZ1ZGLC+7CzV211IOFxwkJEWFUdXH0vfl9W1MiO9/PkxkWDDnzE2npLaVwgPNPPTe3l7P63RrJCFBwoSESErrBre0ii+t3XuA6qZ2/vP82Vx/0iS+8alp/g5pVBktw3+zgCKvx0VuWa9E5CbgJoDcXKuWmvFpdmYcH+45QHpceK9rQXVLjgnjwGFNW2/sKCevrJGKhlYW5hy9o/mOS+fzvXNncccL2/nbB4UU17TwmaU5nDL9YFNXd9NWSHAQE+Ij2O4OsfU3VeXJ9UWEhQRxzfJcYmyuyKD5rEYiIq+JyJZebhf76j27qer9qrpUVZemph597L0xY9HsTKdj/GtnTOtzkUeA5OgjZ7c/vraIu1/Pp7qpvddmqsPFhIcwISGSr54xlZaOLp7dWMKf399zyDndo7a6aySVDW20dY78Do1v76zkN6t29sTzzUc28NjaIi5fnGVJ5Bj57K+mqiuG8eWKgWyvx9lumTGmD+fPz6ShtZOrl+X0e15yTFjPvIlulQ1ttLg7NqbFhff2tF7Ny4pn3Q/O5o4XtvHK1vJDJh1210iCg4RMt7msrK61Z5fJkfKbV/PYWFTHtpI67rh0Ps+5C2z+16fnjGgcY8moGP6rqqVAvYicIM6soGuBf/k5LGMCWmxEKF88dQrhIf0vGpkSE05lY9shI7IqvTrf++qo70t8ZCjLJidT19LBzoqGnvLuPpLQYCHLXb6lZBALRg6H6sY2NhXX9WxIde/buwC4aNGEgJ1lPxr4JZGIyKUiUgScCLwgIq94HdsL/Aa4XkSKRKT7Z8LXgAeAAmAXNmLLmGExPT2GhtZO8soPful7L/o4mBpJt+WTnbkqH+050FPW2dVdIwkisyeRjGyH++r8SlThRxfNJTRY+PuafYSHBDHPJhwOib9GbT2tqtmqGq6q6aq60uvYJFVNUtUY95xtbvlaVZ2nqlNV9es2h8SY4XHu3AyCg4TnNjp7hDS1ddLcfrDvYiB9JIfLTowkMz6CD70TicftIwk+2LTlPXKruLaF1g7f9pm8lVdJSkwYJ0xJZvnkZNq7PCzKSSAsZFQ0zgQs++sZM84lx4Rz8rQUnttYiqoeUhsRgZSYgS3P4k1EWDIxkQ37anvKvIf/RoQGkxwdRrHbtLVmdzVn/vItfv9mwRCvpn9rdldz8rQUgoKEs2Y7u1Ye7zXT3xwbSyTGGC5ckMm+A81sLKrr6R8JCwkiOTqckOBj+5pYmJ1AcW1LzwZdB5u2nL6IzIQISutaKKtr5aaH19Le5WF7WUOfrzdUVY1tlNe3MT/LacY6b14mE5OjOGduus/ec7ywRGKM4Zy5GYQFB/HcxpKeGsktZ00f0sS8BdnOF/amIqdW0tXT2e587UyIjyS/vJH7V++mqb2LeVlx7PHhQo7d81bmTHAWjcyIj+Dt75zJguwEn73neGGJxBhDfGQop89M5flNJZS7y7tfdXwO15006Zhfc15WPEECG/c7uyJ2uH0k3TWSyxZnUVzbwoPv7eG8eRmcMi2VwuqmnoQz3Lr3QpmTGeeT1x/PLJEYYwBnj5Ly+jZe3FxKcJCQGDX4vhFv0eEhTEuLYXOxk0i63Kat0CDna+fceZl82t118UunTmFKSjQdXUrxADfJGqxtJfVkJUSSMMTrMkeyaZzGGMDZvCkmPISP99aQFhs+LEunL8hO4M0dFXR5tGfUlvfr/urKhdx02hQWZCfQ7s40313VSG4fa4MNxbbSemZbbcQnrEZijAEgKiyEK5Y4C0gcy9yR3qyYnU51Uzt//7Dw4Kit4IOJJCI0uKePYnKKM8PdF/0krR1d7K5sZE7mkfupmKGzRGKM6XHtiRMBSI0ZnkSycm46p0xL4Zev5PVs1RvSR00nOTqM2IgQnySSopoWPAqTU0d2OZbxwhKJMabHlNQYbj5zKhcv6nNx7UEREX508VxaO7r4yQvbAAgJ6v1rR0SYkhLN7srhTyTdEx8z4yOH/bWNJRJjzGG+s3IWlxw3PIkEYGpqDF86dQrda1F4N20dcW5aDPkVwz+XpNSd+DjBEolPWGe7Mcbnvv6paWwvrSczIZL0uL6XXJmZHstT64upa+4gPurY9q3vTYlbI0mPH54mO3MoSyTGGJ+LCgvhoRuWHfW8GelOZ/jOioZhXbqktLaVlJjwo66EbI6NNW0ZYwLGjAwnkeQN81IpJXUtTEgY/OKTZmAskRhjAsaE+AhiwkPILx/eRFJa12r9Iz5kicQYEzBEhOnpMYfsjTJUqkppbQuZViPxGUskxpiAMiMtlg37a/neE5tobu8c8uvVt3bS1N5lNRIfskRijAkonzk+mwVZCTy6dj+rtpUf9XxV7bdPpWcOidVIfMYSiTEmoCyZmMQ/bzqBxKhQ3s6rPOr57++qZuVdq1m790Cvx7uTzIQEq5H4ir/2bL9SRLaKiEdElnqVny0i60Rks/vvp7yOLXHLC0TkbhEZ+opyxpiAFBwknD4jlbd3VuI5yrLyn+yrAThkW99uqsqD7+0lNymKBVm2L7uv+KtGsgW4DFh9WHkVcKGqzgeuA/7qdewPwJeA6e7t3BGI0xjjJ2fMTKO6qb1nGfq+bHM3rFpfWNNT9l//2sJbeRV8sLuajftr+dJpU455p0dzdH6ZkKiq28EZoXFY+SdeD7cCkSISDiQBcaq6xn3ew8AlwEsjErAxZsSdMj0FgI/2HGBhTt+7GG5zN6z6ZH8tqkpJXSsPf1DIGzsqSI4OIy02nCvdVY2NbwRyir4cWK+qbUAWUOR1rMgt65WI3CQia0VkbWXl0dtYjTGBJyUmnLiIEPbXNPd5TmNbJ3urm8lOjORAUzt7q5t7+kqKalrYWFTHbefNIiLUZrT7ks9qJCLyGpDRy6H/VNV/HeW5c4E7gXOO5b1V9X7gfoClS5f6Zt9OY4zPZSdGsf9A74nkntfzeXun80Pxs8tz+cXLeawvrGFTUS3RYcEcPzmJLo9yyTCtZGz65rNEoqorjuV5IpINPA1cq6q73OJiwLtumu2WGWPGsJykSHb1saz8K9vK2FLsNGtdtHACd63KZ2dFA7urmpiWFsOD1x0PQNAw7PRo+hdQTVsikgC8ANymqu91l6tqKVAvIie4o7WuBfqt1RhjRr+cxCiKappRPbJhobS2lXlZcXzl9KlkJUSSmxzFnsomCqubyU2OJihILImMEH8N/71URIqAE4EXROQV99DXgWnAf4nIBveW5h77GvAAUADswjrajRnzshMjae3wUNXYfkh5a0cX1U3trJyTwW3nzUJEmJQcTUFFI0U1zUzywZ7vpm/+GrX1NE7z1eHlPwF+0sdz1gLzfByaMSaA5CQ5CWF/TTOpsQf3EimrczaqyvSaZDg5JYrXtjsz4Scm25a6IymgmraMMcZbTyI5rMO9e6OqCfEHlz2ZlHIweViNZGRZIjHGBKzsRKfGUVTTckh599a5h9ZIDiaSXEskI8p2SDTGBKyosBCSo8PYV+3USCrqW1l512rmucudZHrVSLoTSVRYMKkxtqXuSLIaiTEmoM3PjmfNnmpUlc3FddQ0d/BOfhVJ0WGHTDRMj40gIjSIicnRR6yaYXzLEokxJqCtmJ1OYXUzBRWN7Kk6OKfEuzYCznyRBdkJtjijH1jTljEmoK2Ync73n9nCq9vKD+kryexlo6q/3LCMIPt5POIskRhjAlpGfAQLsuNZta2ciNAgZqTHsLeqmZykIxNJZJitqeUPlkiMMQHv7Nnp/HrVTmLDQ1g5L4OfXbag10Ri/MMqgcaYgHf23HQAGto6mZwSzZKJiaTF2ta5gcISiTEm4M1Mj+2ZUzIlxWatBxpLJMaYgCcinD3HqZVMTrVEEmisj8QYMyrcePJkwkOCmZ4W6+9QzGEskRhjRoWcpChuO2+Wv8MwvbCmLWOMMUNiicQYY8yQWCIxxhgzJJZIjDHGDIklEmOMMUNiicQYY8yQWCIxxhgzJJZIjDHGDImoqr9j8CkRqQQKj/HpKUDVMIYzGtg1jx/j8brtmgdmoqqmDvTkMZ9IhkJE1qrqUn/HMZLsmseP8Xjdds2+YU1bxhhjhsQSiTHGmCGxRNK/+/0dgB/YNY8f4/G67Zp9wPpIjDHGDInVSIwxxgyJJRJjjDFDMi4SiYjsFZHNIrJBRNa6ZUkiskpE8t1/E91yEZG7RaRARDaJyGKv17nOPT9fRK7zKl/ivn6B+1zxwzU+KCIVIrLFq8zn19jXe/j5un8oIsXu571BRM73Ona7ew15IrLSq/xct6xARG7zKp8sIh+65Y+KSJhbHu4+LnCPTxqh680RkTdFZJuIbBWRW9zyMf1Z93PdY/mzjhCRj0Rko3vNPzrWOIfrb9EnVR3zN2AvkHJY2S+A29z7twF3uvfPB14CBDgB+NAtTwJ2u/8muvcT3WMfueeK+9zz/HCNpwGLgS0jeY19vYefr/uHwL/3cu4cYCMQn5Ms/gAAB65JREFUDkwGdgHB7m0XMAUIc8+Z4z7nMeBq9/69wFfd+18D7nXvXw08OkLXmwksdu/HAjvd6xrTn3U/1z2WP2sBYtz7ocCH7ucyqDiH82/RZ6wj9R+CP2/0nkjygEyv/0jz3Pv3Adccfh5wDXCfV/l9blkmsMOr/JDzRvg6J3HoF6rPr7Gv9/Dzdff15XI7cLvX41eAE93bK4ef5/6PXAWEuOU953U/170f4p4nfrj2fwFnj5fPupfrHhefNRAFrAeWDzbO4fxb9HUbF01bgAKvisg6EbnJLUtX1VL3fhmQ7t7PAvZ7PbfILeuvvKiX8kAwEtfY13v429fdppwHvZpgBnvdyUCtqnYeVn7Ia7nH69zzR4zbdHEczi/VcfNZH3bdMIY/axEJFpENQAWwCqcGMdg4h/Nv0avxkkhOUdXFwHnAzSJymvdBddLumB4HPRLXGEB/xz8AU4FFQCnwa/+GM/xEJAZ4EviWqtZ7HxvLn3Uv1z2mP2tV7VLVRUA2sAyY5eeQejUuEomqFrv/VgBP43wg5SKSCeD+W+GeXgzkeD092y3rrzy7l/JAMBLX2Nd7+I2qlrv/A3qAP+J83jD4664GEkQk5LDyQ17LPR7vnu9zIhKK82X6d1V9yi0e8591b9c91j/rbqpaC7yJ08w02DiH82/RqzGfSEQkWkRiu+8D5wBbgGeB/9/e2YVYVUVx/PdX03pSyx4KH0pTrEyl1ESDzGQeKsxCIxHMmkKDjJKiYEKilzIzKrSClMQwIyPKBsvyI1HRHPNjZnTI8SN6iSgsLUnLXD2sfZ3jdWYc750P77h+cJh79tl7n732uXPWrL33/HdupcpD+JgrKX1aWu0yCjiSwvnVQJmk3il8LsPHDX8GjkoalVa3TMvU1dG0h41N3aPDyL3sEvfhzxu8rQ+m1S3XAgPwieUqYEBaqdIdn6hcmf7qXg9MSuXz+zBn9yRgXcrfpqT+XwzUmdnrmUud+lk3ZXcnf9ZXSuqVPl+GzwnVFdDO1uyLxmnPCaOOOPAVCbvTsQeoSOlXAGuBemANcHlKF7AQH4usAYZn6noE2J+OhzPpw/Ev8AFgAR0z6bocD+3/xcc0y9vDxqbu0cF2f5Dsqk6/RFdl8lckG34gs7oOX920L12ryPv+bEv9sQLokdIvTef70/V+7WTvbfiQUjWwKx13dfZn3YzdnflZDwF2JttqgTmFtrO1+qKpIyRSgiAIgqLo9ENbQRAEQdsSjiQIgiAoinAkQRAEQVGEIwmCIAiKIhxJEARBUBThSIKSQJJJmp85f0bSi61U9xJJk86ds+j7TJZUJ2l9Ju0mNSjXHpZ0KH1eI2lCVpG1DdozUdINbVV/cPHQ7dxZguCC4ARwv6SXzey3jm5MDkndrEGT6FyUA4+Z2aZcgpnV4PIeSFoCVJrZJ5kyK1urrY0wEagE9rbhPYKLgIhIglLhJL739NP5F/IjCkl/pZ9jJW2Q9Lmkg5JekTRVvsdDjaT+mWrGS9ouaZ+ke1L5rpLmSapKooAzMvVulLSSRl7Ckqak+mslzU1pc/B/qlssaV5LDJY0XdKCjI3vSNqabBkrFymsSw4oV6ZM0hZJOyStkGtTkWzfm+x4TdJoYAIwL0VA/dPxlVzcdKOkQZl7v9tI/9yY+nJXqndAS+wKOh8RkQSlxEKgWtKr51FmKHA9cBjfc2ORmY2Ub4w0C3gq5bsG12nqD6yXdB0uD3LEzEZI6gFslvR1yn8zMNjMDmVvJulqYC5wC/A7rjo90cxekjQOlzzfft6WO71xraUJeKQyBngUqJI0DP/P/heA8WZ2TNJzwGxJC3H5kEFmZpJ6mdkfyRGejoAkrQVmmlm9pFuBt4FxzfTPTOBNM1uWJDa6FmhXUOKEIwlKBjM7Kmkp8CTwdwuLVVmSPpd0AMg5ghrgjky+j82F/+olHcRVVsuAIZlopyeuU/QPsC3fiSRGAN+a2a/pnsvwzbc+a2F7m+OL5AhqgF/SsBiS9uAv+r74Jkab5Zsadge24HLix/FoqBIfzjqDFLmMBlaoYYPPHpksjfXPFqBCUl/gUzOrbwUbgxIkHElQaryBb/DzfibtJGmYVlIX/AWa40Tm86nM+SnO/P7nawUZrlM1y8xWZy9IGgscK6z5RZFte75d3YD/gG/MbEp+QUkjgTtxIb4naIg0cnTB96AY1sS9z+ofM/tQ0nfA3cAqSTPMbN35GBR0DmKOJCgpzOwwvg1oeSb5R3woCXzY55ICqp4sqUuaN+mHi9utBh6Xy5cjaaBcQbo5tgG3S+ojqSu+w+CGAtpTCFuBMWnYKad8PTBFGz3NbBU+xzQ05f8T37YW8709DkmanMpK0tBM3Wf1j6R+wEEzewtXhx3SDjYGFyDhSIJSZD7QJ3P+Hv7y3o3PIRQSLfyEO4Ev8XmC48AifDJ9h6RafDvaZqP4NIz2PC7DvRv43szaRW49DadNB5ZLqsaHngbhzqIypW0CZqciHwHPStqZHMRUoDz14x7g3kz1jfXPA0CtfAe/wcDSNjYxuEAJ9d8gCJpFjS9LDoLTREQSBEEQFEVEJEEQBEFRREQSBEEQFEU4kiAIgqAowpEEQRAERRGOJAiCICiKcCRBEARBUfwPSwucyg3bbbsAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import imageio\n",
        "from google.colab import files"
      ],
      "metadata": {
        "id": "wRib-cm60eRM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "images = []\n",
        "obs = model.env.reset()\n",
        "img = env.render(mode='rgb_array', max_width = 51)\n",
        "for i in range(350):\n",
        "    images.append(img)\n",
        "    action, _ = model.predict(obs)\n",
        "    obs, _, _ ,_ = model.env.step(action)\n",
        "    img = env.render(mode='rgb_array', max_width = 51)\n",
        "\n",
        "imageio.mimsave('easy_maze2d_a2c.gif', [np.array(img) for i, img in enumerate(images) if i%2 == 0], fps=15)\n",
        "files.download(\"easy_maze2d_a2c.gif\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "VOoZ3lnY0WGl",
        "outputId": "48436cca-c49b-4b4e-9a0b-98ce1b534962"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_0fe2d48b-ca46-4af0-91fd-ea756d5e1503\", \"easy_maze2d_a2c.gif\", 206943)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#### higher quality rendering ####\n",
        "for i in range(len(images)):\n",
        "  img = images[i]\n",
        "  a = plt.imshow(img)\n",
        "  a.figure.savefig(str(i) + \"rl_course_medium_maze_PG2_ok.png\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267
        },
        "id": "bnSqin0v80Yz",
        "outputId": "9c8b4155-13bc-4f28-a999-26d75d96e393"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD6CAYAAABuxZF5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAN4UlEQVR4nO3dXawc5X3H8e8vNpSUpAJDalkYalBQIi4SIlk0UbigVFSURoGLCBGlkiuh+qaViFIpgVZqm0iVkpu8XFStrILiizZAk7RG3FBKiNreAIe3BnAdnAgULIOVAgrcpDX8e7HjcnxynJ2zZ19mz/P9SKuzM2fPzH9n93eeeZ6Z2U1VIak971p0AZIWw/BLjTL8UqMMv9Qowy81yvBLjdpU+JNcn+RIkqNJbp9WUZJmL5Me50+yDfghcB3wEvAY8Omqeu6X/I0nFUgzVlXp87jNtPxXAUer6sdV9T/A3cCNm1iepDnaTPgvAn6yavqlbt5pkuxPspJkZRPrkjRl22e9gqo6ABwAd/ulIdlM+I8BF6+a3t3N683rCqTNS3p18X/BZnb7HwMuT3JpkrOBW4D7NrE8SXM0cctfVSeT/DHwALANuKuqnp1aZZJmauJDfROtbE2f391+afPW7vbP41CfpCU289F+bQ2TDCq5ZzdstvxSowy/1CjDLzXKPr8mYn9++dnyS40y/FKjDL/UKPv8S2694+9r++OTXvihrc2WX2qU4ZcaZfilRtnnb9Akx+gdN9h6bPmlRhl+qVGGX2qU4Zca5YCfAAf0WmTLLzXK8EuNMvxSo+zzN6DPST1+OEd7bPmlRhl+qVGGX2qUff6Bm8bx93W+zmnTy9Tys+WXGmX4pUYZfqlR9vmXjP11TYstv9Qowy81yvBLjRob/iR3JTmR5JlV83YkeTDJ893P82dbpqRp69PyfxO4fs2824GHqupy4KFuWksiydjbUPWpfVmey6KNDX9V/Rvw6prZNwIHu/sHgZumXJekGZv0UN/Oqjre3X8Z2HmmBybZD+yfcD2SZmTTx/mrqpKc8eBzVR0ADgD8ssdJmq9Jw/9Kkl1VdTzJLuDENIvSO6ZxUk+fZSxL33grPZdFm/RQ333Avu7+PuDQdMqRNC8Z9580ybeAa4ALgVeAvwD+GbgXuAR4Ebi5qtYOCq63rNNW5qmqw7GVLvvdSs+lj3Web69dn7HhnybDP1xbKTBb6bn0MWn4vbCnAVupD7yVnsuieXqv1CjDLzXK8EuNMvxSoxzwa9Ayj34vc+1DY8svNcrwS40y/FKj7PMP3CQntdgvXh6LPBvRll9qlOGXGmX4pUbZ5x+4Hpdcb3oZapMtv9Qowy81yvBLjdqSff7WPsllrWl84MUk22wWH7Qx5Ndu3PtskvfhuG04ze1hyy81yvBLjTL8UqMMv9SoLTng15IhD4gNubahmMZJXJOy5ZcaZfilRhl+qVH2+dWL35Sz9djyS40y/FKjDL/UKPv86sVj9hs3jW02y+1uyy81yvBLjRob/iQXJ3k4yXNJnk1yWzd/R5IHkzzf/Tx/9uVKmpb0OLd4F7Crqp5I8l7gceAm4A+AV6vqy0luB86vqi+MWdZpK5tVf6b1D/PQfCzi3If13svrvN97FTa25a+q41X1RHf/DeAwcBFwI3Cwe9hBRv8QJC2JDfX5k+wBPgI8AuysquPdr14Gdk61Mkkz1ftQX5L3AN8BPltVP1u9q1FVtXaXftXf7Qf2b7ZQSdM1ts8PkOQs4H7ggar6ajfvCHBNVR3vxgW+X1UfGLMc+/zaMrZ8nz+jJd8JHD4V/M59wL7u/j7gUJ8VzkNVnXaTZmHt+2wet2nqM9p/NfDvwA+At7vZf8qo338vcAnwInBzVb06Zllzafmllkza8vfa7Z8Wwy9N38x2+yVtTV7Ys0h79/7ivJWV+dehJtnyS40y/FKjDL/UKEf7pSXnaL+kDTH8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81yvBLjTL8UqMMv9Qowy81amz4k5yT5NEkTyd5NskXu/mXJnkkydEk9yQ5e/blSpqWsd/Sm9FXgJ5bVW8mOQv4D+A24HPAd6vq7iR/CzxdVX8zZll+S+8Grf0G1j6WebtO8nzHGfL2WOcbdqexjOl8S2+NvNlNntXdCrgW+HY3/yBwU99iJS1erz5/km1JngJOAA8CPwJer6qT3UNeAi46w9/uT7KSZGUaBUuajl7hr6q3qupKYDdwFfDBviuoqgNVtbeq9k5Yo6QZ2NBof1W9DjwMfAw4L8n27le7gWNTrk3SDPUZ7X9fkvO6++8GrgMOM/on8KnuYfuAQ7MqUu+oqtNuW93a5zvJTevrM9r/IUYDetsY/bO4t6q+lOQy4G5gB/Ak8PtV9fMxy3K0f4PGjQavNzq+zNt1GqPfy2SRo/1jwz9Nhn/jDP/yPpc+Bn2oT9LWtH38Q6TF6XPSz1D3Doa+F2PLLzXK8EuNMvxSo+zzD9y4fuLQ+pGbtdWez5DZ8kuNMvxSowy/1Cj7/AvU5+y81s7w0/zY8kuNMvxSowy/1CjDLzXK8EuNMvxSowy/1CjDLzXKk3yWzCy+0UbDMe71neYJXLb8UqMMv9Qowy81yj7/kvGina1lka+nLb/UKMMvNcrwS42yzz8wsziOP5QvjxhKHYsytC8gseWXGmX4pUYZfqlRhl9qlAN+C9TagFdr+ry+i7xQy5ZfapThlxrVO/xJtiV5Msn93fSlSR5JcjTJPUnOnl2ZkqZtIy3/bcDhVdNfAb5WVe8HXgNu3ejKk4y9afpmtd197ZZLr/An2Q38HvB33XSAa4Fvdw85CNw0iwIlzUbflv/rwOeBt7vpC4DXq+pkN/0ScNF6f5hkf5KVJCubqlTSVI0Nf5JPACeq6vFJVlBVB6pqb1XtneTvJc1Gn+P8Hwc+meQG4Bzg14BvAOcl2d61/ruBYxtd+XrHQe0rTt/Qjze3bNAf5lFVd1TV7qraA9wCfK+qPgM8DHyqe9g+4NDMqpQ0dZs5zv8F4HNJjjIaA7hzOiVJmofMc7cjyWkr67Pb7ymw8zGN7T5uGb62s7HOdu3Vh/MMP6lRg7+wZ5JvMNno4JUt0GwMdRBxkrq24l6qLb/UKMMvNcrwS40aXJ9/Fv2mZeuLLcI0ttGybGdPehqx5ZcaZfilRhl+qVGD6/NLQzTJGMDQzwOw5ZcaZfilRhl+qVH2+aV1DK1/Pgu2/FKjDL/UKMMvNcrwS41qYsBvkg8E0WJs9KPA+pjGR5LNaj2LZMsvNcrwS40y/FKjmujzT6PfqGGY1wdxtPCBH7b8UqMMv9Qowy81qok+/zjT6rvN4jjvLPqVy3Q8ehbPf1F99T7rnedrY8svNcrwS40y/FKjtmSff1y/aZn6vNOodZmOR8/jtZnkGP4kdU3juexl72nTK6xsepmn2PJLjTL8UqN67fYneQF4A3gLOFlVe5PsAO4B9gAvADdX1WuzKVPStG2k5f+tqrqyqk51Qm4HHqqqy4GHumlJSyI9Bz9eAPZW1U9XzTsCXFNVx5PsAr5fVR8Ys5zlGWmTllRV9Rrh7dvyF/AvSR5Psr+bt7Oqjnf3XwZ2brBGSQvU91Df1VV1LMmvAw8m+a/Vv6yqOlOr3v2z2L/e7yQtTq/d/tP+IPlL4E3gD3G3Xxqcvrv9Y1v+JOcC76qqN7r7vwN8CbgP2Ad8uft5qMf6fgq8CFzY3V8Gy1LrstQJy1PrstQJ79T6G33/YGzLn+Qy4J+6ye3AP1TVXyW5ALgXuIRRoG+uqld7rTRZWXXUYNCWpdZlqROWp9ZlqRMmq3Vsy19VPwY+vM78/wZ+eyMrkzQcnuEnNWpR4T+woPVOYllqXZY6YXlqXZY6YYJaNzzaL2lrcLdfapThlxo11/AnuT7JkSRHkwzqQqAkdyU5keSZVfN2JHkwyfPdz/MXWeMpSS5O8nCS55I8m+S2bv6g6k1yTpJHkzzd1fnFbv6lSR7p3gf3JDl7kXWulmRbkieT3N9ND7LWJC8k+UGSp5KsdPM29PrPLfxJtgF/DfwucAXw6SRXzGv9PXwTuH7NvKFeuXgS+JOqugL4KPBH3bYcWr0/B66tqg8DVwLXJ/ko8BXga1X1fuA14NYF1rjWbcDhVdNDrnVzV9pW1VxuwMeAB1ZN3wHcMa/196xxD/DMqukjwK7u/i7gyKJrPEPdh4Drhlwv8KvAE8BvMjoTbft674sF17i7C821wP1ABlzrC8CFa+Zt6PWf527/RcBPVk2/1M0bssFfuZhkD/AR4BEGWG+3G/0UcAJ4EPgR8HpVneweMqT3wdeBzwNvd9MXMNxaN32l7Zb8AM9ZqDrzlYuLkuQ9wHeAz1bVz1Z/6ORQ6q2qt4Ark5zH6DTxDy64pHUl+QRwoqoeT3LNouvpYeIrbU+ZZ8t/DLh41fTubt6QvdJdsUj388SC6/l/Sc5iFPy/r6rvdrMHW29VvQ48zGjX+bwkpxqeobwPPg58svvgmrsZ7fp/g2HWSlUd636eYPRP9So2+PrPM/yPAZd3o6dnA7cwujJwyE5duQj9r1ycuYya+DuBw1X11VW/GlS9Sd7XtfgkeTejcYnDjP4JfKp72MLrBKiqO6pqd1XtYfTe/F5VfYYB1prk3CTvPXWf0ZW2z7DR13/OgxQ3AD9k1O/7s0UPmqyp7VvAceB/GfXtbmXU53sIeB74V2DHouvsar2aUZ/vP4GnutsNQ6sX+BDwZFfnM8Cfd/MvAx4FjgL/CPzKorfpmrqvAe4faq1dTU93t2dPZWmjr7+n90qN8gw/qVGGX2qU4ZcaZfilRhl+qVGGX2qU4Zca9X/C6yAsGWmdaAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "years = list(range(len(images)))\n",
        "imagesg = list()\n",
        "for y in years:\n",
        "    imagesg.append(imageio.imread(str(y) + \"rl_course_medium_maze_PG2_ok.png\"))\n",
        "imageio.mimsave(\"medium_maze_evolution_PG2.gif\", imagesg,fps = 15)\n",
        "files.download(\"medium_maze_evolution_PG2.gif\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "DnofhF_o9Eia",
        "outputId": "65fb489d-2084-429e-8e27-a0b80e39382e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_907950a1-3904-4246-88ba-d9c1db5147f7\", \"medium_maze_evolution_PG2.gif\", 6815136)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import Dict\n",
        "import torch as th\n",
        "def mutate(params: Dict[str, th.Tensor]) -> Dict[str, th.Tensor]:\n",
        "    \"\"\"Mutate parameters by adding normal noise to them\"\"\"\n",
        "    return dict((name, param + th.randn_like(param)) for name, param in params.items())"
      ],
      "metadata": {
        "id": "ldryc7ZddHE3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from stable_baselines3.common.evaluation import evaluate_policy\n",
        "# Include only variables with \"policy\", \"action\" (policy) or \"shared_net\" (shared layers)\n",
        "# in their name: only these ones affect the action.\n",
        "# NOTE: you can retrieve those parameters using model.get_parameters() too\n",
        "mean_params = dict(\n",
        "    (key, value)\n",
        "    for key, value in model.policy.state_dict().items()\n",
        "    if (\"policy\" in key or \"shared_net\" in key or \"action\" in key)\n",
        ")\n",
        "\n",
        "# population size of 50 invdiduals\n",
        "pop_size = 25\n",
        "# Keep top 10%\n",
        "n_elite = pop_size // 5\n",
        "# Retrieve the environment\n",
        "env = model.get_env()\n",
        "\n",
        "for iteration in range(3):\n",
        "    # Create population of candidates and evaluate them\n",
        "    population = []\n",
        "    for population_i in range(pop_size):\n",
        "        candidate = mutate(mean_params)\n",
        "        # Load new policy parameters to agent.\n",
        "        # Tell function that it should only update parameters\n",
        "        # we give it (policy parameters)\n",
        "        model.policy.load_state_dict(candidate, strict=False)\n",
        "        # Evaluate the candidate\n",
        "        fitness, _ = evaluate_policy(model, env)\n",
        "        population.append((candidate, fitness))\n",
        "    # Take top 10% and use average over their parameters as next mean parameter\n",
        "    top_candidates = sorted(population, key=lambda x: x[1], reverse=True)[:n_elite]\n",
        "    mean_params = dict(\n",
        "        (\n",
        "            name,\n",
        "            th.stack([candidate[0][name] for candidate in top_candidates]).mean(dim=0),\n",
        "        )\n",
        "        for name in mean_params.keys()\n",
        "    )\n",
        "    mean_fitness = sum(top_candidate[1] for top_candidate in top_candidates) / n_elite\n",
        "    print(f\"Iteration {iteration + 1:<3} Mean top fitness: {mean_fitness:.2f}\")\n",
        "    print(f\"Best fitness: {top_candidates[0][1]:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QSwCkMDtdBTj",
        "outputId": "93240551-c2b7-4be0-c82b-ac0e30bf4df6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1   Mean top fitness: -1965.35\n",
            "Best fitness: -1965.35\n",
            "Iteration 2   Mean top fitness: -1965.35\n",
            "Best fitness: -1965.35\n",
            "Iteration 3   Mean top fitness: -1965.35\n",
            "Best fitness: -1965.35\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.load_parameters"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "id": "pzCZqAUZciPl",
        "outputId": "8edf656c-e5cc-465a-e0ac-37ffe5342b55"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-86195fc7f0ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'A2C' object has no attribute 'load_parameters'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the Evolution Strategy (ES) object\n",
        "es = cma.CMAEvolutionStrategy(flatten(policy_params), sigma0=1)\n",
        "\n",
        "for iteration in range(10):\n",
        "    # Create population of candidates and evaluate them\n",
        "    candidates, fitnesses = es.ask(), []\n",
        "    for candidate in candidates:\n",
        "        # Load new policy parameters to agent.\n",
        "        model.load_parameters(to_dict(candidate, policy_params), exact_match=False)\n",
        "        # Evaluate the agent using stable-baselines predict function\n",
        "        fitnesses.append(evaluate(model.get_env(), model))\n",
        "    # CMA-ES update\n",
        "    es.tell(candidates, fitnesses)\n",
        "    # Display some training infos\n",
        "    mean_fitness = np.mean(sorted(fitnesses)[:int(0.1 * len(candidates))])\n",
        "    print(\"Iteration {:<3} Mean top 10% reward: {:.2f}\".format(iteration, -mean_fitness))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "id": "IL5epog2bTQ4",
        "outputId": "5126a8a4-fd46-4058-d1ea-565ea9b331b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(16_w,33)-aCMA-ES (mu_w=9.4,w_1=19%) in dimension 20861 (seed=253053, Thu Apr 28 13:51:15 2022)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-04bc2a6510e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcandidate\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcandidates\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;31m# Load new policy parameters to agent.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcandidate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpolicy_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexact_match\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0;31m# Evaluate the agent using stable-baselines predict function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mfitnesses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_env\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'A2C' object has no attribute 'load_parameters'"
          ]
        }
      ]
    }
  ]
}