sandbox_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118\n",
    "\n",
    "pip install torchrl wandb hydra-core tqdm gymnasium gymnasium[classic-control] gymnasium[box2d]\n",
    "\n",
    "\n",
    "Additonall\n",
    "\n",
    "pip install torchvision torchviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Step</th>\n",
       "      <th>DQN</th>\n",
       "      <th>DQN_MIN</th>\n",
       "      <th>DQN_MAX</th>\n",
       "      <th>DQN_PER</th>\n",
       "      <th>DQN_PER_MIN</th>\n",
       "      <th>DQN_PER_MAX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>23.285714</td>\n",
       "      <td>16.571429</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>256</td>\n",
       "      <td>16.277778</td>\n",
       "      <td>15.555556</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>384</td>\n",
       "      <td>17.142857</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>512</td>\n",
       "      <td>18.400000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>12.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>640</td>\n",
       "      <td>14.666667</td>\n",
       "      <td>13.333333</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Step        DQN    DQN_MIN    DQN_MAX    DQN_PER  DQN_PER_MIN  DQN_PER_MAX\n",
       "0   128  23.285714  16.571429  30.000000  30.000000    30.000000    30.000000\n",
       "1   256  16.277778  15.555556  17.000000  17.000000    17.000000    17.000000\n",
       "2   384  17.142857  16.000000  18.285714  18.285714    18.285714    18.285714\n",
       "3   512  18.400000  12.800000  24.000000  12.800000    12.800000    12.800000\n",
       "4   640  14.666667  13.333333  16.000000  16.000000    16.000000    16.000000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load a .csv with pandas\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "# Load the data\n",
    "data = pd.read_csv('data.csv')\n",
    "data.head()\n",
    "\n",
    "# Rename columns as\n",
    "# 'DQN', 'DQN_MIN', 'DQN_MAX', 'DQN_PER', 'DQN_PER_MIN', 'DQN_PER_MAX'\n",
    "data.columns = ['Step','DQN', 'DQN_MIN', 'DQN_MAX', 'DQN_PER', 'DQN_PER_MIN', 'DQN_PER_MAX']\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Step</th>\n",
       "      <th>DQN</th>\n",
       "      <th>DQN_MIN</th>\n",
       "      <th>DQN_MAX</th>\n",
       "      <th>DQN_PER</th>\n",
       "      <th>DQN_PER_MIN</th>\n",
       "      <th>DQN_PER_MAX</th>\n",
       "      <th>DQN_CUMSUM</th>\n",
       "      <th>DQN_PER_CUMSUM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>23.285714</td>\n",
       "      <td>16.571429</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>23.285714</td>\n",
       "      <td>30.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>256</td>\n",
       "      <td>16.277778</td>\n",
       "      <td>15.555556</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>39.563492</td>\n",
       "      <td>47.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>384</td>\n",
       "      <td>17.142857</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>18.285714</td>\n",
       "      <td>56.706349</td>\n",
       "      <td>65.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>512</td>\n",
       "      <td>18.400000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>12.800000</td>\n",
       "      <td>75.106349</td>\n",
       "      <td>78.085714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>640</td>\n",
       "      <td>14.666667</td>\n",
       "      <td>13.333333</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>89.773016</td>\n",
       "      <td>94.085714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Step        DQN    DQN_MIN    DQN_MAX    DQN_PER  DQN_PER_MIN  DQN_PER_MAX  \\\n",
       "0   128  23.285714  16.571429  30.000000  30.000000    30.000000    30.000000   \n",
       "1   256  16.277778  15.555556  17.000000  17.000000    17.000000    17.000000   \n",
       "2   384  17.142857  16.000000  18.285714  18.285714    18.285714    18.285714   \n",
       "3   512  18.400000  12.800000  24.000000  12.800000    12.800000    12.800000   \n",
       "4   640  14.666667  13.333333  16.000000  16.000000    16.000000    16.000000   \n",
       "\n",
       "   DQN_CUMSUM  DQN_PER_CUMSUM  \n",
       "0   23.285714       30.000000  \n",
       "1   39.563492       47.000000  \n",
       "2   56.706349       65.285714  \n",
       "3   75.106349       78.085714  \n",
       "4   89.773016       94.085714  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the cumulative sum of the data\n",
    "\n",
    "data['DQN_CUMSUM'] = data['DQN'].cumsum()\n",
    "data['DQN_PER_CUMSUM'] = data['DQN_PER'].cumsum()\n",
    "data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGvCAYAAAC3lbrBAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABbqUlEQVR4nO3deVhUZf8G8HvYBhCHRWVTEFwRxA0FcclQEpUsX63M/Bm5VYamUm5l7omvW7aYVpZYWia9mYqmKW6VpIaR+5YYpg6aCuPGrM/vD+LkEVQGgWGY+3Ndc10953znzJeDOnfP2RRCCAEiIiIiG2Rn6QaIiIiILIVBiIiIiGwWgxARERHZLAYhIiIislkMQkRERGSzGISIiIjIZjEIERERkc1iECIiIiKb5WDpBqoyk8mECxcuoGbNmlAoFJZuh4iIiEpBCIHr16/D398fdnb3n/NhELqPCxcuICAgwNJtEBERURmcO3cO9erVu28Ng9B91KxZE0DhjlSpVBbuhoiIiEpDo9EgICBA+h6/Hwah+yg6HKZSqRiEiIiIrExpTmvhydJERERksxiEiIiIyGbx0Fg5MBqN0Ov1lm6DysjR0RH29vaWboOIiCyAQeghCCGgVquRl5dn6VboIXl4eMDX15e3SSAisjEMQg+hKAR5e3vD1dWVX6JWSAiBW7du4dKlSwAAPz8/C3dERESViUGojIxGoxSCatWqZel26CG4uLgAAC5dugRvb28eJiMisiE8WbqMis4JcnV1tXAnVB6Kfo8814uIyLYwCD0kHg6rHvh7JCKyTQxCREREZLMYhIiIiMhmMQgRERGRzWIQskEvvPACFAoFFAoFHB0d4ePjg8ceewyfffYZTCaTrHbPnj3o1asXPD094ezsjPDwcCxcuBBGo1FWp1Ao4OzsjD///FO2vE+fPnjhhRcq+kciIiIqEwYhG9WjRw9cvHgRZ8+exffff4+YmBiMHj0ajz/+OAwGAwBg7dq16NKlC+rVq4cdO3bg+PHjGD16NGbNmoVnn30WQgjZNhUKBaZMmWKJH4eIiKzNravAmgTg6hmLtsH7CJUjIQRu640PLqwALo72Zl35pFQq4evrCwCoW7cu2rRpg/bt26Nbt25ISUnBgAEDMHz4cDzxxBP4+OOPpfcNGzYMPj4+eOKJJ7BmzRr0799fWjdy5EgsXLgQ48aNQ/PmzcvvhyMiouqlIB+YG1z43/l/AcO2ARa6epdBqBzd1hsROmWLRT776Iw4uDo93K+za9euaNmyJb799lvUqlULV65cweuvv16srnfv3mjSpAm++uorWRDq2LEjTp48iYkTJyItLe2heiEiompKXwB8/uS/4/gFFgtBAA+N0V1CQkJw9uxZnDx5EgDQrFmze9YV1dwpOTkZmzdvxo8//lihfRIRkRXKPw+87QNc+A1GV28MdZ6PNRcs+3QGzgiVIxdHexydEWexzy4PQgjZIba7zwO6k5OTU7FloaGheP755zFx4kT8/PPP5dITERFVA7euAu+ESsMB117CPuGPs7v+wJOt/KF0sMzjjcyaEQoKCpKuNrrzlZiYCAAoKChAYmIiatWqBTc3N/Tr1w+5ubmybeTk5CA+Ph6urq7w9vbGuHHjpJNzi+zcuRNt2rSBUqlEo0aNkJKSUqyXxYsXIygoCM7OzoiKisK+fftk60vTS3lTKBRwdXKwyKu87ox87NgxBAcHo3HjxtL4XnVNmjQpcd306dNx4MABfPfdd+XSExERWbkLWf+eEwRgvv5p7BOFRxy+GBplsRAEmBmE9u/fj4sXL0qvrVu3AgCefvppAMDYsWOxYcMGpKamYteuXbhw4QL69u0rvd9oNCI+Ph46nQ579uzBihUrkJKSIrvSKDs7G/Hx8YiJiUFWVhbGjBmDYcOGYcuWf8+9+frrr5GUlISpU6fiwIEDaNmyJeLi4qQniJemFypu+/btOHToEPr164e4uDh4eXlhwYIFxerWr1+PU6dO3fOy+ICAAIwcORJvvPFGscvsiYjIxmQsBj7uIg3n6Z/BB8b/AADSRnWCv4eLpTorJB7C6NGjRcOGDYXJZBJ5eXnC0dFRpKamSuuPHTsmAIiMjAwhhBCbNm0SdnZ2Qq1WSzVLliwRKpVKaLVaIYQQ48ePF2FhYbLP6d+/v4iLi5PGkZGRIjExURobjUbh7+8vkpOThRCiVL2URn5+vgAg8vPzi627ffu2OHr0qLh9+3apt1dVJCQkiB49eoiLFy+Kv/76S2RmZoq3335buLm5iccff1wYDAYhhBCpqanC3t5eDB8+XPz+++8iOztbLFu2THh6eorhw4fLtglArF27VhpfuXJFuLu7C2dnZ5GQkFCJP13ZWPPvk4ioyjq7R4ipKun1v8m9RP0JaaL+hDSxZOfpCvvY+31/363MJ0vrdDqsXLkSQ4YMgUKhQGZmJvR6PWJjY6WakJAQBAYGIiMjAwCQkZGB8PBw+Pj4SDVxcXHQaDQ4cuSIVHPnNopqirah0+mQmZkpq7Gzs0NsbKxUU5peSqLVaqHRaGSv6mrz5s3w8/NDUFAQevTogR07duC9997DunXrYG9fOEX51FNPYceOHcjJyUHnzp0RHByMYcOGYeLEibJL6kvi5eWFCRMmoKCgoDJ+HCIiqmp+Wwks7yENX9WNRJL+FQDAkI7BeLlLQ0t1JlPmk6W/++475OXlSYdH1Go1nJyc4OHhIavz8fGBWq2Wau4MQUXri9bdr0aj0eD27du4du0ajEZjiTXHjx8vdS8lSU5OxvTp0x/8w1u5lJSUEs+7Kknnzp2xefNmAIXnXT355JNISUnB4MGDUadOHalOlHBS9aRJkzBp0qRy6ZmIiKzI/mXAxtekYV/tNBwQheeVvvhIA7zRq+Qrki2hzDNCn376KXr27Al/f//y7MeiJk2ahPz8fOl17tw5S7dUpTg7O2PdunV4/vnnsXv3bku3Q0REVdHF32Uh6HndBCkEvT+gdZUKQUAZZ4T+/PNPbNu2Dd9++620zNfXFzqdDnl5ebKZmNzcXOkOxr6+vsWu7iq6kuvOmruv7srNzYVKpYKLiwvs7e1hb29fYs2d23hQLyVRKpVQKpWl3Au2ydnZGRMnTrR0G0REVBWd2gqsekoavqQbg92mlgCAz4dE4pEmde71Tosp04zQ8uXL4e3tjfj4eGlZREQEHB0dkZ6eLi07ceIEcnJyEB0dDQCIjo7GoUOHZFd3bd26FSqVCqGhoVLNndsoqinahpOTEyIiImQ1JpMJ6enpUk1peiEiIqJy9NevshA0UT8MW0yRAIBvXo6ukiEIKMOMkMlkwvLly5GQkAAHh3/f7u7ujqFDhyIpKQleXl5QqVQYNWoUoqOj0b59ewBA9+7dERoaikGDBmHu3LlQq9WYPHkyEhMTpZmYl19+GR988AHGjx+PIUOGYPv27VizZg02btwofVZSUhISEhLQtm1bREZGYtGiRbh58yYGDx5c6l6IiIionFzNBpZ1k4bT9M9jtbErAGBdYke0DPCwUGMPZnYQ2rZtG3JycjBkyJBi69555x3Y2dmhX79+0Gq1iIuLw4cffiitt7e3R1paGkaMGIHo6GjUqFEDCQkJmDFjhlQTHByMjRs3YuzYsXj33XdRr149LFu2DHFx/96xuX///rh8+TKmTJkCtVqNVq1aYfPmzbITqB/UCxEREZUDzQXgvVbS8L/6Z5FiLLxa7P0Brat0CAIAhSjpch8CAGg0Gri7uyM/Px8qlUq2rqCgANnZ2QgODoazs7OFOqTywt8nEVEZ5J0DFjWXhsn6AfjI2BsAsPyFdogJ8bZIW/f7/r4bnzVGRERE5ss9Ciz597zbP0x+Ugj6aUIM6nm6Wqozs/Dp80RERGQe3U1ZCEoztkc3XeEjmb4cHmU1IQjgjBARERGZ43Ye8N/60tAkFBipfxUAsGFkJ4TXc7dQY2XDGSEiIiIqnetqWQhaYXgMDbSrAAD/G9HB6kIQwCBkk1544QUoFAooFAo4OjrCx8cHjz32GD777DOYTCZZ7Z49e9CrVy94enrC2dkZ4eHhWLhwYbGnyisUCjg7O+PPP/+ULe/Tp889n1J/v76cnJzQqFEjzJgxAwaDAQCwc+dOaf3dr6JHp0ybNk1aZm9vj4CAALz44ou4evVqGfcWEREBAAw6YEFTaXhZuGOqofC2Nb9OjkVEfU9LdfZQGIRsVI8ePXDx4kWcPXsW33//PWJiYjB69Gg8/vjjUvBYu3YtunTpgnr16mHHjh04fvw4Ro8ejVmzZuHZZ58t9nwxhUKBKVOmlEtfp06dwmuvvYZp06Zh3rx5spoTJ07g4sWLspe3979XJoSFheHixYvIycnB8uXLsXnzZowYMeKh+iIismlCALPkN0Rsp10CAPjtrcdQ2816n8rAc4TKkxCA/pZlPtvRFVAoSl2uVCqlx43UrVsXbdq0Qfv27dGtWzekpKRgwIABGD58OJ544gnZk+aHDRsGHx8fPPHEE1izZg369+8vrRs5ciQWLlyIcePGoXnz5sU+09y+RowYgbVr12L9+vWyh7d6e3sXe6DunRwcHGQ/29NPP43ly5eXqR8iIpunuwnMlj9XNKjgSwDAyVk94eRg3XMqDELlSX+r2B+WSvPGBcCpxkNtomvXrmjZsiW+/fZb1KpVC1euXMHrr79erK53795o0qQJvvrqK1kQ6tixI06ePImJEyciLS3toXop4uLigitXrpT5/WfPnsWWLVvg5ORULv0QEdmcL/vLhkUh6OC07lYfggAeGqO7hISE4OzZszh58iQAoFmzkp8SHBISItXcKTk5GZs3b8aPP/74UH0IIbBt2zZs2bIFXbt2la2rV68e3NzcpFdYWJhs/aFDh+Dm5gYXFxcEBwfjyJEjmDBhwkP1Q0Rkc0xG4MJvQHAXaVGDgpUAgG1JXaBydrRUZ+WKM0LlydG1cGbGUp9dDoQQUNxxiO1+Nx4vaZYlNDQUzz//PCZOnIiff/7Z7M9PS0uDm5sb9Ho9TCYTnnvuOUybNk1W8+OPP6JmzZrS2NFR/pexadOmWL9+PQoKCrBy5UpkZWVh1KhRZvdCRGSzjHpgZm0AwGrv0fhcOxtHRX0AChyc1r3ahCCAQah8KRQPfXjK0o4dO4bg4GA0btxYGnfo0KHEulatWpW4jenTp6NJkyb47rvvzP78mJgYLFmyBE5OTvD395c92LdIcHDwfc8RKrriDADmzJmD+Ph4TJ8+HTNnzjS7HyIim2MySiEIAFwv7sNRUfg/k2mjOlWrEATw0BjdYfv27Th06BD69euHuLg4eHl5YcGCBcXq1q9fj1OnTt3zsviAgACMHDkSb7zxRrHL7B+kRo0aaNSoEQIDA0sMQWUxefJkzJ8/HxcuWGi2jojIWhgNwAwv2aJX9YUhaOOrndC8rvXdJ+hBGIRslFarhVqtxvnz53HgwAHMnj0bTz75JB5//HE8//zzqFGjBj766COsW7cOL774Ig4ePIizZ8/i008/xQsvvIDhw4ejV69e99z+pEmTcOHCBWzbtq3ce7906RLUarXspdfr71kfHR2NFi1aYPbs2eXeCxFRtXHzCjCzlmxRUEHhzRI/e6EtwvyrXwgCGIRs1ubNm+Hn54egoCD06NEDO3bswHvvvYd169bB3t4eAPDUU09hx44dyMnJQefOnREcHIxhw4Zh4sSJskvqS+Ll5YUJEyagoKCg3Htv2rQp/Pz8ZK/MzMz7vmfs2LFYtmwZzp07V+79EBFZvZxfgHkNZIsKrw5TYO8b3dA1xMcyfVUChbjf2bA2TqPRwN3dHfn5+VCpVLJ1BQUFyM7ORnBwMJydnS3UYeUqKCjAk08+iXPnzmHXrl2oU6fOg99kJWzx90lEBADIPQIs+fdc0HH6F5FqfBQA8MXQSHRubH3/1t/v+/tunBGiUnN2dsa6devw/PPPY/fu3ZZuh4iIHtblE7IQ9LupgRSCPh4UYZUhyFy8aozM4uzsjIkTJ5r9vpycHISGht5z/dGjRxEYGPgwrRERkTl+fg/Y+pY0nKQfiq+M3QAAq4ZFoWOj2vd6Z7XCIESVwt/fH1lZWfddT0RElSRjsSwE/c/YSQpB7z7bymZCEMAgRJXEwcFBurcPERFZ0MFUYMsb0nCp4XHMMTwHABgX1xRPtqprqc4sgkHoIfFc8+qBv0cisgn7PwU2JknDV3Ujsd5UeI7Qwmdaom+bepbqzGIYhMqo6LEOt27dgouLi4W7oYd169YtAMUf10FEVG38uABInyENB+jeRIap8FmNAyIDbDIEAQxCZWZvbw8PDw9cunQJAODq6ip7RhdZByEEbt26hUuXLsHDw0O6hxIRUbXy9SDg2Hpp+FW9ycg4XXgBywsdgjDtibB7vbPaYxB6CL6+vgAghSGyXh4eHtLvk4ioWjm3XxaCpigS4eL/ONoZrmFyfChaBnhYrrcqgEHoISgUCvj5+cHb2/u+j3igqs3R0ZEzQURUPV08CHwaKw0n6odhtbEjah/4C9tff7TaPUC1LBiEyoG9vT2/SImIqGq565yg9w19sNrYFQCw4JlWDEH/YBAiIiKqbk5tk4Wgl3RjscXUDgAw48kwdGlS/e8YXVp8xAYREVF1cuUPYFU/aXjMFCCFoKm9Q/F8dJCFGquaGISIiIiqixuXgPfbSMMfjBHoqfsvAODxFn4Y3DHYUp1VWTw0RkREVB3czgPmN5aGdz5FfvZ/wvFcFJ/nWBIGISIiImt36yow99/Znqn6BCkE7Xj9UQTXrmGhxqo+HhojIiKyZtdzZSFoseEJrDDGAQC+HB7FEPQAnBEiIiKyVheygI+7SMOfjGGYZ3gWALB7XAwCa7laqDHrwRkhIiIia3T+gCwEaYUD/k//JgDgfyM6MASVEoMQERGRtbmuBj6JkYbv6PuhqfZzAMDyF9ohor6npTqzOjw0RkREZE0MOmBBU2m41xSCd42F9w3a+0Y3+KicLdWZVWIQIiIishbX1bIQ9Jb+BXxh7A4A+PaVDgxBZWD2obHz58/j//7v/1CrVi24uLggPDwcv/76q7ReCIEpU6bAz88PLi4uiI2NxalTp2TbuHr1KgYOHAiVSgUPDw8MHToUN27ckNUcPHgQnTt3hrOzMwICAjB37txivaSmpiIkJATOzs4IDw/Hpk2bZOtL0wsREZFVyD8vC0Gjda9IIWhR/1ZoE8jDYWVhVhC6du0aOnbsCEdHR3z//fc4evQoFixYAE/Pf3f+3Llz8d5772Hp0qXYu3cvatSogbi4OBQUFEg1AwcOxJEjR7B161akpaVh9+7dePHFF6X1Go0G3bt3R/369ZGZmYl58+Zh2rRp+Pjjj6WaPXv2YMCAARg6dCh+++039OnTB3369MHhw4fN6oWIiKjKu64G3gmVhpuMkVhn6gQASBncDn1a17VUZ9ZPmGHChAmiU6dO91xvMpmEr6+vmDdvnrQsLy9PKJVK8dVXXwkhhDh69KgAIPbv3y/VfP/990KhUIjz588LIYT48MMPhaenp9BqtbLPbtq0qTR+5plnRHx8vOzzo6KixEsvvVTqXh4kPz9fABD5+fmlqiciIip3Br0QU1XSa9PkbqL+hDRRf0Ka+OWPvy3dXZVkzve3WTNC69evR9u2bfH000/D29sbrVu3xieffCKtz87OhlqtRmxsrLTM3d0dUVFRyMjIAABkZGTAw8MDbdu2lWpiY2NhZ2eHvXv3SjWPPPIInJycpJq4uDicOHEC165dk2ru/JyimqLPKU0vd9NqtdBoNLIXERGRxRh0wNJO0vADw5MYoR8LAPjm5WhENahlqc6qDbOC0JkzZ7BkyRI0btwYW7ZswYgRI/Dqq69ixYoVAAC1Wg0A8PHxkb3Px8dHWqdWq+Ht7S1b7+DgAC8vL1lNSdu48zPuVXPn+gf1crfk5GS4u7tLr4CAgAftEiIiooph1APrRwGXjwEAPjX0xHxDfwDA3H4t0DbIy5LdVRtmXTVmMpnQtm1bzJ49GwDQunVrHD58GEuXLkVCQkKFNFiZJk2ahKSkJGms0WgYhoiIqPIZ9cDM2tJwiaE3/msYAAD4fEgkHmlSx1KdVTtmzQj5+fkhNDRUtqxZs2bIyckBAPj6+gIAcnNzZTW5ubnSOl9fX1y6dEm23mAw4OrVq7KakrZx52fcq+bO9Q/q5W5KpRIqlUr2IiIiqlS6m8BnPaThMN1rUgj6YewjDEHlzKwg1LFjR5w4cUK27OTJk6hfvz4AIDg4GL6+vkhPT5fWazQa7N27F9HR0QCA6Oho5OXlITMzU6rZvn07TCYToqKipJrdu3dDr9dLNVu3bkXTpk2lK9Sio6Nln1NUU/Q5pemFiIioSvn7NDDbHzhfeFuaV3WJ2GaKAAAcmR6HJj41Ldld9WTOWdj79u0TDg4O4u233xanTp0Sq1atEq6urmLlypVSzZw5c4SHh4dYt26dOHjwoHjyySdFcHCwuH37tlTTo0cP0bp1a7F3717x008/icaNG4sBAwZI6/Py8oSPj48YNGiQOHz4sFi9erVwdXUVH330kVTz888/CwcHBzF//nxx7NgxMXXqVOHo6CgOHTpkVi/3w6vGiIio0lzLkV0d9tGbA6Srwy7mle57iwqZ8/1tVhASQogNGzaI5s2bC6VSKUJCQsTHH38sW28ymcRbb70lfHx8hFKpFN26dRMnTpyQ1Vy5ckUMGDBAuLm5CZVKJQYPHiyuX78uq/n9999Fp06dhFKpFHXr1hVz5swp1suaNWtEkyZNhJOTkwgLCxMbN240u5f7YRAiIqJKcfOKLAR9MzleCkFGo8nS3Vkdc76/FUIIYdk5qapLo9HA3d0d+fn5PF+IiIgqRkE+MCdQtiio4EsAwB+ze8HeTmGJrqyaOd/ffNYYERGRpegL7hmCzszuBTuGoApn9rPGiIiIqBxcVwNvy+91VxSCTr/dkyGokjAIERERVbbjm2QPUAWAoIJVAIBTb/eEgz2/nisL9zQREVFlunwSWD1AGh4x1f9nJkiBU2/3hCNDUKXi3iYiIqost64Ci9tJw6vCDfG6wqc1MARZBvc4ERFRZbh1FZgbLA23GiPQRvsRAAUOTevOEGQhvGqMiIioot24DMxvJA3f0ffDu8Z+AIATs3pA6WBvqc5sHoMQERFRRdJel4Wghfqn8J6xLwDg18mxDEEWxnk4IiKiinLzbyC5njT81NBTCkGZk2NR201pqc7oHwxCREREFcGoB+Y1lIYphu6YaRgEANj3ZjfUYgiqEhiEiIiIyptBB8ysLQ03GSMxzfACAGDPxK7wrulsocbobgxCRERE5el6LjCrjjQ8YGqEV/RjAAA/T+wKfw8XCzVGJWEQIiIiKi+3rwELmsgW9dXNAACsGhaFugxBVQ6DEBERUXm4dRX4b5BsUdFjMxY83RIdG9Uu4U1kaQxCRERED8uol90scaWhm/TYjC+HR6FfRL17v5csivcRIiIiehintgGr+knDifphWG3sCgBYMrANOjTkTFBVxiBERERUVtdzZSFoqO41pJsiAAA/TYhBPU9XS3VGpcQgREREVBbfvQJkrZKGc/X9pRC08/VHGYKsBIMQERGRudLGykJQX+00HBBNEOJbE9+M6AA3Jb9erQV/U0RERObYPgv49TNp2E07D3+IugCA/43ogBoMQVaFvy0iIqLS2vImkPGBNHxE+w5yhE/hqjGPMARZIf7GiIiISmPty8DvX0nD53UTpBC0LekRNPKuaanO6CEwCBERET3I/k9lIaiHdg6Oi0AAwPbXuqBBHTdLdUYPiTdUJCIiup+sL4GNSdJwgO5NKQR9/WJ7hiArxxkhIiKie/miL/BHujTsrv0vTooAAMBXw9sjqkEtS3VG5YQzQkRERCVJeVwWgk51+QAjnnkcTX1qYsPITohuyBBUHXBGiIiI6G5zAoGCfGnYrmAxLm/xxJqXXLHx1U5wsOc8QnXB3yQREdGdVg+UhaCu2vm4DE8AQOtAD4agaoYzQkREREVmegNGrTRsU7AUV6ECAOx9oxscGYKqHf5GiYiIACD1BVkIekw7VwpB6a91gY/K2UKNUUXijBAREdH+ZcCRtdKwr3YaTol6AIBfJnWDrztDUHXFIERERLZt3yfAptel4RPamTgoGgIAzszuBTs7haU6o0rAQ2NERGS7jqyVhaCntVOkEPQHQ5BNYBAiIiLbdGpr4XlB/+ivfQv7RUjhqrd7wp4hyCbw0BgREdmeM7uAVU9Jw1d1idgrmgEonAliCLIdDEJERGRb1iUCv62UhvHa2TgiggAAx2f2YAiyMWYdGps2bRoUCoXsFRISIq0vKChAYmIiatWqBTc3N/Tr1w+5ubmybeTk5CA+Ph6urq7w9vbGuHHjYDAYZDU7d+5EmzZtoFQq0ahRI6SkpBTrZfHixQgKCoKzszOioqKwb98+2frS9EJERDbmYKosBPXTTsUREQQPV0ecmd0Lzo72FmyOLMHsc4TCwsJw8eJF6fXTTz9J68aOHYsNGzYgNTUVu3btwoULF9C3b19pvdFoRHx8PHQ6Hfbs2YMVK1YgJSUFU6ZMkWqys7MRHx+PmJgYZGVlYcyYMRg2bBi2bNki1Xz99ddISkrC1KlTceDAAbRs2RJxcXG4dOlSqXshIiIbsyMZ+HaYNByvH45M0RQAkDWlO0+MtlXCDFOnThUtW7YscV1eXp5wdHQUqamp0rJjx44JACIjI0MIIcSmTZuEnZ2dUKvVUs2SJUuESqUSWq1WCCHE+PHjRVhYmGzb/fv3F3FxcdI4MjJSJCYmSmOj0Sj8/f1FcnJyqXspjfz8fAFA5Ofnl/o9RERUxZhMQqweKMRUlfR6blKyqD8hTTR+Y5PQ6o2W7pDKmTnf32bPCJ06dQr+/v5o0KABBg4ciJycHABAZmYm9Ho9YmNjpdqQkBAEBgYiIyMDAJCRkYHw8HD4+PhINXFxcdBoNDhy5IhUc+c2imqKtqHT6ZCZmSmrsbOzQ2xsrFRTml5KotVqodFoZC8iIrJyWauAYxukYRftQvxsCgdQeE6QkwMvoLZlZv32o6KikJKSgs2bN2PJkiXIzs5G586dcf36dajVajg5OcHDw0P2Hh8fH6jVagCAWq2WhaCi9UXr7lej0Whw+/Zt/P333zAajSXW3LmNB/VSkuTkZLi7u0uvgICA0u0YIiKqmr59sfDk6H88p3sDfwpfALxZIhUy66qxnj17Sv/dokULREVFoX79+lizZg1cXFzKvbnKNmnSJCQlJUljjUbDMEREZK1ObQUOfi0NO2vfwTlR+D/RvFkiFXmo+UAPDw80adIEp0+fhq+vL3Q6HfLy8mQ1ubm58PUtTN++vr7FrtwqGj+oRqVSwcXFBbVr14a9vX2JNXdu40G9lESpVEKlUsleRERkhbZNl90n6BntW1IIOjmLN0ukfz1UELpx4wb++OMP+Pn5ISIiAo6OjkhPT5fWnzhxAjk5OYiOjgYAREdH49ChQ7Kru7Zu3QqVSoXQ0FCp5s5tFNUUbcPJyQkRERGyGpPJhPT0dKmmNL0QEVE1tSYB+GmhNOyqnY99/9wscf+bsTwniGTMOjT2+uuvo3fv3qhfvz4uXLiAqVOnwt7eHgMGDIC7uzuGDh2KpKQkeHl5QaVSYdSoUYiOjkb79u0BAN27d0doaCgGDRqEuXPnQq1WY/LkyUhMTIRSqQQAvPzyy/jggw8wfvx4DBkyBNu3b8eaNWuwceNGqY+kpCQkJCSgbdu2iIyMxKJFi3Dz5k0MHjwYAErVCxERVUNfPQec+Pf74iXdGJwR/gCA02/3hIM9QxDJmRWE/vrrLwwYMABXrlxBnTp10KlTJ/zyyy+oU6cOAOCdd96BnZ0d+vXrB61Wi7i4OHz44YfS++3t7ZGWloYRI0YgOjoaNWrUQEJCAmbMmCHVBAcHY+PGjRg7dizeffdd1KtXD8uWLUNcXJxU079/f1y+fBlTpkyBWq1Gq1atsHnzZtkJ1A/qhYiIqpnsH2UhqJd2No7+c8foI9PjGIKoRAohhLB0E1WVRqOBu7s78vPzeb4QEVFVlncOWNRcGo7Tv4hU46MAgH1vdIO3ytlCjZElmPP9zWeNERGRdcveDXz37yXyvbWzcEg0AADsmdiVIYjui0GIiIis16mtsqvDogvex0XUQg0ne+wcF4M6NZUWbI6sAYMQERFZp5xfpBB03bE2+t9IwkXUAgDsfTMWbkp+xdGD8U8JERFZn5/fA7a+BQD4064eul2fDcM/X2lHZ8TB1Ylfb1Q6PIWeiIisy7E0KQQBQK9b06UQtHtcDEMQmYV/WoiIyHpsfgP4ZbE07KJdiJsofMTTj+NjEODlaqnOyEoxCBERkXU4f0AWgrpq50sPUN37Rjf48OowKgMeGiMioqrvr1+BT2KkYV/tNOmO0b9MYgiismMQIiKiqu3sT8CybtLwdf1LOCCaAAC+S+wIX3eGICo7HhojIqKqa+ccYGeyNHxBNx47Ta0AAFvHPoLGPjUt1BhVFwxCRERUNX3xH+CP7dJwmO41KQQteLolQxCVCwYhIiKqeo6uk4WgQbqJ+NHUAgDw+ZBIPNKkjqU6o2qGQYiIiKqWuy6Rf1S7AGeFHwBgTt9whiAqVwxCRERUdaTPlIWgztp3cE74AACW/l8b9GjuZ6nOqJpiECIioqrhoy7AxSxpGFbwqXSzxP5tAxiCqEIwCBERkeV9HCMLQW0LlkghaEBkIJL7hluoMaruGISIiMiyfv8auHBAGrYuWIprUAEAYpt5MwRRhWIQIiIiy9mRDOyaIw3vDEHfvtIBbQI9LdUZ2QgGISIisoz/DQMOpUrD0ILPcAuFd4me1ac5QxBVCgYhIiKqfBmLZSEorOBTKQR9MTQSnRvzEnmqHHzWGBERVa6fFgFb3pCGz2jfkk6Mnv5EGEMQVSrOCBERUeU5vQ3YNlUaPqJ9Bzn/3Cdo8XNtEN+Cl8hT5WIQIiKiypF3DljZTxr2006VQtBHgyIQF+Zrqc7IhvHQGBERVbzfVgKLmkvDHto5yBRNAQAJ0fUZgshiOCNEREQV6647RkcXvI+LqAWg8D5B059sfo83ElU8zggREVHFObVNFoLe1A+RQtDAqEAsS2hnocaICnFGiIiIKsafGcCqf88Jmq9/GquMsQCAyfHNMKxzA0t1RiRhECIiovJ3dD2wZpA0nKZ/HinGHgCApf8XgR7NeU4QVQ08NEZEROVLfVgWgmbqB0ohqGuIN0MQVSkMQkREVH6ungGWdpSG3xo74VNjPACgsbcbPnuB5wRR1cIgRERE5ePaWeC91tLwa8OjSNK/AgBoWc8dW5O6WKgxonvjOUJERPTwblwG3m0pDRfon8L7xr4AgPq1XLFuZCdLdUZ0X5wRIiKih1OQDyzrJg0/MsRLIahlgAd2jYuxVGdED8QZISIiKrvbecB/60vDyfrBWGl8DACw5qVoRAZ7WagxotLhjBAREZWNELIQlGaMkkLQkoFtGILIKjAIERGR+UwmYOsUaZhjqoOR+tEAgOS+4egZzqfIk3V4qCA0Z84cKBQKjBkzRlpWUFCAxMRE1KpVC25ubujXrx9yc3Nl78vJyUF8fDxcXV3h7e2NcePGwWAwyGp27tyJNm3aQKlUolGjRkhJSSn2+YsXL0ZQUBCcnZ0RFRWFffv2ydaXphciIjKTQQfM8AT2vIcbLv6Yqk/AI7p3AQDvPtsKAyIDLdwgUemVOQjt378fH330EVq0aCFbPnbsWGzYsAGpqanYtWsXLly4gL59+0rrjUYj4uPjodPpsGfPHqxYsQIpKSmYMuXf/7PIzs5GfHw8YmJikJWVhTFjxmDYsGHYsmWLVPP1118jKSkJU6dOxYEDB9CyZUvExcXh0qVLpe6FiIjMZNABs+pIwyn5T2CFMQ4A8MXQSDzZqq6lOiMqG1EG169fF40bNxZbt24VXbp0EaNHjxZCCJGXlyccHR1FamqqVHvs2DEBQGRkZAghhNi0aZOws7MTarVaqlmyZIlQqVRCq9UKIYQYP368CAsLk31m//79RVxcnDSOjIwUiYmJ0thoNAp/f3+RnJxc6l4eJD8/XwAQ+fn5paonIqrWrucKMVUle9WfsEHUn5AmNvx+3tLdEUnM+f4u04xQYmIi4uPjERsbK1uemZkJvV4vWx4SEoLAwEBkZGQAADIyMhAeHg4fHx+pJi4uDhqNBkeOHJFq7t52XFyctA2dTofMzExZjZ2dHWJjY6Wa0vRyN61WC41GI3sREREKzwma31i2KKjgSwAKjOraCI+38LdMX0QPyezL51evXo0DBw5g//79xdap1Wo4OTnBw8NDttzHxwdqtVqquTMEFa0vWne/Go1Gg9u3b+PatWswGo0l1hw/frzUvdwtOTkZ06dPv89PT0Rkg4QoPCfoDkEFqwAAz7YLwGvdm1qiK6JyYdaM0Llz5zB69GisWrUKzs7OFdWTxUyaNAn5+fnS69y5c5ZuiYjI8u64WSJQFIIU+G+/cMzp16Lk9xBZCbOCUGZmJi5duoQ2bdrAwcEBDg4O2LVrF9577z04ODjAx8cHOp0OeXl5svfl5ubC17fwacO+vr7FrtwqGj+oRqVSwcXFBbVr14a9vX2JNXdu40G93E2pVEKlUsleREQ2SwhgTiBwPhMAcE24SYfDnosKRP92vDqMrJ9ZQahbt244dOgQsrKypFfbtm0xcOBA6b8dHR2Rnp4uvefEiRPIyclBdHQ0ACA6OhqHDh2SXd21detWqFQqhIaGSjV3bqOopmgbTk5OiIiIkNWYTCakp6dLNREREQ/shYiI7mO6R+HjMwDohT1aaz8GAKx9pQNm/yfcgo0RlR+zzhGqWbMmmjdvLltWo0YN1KpVS1o+dOhQJCUlwcvLCyqVCqNGjUJ0dDTat28PAOjevTtCQ0MxaNAgzJ07F2q1GpMnT0ZiYiKUSiUA4OWXX8YHH3yA8ePHY8iQIdi+fTvWrFmDjRs3Sp+blJSEhIQEtG3bFpGRkVi0aBFu3ryJwYMHAwDc3d0f2AsREd3DNHfZsLH2CwDAxlc7IczfvaR3EFmlcn/W2DvvvAM7Ozv069cPWq0WcXFx+PDDD6X19vb2SEtLw4gRIxAdHY0aNWogISEBM2bMkGqCg4OxceNGjB07Fu+++y7q1auHZcuWIS4uTqrp378/Ll++jClTpkCtVqNVq1bYvHmz7ATqB/VCRER3EaJwJugOwQUrAQDfvBzNEETVjkIIISzdRFWl0Wjg7u6O/Px8ni9ERLbhrpmgBgUrYYIdPh4Uge5hJZ9fSVTVmPP9zWeNERERYDQUC0FNC1Jggh0WP9eGIYiqrXI/NEZERFZGCGBmLdmi4IKVELDD9te6oEEdNws1RlTxOCNERGTL7jon6KLwQlDBlxCww/sDWjMEUbXHGSEiIltl0AHrXpEtitZ+AADYM7Er/D1cLNEVUaViECIiskVGvewp8jphjybazwEAnzzfliGIbAYPjRER2aKZtaX/TDU88k8IUmD2f8LxWKjPvd9HVM1wRoiIyJaYjMAML9micYaXAQBfDI1E58Z1SnoXUbXFIEREZCtKCEGFzw4DvhwWhQ6Napf0LqJqjUGIiMgWGA3FLpEvfIo8sOnVzgj1501jyTYxCBERVXcGnezEaABoXFB4TlDGpK7wc+eJ0WS7eLI0EVF1ZtDKQtBuYziCCr6EHg6Y268FQxDZPAYhIqLqbJa3bPi8fhIA4LXHmuCZdgGW6IioSuGhMSKi6qqEB6gC4ANUie7AIEREVN3c9dgMoOjEaAV+GPsImvjUtEhbRFURgxARUXVzjxC04/VHEVy7hkVaIqqqeI4QEVF1IQTw8aPSsEA4SiHo21c6MAQRlYBBiIioOig6HHbhNwDALmMLhGhXAFDgfyOi0SbQ06LtEVVVPDRGRGTt7jon6DtjB4zRJwIAD4cRPQCDEBGRtbvrnKAx+pEAgN+ndoe7i6MFGiKyHjw0RkRkrYz6YpfIFz077MfxMQxBRKXAIEREZI1MJmCm/CGpRfcJWv5COwR4uVqiKyKrw0NjRETWxmQCZshPfm5SsAIm2GHfG93grXK2UGNE1oczQkRE1qQgXxaCzpnqIKjgS+jgiB2vP8oQRGQmzggREVmL/PPAO6GyRZ117wIATr/dEw72/H9bInPxbw0RkTW4kFUsBBWdGH1iVg+GIKIy4owQEVFVl3sE+LiLbFFRCMpO7gWFQmGJroiqBf4vBBFRVXY6HVjSQRp+ZIiXQtCZ2QxBRA+LQYiIqKr6MwNY2VcafmWIQbJhIADg+MwesLNjCCJ6WDw0RkRUFV3IApb3kIaDdeOww9QaAA+HEZUnzggREVU1+edl5wR9bngMO0ytYadgCCIqbwxCRERVyV2XyK83RmOKYTAA4A+eE0RU7nhojIioqrh0DPiwvTR8z9AHCw3PAOBMEFFFYRAiIqoKrmbLQtAUfQI+N8YBKLxPEEMQUcXgoTEiIku7fBJ4r5U03G9qIoWg4zN7QOlgb6HGiKo/zggREVnSX78Cy7pJw+3GVhiiHw8AODsn3lJdEdkMzggREVnK8U2yEJRlaiiFoNNv97RUV0Q2xawgtGTJErRo0QIqlQoqlQrR0dH4/vvvpfUFBQVITExErVq14Obmhn79+iE3N1e2jZycHMTHx8PV1RXe3t4YN24cDAaDrGbnzp1o06YNlEolGjVqhJSUlGK9LF68GEFBQXB2dkZUVBT27dsnW1+aXoiILOboOmD1ANmiPrqZAIDfp3bns8OIKolZf9Pq1auHOXPmIDMzE7/++iu6du2KJ598EkeOHAEAjB07Fhs2bEBqaip27dqFCxcuoG/ff++KajQaER8fD51Ohz179mDFihVISUnBlClTpJrs7GzEx8cjJiYGWVlZGDNmDIYNG4YtW7ZINV9//TWSkpIwdepUHDhwAC1btkRcXBwuXbok1TyoFyIii3nbH1jzvDT8zthB9tgMdxdHS3VGZHvEQ/L09BTLli0TeXl5wtHRUaSmpkrrjh07JgCIjIwMIYQQmzZtEnZ2dkKtVks1S5YsESqVSmi1WiGEEOPHjxdhYWGyz+jfv7+Ii4uTxpGRkSIxMVEaG41G4e/vL5KTk4UQolS9lEZ+fr4AIPLz80v9HiKi+/ppkRBTVdJryhujRP0JaaLdrK3CZDJZujuiasGc7+8yz70ajUasXr0aN2/eRHR0NDIzM6HX6xEbGyvVhISEIDAwEBkZGQCAjIwMhIeHw8fHR6qJi4uDRqORZpUyMjJk2yiqKdqGTqdDZmamrMbOzg6xsbFSTWl6KYlWq4VGo5G9iIjKzeZJwNZ/Z8CTg1fAtfMreOXRhtj3ZiwvkSeyALOvGjt06BCio6NRUFAANzc3rF27FqGhocjKyoKTkxM8PDxk9T4+PlCr1QAAtVotC0FF64vW3a9Go9Hg9u3buHbtGoxGY4k1x48fl7bxoF5KkpycjOnTp5duRxARmePL/sDJzdIwquAD5B5zRFqsH5rXdbdgY0S2zewZoaZNmyIrKwt79+7FiBEjkJCQgKNHj1ZEb5Vu0qRJyM/Pl17nzp2zdEtEVB1smy4LQZEFi5ELL4T41kSYv8qCjRGR2TNCTk5OaNSoEQAgIiIC+/fvx7vvvov+/ftDp9MhLy9PNhOTm5sLX19fAICvr2+xq7uKruS6s+buq7tyc3OhUqng4uICe3t72Nvbl1hz5zYe1EtJlEollEqlGXuDiOgBDn0D/LRQGvbVTsMleKJHmC+WDoqwYGNEBJTDfYRMJhO0Wi0iIiLg6OiI9PR0ad2JEyeQk5OD6OhoAEB0dDQOHToku7pr69atUKlUCA0NlWru3EZRTdE2nJycEBERIasxmUxIT0+XakrTCxFRhTuYCvxvqDSM176NA6IJvGo4MQQRVRFmzQhNmjQJPXv2RGBgIK5fv44vv/wSO3fuxJYtW+Du7o6hQ4ciKSkJXl5eUKlUGDVqFKKjo9G+feHzc7p3747Q0FAMGjQIc+fOhVqtxuTJk5GYmCjNxLz88sv44IMPMH78eAwZMgTbt2/HmjVrsHHjRqmPpKQkJCQkoG3btoiMjMSiRYtw8+ZNDB5c+ITm0vRCRFShTm8Dvh0mDR/VLsBZ4QcAOPDWY5bqiojuZs7laEOGDBH169cXTk5Ook6dOqJbt27ihx9+kNbfvn1bvPLKK8LT01O4urqK//znP+LixYuybZw9e1b07NlTuLi4iNq1a4vXXntN6PV6Wc2OHTtEq1athJOTk2jQoIFYvnx5sV7ef/99ERgYKJycnERkZKT45ZdfZOtL08uD8PJ5IiqTzM9ll8i3nbBS1J+QJupPSBNGIy+RJ6po5nx/K4QQwtJhrKrSaDRwd3dHfn4+VCqe0EhEpfBXJrCsqzTsq52GA6IJAr1csXt8jAUbI7Id5nx/8x7uRETl5c89shDUX/sWDogm6N3SnyGIqIpiECIiKg8H1wDL/31Q6rO6ydgrmgEA3nu2lYWaIqIHYRAiInpYR9cD3w6XhvP0z+AXU+GVsNnJvXjHaKIqzOz7CBER0R1OpwNrBknDVMMjWGzsAwA4OyfeQk0RUWkxCBERldXF34GVfaXhh4YnMNfwLADg5Kye93oXEVUhDEJERGVxaiuw6ilp+JJuLLaY2gHgTBCRNWEQIiIy16Fv7rpj9GwcEUGoU1OJ/W/GWrAxIjIXT5YmIjLHxYOyEPS2/jkcEUFwdbJnCCKyQgxCRESldekY8FFnafidsQM+MT6OZ9rWw9EZPSzYGBGVFQ+NERGVxv5PgY1J0jBWOxenRT2onB0w96mWFmyMiB4GZ4SIiB7k0jFZCJqp/z+cFvXgXVOJg9PiLNgYET0szggREd3P+Uzgk38fmzFP/ww+NfbCZy+0RdcQHws2RkTlgTNCRET3kntUFoK+MMRisbEPQnxrMgQRVROcESIiKsmxNODrgdJwgO5NZJjC8ERLf7w3oLUFGyOi8sQgRER0t79PyUJQvPZtHBHB8HN3ZggiqmYYhIiI7vTti8DBr6VhN+08/CHqAgAyJnWzVFdEVEF4jhARUZGfFslCUA/tHCkE8bEZRNUTZ4SIiABgQQhw/aI0LLpPkEIBZCczBBFVV5wRIiL6epAsBMVr38ZpUQ91PVwYgoiqOc4IEZFtS3kcOPujNEzSvYwjIhgA8PPErvd6FxFVEwxCRGS7/hsM3L4qDUfqRiHNFI2aSgccms47RhPZAgYhIrJNmSmyEDRZPxhppmiMjGmE1+OaWq4vIqpUDEJEZHsO/w/YMFoaDtG9ju2mNljwdEv0i6hnwcaIqLIxCBGRbfnfcODQGmn4sm4MtpvaYPoTYQxBRDaIQYiIbIMQwGdxwLm90qKBukn42RSOab1DkdAhyHK9EZHFMAgRkW2Y7iEbFj077PXuTfBCx2DL9EREFscgRETV3zR32bCLdiH+FL6Y+WQYBkUHWaYnIqoSGISIqPoSothMULuCD3EZHtgwshPC67mX/D4ishkMQkRUPZlMwAxP2aLGBZ9DDwekjeqE5nUZgoiIQYiIqqu7QlCDgpUwwQ5bxz6Cxj41LdQUEVU1DEJEVL2UcDgsqGAVAAVOzOoBpYO9RdoioqqJQYiIqg/tdSBZfi+gohB0dg4fnkpExfHp80RUPZzeJgtBe00hCCpYhXqergxBRHRPnBEiIuv3Zwawsp9sUX/dFEyOb4ZhnRtYqCkisgYMQkRk3TaNA/Z9LFsUVPAlfhj7CJrwpGgiegAGISKyXnfdKPF9Qx8sMDyD/W/Gok5NpYWaIiJrYtY5QsnJyWjXrh1q1qwJb29v9OnTBydOnJDVFBQUIDExEbVq1YKbmxv69euH3NxcWU1OTg7i4+Ph6uoKb29vjBs3DgaDQVazc+dOtGnTBkqlEo0aNUJKSkqxfhYvXoygoCA4OzsjKioK+/btM7sXIrJSd4WgkbpRWGB4BrvHxTAEEVGpmRWEdu3ahcTERPzyyy/YunUr9Ho9unfvjps3b0o1Y8eOxYYNG5Camopdu3bhwoUL6Nu3r7TeaDQiPj4eOp0Oe/bswYoVK5CSkoIpU6ZINdnZ2YiPj0dMTAyysrIwZswYDBs2DFu2bJFqvv76ayQlJWHq1Kk4cOAAWrZsibi4OFy6dKnUvRCRFTKZioWgeO3bSDNF48zsXgis5WqhxojIKomHcOnSJQFA7Nq1SwghRF5ennB0dBSpqalSzbFjxwQAkZGRIYQQYtOmTcLOzk6o1WqpZsmSJUKlUgmtViuEEGL8+PEiLCxM9ln9+/cXcXFx0jgyMlIkJiZKY6PRKPz9/UVycnKpe3mQ/Px8AUDk5+eXqp6IKpj2hhBTVbJX4wlrRf0JaZbujIiqEHO+vx/q8vn8/HwAgJeXFwAgMzMTer0esbGxUk1ISAgCAwORkZEBAMjIyEB4eDh8fHykmri4OGg0Ghw5ckSquXMbRTVF29DpdMjMzJTV2NnZITY2VqopTS9302q10Gg0shcRVRE5e4HZ/rJFzQuW4aWuzXh5PBGVWZmDkMlkwpgxY9CxY0c0b94cAKBWq+Hk5AQPDw9ZrY+PD9RqtVRzZwgqWl+07n41Go0Gt2/fxt9//w2j0VhizZ3beFAvd0tOToa7u7v0CggIKOXeIKIK9fN7wGfdZYuCCr7EqJ5t8Fr3phZqioiqgzJfNZaYmIjDhw/jp59+Ks9+LGrSpElISkqSxhqNhmGIyNKmFX84alDBl5jTNxzPRgZaoCEiqk7KFIRGjhyJtLQ07N69G/Xq/XsnV19fX+h0OuTl5clmYnJzc+Hr6yvV3H11V9GVXHfW3H11V25uLlQqFVxcXGBvbw97e/sSa+7cxoN6uZtSqYRSyatNiKqM9aNkw2+NnZCkfwW7x8XwpGgiKhdmHRoTQmDkyJFYu3Yttm/fjuDgYNn6iIgIODo6Ij09XVp24sQJ5OTkIDo6GgAQHR2NQ4cOya7u2rp1K1QqFUJDQ6WaO7dRVFO0DScnJ0RERMhqTCYT0tPTpZrS9EJEVZQQhTNBBz6XFnUseBdJ+ldwdk48QxARlR9zzsIeMWKEcHd3Fzt37hQXL16UXrdu3ZJqXn75ZREYGCi2b98ufv31VxEdHS2io6Ol9QaDQTRv3lx0795dZGVlic2bN4s6deqISZMmSTVnzpwRrq6uYty4ceLYsWNi8eLFwt7eXmzevFmqWb16tVAqlSIlJUUcPXpUvPjii8LDw0N2NdqDenkQXjVGZAG384pdGdZ6wpdi9b4/Ld0ZEVkJc76/zQpCAEp8LV++XKq5ffu2eOWVV4Snp6dwdXUV//nPf8TFixdl2zl79qzo2bOncHFxEbVr1xavvfaa0Ov1spodO3aIVq1aCScnJ9GgQQPZZxR5//33RWBgoHBychKRkZHil19+ka0vTS/3wyBEVMkuHS8WghpMWCd2n7xk6c6IyIqY8/2tEEIIS81GVXUajQbu7u7Iz8+HSqWydDtE1VuJJ0Wvwp6J3eDv4WKBhojIWpnz/c1njRGR5d3jyrAj0+NQQ8l/poio4vBfGCKyHCGA6R6yRUm6l/Gt6RHeJJGIKgWDEBFZhuYisDBEtqhdwYdI6B6Js10bW6gpIrI1DEJEVLmEAGZ4AcIkW9yw4At8nBCFbs187vFGIqLyxyBERJXHZCwMQXdpqUjFH3O6l/AGIqKKxSBERJUj/y/gnTDZosn6weg5eDJ+b1TbQk0Rka1jECKiirf/U2BjkmxRcMFKHJ/VC0oHews1RUTEIEREFalAA8yRP7h4qeFxzDE8h+zkXlAoFBZqjIioEIMQEVWMEu4N1FU7H3Ne7IeTAR4MQURUJTAIEVH5KuHeQADQqOBz/C+xC1oGFF9HRGQpZj19nojovg59UywEXRYqBBV8iT1vxjEEEVGVwxkhInp495gFalXwEfJQk3eJJqIqi0GIiB6O+hCwtJNs0Q3hjObaz/Dl8Ch0aMhL44mo6mIQIqKy+2YocPgb2aKmBSkIr++DsyM6WKgpIqLSYxAiIvNd+xN4t4Vs0QT9cHxtjMG2pC5o5O1mocaIiMzDIEREpWcyAZ91B/7aL1scXrAM1+HKewMRkdVhECKi0tk+C9g9T7Zomv55pBh74MOBbdAr3M9CjRERlR2DEBHdnxDAZz2Ac79Ii9TCE49qF2L2M5E426aeBZsjIno4DEJEdG9H1wFrnpct+szQAzMMz2Pfm93gXdPZQo0REZUPBiEiKk5/G3jbV7ZILTwRrX0f3cP8cHZQWws1RkRUvhiEiEhuWWyxk6FH6UbiJ+cu+H1aDFTOjhZqjIio/DEIEdG/SnhQauOCz7F8aEe835g3RiSi6odBiIgA9WFgaUfZold1IxEaNwSnujS0UFNERBWPQYjIlt26CswNLra4u2odtoztwnsCEVG1xyBEZKsOpgLfDpMtek33Mjo9PQo/tOYl8URkGxiEiGzN1TPAe62LLd4Svwfz24ZyFoiIbIqdpRsgosqjzfqmWAgapnsNOaMuIq5dGEMQEdkczggR2QDjyW2w/7IflHcsW2Lojb8iJmDpE2FwsOf/ExGRbWIQIqrObl4B5jWA/R2LdhvDMVQ/DgdnPA4XJ/t7vpWIyBYwCBFVR0IA0z2KLf5SxKHLaytwysOl8nsiIqqCGISIqpuFoYDmvGzRFmNbXO71Kf4vOsgyPRERVVEMQkTVxbn9wKexxRZ/HzYP3fsNh70dT4QmIrobgxCRtTu+CVg9oNjiz3wnI+apV9Czdg0LNEVEZB0YhIis1fkDwCcxJa76qV8mhoQ3quSGiIisD4MQkbXRXAQWhhRbfMwUiA+afIaFz7ZBJwdeDUZEVBpm3zxk9+7d6N27N/z9/aFQKPDdd9/J1gshMGXKFPj5+cHFxQWxsbE4deqUrObq1asYOHAgVCoVPDw8MHToUNy4cUNWc/DgQXTu3BnOzs4ICAjA3Llzi/WSmpqKkJAQODs7Izw8HJs2bTK7FyKrYTQAK54oFoLSjO0RVLAKt4ftxuL/awclQxARUamZHYRu3ryJli1bYvHixSWunzt3Lt577z0sXboUe/fuRY0aNRAXF4eCggKpZuDAgThy5Ai2bt2KtLQ07N69Gy+++KK0XqPRoHv37qhfvz4yMzMxb948TJs2DR9//LFUs2fPHgwYMABDhw7Fb7/9hj59+qBPnz44fPiwWb0QWYV9nwAzawHZu2SLmxak4HrvT5CdHI82gZ4Wao6IyIqJhwBArF27VhqbTCbh6+sr5s2bJy3Ly8sTSqVSfPXVV0IIIY4ePSoAiP3790s133//vVAoFOL8+fNCCCE+/PBD4enpKbRarVQzYcIE0bRpU2n8zDPPiPj4eFk/UVFR4qWXXip1Lw+Sn58vAIj8/PxS1ROVu70fCzFVJXtdneIv+i/cINZnnRdGo8nSHRIRVTnmfH+X6331s7OzoVarERv77yW87u7uiIqKQkZGBgAgIyMDHh4eaNu2rVQTGxsLOzs77N27V6p55JFH4OTkJNXExcXhxIkTuHbtmlRz5+cU1RR9Tml6uZtWq4VGo5G9iCzi+EaYFkcBm16XLY7VzsXXXXdj1eh49G7pDzteEk9E9FDK9WRptVoNAPDx8ZEt9/Hxkdap1Wp4e3vLm3BwgJeXl6wmODi42DaK1nl6ekKtVj/wcx7Uy92Sk5Mxffr00v2wROVNCODnd4FtUwHIj1uP0b2C70wdsealDogM9rJMf0RE1RCvGrvDpEmTkJSUJI01Gg0CAgIs2BHZBCGAja8Bv35abFUP7RycsQvCpjGdscjbzQLNERFVb+UahHx9fQEAubm58PPzk5bn5uaiVatWUs2lS5dk7zMYDLh69ar0fl9fX+Tm5spqisYPqrlz/YN6uZtSqYRSqSxxHVG5EwI48Dmw4dViq/obpuMvtxZIHRMNfz4XjIiowpTrOULBwcHw9fVFenq6tEyj0WDv3r2Ijo4GAERHRyMvLw+ZmZlSzfbt22EymRAVFSXV7N69G3q9XqrZunUrmjZtCk9PT6nmzs8pqin6nNL0QmQRN/8GprkXPhT1rhA0V/8Muruvx8LXX8LPE7syBBERVTCzZ4Ru3LiB06dPS+Ps7GxkZWXBy8sLgYGBGDNmDGbNmoXGjRsjODgYb731Fvz9/dGnTx8AQLNmzdCjRw8MHz4cS5cuhV6vx8iRI/Hss8/C398fAPDcc89h+vTpGDp0KCZMmIDDhw/j3XffxTvvvCN97ujRo9GlSxcsWLAA8fHxWL16NX799VfpEnuFQvHAXogqVV4OsCi82OIDpkb40HUE2rR/FC+3r4/xzo4WaI6IyEaZe0najh07BIBir4SEBCFE4WXrb731lvDx8RFKpVJ069ZNnDhxQraNK1euiAEDBgg3NzehUqnE4MGDxfXr12U1v//+u+jUqZNQKpWibt26Ys6cOcV6WbNmjWjSpIlwcnISYWFhYuPGjbL1penlfnj5PJWLK38IsSah2GXwYqpKPDVzufh8T7bQGYyW7pKIqNow5/tbIYQQFsxhVZpGo4G7uzvy8/OhUqks3Q5Zm2tngXdbFlusFp54VrkEL3YLQ982deHsyDtBExGVJ3O+v3nVGFF5y/8L+GYIcG6vbPEs/UDsVvXGMx2aYlNUIFyd+NePiMjS+C8xUXn561cgYzHE0XVQCKO0eKWhG1Z4jsLrPZphUjMf2PMmiEREVQaDENHDMBqAY+uBXz4E/toPAFAAyDCGIt3UGttU/0FitxB837ouHOzL9SJNIiIqBwxCRGVx9QyQ9SVwcA2Q9ycAQCscsMHUAZ8ZesChbksM79wAk8L9OANERFSFMQgRmePyCeDTx4CCfGnRFVETK42PYaUhFpfhgblPtcDTEfWgUDAAERFVdQxCRA9iMgGntwK/LAHO7JCtekM/FN8aO+GR0EB89GhDtAn0tFCTRERUFgxCRPdy6yrw+2pg38fAtWwAgBF2SDe2xn5TU3xijAegwOoX26N9g1qW7ZWIiMqEQYjobhcPApnLC8//0d0AAOSLGlhtfBRfGB/DLdd6SIgOQlaH+vBwdbJws0RE9DAYhIiAwpOff1laeOXXhQPS4mOmAHxh7I61xo7wq1MLCe0CMbA97wFERFRd8F9zsl0mE5C9E0gbW3gX6H/o4YAtxrZYZeyGvSIUMU198EFUILqGePMEaCKiaoZBiGzPravAvk+ArFXSpe9Flhh6Y5mhF4yutfF0+3pY0DGYT4AnIqrGGITINpiMwB/bgQOfAyc3A0YdAOAGXPGNoRO2m1pjt6kF2tb3wuT2gejZ3I/PACMisgEMQlS95f8F/Lay8JV/Tlp82BSETwy9sMXUDo7KGujbri7ejKqPpr41LdgsERFVNgYhqn4MOuDUD4WzP6e3AsIEANDADf8zdMQa46M4JuojMtgLs9oGoFe4L09+JiKyUfzXn6qP3KOFl70f+ga4fVVa/IupGb40dMUWUzu4utbAUxH18EFkIBrWcbNgs0REVBUwCJF10xcA3wwGTmySLb4kPPCtsRO+NsbgvF1ddAmpg7kt/BAX5stzf4iISMIgRNZHiMJ7/awfDeQekq3aZIzEamMMMkRztG/kgxEt/REX5gt3F0cLNUtERFUZgxBZj0vHgCPfAYe/Aa6cLrb6Ce1MaL1b4amIepjb0h++7s6V3yMREVkVBiGq2m7+Dfy6vDD8XD4uLb4tnLDVFIFc4YlFhn64CRekjeqEMH8Vb3pIRESlxiBEVc/ta8DRdcChbyD+3AOFMAIAtMIBu00tsNHYHjvQFu2aBqJP67rYH+LNq76IiKhM+O1BVYO+oPCS90NrIE5ugeKfGx4qAPxuaoAvjI9hi7Edmtaviz6t62JquB88a/CBp0RE9HAYhMhyhAB++wJYPwompTvstPkACsPPcVMA1ho7YZMpEso6DdGnlT82taqLAC9Xy/ZMRETVCoMQVb68HODoOhgzV8D+yikAgJ02HxeFF9Ybo7HW2Bn5qiaID/fDktZ1ed4PERFVGAYhqhy3rgKH/wdd5ko45WYBAO68m89UfQL21e6Lx8L8MC/UF83rMvwQEVHFYxCiinP1DHB8E7RHNsDx/D7YwQQnAEahwH4RgjRjexz26IqY1s3wQit/TK9dw9IdExGRjWEQovJjMgEXf4M4thG6o2lQXj0BAFD+s/qIqT6+MT6CDcYOKFDWwupX2vOwFxERWRSDED0cgxbI3g3jsY0wHNsE5e1cKFAYfgzCDvtMIdhqisAlv25oEd4CA5v5YEqdGgw/RERUJTAIkfluXwNObcWtg+vgmL0djsZbsEfhOT83hDN2mVpgh2iHG4ExiAxthMHNfBBYi1d7ERFR1cMgRA8mBHDpKAqO/4CbRzbD49J+2MOIomiTKzywzRiBvU7t4dr0UTwSGoBpTerATck/XkREVLXxm4pKVqCBOLMD+Qe/h0P2drhpc+EMoOjpXSdM9ZAu2uJcnUdRN6wDujT1xQB/FezseMiLiIisB4MQFfpn1ufG4U24fXQLvK4cgD2M8PhndYFwRIYpFEdd2kLfqDvCmrfGoAZeqOnMp7oTEZH1YhCyZbeuQn96B65mbYLLuV1Q6S/DDYDbP6vPmHzxE1rhsu8jqB3WFV3D6yOGd3YmIqJqhEHIlmhvQJz9CflHt8L4x4/wvHESjhDw+Wd1gXDEHlMYTtaMgqJJd4Q3b4X+QZ5QOtjfd7NERETWikGoOhMC4tIxXDu4Gfrjm1HrSiYcYJAOdwHAKVNd7LVvg1uBXeDXoiuim9ZDVzflvbZIRERUrTAIVTPi2llcPbwdmuM74JmbAQ/DZXjdsT7HVAd7RAtcrhMJ95AYtAkLwXN+PMmZiIhsE4OQtbtxCVcObUX+0XS4q/eglv4iagGo9c/qAuGI/aIZ/nCPhqlxd4SEtUKfQE84O/JwFxERkU0EocWLF2PevHlQq9Vo2bIl3n//fURGRlq6rTIRN6/g4qGd0BzbBpU6A/7abFnwMQg7HBINkV2zDUz1O6Fui65o28APnZ0YfIiIiO5W7YPQ119/jaSkJCxduhRRUVFYtGgR4uLicOLECXh7e1u6vfsyaW/ics5xXDm9D8azGfC6+hvq6nPgD8D/jrqjpvo47RYBfWBn+LfoipaN6qG1U7X/1RIRET00hRBCWLqJihQVFYV27drhgw8+AACYTCYEBARg1KhRmDhx4n3fq9Fo4O7ujvz8fKhUqgrpz2TQ49Jfp3HlzyO4rT4BXDkN1+tnUavgHHzE5RLf84fwx5karaEN6IQ64bEIb9IArgw+REREAMz7/q7W3546nQ6ZmZmYNGmStMzOzg6xsbHIyMgoVq/VaqHVaqWxRqOpkL7OnczCle8mwasgB77Gi/BVGOF7j9o8UQN/OQTikkcrILA96oQ+gqbBQWjoYFchvREREdmSah2E/v77bxiNRvj4+MiW+/j44Pjx48Xqk5OTMX369Arvy87BEa1u7SkcKApPaD5v749rzoEoUAVDUbsRaviFoE5QKHx866K5PUMPERFRRajWQchckyZNQlJSkjTWaDQICAgo98/xCWiCvSET4eIXglqBofAJaIiGDvxVEBERVbZq/e1bu3Zt2NvbIzc3V7Y8NzcXvr7FD0YplUoolRV/M0EHR0dEPTvpwYVERERUoar1MRcnJydEREQgPT1dWmYymZCeno7o6GgLdkZERERVQbWeEQKApKQkJCQkoG3btoiMjMSiRYtw8+ZNDB482NKtERERkYVV+yDUv39/XL58GVOmTIFarUarVq2wefPmYidQExERke2p9vcRehiVcR8hIiIiKl/mfH9X63OEiIiIiO6HQYiIiIhsFoMQERER2SwGISIiIrJZDEJERERksxiEiIiIyGYxCBEREZHNYhAiIiIim8UgRERERDar2j9i42EU3XRbo9FYuBMiIiIqraLv7dI8PINB6D6uX78OAAgICLBwJ0RERGSu69evw93d/b41fNbYfZhMJly4cAE1a9aEQqEo121rNBoEBATg3LlzfI5ZBeJ+rhzcz5WD+7nycF9Xjoraz0IIXL9+Hf7+/rCzu/9ZQJwRug87OzvUq1evQj9DpVLxL1kl4H6uHNzPlYP7ufJwX1eOitjPD5oJKsKTpYmIiMhmMQgRERGRzWIQshClUompU6dCqVRaupVqjfu5cnA/Vw7u58rDfV05qsJ+5snSREREZLM4I0REREQ2i0GIiIiIbBaDEBEREdksBiEiIiKyWQxCFWjx4sUICgqCs7MzoqKisG/fvvvWp6amIiQkBM7OzggPD8emTZsqqVPrZs5+/uSTT9C5c2d4enrC09MTsbGxD/y9UCFz/zwXWb16NRQKBfr06VOxDVYT5u7nvLw8JCYmws/PD0qlEk2aNOG/HaVg7n5etGgRmjZtChcXFwQEBGDs2LEoKCiopG6t0+7du9G7d2/4+/tDoVDgu+++e+B7du7ciTZt2kCpVKJRo0ZISUmp8D4hqEKsXr1aODk5ic8++0wcOXJEDB8+XHh4eIjc3NwS63/++Wdhb28v5s6dK44ePSomT54sHB0dxaFDhyq5c+ti7n5+7rnnxOLFi8Vvv/0mjh07Jl544QXh7u4u/vrrr0ru3LqYu5+LZGdni7p164rOnTuLJ598snKatWLm7metVivatm0revXqJX766SeRnZ0tdu7cKbKysiq5c+ti7n5etWqVUCqVYtWqVSI7O1ts2bJF+Pn5ibFjx1Zy59Zl06ZN4s033xTffvutACDWrl173/ozZ84IV1dXkZSUJI4ePSref/99YW9vLzZv3lyhfTIIVZDIyEiRmJgojY1Go/D39xfJyckl1j/zzDMiPj5etiwqKkq89NJLFdqntTN3P9/NYDCImjVrihUrVlRUi9VCWfazwWAQHTp0EMuWLRMJCQkMQqVg7n5esmSJaNCggdDpdJXVYrVg7n5OTEwUXbt2lS1LSkoSHTt2rNA+q5PSBKHx48eLsLAw2bL+/fuLuLi4CuxMCB4aqwA6nQ6ZmZmIjY2VltnZ2SE2NhYZGRklvicjI0NWDwBxcXH3rKey7ee73bp1C3q9Hl5eXhXVptUr636eMWMGvL29MXTo0Mpo0+qVZT+vX78e0dHRSExMhI+PD5o3b47Zs2fDaDRWVttWpyz7uUOHDsjMzJQOn505cwabNm1Cr169KqVnW2Gp70E+dLUC/P333zAajfDx8ZEt9/HxwfHjx0t8j1qtLrFerVZXWJ/Wriz7+W4TJkyAv79/sb989K+y7OeffvoJn376KbKysiqhw+qhLPv5zJkz2L59OwYOHIhNmzbh9OnTeOWVV6DX6zF16tTKaNvqlGU/P/fcc/j777/RqVMnCCFgMBjw8ssv44033qiMlm3Gvb4HNRoNbt++DRcXlwr5XM4Ikc2aM2cOVq9ejbVr18LZ2dnS7VQb169fx6BBg/DJJ5+gdu3alm6nWjOZTPD29sbHH3+MiIgI9O/fH2+++SaWLl1q6daqlZ07d2L27Nn48MMPceDAAXz77bfYuHEjZs6caenWqBxwRqgC1K5dG/b29sjNzZUtz83Nha+vb4nv8fX1Naueyrafi8yfPx9z5szBtm3b0KJFi4ps0+qZu5//+OMPnD17Fr1795aWmUwmAICDgwNOnDiBhg0bVmzTVqgsf579/Pzg6OgIe3t7aVmzZs2gVquh0+ng5ORUoT1bo7Ls57feeguDBg3CsGHDAADh4eG4efMmXnzxRbz55puws+OcQnm41/egSqWqsNkggDNCFcLJyQkRERFIT0+XlplMJqSnpyM6OrrE90RHR8vqAWDr1q33rKey7WcAmDt3LmbOnInNmzejbdu2ldGqVTN3P4eEhODQoUPIysqSXk888QRiYmKQlZWFgICAymzfapTlz3PHjh1x+vRpKWgCwMmTJ+Hn58cQdA9l2c+3bt0qFnaKwqfg4zrLjcW+Byv0VGwbtnr1aqFUKkVKSoo4evSoePHFF4WHh4dQq9VCCCEGDRokJk6cKNX//PPPwsHBQcyfP18cO3ZMTJ06lZfPl4K5+3nOnDnCyclJfPPNN+LixYvS6/r165b6EayCufv5brxqrHTM3c85OTmiZs2aYuTIkeLEiRMiLS1NeHt7i1mzZlnqR7AK5u7nqVOnipo1a4qvvvpKnDlzRvzwww+iYcOG4plnnrHUj2AVrl+/Ln777Tfx22+/CQBi4cKF4rfffhN//vmnEEKIiRMnikGDBkn1RZfPjxs3Thw7dkwsXryYl89bu/fff18EBgYKJycnERkZKX755RdpXZcuXURCQoKsfs2aNaJJkybCyclJhIWFiY0bN1Zyx9bJnP1cv359AaDYa+rUqZXfuJUx98/znRiESs/c/bxnzx4RFRUllEqlaNCggXj77beFwWCo5K6tjzn7Wa/Xi2nTpomGDRsKZ2dnERAQIF555RVx7dq1ym/ciuzYsaPEf2+L9m1CQoLo0qVLsfe0atVKODk5iQYNGojly5dXeJ8KITivR0RERLaJ5wgRERGRzWIQIiIiIpvFIEREREQ2i0GIiIiIbBaDEBEREdksBiEiIiKyWQxCREREZLMYhIiIiKjS7d69G71794a/vz8UCgW+++47s7chhMD8+fPRpEkTKJVK1K1bF2+//bZZ2+BDV4mIiKjS3bx5Ey1btsSQIUPQt2/fMm1j9OjR+OGHHzB//nyEh4fj6tWruHr1qlnb4J2liYiIyKIUCgXWrl2LPn36SMu0Wi3efPNNfPXVV8jLy0Pz5s3x3//+F48++igA4NixY2jRogUOHz6Mpk2blvmzeWiMiIiIqpyRI0ciIyMDq1evxsGDB/H000+jR48eOHXqFABgw4YNaNCgAdLS0hAcHIygoCAMGzbM7BkhBiEiIiKqUnJycrB8+XKkpqaic+fOaNiwIV5//XV06tQJy5cvBwCcOXMGf/75J1JTU/H5558jJSUFmZmZeOqpp8z6LJ4jRERERFXKoUOHYDQa0aRJE9lyrVaLWrVqAQBMJhO0Wi0+//xzqe7TTz9FREQETpw4UerDZQxCREREVKXcuHED9vb2yMzMhL29vWydm5sbAMDPzw8ODg6ysNSsWTMAhTNKDEJERERklVq3bg2j0YhLly6hc+fOJdZ07NgRBoMBf/zxBxo2bAgAOHnyJACgfv36pf4sXjVGREREle7GjRs4ffo0gMLgs3DhQsTExMDLywuBgYH4v//7P/z8889YsGABWrdujcuXLyM9PR0tWrRAfHw8TCYT2rVrBzc3NyxatAgmkwmJiYlQqVT44YcfSt0HgxARERFVup07dyImJqbY8oSEBKSkpECv12PWrFn4/PPPcf78edSuXRvt27fH9OnTER4eDgC4cOECRo0ahR9++AE1atRAz549sWDBAnh5eZW6DwYhIiIislm8fJ6IiIhsFoMQERER2SwGISIiIrJZDEJERERksxiEiIiIyGYxCBEREZHNYhAiIiIim8UgRERERDaLQYiIiIhsFoMQERER2SwGISIiIrJZDEJERERks/4fsxxkJpO4fHEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the cumulative sum of the data\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(data['Step'], data['DQN_CUMSUM'], label='DQN')\n",
    "plt.plot(data['Step'], data['DQN_PER_CUMSUM'], label='DQN_PER')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_398113/3448629314.py:6: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  matrix = torch.load(filepath)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd4AAAGdCAYAAAC8UhIBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAmSElEQVR4nO3df3RU9Z3/8VcSySSFzGAiScghkWit4YdRDIiRrVqM5OQgR2u+qHzpNkVPd7cNCORrt2R3BanFwZ4u0goE8dDAnpqitIsWT4UvpoewrERCkH5hu4vaUpkWE+yeMgNxmeDMfP9QRkcIzM2duXPn5vk45/PH3Nw7n3di69v3+/O592ZEIpGIAACAJTJTHQAAAEMJiRcAAAuReAEAsBCJFwAAC5F4AQCwEIkXAAALkXgBALAQiRcAAAtdYfWE4XBYJ06cUF5enjIyMqyeHgBgQiQS0enTp1VSUqLMzOTVbmfPnlV/f7/p78nOzlZOTk4CIkocyxPviRMnVFpaavW0AIAE8vl8GjNmTFK+++zZsyovL1dPT4/p7youLtaxY8dslXwtT7x5eXmSpBxJ1LsAkF4iks7q03+XJ0N/f796enrk8x2T2+0e9PcEAgGVlparv79/aCfe8+3lDJF4ASBdWbFU6Ha7TSVeu7I88QIAEJ+PPhlmrrcfEi8AwKZIvAAAWMiZiZf7eAEAsBAVLwDApkIyV7WGEhVIQpF4AQA2RasZAACYRMULALApZ1a8JF4AgE05M/HSagYAwEJUvAAAmwrJ3M5kdjUDAGCAM28notUMAICFBpV4165dq7FjxyonJ0dTp07V/v37Ex0XAGDI+ygBI35PPPGEMjIyYkZFRUX052fPnlVjY6MKCgo0YsQI1dfXq7e31/BvZTjxvvjii2pqatKyZct08OBB3XjjjaqtrdXJkycNTw4AwMCsTbySNGHCBL3//vvRsXfv3ujPFi9erO3bt2vr1q3q6OjQiRMndP/99xuew3DiXbVqlb75zW9q3rx5Gj9+vNavX68vfOEL+slPfmJ4cgAABmZ94r3iiitUXFwcHVdddZUkye/3a+PGjVq1apWmT5+uqqoqtba26o033lBnZ6ehOQwl3v7+fnV3d6umpubTL8jMVE1Njfbt23fRa4LBoAKBQMwAAMCO3nnnHZWUlOiaa67R3Llzdfz4cUlSd3e3zp07F5P/KioqVFZWNmD+G4ihxPvnP/9ZoVBIRUVFMceLiorU09Nz0Wu8Xq88Hk90lJaWGgoQADBUnd/VPNjx8a7mzxd/wWDworNNnTpVmzZt0o4dO9TS0qJjx47py1/+sk6fPq2enh5lZ2dr5MiRMddcKv8NJOm7mpubm+X3+6PD5/Mle0oAgCMkptVcWloaUwB6vd6LzlZXV6fZs2ersrJStbW1+tWvfqVTp07ppZdeSuhvZeg+3quuukpZWVkX7OLq7e1VcXHxRa9xuVxyuVyDjxAAABN8Pp/cbnf0c7w5aeTIkfrSl76kd999V3fffbf6+/t16tSpmKr3UvlvIIYq3uzsbFVVVam9vT16LBwOq729XdXV1YYmBgDg0hJT8brd7pgRb+I9c+aMfve732n06NGqqqrSsGHDYvLf0aNHdfz4ccP5z/CTq5qamtTQ0KDJkyfrlltu0erVq9XX16d58+YZ/SoAAC7B2pckPPbYY5o1a5auvvpqnThxQsuWLVNWVpbmzJkjj8ejRx55RE1NTcrPz5fb7daCBQtUXV2tW2+91dA8hhPvgw8+qA8++EBLly5VT0+PbrrpJu3YseOCDVcAAKSTP/7xj5ozZ47++7//W6NGjdJf/dVfqbOzU6NGjZIkPfPMM8rMzFR9fb2CwaBqa2u1bt06w/NkRCKRSKKDv5RAICCPx6NcSRlWTgwAMC0i6X/08X2tn103TaTzecLv/4Xc7uEmvqdPHk99UmMdDF6SAACwKV6SAAAATKLiBQDYlLWbq6xC4gUA2BSJFwAACzkz8bLGCwCAhah4AQA25cyKl8QLALApbicCAAAmUfECAGwqJHNVqz0rXhIvAMCmnLnGS6sZAAALUfECAGzKmRUviTeN9X2Q6gjicNWrqY4gDjNTHUCcvp3qAC5reEZLqkOAo7CrGQAAmETFCwCwKVrNAABYiMQLAICFnJl4WeMFAMBCVLwAAJtyZsVL4gUA2BS3EwEAAJOoeAEANvWRpCyT19sPiRcAYFPOTLy0mgEAsBAVLwDAppxZ8ZJ4AQA2xa5mAABgEhUvAMCmPpK5+pBWMwAABjgz8Rr+jfbs2aNZs2appKREGRkZevnll5MQFgAAHyVg2I/hxNvX16cbb7xRa9euTUY8AAA4muFWc11dnerq6pIRCwAAnxGSuZ3J9tzVnPQ13mAwqGAwGP0cCASSPSUAwBG4nWhQvF6vPB5PdJSWliZ7SgAAbCvpibe5uVl+vz86fD5fsqcEADiCMzdXJb3V7HK55HK5kj0NAMBxPpKUYfJ6++HJVQAAWMhwxXvmzBm9++670c/Hjh3ToUOHlJ+fr7KysoQGBwAYypxZ8RpOvAcOHNBXvvKV6OempiZJUkNDgzZt2pSwwAAAQx2JV5J05513KhKJJCMWAAAcj2c1AwBsKiRzFa897+Ml8QIAbMpsq9ghrWYAAKzhzMTL7UQAAFiIihcAYFPOrHhJvAAAmzK7Ocqem6toNQMAYCEqXgCATX0kycxzI+xZ8ZJ4AQA25czES6sZAAALUfECAGzKmRUviRcAYFPOTLy0mgEAsBAVbzo7neoA4nDVA6mOIA7/J9UBxGdXS6ojuKy+iP3fyT0843iqQ0DcQjJX8YYTFUhCkXgBADZF4gUAwEIfydyKqD0TL2u8AABYiIoXAGBTVLwAAFjoowSMwVu5cqUyMjK0aNGi6LGzZ8+qsbFRBQUFGjFihOrr69Xb22voe0m8AAB8TldXl5577jlVVlbGHF+8eLG2b9+urVu3qqOjQydOnND9999v6LtJvAAAmwrJXLU7uAdonDlzRnPnztXzzz+vK6+8Mnrc7/dr48aNWrVqlaZPn66qqiq1trbqjTfeUGdnZ9zfT+IFANhUYlrNgUAgZgSDwUvO2tjYqJkzZ6qmpibmeHd3t86dOxdzvKKiQmVlZdq3b1/cvxWJFwDgaKWlpfJ4PNHh9XoHPHfLli06ePDgRc/p6elRdna2Ro4cGXO8qKhIPT09ccfDrmYAgE19JCnDxPUfP3zD5/PJ7XZHj7pcroue7fP5tHDhQu3atUs5OTkm5r00Ei8AwKYSk3jdbndM4h1Id3e3Tp48qZtvvjl6LBQKac+ePVqzZo127typ/v5+nTp1Kqbq7e3tVXFxcdxRkXgBAJB011136fDhwzHH5s2bp4qKCn33u99VaWmphg0bpvb2dtXX10uSjh49quPHj6u6ujrueUi8AAB7ioTNParZ4LV5eXmaOHFizLHhw4eroKAgevyRRx5RU1OT8vPz5Xa7tWDBAlVXV+vWW2+Nex4SLwDAnsIy9/CpJDy46plnnlFmZqbq6+sVDAZVW1urdevWGfoOEi8AwJ5CMvcuezPXfmL37t0xn3NycrR27VqtXbt20N/J7UQAAFjIUOL1er2aMmWK8vLyVFhYqPvuu09Hjx5NVmwAgKEslIBhQ4YSb0dHhxobG9XZ2aldu3bp3LlzmjFjhvr6+pIVHwBgqAonYNiQoTXeHTt2xHzetGmTCgsL1d3drdtvvz2hgQEA4ESmNlf5/X5JUn5+/oDnBIPBmOdiBgIBM1MCAIYKG2yuSoZBb64Kh8NatGiRpk2bdsF9T5/l9XpjnpFZWlo62CkBAEOJQ1vNg068jY2NOnLkiLZs2XLJ85qbm+X3+6PD5/MNdkoAANLeoFrN8+fP16uvvqo9e/ZozJgxlzzX5XIN+EBqAAAGFJa5drFNK15DiTcSiWjBggXatm2bdu/erfLy8mTFBQAY6hy6xmso8TY2NqqtrU2vvPKK8vLyou8f9Hg8ys3NTUqAAAA4iaE13paWFvn9ft15550aPXp0dLz44ovJig8AMFQ5dHOV4VYzAACWoNUMAICFHJp4eUkCAAAWouIFANiTDd/HmwgkXgCAPdFqBgAAZlHxAgDsKSJz7WKb3ohD4gUA2BOtZgAAYBYVLwDAnhxa8ZJ4AQD25NDbiWg1AwBgISpeAIA90WoGAMBCJF7YzrhUBxCHOR+mOoLLe+7JVEcQn0CqA4jD/zue6gjgJKzxAgAAs6h4AQD2FJa5drFNK14SLwDAnmg1AwAAs6h4AQD2xK5mAAAs5NDES6sZAAALUfECAOzJoZurSLwAAHui1QwAAMyi4gUA2JNDK14SLwDAniIyt04bSVQgiUXiBQDYk0MrXtZ4AQCwEBUvAMCeuJ0IAAAL0WqWWlpaVFlZKbfbLbfbrerqar322mvJig0AAMcxVPGOGTNGK1eu1HXXXadIJKLNmzfr3nvv1VtvvaUJEyYkK0YAwFDk0IrXUOKdNWtWzOcVK1aopaVFnZ2dJF4AQGKxxhsrFApp69at6uvrU3V19YDnBYNBBYPB6OdAIDDYKQEASHuGE+/hw4dVXV2ts2fPasSIEdq2bZvGjx8/4Pler1fLly83FSQAYAhyaKvZ8H28119/vQ4dOqQ333xT3/rWt9TQ0KDf/va3A57f3Nwsv98fHT6fz1TAAIAhIqxPk+9ghlNazdnZ2friF78oSaqqqlJXV5d+9KMf6bnnnrvo+S6XSy6Xy1yUAIChx6FrvKafXBUOh2PWcAEAwMAMVbzNzc2qq6tTWVmZTp8+rba2Nu3evVs7d+5MVnwAgKHKoWu8hhLvyZMn9fWvf13vv/++PB6PKisrtXPnTt19993Jig8AMFQ5tNVsKPFu3LgxWXEAADAk8KxmAIA90WoGAMBCDk28vI8XAAALUfECAOyJzVUAAFjo/JOrzFxvQ7SaAQCwEIkXAGBP4QQMA1paWlRZWSm32y23263q6mq99tpr0Z+fPXtWjY2NKigo0IgRI1RfX6/e3l7DvxaJFwBgT2ZekDCIHdFjxozRypUr1d3drQMHDmj69Om699579R//8R+SpMWLF2v79u3aunWrOjo6dOLECd1///2Gfy3WeAEA9mTx7USzZs2K+bxixQq1tLSos7NTY8aM0caNG9XW1qbp06dLklpbWzVu3Dh1dnbq1ltvjXseKl4AgKMFAoGYEc+LfUKhkLZs2aK+vj5VV1eru7tb586dU01NTfSciooKlZWVad++fYbiIfECAOwpQWu8paWl8ng80eH1egec8vDhwxoxYoRcLpf+7u/+Ttu2bdP48ePV09Oj7OxsjRw5Mub8oqIi9fT0GPq1aDUDAOwpQa1mn88nt9sdPXypd8Rff/31OnTokPx+v37+85+roaFBHR0dJoK4EIkXAOBo53cpxyM7O1tf/OIXJUlVVVXq6urSj370Iz344IPq7+/XqVOnYqre3t5eFRcXG4qHxJvGhl9+mSL1NqU6gMvr+9+pjiA+w/9XqiMALGaDZzWHw2EFg0FVVVVp2LBham9vV319vSTp6NGjOn78uKqrqw19J4kXAGBPEZl7+lTE2OnNzc2qq6tTWVmZTp8+rba2Nu3evVs7d+6Ux+PRI488oqamJuXn58vtdmvBggWqrq42tKNZIvECACBJOnnypL7+9a/r/fffl8fjUWVlpXbu3Km7775bkvTMM88oMzNT9fX1CgaDqq2t1bp16wzPkxGJRAz+N4E5gUBAHo9HuZIyrJwYGEDf/011BPEZPiPVEQAfF5H/I8nv98e9bmrU+Tzhf0xyD7wP6vLfE5Q8P0xurINBxQsAsCeHvp2I+3gBALAQFS8AwJ5ssKs5GUi8AAB7IvECAGAh1ngBAIBZVLwAAHui1QwAgIXCMpc8aTUDAAAqXgCAPTl0cxWJFwBgTw5d46XVDACAhah4AQD25NBWs6mKd+XKlcrIyNCiRYsSFA4AAJ8IJWDY0KATb1dXl5577jlVVlYmMh4AABxtUIn3zJkzmjt3rp5//nldeeWViY4JAAAq3s9qbGzUzJkzVVNTc9lzg8GgAoFAzAAA4LLCCRg2ZHhz1ZYtW3Tw4EF1dXXFdb7X69Xy5csNBwYAGOJ4cpXk8/m0cOFCvfDCC8rJyYnrmubmZvn9/ujw+XyDChQAACcwVPF2d3fr5MmTuvnmm6PHQqGQ9uzZozVr1igYDCorKyvmGpfLJZfLlZhoAQBDR0jm7r2x6RqvocR711136fDhwzHH5s2bp4qKCn33u9+9IOkCADBoDr2P11DizcvL08SJE2OODR8+XAUFBRccBwAAF+LJVQAAe6LVfHG7d+9OQBgAAHyOQ1vNvCQBAAAL0WoGANgTrWYAACzk0MRLqxkAAAtR8QIA7CkicxukIokKJLFIvAAAewpJyjB5vQ2ReAEA9uTQxMsaLwAAFqLiBQDYk0MfoEHiBQDYE61mAABgFhUvAMCeaDUDAGAhh7aaSbxprG9GqiOIwx2pDiAOd89OdQRx6Xt0a6pDuKzhP051BID9kXgBAPYUlrmqlVYzAAAGhGWu1WzTxMuuZgAALETFCwCwJ7Obo9hcBQCAASReAAAsxBovAAAwi4oXAGBPtJoBALAQrWYAAGAWFS8AwJ7MVqw2rXhJvAAAewpJipi43qaJl1YzAAAWouIFANgTrWYAACxEqxkAAJhlKPE+8cQTysjIiBkVFRXJig0AMJSFEjBsyHCrecKECXr99dc//YIr6FYDAJKANd5PLrjiChUXFycjFgAAPhWWuTVeM9cmkeE13nfeeUclJSW65pprNHfuXB0/fjwZcQEA4EiGKt6pU6dq06ZNuv766/X+++9r+fLl+vKXv6wjR44oLy/votcEg0EFg8Ho50AgYC5iAMDQYPZZzU6oeOvq6jR79mxVVlaqtrZWv/rVr3Tq1Cm99NJLA17j9Xrl8Xiio7S01HTQAIAhwOLNVV6vV1OmTFFeXp4KCwt133336ejRozHnnD17Vo2NjSooKNCIESNUX1+v3t5eQ/OYup1o5MiR+tKXvqR33313wHOam5vl9/ujw+fzmZkSAICk6OjoUGNjozo7O7Vr1y6dO3dOM2bMUF9fX/ScxYsXa/v27dq6das6Ojp04sQJ3X///YbmMbUl+cyZM/rd736nv/7rvx7wHJfLJZfLZWYaAMBQFJKlreYdO3bEfN60aZMKCwvV3d2t22+/XX6/Xxs3blRbW5umT58uSWptbdW4cePU2dmpW2+9Na55DFW8jz32mDo6OvSHP/xBb7zxhr761a8qKytLc+bMMfI1AABcXjgBQx/vLfrs+Oy+o0vx+/2SpPz8fElSd3e3zp07p5qamug5FRUVKisr0759++L+tQwl3j/+8Y+aM2eOrr/+ej3wwAMqKChQZ2enRo0aZeRrAACwTGlpacxeI6/Xe9lrwuGwFi1apGnTpmnixImSpJ6eHmVnZ2vkyJEx5xYVFamnpyfueAy1mrds2WLkdAAABi9BrWafzye32x09HM/yZ2Njo44cOaK9e/eaCODieOwUAMCeEpR43W53TOK9nPnz5+vVV1/Vnj17NGbMmOjx4uJi9ff369SpUzFVb29vr6EHS/GSBAAAJEUiEc2fP1/btm3Tr3/9a5WXl8f8vKqqSsOGDVN7e3v02NGjR3X8+HFVV1fHPQ8VLwDAniKy9CEYjY2Namtr0yuvvKK8vLzouq3H41Fubq48Ho8eeeQRNTU1KT8/X263WwsWLFB1dXXcO5olEi8AwKbMvmDI6LUtLS2SpDvvvDPmeGtrq77xjW9Ikp555hllZmaqvr5ewWBQtbW1WrdunaF5SLwAAFuyOvFGIpcvr3NycrR27VqtXbt2cEGJNV4AACxFxQsAsKXPPANj0NfbEYkXAGBLVrearUKrGQAAC1HxAgBsiVYzAAAWotUMAABMo+IFANhSWOaqVlrNAAAYwBov7Kck1QHE4dVUBxCHvK2pjsAx+uJ7v3hKDb/8G+GApCLxAgBsyambq0i8AABbIvECAGAhp67xcjsRAAAWouIFANgSrWYAACxEqxkAAJhGxQsAsCWeXAUAgIWcusZLqxkAAAtR8QIAbMmpm6tIvAAAW6LVDAAATKPiBQDYklMrXhIvAMCWnLrGa7jV/Kc//Ulf+9rXVFBQoNzcXN1www06cOBAMmIDAAxhoQQMOzJU8f7lL3/RtGnT9JWvfEWvvfaaRo0apXfeeUdXXnllsuIDAMBRDCXep59+WqWlpWptbY0eKy8vT3hQAABEZK5dHElUIAlmqNX8y1/+UpMnT9bs2bNVWFioSZMm6fnnn09WbACAIcyprWZDiff3v/+9WlpadN1112nnzp361re+pUcffVSbN28e8JpgMKhAIBAzAAAYqgy1msPhsCZPnqynnnpKkjRp0iQdOXJE69evV0NDw0Wv8Xq9Wr58uflIAQBDilNvJzJU8Y4ePVrjx4+POTZu3DgdP358wGuam5vl9/ujw+fzDS5SAMCQEk7AsCNDFe+0adN09OjRmGNvv/22rr766gGvcblccrlcg4sOAACHMZR4Fy9erNtuu01PPfWUHnjgAe3fv18bNmzQhg0bkhUfAGCIotUsacqUKdq2bZt+9rOfaeLEiXryySe1evVqzZ07N1nxAQCGKKfuajb8yMh77rlH99xzTzJiAQDA8XhWMwDAlpz6rGYSLwDAlsIy1y4m8QIAYIBTK17DbycCAACDR8ULALAlp95OROIFANiSUxMvrWYAACxExQsAsCWnbq4i8QIAbIlWMwAAMI2KFwBgS06teEm8AABbisjcOm0kUYEkGK1mAAAsRMULALAlWs2wn45UBxCHWakOIA6nUx2AgwRSHQCchNuJAACwkFMrXtZ4AQCwEBUvAMCWnFrxkngBALbk1DVeWs0AAFiIihcAYEu0mgEAsFBY5pInrWYAAGxuz549mjVrlkpKSpSRkaGXX3455ueRSERLly7V6NGjlZubq5qaGr3zzjuG5iDxAgBsKZyAYVRfX59uvPFGrV279qI//8EPfqAf//jHWr9+vd58800NHz5ctbW1Onv2bNxz0GoGANhSKtZ46+rqVFdXd9GfRSIRrV69Wv/0T/+ke++9V5L0L//yLyoqKtLLL7+shx56KK45qHgBAIjDsWPH1NPTo5qamugxj8ejqVOnat++fXF/DxUvAMCWEnUfbyAQ+xBxl8sll8tl+Pt6enokSUVFRTHHi4qKoj+LBxUvAMCWQgkYklRaWiqPxxMdXq/X0t/j86h4AQC2lKg1Xp/PJ7fbHT0+mGpXkoqLiyVJvb29Gj16dPR4b2+vbrrppri/h4oXAOBobrc7Zgw28ZaXl6u4uFjt7e3RY4FAQG+++aaqq6vj/h4qXgCALaXiWc1nzpzRu+++G/187NgxHTp0SPn5+SorK9OiRYv0/e9/X9ddd53Ky8v1+OOPq6SkRPfdd1/ccxiqeMeOHauMjIwLRmNjo5GvAQDgss4/uWqwYzCJ98CBA5o0aZImTZokSWpqatKkSZO0dOlSSdLf//3fa8GCBfqbv/kbTZkyRWfOnNGOHTuUk5MT9xyGKt6uri6FQp923I8cOaK7775bs2fPNvI1AADY0p133qlIJDLgzzMyMvS9731P3/ve9wY9h6HEO2rUqJjPK1eu1LXXXqs77rhj0AEAAHAxvCThc/r7+/XTn/5UTU1NysjIGPC8YDCoYDAY/fz5+6kAALgY3sf7OS+//LJOnTqlb3zjG5c8z+v1xtw/VVpaOtgpAQBIe4NOvBs3blRdXZ1KSkoueV5zc7P8fn90+Hy+wU4JABhCEvUADbsZVKv5vffe0+uvv65//dd/vey5g300FwBgaKPV/Bmtra0qLCzUzJkzEx0PAACOZrjiDYfDam1tVUNDg664gudvAACSg13Nn3j99dd1/PhxPfzww8mIBwAASSTeqBkzZlzy5mIAABIhInPrtHbNVLwkAQAAC7FICwCwJVrNAABYyKmJl1YzAAAWouIFANiSUx+gQeIFANgSrWYAAGAaFS8AwJZoNQMAYCFazQAAwDQqXgCALYVlrmql1QwAgAGs8QIAYKGQzK2H2nWNl8SbxoYfS3UEcfhxqgOApfjnDVwWiRcAYEtUvAAAWMipa7zcTgQAgIWoeAEAtkSrGQAAC9FqBgAAplHxAgBsiSdXAQBgoZCkDJPX2xGtZgAALETFCwCwJaduriLxAgBsyamtZhIvAMCWnJp4WeMFAMBCVLwAAFtijRcAAAvRagYAAKYZSryhUEiPP/64ysvLlZubq2uvvVZPPvmkIpFIsuIDAAxREX3abh7MsGtmMtRqfvrpp9XS0qLNmzdrwoQJOnDggObNmyePx6NHH300WTECAIYgs61iu7aaDSXeN954Q/fee69mzpwpSRo7dqx+9rOfaf/+/UkJDgAApzHUar7tttvU3t6ut99+W5L0m9/8Rnv37lVdXd2A1wSDQQUCgZgBAMDlhBIw7MhQxbtkyRIFAgFVVFQoKytLoVBIK1as0Ny5cwe8xuv1avny5aYDBQAMLWGZ29Vs19uJDFW8L730kl544QW1tbXp4MGD2rx5s374wx9q8+bNA17T3Nwsv98fHT6fz3TQAACkK0MV73e+8x0tWbJEDz30kCTphhtu0HvvvSev16uGhoaLXuNyueRyucxHCgAYUthcJenDDz9UZmZskZyVlaVw2K4FPQAgXZF4Jc2aNUsrVqxQWVmZJkyYoLfeekurVq3Sww8/nKz4AABDlFPXeA0l3meffVaPP/64vv3tb+vkyZMqKSnR3/7t32rp0qXJig8AAEfJiFj82KlAICCPx6NcmfsvGQCA9SKS/keS3++X2+1Oyhzn88RYmXuucVjSH5TcWAeDlyQAAGzJbKvYrq1mXpIAAICFqHgBALYUkrkXHdi14iXxAgBsyamJl1YzAAAWouIFANiSUzdXkXgBALZEqxkAAJhGxQsAsKWwzFW8lj4dygAqXgCALYUTMAZj7dq1Gjt2rHJycjR16lTt37/f1O/xeSReAIAthRIwjHrxxRfV1NSkZcuW6eDBg7rxxhtVW1urkydPmv59ziPxAgDwiVWrVumb3/ym5s2bp/Hjx2v9+vX6whe+oJ/85CcJm8PyNd7z72Swa+8dADCw8//utuL9OiGZe5nO+QgDgUDMcZfLJZfLdcH5/f396u7uVnNzc/RYZmamampqtG/fPhORxLI88Z4+fVqSdNbqiQEACXP69Gl5PJ6kfHd2draKi4vV09Nj+rtGjBih0tLSmGPLli3TE088ccG5f/7znxUKhVRUVBRzvKioSP/1X/9lOpbzLE+8JSUl8vl8ysvLU0aG+RcDBgIBlZaWyufz2eq1T+mGv2Ni8HdMHP6WiZHov2MkEtHp06dVUlKSgOguLicnR8eOHVN/f7/p74pEIhfkmotVu1ayPPFmZmZqzJgxCf9et9vN/zkTgL9jYvB3TBz+lomRyL9jsirdz8rJyVFOTk7S5/msq666SllZWert7Y053tvbq+Li4oTNw+YqAAD0cYu7qqpK7e3t0WPhcFjt7e2qrq5O2Dw8QAMAgE80NTWpoaFBkydP1i233KLVq1err69P8+bNS9gcaZ94XS6Xli1blvKefbrj75gY/B0Th79lYvB3NObBBx/UBx98oKVLl6qnp0c33XSTduzYccGGKzMyIlbsCQcAAJJY4wUAwFIkXgAALETiBQDAQiReAAAslPaJN9mvb3I6r9erKVOmKC8vT4WFhbrvvvt09OjRVIeV9lauXKmMjAwtWrQo1aGknT/96U/62te+poKCAuXm5uqGG27QgQMHUh1WWgmFQnr88cdVXl6u3NxcXXvttXryyScteb4yLi+tE68Vr29yuo6ODjU2Nqqzs1O7du3SuXPnNGPGDPX19aU6tLTV1dWl5557TpWVlakOJe385S9/0bRp0zRs2DC99tpr+u1vf6t//ud/1pVXXpnq0NLK008/rZaWFq1Zs0b/+Z//qaefflo/+MEP9Oyzz6Y6NCjNbyeaOnWqpkyZojVr1kj6+AkjpaWlWrBggZYsWZLi6NLTBx98oMLCQnV0dOj2229PdThp58yZM7r55pu1bt06ff/739dNN92k1atXpzqstLFkyRL9+7//u/7t3/4t1aGktXvuuUdFRUXauHFj9Fh9fb1yc3P105/+NIWRQUrjivf865tqamqix5Lx+qahxu/3S5Ly8/NTHEl6amxs1MyZM2P+d4n4/fKXv9TkyZM1e/ZsFRYWatKkSXr++edTHVbaue2229Te3q63335bkvSb3/xGe/fuVV1dXYojg5TGT66y6vVNQ0k4HNaiRYs0bdo0TZw4MdXhpJ0tW7bo4MGD6urqSnUoaev3v/+9Wlpa1NTUpH/4h39QV1eXHn30UWVnZ6uhoSHV4aWNJUuWKBAIqKKiQllZWQqFQlqxYoXmzp2b6tCgNE68SLzGxkYdOXJEe/fuTXUoacfn82nhwoXatWuX5W9UcZJwOKzJkyfrqaeekiRNmjRJR44c0fr160m8Brz00kt64YUX1NbWpgkTJujQoUNatGiRSkpK+DvaQNomXqte3zRUzJ8/X6+++qr27NmTlNc2Ol13d7dOnjypm2++OXosFAppz549WrNmjYLBoLKyslIYYXoYPXq0xo8fH3Ns3Lhx+sUvfpGiiNLTd77zHS1ZskQPPfSQJOmGG27Qe++9J6/XS+K1gbRd47Xq9U1OF4lENH/+fG3btk2//vWvVV5enuqQ0tJdd92lw4cP69ChQ9ExefJkzZ07V4cOHSLpxmnatGkX3M729ttv6+qrr05RROnpww8/VGZm7L/es7KyFA6HUxQRPittK17Jmtc3OV1jY6Pa2tr0yiuvKC8vTz09PZI+ftF1bm5uiqNLH3l5eResiw8fPlwFBQWslxuwePFi3XbbbXrqqaf0wAMPaP/+/dqwYYM2bNiQ6tDSyqxZs7RixQqVlZVpwoQJeuutt7Rq1So9/PDDqQ4NkhRJc88++2ykrKwskp2dHbnlllsinZ2dqQ4prUi66GhtbU11aGnvjjvuiCxcuDDVYaSd7du3RyZOnBhxuVyRioqKyIYNG1IdUtoJBAKRhQsXRsrKyiI5OTmRa665JvKP//iPkWAwmOrQEIlE0vo+XgAA0k3arvECAJCOSLwAAFiIxAsAgIVIvAAAWIjECwCAhUi8AABYiMQLAICFSLwAAFiIxAsAgIVIvAAAWIjECwCAhUi8AABY6P8DgTkzysiP830AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Load the .pt file\n",
    "filepath = \"final_project/visiting_count.pt\"\n",
    "matrix = torch.load(filepath)\n",
    "\n",
    "# Plot the heatmap\n",
    "plt.imshow(matrix, cmap='hot')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.58817514, -0.5742401 , -0.56959508,  1.73201032])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "arr = np.array([1,7,9, 1000])\n",
    "norm_arr = (arr - arr.mean()) / arr.std()\n",
    "norm_arr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.0)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_arr.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.00600601, 0.00800801, 1.        ])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_arr_max_min = (arr - arr.min()) / (arr.max() - arr.min())\n",
    "norm_arr_max_min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.2535035035035035)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_arr_max_min.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method IPythonKernel._clean_thread_parent_frames of <ipykernel.ipkernel.IPythonKernel object at 0x7febd6387f10>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/ipykernel/ipkernel.py\", line 775, in _clean_thread_parent_frames\n",
      "    def _clean_thread_parent_frames(\n",
      "\n",
      "KeyboardInterrupt: \n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_regression\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import torch\n",
    "\n",
    "# Create a synthetic regression dataset\n",
    "X, y = make_regression(n_samples=1000, n_features=10, noise=0.1)\n",
    "X = torch.tensor(X, dtype=torch.float32)\n",
    "y = torch.tensor(y, dtype=torch.float32).view(-1, 1)\n",
    "\n",
    "# Create a DataLoader\n",
    "dataset = TensorDataset(X, y)\n",
    "dataloader = DataLoader(dataset, batch_size=32, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class ToyModel(nn.Module):\n",
    "    def __init__(self, input_size, output_size):\n",
    "        super(ToyModel, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_size, 50)\n",
    "        self.fc2 = nn.Linear(50, 20)\n",
    "        self.fc3 = nn.Linear(20, output_size)\n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.relu(self.fc1(x))\n",
    "        x = self.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "# Instantiate the model\n",
    "model = ToyModel(input_size=10, output_size=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100, Learning Rate: 0.1\n",
      "Epoch 2/100, Learning Rate: 0.1\n",
      "Epoch 3/100, Learning Rate: 0.1\n",
      "Epoch 4/100, Learning Rate: 0.1\n",
      "Epoch 5/100, Learning Rate: 0.1\n",
      "Epoch 6/100, Learning Rate: 0.1\n",
      "Epoch 7/100, Learning Rate: 0.1\n",
      "Epoch 8/100, Learning Rate: 0.1\n",
      "Epoch 9/100, Learning Rate: 0.1\n",
      "Epoch 10/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 11/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 12/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 13/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 14/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 15/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 16/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 17/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 18/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 19/100, Learning Rate: 0.010000000000000002\n",
      "Epoch 20/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 21/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 22/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 23/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 24/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 25/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 26/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 27/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 28/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 29/100, Learning Rate: 0.0010000000000000002\n",
      "Epoch 30/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 31/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 32/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 33/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 34/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 35/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 36/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 37/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 38/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 39/100, Learning Rate: 0.00010000000000000003\n",
      "Epoch 40/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 41/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 42/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 43/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 44/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 45/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 46/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 47/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 48/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 49/100, Learning Rate: 1.0000000000000004e-05\n",
      "Epoch 50/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 51/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 52/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 53/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 54/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 55/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 56/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 57/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 58/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 59/100, Learning Rate: 1.0000000000000004e-06\n",
      "Epoch 60/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 61/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 62/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 63/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 64/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 65/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 66/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 67/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 68/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 69/100, Learning Rate: 1.0000000000000005e-07\n",
      "Epoch 70/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 71/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 72/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 73/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 74/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 75/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 76/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 77/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 78/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 79/100, Learning Rate: 1.0000000000000005e-08\n",
      "Epoch 80/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 81/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 82/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 83/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 84/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 85/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 86/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 87/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 88/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 89/100, Learning Rate: 1.0000000000000005e-09\n",
      "Epoch 90/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 91/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 92/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 93/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 94/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 95/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 96/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 97/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 98/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 99/100, Learning Rate: 1.0000000000000006e-10\n",
      "Epoch 100/100, Learning Rate: 1.0000000000000006e-11\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABGXUlEQVR4nO3deVyVZf7/8fdZWAQFFxRccI1yTQ2UMI1KC9Ms1JnMr5Pk+Mux1DQaLVcqK9LUbHE0y7KpHM0WRy11DG2V3DDLXNpMSwNEExQVlHP//khuPYHKUeAI9+v5eJzHyH2uc87n3Dbx7ro/13XbDMMwBAAAYCF2bxcAAABQ3ghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAADAcghAAEqscePGuueee7xdhqX8/PPPstlsmjZtWpl/1vz582Wz2fTzzz97/NqPP/5YNptNH3/8canXBZQFAhBQzgp/yWzatMnbpVQoNpvN7REUFKTY2Fh98MEHF/2eCxYs0MyZM0uvyLMsW7ZMsbGxqlOnjgICAtS0aVPdeeedWrlyZZl8HgDPOL1dAICKY9euXbLbvfffTTfffLMGDhwowzC0Z88ezZ49W7169dKKFSsUFxfn8fstWLBA27Zt06hRo0q1zmnTpmn06NGKjY3V2LFjFRAQoB9++EEfffSRFi5cqO7du5fq5wHwHAEIsKhTp07J5XLJ19e3xK/x8/Mrw4ou7Morr9Tf/vY38+e+ffuqZcuWeu655y4qAJWFU6dOafLkybr55pv1v//9r8jzmZmZXqgKwJ9xCQy4TO3bt09///vfFRoaKj8/P7Vq1Uqvvvqq25j8/HxNmjRJkZGRCg4OVmBgoLp06aK1a9e6jTu7j2TmzJlq1qyZ/Pz8tH37dj366KOy2Wz64YcfdM8996h69eoKDg7WoEGDdOzYMbf3+XMPUOHlvC+++EKJiYmqXbu2AgMD1bt3bx04cMDttS6XS48++qjq1aungIAA3Xjjjdq+ffsl9RW1aNFCISEh+vHHH92O//e//1XPnj1Vr149+fn5qVmzZpo8ebIKCgrMMTfccIM++OAD7dmzx7ys1rhxY/P5vLw8JSUl6YorrpCfn5/Cw8M1ZswY5eXlnbemrKws5eTk6Lrrriv2+Tp16rj9fOLECT366KO68sor5e/vr7p166pPnz5FvpMkzZ071/y769ChgzZu3FhkzM6dO/WXv/xFNWvWlL+/v6KiorR06dIi47799lvddNNNqlKliho0aKAnnnhCLperyDibzaZHH320yPGS/r2tX79e3bt3V3BwsAICAhQbG6svvvjigq8DyhozQMBlKCMjQ9dee61sNpuGDx+u2rVra8WKFRo8eLBycnLMSzY5OTl65ZVX1L9/f9177706cuSI5s2bp7i4OG3YsEHt2rVze9/XXntNJ06c0JAhQ+Tn56eaNWuaz915551q0qSJkpOTlZaWpldeeUV16tTRlClTLljviBEjVKNGDSUlJennn3/WzJkzNXz4cC1atMgcM3bsWE2dOlW9evVSXFyctm7dqri4OJ04ceKiz1N2drZ+//13NWvWzO34/PnzVbVqVSUmJqpq1apas2aNJk2apJycHD3zzDOSpPHjxys7O1u//vqrnn32WUlS1apVJf0R1m6//XZ9/vnnGjJkiFq0aKFvvvlGzz77rL777jstWbLknDXVqVNHVapU0bJlyzRixAi3c/xnBQUFuu2225SSkqK77rpLI0eO1JEjR7R69Wpt27bN7XstWLBAR44c0T/+8Q/ZbDZNnTpVffr00U8//SQfHx9Jf4Sa6667TvXr19cjjzyiwMBAvf3224qPj9e7776r3r17S5LS09N144036tSpU+a4uXPnqkqVKp7/JZzHmjVrdOuttyoyMlJJSUmy2+167bXXdNNNN+mzzz5Tx44dS/XzAI8YAMrVa6+9ZkgyNm7ceM4xgwcPNurWrWtkZWW5Hb/rrruM4OBg49ixY4ZhGMapU6eMvLw8tzG///67ERoaavz97383j+3evduQZAQFBRmZmZlu45OSkgxJbuMNwzB69+5t1KpVy+1Yo0aNjISEhCLfpVu3bobL5TKPP/jgg4bD4TAOHz5sGIZhpKenG06n04iPj3d7v0cffdSQ5Pae5yLJGDx4sHHgwAEjMzPT2LRpk9G9e3dDkvHMM8+4jS08P2f7xz/+YQQEBBgnTpwwj/Xs2dNo1KhRkbFvvPGGYbfbjc8++8zt+Jw5cwxJxhdffHHeWidNmmRIMgIDA41bb73VePLJJ43NmzcXGffqq68akowZM2YUea7wfBb+3dWqVcs4dOiQ+fx///tfQ5KxbNky81jXrl2NNm3auH1Hl8tldOrUyYiIiDCPjRo1ypBkrF+/3jyWmZlpBAcHG5KM3bt3m8clGUlJSUXq+/M/C2vXrjUkGWvXrjU/NyIiwoiLi3P7Z+PYsWNGkyZNjJtvvrmYMweUHy6BAZcZwzD07rvvqlevXjIMQ1lZWeYjLi5O2dnZSktLkyQ5HA6zh8flcunQoUM6deqUoqKizDFn69u3r2rXrl3s5w4dOtTt5y5duujgwYPKycm5YM1DhgyRzWZze21BQYH27NkjSUpJSdGpU6d0//33u71uxIgRF3zvs82bN0+1a9dWnTp1FBUVpZSUFI0ZM0aJiYlu486eyThy5IiysrLUpUsXHTt2TDt37rzg5yxevFgtWrRQ8+bN3c7/TTfdJElFLjH+2WOPPaYFCxaoffv2WrVqlcaPH6/IyEhdc8012rFjhznu3XffVUhISLHn4ezzKUn9+vVTjRo1zJ+7dOkiSfrpp58kSYcOHdKaNWt05513mt85KytLBw8eVFxcnL7//nvt27dPkvThhx/q2muvdZuBqV27tgYMGHDBc1NSX331lb7//nv93//9nw4ePGjWk5ubq65du+rTTz8t9pIbUF64BAZcZg4cOKDDhw9r7ty5mjt3brFjzm6kff311zV9+nTt3LlTJ0+eNI83adKkyOuKO1aoYcOGbj8X/rL9/fffFRQUdN6az/daSWYQuuKKK9zG1axZ0+2X+oXccccdGj58uPLz87Vx40Y99dRTOnbsWJGVad9++60mTJigNWvWFAlw2dnZF/yc77//Xjt27DhnWCxJI3P//v3Vv39/5eTkaP369Zo/f74WLFigXr16adu2bfL399ePP/6oq666Sk7nhf9VfKFz/MMPP8gwDE2cOFETJ048Z93169fXnj17FB0dXeT5q6666oJ1lNT3338vSUpISDjnmOzsbI/+/oHSRAACLjOF/1X8t7/97Zy/PK6++mpJ0ptvvql77rlH8fHxGj16tOrUqSOHw6Hk5ORim2jP1+PhcDiKPW4YxgVrvpTXeqJBgwbq1q2bJKlHjx4KCQnR8OHDdeONN6pPnz6SpMOHDys2NlZBQUF6/PHH1axZM/n7+ystLU0PP/xwiWYdXC6X2rRpoxkzZhT7fHh4eIlrDgoK0s0336ybb75ZPj4+ev3117V+/XrFxsaW+D2kC5/jwu/1z3/+85wr4v4cQC/F2Q3lxSms55lnninSi1aosOcK8AYCEHCZqV27tqpVq6aCggLzl/25vPPOO2ratKnee+89t0smSUlJZV2mRxo1aiTpj1mKs2ehDh48aM5gXIx//OMfevbZZzVhwgT17t3b3In44MGDeu+993T99debY3fv3l3k9X++zFSoWbNm2rp1q7p27XrOMRcjKipKr7/+un777Tfzc9avX6+TJ0+ajcwXq2nTppIkHx+fC/5z06hRI3OG5my7du0qcqxGjRo6fPiw27H8/HzzO5xLYQN3UFDQBesBvIEeIOAy43A41LdvX7377rvatm1bkefPXl5eOCtw9kzL+vXrlZqaWvaFeqBr165yOp2aPXu22/EXX3zxkt7X6XTqoYce0o4dO/Tf//5XUvHnJD8/X//617+KvD4wMLDYS2J33nmn9u3bp5dffrnIc8ePH1dubu45azp27Ng5z/+KFSsknbnU1LdvX2VlZRV7HjydPatTp45uuOEGvfTSS8WGk7P/uenRo4e+/PJLbdiwwe35t956q8jrmjVrpk8//dTt2Ny5cy84AxQZGalmzZpp2rRpOnr06HnrAbyBGSDAS1599dVib4swcuRIPf3001q7dq2io6N17733qmXLljp06JDS0tL00Ucf6dChQ5Kk2267Te+995569+6tnj17avfu3ZozZ45atmxZ7C8dbwkNDdXIkSM1ffp03X777erevbu2bt2qFStWKCQk5JJmWe655x5NmjRJU6ZMUXx8vDp16qQaNWooISFBDzzwgGw2m954441iA0VkZKQWLVqkxMREdejQQVWrVlWvXr1099136+2339bQoUO1du1aXXfddSooKNDOnTv19ttva9WqVYqKiiq2nmPHjqlTp0669tpr1b17d4WHh+vw4cNasmSJPvvsM8XHx6t9+/aSpIEDB+rf//63EhMTtWHDBnXp0kW5ubn66KOPdP/99+uOO+7w6FzMmjVLnTt3Vps2bXTvvfeqadOmysjIUGpqqn799Vdt3bpVkjRmzBi98cYb6t69u0aOHGkug2/UqJG+/vprt/f8f//v/2no0KHq27evbr75Zm3dulWrVq1SSEjIeWux2+165ZVXdOutt6pVq1YaNGiQ6tevr3379mnt2rUKCgrSsmXLPPp+QKny1vIzwKoKl46f6/HLL78YhmEYGRkZxrBhw4zw8HDDx8fHCAsLM7p27WrMnTvXfC+Xy2U89dRTRqNGjQw/Pz+jffv2xvLly42EhAS35d2FS6n/vFzcMM4sgz9w4ECxdZ69JPpcy+D/vKT/z0uiDeOPJfsTJ040wsLCjCpVqhg33XSTsWPHDqNWrVrG0KFDL3jeJBnDhg0r9rnC5fSFn/fFF18Y1157rVGlShWjXr16xpgxY4xVq1YVqeno0aPG//3f/xnVq1c3JLmds/z8fGPKlClGq1atDD8/P6NGjRpGZGSk8dhjjxnZ2dnnrPPkyZPGyy+/bMTHx5t/LwEBAUb79u2NZ555psi2BceOHTPGjx9vNGnSxPx7/stf/mL8+OOPhmGc/+9OxSxR//HHH42BAwcaYWFhho+Pj1G/fn3jtttuM9555x23cV9//bURGxtr+Pv7G/Xr1zcmT55szJs3r8jfeUFBgfHwww8bISEhRkBAgBEXF2f88MMPF1wGX2jLli1Gnz59jFq1ahl+fn5Go0aNjDvvvNNISUk55zkEyoPNMEq5SxEASujw4cOqUaOGnnjiCY0fP97b5QCwEHqAAJSL48ePFzlWeCf2G264oXyLAWB59AABKBeLFi3S/Pnz1aNHD1WtWlWff/65/vOf/+iWW245532zAKCsEIAAlIurr75aTqdTU6dOVU5OjtkY/cQTT3i7NAAWRA8QAACwHHqAAACA5RCAAACA5dADVAyXy6X9+/erWrVqpboNPgAAKDuGYejIkSOqV69ekZsk/xkBqBj79+/36GaHAADg8vHLL7+oQYMG5x1DACpGtWrVJP1xAoOCgrxcDQAAKImcnByFh4ebv8fPhwBUjMLLXkFBQQQgAAAqmJK0r9AEDQAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALIcABAAALMfrAWjWrFlq3Lix/P39FR0drQ0bNpxz7Lfffqu+ffuqcePGstlsmjlz5iW/JwAAsB6vBqBFixYpMTFRSUlJSktLU9u2bRUXF6fMzMxixx87dkxNmzbV008/rbCwsFJ5TwAAYD02wzAMb314dHS0OnTooBdffFGS5HK5FB4erhEjRuiRRx4572sbN26sUaNGadSoUaX2noVycnIUHBys7OzsUr0Zas6Jk8o5frLU3q+0hAb5y8fh9clAAAAuiSe/v712N/j8/Hxt3rxZY8eONY/Z7XZ169ZNqamp5fqeeXl5ysvLM3/Oycm5qM+/kDe/3KOpK3eVyXtfima1A/W/B2PlsF/47rkAAFQGXgtAWVlZKigoUGhoqNvx0NBQ7dy5s1zfMzk5WY899thFfaYnnHab/JyXz0yLISn/lEs/HshVbv4pBfn7eLskAADKhdcC0OVk7NixSkxMNH/OyclReHh4qX/OkOubacj1zUr9fS9WgctQs3Ef/vHnAq9dCQUAoNx5LQCFhITI4XAoIyPD7XhGRsY5G5zL6j39/Pzk5+d3UZ9ZkZ19xeuUiwAEALAOr12P8fX1VWRkpFJSUsxjLpdLKSkpiomJuWzeszKz2Wxynk5BLu/1wgMAUO68egksMTFRCQkJioqKUseOHTVz5kzl5uZq0KBBkqSBAweqfv36Sk5OlvRHk/P27dvNP+/bt09fffWVqlatqiuuuKJE7wl3DrtNp1wGM0AAAEvxagDq16+fDhw4oEmTJik9PV3t2rXTypUrzSbmvXv3ym4/M0m1f/9+tW/f3vx52rRpmjZtmmJjY/Xxxx+X6D3hzmm3KU/0AAEArMWr+wBdrspqH6DL0dWPrlLOiVNa81Csmtau6u1yAAC4aJ78/r581mTDK5ynN0As4BIYAMBCCEAWZ7f90QRNDxAAwEoIQBZXuAqMGSAAgJUQgCzOQQACAFgQAcjinA4ugQEArIcAZHHMAAEArIgAZHGFPUCnXC4vVwIAQPkhAFmcw84yeACA9RCALO70NkD0AAEALIUAZHHmDBC3wgAAWAgByOLMfYC4IwoAwEIIQBbHKjAAgBURgCzuzCowAhAAwDoIQBZ3ZgaIZfAAAOsgAFmcOQNEEzQAwEIIQBZHDxAAwIoIQBbnoAcIAGBBBCCLc57eB8jFMngAgIUQgCzOQQ8QAMCCCEAW56QHCABgQQQgi6MHCABgRQQgi3M62AcIAGA9BCCLs9uYAQIAWA8ByOLoAQIAWBEByOIcp5fBE4AAAFZCALK4Mz1ABCAAgHUQgCyOVWAAACsiAFkcPUAAACsiAFncmRkglsEDAKyDAGRxDhszQAAA6yEAWZzDwb3AAADWQwCyOLMHiLvBAwAshABkcewDBACwIgKQxTlZBg8AsCACkMUVrgIroAcIAGAhBCCLYwYIAGBFBCCLs5sbIbIPEADAOghAFscMEADAighAFlfYA+RiGTwAwEIIQBbnPL0Mno0QAQBWQgCyOAc3QwUAWBAByOLoAQIAWBEByOIK7wXGDBAAwEoIQBZXeDd4ZoAAAFZCALI4J/sAAQAsiABkcTRBAwCsiABkcU56gAAAFkQAsjhH4T5ABCAAgIUQgCzOySUwAIAFEYAszsE+QAAACyIAWRxN0AAAKyIAWRwBCABgRQQgi6MHCABgRQQgizvTA8RGiAAA6yAAWZzz9DJ4ZoAAAFZCALI4VoEBAKzI6wFo1qxZaty4sfz9/RUdHa0NGzacd/zixYvVvHlz+fv7q02bNvrwww/dnj969KiGDx+uBg0aqEqVKmrZsqXmzJlTll+hQivsATIMyUUIAgBYhFcD0KJFi5SYmKikpCSlpaWpbdu2iouLU2ZmZrHj161bp/79+2vw4MHasmWL4uPjFR8fr23btpljEhMTtXLlSr355pvasWOHRo0apeHDh2vp0qXl9bUqFPvpACQxCwQAsA6vBqAZM2bo3nvv1aBBg8yZmoCAAL366qvFjn/uuefUvXt3jR49Wi1atNDkyZN1zTXX6MUXXzTHrFu3TgkJCbrhhhvUuHFjDRkyRG3btr3gzJJVOc8KQC6DAAQAsAavBaD8/Hxt3rxZ3bp1O1OM3a5u3bopNTW12Nekpqa6jZekuLg4t/GdOnXS0qVLtW/fPhmGobVr1+q7777TLbfccs5a8vLylJOT4/awCgczQAAAC/JaAMrKylJBQYFCQ0PdjoeGhio9Pb3Y16Snp19w/AsvvKCWLVuqQYMG8vX1Vffu3TVr1ixdf/3156wlOTlZwcHB5iM8PPwSvlnFcvYMUEEBAQgAYA1eb4IubS+88IK+/PJLLV26VJs3b9b06dM1bNgwffTRR+d8zdixY5WdnW0+fvnll3Ks2LvcZ4DYCwgAYA1Ob31wSEiIHA6HMjIy3I5nZGQoLCys2NeEhYWdd/zx48c1btw4vf/+++rZs6ck6eqrr9ZXX32ladOmFbl8VsjPz09+fn6X+pUqJJvNJofdpgKXwV5AAADL8NoMkK+vryIjI5WSkmIec7lcSklJUUxMTLGviYmJcRsvSatXrzbHnzx5UidPnpTd7v61HA6HXMxunBN7AQEArMZrM0DSH0vWExISFBUVpY4dO2rmzJnKzc3VoEGDJEkDBw5U/fr1lZycLEkaOXKkYmNjNX36dPXs2VMLFy7Upk2bNHfuXElSUFCQYmNjNXr0aFWpUkWNGjXSJ598on//+9+aMWOG177n5c5h435gAABr8WoA6tevnw4cOKBJkyYpPT1d7dq108qVK81G571797rN5nTq1EkLFizQhAkTNG7cOEVERGjJkiVq3bq1OWbhwoUaO3asBgwYoEOHDqlRo0Z68sknNXTo0HL/fhUFN0QFAFiNzTDY/OXPcnJyFBwcrOzsbAUFBXm7nDLX7vH/6fCxk/ooMVZX1Knq7XIAALgonvz+rnSrwOA5ZoAAAFZDAMJZTdA0igMArIEABDlP91kxAwQAsAoCEFgGDwCwHAIQzADEDBAAwCoIQCAAAQAshwAEVoEBACyHAAR6gAAAlkMAwlkzQCyDBwBYAwEIZ2aACpgBAgBYAwEI7AMEALAcAhBUeL9ZeoAAAFZBAII5A+TivrgAAIsgAIEeIACA5RCAwD5AAADLIQCBfYAAAJZDAIKcDvYBAgBYCwEIcpxugmYGCABgFQQg6PQEED1AAADLIADBnAEiAAEArIIABHMVGJfAAABWQQCCHA6WwQMArIUABGaAAACWQwCCuQ8Qy+ABAFZBAIIcNmaAAADWQgDCmR4g7gUGALAIAhDO3AuMu8EDACyCAAT2AQIAWA4BCKwCAwBYDgEIZ1aB0QMEALAIAhCYAQIAWA4BCOwDBACwHAIQzADEDBAAwCoIQDAvgblYBg8AsAgCEMxl8KdoggYAWAQBCGc2QuQSGADAIghAoAcIAGA5BCDI6WAGCABgLQQgyG7eDZ5l8AAAayAAgR4gAIDlEIBw1kaIBCAAgDUQgEAPEADAcghAOLMPEAEIAGARBCDQAwQAsBwCENgHCABgOQQg0AQNALAcAhDOmgFiHyAAgDVcUgA6ceJEadUBLzLvBk/+AQBYhMcByOVyafLkyapfv76qVq2qn376SZI0ceJEzZs3r9QLRNljBggAYDUeB6AnnnhC8+fP19SpU+Xr62seb926tV555ZVSLQ7lw3l6GTw9QAAAq/A4AP373//W3LlzNWDAADkcDvN427ZttXPnzlItDuWDVWAAAKvxOADt27dPV1xxRZHjLpdLJ0+eLJWiUL7MfYAKCEAAAGvwOAC1bNlSn332WZHj77zzjtq3b18qRaF8MQMEALAap6cvmDRpkhISErRv3z65XC6999572rVrl/79739r+fLlZVEjyhj7AAEArMbjGaA77rhDy5Yt00cffaTAwEBNmjRJO3bs0LJly3TzzTeXRY0oY+YlMIMABACwBo9ngCSpS5cuWr16dWnXAi85ewbIMAzZbDYvVwQAQNnyeAaoadOmOnjwYJHjhw8fVtOmTT0uYNasWWrcuLH8/f0VHR2tDRs2nHf84sWL1bx5c/n7+6tNmzb68MMPi4zZsWOHbr/9dgUHByswMFAdOnTQ3r17Pa7NKgqXwUtcBgMAWIPHAejnn39WQUFBkeN5eXnat2+fR++1aNEiJSYmKikpSWlpaWrbtq3i4uKUmZlZ7Ph169apf//+Gjx4sLZs2aL4+HjFx8dr27Zt5pgff/xRnTt3VvPmzfXxxx/r66+/1sSJE+Xv7+/ZF7UQh+PMjA+N0AAAK7AZRskaP5YuXSpJio+P1+uvv67g4GDzuYKCAqWkpGj16tXatWtXiT88OjpaHTp00Isvvijpj6X04eHhGjFihB555JEi4/v166fc3Fy3Zutrr71W7dq105w5cyRJd911l3x8fPTGG2+UuI4/y8nJUXBwsLKzsxUUFHTR71NRnDhZoOYTV0qSvn0sToF+F3VlFAAAr/Lk93eJf9PFx8dLkmw2mxISEtye8/HxUePGjTV9+vQSF5mfn6/Nmzdr7Nix5jG73a5u3bopNTW12NekpqYqMTHR7VhcXJyWLFki6Y8A9cEHH2jMmDGKi4vTli1b1KRJE40dO9asvzh5eXnKy8szf87JySnx96gM7DZmgAAA1lLiS2Aul0sul0sNGzZUZmam+bPL5VJeXp527dql2267rcQfnJWVpYKCAoWGhrodDw0NVXp6erGvSU9PP+/4zMxMHT16VE8//bS6d++u//3vf+rdu7f69OmjTz755Jy1JCcnKzg42HyEh4eX+HtUBoWrwCR6gAAA1uDxtY7du3eXRR2lwnX6Zp533HGHHnzwQUlSu3bttG7dOs2ZM0exsbHFvm7s2LFuM0s5OTmWCkF2u002m2QYBCAAgDVcVLNHbm6uPvnkE+3du1f5+fluzz3wwAMleo+QkBA5HA5lZGS4Hc/IyFBYWFixrwkLCzvv+JCQEDmdTrVs2dJtTIsWLfT555+fsxY/Pz/5+fmVqO7Kymm36WSBQQACAFiCxwFoy5Yt6tGjh44dO6bc3FzVrFlTWVlZCggIUJ06dUocgHx9fRUZGamUlBSzP8flciklJUXDhw8v9jUxMTFKSUnRqFGjzGOrV69WTEyM+Z4dOnQo0oj93XffqVGjRp5+VUtxnA5Ap07PogEAUJl5HIAefPBB9erVS3PmzFFwcLC+/PJL+fj46G9/+5tGjhzp0XslJiYqISFBUVFR6tixo2bOnKnc3FwNGjRIkjRw4EDVr19fycnJkqSRI0cqNjZW06dPV8+ePbVw4UJt2rRJc+fONd9z9OjR6tevn66//nrdeOONWrlypZYtW6aPP/7Y069qKX/sBeRiBggAYAkeB6CvvvpKL730kux2uxwOh/Ly8tS0aVNNnTpVCQkJ6tOnT4nfq1+/fjpw4IAmTZqk9PR0tWvXTitXrjQbnffu3Sv7WZv0derUSQsWLNCECRM0btw4RUREaMmSJWrdurU5pnfv3pozZ46Sk5P1wAMP6KqrrtK7776rzp07e/pVLYUbogIArKTE+wAVql27ttatW6eIiAhdeeWVeuGFFxQXF6edO3cqMjJSubm5ZVVrubHaPkCSdM3k1TqUm6//PXi9rgyt5u1yAADwWJnsA1Soffv22rhxoyIiIhQbG6tJkyYpKytLb7zxhttMDCoWcwaogBkgAEDl5/GtMJ566inVrVtXkvTkk0+qRo0auu+++3TgwAG99NJLpV4gykfhXkAu7ggPALAAj2eAoqKizD/XqVNHK1euLNWC4B30AAEArMTjGaBzSUtL82gnaFxeCmeAClgGDwCwAI8C0KpVq/TPf/5T48aN008//SRJ2rlzp+Lj49WhQwdzJ2ZUPPQAAQCspMSXwObNm6d7771XNWvW1O+//65XXnlFM2bM0IgRI9SvXz9t27ZNLVq0KMtaUYacp7cbYB8gAIAVlHgG6LnnntOUKVOUlZWlt99+W1lZWfrXv/6lb775RnPmzCH8VHB2eoAAABZS4gD0448/6q9//askqU+fPnI6nXrmmWfUoEGDMisO5cfsAWIVGADAAkocgI4fP66AgABJks1mk5+fn7kcHhVfYQ9QAT1AAAAL8GgZ/CuvvKKqVatKkk6dOqX58+crJCTEbUxJb4aKy4uTS2AAAAspcQBq2LChXn75ZfPnsLAwvfHGG25jbDYbAaiCMmeACEAAAAsocQD6+eefy7AMeJvTUTgDxFYGAIDKr9Q2QkTF5mAZPADAQghAkCSdngCiBwgAYAkEIEg6MwPkIgABACyAAARJrAIDAFgLAQiSJIeDVWAAAOvwaB8gScrJySn2eOHmiL6+vpdcFMofM0AAACvxOABVr15dNpvtnM83aNBA99xzj5KSkmS3M8FUUZzZB4hl8ACAys/jADR//nyNHz9e99xzjzp27ChJ2rBhg15//XVNmDBBBw4c0LRp0+Tn56dx48aVesEoG8wAAQCsxOMA9Prrr2v69Om68847zWO9evVSmzZt9NJLLyklJUUNGzbUk08+SQCqQLgXGADASjy+RrVu3Tq1b9++yPH27dsrNTVVktS5c2ft3bv30qtDuXFwN3gAgIV4HIDCw8M1b968IsfnzZun8PBwSdLBgwdVo0aNS68O5cbJTtAAAAvx+BLYtGnT9Ne//lUrVqxQhw4dJEmbNm3Szp079c4770iSNm7cqH79+pVupShTDnqAAAAW4nEAuv3227Vz50699NJL+u677yRJt956q5YsWaLGjRtLku67775SLRJlz8nd4AEAFuJxAJKkJk2a6Omnny7tWuBF5gwQTdAAAAu4qAB0+PBhbdiwQZmZmXL9ad+YgQMHlkphKF9O9gECAFiIxwFo2bJlGjBggI4ePaqgoCC3TRFtNhsBqIKy0wMEALAQj1eBPfTQQ/r73/+uo0eP6vDhw/r999/Nx6FDh8qiRpSDwhkgF8vgAQAW4HEA2rdvnx544AEFBASURT3wEsfpZfD0AAEArMDjABQXF6dNmzaVRS3wIlaBAQCsxOMeoJ49e2r06NHavn272rRpIx8fH7fnb7/99lIrDuWHfYAAAFbicQC69957JUmPP/54kedsNpsKCgouvSqUO6eDGSAAgHV4HID+vOwdlcOZGSD+fgEAlZ/HPUConBw2ZoAAANZRohmg559/XkOGDJG/v7+ef/7584594IEHSqUwlC8HTdAAAAspUQB69tlnNWDAAPn7++vZZ5895zibzUYAqqAKe4BoggYAWEGJAtDu3buL/TMqj8J9gJgBAgBYAT1AkHRmHyBmgAAAVuDxKrCCggLNnz9fKSkpxd4Mdc2aNaVWHMoPPUAAACvxOACNHDlS8+fPV8+ePdW6dWu3m6Gi4mIGCABgJR4HoIULF+rtt99Wjx49yqIeeIndnAFiHyAAQOXncQ+Qr6+vrrjiirKoBV505l5gXi4EAIBy4HEAeuihh/Tcc8/JMLhUUpk4mAECAFiIx5fAPv/8c61du1YrVqxQq1atitwM9b333iu14lB+nKeXwdMDBACwAo8DUPXq1dW7d++yqAVexCowAICVeBSATp06pRtvvFG33HKLwsLCyqomeIG5CqyAAAQAqPw86gFyOp0aOnSo8vLyyqoeeAkzQAAAK/G4Cbpjx47asmVLWdQCL3KwDxAAwEI87gG6//779dBDD+nXX39VZGSkAgMD3Z6/+uqrS604lJ/CS2AuVvcBACzA4wB01113SZLbXd9tNpsMw5DNZlNBQUHpVYdyY84AsREQAMACPA5A3A2+cnJyN3gAgIV4HIAaNWpUFnXAyxwOeoAAANbhcQAqtH37du3du1f5+flux2+//fZLLgrlz8kqMACAhXgcgH766Sf17t1b33zzjdn7I8m8Kzw9QBWT3XZmBqiwnwsAgMrK42XwI0eOVJMmTZSZmamAgAB9++23+vTTTxUVFaWPP/64DEpEeSicAZIkJoEAAJWdxwEoNTVVjz/+uEJCQmS322W329W5c2clJye7rQzzxKxZs9S4cWP5+/srOjpaGzZsOO/4xYsXq3nz5vL391ebNm304YcfnnPs0KFDZbPZNHPmzIuqzSoKe4AkLoMBACo/jwNQQUGBqlWrJkkKCQnR/v37Jf3RHL1r1y6PC1i0aJESExOVlJSktLQ0tW3bVnFxccrMzCx2/Lp169S/f38NHjxYW7ZsUXx8vOLj47Vt27YiY99//319+eWXqlevnsd1Wc3ZM0AEIABAZedxAGrdurW2bt0qSYqOjtbUqVP1xRdf6PHHH1fTpk09LmDGjBm69957NWjQILVs2VJz5sxRQECAXn311WLHP/fcc+revbtGjx6tFi1aaPLkybrmmmv04osvuo3bt2+fRowYobfeeqvIHetRlOOsAHTKxV5AAIDKzeMANGHCBLlO/4J8/PHHtXv3bnXp0kUffvihnn/+eY/eKz8/X5s3b1a3bt3OFGS3q1u3bkpNTS32NampqW7jJSkuLs5tvMvl0t13363Ro0erVatWHtVkVYX7AEnMAAEAKj+PV4HFxcWZf77iiiu0c+dOHTp0SDVq1PB45VBWVpYKCgoUGhrqdjw0NFQ7d+4s9jXp6enFjk9PTzd/njJlipxOZ4l7kvLy8txu8JqTk1PSr1BpnDUBxF5AAIBKz+MZoEI//PCDVq1apePHj6tmzZqlWdMl2bx5s5577jnNnz+/xIEsOTlZwcHB5iM8PLyMq7z82Gw27ggPALAMjwPQwYMH1bVrV1155ZXq0aOHfvvtN0nS4MGD9dBDD3n0XiEhIXI4HMrIyHA7npGRobCwsGJfExYWdt7xn332mTIzM9WwYUM5nU45nU7t2bNHDz30kBo3blzse44dO1bZ2dnm45dffvHoe1QW3BEeAGAVHgegBx98UD4+Ptq7d68CAgLM4/369dPKlSs9ei9fX19FRkYqJSXFPOZyuZSSkqKYmJhiXxMTE+M2XpJWr15tjr/77rv19ddf66uvvjIf9erV0+jRo7Vq1api39PPz09BQUFuDysy7whPAAIAVHIe9wD973//06pVq9SgQQO34xEREdqzZ4/HBSQmJiohIUFRUVHq2LGjZs6cqdzcXA0aNEiSNHDgQNWvX1/JycmS/tiIMTY2VtOnT1fPnj21cOFCbdq0SXPnzpUk1apVS7Vq1XL7DB8fH4WFhemqq67yuD4rYQYIAGAVHgeg3Nxct5mfQocOHZKfn5/HBfTr108HDhzQpEmTlJ6ernbt2mnlypVmo/PevXtlP2uFUqdOnbRgwQJNmDBB48aNU0REhJYsWaLWrVt7/Nlwd+Z+YCyDBwBUbjaj8GZeJdSjRw9FRkZq8uTJqlatmr7++ms1atRId911l1wul955552yqrXc5OTkKDg4WNnZ2Za6HBb1xEfKOpqnlaO6qHmYdb43AKBy8OT3t8czQFOnTlXXrl21adMm5efna8yYMfr222916NAhffHFFxddNLyvcAboVAGXwAAAldtF7QT93XffqXPnzrrjjjuUm5urPn36aMuWLWrWrFlZ1IhywjJ4AIBVeDwDJEnBwcEaP36827Fff/1VQ4YMMZuRUfHQBA0AsIqL3gjxzw4ePKh58+aV1tvBC8xl8J61hQEAUOGUWgBCxeegBwgAYBEEIJjoAQIAWAUBCCano7AHiH2AAACVW4mboPv06XPe5w8fPnyptcDLHKc3nGQGCABQ2ZU4AAUHB1/w+YEDB15yQfCe0xNArAIDAFR6JQ5Ar732WlnWgcuAkxkgAIBF0AMEE03QAACrIADBVNgETQACAFR2BCCY2AkaAGAVBCCYnOYlMJbBAwAqNwIQTMwAAQCsggAEE03QAACrIADBVLgRIvcCAwBUdgQgmLgbPADAKghAMNEDBACwCgIQTE56gAAAFkEAgsmcAaIHCABQyRGAYGIfIACAVRCAYLLTAwQAsAgCEEz0AAEArIIABFPhPkAEIABAZUcAgsnJJTAAgEUQgGDiVhgAAKsgAMHEDBAAwCoIQDA5HCyDBwBYAwEIJoeNGSAAgDUQgGCiBwgAYBUEIJjYBwgAYBUEIJgcDvYBAgBYAwEIJlaBAQCsggAEEz1AAACrIADBxAwQAMAqCEAwnZkBYh8gAEDlRgCCiUtgAACrIADBxDJ4AIBVEIBgctj/+MeBHiAAQGVHAIKJGSAAgFUQgGAq7AE6VUAAAgBUbgQgmJgBAgBYBQEIJru5DxDL4AEAlRsBCKbCGSAmgAAAlR0BCCYHM0AAAIsgAMHkPL0MvoAmaABAJUcAgsnBvcAAABZBAILJ6WAVGADAGghAMDEDBACwCgIQTA4bM0AAAGsgAMHE3eABAFZBAIKJHiAAgFUQgGBiHyAAgFUQgGAq3AfIZUguZoEAAJUYAQimwhkgSSowCEAAgMrrsghAs2bNUuPGjeXv76/o6Ght2LDhvOMXL16s5s2by9/fX23atNGHH35oPnfy5Ek9/PDDatOmjQIDA1WvXj0NHDhQ+/fvL+uvUeE5zw5AzAABACoxrwegRYsWKTExUUlJSUpLS1Pbtm0VFxenzMzMYsevW7dO/fv31+DBg7VlyxbFx8crPj5e27ZtkyQdO3ZMaWlpmjhxotLS0vTee+9p165duv3228vza1VIZ88AsRcQAKAysxmGd691REdHq0OHDnrxxRclSS6XS+Hh4RoxYoQeeeSRIuP79eun3NxcLV++3Dx27bXXql27dpozZ06xn7Fx40Z17NhRe/bsUcOGDS9YU05OjoKDg5Wdna2goKCL/GYVz8kClyLGr5AkbU26RcFVfLxcEQAAJefJ72+vzgDl5+dr8+bN6tatm3nMbrerW7duSk1NLfY1qampbuMlKS4u7pzjJSk7O1s2m03Vq1cvlborq8KNECUugQEAKjenNz88KytLBQUFCg0NdTseGhqqnTt3Fvua9PT0Ysenp6cXO/7EiRN6+OGH1b9//3Omwby8POXl5Zk/5+TkePI1Kg273Sa77Y9VYCyFBwBUZl7vASpLJ0+e1J133inDMDR79uxzjktOTlZwcLD5CA8PL8cqLy+FS+GZAQIAVGZeDUAhISFyOBzKyMhwO56RkaGwsLBiXxMWFlai8YXhZ8+ePVq9evV5rwWOHTtW2dnZ5uOXX365yG9U8ZmbIRYQgAAAlZdXA5Cvr68iIyOVkpJiHnO5XEpJSVFMTEyxr4mJiXEbL0mrV692G18Yfr7//nt99NFHqlWr1nnr8PPzU1BQkNvDqpzcDwwAYAFe7QGSpMTERCUkJCgqKkodO3bUzJkzlZubq0GDBkmSBg4cqPr16ys5OVmSNHLkSMXGxmr69Onq2bOnFi5cqE2bNmnu3LmS/gg/f/nLX5SWlqbly5eroKDA7A+qWbOmfH19vfNFKwi7eTsMAhAAoPLyegDq16+fDhw4oEmTJik9PV3t2rXTypUrzUbnvXv3ym4/M1HVqVMnLViwQBMmTNC4ceMUERGhJUuWqHXr1pKkffv2aenSpZKkdu3auX3W2rVrdcMNN5TL96qoCmeAXOwEDQCoxLy+D9DlyKr7AElSxyc/UuaRPH34QBe1rGet7w4AqNgqzD5AuPzQAwQAsAICENw4HIU9QOwDBACovAhAcMM+QAAAKyAAwY2DVWAAAAsgAMFN4f3AmAECAFRmBCC4cdAEDQCwAAIQ3DgdBCAAQOVHAIIbeoAAAFZAAIKbM/sAsQweAFB5EYDghhkgAIAVEIDghiZoAIAVEIDgxnF6I8RTBQQgAEDlRQCCG7MHiHvkAgAqMQIQ3HAJDABgBQQguHHSBA0AsAACENyYM0AFLIMHAFReBCC4YQYIAGAFBCC4sdMDBACwAAIQ3DADBACwAgIQ3BTuA+QiAAEAKjECENwwAwQAsAICENywDxAAwAoIQHDDDBAAwAoIQHDjcBTOALEPEACg8iIAwY3DxgwQAKDyIwDBjZMeIACABTi9XQAuL4XL4Fdvz9DPB495uZpz83faNarblWpZL8jbpQAAKiACENyEBftJkn7LPqHfsk94uZrzq+rn1Ix+7bxdBgCgAiIAwU2faxqoVqCfck6c9HYp57R9f45e+Xy3fszK9XYpAIAKigAENz4Ou7q1DPV2GefVqt4RvfL5bv104KgMw5DtdOM2AAAlRRM0KpxGtQJks0lHTpzSwdx8b5cDAKiACECocPx9HKpfvYok6acDXAYDAHiOAIQKqWntqpKk3VlHvVwJAKAiIgChQmoaEiiJGSAAwMUhAKFCalr7jwD0IwEIAHARCECokJqcngHiEhgA4GIQgFAhFfYA7T10TKcKuHErAMAzBCBUSHWD/OXvY9fJAkO//n7c2+UAACoYAhAqJLvdpsa1TjdCcxkMAOAhAhAqrGanL4OxEgwA4CkCECqswkbon7gnGADAQwQgVFiFS+F/OsAlMACAZwhAqLDOLIVnBggA4BkCECqspiF/9ABl5OTpaN4pL1cDAKhICECosIIDfFQr0FeS9DOzQAAADxCAUKGduSUGfUAAgJIjAKFCK7wMRh8QAMATBCBUaE1qc1d4AIDnCECo0JqyEgwAcBEIQKjQzt4LyDAML1cDAKgoCECo0BrWDJTdJuXmFyjzSJ63ywEAVBAEIFRovk67wmsGSKIPCABQcgQgVHhNQ7grPADAMwQgVHhNT98VfjczQACAEiIAocLjrvAAAE8RgFDhFa4EYyk8AKCknN4uALhUhbtB7z10TC+kfC+bzcsFnUejWoHq2aau7PbLuEgAsIDLIgDNmjVLzzzzjNLT09W2bVu98MIL6tix4znHL168WBMnTtTPP/+siIgITZkyRT169DCfNwxDSUlJevnll3X48GFdd911mj17tiIiIsrj66CchQb5KcjfqZwTpzR99XfeLueC3vhyj6b2vVqNT1+6AwCUP5vh5d3jFi1apIEDB2rOnDmKjo7WzJkztXjxYu3atUt16tQpMn7dunW6/vrrlZycrNtuu00LFizQlClTlJaWptatW0uSpkyZouTkZL3++utq0qSJJk6cqG+++Ubbt2+Xv7//BWvKyclRcHCwsrOzFRQUVOrfGaUvZUeGVm/P8HYZ53XKZejDb37TsfwC+fvYNSauue7p1JjZIAAoJZ78/vZ6AIqOjlaHDh304osvSpJcLpfCw8M1YsQIPfLII0XG9+vXT7m5uVq+fLl57Nprr1W7du00Z84cGYahevXq6aGHHtI///lPSVJ2drZCQ0M1f/583XXXXResiQCEsvLLoWN6+N2vte7Hg5KkqEY1dHPLUC9XdWGBfk4FV/ExH4F+zsv6UmMhu80mW+H/2lQhagasopqfj4IDfEr1PT35/e3VS2D5+fnavHmzxo4dax6z2+3q1q2bUlNTi31NamqqEhMT3Y7FxcVpyZIlkqTdu3crPT1d3bp1M58PDg5WdHS0UlNTiw1AeXl5yss7s4twTk7OpXwt4JzCawborf8XrQUb9uqpD3Zo057ftWnP794uCwDK3f03NNOY7s299vleDUBZWVkqKChQaKj7fwGHhoZq586dxb4mPT292PHp6enm84XHzjXmz5KTk/XYY49d1HcAPGWz2TQgupFir6yteZ/vVvbxk94u6bwMQzqad0rZx08q5/hJZR8/qWP5Bd4u64IMw5ChP+o3DEMFFfRecRW0bOCCnF6+/H9ZNEF729ixY91mlXJychQeHu7FimAFDWoEKKlXK2+XAQCW5NV9gEJCQuRwOJSR4d68mpGRobCwsGJfExYWdt7xhf/ryXv6+fkpKCjI7QEAACovrwYgX19fRUZGKiUlxTzmcrmUkpKimJiYYl8TExPjNl6SVq9ebY5v0qSJwsLC3Mbk5ORo/fr153xPAABgLV6/BJaYmKiEhARFRUWpY8eOmjlzpnJzczVo0CBJ0sCBA1W/fn0lJydLkkaOHKnY2FhNnz5dPXv21MKFC7Vp0ybNnTtX0h/9FaNGjdITTzyhiIgIcxl8vXr1FB8f762vCQAALiNeD0D9+vXTgQMHNGnSJKWnp6tdu3ZauXKl2cS8d+9e2e1nJqo6deqkBQsWaMKECRo3bpwiIiK0ZMkScw8gSRozZoxyc3M1ZMgQHT58WJ07d9bKlStLtAcQAACo/Ly+D9DliH2AAACoeDz5/c3NUAEAgOUQgAAAgOUQgAAAgOUQgAAAgOUQgAAAgOUQgAAAgOUQgAAAgOUQgAAAgOUQgAAAgOV4/VYYl6PCzbFzcnK8XAkAACipwt/bJbnJBQGoGEeOHJEkhYeHe7kSAADgqSNHjig4OPi8Y7gXWDFcLpf279+vatWqyWazlep75+TkKDw8XL/88gv3GStjnOvyw7kuP5zr8sO5Lj+lda4Nw9CRI0dUr149txupF4cZoGLY7XY1aNCgTD8jKCiI/0OVE851+eFclx/OdfnhXJef0jjXF5r5KUQTNAAAsBwCEAAAsBwCUDnz8/NTUlKS/Pz8vF1Kpce5Lj+c6/LDuS4/nOvy441zTRM0AACwHGaAAACA5RCAAACA5RCAAACA5RCAAACA5RCAytGsWbPUuHFj+fv7Kzo6Whs2bPB2SRVecnKyOnTooGrVqqlOnTqKj4/Xrl273MacOHFCw4YNU61atVS1alX17dtXGRkZXqq48nj66adls9k0atQo8xjnuvTs27dPf/vb31SrVi1VqVJFbdq00aZNm8znDcPQpEmTVLduXVWpUkXdunXT999/78WKK6aCggJNnDhRTZo0UZUqVdSsWTNNnjzZ7V5SnOuL8+mnn6pXr16qV6+ebDablixZ4vZ8Sc7roUOHNGDAAAUFBal69eoaPHiwjh49Wir1EYDKyaJFi5SYmKikpCSlpaWpbdu2iouLU2ZmprdLq9A++eQTDRs2TF9++aVWr16tkydP6pZbblFubq455sEHH9SyZcu0ePFiffLJJ9q/f7/69Onjxaorvo0bN+qll17S1Vdf7Xacc106fv/9d1133XXy8fHRihUrtH37dk2fPl01atQwx0ydOlXPP/+85syZo/Xr1yswMFBxcXE6ceKEFyuveKZMmaLZs2frxRdf1I4dOzRlyhRNnTpVL7zwgjmGc31xcnNz1bZtW82aNavY50tyXgcMGKBvv/1Wq1ev1vLly/Xpp59qyJAhpVOggXLRsWNHY9iwYebPBQUFRr169Yzk5GQvVlX5ZGZmGpKMTz75xDAMwzh8+LDh4+NjLF682ByzY8cOQ5KRmprqrTIrtCNHjhgRERHG6tWrjdjYWGPkyJGGYXCuS9PDDz9sdO7c+ZzPu1wuIywszHjmmWfMY4cPHzb8/PyM//znP+VRYqXRs2dP4+9//7vbsT59+hgDBgwwDINzXVokGe+//775c0nO6/bt2w1JxsaNG80xK1asMGw2m7Fv375LrokZoHKQn5+vzZs3q1u3buYxu92ubt26KTU11YuVVT7Z2dmSpJo1a0qSNm/erJMnT7qd++bNm6thw4ac+4s0bNgw9ezZ0+2cSpzr0rR06VJFRUXpr3/9q+rUqaP27dvr5ZdfNp/fvXu30tPT3c51cHCwoqOjOdce6tSpk1JSUvTdd99JkrZu3arPP/9ct956qyTOdVkpyXlNTU1V9erVFRUVZY7p1q2b7Ha71q9ff8k1cDPUcpCVlaWCggKFhoa6HQ8NDdXOnTu9VFXl43K5NGrUKF133XVq3bq1JCk9PV2+vr6qXr2629jQ0FClp6d7ocqKbeHChUpLS9PGjRuLPMe5Lj0//fSTZs+ercTERI0bN04bN27UAw88IF9fXyUkJJjns7h/p3CuPfPII48oJydHzZs3l8PhUEFBgZ588kkNGDBAkjjXZaQk5zU9PV116tRxe97pdKpmzZqlcu4JQKg0hg0bpm3btunzzz/3dimV0i+//KKRI0dq9erV8vf393Y5lZrL5VJUVJSeeuopSVL79u21bds2zZkzRwkJCV6urnJ5++239dZbb2nBggVq1aqVvvrqK40aNUr16tXjXFdyXAIrByEhIXI4HEVWw2RkZCgsLMxLVVUuw4cP1/Lly7V27Vo1aNDAPB4WFqb8/HwdPnzYbTzn3nObN29WZmamrrnmGjmdTjmdTn3yySd6/vnn5XQ6FRoayrkuJXXr1lXLli3djrVo0UJ79+6VJPN88u+USzd69Gg98sgjuuuuu9SmTRvdfffdevDBB5WcnCyJc11WSnJew8LCiiwUOnXqlA4dOlQq554AVA58fX0VGRmplJQU85jL5VJKSopiYmK8WFnFZxiGhg8frvfff19r1qxRkyZN3J6PjIyUj4+P27nftWuX9u7dy7n3UNeuXfXNN9/oq6++Mh9RUVEaMGCA+WfOdem47rrrimzn8N1336lRo0aSpCZNmigsLMztXOfk5Gj9+vWcaw8dO3ZMdrv7r0KHwyGXyyWJc11WSnJeY2JidPjwYW3evNkcs2bNGrlcLkVHR196EZfcRo0SWbhwoeHn52fMnz/f2L59uzFkyBCjevXqRnp6urdLq9Duu+8+Izg42Pj444+N3377zXwcO3bMHDN06FCjYcOGxpo1a4xNmzYZMTExRkxMjBerrjzOXgVmGJzr0rJhwwbD6XQaTz75pPH9998bb731lhEQEGC8+eab5pinn37aqF69uvHf//7X+Prrr4077rjDaNKkiXH8+HEvVl7xJCQkGPXr1zeWL19u7N6923jvvfeMkJAQY8yYMeYYzvXFOXLkiLFlyxZjy5YthiRjxowZxpYtW4w9e/YYhlGy89q9e3ejffv2xvr1643PP//ciIiIMPr3718q9RGAytELL7xgNGzY0PD19TU6duxofPnll94uqcKTVOzjtddeM8ccP37cuP/++40aNWoYAQEBRu/evY3ffvvNe0VXIn8OQJzr0rNs2TKjdevWhp+fn9G8eXNj7ty5bs+7XC5j4sSJRmhoqOHn52d07drV2LVrl5eqrbhycnKMkSNHGg0bNjT8/f2Npk2bGuPHjzfy8vLMMZzri7N27dpi//2ckJBgGEbJzuvBgweN/v37G1WrVjWCgoKMQYMGGUeOHCmV+myGcdZ2lwAAABZADxAAALAcAhAAALAcAhAAALAcAhAAALAcAhAAALAcAhAAALAcAhAAALAcAhAAnIPNZtOSJUu8XQaAMkAAAnBZuueee2Sz2Yo8unfv7u3SAFQCTm8XAADn0r17d7322mtux/z8/LxUDYDKhBkgAJctPz8/hYWFuT1q1Kgh6Y/LU7Nnz9att96qKlWqqGnTpnrnnXfcXv/NN9/opptuUpUqVVSrVi0NGTJER48edRvz6quvqlWrVvLz81PdunU1fPhwt+ezsrLUu3dvBQQEKCIiQkuXLjWf+/333zVgwADVrl1bVapUUURERJHABuDyRAACUGFNnDhRffv21datWzVgwADddddd2rFjhyQpNzdXcXFxqlGjhjZu3KjFixfro48+cgs4s2fP1rBhwzRkyBB98803Wrp0qa644gq3z3jsscd055136uuvv1aPHj00YMAAHTp0yPz87du3a8WKFdqxY4dmz56tkJCQ8jsBAC5eqdxSFQBKWUJCguFwOIzAwEC3x5NPPmkYhmFIMoYOHer2mujoaOO+++4zDMMw5s6da9SoUcM4evSo+fwHH3xg2O12Iz093TAMw6hXr54xfvz4c9YgyZgwYYL589GjRw1JxooVKwzDMIxevXoZgwYNKp0vDKBc0QME4LJ14403avbs2W7Hatasaf45JibG7bmYmBh99dVXkqQdO3aobdu2CgwMNJ+/7rrr5HK5tGvXLtlsNu3fv19du3Y9bw1XX321+efAwEAFBQUpMzNTknTfffepb9++SktL0y233KL4+Hh16tTpor4rgPJFAAJw2QoMDCxySaq0VKlSpUTjfHx83H622WxyuVySpFtvvVV79uzRhx9+qNWrV6tr164aNmyYpk2bVur1Aihd9AABqLC+/PLLIj+3aNFCktSiRQtt3bpVubm55vNffPGF7Ha7rrrqKlWrVk2NGzdWSkrKJdVQu3ZtJSQk6M0339TMmTM1d+7cS3o/AOWDGSAAl628vDylp6e7HXM6nWaj8eLFixUVFaXOnTvrrbfe0oYNGzRv3jxJ0oABA5SUlKSEhAQ9+uijOnDggEaMGKG7775boaGhkqRHH31UQ4cOVZ06dXTrrbfqyJEj+uKLLzRixIgS1Tdp0iRFRkaqVatWysvL0/Lly80ABuDyRgACcNlauXKl6tat63bsqquu0s6dOyX9sUJr4cKFuv/++1W3bl395z//UcuWLSVJAQEBWrVqlUaOHKkOHTooICBAffv21YwZM8z3SkhI0IkTJ/Tss8/qn//8p0JCQvSXv/ylxPX5+vpq7Nix+vnnn1WlShV16dJFCxcuLIVvDqCs2QzDMLxdBAB4ymaz6f3331d8fLy3SwFQAdEDBAAALIcABAAALIceIAAVElfvAVwKZoAAAIDlEIAAAIDlEIAAAIDlEIAAAIDlEIAAAIDlEIAAAIDlEIAAAIDlEIAAAIDlEIAAAIDl/H99SclNEOMNfgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.optim as optim\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Assuming you have already defined your model, loss function, and optimizer\n",
    "# model = ...\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.1)\n",
    "\n",
    "\n",
    "# Define a learning rate scheduler\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n",
    "\n",
    "# Variables to store the learning rates over time\n",
    "lrs = []\n",
    "epochs = 100\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    # Training loop\n",
    "    for inputs, targets in dataloader:\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "    # Step the scheduler\n",
    "    scheduler.step()\n",
    "\n",
    "    # Store the learning rate\n",
    "    lrs.append(optimizer.param_groups[0]['lr'])\n",
    "\n",
    "    # Optionally, print the learning rate\n",
    "    print(f\"Epoch {epoch+1}/{epochs}, Learning Rate: {optimizer.param_groups[0]['lr']}\")\n",
    "\n",
    "# Plotting the learning rate over epochs\n",
    "plt.plot(range(epochs), lrs)\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Learning Rate')\n",
    "plt.title('Learning Rate Schedule')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://wandb.ai/safijari/dqn-tutorial/reports/Deep-Q-Networks-DQN-With-the-Cartpole-Environment--Vmlldzo4MDc2MQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data[\"action_value\"] # shape (batch_size, num_actions)\n",
    "# Get the mean q_value from the batch data[\"action_value\"]\n",
    "# data[\"action_value\"].mean()\n",
    "\n",
    "import torch\n",
    "# Create a toy tensor with shape (10, 2)\n",
    "tensor = torch.rand(10, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4304, 0.3132],\n",
       "        [0.7245, 0.4431],\n",
       "        [0.9938, 0.1911],\n",
       "        [0.6044, 0.1895],\n",
       "        [0.4958, 0.2732],\n",
       "        [0.5166, 0.9291],\n",
       "        [0.2419, 0.7202],\n",
       "        [0.1368, 0.6574],\n",
       "        [0.7252, 0.6230],\n",
       "        [0.1141, 0.6644]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.4994)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor.mean(-1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.4994)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7efc1421c130>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import time\n",
    "\n",
    "from torchrl.envs import GymEnv, StepCounter, TransformedEnv\n",
    "from tensordict.nn import TensorDictModule as Mod, TensorDictSequential as Seq\n",
    "from torchrl.modules import EGreedyModule, MLP, QValueModule\n",
    "from torchrl.collectors import SyncDataCollector\n",
    "from torchrl.data import LazyTensorStorage, ReplayBuffer\n",
    "from torch.optim import Adam\n",
    "from torchrl.objectives import DQNLoss, SoftUpdate\n",
    "from torchrl._utils import logger as torchrl_logger\n",
    "from torchrl.record import CSVLogger, VideoRecorder\n",
    "from torchrl.modules import QValueActor\n",
    "from torchrl.data import CompositeSpec\n",
    "\n",
    "torch.manual_seed(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1: Rewards = [0.0, 1.0], Average reward over last 100 episodes = 0.5\n",
      "Episode 2: Rewards = [1.0, 2.0], Average reward over last 100 episodes = 1.0\n",
      "Episode 3: Rewards = [2.0, 3.0], Average reward over last 100 episodes = 1.5\n",
      "Episode 4: Rewards = [3.0, 4.0], Average reward over last 100 episodes = 2.0\n",
      "Episode 5: Rewards = [4.0, 5.0], Average reward over last 100 episodes = 2.5\n",
      "Episode 6: Rewards = [5.0, 6.0], Average reward over last 100 episodes = 3.0\n",
      "Episode 7: Rewards = [6.0, 7.0], Average reward over last 100 episodes = 3.5\n",
      "Episode 8: Rewards = [7.0, 8.0], Average reward over last 100 episodes = 4.0\n",
      "Episode 9: Rewards = [8.0, 9.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 10: Rewards = [9.0, 0.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 11: Rewards = [0.0, 1.0], Average reward over last 100 episodes = 4.136363506317139\n",
      "Episode 12: Rewards = [1.0, 2.0], Average reward over last 100 episodes = 3.9166667461395264\n",
      "Episode 13: Rewards = [2.0, 3.0], Average reward over last 100 episodes = 3.807692289352417\n",
      "Episode 14: Rewards = [3.0, 4.0], Average reward over last 100 episodes = 3.7857143878936768\n",
      "Episode 15: Rewards = [4.0, 5.0], Average reward over last 100 episodes = 3.8333332538604736\n",
      "Episode 16: Rewards = [5.0, 6.0], Average reward over last 100 episodes = 3.9375\n",
      "Episode 17: Rewards = [6.0, 7.0], Average reward over last 100 episodes = 4.088235378265381\n",
      "Episode 18: Rewards = [7.0, 8.0], Average reward over last 100 episodes = 4.277777671813965\n",
      "Episode 19: Rewards = [8.0, 9.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 20: Rewards = [9.0, 0.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 21: Rewards = [0.0, 1.0], Average reward over last 100 episodes = 4.309523582458496\n",
      "Episode 22: Rewards = [1.0, 2.0], Average reward over last 100 episodes = 4.181818008422852\n",
      "Episode 23: Rewards = [2.0, 3.0], Average reward over last 100 episodes = 4.1086955070495605\n",
      "Episode 24: Rewards = [3.0, 4.0], Average reward over last 100 episodes = 4.083333492279053\n",
      "Episode 25: Rewards = [4.0, 5.0], Average reward over last 100 episodes = 4.099999904632568\n",
      "Episode 26: Rewards = [5.0, 6.0], Average reward over last 100 episodes = 4.153846263885498\n",
      "Episode 27: Rewards = [6.0, 7.0], Average reward over last 100 episodes = 4.240740776062012\n",
      "Episode 28: Rewards = [7.0, 8.0], Average reward over last 100 episodes = 4.357142925262451\n",
      "Episode 29: Rewards = [8.0, 9.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 30: Rewards = [9.0, 0.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 31: Rewards = [0.0, 1.0], Average reward over last 100 episodes = 4.370967864990234\n",
      "Episode 32: Rewards = [1.0, 2.0], Average reward over last 100 episodes = 4.28125\n",
      "Episode 33: Rewards = [2.0, 3.0], Average reward over last 100 episodes = 4.2272725105285645\n",
      "Episode 34: Rewards = [3.0, 4.0], Average reward over last 100 episodes = 4.205882549285889\n",
      "Episode 35: Rewards = [4.0, 5.0], Average reward over last 100 episodes = 4.214285850524902\n",
      "Episode 36: Rewards = [5.0, 6.0], Average reward over last 100 episodes = 4.25\n",
      "Episode 37: Rewards = [6.0, 7.0], Average reward over last 100 episodes = 4.3108110427856445\n",
      "Episode 38: Rewards = [7.0, 8.0], Average reward over last 100 episodes = 4.3947367668151855\n",
      "Episode 39: Rewards = [8.0, 9.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 40: Rewards = [9.0, 0.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 41: Rewards = [0.0, 1.0], Average reward over last 100 episodes = 4.402439117431641\n",
      "Episode 42: Rewards = [1.0, 2.0], Average reward over last 100 episodes = 4.333333492279053\n",
      "Episode 43: Rewards = [2.0, 3.0], Average reward over last 100 episodes = 4.2906975746154785\n",
      "Episode 44: Rewards = [3.0, 4.0], Average reward over last 100 episodes = 4.2727274894714355\n",
      "Episode 45: Rewards = [4.0, 5.0], Average reward over last 100 episodes = 4.277777671813965\n",
      "Episode 46: Rewards = [5.0, 6.0], Average reward over last 100 episodes = 4.304347991943359\n",
      "Episode 47: Rewards = [6.0, 7.0], Average reward over last 100 episodes = 4.3510637283325195\n",
      "Episode 48: Rewards = [7.0, 8.0], Average reward over last 100 episodes = 4.416666507720947\n",
      "Episode 49: Rewards = [8.0, 9.0], Average reward over last 100 episodes = 4.5\n",
      "Episode 50: Rewards = [9.0, 0.0], Average reward over last 100 episodes = 4.5\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "window_size = 100\n",
    "reward_queue = deque(maxlen=window_size)\n",
    "\n",
    "for episode in range(50):\n",
    "    # Example tensor containing multiple rewards for an episode\n",
    "    episode_rewards = torch.tensor([episode % 10, (episode + 1) % 10], dtype=torch.float32)\n",
    "    \n",
    "    # Extend the deque with all rewards in the tensor\n",
    "    reward_queue.extend(episode_rewards.detach().cpu().numpy())\n",
    "    \n",
    "    # Calculate the average reward over the current window\n",
    "    average_reward = np.mean(reward_queue)\n",
    "    print(f\"Episode {episode + 1}: Rewards = {episode_rewards.tolist()}, Average reward over last {window_size} episodes = {average_reward}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchrl.envs import (\n",
    "    CatFrames,\n",
    "    DoubleToFloat,\n",
    "    EndOfLifeTransform,\n",
    "    GrayScale,\n",
    "    GymEnv,\n",
    "    NoopResetEnv,\n",
    "    Resize,\n",
    "    RewardSum,\n",
    "    SignTransform,\n",
    "    StepCounter,\n",
    "    ToTensorImage,\n",
    "    TransformedEnv,\n",
    "    VecNorm,\n",
    "    ObservationNorm,\n",
    "    CenterCrop,\n",
    "    TimeMaxPool,\n",
    "    RewardClipping\n",
    ")\n",
    "\n",
    "import gymnasium as gym\n",
    "\n",
    "from torchrl.envs import GymWrapper\n",
    "import numpy as np\n",
    "np.float_ = np.float64\n",
    "\n",
    "import ale_py\n",
    "\n",
    "gym.register_envs(ale_py)\n",
    "\n",
    "def make_env(env_name=\"Asteroids\", frame_skip = 4, \n",
    "             device=\"cpu\", seed = 0, cropping = False, is_test=False):\n",
    "    \n",
    "    full_game_name = f'ALE/{env_name}-v5'\n",
    "\n",
    "    env = gym.make(env_name,\n",
    "                    repeat_action_probability=0.25, # Sticky actions with probability 0.25, as suggested by (Machado et al., 2017).\n",
    "                   )\n",
    "    env = GymWrapper(env,\n",
    "                     frame_skip=frame_skip, \n",
    "                     from_pixels=True, \n",
    "                     pixels_only=False, \n",
    "                     device=device,\n",
    "                     categorical_action_encoding=True)\n",
    "\n",
    "    \n",
    "    env = TransformedEnv(env)\n",
    "    env.append_transform(NoopResetEnv(noops=30, random=True)) # NOTE: Cartpole with no noops will fall into reset in the begining\n",
    "                                                                # I could use an small noop reset to avoid this, but I think is not necesary\n",
    "                                                                # in this case. Analyze this later\n",
    "    if not is_test:\n",
    "        env.append_transform(EndOfLifeTransform()) # NOTE: Check my environment is not based on lives (so not important)\n",
    "        # env.append_transform(SignTransform(in_keys=[\"reward\"])) #NOTE: this function simplifies the reward to -1, 0, 1\n",
    "        env.append_transform(RewardClipping(clamp_min =-1, clamp_max=1))\n",
    "    env.append_transform(ToTensorImage()) \n",
    "    env.append_transform(GrayScale())\n",
    "    env.append_transform(TimeMaxPool(T=2, in_keys = [\"pixels\"]))\n",
    "    if cropping:\n",
    "        env.append_transform(CenterCrop(400, in_keys = [\"pixels\"]))\n",
    "    env.append_transform(Resize(84, 84))\n",
    "    env.append_transform(CatFrames(N=frame_skip, dim=-3))\n",
    "    env.append_transform(RewardSum())\n",
    "    env.append_transform(StepCounter(max_steps=27000)) # NOTE: Cartpole-v1 has a max of 500 steps\n",
    "    env.append_transform(DoubleToFloat())\n",
    "    # env.append_transform(VecNorm(in_keys=[\"pixels\"])) # NOTE: this normalization could not be necessary as in Mnih et al. 2015\n",
    "                                                        # they only normalize to 0-1 the pixels\n",
    "    env.set_seed(seed)\n",
    "\n",
    "    # NOTE: a rollout will be take a trajectory of frames and group the frames by N=4 sequentially\n",
    "    # so that the output will be 7x4x84x84 because with a rollout of 10 steps we will have 7 groups of\n",
    "    # 4 frames each\n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/gymnasium/envs/registration.py:521: UserWarning: \u001b[33mWARN: Using the latest versioned environment `Asteroids-v4` instead of the unversioned environment `Asteroids`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    }
   ],
   "source": [
    "env = make_env(env_name=\"Asteroids\", \n",
    "               frame_skip = 4, \n",
    "               device=\"cpu\", #\n",
    "               seed = 0,  \n",
    "               is_test=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorDict(\n",
      "    fields={\n",
      "        action: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        collector: TensorDict(\n",
      "            fields={\n",
      "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
      "            batch_size=torch.Size([10]),\n",
      "            device=None,\n",
      "            is_shared=False),\n",
      "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        end-of-life: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        lives: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        next: TensorDict(\n",
      "            fields={\n",
      "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                end-of-life: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                lives: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "            batch_size=torch.Size([10]),\n",
      "            device=None,\n",
      "            is_shared=False),\n",
      "        pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "    batch_size=torch.Size([10]),\n",
      "    device=None,\n",
      "    is_shared=False)\n"
     ]
    }
   ],
   "source": [
    "# Define a random policy\n",
    "from torchrl.collectors import SyncDataCollector\n",
    "from torchrl.envs.utils import RandomPolicy\n",
    "    \n",
    "policy = RandomPolicy(env.action_spec)\n",
    "\n",
    "# Define a SyncDataCollector and collect some data\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn = env,\n",
    "    policy = policy,\n",
    "    frames_per_batch=10\n",
    "    ) \n",
    "\n",
    "\n",
    "for data in collector:\n",
    "    print(data)\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7YAAADeCAYAAAAJtZwyAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAknUlEQVR4nO3d6W8bd37H8c8cHA4vUdRhKbKycZwmmzabLdB2i0Wz3f+6DwsU6KJ9UKBIt0AOd+NbpymT4k3O1QfGTGTHhyjZ4vxm3i9AcA6aGpHzpvAd/mZoJUmSCAAAAAAAQ9mr3gAAAAAAAK6DwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABjNvewNLcv6kNsBFEaSJKvehLeiZeBy8twyHQOXk+eOJVoGLusyLV96sC0K27blOM5LLyRxHCuKIklvf9Acx5Ft27Isa6kXoiRJFMfxS3+m22JZ1i+25zL3J0lRFGXbXVYXH79lnpckSbKvOI4Vx/EH3lK8b7RcLLRcTnRcLHRcXrRcLKa2XKrB1rIs2batSqXy0hMUhuFLQbyJ4zjZk2zbl1/FnSSJoijKvkeSJNlO8rrtucz9pdIdJu9HJD8U27blum72WC7zOKbPiSR+iRqGlouHlsuHjouHjsuJlovH1JYLO9imO/bFJ8W2bfm+r2az+VI8w+FQg8FAcRwrDMPX7sSWZcn3fVWrVTmOo0ql8sbvm95vGmkYhppOpwrDUMPhUIvFQo7jyPM8eZ6n9fV1ue6Lp+LVSNMdKd3BpBcvFJI0GAw0HA5fOiJWNq7rqtFoZM/Jm14Q0yjT/SBJEs1mMwVBoOl0qtFoVNoXr7yj5XKg5WKj43Kg4+Kj5XIwteVCDrbpg+u6rur1eraTO46jTqejvb29LEhJevr0qR48eKAgCBRF0WufAMdx1G631W63Va1WVavVXvskO46jarWaheq6rkajkc7OzjSZTBQEgRaLharVqprNplqtlj7//HM1m80s+iAINB6PlSRJ9j1835fv+5KUbeejR480n89fCrVsfN/X9va2PM9TvV6X53m/uI1lWfI8T67rynVdVatVRVGks7MzjcdjPXv2LHu8kS+0XB60XFx0XB50XGy0XB6mtlzIwVZ6eVmE67rZn9VqVfV6/aXwPM9bam3/296WT7+n4zjyfV+e5ykMw+zPNKR07brruvJ9X/V6XWEYKooiWZalyWSS3Z9lWapUKmo0GpKkxWKhIAiyIygs2fn54guvOxfAtu3s6J3ruqrVaorjWOPxWGEYvrQvIH9ouVxouZjouFzouLhouVxMa7mQg216Anu1WtX6+roqlYqq1apc19Xa2lp2lEl6sTQhfeDf9uBHUaRer6fxeJwF87rb12o17ezsqFarZUeBFotFFv7FHSQ96pUuwYjjWIvFQsPhUI8ePcqOPLmuq93dXa2vr2fhXlz7XmbT6VQnJyfZ45G+4Pm+ny2NSZespMspPM9TkiRZiMucz4GbRcvlQcvFRcflQcfFRsvlYWrLhRxsL+7UaQDpUolarZa9ZZ6eaH7xql9vkiSJJpOJptNp9j1ep9VqqdVqybKsLOr0pPiLR6EurkdPb5P+ndlsprOzM81mM9XrdVUqFbVarewFJX1L/9WjKGVcKhEEgc7Pz196LNIlKOmLa7ofpI9P+txffF6QT7RcHrRcXHRcHnRcbLRcHqa2XMjBNj1JfTabqdfrZUcR0pPDLx6ZSV08+flN6+kvXnHtTTv5xSuzAbgeWgbMR8dAMdAy8q6Qg216sncYhprP57IsS9VqNbvS2u7u7ktHdNIjMum/p1f1enVtfRrT26JKr6DGunzg+mgZMB8dA8VAy8i7Qg62aUjpmu/0fID0ZPCLR3ziOM7WjafLEMIwVBAECoLgtQFeVvrBxOn3u/h9039+9TbSz2/t27adLe1I162nt794+fEyH8FKT1xPn2/LsrKlMemSmFeXwlz8MG9eIPONlsuDlouLjsuDjouNlsvD1JYLOdimwdXrde3u7mbRue6Lz2SyLOulB7zZbOpXv/qVoihSEAQKw1DPnj1Tr9fLjkpddse+GO5isdBsNtN8Ps/u92J46RM/n8+zsBzHUavV0t27dxVFUXYCe6vVyo5ULRYLRVH0xs8EK5N6vZ5djjy9/PvF2C7GmL7ops9n+pyU9TPKTEDL5UHLxUXH5UHHxUbL5WFqy4UcbNPlDpVKRc1m86UnJN3BoyhSFEXZbev1ehZNFEUaDAZyHGfpIw7p0Z00jPQr/X4XQ0njS0NNdwDXddVsNrOjXZb14nOi0v+f3h/nG7x4kU0/X6vZbL72g70vXoggfY7Txz19XpBPtFwetFxcdFwedFxstFweprZcyMH24mdVnZycZNGlkaVvradPSLrjXzzKMxgMsiM3ywiCILtseRAEOjk50WKx0Gg00mKx0GKxyG43Go0URZHu37+fvSCk0c7nc0nKTsJPPyss/fnCMNTz588VBEGpl+4sFgs9f/5crutqMBhkFzB41cXnP71q22g00nw+13A4LPVjmGe0XB60XFx0XB50XGy0XB6mtmwllzwc8bZLdefNxXMAXo3sTT/Hqw/DYrHIliIs86RY1osPe7YsK3vL/uIJ72ko6falLwSvXkXu1e1Jt/viEaR0Ocbrbl8WF5/ji5d7f51Xjyylz0m6ROZ9yftzQcuX/960fHNoeTl0fPnvTcc3h46XR8uX/960fHNMbbmQ79imP3j6dvhVPkfpOssQ0rDSt+TTeF+9vzToMAyXemFL7yM92lXW6KSfA0rP61jmcUyPHpb5iFze0XJ50HJx0XF50HGx0XJ5mNpyId+xveg6232dHfqyR67edtt3KXNwF11333zfj2Penxdafj/fl5bfP1q+PDp+P9+Xjt8/Ol4OLb+f70vL75+JLRd+sAVuWt5fEGkZuJw8t0zHwOXkuWOJloHLukzLy68hAAAAAAAgRxhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0dxVbwCwKo7jyLIsJUmiOI4lSUmSrHirACyLlgHz0TFQDKtsmXdsUUqWZcm2bTmOI9u2ZVnWqjcJwBXQMmA+OgaKYdUt844tSsVxHHmep0qloo8//lidTke9Xk+np6cKgkCj0UhhGK56MwG8Ay0D5qNjoBjy0jKDLUrFdV21Wi2tra3pj3/8o7788kv98MMP+q//+i8Nh0MtFgt+iQIGoGXAfHQMFENeWmYpMkqlUqmo1Wqp3W5re3tbOzs7unXrljY3N7W+vi7P81gCBRiAlgHz0TFQDHlpmXdsUQq2bcu2bW1tbekf//EfdevWLf3+97/Xl19+qa2tLbXbbR0dHanf72s+nysIAo4SAzlEy4D56Bgohry1zGCLUkhPZG80Gtrb29NHH32k27dv6/bt2wrDUOPxWJVKRc1mU67rKoqiVW8ygNegZcB8dAwUQ95aZrBF4VmWpXa7rVarpTt37uirr77Szs6OOp2OHMdRu93WnTt35LquvvjiC1mWpcPDQx0eHq560wFcQMuA+egYKIY8tsxgi8KzLEudTkf7+/v64osv9Hd/93fa3t7OwtvY2FCz2VSz2dTXX38t3/cVhqGOjo74DD0gR2gZMB8dA8WQx5a5eBRKwfd9ra2tqdlsqlaryfd9OY4j6cUyCtd15Xme2u22Njc3Va/XV7zFAF6HlgHz0TFQDHlrmXdsUXi2bWtzc1N3797V/v6+Op2OWq3WS+Gl6/8/++wz1et1/eUvf5FlWRwdBnKElgHz0TFQDHlsmXdsUQqVSkW+76tarcpxHNm2/dJlxy3Lkm3b8n1f9XpdlUplhVsL4E1oGTAfHQPFkLeWeccWhZckiY6Pj/Xjjz/KdV09ffpUk8lEjUZD1Wo1C/D8/Fzff/+9Hj58qKOjoxVvNYBX0TJgPjoGiiGPLTPYovDiOFav19ODBw+0tram4+NjRVGkJEmUJEl2NGk4HOr+/fv68ccf9ezZM5Y8ATlDy4D56Bgohjy2zGCLUpjNZhoOhzo8PNS3336rTqejjz76SOvr69kyiuPjYx0dHen09FTj8ZhfokAO0TJgPjoGiiFvLTPYovCSJNH5+bnG47H6/b4ODg5Ur9f1+eef69atW2q329re3la329Wf//xnPXz4UPP5fNWbDeAVtAyYj46BYshjywy2KIV0acRkMtGzZ8/k+75arZbiONZ0OlUURXr+/LkGg0H27wDyh5YB89ExUAx5a9lKLvl+8MUrXAEmStf6VyoVOY6jdrst3/fleZ5839disdCTJ0+yZRJXXSqR9+VStAzT0TIdw3x0/AItw3R5apnBFqVl23Z2WfL0M7XCMLz2L0F+iQI3q4wt0zGKpowdS7SM4lllyyxFRmklSaI4jl/6pZL3X4AAfomWAfPRMVAMq2yZwRaldZ3lEADyg5YB89ExUAyrbNleyXcFAAAAAOA9YbAFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0RhsAQAAAABGY7AFAAAAABiNwRYAAAAAYDQGWwAAAACA0dxVb8CHZlnWG/9fkiQ3uCUAroOWAfPRMVAMtIw8KuRga9u2bNuWZVnZ16uSJMm+4jhWHMcr2FIAb0PLgPnoGCgGWkbeFXKwtSxLjuPIsqwswNeJokhxHHNkCcgpWgbMR8dAMdAy8q4wg61lWfI8T47jaG9vT7dv35bv++p0OqpUKr+4fRiG6na7mkwmOjk50eHhocIwVBAEhFhC6f6zvb0t3/fluq48z3vtbSeTiXq9noIg0GQyURiGN7y1xUbLuA5azgc6xnXQcX7QMq7jplsuzGBr27Z831e1WtWvf/1r/e53v9P6+ro+++wz1Wq1X9x+Op3qxx9/VLfb1f/8z//o/Pxc8/lcURQpiqIV/ARYlfTIo+/7unPnjjqdjhqNxmv3G0nqdrv6v//7P02nU4VhyC/R94yWcVW0nB90jKui43yhZVzVKlouzGDruq62t7e1tram27dva29vT+12W1tbW6pWq7+4/Xw+12AwULVaVa/X0+npqcbjsY6PjzWbzRQEAQEWXLqEptFoqN1uq91u65NPPtHW1pbq9bqazeZr/16r1VIYhhoOh4rjWK7rarFYaD6f3+TmFxYtY1m0nD90jGXRcT7RMpa1ypYLM9iura3pn//5n/Xpp5/qt7/9rb7++mt5nqe1tTXZ9i8/1SiOY+3v72s2m+mzzz7TX/3VX+nk5ET/9m//pm63q+fPn2swGKzgJ8FNcRxHjuPo7t27+vu//3vt7Ozom2++0UcffaRGo6FGo/Hav3dycqK//OUv6na7+vd//3c9fvxYx8fHOjw85CIJ7wEtY1m0nD90jGXRcT7RMpa1ypYLM9i6rqvNzU3t7u7q1q1b2t7eluu68n3/jVdt8zwvW8c9Go3kuq42NjY0m800Go1W8FPgJqVX96vX69rb29POzo4++eQT7e7uqtFoqF6vv/bv+b6vIAjUbDZ17949jcdjjUYjVSoVfom+B7SMZdFy/tAxlkXH+UTLWNYqWy7MYBsEgc7OznR4eKjd3V2dn5+rVqtlJ7y/yrIsua4rx3G0u7srz/O0s7Oj0Wikk5MT/elPf1K3213BT4KbkF6m3rZtra2taX9/X9vb22q1WtnJ7W/SbDazJRVhGOo3v/mNvv/+e3377bec2/Me0DKWQcv5RMdYBh3nFy1jGatuuTCDbRRF6vV6qtfr6vf7Go/Hkl4soXhdeJKyB3d7e1sbGxva3NzUZDLR6empfvrpJ3377bc3tflYgTS8RqOhW7duZWv/q9XqG/cZSarVaqrVagrDUI1GQ+PxWGtra1y04j2hZSyLlvOHjrEsOs4nWsayVtlyIQbbVz8w+jr34TjOa88ZQPGkHx4+GAz09OlTLRaL7DL274pPehFurVaTZVnZ0gouiHA9tIyroOV8oWNcBR3nDy3jKlbZsvGDrW3bcl03W/bwtg+MfpM02PRk5/S+UFxJkiiKIiVJooODA/3Hf/yH9vb2tLe3J9d1tb6+/tqr/V3kOI7a7baazaa2traypRO4GlrGVdByvtAxroKO84eWcRWrbtn4wbZSqajRaKjVaqnZbKrRaGRruJeNJ0mS7AvFZlmWKpVK9qL96ov1ZfeB9Ehkej+4OlrGVdByvtAxroKO84eWcRWrbtn46u/cuaN/+Id/0Pb2tv7pn/5Ju7u72ZXbLvtgRFGkMAw1m800mUw0Ho+1WCxuYOtx09KLGniep08//VRbW1v66quv9M0336jT6ejTTz+91NEkvH+0jGXQcj7RMZZBx/lFy1hGXlo2frDtdDr66quvtLOzo9/85jfa2dnJTj6+rDiOFQSBwjDUfD7XbDbjvIyCSk9or1Qq2t7e1scff6zPP/9cX3/9tVqtljY3N994Cfs34Qjk+0HLWAYt5xMdYxl0nF+0jGXkpWXjB1vf99XpdNTpdFSv1+V53tLLT4bDobrdro6Pj/Xtt9/q8PBQJycnH2iLsUppdLVaTXfv3tXf/M3f6O7du1pfX1etVsuWTVwmvDAM9ezZM43HY92/f18PHjzgfJ5roGUsg5bziY6xDDrOL1rGMvLSsvGDba1W087Ojra3t7W2tqZGo7H0ye3Pnz/XvXv39PDhQ/3rv/6rnjx5ovPz8w+0xVgl27bleZ5arZZ++9vf6o9//KM2Nja0u7srx3GW2ncWi4V++umn7AX7z3/+M0cir4GWsQxazic6xjLoOL9oGcvIS8vGD7ZJkmTLHJIkufLlyNNLUy8WC83nc14MCyzdRy6elH6Vq/3FcazpdKrhcKjRaKThcMh+cw20jGXRcv7QMZZFx/lEy1hWHlo2frAdDod68uSJwjDUF198caX40iMM9Xqdz9gquPQFNj3vYzabKQiCK91XFEXqdrt6+vSpHj9+rEePHrHs6RpoGcug5XyiYyyDjvOLlrGMvLRs/GAbBIGGw6FarVZ2VEnSUvGlb59XKhU+X6sE0kvOR1GkIAiyz9u6yv0sFgtNp1NNJhONRiN+iV4DLWNZtJw/dIxl0XE+0TKWlYeWjR9s+/2+7t27p/l8rsFgoCAIlv6MrWazqZ2dHZ2fn2t9fV3tdlthGGo6nX7ALccqxHGs+Xyu8XisR48eaWNjQ1EU6eOPP176Rdd1XX388ceyLEuHh4dqNptXPjoFWsZyaDmf6BjLoOP8omUsIy8tGz/Ynp+f66efflKSJBqPxwqCQJZlLfUgNhoNua6rfr+vTqejVqul4XD4AbcaqxJFURbe48eP5fu+2u224jhe+r48z9P+/r6azaZ++OEHNRoNfoleAy1jGbScT3SMZdBxftEylpGXlo0fbIMg0Hg8Vr/f18OHD+V5nra3t7WzsyPbti8VYPrWefqFYkvPA5hMJjo/P1e/31ev11Oj0VC9Xl/qRTs9OT694ttVL64AWsbyaDl/6BjLouN8omUsKw8tGz/YTiYTHR8faz6f61/+5V/03//93/rDH/6gb775Rp7nXeqE9SiKtFgstFgsFIahwjC80hEGmCG90t/p6akkaWtrSz/88IM6nY7u3r2rVqt1qfuxLEuVSkW+72fnj3C1v6ujZSyLlvOHjrEsOs4nWsay8tCy8YPtxbe+T05OFMexer2e5vO5LMu61BGi9CTn9LLmF0+SRzElSaLZbKbRaKR+v6+zszNJLz47K47jSx8dSo9OXfwTV0PLuApazhc6xlXQcf7QMq5i1S0XZrBNkkQ//vijDg4OtL+/r08//VSdTke1Wu2db333+309fvxYDx8+1PHxsbrdriaTyQ39BFiFKIp0cnKi8/NzhWGo0Wikvb09NZtNJUmier2uer3+1vsIw1CHh4c6Pj7W8fGxptMp5/NcAy3jKmg5X+gYV0HH+UPLuIpVt2z8YJt+ZlIYhjo4OFClUtHjx491eHioOI51586dd97HZDLRycmJTk9P1ev1NBgMuER8wcVxrPPzc52fn2s+n6vf7+vu3bv6wx/+oK2tLbmue6nwut2ujo6O1Ov1NJvN2G+ugZZxFbScL3SMq6Dj/KFlXMWqWzZ+sE2lJ6bHcazBYKCjoyNZlqVer6c4jlWr1VSpVF66fXoEoNvt6uDgQKenp5rP51f+3CWYKQgCTadTDQYDDQYD9ft9VatVtVot2bb9iyUT6Qv9bDZTv99Xt9vNXqx5wb4+WsZV0XJ+0DGuio7zhZZxVatouTCDraQsmJOTE3333XcajUb61a9+pdlsplu3bqndbr902+fPn2s0Gun+/fv67rvvdHx8rPF4zMUGSmY+n+v58+dqNBo6OjrS+vq6fN/XxsaGbNuW67ovxReGoSaTSfYC/+DBA52dnWk+n/NL9D2hZVwFLecLHeMq6Dh/aBlXsYqWCzXYSj8fKUofyNPTU4VhKNu2XwpqsVjo2bNnGg6HOjs7U6/X03A45EWwhOI4zs4lOTs70+npqTqdjra3t+W6rjzPe+nKf5PJRL1eT8+fP8/2m9lsxpX+3jNaxrJoOX/oGMui43yiZSxrFS1bySXXBJj0WWBra2tqtVra3NzU119/rbW1Ne3u7qrT6WS3WSwWOjo60nA41P3793Xv3j3NZjP1ej0uNlAy6RXaGo2Gfv3rX2tjY0O///3v9bvf/U71el0bGxty3Z+PAR0cHOh///d/dXZ2pv/8z//UwcFBFqIJn9VGyyiqMrVMxyiqMnUs0TKKaxUtF+4dW0nZWu7hcCjbttVsNtXtdrW+vp7dJggCHR4eajQa6eDgQE+ePMn9ix8+jDSW6XSqBw8e6OjoSJubm9rb21Oj0VAcx/I8L7v9wcGBvvvuO/V6PT158kSnp6daLBbsPx8ALWMZtJxPdIxl0HF+0TKWsYqWCznYptLlEGmEtVot+39hGGo4HGo+n2swGKxwK5EXcRxrOp0qDEPdu3dPcRyrWq1qbW3tpUvan56e6sGDB5pMJjo/P1cQBJw38oHRMpZBy/lEx1gGHecXLWMZN9lyIZciX3Txg4Bf/Rkuvq3N0SSkLMtSpVKR53myLOsXV26Loij7oOkoin6x9j/v+xItoyyK3DIdoyyK3LFEyyiPm2i58IMtcBWO42RHkS6+eEs/nwyfXv7+VXl/EadllElRW6ZjlElRO5ZoGeXyoVtmsAXe4G37/Nuy4ZcokC9FbJmOUTZF7FiiZZTPh2y50OfYAteR91+GAC6HlgHz0TFQDB+yZfvdNwEAAAAAIL8YbAEAAAAARmOwBQAAAAAYjcEWAAAAAGA0BlsAAAAAgNEYbAEAAAAARmOwBQAAAAAYjcEWAAAAAGA0BlsAAAAAgNEYbAEAAAAARmOwBQAAAAAYjcEWAAAAAGA0BlsAAAAAgNEYbAEAAAAARmOwBQAAAAAYjcEWAAAAAGA0d9UbcBmWZWX/nCTJCrcEwHXQMmA+OgaKgZZRNLkebC3LkmVZ8n1f1WpVURRpNpspjmNFUUSEgCFoGTAfHQPFQMsoqtwvRbYsS9VqVc1mU77vy3VdOY7z0lEmAPlHy4D56BgoBlpGEeX2HVvbtlWtVuW6rnZ2drS7u6vpdKqzszOFYajJZKLFYqH5fK7pdLrqzQXwBrQMmI+OgWKgZRRZbgdbx3HUarVUq9X05Zdf6q//+q81HA719OlTzWYzdbtdTSYT9Xo9zedzxXG86k0G8Bq0DJiPjoFioGUUWW4HW9u2ValUVK1WVavV1Gw2JUntdjs7H8DzPIVhqMFgoDAMFQQBAQI5Q8uA+egYKAZaRpHlcrC1LEuVSkWdTkdra2va3NzU2tqams2m2u22oijScDjUfD7X0dGROp2ORqORHj9+rNFoxEnvQE7QMmA+OgaKgZZRdLkbbC3Lkm3bchxH9XpdjUZDvu+rVqtJkhqNhpIkUavV0mKxkCSNRiNVKhUdHR2tctMBXEDLgPnoGCgGWkYZ5G6wrdfrarVaarfbun37ttbX17W2tvaL27muK8uytLa2pu3tbVUqFTUaDU0mE4VhqDAMV7D1AFK0DJiPjoFioGWUQe4G22azqd3dXW1ubuqzzz5Tu93+RXiWZcnzPEkvzgnY39+X7/tqNBoaDoeazWZ8DhewYrQMmI+OgWKgZZRBbgZb27Zl27Y8z1Oj0VCtVlO1WpXv+7LtN3/cruu68jxP9XpdGxsbCoJAvV4vW0ZBfMDNomXAfHQMFAMto0xyMdhaliXf97MT2m/duqX19XWtr6+r1Wq9NTzP87S+vq5qtaq//du/Vb/f1/fff6/RaKQoihRF0Q3+JEC50TJgPjoGioGWUTa5GGwvXnrc9/3sq1KpyHXfvonp343jWO12Ozsp3rZtJUmiOI45qgTcEFoGzEfHQDHQMspmpYOtZVlyHEe+7+vOnTvqdDra39/XJ598It/3s3X+77oP27bluq4ajUb2wdONRkOLxULj8ZjwgA+MlgHz0TFQDLSMslrpYJtedrxarWpvb097e3va3d3V/v5+dk7Au1iWJcuy5Lqu6vV6dkSpXq/LsixNp1M+VBr4wGgZMB8dA8VAyyird+/ZH1gaThqabdvZf7Ms60r3dfHfAdwMWgbMR8dAMdAyymjlg6308odGO45z6aNJb7uvq/59AFdHy4D56BgoBlpG2ax870yP+lznSNKr93Xd+wGwPFoGzEfHQDHQMspopYPtxRPTK5XKOz9T61335TiOXNfNviqVCvEBN4CWAfPRMVAMtIyyysU7tunJ6enXde4rXXLhui7LJYAbRMuA+egYKAZaRhmt9KrISZIoDEMtFgv1+33VajVZliXP8+S6rjzPk+M4l1r2EMexJpOJZrOZRqNR9s9cihz48GgZMB8dA8VAyyirlQ62cRxrsVhoOp3q9PRUcRwriqLss7fW19ezD5F+V3hRFKnf72s8HqvX66nf7ysIAkVRdEM/DVBetAyYj46BYqBllNXK37G1LCs7GjQYDFSr1VSv17M/L16m/G3iONZsNtN4PNZsNlMURXy+FnBDaBkwHx0DxUDLKKuVDrbSi2Dm87kODg50enqqg4MDtVotbW1tqVKpaHNzU/V6/Z3nBiwWCx0eHurk5ETdblfz+VxxHLNUArghtAyYj46BYqBllNHKB1vpRXzj8ViWZWVHhdKjTK1WS9Vq9a1/P0kSRVGk0WikwWCg6XSqKIqIDrhhtAyYj46BYqBllE0uBlvpRTzpye7z+Vyj0UiPHz/WaDTSRx99JElyHEfVavWlk90Hg4HOzs7U7/fV7XZ1fn7OSe3ACtEyYD46BoqBllEmuRpsJSkMQyVJkoX3/PlzVSoVNZtNVatVVSoVOY6T/b3hcKhHjx7p/Pw8C3A+n6/qxwBKj5YB89ExUAy0jDLJzWCbSpJEcRwrDEONRiPFcaxnz56pWq2q0WjIsqwsPtu2NZvNNBwONRqNNJvNuFIbkBO0DJiPjoFioGWUQS4H2yiKNJlM9PTpU7muq8FgoIcPH+rWrVv66quvVK/X1Wg05Hmeut2uHj16pNFopH6/r8lkwjIJIAdoGTAfHQPFQMsog9wNtqn0am5BEGgwGChJEnmep+FwmF1mPAxDTadTzWYzzedzhWHIJciBnKFlwHx0DBQDLaPIcjvYSj8vmxiPxwqCQIvFQovFQp7nyfM8ua6rbrerZ8+eKQgChWG46k0G8Bq0DJiPjoFioGUUlZVccl1BepW0VXJdV7VaLftAacuyFARBdpU2LkGOPMj7PkjLwOXkeR+kY+By8r4P0jJwOZfZB3P9ju2rkiRREASyLCsLL10ekV7OHED+0TJgPjoGioGWURRGvWMrvX47CA55kvf9kZaBy8nz/kjHwOXkfX+kZeByCveOrURkQFHQMmA+OgaKgZZRBPaqNwAAAAAAgOtgsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGI3BFgAAAABgNAZbAAAAAIDRGGwBAAAAAEZjsAUAAAAAGM1KkiRZ9UYAAAAAAHBVvGMLAAAAADAagy0AAAAAwGgMtgAAAAAAozHYAgAAAACMxmALAAAAADAagy0AAAAAwGgMtgAAAAAAozHYAgAAAACMxmALAAAAADDa/wPwP1r3t7SKDwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1200x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVR0lEQVR4nO3dSWycB/3/8ffzzDzPLJ7MeJk4sZ3YCa1rJ3HdmrgUqiSAEA2KBImqAocKEJUqOHPggLhw4Yg4c+CGKlFahKjEIoLUiqZNII7TNMFLs3iLG28Zb+NZH/8O/nvAtZ3/DPHMM8vnJeUSjx8/Y7/95JtnNdbX19cRERGRmma6vQIiIiLiPg0EIiIiooFARERENBCIiIgIGghEREQEDQQiIiKCBgIRERFBA4GIiIgA3nxfaBhGMddD9lgx7jelBipLse45pg4qi7YFkm8DeQ8EpWIYBj6fD4BsNksmk9nyZgzDwLZtTHPnnRuO4+Q+zzRNbNveNd719XWSyWTRNpyFKOR9VTs1oAZAHagDNVDqBspiIDAMA8uycn9aW1sxDIPFxUVmZ2dJJpO511qWRVNTE8FgcMsyvF4vtm0Tj8dzn1dXV0drays+n49sNksymcQ0TSzLwjAMMpkMd+7cIZ1Oux5BIe+rGqkBNQDqANSBGnCvgbIYCDweD6FQiFAoRCAQoKenB4/Hw9jYGMvLy9sCaGhooKGhYcvnRyIR6uvrmZ2dZXx8nNnZWcLhMN3d3ezbt49kMsn09DSWZdHY2Ijf7ycej3P//v1tU6cb/vt9eTweTNPE4/Gwb98+9u3bt+V9VSM1oAZAHYA6UAPuNVAWA8HmD6W5uZlAIEA4HAbA5/Nt272zurrK0NDQlr+3bZve3l4OHDiA3+/HsixgI4xgMIhlWczPz3PlyhXC4TDPPvts7msYhlEWx8P++31Fo1FCoRCRSIRAIEAwGNzyvqqRGlADoA5AHagB9xooi4EgkUgwPj7OgwcPCIVCtLa25o6feL1ePB4PjuPkprZsNrvl803TxHEcN1Zd9ogaEFAHogbcVBYDgWVZRKNR6uvrCQQCOI5DJpMhGAzS0dFBNBplamqKeDy+7Ye/KZPJkEwmSafTuRMtHMchkUhgWRZ+v5/Ozk78fj/BYJBMJkMqlXJ919Amn8/HgQMHCIVC+P1+bNvGtm08Hs+291WN1IAaAHUA6kANuNdAWQwEpmnmfjBer5e5uTkMwyCZTGJZFqFQCK/Xi2maOwbgOA5LS0tMT0+zvLzMysoKsDFpzs7Osra2RjqdJhQKYRgGy8vLJBIJkslkWRwvgo3vQSAQIBQKARvvKZlMEovFSCQSxGKx3PuqRmpADYA6AHWgBtxrwFjP890X87jK5iUWXq9316+ztra2ZTfRp9m2jWVZuWkynU7j8Xjw+XyPvHRjbW2tbALw+Xx4PJ4dP755iUk6nc5reZV27bEaqIwGQB0UWyV0oAaKy60GymIgkL1XaRsB2XuVOBDI3tO2QPJtQLcuFhEREQ0EIiIiooFARERE0EAgIiIiaCAQERERNBCIiIgIGghEREQEDQQiIiKCBgIRERFBA4GIiIiggUBERETQQCAiIiJoIBARERE0EIiIiAgaCERERAQNBBXDMAw9g7zGqQEBdSDFa0ADQQUwDAOfz0cgEMDj8bi9OuICNSCgDqS4DXj3dGlSFM3NzZw/f56Wlhb+8Ic/cP36ddbX191eLSkhNSCgDqS4DWggKHOWZfHiiy9y7tw5Dh8+TCgU4vbt26yuruI4jturJyWgBgTUgRS/AQ0EZczj8dDW1saZM2c4ceIE0WiUVCrF8ePHuXnzJisrK26vohSZGhBQB1KaBnQOQRmzLItjx45x6tQp2traiEQi9PT0cPr0aSKRiNurJyWgBgTUgZSmAQ0EZczn83Hy5EkOHjxIIBDAMAyCwSBnzpzRRqBGqAEBdSClaUADQRlbW1vjb3/7G4ODg0xMTLC4uEgsFuN3v/sds7Ozbq+elIAaEFAHUpoGdA5BGctkMty5c4df/epXHDt2jPb2dsLhMFeuXGF5ednt1ZMSUAMC6kC2NvDZz36Wzs5Ompub97QBDQRlzHEc5ubm+POf/8zw8DDt7e3s37+fu3fvkkql3F49KQE1IKAO5D8N/PWvfyWVSmEYBqZpMj09vWcNaCAoc47j8PDhQxYXF/nwww8xTVMbgBqjBgTUgWw0EI/HCQQCNDc309DQQCQSYW1tbU9a0DkEFcJxHDKZjDYANUwNCKiDWmYYBqdPn+bChQt84xvfoL+/nx/+8Ie0trbuya2MtYdARERcZxgG4XCY/v7+3Fn0AMlkkhs3bjA/P1/TQ5BpmoTDYS5cuMDTTz9NKBTCMAzOnz/PBx98wMrKCnNzc4/1NTQQiIiIq2zbpqGhgdOnT/PVr36VUCiEaZqsr6+ztrZGS0sLN27cYHR0tKZPorQsi5aWFsLhcO45BgcPHiQcDmPb9mMvXwOBiIi4KhAI0NHRwXe/+12+/OUvU1dXl9tDkEgk6Ojo4OLFiziOw61bt2p2T4HjOCwvL7O8vEwkEsG2bVZWVkgmk2Sz2cdevgYCERFxlW3bNDY20tfXh9/v33I83O/386UvfYmuri4+//nP8+Mf/5jbt2+TyWRcXOPScxyHhYUFfv3rXzMyMsLnPvc5Tp48yeuvv87Vq1eZmZl57K9hrOf5mCQ9f7uyFOMJaGqgshTrKXjqoLJUwrYgGAxy6NAhvv/97/Paa6/R0NCAaW495z2ZTHL37l1eeeUVbty4QTqd3tN1qBQ+n49gMEhLSwv9/f1cunSJyclJEonErp+TbwPaQyAiIq6pq6vjiSee4Pnnn+fUqVMEg8EdBw6Px4NlWdsGhVqTTCZJpVIkk0lWV1eZmZmprvsQGIaxLYD19XU957uGqAEBdVCLIpEIJ06c4Ny5c/T39+Pz+bY14DgOi4uLTE1NPfJ/wrVifX2deDzO2NjYni63LAYCv9+P1+vNnTUJkE6nSaVSNbtb6FFM09zxF6aSN5pqoDDV2ACog0JVQwfNzc08//zzXLhwYdfXpFIpbt68yZtvvsknn3yyJyfQVYu9bMDVgcA0TRobG3n11Vd55plnOHjwYO5jV69e5R//+AfvvvsusVjMvZUsM5Zl8eKLL9LU1IRlWbm//+CDD5iYmGBpacnFtSucGihctTUA6uB/US0dpNNpkskkyWQSn8+342symQzj4+NcunTpsa+1ryZ73YCrA0EoFOK1117jm9/8Ji0tLQSDwdzHOjo66O7upquri9/+9rdMTk7W9FTo9XppaWnh7NmzfOc73yEajW4J4P333+fixYtcuXKFjz/+2MU1LYwayF+1NgDqoBDV1sHExARDQ0PcuXOHY8eO7fgav9/P8ePH+frXv87s7KwaKFIDru8hCIVC1NfXE4lECAQCuY8FAoHc2ZQzMzO88cYbxONxF9fWXaZp0tDQwHPPPUdPTw+RSGTLblWv14vP56OhoWHXKbscqYH8VWsDoA4KUW0drK6uMjU1xUcffcShQ4dyd+D7b16vl0OHDnHmzBnu3LmjBorUgKsDQTabZXJykocPH9LY2LhlI2BZFm1tbUQiEebm5nj77bdrOgDDMLBtm2g0im3b235hjh49SiQSoa2tDcdxXFrLwqmB/FVrA6AOClFtHWSzWcbHx3n//fd59tlnOXr0KF7v9n+a9u/fz8mTJ3nw4IEaKFIDrl2/4fF4sG17y1TzaYZh5CadWpfJZJicnOT1119nampqx8tMGhsb6ejo4PDhwy6sYeHUQGGqsQFQB4Wqxg5GR0d54403GBoa2vWGQ2rgP4rVgGt7CPr6+vjWt77Fyy+/TGtr6473Yc5kMqyurjI/P18Rk26x2LZNb28vp0+f5pVXXuHIkSN7ct9qt6mB/FVrA6AOClGtHfT19fHtb3+bkydP7vp+1MCGYjbg2kDQ1NREd3c3ra2tu058CwsLXL9+nd///vesra2VeA3Lh9/vp7+/n7Nnz9LZ2bnjLiKAsbExBgYGuHbtmgtrWTg1kL9qbQDUQSGqtYOmpia6urpoamra9cZDamBDMRtwbSAwTRPbth+5+ycejzM1NcXly5dr9mEWsHGCyOHDh3nqqacIh8O7vm5iYoLBwUGuX79ewrX736mB/FVrA6AOClGtHaiB/BWzAdfOIXj48CGjo6OP3PXj8/mor6+v+XunO47D2toaq6urj3zdwsICd+/eZWRkpERr9njUQP6qtQFQB4Wo1g7UQP4KbWB0dDTvZbs2ENy7d4+LFy8Sj8d3vaNSNBqlp6eH9vb2R55wVO2Wlpa4cuUK77333iNf19vbyxe/+EWi0WiJ1uzxqIH8VWsDoA4KUa0dqIH85dtAf38/L7/8Ml/72tfyXrZrA8HS0hIfffQRv/jFL5iamtrxtqTZbJZUKlVzj7n8NMdxmJqa4saNG1y7dm3XXxi/308wGNzxkp1ypAbyV60NgDooRLV2oAbyl28D0WiUL3zhC/zsZz/Le9muDQTJZJKZmRn+9Kc/cfv2bZaXl7e9ZmVlhampKRYWFmr6rFKABw8ecO3aNf74xz8Si8V2vEvX4uIi8/PzJJNJF9awcGqgMNXYAKiDQlVjB2qgMDMzM9y6dYsrV67sOhDYtk1TUxM9PT15L9e18TGbzbK0tMTVq1cZHBykubl5y92WstksMzMz3Lx5Uw+zAObm5lhdXWVhYYGXXnopd+e2TZtT4+jo6P/32FK5UAOFqcYGQB0Uqho7UAOFWVpaYmJiguHhYRzH2fXKDMMwCnpctOv7k9LpNO+99x779+8nEAhw5MgRYOMMyYGBAS5duqSnnP0/iUSCyclJhoaGcneo2twluLi4yOjoKENDQxV3By81kL9qbQDUQSGqtQM1kJ/Ozk6+8pWv8IMf/GDX8ykcxyGRSLCwsMChQ4fyWq6xnuczEot5ZmcgEKCrq4uurq5cAPfu3WN4eJjh4eGavub00zweD52dnXzve9+jp6eH5uZmAN58803effddhoaGiMViRXn8qRooD242AOqgXGhbULteeOEFzp8/z49+9KNdzxNZWFhgYGCAX/7yl7z99tt5LbcsBgKAcDhMOBwmFAoBG8eLlpaWKuYRnqUUDAbp6elh//79uV2F//73v5mens4dU6y0jQCogUK41QCog3KibUFteuaZZzh37hw/+clPct+jT4vFYvzrX//i5z//OX//+9/zWq7rhww26Yedv3g8zsDAAKZp5n4xM5kMjuMU7R+BUlAD+avWBkAdFKJaO1ADjxaLxfjkk0+IxWLU1dXtOKDZts2+ffs4ePBg3sstm4FAClPrl96IGpAN6qD2PHjwgJGREe7du8eBAwewLGvba4LBIK2trfT19eW9XNcuOxQREZHCpVIpbt26xU9/+lMuX76842WajuOQTCZ3/NhuNBCIiEhJbF4GV8ilcLKd4zgsLy/z4Ycf8tZbbzE7O7vtNcvLy9y9e5d33nkn7+XqkIGIiJTEgQMH8Pv9ZDIZZmdnSaVSFX2ug5symQyLi4vcuHGD27dvU19fT2NjI7CxB2HzPgVjY2N5L1MDgYiIFJ3H4+HJJ58kGo2STCa5desWc3NzJBKJmr/R0P9qfX2d8fFxLl++jG3bPP3008DGSYcDAwP885//JBaL5b28srnsUPZWJV5qJHurUi87lL1VDtsCwzAIh8O8+uqrtLW1YVkWS0tL/OUvf2F4eJhYLKYbDj2GpqYmOjs76e3tBTYGgsHBQe7evUs6nc67AQ0EVaocNgLiLg0EAu5vCzweD3V1dfT29nL27FmampqwLCt3HPytt95ieHiY+fn5PV/PWmGaJl6vN3e1geM4pNPp3BUo+TagQwYiIlI0lmVRX19PX18fkUgEr9eLYRh4PB7C4TBdXV2kUilSqVRBZ8TLfziOk/sePg6d6ikiIkVhWRYNDQ20t7dz9OhRAoHAlisMTNPkyJEjdHR0EA6HXVxTAQ0EIiJSJA0NDXR3d3P69GmCweCOhxra2to4fPgwTU1NLqyh/LeSDASGYeD3+2lra8Pv9+sYZA1SAwLqoNa0t7fz1FNP0d7evutDeGBjcPjMZz6jHlxWknMIQqEQHR0dnDp1iqGhIWZmZpiZmWFubq4UX17KgBoQUAe1JpFI5HWvgXQ6XXGPaq5GRR8INi83aWlp4cknnyQQCDA7O8v4+Dgff/wx9+/fL/YqiMvUgIA6qEWZTIZMJpN76uJuewA2H8RjGIZuVOSiog8Etm0TjUZpaWmhvr6e+vp64vE4Bw4cwOfz8eDBA92UosqpAQF1UIvS6TQrKyvEYjEikQiBQGDHoaCuro6DBw9imiaO47iwpgIlOIegu7ub5557LncHJdh4CtOhQ4c4ceIE4XBY97WucmpAQB3UorGxMd555x1+85vfcO3atV0vizNNE8uy2LdvnxpwUdG+86ZpUl9fT0dHB62trUQikS0fDwaDdHR08NJLL2lDUKXUgIA6qGWZTIZYLMbU1BQPHz7c9X//aqA8FOW7bpomwWCQvr4+jh8/TmNj47YfsGmaubONw+Hwjs9zlsqlBgTUgWwwTZNQKLTrP/RqoDwUZSDYvLToiSeeoK2tjbq6uh1fs3lLy2Aw+MhLUqTyqAEBdSAbP1/Lsqirq9v1pEI1UB6KNhBsRuDxeB55benm87F1/Wl1UQMC6kD+00A+hwHUgLuKNhBsToSPisAwDGzbJhAIaCKsMmpAQB2IGqgkRfmuZ7NZVlZWGB0dxe/309TURDAYxOPxbHvd/fv3mZmZYW1trRirIi5RAwLqQNRAJSnKHgLHcUgkEgwPD3P16lWmpqZIJpPbXpdOp7l37x5LS0t6FnaVUQMC6kDUQCUp2n6ZVCrFyMgI09PT2LZNOBzG5/PlpsJsNsvq6iojIyPE43HdjKIKqQEBdSBqoFIU9UCN4zisrq5y8+ZNstksL7zwAtFoFNM0uX//PoODg0xMTOjuZFVMDQioA1EDlaDoZ25kMhnGxsaYn59nYmKCs2fP0tDQwOTkJDdv3nzkzSqkOqgBAXUgaqDcGet5PknicS8D8Xq91NXV0dXVRTAYZGFhgbGxMRYXFx9rubKzYjwgRA1UlmI9JEYdVBZtCyTfBko2EGzaPLs0k8mQSqW0e6hIynEjsEkNlEa5DgSb1EFpaFsgZTsQSGmU80ZASqPcBwIpDW0LJN8G9AQJERER0UAgIiIiGghEREQEDQQiIiKCBgIRERFBA4GIiIiggUBERETQQCAiIiJoIBARERE0EIiIiAgaCERERAQNBCIiIoIGAhEREUEDgYiIiKCBQERERNBAICIiImggEBERETQQiIiICGCsr6+vu70SIiIi4i7tIRARERENBCIiIqKBQERERNBAICIiImggEBERETQQiIiICBoIREREBA0EIiIiggYCERERAf4P1kq3A5Sql+MAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAXr0lEQVR4nO3d21Mb58EG8GdX2tUBoQPInITBroPBDiGhJk3tMZ5k0sQzmWnjSdrmItNTZjJt/4FetRf9Bzq56U2n06tOM9M0yXTimaR1fRFP6sQk2NgEB5A5IwOSACF0Wmm1+i74pIQCjhSzWh2e30xukMCvwsPq0e77vivkcrkciIiIqK6JRg+AiIiIjMdCQERERCwERERExEJAREREYCEgIiIisBAQERERWAiIiIgILAREREQEwFzsEwVB0HMcdMj02G+KGagueu05xhxUFx4LqNgMFF0IykUQBFgsFgBANpuFqqq7XowgCJBlGaK4/8kNTdMK3yeKImRZPjC8uVwOiqLoduAsRSmvq9YxA8wAwBwwB8xAuTNQEYVAEARIklT4r6OjA4IgYGtrC6FQCIqiFJ4rSRKam5tht9t3/Qyz2QxZlpFIJArf19DQgI6ODlgsFmSzWSiKAlEUIUkSBEGAqqqYnZ1FJpMxPASlvK5axAwwAwBzADAHzIBxGaiIQmAymeBwOOBwOGCz2dDf3w+TyYSFhQVsb2/vCYDH44HH49n1/S6XC263G6FQCIuLiwiFQnA6nejr60NjYyMURcHKygokSUJTUxOsVisSiQTu37+/p3Ua4auvy2QyQRRFmEwmNDY2orGxcdfrqkXMADMAMAcAc8AMGJeBiigE+V9KS0sLbDYbnE4nAMBisew5vROPxzE5Obnr67IsY2BgAK2trbBarZAkCcBOMOx2OyRJwvr6OkZGRuB0OvHEE08U/g1BECriethXX5fX64XD4YDL5YLNZoPdbt/1umoRM8AMAMwBwBwwA8ZloCIKQSqVwuLiItbW1uBwONDR0VG4fmI2m2EymaBpWqG1ZbPZXd8viiI0TTNi6HRImAECmANiBoxUEYVAkiR4vV643W7YbDZomgZVVWG329Hd3Q2v14tAIIBEIrHnl5+nqioURUEmkylMtNA0DalUCpIkwWq1oqenB1arFXa7HaqqIp1OG35qKM9isaC1tRUOhwNWqxWyLEOWZZhMpj2vqxYxA8wAwBwAzAEzYFwGKqIQiKJY+MWYzWaEw2EIggBFUSBJEhwOB8xmM0RR3DcAmqYhGo1iZWUF29vbiMViAHaaZigUQjKZRCaTgcPhgCAI2N7eRiqVgqIoFXG9CNj5f2Cz2eBwOADsvCZFURCJRJBKpRCJRAqvqxYxA8wAwBwAzAEzYFwGhFyRr17P6yr5JRZms/nAfyeZTO46TfS/ZFmGJEmFNpnJZGAymWCxWB64dCOZTFZMACwWC0wm076P55eYZDKZon5eta09ZgaqIwMAc6C3asgBM6AvozJQEYWADl+1HQTo8FVjIaDDx2MBFZsBbl1MRERELARERETEQkBERERgISAiIiKwEBARERFYCIiIiAgsBERERAQWAiIiIgILAREREYGFgIiIiMBCQERERGAhICIiIrAQEBEREVgIiIiICCwEREREBBaCqiEIAu9BXueYAQKYA9IPC0EVEAQBFosFNpsNJpPJ6OGQAZgBApgD0pfZ6AHQ12tpacGLL76I9vZ2/POf/8Tt27eRy+WMHhaVETNAAHNA+mIhqHCSJOH555/HCy+8gKNHj8LhcGBmZgbxeByaphk9PCoDZoAA5oD0x0JQwUwmE3w+Hy5cuIBHH30UXq8X6XQap0+fxsTEBGKxmNFDJJ0xAwQwB1QenENQwSRJwqlTp3D+/Hn4fD64XC709/djeHgYLpfL6OFRGTADBDAHVB4sBBXMYrHgzJkzaGtrg81mgyAIsNvtuHDhAg8CdYIZIIA5oPJgIahgyWQS//nPfzA2NoalpSVsbW0hEongH//4B0KhkNHDozJgBghgDqg8OIeggqmqitnZWfzpT3/CqVOn0NXVBafTiZGREWxvbxs9PCoDZoCA3Tn49re/jZ6eHrS0tDAHdcpqtcJqtcJisWBtbe3Qfi4LQQXTNA3hcBgffPABpqam0NXVhSNHjmBubg7pdNro4VEZMAMEfJmDf//730in0xAEAaIoYmVlhTmoM4IgoK2tDZ2dnXC73bhy5QrS6fShLD9lIahwmqZhc3MTW1tbuHPnDkRR5AGgzjADBOzkIJFIwGazoaWlBR6PBy6XC8lkknmoI1arFWfPnsX3vvc9HDt2DHfu3MHq6uqhZIBzCKqEpmlQVZV/+HWMGahvgiBgeHgYly5dwg9+8AMMDQ3hV7/6FTo6OriVcZ3IZ+Dll1/Gyy+/jHPnzh1qBniGgIgMJwgCnE4nhoaGCrPoAUBRFIyPj2N9fb2ui5AoinA6nbh06RIee+wxOBwOCIKAF198EZ988glisRjC4bDRw3wozMCDlSMDLAREZChZluHxeDA8PIznnnsODocDoigil8shmUyivb0d4+Pj8Pv9dT2BTpIktLe3w+l0Fu5j0NbWBqfTCVmWDR7dw2EGiqN3BlgIiMhQNpsN3d3d+OlPf4pnnnkGDQ0NhU+HqVQK3d3duHr1KjRNw927d+v2U6Kmadje3sb29jZcLhdkWUYsFoOiKMhms0YP76EwA8XROwMsBERkKFmW0dTUhMHBQVit1l3XQq1WK55++mn09vbiu9/9Ln7zm99gZmYGqqoaOOLy0zQNGxsb+Mtf/oLp6Wl85zvfwZkzZ/Dmm29idHQUwWDQ6CE+FGbg65UjA0KuyLUKnLRSXfS4AxozUF30ugveYefAbrejs7MTv/jFL/D666/D4/FAFHfPd1YUBXNzc3j11VcxPj6OTCZzqGOoFhaLBXa7He3t7RgaGsL169exvLyMVCp14PdUw7GAGSienhngGQIiMkxDQwNOnDiBp556CufPn4fdbt/3zcZkMkGSpD1vEvVGURSk02koioJ4PI5gMFj1p8+ZgdLomYGKKASCIOwJQC6X432+6wgzUJ9cLhceffRRvPDCCxgaGoLFYtmTA03TsLW1hUAg8MBPQfUil8shkUhgYWHB6KEcCmagdHploCIKgdVqhdlsLsyaBIBMJoN0Ol23p4UeRBTFff9gqvnNkxkoTa1koKWlBU899RQuXbp04HPS6TQmJibw9ttvY3V1teon0B2mWsgBM1A5DC0EoiiiqakJr732Gh5//HG0tbUVHhsdHcVHH32Ea9euIRKJGDfICiNJEp5//nk0NzdDkqTC1z/55BMsLS0hGo0aOLrSMQOlq6UMZDIZKIoCRVFgsVj2fY6qqlhcXMT169erfq39YaqVHDADlcPQQuBwOPD666/jRz/6Edrb22G32wuPdXd3o6+vD729vfj73/+O5eXlum6FZrMZ7e3tuHjxIn7yk5/A6/XuOgh8/PHHuHr1KkZGRnDv3j0DR1oaZqB4tZiBpaUlTE5OYnZ2FqdOndr3OVarFadPn8b3v/99hEIh5qDGclBsBk6cOIFz587hs88+K/MI64fhZwgcDgfcbjdcLhdsNlvhMZvNVphNGQwG8dZbbyGRSBg4WmOJogiPx4Mnn3wS/f39cLlcu06vm81mWCwWeDyeA1t2JWIGileLGYjH4wgEAvj888/R2dlZ2H3tq8xmMzo7O3HhwgXMzs4yBzWWg2Iz0NjYuOsMIh0+QwtBNpvF8vIyNjc30dTUtOvNQJIk+Hw+uFwuhMNhXL58ua4PAoIgQJZleL1eyLK85w/m+PHjcLlc8Pl80DTNoFGWjhkoXi1mIJvNYnFxER9//DGeeOIJHD9+HGbz3sPSkSNHcObMGaytrTEHNZaDfAauX7+OkydP4tSpU3t23aumORHVzLD1GyaTCbIs72q2/0sQhELbrXeqqmJ5eRlvvvkmAoHAvstMmpqa0N3djaNHjxowwtIxA6WpxQwAgN/vx1tvvYXJyckDN5thDr5UizmYnJzEn//8Z/zxj3/E5ubmniKjKArm5+cxMjJi0Ajrg2FnCAYHB/HjH/8YP/zhD9HR0bHvPsyqqiIej2N9fb0qmq5eZFnGwMAAhoeH8eqrr+LYsWNVv3c5wAyUolYzAOzk4JVXXsGZM2cOfE3MwY5azkE6ncatW7ewvLwMm82GxsZGCIIATdMQDAYxOzuLyclJo4dZ0wwrBM3Nzejr60NHR8eBrX9jYwO3b9/Gu+++i2QyWeYRVg6r1YqhoSFcvHgRPT09+54mBICFhQXcvHkTt27dMmCUpWMGilerGQB2ctDb24vm5uYDN51hDnbUcg7yKwn++te/4umnny68vkQigStXruDDDz/E2tqa0cOsOPmzZ1/NQS6XQzabLbk8G1YIRFGELMsPPAWYSCQQCARw48aNqt+N62GYzWYcPXoUJ0+ehNPpPPB5S0tLGBsbw+3bt8s4um+OGSherWYAYA5KUcs5yO/V//777yMWixXmEsRiMXzwwQeYmpqqmqWU5WIymdDU1ASfzwer1Vr4eiqVwtraGjY2NqAoStE/z7BCsLm5Cb/fj+eee+7ATwUWiwVut7vu99DXNA3JZBLxePyBz9vY2MDc3Bymp6fLNLKHwwwUr1YzADAHpSg1B36/v0wjOxyqqmJ6ehpzc3OFuUW5XA6ZTKaul5rux2QyoaGhAefPn8crr7wCn89XeGx1dRXvvfcerl27hkAgUPTPNKwQzM/P4+rVq/j5z3++61aXX+X1etHf34+uri5EIpG6vXYYjUYxMjKCzs5O9Pf3H/i8gYEBhEIhXLlypYyj++aYgeLVagYA5qAUxeZgaGgIAKpyQ69cLlfXZ4GK5Xa7MTw8jDfeeAOtra275pKk02k8/vjjGB0dxd/+9reif6Zhqwyi0Sg+//xz/OEPf0AgENh3e9psNot0Ol13t7n8X5qmIRAIYHx8HLdu3TpwCY7VaoXdbt932VYlYgaKV6sZAJiDUhSbA6/Xi7Nnz+L3v/99mUdI5ZLJZLCxsYHZ2VkoilK4H0x+WerRo0fxzDPPlJQBwwqBoigIBoN4//33MTMzg+3t7T3PicViCAQC2NjYqNtPBHlra2u4desW3nvvPUQikX1Pn21tbWF9fb2ka0ZGYgZKU4sZAJiDUgWDQdy9excjIyMHFgJZltHc3PzAswhUvfJv/NlsFrFYbM+xQBAEWK1WHDlypKQMGPYxIpvNIhqNYnR0FGNjY2hpadm141Y2m0UwGMTExARvZgEgHA4jHo9jY2MDL730UmEHv7z8Jwe/3/+11xcrBTNQmlrMAMAclCoajWJpaQlTU1PQNO3AeReCINT9rYJrlSzLcDqdaGlpgc1mO7QMGH5eMZPJ4L///S+OHDkCm82GY8eOAdiZJXvz5k1cv36dd7v7f6lUCsvLy5icnCzsUpY/Nby1tQW/34/Jycmq28WNGSherWYAYA6K1dPTg2effRa//OUvD9zUS9M0pFIpbGxsoLOzs8wjJL3lM/DrX/8aJ0+e3HfezTfJgOGFAAAuX74Mv9+P3t7ewkFgfn4eU1NTmJqaMnZwFSSXyyGZTOJ3v/sdfvazn6G/vx8tLS0AgLfffhvXrl2r2o07mIHi1HIGAOagGE6nEx0dHThx4sSBqy4ikQhu3ryJN954A5cvXy7zCElvemWgIgpBMpnE7OwswuFwYd1sLBZDNBqt601I9pPf9/vdd9/FRx99VDhl/MUXX2BlZWXf66/VgBkoXq1mAGAOihGPxxGJRJBKpeBwOPZ9Tv40cTWeKaKvp1cGKqIQADvXxbjpRHESiQRu3rwJURQL7VBVVWiaVtU3AWEGilerGQCYg68TiUSwurqKSCRy4DJNWZZ5d8AaplcGOOOkSqmqinQ6DUVRoCgKstls1b8RUGmYgfq0traG6elpzM/PH7gM0263o6OjA4ODg2UeHZWDXhlgISAiqiLpdBp3797Fb3/7W9y4cWPfS0SapkFRlKq+fEQH0ysDLAREVBb5JVBcCvdwNE3D9vY27ty5g3feeQehUGjPc7a3tzE3N4cPP/zQgBEejBk4HHploGLmEBBRbWttbYXVaoWqqgiFQkin07zE8Q2pqoqtrS2Mj49jZmYGbrcbTU1NAHY+Peb3KVhYWDB4pLsxA4dHjwywEBCR7kwmEx555BF4vV4oioK7d+8iHA4jlUrV/UZD31Qul8Pi4iJu3LgBWZbx2GOPAfhyudmnn35aUfcyYAYO32FnQMgVWc/q/S5j1UaP1s0MVBe9PnmVmgNBEOB0OvHaa6/B5/NBkiREo1H861//wtTUFCKRCDccegjNzc3o6enBwMAAgJ03g7GxMczNzSGTyVTEsYAZ0NdhZYCFoEZVwkGAjFUJhSB/i9aBgQFcvHgRzc3NkCSpcA30nXfewdTUFNbX13UZaz0QRRFmsxmSJAHYub6cyWQKs8+NPhYwA/o7rAzwkgER6UaSJLjdbgwODsLlcsFsNkMQBJhMJjidTvT29iKdTiOdTnNG/DekaVrh/2ElYgb0d1gZ4FRPItKFJEnweDzo6urC8ePH99yERRRFHDt2DN3d3XA6nQaOlPTCDFQXFgIi0oXH40FfXx+Gh4dht9v3Pc3s8/lw9OhRNDc3GzBC0hszUF3KUgjy92b2+XywWq28Fl2HmIH609XVhZMnT6Krq6twR8b9eDwefOtb32ImahAzUF3KMofA4XCgu7sb58+fx+TkJILBIILBIMLhcDn+eaoAzED9SaVSRa0zz2QyvAlPjWIGqovuhSC/3KS9vR2PPPIIbDYbQqEQFhcXce/ePdy/f1/vIZDBmIH6pKoqVFUt3GPhoE9/+ZuwCILATWpqTLEZyGazXHZYAXQvBLIsw+v1or29HW63G263G4lEAq2trbBYLFhbW+OmFDWOGahPmUwGsVgMkUgELpcLNptt3zeEhoYGtLW1QRRFaJpmwEhJL8VkQFVVpFIp3t66AuheCPr6+vDkk0+it7e38DW73Y7Ozk5ks1l89tln2Nra4oGghjED9WlhYQGbm5v44osvcPbsWQwNDcFisex5niiKkCQJjY2NzEGNKSYD6+vruHfvHsbGxniGyGC6TSoURRFutxvd3d3o6OiAy+Xa9bjdbkd3dzdeeuklOJ1O3uyiBjED9U1VVUQiEQQCAWxubh74Rs8c1K58BpaWljA7O4twOAxFUQqPJ5NJLC8vY21tbdfXyRi6/OWJogi73Y7BwUGcPn0aTU1Ne/7IRVEszDp3Op2FHZaoNjADlCeKIhwOx4Fv9MxBbdM0DalUCn6/H59++inu3buHzc1N5HI5hMNhjI+PY35+nmcHKoAulwzyS8xOnDhRWGa233PyW1ra7XaYzWY2xBrCDBCw8zuWJAkNDQ0HTihjDmpfNpvF6uoq7ty5A0VRkEwm4fP5sLCwgJmZGa42qhC6FYL8gcBkMj1wbWn+/thcf1pbmAECvsxBMZcBmIPapmkaFhcXEYlE4Pf70dnZic3NTc4bqSC6FYL8p4IHHQgEQYAsy7DZbA/ctIKqDzNAAHNAe0WjUUSjUSwsLADQ7yZcVDpd/vKy2SxisRj8fj+sViuam5tht9thMpn2PO/+/fsIBoNcclJjmAECmAM6GItA5dFlUmF+EsnU1BRGR0cRCAT2vSaYyWQwPz+PaDTKTSlqDDNAAHNAVE10OzeXTqcxPT2NlZUVyLIMp9MJi8VS+GSQzWYRj8cxPT2NRCLBa0g1iBkggDkgqha6XqzTNA3xeBwTExPIZrM4d+4cvF4vRFHE/fv3MTY2hqWlJe5SV8OYAQKYA6JqoPvsHVVVsbCwgPX1dSwtLeHixYvweDxYXl7GxMTEAzcsodrADBDAHBBVOiFX5MyOh10KZDab0dDQgN7eXtjtdmxsbGBhYQFbW1sP9XNpf3pM2GEGqotek7aYg+pSiccCKq9iM1C2QpCXn2GsqirS6TRPEeqkkg8CzEB5VGohyGMOyqOSjwVUHhVbCKg8eBCgSi8EVB48FlCxGeBdRIiIiIiFgIiIiFgIiIiICCwEREREBBYCIiIiAgsBERERgYWAiIiIwEJAREREYCEgIiIisBAQERERWAiIiIgILAREREQEFgIiIiICCwERERGBhYCIiIjAQkBERERgISAiIiKwEBAREREAIZfL5YweBBERERmLZwiIiIiIhYCIiIhYCIiIiAgsBERERAQWAiIiIgILAREREYGFgIiIiMBCQERERGAhICIiIgD/B05Cjv5SkQuNAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZ6klEQVR4nO3dyW8b5/0G8GeGnOFwEReJ2qmtjizZkeUoVuLasIwEaWLAQBsjaZsCQds0QNAW7amHnppD/4Eil16Kor0UDdA0CYIYjVPHBxuJYyvRYitytFi7ZEnUQoqiSA6X4e/gH1krkhzaFjlcng+QiziS3lEez3zn3UZIJpNJEBERUUkT9W4AERER6Y8FAREREbEgICIiIhYEREREBBYEREREBBYEREREBBYEREREBBYEREREBMCY6YGCIGSzHbTPsrHfFDNQWLK15xhzUFh4LaBMM5BxQZArgiDAZDIBABKJBOLx+LaTEQQBsixDFHfv3NA0Lf19oihCluU9w5tMJqGqatYunA/iQc6r2DEDzADAHDAHgMFggNPphCiKUFUVW1tbSCQS6c+NRiNsNls6J98UjUahqipCoRBkWUZZWRmMxp23vWQyiUQiAb/fv+3n6+VBzmtff+++/rSHJAgCJElK/1dXVwdBELCxsYGVlRWoqpo+VpIkVFRUwGKxbPsZRqMRsiwjFAqlv89qtaKurg4mkwmJRAKqqkIURUiSBEEQEI/HMTk5iVgspvuF4EHOqxgxA8wAwBwAzEHqZmi1WmG1WtHT0wOj0YiJiQn09fXB5/Olj7Varejs7ERNTU36a6Iowmq1wmazYXFxEbdv38bAwABqa2tx+vRpuFwuRKNR+Hw+iKKIsrIySJKEYDCIDz74AJubm7oXBZmeV39//77+3rwoCAwGA2w2G2w2G8xmMzo6OmAwGDAzM4PNzc0dFwGXywWXy7Xt+x0OB5xOJ1ZWVjA7O4uVlRXY7Xa0t7ejrKwMqqpicXERkiShvLwciqIgFArhzp07O5489HDveRkMBoiiCIPBgLKyMpSVlW07r2LEDDADAHMAMAcmkwkejwcejwdVVVX4zW9+A0mS8PHHH2NqampHQXD48GG0t7ene38kSUJraytaW1sxMDCACxcuYHBwEC0tLXjttdfQ1NSE9fV1XLt2DTabDW1tbXC73fB6vbh8+TJCoVBeFASp85JlGUajESaTCc3NzWhqakqfV1EWBKl/mFVVVTCbzbDb7QDuBuObXXxbW1sYGRnZ9nVZltHZ2Ynq6mooigJJkgDcvThYLBZIkoS1tTX09vbCbrfjiSeeSP8OQRDyYjzs3vNyu92w2WxwOBwwm82wWCzbzqsYMQPMAMAcAMyB1WrFkSNH8OSTT6KyshItLS1IJpNwOp07uvsXFxfx97//HQaDIf01h8OB3/72tzh+/Hj67wcAiqKgqqoKNpsNw8PDePPNN9HS0oLf/e53aG1tBXC3dyIfMnDveXV2dqKlpQXNzc2orq5GdXX1tvPaT3lREEQiEczOzmJ5eRk2mw11dXXpMTSj0QiDwQBN09KV+zerN1EUoWmaHk2nfcIMEMAcELC2tob//Oc/uHr1KjweD06fPp0e+1cUBYqiIBqNpnOgquqOojAfhn4KUV4UBJIkwe12w+l0wmw2Q9M0xONxWCwWNDU1we12Y2Fh4b5dOfF4HKqqIhaLpSfbaJqGSCQCSZKgKApaW1uhKAosFgvi8Tii0WjehMZkMqG6uho2mw2KokCWZciyDIPBsOO8ihEzwAwAzAHAHNhsNhw9ehStra2oqqpCLBZDOBxGTU0Nzp49i+7ubnzyySfwer0Ih8MAts+i1zQNoVAIPp8Pm5ubiEQiAABVVeH3+1FWVgaXy4Wf/OQnKC8vR01NDYLBIDY2NvKmmHS5XHj66afh8XjgdrtRVlYGu90OWZaxsbGx7bz2U14UBKIopv9xGo1GrK6uQhAEqKoKSZJgs9lgNBohiuKuFwFN0xAIBLC4uIjNzU0Eg0EAd582VlZWEA6HEYvFYLPZIAhC+o+pqmpejBkCd/8GZrM53Q2kaVo6wJFIBH6/P31exYgZYAYA5gBgDlLDRrW1tbDZbBgcHITBYEAgEIDD4YAsyzCbzduGCe4Vj8cxMzODzz77DDMzM5ifn0cymYTP50N/fz+WlpYQCoVQX18Pk8mE+fl5rK2tYX19HeFwOC+KAkmSUFlZiYaGBgiCkF4BMT4+Dp/Ph9HRUczPz+/77xWSGf4LyOa4SmqZzf3Gb1L/o/ZqrizLkCQp/UQRi8VgMBhgMpnuu3wnHA7nzUXAZDLtGfLUMqNYLJbRzyu0tcfMQGFkAGAOsq0QcpDNDBiNRtjtdlgslj1/z+rqKlRV3fXmLQhC+ok6Go0iHA5jc3MTiqLA6XTuOv8itexwZWUlL3pfJEmC0+mEoii7fq6qavq8MpFpBvKiIKD9V2gXAdp/hVgQ0P7jtYAyzQC3LiYiIiIWBERERMSCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCgIiIiMCCoGAIgsB3kJc4ZoCIsokFQQEQBAEmkwlmsxkGg0Hv5pAOmAEiyjaj3g2gb1dVVYUXX3wRtbW1+OCDD3Djxg0kk0m9m0U5xAwQUbaxIMhzkiThhRdewNmzZ9HQ0ACbzYaJiQlsbW1B0zS9m0c5wAwQUS6wIMhjBoMB9fX1OH36NB5//HG43W5Eo1EcPnwYw8PDCAaDejeRsowZIKJc4RyCPCZJEg4dOoRTp06hvr4eDocDHR0d6OnpgcPh0Lt5lAPMABHlCguCPGYymXDs2DHU1NTAbDZDEARYLBacPn2aN4MSwQwQUa6wIMhj4XAYn3zyCQYHBzE3N4eNjQ34/X78+9//xsrKit7NoxxgBogoVziHII/F43FMTk7iL3/5Cw4dOoTGxkbY7Xb09vZic3NT7+ZRDtybgSeffBKtra2oqqpiBkqYoihQFAUmkwnLy8t6N4eKCAuCPKZpGlZXV3HhwgWMjo6isbERlZWVmJqaQjQa1bt5lAOpDPz3v/9FNBqFIAgQRRGLi4vMQAkSBAE1NTXweDxwOp24ePEiotEol6DSvmBBkOc0TYPP58PGxgZu3rwJURR5IygxmqYhFArBbDajqqoKLpcLDocD4XCYWSgxiqLgxIkT+N73vofm5mbcvHkTS0tLzAHtC84hKBCapiEej/MffgkSBAE9PT04d+4cfvCDH6C7uxu/+tWvUFdXx62MS0gqBy+//DJefvllnDx5kjmgfcUeAtKVIAiw2+3o7u5Oz6IHAFVVMTQ0hLW1tZIugkRRhN1ux7lz53DkyBHYbDYIgoAXX3wR165dQzAYxOrqqt7NfGTMwf2VSg5IXywISDeyLMPlcqGnpwfPP/88bDYbRFFEMplEOBxGbW0thoaGMD4+XtIT6CRJQm1tLex2e/o9BjU1NbDb7ZBlWefWPTrmIDPFngPSHwsC0o3ZbEZTUxN+9rOf4dlnn4XVak0/GUYiETQ1NeHSpUvQNA23bt0q2SdETdOwubmJzc1NOBwOyLKMYDAIVVWRSCT0bt4jYw4yU+w5IP2xICDdyLKM8vJydHV1QVGUbeOgiqLgmWeeQVtbG7773e/i97//PSYmJhCPx3Vsce5pmob19XX87W9/w9jYGJ5++mkcO3YMb7/9Nvr6+uD1evVu4iNjDr5dKeSA9CckM1yvwkkrhSUby5D2OwMWiwUejwe/+MUv8MYbb8DlckEUt89zVVUVU1NTePXVVzE0NIRYLLavbSgUJpMJFosFtbW16O7uxtWrVzE/P49IJLLn92RrKRpzoJ98yQHvB4Ul0wywh4B0YbVaceDAARw/fhynTp2CxWLZ9SJjMBggSdKOG0SpUVUV0WgUqqpia2sLXq+3KLrOmYMHU6w5oPyQFwWBIAg7LgLJZJKbbRQxh8OBxx9/HGfPnkV3dzdMJtOODGiaho2NDSwsLNz3CahUJJNJhEIhzMzM6N2UfcMcPLhizAHlh7woCBRFgdFoTM+cBYBYLIZoNFqyXYP3I4rirhfNQiqgqqqqcPz4cZw7d27PY6LRKIaHh/Huu+9iaWmJE6fuUQwZAJgDonyia0EgiiLKy8vx+uuv4+jRo6ipqUl/1tfXh08//RRXrlyB3+/Xr5F5RpIkvPDCC6ioqIAkSemvX7t2DXNzcwgEAjq2LnOxWAyqqkJVVZhMpl2PicfjmJ2dxdWrV7nG+h7FkgGAOSDKJ7oWBDabDW+88QZ+9KMfoba2FhaLJf1ZU1MT2tvb0dbWhn/961+Yn58v6ScDo9GI2tpanDlzBj/96U/hdru33Qw+//xzXLp0Cb29vbh9+7aOLc3M3NwcRkZGMDk5iUOHDu16jKIoOHz4ML7//e9jZWWFGSiyDACZ5+DAgQM4efIkvvzyyxy3kKh06N5DYLPZ4HQ64XA4YDab05+Zzeb0jFqv14t33nkHoVBIx9bqSxRFuFwuPPXUU+jo6IDD4dg2xGI0GmEymeByufZ80sonW1tbWFhYwFdffQWPx5Peee1eRqMRHo8Hp0+fxuTkJDNQZBkAMs9BWVnZth5EItp/uhYEiUQC8/Pz8Pl8KC8v31YQSJKE+vp6OBwOrK6u4vz58yV9MxAEAbIsw+12Q5blHRfNlpYWOBwO1NfXQ9M0nVqZuUQigdnZWXz++ed44okn0NLSAqNxZxwrKytx7NgxLC8vMwNFlgHgfzm4evUqDh48iEOHDu3Yda/Q5kUQFSrd1vAYDAbIsrztCeebBEFIP/WUung8jvn5ebz99ttYWFjYdalReXk5mpqa0NDQoEMLH9z4+DjeeecdjIyM7LnRDDPwP8WYAQAYGRnBX//6V/z5z3+Gz+fbUcyoqorp6Wn09vbq1EKi0qBbD0FXVxd+/OMf44c//CHq6up23Ys7Ho9ja2sLa2trBfPEkw2yLKOzsxM9PT149dVX0dzcXBR7l3d1deGVV17BsWPH9jwfZuCuYs1ASjQaxcDAAObn52E2m1FWVgZBEKBpGrxeLyYnJzEyMqJ3M4mKmm4FQUVFBdrb21FXV7fn09/6+jpu3LiB999/H+FwOMctzB+KoqC7uxtnzpxBa2vrrt3FADAzM4P+/n4MDAzo0MoHV1FRgba2NlRUVOy54QwzcFexZiAltZLgH//4B5555pn0OYZCIVy8eBGXL1/G8vKy3s3MO6ketHuzkEwmkUgkSrqApoejW0EgiiJkWb5vV3AoFMLCwgKuX79e0rtxGY1GNDQ04ODBg7Db7XseNzc3h8HBQdy4cSOHrXt4zEDmijUDKam9+j/66CMEg8H0XIJgMIgLFy5gdHS0oJZT5oLBYEB5eTnq6+uhKEr665FIBMvLy1hfX4eqqjq2kAqNbgWBz+fD+Pg4nn/++T2fDk0mE5xOZ8nvm61pGsLhMLa2tu573Pr6OqampjA2Npajlj0aZiBzD5qB8fHxHLVs/8TjcYyNjWFqaio9tyiZTCIWi5X0ctPdGAwGWK1WnDp1Cq+88grq6+vTny0tLeHDDz/ElStXsLCwoGMrqdDoVhBMT0/j0qVLeO2117a97vRebrcbHR0daGxshN/vL9kusEAggN7eXng8HnR0dOx5XGdnJ1ZWVnDx4sUctu7hMQOZyzQD3d3dAFCwm3klk8mS7gnKlNPpRE9PD9566y1UV1dvm08SjUZx9OhR9PX14Z///KeOraRCo9sqg0AggK+++gp/+tOfsLCwsOsWxYlEAtFotORedfpNmqZhYWEBQ0NDGBgY2HMZlqIosFgsuy7fy0fMQOYyzYDb7caJEyfwxz/+McctpFyKxWJYX1/H5OQkVFVNvw8mtTS1oaEBzz77LHNAD0S3gkBVVXi9Xnz00UeYmJjA5ubmjmOCwSAWFhawvr5esk+GKcvLyxgYGMCHH34Iv9+/axfqxsYG1tbWCmbckBl4MF6vF7du3UJvb++eBYEsy6ioqLhvLwIVttSNP5FIIBgM7rgWCIIARVFQWVnJHNAD0e1RMpFIIBAIoK+vD4ODg6iqqtq281oikYDX68Xw8DBfaAJgdXUVW1tbWF9fx0svvZTexTEl9QQ5Pj7+rePM+YIZeDCBQABzc3MYHR2Fpml7zrsQBKHkXxNczGRZht1uR1VVFcxmM3NA+0b3vuVYLIbPPvsMlZWVMJvNaG5uBnB3tnR/fz+uXr3KNx7+v0gkgvn5eYyMjKR3q0sND2xsbGB8fBwjIyMFt5sfM5CZ1tZWPPfcc/jlL3+554ZemqYhEolgfX0dHo8nxy2kXEjl4Ne//jUOHjy469wb5oAehu4FAQCcP38e4+PjaGtrS98MpqenMTo6itHRUX0bl0eSySTC4TDefPNN/PznP0dHRweqqqoAAO+++y6uXLlSsJu3MAPfzm63o66uDgcOHNhz1YXf70d/fz/eeustnD9/PsctpFxgDihb8qIgCIfDmJycxOrqanr9dDAYRCAQKOnNaHaT2vv9/fffx6effpoeNvj666+xuLi46zh8IWAGvt3W1hb8fj8ikQhsNtuux6S6iAutl4gyxxxQtuRFQQDcHR/lxiOZCYVC6O/vhyiK6SeEeDwOTdMK+kUwzMD9+f1+LC0twe/377lMU5ZlvhmwyDEHlC2ccVKg4vE4otEoVFWFqqpIJBIFXQzQt1teXsbY2Bimp6f3XIZpsVhQV1eHrq6uHLeOcoU5oGxhQUBUIKLRKG7duoU//OEPuH79+q7DQ5qmQVXVgh06om/HHFC2sCCgrEstf+ISqEejaRo2Nzdx8+ZNvPfee1hZWdlxzObmJqampnD58mUdWnh/zMH+KPQcUP7KmzkEVLyqq6uhKAri8ThWVlYQjUY5vPGQ4vE4NjY2MDQ0hImJCTidTpSXlwO4++SY2qdgZmZG55buxBzsn0LOAeUvFgSUVQaDAY899hjcbjdUVcWtW7ewurqKSCRS8hsNPaxkMonZ2Vlcv34dsizjyJEjAP631OyLL77Iu3cZMAf7rxBzQPlNSGZYopf62+YKTTaevB40A4IgwG634/XXX0d9fT0kSUIgEMDHH3+M0dFR+P1+bjj0CCoqKtDa2orOzk4Ad28Eg4ODmJqaQiwWy9rTN3OQX/TIAe8HhSXTDLAgKFJ6XwRSr2ft7OzEmTNnUFFRAUmS0uOf7733HkZHR7G2trbv7SwVoijCaDRCkiQAd8eWY7FYeuZ5PhQEzEH26ZED3g8KS6YZ4JABZYUkSXA6nejq6oLD4YDRaIQgCDAYDLDb7Whra0M0GkU0GuVM6IekaVr6b5ivmIPsK4QcUGHgdF/ad5IkweVyobGxES0tLTtewCKKIpqbm9HU1AS73a5jSymbmAOiwsKCgPady+VCe3s7enp6YLFYdu1erK+vR0NDAyoqKnRoIeUCc0BUWHJSEKTez11fXw9FUTj+VOQaGxtx8OBBNDY2pt/GuBuXy4XvfOc7zEORYg6ICktO5hDYbDY0NTXh1KlTGBkZgdfrhdfrxerqai5+PeVYJBLJaI15LBbjy1eKGHNAVFiyXhCklhzV1tbiscceg9lsxsrKCmZnZ3H79m3cuXMn202gHIvH44jH4+n3K+z15Jd6AYsgCNygpghlmoNEIsFlh0R5IOsFgSzLcLvdqK2thdPphNPpRCgUQnV1NUwmE5aXl7kxSZGJxWIIBoPw+/1wOBwwm8273gysVitqamogiiI0TdOhpZRNmeQgHo8jEonwFddEeSDrBUF7ezueeuoptLW1pb9msVjg8XiQSCTw5ZdfYmNjgzeEIjIzMwOfz4evv/4aJ06cQHd3N0wm047jRFGEJEkoKytjBopQJjlYW1vD7du3MTg4yF4iIp1lbVKhKIpwOp1oampCXV0dHA7Hts8tFguamprw0ksvwW6384UnRSQej8Pv92NhYQE+n2/PGz0zUNxSOZibm8Pk5CRWV1ehqmr683A4jPn5eSwvL2/7OhHpIytXYFEUYbFY0NXVhcOHD6O8vHzHxV4UxfTKA7vdnt5li4qHKIqw2Wx73uiZgeKnaRoikQjGx8fxxRdf4Pbt2/D5fEgmk1hdXcXQ0BCmp6fZO0CUB7IyZJBaZnjgwIH0UsPdjklta2qxWGA0GvmUUEQEQYAkSbBarXtOJmMGSkMikcDS0hJu3rwJVVURDodRX1+PmZkZTExMcLURUZ7IWkGQuiEYDIb7ri9OvSOda5CLSyoDmQwDMAPFT9M0zM7Owu/3Y3x8HB6PBz6fj3NHiPJI1gqC1NPh/W4IgiBAlmWYzeb7blxChYcZoN0EAgEEAgHMzMwAyN4LmIjowWXlCpxIJBAMBjE+Pg5FUVBRUQGLxQKDwbDjuDt37sDr9XLZUZFhBuh+WAgQ5Z+sTCpMTSQaHR1FX18fFhYWdh0bjsVimJ6eRiAQ4MYkRYYZICIqLFnro41GoxgbG8Pi4iJkWYbdbofJZEo/ISYSCWxtbWFsbAyhUIjjiEWIGSAiKhxZHbTVNA1bW1sYHh5GIpHAyZMn4Xa7IYoi7ty5g8HBQczNzXGnwiLGDBARFYasz+KKx+OYmZnB2toa5ubmcObMGbhcLszPz2N4ePi+G9dQcWAGiIjyn5DMcHbPoy4JMxqNsFqtaGtrg8Viwfr6OmZmZrCxsfFIP5d2l41JW8xAYcnWxD0uDy0s+XgtoNzKNAM5KwhSUjPN4/E4otEou4qzJJ8vAsxAbrAgICC/rwWUG3lbEFBu8CJALAgI4LWAMs8A3yZDRERELAiIiIiIBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQEREREBEJLJZFLvRhAREZG+2ENARERELAiIiIiIBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQERERGBBQEREREB+D8dU5WpRViLgAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdwUlEQVR4nO3daWwcd/0/8PfM7Mzel+9jHdskG9et41ymoVGcNrRpUAu0oqBWqkClogIESMADHtHHqE9Qn/AMgZCACqq2VA1NShOJFuqmbhO7cQ4f8e2Nzz28u97d2WP2/yB/L3Ft5+fEXs+u9/2S8mQ9GX8m+ex3PjPfS8hms1kQERFRSRP1DoCIiIj0x4KAiIiIWBAQERERCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICYNjogYIg5DMO2mL5WG+KOVBc8rXmGPOguLAtoI3mwIYLgu0iCAKMRiMAIJPJIJ1Or7gYQRCgKApEce2XG5qm5f6eKIpQFGXd5M1ms1BVNW8N5924m+va6ZgDzAEAkCQJLpcLoihCVVUsLS0hk8nkfm4wGGCz2XK58kXJZBKqqiIWi0FRFNjtdhgMq5u8bDaLTCaDUCi04vx6uZvr2unYFmxvW1AQBYEgCJBlOfenrq4OgiBgcXER8/PzUFU1d6wsyygvL4fFYllxDoPBAEVREIvFcn/ParWirq4ORqMRmUwGqqpCFEXIsgxBEJBOpzEyMoJUKqV7EtzNde1EzAHmAPC/m6HVaoXVakVnZycMBgOGh4dx8eJFBIPB3LFWqxXt7e2oqanJfSaKIqxWK2w2G6anp3Hjxg309PSgtrYWx48fh9vtRjKZRDAYhCiKsNvtkGUZ0WgUb7/9NiKRiO5FwUav69KlSzpGmT9sC/RrCwqiIJAkCTabDTabDWazGW1tbZAkCePj44hEIqsSwO12w+12r/j7TqcTLpcL8/PzmJiYwPz8PBwOB+677z7Y7Xaoqorp6WnIsoyysjKYTCbEYjHcvHlzVdWph9uvS5IkiKIISZJgt9tht9tXXNdOxBxgDgCA0WiEx+OBx+NBVVUVfvKTn0CWZbz33nsYHR1dVRDcf//9uO+++3JPfbIsw+v1wuv1oqenB2fPnkVvby+am5vxwgsvoLGxEYFAABcuXIDNZkNLSwsqKiowNzeHDz74ALFYrCAKguXrUhQFBoMBRqMRTU1NaGxszF3XTi0I2Bbo1xYUREGw/J9SVVUFs9kMh8MB4Fbj8MXXO0tLS+jv71/xuaIoaG9vR3V1NUwmE2RZBnArMSwWC2RZht/vR3d3NxwOBw4cOJD7HYIgFER/2O3XVVFRAZvNBqfTCbPZDIvFsuK6diLmAHMAuHUz3LdvHw4dOoTKyko0Nzcjm83C5XKtet0/PT2NP/7xj5AkKfeZ0+nET3/6Uxw5ciT3bwgAJpMJVVVVsNlsuHr1Kl5++WU0Nzfjl7/8JbxeL4BbT1+FkAe3X1d7ezuam5vR1NSE6upqVFdXr7iunYhtgX5tQUEUBIlEAhMTE5idnYXNZkNdXV2u/8RgMECSJGialqvavljBi6IITdP0CJ22CHOAAMDv9+Pdd99FV1cXPB4Pjh8/nuv7N5lMMJlMSCaTuVxQVXXVzaAQXvnSvWNboJ+CKAhkWUZFRQVcLhfMZjM0TUM6nYbFYkFjYyMqKirg8/nu+DovnU5DVVWkUqncQAtN05BIJCDLMkwmE7xeL0wmEywWC9LpNJLJZME0HEajEdXV1bDZbDCZTFAUBYqiQJKkVde1EzEHmAMAYLPZsH//fni9XlRVVSGVSiEej6OmpgZPPPEEOjo6cO7cOczNzSEejwNYOYJa0zTEYjEEg0FEIhEkEgkAgKqqCIVCsNvtcLvdeO6551BWVoaamhpEo1EsLi4WzE3E7XbjwQcfhMfjQUVFBex2OxwOBxRFweLi4orr2onYFujXFhREQSCKYu4/xmAwYGFhAYIgQFVVyLIMm80Gg8EAURTXTABN0xAOhzE9PY1IJIJoNArgVqU5Pz+PeDyOVCoFm80GQRByXyhVVQuivwi49W9gNptzrwI1Tcs1YolEAqFQKHddOxFzgDkA/O91cW1tLWw2G3p7eyFJEsLhMJxOJxRFgdlsXtFNcLt0Oo3x8XF89NFHGB8fx9TUFLLZLILBIC5duoSZmRnEYjHU19fDaDRiamoKfr8fgUAA8Xi8IIoCWZZRWVmJhoYGCIKQmwExNDSEYDCIgYEBTE1N6R1m3rAt0K8tELIbvPp89qssT7G4Ux/e8pd1vXAVRYEsy7lqMpVKQZIkGI3GO07diMfjBZMARqNx3YZueYpJKpXa0PmKbe4xc6A4cgDIbx4YDAY4HA5YLJZ1f8/CwgJUVV3z5i0IQu6JOplMIh6PIxKJwGQyweVyrdnvujztcH5+viDewMiyDJfLBZPJtObPVVXNXddGsC1gW7DRayqIgoC2XrE1ArT1irEgoK3HtoA2mgNcupiIiIhYEBARERELAiIiIgILAiIiIgILAiIiIgILAiIiIkKBLExERESla3lr4ts3KQJuzbf3+/26bzhVKlgQEBGRriwWC2pqanDy5EkAtwqEdDqNaDSKs2fPYnFxsSAWjdrpWBAQEZFuPB4PvvrVr+K5557Do48+mltJMJPJIBqN4u9//zv+8pe/4Pr16wgGgwWxkuBOxYKAiIh0Y7VaUV9fj9bWViiKkvvcYDDAYDDg61//OsrLy3H27Fm8++67mJ2d1THanY2DComISDeqqiIYDOLmzZurnv4lSUJdXR0OHz6M1tZWWK1WnaIsDSwIiIhIF5IkIZPJYHZ2Fv39/Wt2B0iShMrKSpSVla25ORVtHXYZEBGRLmpra/HMM8/ga1/7Gk6cOLHuToS0PVgQEBHRthNFES0tLXjuueewf//+FeMHbheJRPDGG2/gvffew/z8/DZHWVpYjhERkW6MRuO6xQAApFIpXL9+HUNDQ4hGo9sYWelhQVAkBEHgHuREtGNks1kEg0GEQiHEYrF1j1tekyCZTCKZTG5jhKWHBUEREAQBRqMRZrMZkiTpHQ4R0aZls1lcv34dg4ODd+wKcDgcOHToEPbs2bON0ZUmjiEoAlVVVXjqqadQW1uLt99+G59//jkX5yCiopdIJPDvf/8bNpsNFosF1dXVq44RRRFWqxVGo1GHCEsLC4ICJ8syHn/8cTzxxBNoaGiAzWbD8PAwlpaWoGma3uEREd2zbDaLc+fOIRQKIRKJ4KWXXlo10yAajeL69euYnJzUKcrSwYKggEmShPr6ehw/fhwPPPAAKioqkEwmcf/99+Pq1ascYENERW9+fh6Dg4Po7e3F0tISrFZrrijQNA2hUAjj4+NYWFjQOdKdj2MICpgsy2htbcWxY8dQX18Pp9OJtrY2dHZ2wul06h0eEdGWCIVCGBgYwPDwMKLRKBKJBBKJBCKRCAYHBzEyMgK/3693mDse3xAUMKPRiMOHD6OmpgZmsxnArV3Bjh8/jnfffRc+n0/nCImINi8YDOKzzz7Dz3/+c/zsZz9DeXk5NE2D3+/HK6+8ghs3biASiegd5o4nZDc4Oo1T3rafoig4dOgQfvOb32D37t1wOBzIZDL4xS9+gTNnztxxZG4+Bh0yB4pLvgaeMg+KS7G0BYIgwGQyoaysLDebKpPJYGFhAalUimOmNmGjOcCCoICJooiKigo8+uijOHToELxeL6qqqvDiiy9ibGwMiURi3b9bLI0AbZzJZILJZILRaNzQjm8sCAgovrZAkqTc+bPZLDKZTN5+V6nYaA6wy6CAaZqGhYUF/Otf/0IymYQgCBBFEdPT01ygo8QIgoCamhp4PB64XC68//77SCaTnH5KOw4LAP2wIChwmqYhFovBbDajqqoKbrcbTqcT8XicRUEJMZlMeOihh/DYY4+hqakJly9fxszMDHOAiLYMZxkUOEEQ0NnZiaeffhrf/OY30dHRgR/96Eeoq6vjq9sSsZwDzzzzDJ555hkcPXqUOUBEW07XNwSCIMDhcKCjowNmsznXuKmqir6+Pvj9/pJ+AhJFEQ6HA08//TT27dsHm80GQRDw1FNP4cKFC4hGo0U/N5c5cGelkANEVBh0KwgURYHb7UZnZydOnjwJm80GURSRzWYRj8dRW1uLvr4+DA0NlfR0E1mWUVtbC4fDkRt5W1NTA4fDcccdwooBc2BjdnIOEFHh0K0gMJvNaGxsxPe+9z2cOHECVqs193SYSCTQ2NiI8+fPQ9M0XLt2rWSfEjVNQyQSQSQSgdPphKIoiEajUFW16AffMAc2ZifnABEVDl3fEJSVleHgwYMwmUwr+kJNJhMeeeQRtLS04Ctf+Qp+9atfYXh4GOl0Wq9wdaFpGgKBAP7whz9gcHAQDz74IA4fPozXXnsNFy9exNzcnN4hbgpz4P+203OAiAqHbusQWCwWeDwefP/738dLL70Et9u9alMLVVUxOjqK559/Hn19fUilUlsaQ7EwGo2wWCyora1FR0cHurq6MDU1VfTrEDAHNq5QcgDgOgTFphjaAsqvgl6HwGq1Yvfu3Thy5AiOHTsGi8WyZoJJkgRZllfdJEqNqqpIJpNQVRVLS0uYm5sr+tfnzIG7sxNzgIgKiy4FgdPpxAMPPIAnnngCHR0dMBqNq24GmqZhcXERPp/vjk9BpSKbzSIWi2F8fFzvULYEc+Du7bQcoNIiCAIX0ipwuhQEVVVVOHLkCJ5++ul1j0kmk7h69SreeOMNzMzMcPDUbURRXPPmWUxfNuYAUekQRRGyLHNPggKnS0GQSqWgqipUVYXRaFzzmHQ6jYmJCXR1dXGe9W1kWcbjjz+O8vJyyLKc+/zChQuYnJxEOBzWMbqNYw4QlQaz2Yzq6mocP34cFy5cgN/vx9LSEt/6FSBdCoLJyUn09/djZGQEra2tax5jMplw//334xvf+Abm5+cxNTVV0k+IBoMBtbW1OHXqFL773e+ioqJiRUHw8ccf4/z58+ju7saNGzd0jHRjNpoDu3fvxtGjR/HZZ59tc4REtBXKysrQ2tqKtrY2VFdXY3p6GhMTE7hy5QoCgYDe4dFtdCkIlpaW4PP5cOXKFXg8ntzqaysCMxjg8Xhw/PhxjIyM4PXXX0csFtMj3IIgiiLcbje+/OUvo62tDU6nM7dIDXDr38toNMLtdq/7xF1INpoDdrsdNTU1OkVJRJths9lQWVmZ25TLbDbD4XDAbrcjkUjg008/Laquzp1Ol4Igk8lgYmICH3/8MQ4cOIDm5mYYDKtDqaysxOHDhzE7O4vTp0+XdEEgCAIURUFFRQUURVl182xubobT6UR9fX1R9NEt50BXVxf27t2L1tbWVavusaEgKm5VVVXweDyorq6GKIowm80wmUywWCxIpVK4cuUK4vE4v+sFQre5XENDQ3j99dfR39+/7mIzgiDknnxLXTqdxtTUFF577TX4fL41p5yVlZWhsbERDQ0NOkR49/r7+/H73/8ev/vd7xAMBlcVMqqqYmxsDN3d3TpFSET3ymAwoKWlBfv27UNTU1Puc0EQYLFY0NzcjKNHj8LpdOoXJK2g20qFBw8exLPPPovDhw+vux57Op3G0tIS/H5/UTz15ouiKGhvb0dnZyeef/55NDU17Zg17JPJJHp6ejA1NQWz2Qy73Q5BEKBpGubm5jAyMoL+/n69wySiuyDLMh577DEcOHBgzS4/g8GA8vJytLW1YWBgAOFwuKTb+EKhW0FQXl6OlpYWlJeXr7voTCAQwOeff4633noL8Xh8myMsHCaTCR0dHTh16hS8Xu+aXQYAMD4+jkuXLqGnp0eHKO/N8kyCP//5z3jkkUdy1xeLxfD+++/jgw8+wOzsrN5hFpzlt2e350E2m0Umk2HDSrqSJAk2mw1erxcVFRUwmUyrjhEEAZIkoaysDA6HAwsLCyXdxhcK3QoCURShKModuwNisRh8Ph8++eSTkl6VzWAwoKGhAXv37oXD4Vj3uMnJSfT29uLzzz/fxug2Z3mt/jNnziAajebGEkSjUZw9ezb39ED/s9yQ1tfXr2hsE4kEZmdnEQgEoKqqjhFSKVtu28vKymA2m1cMfv4io9EIo9F4x2No++hWEASDQQwNDeHkyZPrviEwGo1wuVwlv262pmmIx+NYWlq643GBQACjo6MYGhrapsi2RjqdxuDgIEZHR3MNQzabRSqVKumppmuRJAlWqxXHjh3Ds88+i/r6+tzPZmZm8M477+DDDz+Ez+fTMUoqZZqm5dYZ+b++v8lkEslkkt/zTdjKFSB1KwjGxsZw/vx5vPDCCyu2vb1dRUUF2trasGvXLoRCoZJ9FRoOh9Hd3Q2Px4O2trZ1j+vo6AAAhEKhbYps62Sz2ZJ+C7RRLpcLnZ2dePXVV1FdXb1iLEkymcT+/ftx8eJF/PWvf9UxSip1mqYhmUwinU4jk8ms+wYglUohnU5zlsE9kiQJra2tmJubQzgc3vRiT7oVBOFwGFeuXMFvf/tbvPjii6iurl6x0A5wa2raclKVMk3T4PP50NfXh56eHhw4cGDdAuqhhx4qmlkGdPdSqRQCgQBGRkbgcrlWdLkpioKGhgY4HA60tLToGCWVskwmg2g0ivfffx91dXXYs2cPWltbV+Tq8hvA7u5uzM/P82HgHhiNRng8Hrz88sv4z3/+g66uLly6dGlT59StIFBVFXNzczhz5gwefvhhWCwWlJWVrTgmGo3C5/MhEAiU7NuBZXNzc7h27Rq6u7uxf//+NQsCRVFQXl4Ol8u1/QFS3gmCAEEQcg3uF1+zCoIAk8mU678l0ks6ncbo6Cii0SiMRiPq6+tXDCBPp9MIh8OYnJxEPB4v+fb9bkmShPLychw7dgxHjx6F2WzObRUfDAbv+by6FQSZTAbhcBgXL15Eb28vqqqqVqy+l8lkMDc3h6tXr3JjGyD35RkYGICmaeuOuxAEoeS3Ct6pFEWBw+FAVVUVzGYzc4AKWiQSAQD4fL7ctGKLxQIAiMfjmJqaQiAQQCqV0jPMomQ2m9HU1ISnnnoKdXV1sFqtCAQC6Onp2dQy77oVBMtSqRQ++ugjVFZW5i4SuDVi/tKlS+jq6mLCAPB6vXj00Ufxwx/+cN3+OE3TkEgkEAgE4PF4tjlCyrflHPjxj3+MvXv3rvmWiDlAhSQSieDKlSu4efMmTpw4gUOHDkFRlNzqs7Ozs3w7cA92796Nhx9+ODco3+124+jRowiHw+jp6bnnB2jdCwIAOH36NIaGhtDS0pIrCMbGxjAwMICBgQF9gysQDocDdXV12L1797qzLkKhEC5duoRXX30Vp0+f3uYIKd+YA1SMlqfD/vOf/8TY2BhMJhOi0Si7ggtQQRQE8XgcIyMjWFhYyM2hj0ajCIfDXKzi/1taWkIoFEIikYDNZlvzmOXXxKW858NOxhygYrXcRdzf3w+DwYB0Os0c3YSZmRn09vbi3Llz2LNnD6qrqzEwMIAPP/xwU0VWQRQEwK0+ci5As75QKISZmRmEQqF1p2kqisLdAXcw5gAVM03T4Pf79Q5jRwgEArh69Sr+8Y9/YP/+/fB6veju7kZ3d/empnAWTEFAdzY7O4vBwUGMjY2tOUUTACwWC+rq6nDw4EEdIqR8Yw4QEXBr7N3ExAT+9Kc/ob6+Hnv27MHCwgImJiY2dV4ORS4SyWQS165dw69//Wt88sknuRG8t9M0DaqqrvkzKn7MASL6Ip/Phw8//BDXrl3b9LnyXhAsT4HiNKjN0TQNkUgEly9fxptvvon5+flVx0QiEYyOjuKDDz7QIcL1MQe2RjHnABHlTzab3ZLVHvPeZVBdXQ2TyYR0Op1bkYrLVN6bdDqNxcVF9PX1YXh4GC6XK7cATTKZzK1TMD4+rnOkKzEHtk6x5gARFb68FgSSJGHPnj2oqKiAqqq4du0aFhYWkEgkSn6hoXuVzWYxMTGBTz75BIqiYN++fQD+N93s008/Lai9DJgDW6/YcoCIioOQ3eCj2t3uOCgIAhwOB1588UXU19dDlmWEw2G89957GBgYQCgU4oJDm1BeXg6v14v29nYAt24Gvb29GB0dRSqVyssTOHOgsOiRA8Dd5wHpqxDaAtLXRnMgLwXB8hat7e3tOHXqFMrLyyHLcq4P9M0338TAwACnoGyCKIowGAy5kebLW44ubwSldyPAHMg/PXIA4M2g2OjdFpD+NpoDeekykGUZLpcLBw8ehNPphMFggCAIkCQptxPb8j7YHA19b5a3Fy3UXcKYA/lX6DlARMVly4d9y7IMt9uNXbt2obm5edUmLKIooqmpCY2NjXA4HFv966kAMAeIiIrPlhcEbrcb9913Hzo7O2GxWNZ8tVRfX4+GhgaUl5dv9a+nAsAcICIqPlteEOzatQt79+7Frl27YDCs3yPhdrvxpS99iX1ROxBzgIhoe0mSBKPRCLPZnPtjMpnWXNF0PVs+hiCRSGxonnkqleLmFjsUc4BoZ7NYLEgkEtytsEDY7Xbs27cPnZ2d2LVrV+7zcDiMK1eubPg8W14QpNNppNNpZDIZZLPZdZ/+ljdhEQSBi9TsMBvNgUwmw2mHREXoyJEjGBwchN/vRyKR0DucktfR0YETJ07gxIkTqK2tzX0eiUSwe/fuDZ9nywuCVCqFaDSKUCgEp9MJs9m85g3BarWipqYGoiiyytxhNpID6XQaiUSC21sTFaHvfOc7eOedd9DX14epqSm9wylpgiDg2LFjeOyxx3DgwAGYzebczzKZDPbs2bPhc215QTA+Po5gMIjr16/joYceQkdHB4xG46rjRFGELMuw2+1YXFxkUbCDbCQH/H4/bty4gd7eXr4hIioyP/jBD1BWVoa//e1vLAh0ls1mMTQ0hL179+ZmdS2TJAk2m23D59ryQYXpdBqhUAg+nw/BYHDdG73FYkFjYyO+9a1vweFwcOObHWQ5ByYnJzEyMoKFhQWoqpr7eTwex9TUFGZnZ1d8TkTFQZZlyLJ8x0HDtD2MRiMMBsOW3EPz9r8piiJsNtu6QYqiCJPJhPr6ejgcDsTjcd4cdhBN05BIJDA0NAS73Q6v14u6ujq4XC4sLCygr68PY2NjfDtAVITi8ThisRgXxdKZoij49re/jSeffBLt7e1wOp2bOl9eCgJBECDLMqxW67oDypZXrbNarbBYLDAYDCwIdphMJoOZmRlcvnwZqqoiHo+jvr4e4+PjGB4exsLCgt4hEtE9GB4e5ndYZ8v32ZMnT+Lo0aOoqamBoiibOmfeCgJBEDb0CmP5OM5F35k0TcPExARCoRCGhobg8XgQDAY5boSoiL3zzjt48803cePGDb1DKVnLD9XNzc1wu92bLgaAbXhDcKeiQBAEKIoCs9nMvqgdLhwOIxwOY3x8HED+Nt4hovyrq6uDwWDg91hH2WwWqVQKly9fhtfrhd1u3/Q58zKSL5PJIBqNYmhoCFNTU4hEIshkMmsed/PmTczNzXH6WYnIZrNsRIiK3OHDh1FXVweLxaJ3KCVruSD473//C7/fv+Y99m7lpSBYHlA2MDCAixcvwufzrTk+IJVKYWxsDOFwmAvUEBEVCYvFAlmWOTtMZ5lMBv39/ejq6sLg4OCmi4K8vadPJpMYHBzE9PQ0FEWBw+GA0WiEJEkAbl3I0tISBgcHEYvF2J9MRFQkAoEA4vE40um03qGUtGw2i7GxMbz11lsQBAENDQ13te7AF+W1vNM0DUtLS7h69Sq6u7vh9/tzN/6bN2/i448/xuTk5Ja86iAiou1x5swZDA0NYXFxUe9QSt7i4iLOnz+Pc+fO4fr165s6V95H8qXTaYyPj8Pv92NychKnTp2C2+3G1NQUrl69esfFi4iIqPB8+umnCIVCbLsLRCqVwvDwMM6dO4fa2lrU1tZCkiTEYjGMj4+jtbV1Q+cRshsc4bXZaYEGgwFWqxUtLS2wWCwIBAIYHx9nhZkn+Ri4x6mhxSVfgzeZB8UlH3nQ0NCAubk5LkxUQMrLy+H1evHkk0/mtp6PRqMYHh7GK6+8sqFzbFtBsMxisUCSJKTTaSSTSXYX5AkLAmJBQADbglKiKAra29tRWVkJSZIQj8fh8/k23JWw7QUBbQ82AsSCgAC2BbTxHOCcESIiImJBQERERCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICICQzWazegdBRERE+uIbAiIiImJBQERERCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICCwIiIiICMD/A4oItQ2r2nQhAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAYrUlEQVR4nO3dW2wcVx0G8G92dmZnr961Ha+93sQ28SYxJM7NaqgVl4Y2AhWkVgqokZAQICqBgBceeKJvvPStL7whEBIXgdRwSWkTJZVIgagkJLbiJL7F8SV24ttevLve3dnLDA+Rl7q2g9Ps7Mza30/qy3rtOdN8e+a/Z+acI+i6roOIiIh2NJvZDSAiIiLzsSAgIiIiFgRERETEgoCIiIjAgoCIiIjAgoCIiIjAgoCIiIjAgoCIiIgA2Lf6RkEQjGwHVZgR600xA7XFqDXHmIPawr6AtpqBLRcE1SIIAhwOBwCgVCqhWCyuORlBECDLMmy2jQc3NE0r/57NZoMsy5uGV9d1qKpqWMf5NJ7mvLY7URTh9/ths9mgqipWVlZQKpXKP7fb7fB4POWcfFI+n4eqqshkMpBlGV6vF3b7+qjruo5SqYREIrHm75vlac5rJ2BfwL6AGahuBixREAiCAEmSyv+FQiEIgoDl5WUsLi5CVdXyeyVJQkNDA1wu15q/YbfbIcsyMplM+ffcbjdCoRAcDgdKpRJUVYXNZoMkSRAEAcViEffv30ehUDA9BE9zXtvR6sXQ7XbD7Xajr68Pdrsd4+PjuHHjBuLxePm9brcb3d3daG5uLr9ms9ngdrvh8Xjw6NEj3Lt3D/39/WhpacELL7yAQCCAfD6PeDwOm80Gr9cLSZKQTqfxl7/8BalUyvSiYKvndfPmTRNbaSz2BewLmAHzMmCJgkAURXg8Hng8HjidThw8eBCiKGJqagqpVGpdAAKBAAKBwJrfr6urg9/vx+LiIqanp7G4uAifz4cDBw7A6/VCVVU8evQIkiShvr4eiqIgk8ng4cOH66pOM3z8vERRhM1mgyiK8Hq98Hq9a85rO3I4HAiHwwiHw2hqasIPfvADSJKEixcvYmJiYl1B8NnPfhYHDhwoV/uSJCESiSASiaC/vx8XLlzAwMAAOjo68K1vfQttbW2IxWL46KOP4PF4sH//fjQ2NmJhYQFXrlxBJpOxREGwel6yLMNut8PhcKC9vR1tbW3l89rOBQH7AvYFzIB5GbBEQbD6j9LU1ASn0wmfzwfg8UXik8M7KysrGB4eXvO6LMvo7u5GMBiEoiiQJAnA42C4XC5IkoRoNIpr167B5/PhyJEj5WMIgmCJ+2EfP6/GxkZ4PB7U1dXB6XTC5XKtOa/tyO1249ChQzh27Bh27dqFjo4O6LoOv9+/brj/0aNH+NWvfgVRFMuv1dXV4Yc//CFOnDhR/v8HAIqioKmpCR6PB3fu3MGbb76Jjo4O/PjHP0YkEgHwuOq2QgY+fl7d3d3o6OhAe3s7gsEggsHgmvPartgXsC9gBszLgCUKglwuh+npaczPz8Pj8SAUCpXvn9jtdoiiCE3TylXbJ7/J2Ww2aJpmRtOpQqLRKN577z1cvXoV4XAYL7zwQvnev6IoUBQF+Xy+nANVVdd1AlYY6qNnw76AmAHzWKIgkCQJjY2N8Pv9cDqd0DQNxWIRLpcLbW1taGxsxOzs7BOHdYvFIlRVRaFQKD9ooWkacrkcJEmCoiiIRCJQFAUulwvFYhH5fN4yFxCHw4FgMAiPxwNFUSDLMmRZhiiK685rO/J4PDh8+DAikQiamppQKBSQzWbR3NyMV155BT09Pbh8+TIWFhaQzWYBrH1yVtM0ZDIZxONxpFIp5HI5AICqqkgkEvB6vQgEAjh79izq6+vR3NyMdDqN5eVly3QegUAAzz33HMLhMBobG+H1euHz+SDLMpaXl9ec13bFvoB9ATNgXgYsURDYbLbyP4zdbsfS0hIEQYCqqpAkCR6PB3a7HTabbcMAaJqGZDKJR48eIZVKIZ1OA3hcaS4uLiKbzaJQKMDj8UAQhHLHqqqqJe4XAY//HzidzvKQsKZp5YtZLpdDIpEon9d2tDpM2NLSAo/Hg4GBAYiiiGQyibq6OsiyDKfTueY2wccVi0VMTU3hX//6F6ampjAzMwNd1xGPx3Hz5k3Mzc0hk8mgtbUVDocDMzMziEajiMViyGazligKJEnCrl27sHv3bgiCUJ4BMTY2hng8jpGREczMzJjdTEOxL2BfwAyYlwFB3+LZG3lfZXWKxZPu5a522ps1V5ZlSJJUriYLhQJEUYTD4Xji1I1sNmuZADgcjk0veKtTTAqFwpb+Xq3NPbbb7fD5fHC5XJseZ2lpCaqqbnjxFgSh/I06n88jm80ilUpBURT4/f4N77etTjtcXFy0xDcuSZLg9/uhKMqGP1dVtXxeW1GL6xCwL2BfwAyYlwFLFARUebXWCVDl1WJBQJXHvoC2mgEuXUxEREQsCIiIiIgFAREREYEFAREREYEFAREREYEFAREREcEiCxMR0c61ui3txzeoAR7PtY5Go6ZvOkXGYwasgQUBEZnK5XKhubkZp0+fBvD44lAsFpFOp3HhwgUsLy9bYuEoMg4zYA0sCIjINOFwGF/84hdx9uxZvPTSS+VV5EqlEtLpNP74xz/it7/9LYaGhhCPxy2xihxVFjNgHSwIiMg0brcbra2t6OrqgizL5dftdjvsdju++tWvoqGhARcuXMB7772H+fl5E1tLRmAGrIMPFRKRaVRVRTwex8OHD9d98xNFEaFQCMePH0dXVxfcbrdJrSQjMQPWwYKAiEwhiiJKpRLm5+cxPDy84VCwKIrYtWsX6uvrN9ygimobM2AtvGVARKZoaWnBmTNn8OUvfxmnTp3adBc62r6YAWthQUBEVWez2bB//36cPXsWhw8fXnPv+ONSqRTeeecdXLx4EYuLi1VuJRmJGbAelmNEZBqHw7HphQAACoUChoaGMDY2hnQ6XcWWUbUwA9bBgoCIqk7XdcTjcSQSCWQymU3ftzofPZ/PI5/PV7GFZDRmwHpYEBBR1em6jqGhIYyOjj5xGNjn8+HYsWPo7OysYuuoGpgB6+EzBERkilwuh7///e/weDxwuVwIBoPr3mOz2eB2u+FwOExoIRmNGbAWFgREZApd13H58mUkEgmkUim88cYb654yT6fTGBoawoMHD0xqJRmJGbAW3jIgItMsLi5idHQUAwMDWFlZgaZp5Z9pmoZEIoGpqSksLS2Z2EoyEjNgHSwIiMhUiUQCIyMjGB8fRzqdRi6XQy6XQyqVwujoKO7fv49oNGp2M8lAzIA1CPoWd4oQBMHotlAFGbEBCDNQW4zaBKbSOVi9R3zs2DH86Ec/QkNDAzRNQzQaxVtvvYV79+4hlUqt+eZIW1cLfQEzYKytZoAFwTZVC50AGatWCoLVv6koCurr6yGKIoDHu90tLS2hUCjwQvAMaqUvYAaMw4Jgm1EUBYqiwOFwbGm3r1rpBMg4tVQQrBJFsfz3dV1HqVQy7Fg7Ra31BcxA5W01A5xlUAMEQUBzczPC4TD8fj8uXbqEfD7PfcFp22HnT8yAeVgQ1ABFUfD888/j5ZdfRnt7O27duoW5uTmu2kVERBXDWQYWJwgC+vr6cObMGZw5cwa9vb343ve+h1AoxCF8IiKqGFNHCARBgM/nQ09PD5xOZ/kCp6oqBgcHEY1Gd/S3YJvNBp/Ph9deew2HDh2Cx+OBIAh49dVX8dFHHyGdTnNuLhERVYRpBYEsywgEAujr68Pp06fh8Xhgs9mg6zqy2SxaWlowODiIsbExpFIps5ppOkmS0NLSAp/PV37ytrm5GT6f74k7hBERET0N0woCp9OJtrY2fPOb38SpU6fgdrvLIwS5XA5tbW344IMPoGka7t69u2NHCjRNQyqVQiqVQl1dHWRZRjqdhqqqfPiGiIgqxtQRgvr6ehw9ehSKoqy5H64oCl588UXs378fn//85/GTn/wE4+PjKBaLZjXXFJqmIRaL4Ze//CVGR0fx3HPP4fjx4/j973+PGzduYGFhwewmEhHRNmHaOgQulwvhcBjf/va38cYbbyAQCKzb1EJVVUxMTOAb3/gGBgcHUSgUKtqGWuFwOOByudDS0oKenh5cvXoVMzMzyOVym/5Orc09psqrxXUIqPLYF5Cl1yFwu93Yu3cvTpw4gZMnT8Llcm0YMFEUIUnSukJhp1FVFfl8HqqqYmVlBQsLCzv2FgoRERnDlIKgrq4On/vc5/DKK6+gp6cHDodjXUGgaRqWl5cxOzv7xG/CO4Wu68hkMpiamjK7KURPTRAELqS1wzED1mdKQdDU1IQTJ07gtdde2/Q9+Xwed+7cwTvvvIO5uTk+QEdUo2w2GyRJ4nr0OxgzUBtMGYsvFApQVRWqqm76nmKxiOnpaVy9ehVLS0sMEVENcjqd2LNnD15//XV0dnaioaEBiqKY3SyqImagdpgyQvDgwQMMDw/j/v376Orq2vA9iqJg79696O3txX/+858qt5CIKqG+vh5dXV04ePAggsEgHj16hOnpady+fRuxWMzs5lEVMAO1w5SCYGVlBbOzs7h9+zbC4XB5Bb41DbPb4fV60dzcbEYTiegZeTwe7Nq1q7wpl9PphM/ng9frRS6Xw/Xr13lPeZtjBmqLKQVBqVQq3w7Yt28furq61q26x5AQ1bampiaEw2EEg0HYbDY4nU4oigKXy4VCoYDbt28jm83ys76NMQO1xbT5fMPDw/jFL36Bn//854jH4+ueEVBVFZOTk7h27ZpJLSSiT8tut2P//v04dOgQ2tvby68LggCXy4WOjg709vairq7OvEaSoZiB2mPqBP98Po/+/n7MzMwgnU6Xq0RN07CwsID79+9jeHjYzCYS0VOSJAmnT5/GkSNHNrzlZ7fb0dDQgIMHD8Lr9e74dUa2I2agNpm62+HqTILf/OY3ePHFFxGJRCDLMjKZDC5duoQrV65gfn7ezCZakiAIsNvta5670HUdpVKJszHIVKIowuPxIBKJoLGxccOnyQVBgCiKqK+vh8/nw9LSErLZrAmtJSMwA7XL1IJgda3+999/H+l0uvwsQTqdxoULFzAyMoJkMmlmEy1n9UPU2tq65oOWy+UwPz+PWCz2xOmcREay2WzlfUqcTmd5h86NOBwOOByOJ76Hag8zULtMLQiAx6MEo6OjmJiYKIdC13UUCgUuRvQJoijC7Xbj5MmTeP3119Ha2lr+2dzcHM6fP48PP/wQs7OzJraSdjJN08rrjPy/z28+n0c+n+fn/BlYcfU/ZqC6KpkB0wsC4HEBwLX5/z+/34++vj68/fbbCAaDa2Zm5PN5HD58GDdu3MDvfvc7E1tJO52macjn8ygWiyiVSpt++ysUCigWi5a7oNUKURTR1dWFhYUFJJNJSy3xzgxUR6UzYImCgLamUCggFovh/v378Pv9cDgc5Z/Jsozdu3fD5/Nh//79JraSdrJSqYR0Oo1Lly4hFAqhs7MTXV1da7K6OgJ47do1LC4u8svAp+BwOBAOh/Hmm2/iH//4B65evYqbN2+a3SwAzEC1GJEBFgQ1QhAECIJQ/rB9cohNEAQoilK+d0dklmKxiImJCaTTaTgcDrS2tqKhoaH8JHmxWEQymcSDBw+QzWb5IOxTEkURDQ0NOHnyJHp7e+F0OstbxcfjcbObB4AZMJpRGWBBUCNkWYbP50NTUxOcTuem03QEQeAUHjJdKpUCAMzOzmJmZgZOpxMulwsAkM1mMTMzg1gshkKhYGYza5LT6UR7ezteffVVhEIhuN1uxGIx9Pf3W2qZd2bAOEZlgAVBjYhEInjppZfw/e9/H/v27Vu31DPw+L5dLpdDLBZDOBw2oZVE/5NKpXD79m08fPgQp06dwrFjxyDLMubn5/Huu+9ifn6e3ww/hb179+ILX/gCTp8+DZvNhkAggN7eXiSTSfT391vqAT1mwBhGZYAFQY3w+XwIhULYu3fvhsUAACQSCdy8eRNvv/023n333Sq3kGi91emwf/vb3zA5OQlFUZBOpxGLxXgh2CGYgdrBgqBGrKysIJFIIJfLwePxbPie1VsFmUymmk0jeqJSqYRkMonh4WHY7XYUi0Vm9BnMzc1hYGAAly9fRmdnJ4LBIEZGRvDhhx9a9gLLDFSWURlgQVAjEokE5ubmkEgk4Ha7NxwlkGWZO0SSJWmahmg0anYztoVYLIY7d+7gz3/+Mw4fPoxIJIJr167h2rVrlp6+xwxUjlEZYEFQI+bn5zE6OorJyUkEg0FIkrTuPS6XC6FQCEePHjWhhURUDYVCAdPT0/j1r3+N1tZWdHZ2YmlpCdPT02Y3jarEqAywIKgR+Xwed+/exU9/+lP87Gc/w+HDh+H1ete8R9M0qKpafrqXiLa32dlZPHz40OxmkIkqmQHD56etToPjVLhno2kaUqkUbt26hXPnzmFxcXHde1KpFCYmJnDlyhUTWkhEZtB13dK3Csh4lcqA4SMEwWAQiqKgWCyWV6RieD+dYrGI5eVlDA4OYnx8HH6/v7wIUT6fx4MHDzAyMoKpqSmTW0pERLXG0IJAFEV0dnaisbERqqri7t27WFpaQi6Xs9Rc2Vqi6zqmp6fx73//G7Is49ChQwD+N+Xw+vXrSCQS5jaSiIhqjqBv8ev6ZnPfn/R+n8+H73znO2htbYUkSUgmk7h48SJGRkaQSCS4QtUzaGhoQCQSQXd3N4DHBcHAwAAmJiZQKBQMGYV52gyQuYwaiWMOagv7AtpqBgwpCFa36e3u7saXvvQlNDQ0QJKk8n3wc+fOYWRkhFNQnoHNZoPdbi/PNljdcrRYLAJgJ0AsCOgx9gW01QwYcstAkiT4/X4cPXoUdXV1sNvtEAQBoiiWd+Nb3QebT8R/Oqvbi3KXMCIiqoSKP/ovSRICgQD27NmDjo6OdRvx2Gw2tLe3o62tDT6fr9KHJyIiok+h4gVBIBDAgQMH0NfXB5fLteHQUmtrK3bv3o2GhoZKH56IiIg+hYrfMtizZw/27duHPXv2wG7f/M8HAgF85jOfweDgIKchEhE9A1EUYbfb14zG6rrO2Vw7SCUyUPGCIJfLbWmtgUKhwM0tiGqQy+VCLpez7EY6O43X68WhQ4fQ19eHPXv2lF9PJpO4ffu2IcdkBqylUhmoeEFQLBZRLBZRKpWg6/qmT6OWSiVOOySqQSdOnMDo6Cii0ShyuZzZzdnxenp6cOrUKZw6dQotLS3l11OpFPbu3WvIMZkBa6lUBipeEBQKBaTTaSQSCdTV1cHpdK4rCorFInK5HLLZbKUPT0QG+/rXv47z589jcHAQMzMzZjdnRxMEASdPnsTLL7+MI0eOwOl0ln9WKpXQ2dlpyHGZAeuoZAYqXhBMTU0hHo9jaGgIzz//PHp6euBwONa8JxqN4t69exgYGODzA0Q15rvf/S7q6+vxhz/8gRcDk+m6jrGxMezbt688q2uVKIrweDyGHJcZsI5KZsCQWwaJRAKqqiIYDKK9vR2NjY3loiCbzWJmZgbz8/NQVbXShycig0mSBEmSnvjQMFWHw+FY9yBZNTAD1lHJDBjyr6lpGnK5HMbGxuD1ehGJRBAKheD3+7G0tITBwUFMTk5ydICoBmWzWWQyGS6KZTJZlvG1r30NX/nKV9Dd3Y26urqqHZsZsIZKZ8Cw8q5UKmFubg63bt2CqqrIZrNobW3F1NQUxsfHsbS0ZNShichA4+Pj/AybTBAESJKE06dPo7e3F83NzZBluWrHZwbMZ0QGDB3v0TQN09PTSCQSGBsbQzgcRjwex/LyMqerENWo8+fP49y5c7h3757ZTdmxVpeC7+joQCAQqGoxADADVmBEBqpyAyiZTCKZTGJqagqAcZuuEJHxQqEQ7HY7P8cm0nUdhUIBt27dQiQSgdfrrerxmQHzGZGBqj6Jous6A0RU444fP45QKASXy2V2U3as1YvBP//5T0Sj0aqvSMgMmM+IDFT30VQiqnkulwuSJFX9yXZaq1QqYXh4GFevXsXo6GhViwJmwBoqnQHOGSGipxKLxZDNZlEsFs1uyo6m6zomJyfxpz/9CYIgYPfu3YatO/BJzIA1VDoDLO+I6Km8//77GBsbw/LystlN2fGWl5fxwQcf4PLlyxgaGqracZkB66hkBlgQENFTuX79OhKJBGcKWUShUMD4+DguX76MmZmZ8rBxJpMxrEhgBqylUhngLQMieir9/f1IpVJmN4M+ZnJyEn/9619RKpXKW8+n02mMj4/jrbfeqvjxmAHrqUQGBH2Lj/1vtmshWZMRszmYgdpi1Iwe5sCaZFlGd3c3du3aBVEUkc1mMTs7a8goATNgTc+aARYE2xQLAmJBQAD7Atp6BvgMAREREbEgICIiIhYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBBYEREREBEDQdV03uxFERERkLo4QEBEREQsCIiIiYkFAREREYEFAREREYEFAREREYEFAREREYEFAREREYEFAREREYEFAREREAP4L6b51ZHD9RBMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAARoUlEQVR4nO3dS28b56HG8Wc4F96GukuUZSWWEAuOU1s2nLQBjMSF2xoo2gJdNAm8Sxftqsmm+3yAfIquuinQa9rURlWgLdogcNrIaGzIlq34EtmKIlGURJp3cs7CEI9V2T5MD8mZof4/wBuSEt8xH837cK6G53meAADAvhbxewAAAMB/FAIAAEAhAAAAFAIAACAKAQAAEIUAAACIQgAAAEQhAAAAkqxWX2gYRifHgTbrxPWmyEC4dOqaY+QgXFgXoNUMtFwIusU0TQ0MDCgSiahcLuvBgweq1+vN5y3Lkuu6ikajj/35SqWicrmsQqEgx3GUSqVkWXsX0/M81et1bW5u7vr9fvkyy9XrDMNo/j/U63XVarVdgTYMQ47jKBJ5/AauRqPR/LlIJCLHcZ64AvM8T+VyuWOT55fxZZZrPyAH5ID5oLvzQSAKwc7CJ5NJJZNJvfrqq7IsS0tLS/rXv/6lbDbbfG0ymdTs7KzGx8ebj0UiESWTSbmuq5WVFd28eVPz8/M6cOCAzpw5o8HBQVUqFWWzWUUiEaVSKdm2rXw+r9/+9rfK5XK+h6DV5fr44499HGXnGIYh27ab/yYmJmQYhra2trS2tqZyudx8rW3bGh4eViKR2PU7LMuS4zgqFArNn0smk5qYmFA0GlW9Xle5XFYkEpFt2zIMQ7VaTZ9++qmq1arvk8GXWa5eRQ7IAfOBf/NBIApBNBrV5OSkJicnNTY2pp/85CeybVsXL17UrVu39gTghRde0PPPP99s+7Zta2ZmRjMzM5qfn9eFCxd0+fJlTU9P64c//KEOHTqkjY0Nffjhh3JdV0eOHNHIyIi++OIL/fWvf1WhUAhEAHaWy3EcWZalaDSqqakpHTp0qLlcvVoITNOU67pyXVfxeFzHjh2TaZq6c+eOcrncnolgcHBQg4ODu36+v79fAwMDWltb0927d7W2tqa+vj49//zzSqVSKpfLWllZkW3bGhoaUiwWU6FQ0P379/d8+/TDo8tlmqYikYhM01QqlVIqldq1XL2KHJAD5gP/5oNAFIJkMqnjx4/r1KlTGh0d1fT0tDzP08DAwJ7NOysrK/rZz34m0zSbj/X39+utt97Syy+/rJGREbmuK0mKxWIaGxuT67q6evWq3nnnHU1PT+unP/2pZmZmJD1so0HYH/bocs3Ozmp6elpTU1NKp9NKp9O7lqsX7aycx8bGFI/H1dfXJ+nhyuE/P58HDx7o2rVrux53HEezs7NKp9OKxWKybVvSwwkikUjItm1lMhldunRJfX19OnnyZPM9DMMIRAYeXa6dz7u/v1/xeFyJRGLXcvUqckAOmA/8mw8CUQgymYzef/99ffDBB5qcnNSZM2ea+3pisZhisZgqlYoajUZzX99/rgSCsKkP/71SqaS7d+9qdXVVrutqYmKiuR/VsiyZptn8/CXtafCRSESNRsOPoaONyAGYD/wTiELguq5OnDihmZkZjY2NqVqtqlgsanx8XN/5znf00ksvaW5uTl988YWKxaKk3UdNNhoNFQoFZbNZ5XI5lUolSVK5XNbm5qZSqZQGBwd1/vx5DQ0NaXx8XPl8XltbW4FZeQwODuprX/uaJicnNTIyolQqpb6+PjmOo62trV3L1Yts29bIyIgGBgYUj8fVaDRUq9WUSCR06NAhjYyM6N69e0/dnFer1VQul1WtVpsHXDUaDZVKJdm2rVgsppmZGcViMSUSCdVqNVUqlcCsOKLRqNLptFzXVSwWk+M4chxHpmnuWa5eRQ7IAfOBf/NBIArBzmbCAwcOyHVdXb58WaZpant7W/39/XIcR/F4fNdmoUfVajXduXNH//jHP3Tnzh0tLy/L8zxls1l9/PHH+vzzz1UoFHTw4EFFo1EtLy8rk8loY2NDxWIxECGwbVujo6N65plnZBhG84jXGzduKJvN6vr161peXvZ7mB0TiUSaK2jLsrS+vi7DMFQul2XbtlzXlWVZikQij50IGo2Gtre3tbKyolwup3w+L+nhN861tTUVi0VVq1W5rivDMJp/UOVyORD7jaWH/wfxeLy5KbDRaDRXYqVSSZubm83l6lXkgBwwH/g3Hxhei38BndyvYlmW+vr6lEgknvg+6+vrKpfLj/2wDMNoNqhKpaJisahcLqdYLKaBgYHH7m/bOc1kbW0tEG3btm0NDAwoFos99vlyudxcrlaE7dzjnVOtnrYPb+eP9UnL5jiObNtufqusVqsyTVPRaPSpp3AVi8XATATRaPSJK7qdU82q1WpLvy+M1yEgB+HIAfNBZ/k1HwSiEKD9wrYSQPuFsRCg/VgXoNUMcOliAABAIQAAABQCAAAgCgEAABCFAAAAiEIAAAAUkAsTYX/auSXtozenkR6eZ53JZHy/wQi6gxwAwUAhgG8SiYTGx8d17tw5SQ8nhlqtpnw+rwsXLmhraysQFwlBZ5EDIBgoBPDF5OSkvvGNb+j8+fP65je/2byCXL1eVz6f1y9+8Qv9/Oc/18LCgrLZbCCuIIf2IwdAcFAI4ItkMqmDBw/q6NGjchyn+bhlWbIsS9/73vc0PDysCxcu6P3339fq6qqPo0WnkAMgODioEL4ol8vKZrO6f//+nm99pmlqYmJCL774oo4ePapkMunTKNFp5AAIDgoBus40TdXrda2ururatWuP3QxsmqZGR0c1NDT02JuRIPzIARAs7DJA1x04cEA/+MEP9O1vf1tnz5594h3o0NvIARAsFAJ0VSQS0ZEjR3T+/HmdOHFi137jR+VyOf3yl7/UxYsXtba21uVRotPIARA8VHL4IhqNPnESkKRqtaqFhQXduHFD+Xy+iyNDN5EDIDgoBOgqz/OUzWa1ubmpQqHwxNftnIteqVRUqVS6OEJ0AzkAgodCgK7yPE8LCwtaXFx86ibgvr4+nTp1SocPH+7i6NAt5AAIHo4hQNeVSiX95S9/keu6SiQSSqfTe14TiUSUTCYVjUZ9GCG6gRwAwUIhQNd5nqe5uTltbm4ql8vpxz/+8Z4jzPP5vBYWFvTZZ5/5NEp0GjkAgoVdBvDF2tqaFhcXdfnyZT148ECNRqP5XKPR0Obmpu7cuaP19XUfR4lOIwdAcFAI4JvNzU1dv35dS0tLyufzKpVKKpVKyuVyWlxc1KeffqpMJuP3MNFh5AAIBsNr8W4hhmF0eixoo07cBKbdGdjZP3zq1Cm9/fbbGh4eVqPRUCaT0bvvvqubN28ql8vt+taI1nXqRkDkIFzCsC5AZ7WaAQpBjwrLSsAwDMViMQ0NDck0TUkP73S3vr6uarXKJPD/EJZCsPM7yUFnhGVdgM6hEOxzYVsJmKbZ/P2e56ler3fsvfaLMBWCHeSg/cK2LkD7tZoBzjJAILDih0QOAD9xUCEAAKAQAAAACgEAABCFAAAAiEIAAABEIQAAAKIQAAAAUQgAAIAoBOgwrmgGiRwAYUAhQMdEIhE5jrPnHvfYX8gBEA5cuhgdEY/HlU6ndebMGX344YfKZDJ68OCBSqWS30NDF5EDIDwoBOiIoaEhHT16VMeOHVM6ndbKyoru3r2rK1euaGNjw+/hoUvIARAeFAK0neu6Gh0d1eTkpAYGBhSPx9XX16dUKqVSqaSPPvqoY3fiQ3CQAyBcKARou7GxMU1OTiqdTisSiSgejysWiymRSKharerKlSsqFotMBj2OHADhwlE+aCvLsnTkyBEdP35cU1NTzccNw1AikdD09LROnz6t/v5+/waJjiMHQPhQCNA2tm3r3LlzOnnypMbHx/c8b1mWhoeHdezYMaVSKY4671HkAAgn/hLRFqZpynVdzczMaGRkRLFYbM9rDMOQaZoaGhpSX1+fotGoDyNFJ5EDILwoBGiLnXPNh4aGFI/HZZrmE18bjUYVjUaf+hqEEzkAwotCgLZoNBqqVqsql8uq1+tPfW2lUlGlUvk/X4cnC+qV/8hBdwU1BwgnCgHaptFoqFKpqFarPXUlX61WVavVOLr8v2Sapr7yla9obGzssZvk/UYOuiPoOUD4UAjQFvV6Xfl8Xn/605908eJFffLJJyqXy7te43meqtWqLl26pLW1NVUqFZ9GG17RaFRTU1N655139MYbb+iFF17we0i7kIPuCHoOEE5chwBtU6vVdOvWLeXzeUWjUR08eFDDw8PNo8hrtZq2t7f12WefqVgsqtFo+DzicDFNU8PDw3rllVd0+vRpxeNxlctl3bp1S9ls1u/hNZGDzgpLDhA+FAK0VS6XkyTdu3dPy8vLisfjSiQSkqRisajl5WVtbGyoWq36OcxQisfjmpqa0ve//31NTEwomUxqY2ND8/Pz+uc//+n38HYhB50TphwgXCgEaLtcLqcrV67o/v37Onv2rE6dOiXHcbS6uqrf//73Wl1d5Vvhf+G5557T17/+dZ07d06RSESDg4M6ffq0tre3NT8/H7iD88hBZ4QtBwgPCgE6olQqaXV1VX/4wx90+/ZtxWIx5fN5bWxsMAnsI+QACA8KATqmXq9re3tb165dk2VZqtVqKhQKfg8rtD7//HNdvnxZc3NzOnz4sNLptK5fv66//e1vgZ5cyUF7hTUHCD4KATqq0Wgok8n4PYyesLGxoatXr+o3v/mNTpw4oZmZGV26dEmXLl0K/Kl75KB9wpwDBJvhtZggLoARLp1YMZCB4Dh48KAOHz6s9fV1Xb169bGv6dTkQA6Cw68ckIFwaTUDFIIexUqg9+18Hk/6rCkE+4MfOSAD4dJqBthlAIQUm4chkQO0D1cqBAAAFAIAAEAhAAAAohAAAABRCAAAgCgEAABAFAIAACCuQwCEmmmasixLkcj/dnvP87jj3T5DDtAOFAK0LJFIqFQqcQOVgEilUjp+/LheffVVPfvss83Ht7e3deXKlY69LzkIFr9ygN5DIUDLXn75ZS0uLiqTyahUKvk9nH3vpZde0tmzZ3X27FkdOHCg+Xgul9Nzzz3XsfclB8HiVw7QeygEaNnrr7+u9957T5988omWl5f9Hs6+ZhiGXnnlFX3rW9/SyZMnFY/Hm8/V63UdPny4Y+9NDoLDzxyg93BQIVr2ox/9SG+++aa++tWv+j2Ufc/zPN24cUNLS0va2tra9ZxpmnJdt2PvTQ6Cw88coPdQCNAy27Zl27Ysiw1LfotGo3sOIusWchAcfuYAvYe/aLSsWCyqUCioUqn4PZR9zXEcvfbaa/rud7+r2dlZ9ff3d/X9yUEw+J0D9B4KAVq2tLSkpaUlra+v+z2UfcswDNm2rXPnzun06dMaHx+X4zhdHQM58F8QcoDeQyFAy9577z396le/0s2bN/0eyr5lGIZM09T09LQGBwd9mQTIgf+CkAP0HnY8oWUTExOyLEue5/k9lH3L8zxVq1X9+9//Vj6f92UM5MB/QcgBeg+FAC178cUXNTExoUQi4fdQ9q2dieDvf/+7MpmML1eiIwf+C0IO0HsoBGhZIpGQbdsc0eyzer2ua9eu6YMPPtDi4mLXJwNyEAx+5wC9h2MI0LKNjQ0Vi0XVajW/h7KveZ6n27dv69e//rUMw9AzzzzT1fPNyUEw+J0D9B4qPlr2xz/+UTdu3NhzARR039bWlv785z9rbm5OCwsLXX1vchAcfuYAvYdCgJZ99NFH2tzc5KY2AVGtVrW0tKS5uTktLy83NxkXCoWOTg7kIFj8ygF6D7sM0LL5+Xnlcjm/h4FH3L59W7/73e9Ur9f17LPPyrIs5fN5LS0t6d133+3Ie5KD4PEjB+g9htfiuUOGYXR6LGijTpwSRgaCyXEczc7OanR0VKZpqlgs6t69ex37dkgOgqmbOSAD4dLqfEAh6FEUAnTqOgHkIFxYF6DVDHAMAQAAoBAAAAAKAQAAEIUAAACIQgAAAEQhAAAAohAAAABRCAAAgCgEAABAFAIAACAKAQAAEIUAAACIQgAAAEQhAAAAohAAAABRCAAAgCgEAABAFAIAACDJ8DzP83sQAADAX2whAAAAFAIAAEAhAAAAohAAAABRCAAAgCgEAABAFAIAACAKAQAAEIUAAABI+h9aqf1LJgC7MgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAARp0lEQVR4nO3dyW8b5+HG8Wc4C3ftEmVZiSXEguPUlg0nbQAjceG2BoK2QA9d4Ft7SE5tLrnnD8hf0VMvBbqmTW1UBdqiDQKnjYzGhmzZipfIVhSJoiTSJIfb/A6GGCuyHSY/DmdIfz+ALySteUd68M7DWQ3P8zwBAIAnWiToAQAAgOBRCAAAAIUAAABQCAAAgCgEAABAFAIAACAKAQAAEIUAAABIslr9oGEYfo4DbebH/abIQHfx655j5KC7MBeg1Qy0XAg6xTAMRaNRSVK9XletVtu1MoZhyHEcRSIP37nRaDSa/y8SichxnEeG1/M8ua7r28T5ZXyZ9ep1ZIAMSJJpmhoYGFAkEpHrurp3757q9XrzfcuylEqlmln5vEqlItd1VSwW5TiO0um0LGvvlOd5nur1ujY3N3f9/KB8mfXqdWSgsxkIRSEwDEO2bTf/TUxMyDAMbW1taW1tTa7rNj9r27aGh4eVSCR2/QzLsuQ4jorFYvP/JZNJTUxMKBqNql6vy3VdRSIR2bYtwzBUq9X00UcfqVqtBr5B+DLr1YvIABmQPpsIk8mkksmkXn75ZVmWpaWlJf33v/9VLpdrfjaZTGp2dlbj4+PN1yKRiJLJpFKplFZWVnT9+nXNz89r3759OnXqlAYHB1WpVJTL5RSJRJROp2XbtgqFgv7whz8on88HvkFodb0++OCDAEfpHzIQXAZCUQhM01QqlVIqlVI8HteRI0dkmqZu3bqlfD6/Z2MwODiowcHBXf+/v79fAwMDWltb0+3bt7W2tqa+vj49++yzSqfTcl1XKysrsm1bQ0NDisViKhaLunv37p5voEF4cL1M01QkEpFpmkqn00qn07vWqxeRATIgSdFoVJOTk5qcnNTY2Jh+/vOfy7ZtnT9/Xjdu3NizMXjuuef07LPPNvcA2batmZkZzczMaH5+XufOndPFixc1PT2tn/3sZzpw4IA2Njb03nvvKZVK6dChQxoZGdGnn36qf/zjHyoWi6HYGOysl+M4sixL0WhUU1NTOnDgQHO9erUQkIHgMhCKQrAzQY+NjSkej6uvr0/S/WB8flfvvXv3dOXKlV2vO46j2dlZZTIZxWIx2bYt6f5GIpFIyLZtZbNZXbhwQX19fTp+/HhzGYZhhOJ42IPrNTIyolQqpf7+fsXjcSUSiV3r1YvIABmQ7k+ER48e1YkTJzQ6Oqrp6Wl5nqeBgYE9u3pXVlb0y1/+UqZpNl/r7+/XL37xC7344ovN36EkxWIxjY2NKZVK6fLly3rzzTc1PT2tN954QzMzM5LufzMNQw4eXK/Z2VlNT09rampKmUxGmUxm13r1IjIQXAZCUQjK5bJu376t1dVVpVIpTUxMNI+lWpYl0zTVaDSa3+A+394ikYgajUYQQ0ebkAFIUjab1TvvvKN3331Xk5OTOnXqVPO4bywWUywWU6VSaWbBdd09xTAMh3/w1ZGB4ISiENi2rZGREQ0MDCgej6vRaKhWqymRSOjAgQMaGRnRnTt3Hrsrp1aryXVdVavV5klXjUZD5XJZtm0rFotpZmZGsVhMiURCtVpNlUolNKGJRqPKZDJKpVKKxWJyHEeO48g0zT3r1YvIABmQpFQqpWPHjmlmZkZjY2OqVqsqlUoaHx/Xd7/7Xb3wwguam5vTp59+qlKpJGn3GdSNRkPFYlG5XE75fF7lclmS5LquNjc3lU6nNTg4qLNnz2poaEjj4+MqFAra2toKTaEcHBzUN77xDU1OTmpkZETpdFp9fX1yHEdbW1u71qsXkYHgMhCKQhCJRJqTtGVZWl9fl2EYcl1Xtm0rlUrJsixFIpGHbgwajYa2t7e1srKifD6vQqEg6f63zrW1NZVKJVWrVaVSKRmG0fxluq4bimPH0v3fQTweb+4GajQazQCXy2Vtbm4216sXkQEyIH126Gjfvn1KpVK6ePGiTNPU9va2+vv75TiO4vH4rl3ED6rVarp165b+/e9/69atW1peXpbnecrlcvrggw/0ySefqFgsav/+/YpGo1peXlY2m9XGxoZKpVIoNgi2bWt0dFRPPfWUDMNonv1+7do15XI5Xb16VcvLy0EP0zdkILgMGF6LM6Gfx1V2Lrd63PGbnT/Uo4brOI5s225+s6xWqzJNU9Fo9LGXcZVKpdBsDKLR6CNDvnO5WbVabennddu1x2SgOzIg+ZsDy7LU19enRCLxyOWsr6/Ldd2HTtyGYTS/TVUqFZVKJeXzecViMQ0MDDz0HIydS87W1tZCsQfGtm0NDAwoFos99H3XdZvr1YpumwvIQHAZCEUhQPt12ySA9uvGQoD2Yy5Aqxng1sUAAIBCAAAAKAQAAEAUAgAAIAoBAAAQhQAAACjAGxPtPJb2wQfUSPevtc5ms4E/XAL+IwMAEB6BFYJEIqHx8XGdOXNG0v2NQ61WU6FQ0Llz57S1tRWKG0TAP2QAAMIjkEIwOTmpb33rWzp79qy+/e1vN+8iV6/XVSgU9Otf/1q/+tWvtLCwoFwuF4q7yKG9yAAAhEsghSCZTGr//v06fPiwHMf5bDCWJcuy9P3vf1/Dw8M6d+6c3nnnHa2urgYxTPiIDABAuARyUqHrusrlcrp79+6eb36maWpiYkLPP/+8Dh8+rGQyGcQQ4TMyAADh0vFCYJqm6vW6VldXdeXKlYfuCjZNU6OjoxoaGnrogyjQ3cgAAIRPxw8Z7Nu3Tz/84Q/1yiuv6PTp0498Ch16FxkAgPDpaCGIRCI6dOiQzp49q2PHju06dvygfD6v3/zmNzp//rzW1tY6OUT4jAwAQDgF8tUsGo0+ckMgSdVqVQsLC7p27ZoKhUIHR4ZOIQMAEC4dLQSe5ymXy2lzc1PFYvGRn9u5Hr1SqahSqXRwhPAbGQCAcOp4IVhYWNDi4uJjdwP39fXpxIkTOnjwYAdHh04gAwAQTh0/qbBcLuvvf/+7UqmUEomEMpnMns9EIhElk0lFo9FODw8dQAYAIHw6Xgg8z9Pc3Jw2NzeVz+f12muv7TnLvFAoaGFhQR9//HGnh4cOIAMAED6BnFS4tramxcVFXbx4Uffu3VOj0Wi+12g0tLm5qVu3bml9fT2I4aEDyAAAhEtgF4Bvbm7q6tWrWlpaUqFQULlcVrlcVj6f1+Lioj766CNls9mghocOIAMAEB6G1+JTYwzDaOuCd44RnzhxQq+//rqGh4fVaDSUzWb11ltv6fr168rn87u+OaJ1fjwMiAx0F78eCNXuHMBf3TAXwF+tZiCwQrDzM2OxmIaGhmSapqT7T7tbX19XtVplQ/D/0C2TABnwD4UAUvfMBfBPVxSCHaZpNn++53mq1+u+LetJ0W2TABloPwoBpO6bC9B+rWYgkMcffx6TP8gAAASLp8oAAAAKAQAAoBAAAABRCAAAgCgEAABAFAIAACAKAQAAEIUAAADI50LA3axABgCgO/hWCCKRiBzH2fOcezw5yAAAdA9fbl0cj8eVyWR06tQpvffee8pms7p3757K5bIfi0MIkQEA6C6+FIKhoSEdPnxYR44cUSaT0crKim7fvq1Lly5pY2PDj0UiZMgAAHSXtheCVCql0dFRTU5OamBgQPF4XH19fUqn0yqXy3r//fd9ewobwoEMAED3aXshGBsb0+TkpDKZjCKRiOLxuGKxmBKJhKrVqi5duqRSqcQGoYeRAQDoPm0928uyLB06dEhHjx7V1NRU83XDMJRIJDQ9Pa2TJ0+qv7+/nYtFiJABAOhObSsEtm3rzJkzOn78uMbHx/e8b1mWhoeHdeTIEaXTac4870FkAAC6V1tmZNM0lUqlNDMzo5GREcVisT2fMQxDpmlqaGhIfX19ikaj7Vg0QoIMAEB3a0sh2LnefGhoSPF4XKZpPvKz0WhU0Wj0sZ9B9yEDANDd2lIIGo2GqtWqXNdVvV5/7GcrlYoqlcoXfg6PFsa7/5GBzgpjBgB0t7YdxG00GqpUKqrVao+d6KvVqmq1GmeYf0WmaeprX/uaxsbGHrpbPkhkoDPCnAEA3asthaBer6tQKOivf/2rzp8/rw8//FCu6+76jOd5qlarunDhgtbW1lSpVNqx6CdKNBrV1NSU3nzzTf3kJz/Rc889F/SQmshAZ4Q5AwC6W9vuQ1Cr1XTjxg0VCgVFo1Ht379fw8PDzTPJa7Watre39fHHH6tUKqnRaLRr0U8E0zQ1PDysl156SSdPnlQ8Hpfrurpx44ZyuVzQw5NEBvzWDRkA0L3aemOifD4vSbpz546Wl5cVj8eVSCQkSaVSScvLy9rY2FC1Wm3nYp8I8XhcU1NT+sEPfqCJiQklk0ltbGxofn5e//nPf4IeXhMZ8E+3ZABAd2r7nQrz+bwuXbqku3fv6vTp0zpx4oQcx9Hq6qr+9Kc/aXV1lW+GX8Ezzzyjb37zmzpz5owikYgGBwd18uRJbW9va35+PlQn6JEBf3RTBgB0H18eblQul7W6uqo///nPunnzpmKxmAqFgjY2NtgQPCHIAAB0F18KgXT/JLPt7W1duXJFlmWpVqupWCz6tbie98knn+jixYuam5vTwYMHlclkdPXqVf3zn/8M7QaWDLRXN2YAQPfwrRBI9y9Dy2azfi7iibGxsaHLly/r97//vY4dO6aZmRlduHBBFy5cCPXle2Sgfbo1AwC6g+G1OJNwI5Tw2L9/vw4ePKj19XVdvnz5oZ/xYwNBBsIjqAxI5KDbMBeg1QxQCLrUzt/jUX8+JoHeF0QGHlwuugNzAVrNgK+HDOAfdhGDDABoJ54/CwAAKAQAAIBCAAAARCEAAACiEAAAAFEIAACAKAQAAEDch6CrmaYpy7IUiXzW6zzP46l3TxAyAKBdWi4EiURC5XKZh6iERDqd1tGjR/Xyyy/r6aefbr6+vb2tS5cu+bJMMhAuQWQAQO9quRC8+OKLWlxcVDabVblc9nNMaMELL7yg06dP6/Tp09q3b1/z9Xw+r2eeecaXZZKBcAkiAwB6V8uF4Mc//rHefvttffjhh1peXvZzTPgChmHopZde0ne+8x0dP35c8Xi8+V69XtfBgwd9WS4ZCI+gMgCgd7V8UuGrr76qn/70p/r617/u53jQAs/zdO3aNS0tLWlra2vXe6ZpKpVK+bJcMhAeQWUAQO9quRDYti3btmVZnIcYtGg0uudEsk4gA+ERVAYA9K6WZ/ZSqaRisahKpeLnePAFHMfRj370I33ve9/T7Oys+vv7O7ZsMhAOQWYAQO9quRAsLS1paWlJ6+vrfo4Hj2EYhmzb1pkzZ3Ty5EmNj4/LcZyOLZ8MBC/oDADoXS0Xgrffflu//e1vdf36dT/Hg8cwDEOmaWp6elqDg4Md3xCQgeAFnQEAvavlA5ATExOyLEue5/k5HjyG53mqVqv63//+p0Kh0PHlk4HgBZ0BAL2r5ULw/PPPa2JiQolEws/x4DF2Ngb/+te/lM1mO343OjIQvKAzAKB3tVwIEomEbNvmrOaA1et1XblyRe+++64WFxc7ukEgA+EQZAYA9K6WzyHY2NhQqVRSrVbzczz4Ap7n6ebNm/rd734nwzD01FNPdeyaczIQDkFmAEDvavmr3l/+8hddu3Ztz01Q0HlbW1v629/+prm5OS0sLHRsuWQgPILKAIDe1XIheP/997W5ucmDbUKiWq1qaWlJc3NzWl5ebu42LhaLvm0gyEC4BJEBAL2r5UMG8/Pzyufzfo4FX9LNmzf1xz/+UfV6XU8//bQsy1KhUNDS0pLeeuutti+PDIRPpzMAoHcZXovXkBmG4fdY8BU4jqPZ2VmNjo7KNE2VSiXduXPHl2+IZCCcOpkBiRx0Gz8uEyYD3aXVDFAIehSTAPy6XwQ56C7MBWg1A1w/BgAAKAQAAIBCAAAARCEAAACiEAAAAFEIAACAKAQAAEAUAgAAIAoBAAAQhQAAAIhCAAAARCEAAACiEAAAAFEIAACAKAQAAEAUAgAAIAoBAAAQhQAAAEgyPM/zgh4EAAAIFnsIAAAAhQAAAFAIAACAKAQAAEAUAgAAIAoBAAAQhQAAAIhCAAAARCEAAACS/g96Af1L30md1wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAYLUlEQVR4nO3dW2wT2eEG8G9uvo6d2AnOzUACCQE2AZSF7hYttFBYbWnVPnR3iypV3Ydtpd6kqi/ty0rt41bqW9W+VGr70Faq1O0l7RYEqHQpEc3CJmKBhEBIAiHBJL7Fl7E9Y8//AcV/sknASXxL8v0kXuKx50xymPP5nDPnCKZpmiAiIqJNTax0AYiIiKjyGAiIiIiIgYCIiIgYCIiIiAgMBERERAQGAiIiIgIDAREREYGBgIiIiADIhR4oCEIpy0FFVor1plgH1pdSrTnGerC+8F5AhdaBggNBuQiCAKvVCgDIZrMwDGPBxQiCAIvFAlFcunMjl8vl3yeKIiwWy7KV1zRNpNPpkt04V2Il17XRSZKE2tpaiKKIdDqNRCKBbDabf12WZaiqmq8nn5TJZJBOp5FMJmGxWOByuSDLi6u6aZrIZrOIRCILPr9SVnJdmwHrAesB24PytgdVEQgEQYCiKPl/zc3NEAQB0WgUMzMzSKfT+WMVRUFdXR0cDseCz5BlGRaLBclkMv8+p9OJ5uZmWK1WZLNZpNNpiKIIRVEgCAIMw8C9e/eg63rFK8FKrmsjmr8JOp1OOJ1OHDlyBLIsY3R0FNeuXUM4HM4f63Q6sW/fPjQ2NuZ/JooinE4nVFXF9PQ07t69i4GBATQ1NeHo0aPweDzIZDIIh8MQRREulwuKoiAej+Nvf/sbYrFYxRuDQq/ro48+qmApS4v1gPWA7UHl2oOqCASSJEFVVaiqCrvdjq6uLkiShImJCcRisUUVwOPxwOPxLHh/TU0NamtrMTMzg/v372NmZgZutxu7d++Gy+VCOp3G9PQ0FEWB1+uFzWZDMpnE1NTUotRZCU9flyRJEEURkiTB5XLB5XItuK6NyGq1wu/3w+/3w+fz4bvf/S4URcHZs2cxNja2qCHYu3cvdu/enU/7iqKgo6MDHR0dGBgYwJkzZzA4OIi2tja89dZb2L59O0KhEK5cuQJVVdHZ2Yn6+no8fvwY//nPf5BMJquiIZi/LovFAlmWYbVa0draiu3bt+eva6M2BADrwSevazPWA7YHlWsPqiIQzP9RfD4f7HY73G43gCc3h0927yQSCQwPDy/4ucViwb59+9DQ0ACbzQZFUQA8qRgOhwOKoiAYDKK/vx9utxsHDhzIn0MQhKoYD3v6uurr66GqKmpqamC32+FwOBZc10bkdDrR3d2Nnp4ebNmyBW1tbTBNE7W1tYu6eaenp/Gb3/wGkiTlf1ZTU4Pvfe97eOmll/K/PwCw2Wzw+XxQVRU3b97EO++8g7a2Nvzwhz9ER0cHgCepuxrqwNPXtW/fPrS1taG1tRUNDQ1oaGhYcF0bFesB6wHbg8q1B1URCFKpFO7fv49AIABVVdHc3JwfP5FlGZIkIZfL5VPbJxO8KIrI5XKVKDoVSTAYxPvvv4++vj74/X4cPXo0P+Zrs9lgs9mQyWTy9SCdTi+6CVRDVx+tDesBsT2onKoIBIqioL6+HrW1tbDb7cjlcjAMAw6HA9u3b0d9fT0ePnz4zO48wzCQTqeh63p+okUul0MqlYKiKLDZbOjo6IDNZoPD4YBhGMhkMlVz47BarWhoaICqqrDZbLBYLLBYLJAkadF1bUSqqmL//v3o6OiAz+eDruvQNA2NjY04deoUDh48iPPnz+Px48fQNA3AwpmzuVwOyWQS4XAYsVgMqVQKAJBOpxGJROByueDxeHD69Gl4vV40NjYiHo8jGo1Wzc3D4/HgU5/6FPx+P+rr6+FyueB2u2GxWBCNRhdc10bFesB6wPagcu1BVQQCURTzfxhZljE7OwtBEJBOp6EoClRVhSzLEEVxyQqQy+UwNzeH6elpxGIxxONxAE+S5szMDDRNg67rUFUVgiDk/0Ol0+mqGC8CnvwO7HZ7viswl8vlb2KpVAqRSCR/XRvRfDdhU1MTVFXF4OAgJEnC3NwcampqYLFYYLfbF3QPP80wDExMTODy5cuYmJjA5OQkTNNEOBzGRx99hEePHiGZTKKlpQVWqxWTk5MIBoMIhULQNK0qGgNFUbBlyxZs3boVgiDkZ77fuXMH4XAYt2/fxuTkZKWLWVKsB6wHbA8q1x4IZoFXX8pxlflHLJ41hjf/n3W54losFiiKkk+Tuq5DkiRYrdZnPrqhaVrVVACr1brsjW7+ERNd1wv6vPX27LEsy3C73XA4HMueZ3Z2Ful0esmbtiAI+W9SmUwGmqYhFovBZrOhtrZ2yfG2+cfNZmZmqqL3RVEU1NbWwmazLfl6Op3OX1ch1uM6BKwH66MePD2fwzTNogYptgerbw8EQVjy+gqt11URCKj41lsgoOJbj4GAiq8U9eDtt98G8KQRjcfj+Pe//41QKFQVPSybld1uh8/nw/HjxxeFgl//+tcFfUZVDBkQEdH68aMf/QjAk2+eU1NTGBoaqqp5GJvR/PoNbW1tOHXqFFRVXbY3ZDkMBEREtCLzj2oahgGLxbLs8AaVhyiKcDgc8Pl8aG9vR0dHx6oCATc3IiKiVclms9B1nT0DFWaz2dDY2Iiuri50dXUtuWZDIdhDQEREKzY/EXNkZASJRKIqJuNtVq2trXjzzTfx7W9/G/X19av+HAYCIiJaMU3TcPnyZfziF7/A2NhYVTyhsVnNb97l9XrX9DkcMiAiohWTZRlOpxN1dXXIZDKVLs6mFg6H8fjxYwSDwTV9TsV6COa3onx6UwrgSTdUMBis+AYjRES0PFmWUVdXh9bW1koXZdMLh8OYnp7G1NQU6urqVjyZcF7FAoHD4UBjYyNOnjwJ4ElAMAwD8XgcZ86cQTQaZRcUEVGVkmUZHo8H27Ztq3RRNr25uTk8ePAAIyMj2LlzJ5xO5/qZVOj3+3H8+HGcPn0an/vc5/JpJpvNIh6P409/+hN+//vfY2hoCOFwmJNViIiqTDabRSwWQyAQWPYYURQhCELRVzOkxa5cuZJfrvmNN97IL828EhUJBE6nEy0tLdizZw8sFsv/F0aWIcsyvvjFL6Kurg5nzpzB+++//8wKR0RE5adpGsbGxtDX17fk66IooqurC3a7HYlEAuPj49A0jcPBJaJpGiYnJ9Hb24tXX30VDodj2aWPl1ORSYXpdBrhcBhTU1OLvv1LkoTm5ma8+OKL2LNnD5xOZyWKSEREy4jH47h16xauX7++7EZLDocDO3fuxP79+9HT04Oenh40NDQ8c58KWr35vRiGh4dx7do1PHz4ML9hU6HK3kMgSRKy2SwCgQCGh4fx8ssvL6ockiRhy5Yt8Hq9S25GQkRElTM6OoozZ86gr68PoVBo0euiKKKurg47duzI79rY3t4ORVEwMjKCQCDAJxNKIJvN4vHjx+jt7YWu6+ju7obVai144mfZA0FTUxO+8pWv4LXXXsOxY8dWPRuSiIgq41vf+hbu3buHaDS6aMc9SZKgqiq6u7vh9Xrz21U7HA58/vOfhyzLSCQSSwYJWptcLodQKIQ//OEPuHTpEurr6yGKIi5dulTQ+8saCERRRGdnJ06fPo39+/cvmD/wtFgshj//+c84e/YsZmZmyllEIiJ6jsHBQRiGseREQavVisbGRnz2s5+Fqqr5cWxBEOBwOLB3714ATybBRaPRspZ7s0ilUrh37x7Gx8dX9L6KfD23Wq3LhgEA0HUdQ0NDuHPnDuLxeBlLRkREz5PJZJYMAy6XC9u2bcOhQ4cWhIF5siyjqakJra2tcLvd5SrupmOaJgzDQCaTWdHQTFkDgWmaCIfDiEQiSCaTyx43vybBSi+GiIgqx+PxYPv27di7dy8kSVo0P0wURbjdbtTX18Pj8XByYZUpeyAYGhrCyMjIM4cC3G43enp60N7eXsbSERHRWvh8PuzYsQM+n2/Zxn5+q96WlhYGgipT9kmFqVQKFy9ehKqqcDgcaGhoWHSMKIpwOp2wWq3lLh4REa2SpmnPHeY1TRO6riMWi3HRuSpT9kBgmibOnz+PSCSCWCyGb37zm4ueNIjH4xgaGsKDBw/KXTwiIlqlbDYLwzCg6zokSVryKbL5VQu5NH31qcikwvk9tAcHB5FIJBZMTsnlcohEIpiYmMDs7GwlikdERKsQjUYxMTGBwcFBhMPhJRv9RCKBYDCImZkZ9hBUmYptbhSJRHD79m2Mjo5ix44d+acO0uk0RkZGcO/evTVv5UhEROUzMzODRCKBQCCAU6dOwWq1LthoJ5fLIRwO48GDB+wBrkIVCwThcBhXr17FD37wA3z/+99HXV0dcrkcgsEg3n33Xdy9exexWKxSxSMiohUyDANzc3OIx+MYGxuDx+OBJEmw2+0AnqwxM79NL58gqz6CWWCfTSlmgwqCAJvNBq/Xm39eNZvNYnZ2Frquc3esNShFVxxnBK8vpeqOZT1YXyp1L1BVFS+88AJ2796Nzs5O+Hw+9Pf3o6+vD3fv3kUqlSp6uWhphdaBivUQAE8KqWkaHj16lK9gpmlyNywionUukUjgzp07+TkDTU1NGBoawuzsLNLpdKWLR0uoaA8BlQ57CIg9BARU/l7gcDjg9Xrh9XoRCoUQCoWeuTAdFV+hdYCBYIOq9E2AKo+BgADeC6jwOsCtBomIiIiBgIiIiBgIiIiICAwEREREBAYCIiIiAgMBERERgYGAiIiIUOJAwGdViYiI1oeSBQJRFGGxWJbcD5uIiIiqS0n2MrDb7WhoaMDRo0dx5coVBINBJBIJbmZBRERUpUoSCLxeL/bs2YOuri40NDRgenoa9+/fx40bNxAKhUpxSiIiIlqDogcCVVWxZcsW+P1+1NbWwm63w+12w+VyIZVK4cMPPyzZGutERES0OkUPBD6fD36/Hw0NDRBFEXa7HTabDQ6HA7qu48aNG9A0jaGAiIioihR1xp8sy+js7ER3dzdaW1vzPxcEAQ6HA21tbTh8+DBqamqKeVoiIiJao6IFAkVRcPLkSRw4cACNjY2LXpdlGXV1dejq6oLL5eLTB0RERFWkKK2yJElQVRUdHR2or6+HzWZbdIwgCJAkCV6vF263G1artRinJiIioiIoSiCYX3PA6/XCbrdDkqRlj7VarbBarc88hoiIiMqrKIEgl8tB13Wk02lks9lnHpvJZJDJZJ57HC2PK0ASEVGxFW0gP5fLIZPJwDCMZzb2uq7DMAw+ZbBKkiThhRdegM/nW3JohoiIaDWKEgiy2Szi8TjOnTuHs2fP4uOPP0Y6nV5wjGma0HUd/f39mJmZQSaTKcapNxWr1YrW1la88847ePPNN7F3795KF4mIiDaIoq1DYBgGxsbGEI/HYbVa0dLSgrq6uvzTBIZhYG5uDg8ePICmacjlcsU69aYgSRLq6urwyiuv4PDhw7Db7Uin0xgbG0M4HK508YiIaJ0r6sJEsVgMAPDw4UNMTk7CbrfD4XAAADRNw+TkJEKhEHRdL+ZpNwW73Y7W1lZ8+ctfRnNzM5xOJ0KhEAYGBnD16tVKF4+IiNa5oq9UGIvFcOPGDUxNTeHYsWPo6emBxWJBIBDAP/7xDwQCAfYOrMLOnTvxmc98BidPnoQoivB4PDh8+DDm5uYwMDDASZpERLQmJdncKJVKIRAI4J///CfGx8dhs9kQj8cRCoUYBoiIiKpQSQIB8GSi4dzcHIaHhyHLMgzDQDKZLNXpNrxHjx5hcHAQ58+fR3t7OxoaGnD79m188MEHDFlERLRmJQsEwJNHEYPBYClPsWmEQiHcvHkTf/3rX7F//350dHSgv78f/f39fISTiIjWTDALbE24GE71aGlpQXt7O2ZnZ3Hz5s0ljylFSGAdWF9KFRRZD9YX3guo0DrAQLBOzf89lvvz8SZADAQE8F5AhdeBkg4ZUOlwmICIiIqJexATERERAwERERExEBAREREYCIiIiAgMBERERAQGAiIiIgIDAREREYHrEKxrkiRBlmWI4v/nOtM0ufMhERGtWMGBwOFwIJVKcSOdKuFyudDd3Y0jR45g27Zt+Z/Pzc3hxo0bFSwZERGtRwUHgpdeegkjIyMIBoNIpVKlLBMV4ODBgzh27BiOHTuGpqam/M9jsRh27txZwZIREdF6VHAgeOONN9Db24uPP/4Yk5OTpSwTPYcgCHjllVdw4sQJHDhwAHa7Pf9aNptFe3t7BUtHRETrUcGTCt9++2184xvfwKFDh0pZHiqAaZq4c+cORkdHEY1GF7wmSRJUVa1QyYiIaL0qOBAoigJFUSDLnIdYaVarddFkQiIiorUouHXXNA3JZBKZTKaU5aHnsFgseP311/GFL3wB+/btQ01NTaWLREREG0DBgWB0dBSjo6OYnZ0tZXnoGQRBgKIoOHnyJA4fPozGxkZYLJZKF4uIiCpIFEUoigJJkpBMJlf/OYUe2Nvbi/feew8DAwOrPhmtjSAIkCQJbW1t8Hg8DANERARVVdHU1IT29vY1DSUX/M7m5mbIsgzTNFd9Mlob0zSh6zquX7+OeDxe6eIQEVEVePXVV/HjH/8YP//5z9Ha2gpJklb1OQUHghdffBHNzc1wOByrOhGt3Xwg+O9//4tgMMgVCYloXXO5XNi1axf27duX/9fZ2QmPx8NJ0wUQBAHt7e340pe+hBMnTqCrqwtf//rX4fV6VxUKVrRSoaIo/CNVWDabxfDwMPr6+iBJEnbt2rXqNEhEVAmSJMHlcuHQoUM4ePAgPB5P/rVoNIrBwUHcvHkTk5OTMAyjgiWtbrIs48iRI+ju7kZLSwsEQcDx48dx5swZZDKZRY+lP/fzCj0wFApB0zT+cSrMNE2Mj4/jL3/5CwRBwNatW7nuABGtK1arFX6/H6+//jpee+01+Hy+/Guzs7M4f/48PB4PLly4gEAgwCXzlyAIAiwWC06cOAG/3w+bzQbTNNHT04Ndu3Zhenq6dIHgX//6F+7cubPiE1DxRaNRXLhwAW63GwcOHOBiUUS0rjidTnR3d+NrX/sanE4nBEHIv+b3+/HVr34Vn/70p3HgwAH87Gc/QygU4pfRMii4///DDz9EJBJhUqsSuq5jdHQU58+fx+TkZH4+QTKZxNDQUIVLR0S0vHg8joGBAfz2t7/Fo0ePFr1us9nQ0tKCkydPwuVycVh0CaZpQtM0/OpXv8Lvfvc7fPDBB5iYmEBvby8uX76MQCCw4s8suIdgYGAAsVhsxSeg0hkfH8ff//53ZLNZbNu2DbIsIx6PY3R0FO+++26li0dEtIjNZoPP50N3dze6u7vhdDoXHSMIAmRZhs1mW9B7QAvlcjkMDQ0hlUrh+vXr6OjowODgIGZmZla1iGDBgYAbGlWfYDCIWCwGwzCwZcsWSJIETdPw8OFDBgIiqkoejwednZ04ceLEsoEgl8shlUohGAzCMAw+7v4MwWAQwWAQDx48wK1btzA5OYlEIrGq3xk3JljnMpkMrl69WuliEBEVZPv27Th27BjeeuutZRdXS6fTmJ6exqVLl5BIJDhUXYBAILCqYYKnMRAQEVFZiaL4zI3y4vE4bt68iV/+8peYnZ1lD0GZcFEBIiIqm6mpKdy6dQtTU1PLHqOqKvx+P/bv3885BGXEQEBERGUTjUbx8OFDjI+PI51OL/nt32KxoKamBs3NzRUo4ebFQEBERGWjaRqmp6dx7dq1/KTBT5rfvY9PGZQXAwEREZVNJpPB8PAwfvrTn+K9997D7OzsomPmnzAYHx/n/IEyYiAgIqKyymazmJubw7lz5/D48eMFG7WZpolwOIx79+7h2rVrfMKgjPiUARERlV0ul8Pw8DB6e3sRjUbR1dUFAIhEIrh8+TIuXLiASCRS2UJuMgwERES0IqIorvmbu2mamJqawrlz5xCPxxEKhQA82dzo4sWL+N///gdN04pR3E1ntbsSC2aBAzSc2LG+lGLcjXVgfSnV2CvrwfpSinrgdruhaRqy2eyaP19RFDidTrhcLgCAYRiYm5tDMpnk/IFVkCQJDocDgiDk/68W2tPCHgIiIlqRn/zkJ/jjH/+I27dvr3mPG13XEYlEODxQBJIkYevWrfjOd74Dr9e74k2h2EOwQbGHgNhDQEBp6sHLL7+M0dFRRKNR6Lpe9M+n1REEAU6nE3v37oXFYskHgosXLxb2fgaCjYmBgBgICChNPaitrUUymeTGQ1VIkiSoqrpgHsH8/IznYSDYoBgIiIGAAN4LqPA6wHUIiIiIiIGAiIiIGAiIiIgIDAREREQEBgIiIiLCCp4yICIioo2LPQRERETEQEBEREQMBERERAQGAiIiIgIDAREREYGBgIiIiMBAQERERGAgICIiIjAQEBEREYD/A4/qZpvMttNEAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inmediate reward tensor([0.])\n",
      "episode_reward tensor([0.])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWUklEQVR4nO3d6W8baR0H8O8cHo/tsR07jnM3RzdNCXHaXRouaVfdCy2VWKRyIyHtC4TEIV7AC3gBEvwBvEFoeYNgeQMveEOphIraLWWrRt2oadNq2yY9cjdOnDg+4vsaXkQeNZtj49ZXku9H6ht7HD+T/Drzned55hlB13UdREREdKiJtW4AERER1R4DARERETEQEBEREQMBERERgYGAiIiIwEBAREREYCAgIiIiMBAQERERAHmvGwqCUMl2UJlVYr0p1sD+Uqk1x1gH+wuPBbTXGthzIKgWSZLQ0NAAURSRTqcRj8eRz+eN92VZhqZpMJvN234+k8kgnU4jkUhAURTY7XbI8tbd1HUd+Xwe4XB408+vlVL266BjDbAGgI2TTvF3kc/nkcvlNh3YBEGAoigQxe07OguFgvE5URShKMqOJzJd15FOpysWokpRyn4ddKyB6tZAXQSC4oHQZrPBZrPh5ZdfhizLePz4McbGxhAKhYxtbTYbhoaG0NLSYrwmiiJsNhs0TYPf78ejR49w69YttLa24pVXXoHL5UImk0EoFIIoirDb7TCZTIjFYjh37hzW19drfkLY637dvHmzhq2sHNYAawDYOBCaTCbjX1tbGwRBQCQSwcrKCtLptLGtyWRCY2MjrFbrpp8hyzIURUEikTA+Z7PZ0NbWBrPZjHw+j3Q6DVEUYTKZIAgCcrkcpqamkM1ma35CKGW/DiLWQO1qoC4CgdlsRkdHBzo6OuD1evHjH/8YJpMJ//nPfzA9Pb3lZDAwMIDjx48bSc9kMqGvrw99fX24desWLly4gPHxcfT09OCdd95BV1cX1tbWcP36dWiahv7+fng8HgQCAfzvf/9DIpGoi5NBcb8URYEsyzCbzeju7kZXV5exXwf1ZMAaYA0AG71DmqZB0zRYLBYMDg5CkiTMzs5ifX19y8nA5XLB5XJt+rzT6URDQwNWVlYwNzeHlZUVOBwOHD9+HHa7Hel0Gn6/HyaTCW63G6qqIpFIYHFxccsVaC08vV+SJEEURUiSBLvdDrvdvmm/DiLWQO1qoC4Cgc1mg8/nw0svvYSmpib09PRA13U0NDRs6er1+/34y1/+AkmSjNecTid+8pOf4HOf+xw8Hg80TQMAqKoKr9cLTdNw9+5d/PrXv0ZPTw9+9rOfoa+vD8BG4qqH8bCn92toaAg9PT3o7u5Gc3MzmpubN+3XQcQaYA0AMA7QXq8XFosFDocDwEZg/PjfKB6PY2JiYtPriqJgaGgIzc3NUFUVJpMJwMZJwmq1wmQyIRgMYnR0FA6HAydPnjS+QxCEuqiDp/er+Dd3Op2wWCywWq2b9usgYg3UrgbqIhAEg0H8+9//xsjICDo6OvDKK68Y476qqkJVVWQyGRQKBWOc5+MFUA/dPPTsWAMEAKlUCnNzc1heXoamaWhrazPGUmVZhiRJRg0A2NKrI4oiCoVCLZpOZcIaqJ26CASapuHEiRPo6+uD1+tFNptFMplES0sLzpw5g1OnTuHSpUsIBAJIJpMANs+aLBQKSCQSCIVCWF9fRyqVAgCk02mEw2HY7Xa4XC58+9vfhtvtRktLC2KxGCKRSN0Ujsvlwmc/+1l0dHTA4/HAbrfD4XBAURREIpFN+3UQsQZYA8DG1aHH40FDQwMsFgsKhQJyuRysViu6urrg8Xjw5MmTXYd4crkc0uk0stmsMemqUCgglUrBZDJBVVX09fVBVVVYrVbkcjlkMpm6CZNmsxnNzc3QNA2qqkJRFCiKAkmStuzXQcQaqF0N1EUgKHYRtba2QtM0jI+PQ5IkRKNROJ1OKIoCi8WyqYv4ablcDrOzs7h27RpmZ2exsLAAXdcRCoVw8+ZNLC0tIZFIoL29HWazGQsLCwgGg1hbW0MymayLE4LJZEJTUxM6OzshCIIx+/3hw4cIhUKYnJzEwsJCrZtZMawB1gCwcXVXPEjLsozV1VUIgoB0Og2TyQRN0yDLMkRR3PZkUCgUEI1G4ff7sb6+jlgsBmDjqnNlZQXJZBLZbBaapkEQBCNkpdPpuhg7BjZ+BxaLxRgeKhQKRrBNpVIIh8PGfh1ErIHa1YCg73HvKzmuIssyHA4HrFbrjt+zurqKdDq97YFbEATjaiqTySCZTGJ9fR2qqqKhoWHbsZbiLWcrKyt1kbZNJhMaGhqgquq276fTaWO/9mK/3XvMGtgfNQBg05wOXdfLGqaKt1vtNq+jGOB22j9FUWAymYwry2w2C0mSYDabd72NK5lM1s3JwGw27xh+i7ebZbPZTa8LgrDt/lXkSpI1UFG1qoG6CARUfvstEFD5VerA9v3vfx/AxgE0Fovhv//9L9bW1uqil+Wwslgs8Hq9eO2117acEP70pz+V/ftYA/WnHDVQF0MGRLR//OIXvwCwcdWxuLiI+/fv19VcjMOoeO9+T08Pzpw5A03TdrwSLgfWQP0pRw0wEBBRSYq3a+ZyOSiKsuMQB1WHKIqwWq3wer144YUX0NfXV/FAwBqoL+WqAT7ciIieST6fRzab5VVhjamqipaWFgwODmJwcHDb+/UrpTh/gDVQW+WqAfYQEFHJipMxHzx4gHg8XhcTsQ6r7u5ufPOb38QPf/hDeDyeqn53KpVCKBTatgaenuBW61VAD7py1QADARGVLJlM4tq1a/jDH/6A6enpurhL47AqPsDL7XZX/btTqRTW19fR2dmJmZmZTXXQ1tYGn8+HWCyGBw8eIBqNHvh1NGqlXDXAIQMiKpksy7DZbGhsbEQmk6l1cw61UCiEQCCAYDBY9e/WNA09PT149dVXN80jEEURx44dw6c+9SkMDw/j9ddfx4kTJ+ByuQ70ssu1Uq4aYA8BEZVMlmU0Njaiu7u71k059EKhEPx+PxYXF9HY2FjRyYQfZ7FY0NraioGBAWNtAlEUoWkaent70d7ebmzjdDoRj8cxNze35f55ej6hUAhLS0tYWVlBU1PTM/8cBgIiKpksy3C5XDhy5Eitm3LoRaNRzM/P48GDBzh69ChsNltV1wz5+MJEFosFvb29aGlpgcVigdlshtlshtPpRCwWMxbX4ryT8olGo8bKq8+DgYCISpbP57G+vo7l5eUdtxFFEYIglH0lO9rq+vXrxlK93/jGN4xleSstmUxiaWkJk5OTyOVyEEURHo8Hp0+fRldX16beCkmScOzYMeN5AhMTExVvH5WGcwiIqGTJZBLT09MYGRnZ9n1RFDE4OIhTp05hYGAAmqbtuAwrPb9kMomFhQWcP3++qgsEBQIBjIyM4M9//jNSqRQGBgbwhS98AW1tbdsOXXg8HnR2dqKjo6OqQxu0N+whIKKSxGIx3Lt3D3fu3NnxYUtWqxVHjx5FU1MTUqkU3G43Hj16hHA4XDfrxR8kxXX4JyYmMDY2hhdffBHNzc2QJGnTcwfKrXhSDwaDMJvN6OzsxNGjR42H8nycoihwuVxwu90wm83Gk0vp+a2vrxsPdXvW3iEGAiIqyePHj3HhwgWMjIxsO2YpiiIaGxvR29trPLnxhRdegMlkwoMHD7C8vMw7Eyogn88jEAjg/PnzyGaz8Pl8MJvNFZ34qWkajhw5AqfTiUwmg6amJni9XiiKsuNnLBYL7HY7VFVlICijQCCA27dv4+zZs7v+/nfDQEBEJfnBD36AqakpRCKRLbPFJUmCpmnw+Xxwu93GI6utViu+/OUvQ5ZlxOPx5578RFsVCgWsra3hb3/7G65evQqPxwNRFHH16tWKfafdbkd7ezs8Hg/m5uaQTqc/8Q6CTCaDVCrFOw3KLBAIYHx8HIFAAC0tLc/UM8RBHCIqyfj4ONbW1rY9oJvNZrS0tOD06dNobGw05g0IggCr1YqBgQEMDw/D6XRWu9mHRiqVwtTUFG7cuIHR0dGKf1c4HEYsFkM+nzcey7ubhoYGeL1e1kCZhcNh3LhxAz/96U8xMTGBRCJR8s9gICCikmQymW0nrdntdhw5cgTDw8PbTiKUZRmtra3o7u6Gw+GoVnMPHV3XkcvlkMlkKj40Ew6H8ejRIywvLyMej2NhYWHHeSVFoihCURQuUFRmuVwOkUgEo6OjmJiYeKZeOAYCIioLl8uFrq4uDAwMQJKkLRObRFGEw+GAx+OBy+Wq6r3yVF6FQgHRaBQzMzO4ffs2wuEwUqkUotEoIpHIrp8VBAGyLMPhcLAGykjXdWSzWSwuLmJ8fBxzc3OIRqMl3XHCQEBEZeH1etHb2wuv17vjgb74mNb29naeDPaxaDSKO3fu4MqVK7h8+bJx14jVat3xDoMi1kBl6bqOS5cu4fLly7hz5w6i0eieP8tJhURUFslkErFYbNdtilcxXKlufzt79iyCwSCWl5cRCASM11kD9WF0dBSzs7P4xz/+gcbGRly+fHlPn2MgIKKyKE4oy2azkCRp24VniqsW8umI+9vY2Jjxt376pM4aqA+6riMYDBrDOnvFIQMiKotIJILZ2VmMj48jFApte8CPx+MIBoNYWVnh1eE+Fo1GkUgkttxpwhqoH9lsFolEgkMGRFR9KysriMfjWF5expkzZ2A2mzc9aKdQKCAUCmF+fh7z8/M1bi1VAmtgf2MgIKKyyOVyiEajiMVimJ6ehsvlgiRJsFgsADaWVi0+ppcrFR5MrIH9TdD32GfD2aD7SyW64lgD+0ulumP3UgeapuHTn/40jh8/jv7+fni9XoyOjmJkZASPHj1CKpWqSNtoq1odC1gD9WOvNcAeAiIqu3g8jocPHxrjxa2trbh//z5WV1eRTqdr3TyqAtbA/sMeggOKPQRUyx6CIqvVCrfbDbfbjbW1NaytrT3Tkqr07Gp9LGAN1N5ea4CB4ICq9UGAaq8eAgHVHo8FtNca4G2HRERExEBAREREDAREREQEBgIiIiICAwERERGBgYCIiIjAQEBERERgICAiIiIwEBAREREYCIiIiAgMBERERAQGAiIiIgIDAREREYGBgIiIiMBAQERERGAgICIiIjAQEBERERgIiIiICAwEREREBAYCIiIiAgMBERERgYGAiIiIwEBAREREYCAgIiIiMBAQERERGAiIiIgIDAREREQEBgIiIiICAwERERGBgYCIiIjAQEBERERgICAiIiIwEBAREREYCIiIiAgMBERERAQGAiIiIgIDAREREYGBgIiIiMBAQERERGAgICIiIjAQEBEREQC51g0gIqLDR5Ik9PX1wWazQZIk4/XZ2VlEIhGkUqkatu5wYiAgIqKqEUURDocD/f39+M53voPGxkaYzWbj/Q8++AA3btzAw4cPEQwGa9jSw4eBgIhoHxNFESaTCZIkIZFI1Lo5n0gURTQ2NuL06dP4yle+gqamJiiKYrzvdrtht9thsVgwPj6OUChUw9buD+WqAQYCIqJ9TNM0uN1uOBwOfPTRRygUCrVu0q5kWUZrayvefvttHDlyBLK8+TT0+uuvo7e3F0NDQ3jvvfdw8eLFut+nWitXDXBSIRHRPvalL30Jv/zlL/G73/0O3d3dm8bj61E2m8Xjx4/x+9//HlNTU9vOFejp6cHw8DB8Ph8EQahBK/eXctUAewiIqCbsdjtaW1uhqqrxWjqdRiAQQCQS4VXhJxAEAUePHsXbb7+NL37xi7DZbPje976Hd999F2tra8jn87Vu4haSJKGtrQ0nT57E2bNn0dzcDJPJtO22giAwDHyCctcAAwERVZUkSbDb7RgeHsapU6fgcrmM9yKRCMbHx3H37l0sLCwgl8vVsKX1TZZlvPzyy/D5fGhvb4cgCHjttddw4cIFZDIZRCKRWjdxC7PZjJMnT+LNN9/EZz7zmS13GBTl83kkEgmEw2Houl6Dlu4P5a4BBgIiqiqz2YyOjg58/etfx1tvvQWv12u8t7q6ikuXLsHlcuH999/H8vIyewq2IQgCFEXBG2+8gY6ODqiqCl3X8dJLL+HYsWPw+/11GQhUVcXw8DDeeust9Pb27tgDkMlkkEwmEY1GIYoia2AblagBziEgoqqy2Wzw+Xz47ne/i87OTqiqavzr6OjAt771LfzqV7/Cz3/+c3g8ni2Tzmh/EwQBkiR94nCAw+HA8PAw3G43a6BKGAiIqKpisRhu3bqF9957D0tLS1veV1UV7e3tePPNN2G32+t+klwt6LqOZDKJP/7xj/jrX/+KDz74ALOzszh//jyuXbuG5eXlWjdxW7FYDLdv38bo6Oiu27EGPtnTNfCvf/0Lk5OTyOVymJ6exs2bNxEIBEr+mYxdRFQ1qqrC6/XC5/PB5/PBZrNt2UYQBMiyDFVVOalsF4VCAffv30cqlcKdO3fQ19eH8fFxrKysIJPJ1Lp528pmswiFQlhdXd11O9bA3hQKBUxOTmJmZgYDAwMQRRF2ux1erxd+v7/k1R4ZCIioalwuF/r7+/HGG2/sGAgKhQJSqRSCwSByuRwnle0iGAwiGAxifn4e9+7dw8LCAuLxeN3+znRdRyqVQjqd3nU71sDeZbNZKIoCRVEgiiJcLheGhoaMOQSl3GnAQEBEVdPV1YVXX30V77zzzqbV6Z6WTqfh9/tx9epVxONxTijbg+Xl5bodJngWrIG9EQQB/f39OH36NAYGBiAIApxOJ7761a9ibm4Oc3NzJa1cyEBARFUliuKuk8RisRju3r2Ld999F6urq7w6PGDy+Tzy+Tx0Xd9xOIA1UBucVEhEVbO4uIh79+5hcXFxx200TUNHRwdOnDjB8eMDqFgDT5482XEb1sDe6LqOyclJnDt3DufOncPo6ChCoRD++c9/GvNLSsEeAiKqmkgkgidPnmBmZsZ4qM3HD/iKosDpdKKtra1GraRKYg2UVzQaxZUrVzA/P4/u7m68+OKLuH79OpaWlkoeamEgIKKqSSaT8Pv9GBsbQ29vL5qamrYsXVt8chtnmB9MrIHyKhQKuHHjBj766CO43W7cvHkTd+/efaanHjIQEFHVZDIZTExM4Le//S0kScLXvvY1tLa2btqmOLt8ZmaGY8cHEGugMlKpFBYXF3cdjvsknENARFWVz+cRjUZx8eJFBAKBTbdF6bqOUCiEqakpjI2NcXb5AcUaqE/sISCiqisUCpiYmMD58+cRiUQwODgIAAiHw7h27Rref/99hMPh2jaSKoo1UH8YCIioJOV42Iyu61hcXMTFixcRi8WwtrYGYOPhRleuXMGHH36IZDJZjuYeOqJY+Y5f1kB9e9YaEPQ9DtBwYsf+UolxN9bA/lKpsVeHw4FkMmncS/48TCYTbDYb7HY7ACCXyyEajSKRSHDs+BlIkgSr1QpBEIz/r5W4ymYN1K/nqQH2EBBRSX7zm9/g73//OyYnJ7G+vv5cPyubzSIcDrNruAwkSUJnZyd+9KMfwe12V/SBQKyB+vS8NcAeggOKPQRUqaurz3/+83j8+DEikQiy2WxFvoNKJwgCbDYbBgYGoCiKcTK4cuVK2b+LNVCfnrcGGAgOKAYCqlQgaGhoQCKR4ENn6pAkSdA0bdMYcnFsvpxYA/XreWqAgeCAYiCgSh2oWQf7C48FtNca4DoERERExEBAREREDAREREQEBgIiIiICAwERERGhhLsMiIiI6OBiDwERERExEBAREREDAREREYGBgIiIiMBAQERERGAgICIiIjAQEBERERgIiIiICAwEREREBOD/MBhrPC4dSTwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the first 4 frames of the first rollout\n",
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(12, 12))\n",
    "\n",
    "for j in range(10):\n",
    "    print(\"inmediate reward\", data[\"next\"][\"reward\"][j])\n",
    "    print(\"episode_reward\", data[\"episode_reward\"][j])\n",
    "    for i in range(4):\n",
    "        plt.subplot(1, 4, i+1)\n",
    "        plt.imshow(data[\"pixels\"][j, i].cpu().numpy(), cmap=\"gray\")\n",
    "        plt.axis(\"off\")\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = make_env(env_name=\"LunarLander-v2\", \n",
    "               frame_skip = 4, \n",
    "               device=\"cpu\", #\n",
    "               seed = 0,  \n",
    "               is_test=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "env = make_env(env_name=\"CarRacing-v2\", \n",
    "               frame_skip = 4, \n",
    "               device=\"cpu\", #\n",
    "               seed = 0,  \n",
    "               is_test=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = make_env(env_name=\"Acrobot-v1\", \n",
    "               frame_skip = 4, \n",
    "               device=\"cpu\", #\n",
    "               seed = 0,  \n",
    "               is_test=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "env = make_env(env_name=\"CartPole-v1\", \n",
    "               frame_skip = 4, \n",
    "               device=\"cpu\", #\n",
    "               seed = 0,  \n",
    "               is_test=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acrobot-v1',\n",
       " 'Ant-v2',\n",
       " 'Ant-v3',\n",
       " 'Ant-v4',\n",
       " 'BabyAI-ActionObjDoor-v0',\n",
       " 'BabyAI-BlockedUnlockPickup-v0',\n",
       " 'BabyAI-BossLevel-v0',\n",
       " 'BabyAI-BossLevelNoUnlock-v0',\n",
       " 'BabyAI-FindObjS5-v0',\n",
       " 'BabyAI-FindObjS6-v0',\n",
       " 'BabyAI-FindObjS7-v0',\n",
       " 'BabyAI-GoTo-v0',\n",
       " 'BabyAI-GoToDoor-v0',\n",
       " 'BabyAI-GoToImpUnlock-v0',\n",
       " 'BabyAI-GoToLocal-v0',\n",
       " 'BabyAI-GoToLocalS5N2-v0',\n",
       " 'BabyAI-GoToLocalS6N2-v0',\n",
       " 'BabyAI-GoToLocalS6N3-v0',\n",
       " 'BabyAI-GoToLocalS6N4-v0',\n",
       " 'BabyAI-GoToLocalS7N4-v0',\n",
       " 'BabyAI-GoToLocalS7N5-v0',\n",
       " 'BabyAI-GoToLocalS8N2-v0',\n",
       " 'BabyAI-GoToLocalS8N3-v0',\n",
       " 'BabyAI-GoToLocalS8N4-v0',\n",
       " 'BabyAI-GoToLocalS8N5-v0',\n",
       " 'BabyAI-GoToLocalS8N6-v0',\n",
       " 'BabyAI-GoToLocalS8N7-v0',\n",
       " 'BabyAI-GoToObj-v0',\n",
       " 'BabyAI-GoToObjDoor-v0',\n",
       " 'BabyAI-GoToObjMaze-v0',\n",
       " 'BabyAI-GoToObjMazeOpen-v0',\n",
       " 'BabyAI-GoToObjMazeS4-v0',\n",
       " 'BabyAI-GoToObjMazeS4R2-v0',\n",
       " 'BabyAI-GoToObjMazeS5-v0',\n",
       " 'BabyAI-GoToObjMazeS6-v0',\n",
       " 'BabyAI-GoToObjMazeS7-v0',\n",
       " 'BabyAI-GoToObjS4-v0',\n",
       " 'BabyAI-GoToObjS6-v0',\n",
       " 'BabyAI-GoToOpen-v0',\n",
       " 'BabyAI-GoToRedBall-v0',\n",
       " 'BabyAI-GoToRedBallGrey-v0',\n",
       " 'BabyAI-GoToRedBallNoDists-v0',\n",
       " 'BabyAI-GoToRedBlueBall-v0',\n",
       " 'BabyAI-GoToSeq-v0',\n",
       " 'BabyAI-GoToSeqS5R2-v0',\n",
       " 'BabyAI-KeyCorridor-v0',\n",
       " 'BabyAI-KeyCorridorS3R1-v0',\n",
       " 'BabyAI-KeyCorridorS3R2-v0',\n",
       " 'BabyAI-KeyCorridorS3R3-v0',\n",
       " 'BabyAI-KeyCorridorS4R3-v0',\n",
       " 'BabyAI-KeyCorridorS5R3-v0',\n",
       " 'BabyAI-KeyCorridorS6R3-v0',\n",
       " 'BabyAI-KeyInBox-v0',\n",
       " 'BabyAI-MiniBossLevel-v0',\n",
       " 'BabyAI-MoveTwoAcrossS5N2-v0',\n",
       " 'BabyAI-MoveTwoAcrossS8N9-v0',\n",
       " 'BabyAI-OneRoomS12-v0',\n",
       " 'BabyAI-OneRoomS16-v0',\n",
       " 'BabyAI-OneRoomS20-v0',\n",
       " 'BabyAI-OneRoomS8-v0',\n",
       " 'BabyAI-Open-v0',\n",
       " 'BabyAI-OpenDoor-v0',\n",
       " 'BabyAI-OpenDoorColor-v0',\n",
       " 'BabyAI-OpenDoorDebug-v0',\n",
       " 'BabyAI-OpenDoorLoc-v0',\n",
       " 'BabyAI-OpenDoorsOrderN2-v0',\n",
       " 'BabyAI-OpenDoorsOrderN2Debug-v0',\n",
       " 'BabyAI-OpenDoorsOrderN4-v0',\n",
       " 'BabyAI-OpenDoorsOrderN4Debug-v0',\n",
       " 'BabyAI-OpenRedBlueDoors-v0',\n",
       " 'BabyAI-OpenRedBlueDoorsDebug-v0',\n",
       " 'BabyAI-OpenRedDoor-v0',\n",
       " 'BabyAI-OpenTwoDoors-v0',\n",
       " 'BabyAI-Pickup-v0',\n",
       " 'BabyAI-PickupAbove-v0',\n",
       " 'BabyAI-PickupDist-v0',\n",
       " 'BabyAI-PickupDistDebug-v0',\n",
       " 'BabyAI-PickupLoc-v0',\n",
       " 'BabyAI-PutNextLocal-v0',\n",
       " 'BabyAI-PutNextLocalS5N3-v0',\n",
       " 'BabyAI-PutNextLocalS6N4-v0',\n",
       " 'BabyAI-PutNextS4N1-v0',\n",
       " 'BabyAI-PutNextS5N1-v0',\n",
       " 'BabyAI-PutNextS5N2-v0',\n",
       " 'BabyAI-PutNextS5N2Carrying-v0',\n",
       " 'BabyAI-PutNextS6N3-v0',\n",
       " 'BabyAI-PutNextS6N3Carrying-v0',\n",
       " 'BabyAI-PutNextS7N4-v0',\n",
       " 'BabyAI-PutNextS7N4Carrying-v0',\n",
       " 'BabyAI-Synth-v0',\n",
       " 'BabyAI-SynthLoc-v0',\n",
       " 'BabyAI-SynthS5R2-v0',\n",
       " 'BabyAI-SynthSeq-v0',\n",
       " 'BabyAI-UnblockPickup-v0',\n",
       " 'BabyAI-Unlock-v0',\n",
       " 'BabyAI-UnlockLocal-v0',\n",
       " 'BabyAI-UnlockLocalDist-v0',\n",
       " 'BabyAI-UnlockPickup-v0',\n",
       " 'BabyAI-UnlockPickupDist-v0',\n",
       " 'BabyAI-UnlockToUnlock-v0',\n",
       " 'BipedalWalker-v3',\n",
       " 'BipedalWalkerHardcore-v3',\n",
       " 'Blackjack-v1',\n",
       " 'CarRacing-v2',\n",
       " 'CartPole-v0',\n",
       " 'CartPole-v1',\n",
       " 'CliffWalking-v0',\n",
       " 'FrozenLake-v1',\n",
       " 'FrozenLake8x8-v1',\n",
       " 'GymV21Environment-v0',\n",
       " 'GymV26Environment-v0',\n",
       " 'HalfCheetah-v2',\n",
       " 'HalfCheetah-v3',\n",
       " 'HalfCheetah-v4',\n",
       " 'Hopper-v2',\n",
       " 'Hopper-v3',\n",
       " 'Hopper-v4',\n",
       " 'Humanoid-v2',\n",
       " 'Humanoid-v3',\n",
       " 'Humanoid-v4',\n",
       " 'HumanoidStandup-v2',\n",
       " 'HumanoidStandup-v4',\n",
       " 'InvertedDoublePendulum-v2',\n",
       " 'InvertedDoublePendulum-v4',\n",
       " 'InvertedPendulum-v2',\n",
       " 'InvertedPendulum-v4',\n",
       " 'LunarLander-v2',\n",
       " 'LunarLanderContinuous-v2',\n",
       " 'MiniGrid-BlockedUnlockPickup-v0',\n",
       " 'MiniGrid-DistShift1-v0',\n",
       " 'MiniGrid-DistShift2-v0',\n",
       " 'MiniGrid-DoorKey-16x16-v0',\n",
       " 'MiniGrid-DoorKey-5x5-v0',\n",
       " 'MiniGrid-DoorKey-6x6-v0',\n",
       " 'MiniGrid-DoorKey-8x8-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-16x16-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-5x5-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-6x6-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-8x8-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-Random-5x5-v0',\n",
       " 'MiniGrid-Dynamic-Obstacles-Random-6x6-v0',\n",
       " 'MiniGrid-Empty-16x16-v0',\n",
       " 'MiniGrid-Empty-5x5-v0',\n",
       " 'MiniGrid-Empty-6x6-v0',\n",
       " 'MiniGrid-Empty-8x8-v0',\n",
       " 'MiniGrid-Empty-Random-5x5-v0',\n",
       " 'MiniGrid-Empty-Random-6x6-v0',\n",
       " 'MiniGrid-Fetch-5x5-N2-v0',\n",
       " 'MiniGrid-Fetch-6x6-N2-v0',\n",
       " 'MiniGrid-Fetch-8x8-N3-v0',\n",
       " 'MiniGrid-FourRooms-v0',\n",
       " 'MiniGrid-GoToDoor-5x5-v0',\n",
       " 'MiniGrid-GoToDoor-6x6-v0',\n",
       " 'MiniGrid-GoToDoor-8x8-v0',\n",
       " 'MiniGrid-GoToObject-6x6-N2-v0',\n",
       " 'MiniGrid-GoToObject-8x8-N2-v0',\n",
       " 'MiniGrid-KeyCorridorS3R1-v0',\n",
       " 'MiniGrid-KeyCorridorS3R2-v0',\n",
       " 'MiniGrid-KeyCorridorS3R3-v0',\n",
       " 'MiniGrid-KeyCorridorS4R3-v0',\n",
       " 'MiniGrid-KeyCorridorS5R3-v0',\n",
       " 'MiniGrid-KeyCorridorS6R3-v0',\n",
       " 'MiniGrid-LavaCrossingS11N5-v0',\n",
       " 'MiniGrid-LavaCrossingS9N1-v0',\n",
       " 'MiniGrid-LavaCrossingS9N2-v0',\n",
       " 'MiniGrid-LavaCrossingS9N3-v0',\n",
       " 'MiniGrid-LavaGapS5-v0',\n",
       " 'MiniGrid-LavaGapS6-v0',\n",
       " 'MiniGrid-LavaGapS7-v0',\n",
       " 'MiniGrid-LockedRoom-v0',\n",
       " 'MiniGrid-MemoryS11-v0',\n",
       " 'MiniGrid-MemoryS13-v0',\n",
       " 'MiniGrid-MemoryS13Random-v0',\n",
       " 'MiniGrid-MemoryS17Random-v0',\n",
       " 'MiniGrid-MemoryS7-v0',\n",
       " 'MiniGrid-MemoryS9-v0',\n",
       " 'MiniGrid-MultiRoom-N2-S4-v0',\n",
       " 'MiniGrid-MultiRoom-N4-S5-v0',\n",
       " 'MiniGrid-MultiRoom-N6-v0',\n",
       " 'MiniGrid-ObstructedMaze-1Dl-v0',\n",
       " 'MiniGrid-ObstructedMaze-1Dlh-v0',\n",
       " 'MiniGrid-ObstructedMaze-1Dlhb-v0',\n",
       " 'MiniGrid-ObstructedMaze-1Q-v0',\n",
       " 'MiniGrid-ObstructedMaze-1Q-v1',\n",
       " 'MiniGrid-ObstructedMaze-2Dl-v0',\n",
       " 'MiniGrid-ObstructedMaze-2Dlh-v0',\n",
       " 'MiniGrid-ObstructedMaze-2Dlhb-v0',\n",
       " 'MiniGrid-ObstructedMaze-2Dlhb-v1',\n",
       " 'MiniGrid-ObstructedMaze-2Q-v0',\n",
       " 'MiniGrid-ObstructedMaze-2Q-v1',\n",
       " 'MiniGrid-ObstructedMaze-Full-v0',\n",
       " 'MiniGrid-ObstructedMaze-Full-v1',\n",
       " 'MiniGrid-Playground-v0',\n",
       " 'MiniGrid-PutNear-6x6-N2-v0',\n",
       " 'MiniGrid-PutNear-8x8-N3-v0',\n",
       " 'MiniGrid-RedBlueDoors-6x6-v0',\n",
       " 'MiniGrid-RedBlueDoors-8x8-v0',\n",
       " 'MiniGrid-SimpleCrossingS11N5-v0',\n",
       " 'MiniGrid-SimpleCrossingS9N1-v0',\n",
       " 'MiniGrid-SimpleCrossingS9N2-v0',\n",
       " 'MiniGrid-SimpleCrossingS9N3-v0',\n",
       " 'MiniGrid-Unlock-v0',\n",
       " 'MiniGrid-UnlockPickup-v0',\n",
       " 'MountainCar-v0',\n",
       " 'MountainCarContinuous-v0',\n",
       " 'Pendulum-v1',\n",
       " 'Pusher-v2',\n",
       " 'Pusher-v4',\n",
       " 'Reacher-v2',\n",
       " 'Reacher-v4',\n",
       " 'Swimmer-v2',\n",
       " 'Swimmer-v3',\n",
       " 'Swimmer-v4',\n",
       " 'Taxi-v3',\n",
       " 'Walker2d-v2',\n",
       " 'Walker2d-v3',\n",
       " 'Walker2d-v4',\n",
       " 'phys2d/CartPole-v0',\n",
       " 'phys2d/CartPole-v1',\n",
       " 'phys2d/Pendulum-v0',\n",
       " 'tabular/Blackjack-v0',\n",
       " 'tabular/CliffWalking-v0']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.available_envs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ale_py import ALEInterface\n",
    "ale = ALEInterface()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A.L.E: Arcade Learning Environment (version 0.10.1+unknown)\n",
      "[Powered by Stella]\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "import ale_py\n",
    "\n",
    "gym.register_envs(ale_py)\n",
    "\n",
    "env = gym.make('ALE/Breakout-v5')\n",
    "obs, info = env.reset()\n",
    "obs, reward, terminated, truncated, info = env.step(env.action_space.sample())\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 False False {'lives': 5, 'episode_frame_number': 4, 'frame_number': 4}\n"
     ]
    }
   ],
   "source": [
    "# print reward, terminated, truncated, info\n",
    "print(reward, terminated, truncated, info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(210, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAGhCAYAAADY5IdbAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAlpklEQVR4nO3df3RU9Z3/8ddMfgwg+UGAZDIaflbBKlBAzebUKpQsJHhQK7srFM9il0OKDfRI2q2bc5RfZ8+Gatf1qCzunrVQT0UsXcWV3WWXH5LUJUQBkWpplrDRoGRCCyZDApn8mM/3j36ZdpoESD5zMxnyfJzzOSdzP5/7ue+5JC/u3HtnxmWMMQIA9Ik71gUAQDwjRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcBCTEN006ZNGjdunIYMGaLc3Fy99957sSwHAHotZiH6+uuvq6SkRGvXrtXRo0c1bdo0zZs3T2fPno1VSQDQa65YfQBJbm6u7rzzTr344ouSpFAopJycHK1atUp/8zd/c8V1Q6GQzpw5o5SUFLlcrv4oF8AgY4zRhQsX5PP55Hb3fLyZ2I81hbW1tenIkSMqLS0NL3O73crPz1dlZWWX8cFgUMFgMPz4888/15e//OV+qRXA4Hb69GnddNNNPfbH5OX8b3/7W3V2diorKytieVZWlvx+f5fxZWVlSktLCzcCFEB/SUlJuWJ/XFydLy0tVVNTU7idPn061iUBGCSudsowJi/nR40apYSEBDU0NEQsb2hokNfr7TLe4/HI4/H0V3kAcM1iciSanJysmTNnat++feFloVBI+/btU15eXixKAoA+icmRqCSVlJRo6dKluuOOO3TXXXfpueeeU0tLi771rW/FqiQA6LWYhejDDz+s3/zmN1qzZo38fr++8pWvaPfu3V0uNgHAQBaz+0RtBAIBpaWlxbqMmMnIyFB6enpU52xqatK5c+e67Rs+fLgyMzOjur1Lly6pvr6+2z6PxyOfzxfVe4A7Ojr0+eefq7OzM2pz2vB6vRo2bFhU5/zNb36jCxcuRHVOJ9xwww09HixdvHix2zt0YqmpqUmpqak99sfsSBR9l5eXp3vvvTeqcx48eFA7d+7stm/SpEl6+OGHo7q9U6dO6V/+5V+6DbXMzEwtW7ZMycnJUdteY2OjXnzxRQUCgajN2Vdut1v33XefJk2aFNV5//Vf/1VVVVVRndMJEyZM0COPPNLtf5InTpzQ1q1bFU/HdoRoHHK73UpMjO4/3ZXekeFyuZSQkBDVI8OrbS8xMTGqzzHa9dtKSEjo13/DgeTy7293/x4JCQkxqMgOIXqdudr/4NEOkoG2PSe22d/i6SgMhOh15/jx4zp+/Hi3fbfddptmzJgR1e3V1dWpoqKi274bb7xRs2bNiuoRUmNjo/77v/9bbW1tXfpSU1M1b948DRkyJGrb62/GGFVUVKiurq7X6/ZlHdgjRK8z9fX1+uCDD7rtS09Pj3qIfvHFFz1ur7W1VbNmzYrq9i5duqQPP/xQra2tXfpGjRqlOXPmRHV7sfB///d/+uUvfxnrMnCN4uMkCgAMUByJAgPMqFGjlJOT0+v1zp8/r5aWFgcqwpUQosAAU1BQoFAo1Ov13nzzTb4dIgYIUWAAcblcSkpK6tO68Xh70PWAEAVipC+3MsX77VvXI0IU6GehUEjl5eX68MMPe71ubm6uxo0bF/2i0GeEKBAD1dXVfVpv4sSJhOgAwy1OAGCBI9HrzPDhw7v9dgDp6t8V0xdDhw5VdnZ2t+f3RowYEfXtJSUlKSsrK+KLC/9we/Hy/vERI0b06dsahg4d6kA1sEGIXmdyc3M1c+bMbvui/YEXkvSlL31JK1eu7LbP7XZH/ULIyJEjVVRU1G2fy+WKi6+Rcbvduv/++3XLLbf0et2+XrmHcwjR60xSUlK//qElJCT069GR2+2+Lo7GPB7PdfE8wDlRALDCkWgc+uijj9TY2BjVOc+cOdNjX11dXY8f2NxXjY2NPb4r54svvtDbb78d1fObwWBQly5ditp8NkKhkA4ePKgTJ05Edd7a2tqozueUzz//vMffp/Pnz8fdRwHy9SAAcAVX+3oQXs4DgIW4fjmfkZERN7e0AIgvoVBI58+fv+q4uA7RFStWxPWnmAMYuFpbW/V3f/d3Vx0X1yE6fPhwQhSAI671vmpeCwOABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALEQ9RMvKynTnnXcqJSVFmZmZevDBB7t8s+GsWbPkcrki2ooVK6JdCgA4LuohWl5eruLiYh06dEh79uxRe3u75s6dq5aWlohxy5cvV319fbg9/fTT0S4FABwX9Q8g2b17d8TjrVu3KjMzU0eOHNE999wTXj5s2LAev5USAOKF4+dEm5qaJP3usz//0KuvvqpRo0bp9ttvV2lpqS5evNjjHMFgUIFAIKIBwEDg6EfhhUIhPf744/rqV7+q22+/Pbz8m9/8psaOHSufz6fjx4/riSeeUHV1td54441u5ykrK9P69eudLBUA+sTREC0uLtZHH32kd999N2L5H35v+JQpU5Sdna05c+bo1KlTmjhxYpd5SktLVVJSEn4cCASUk5PjXOEAcI0cC9GVK1dq165dqqio0E033XTFsbm5uZKkmpqabkPU4/HI4/E4UicA2Ih6iBpjtGrVKr355ps6cOCAxo8ff9V1jh07JknKzs6OdjkA4Kioh2hxcbG2bdumt956SykpKfL7/ZKktLQ0DR06VKdOndK2bds0f/58jRw5UsePH9fq1at1zz33aOrUqdEuBwAcFfUQ3bx5s6Tf3VD/h7Zs2aJHH31UycnJ2rt3r5577jm1tLQoJydHCxcu1JNPPhntUgDAcY68nL+SnJwclZeXR3uzABATvHceACwQogBgIa6/d74vrna6AcD1x+VyOTb3oArRtrY27d+/P/xWVADXv7S0NH39619XcnKyI/MPqhDt6OjQhx9+qIaGhliXAqCfZGdn695773Vsfs6JAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC1EP0XXr1snlckW0yZMnh/tbW1tVXFyskSNHavjw4Vq4cKEaGhqiXQYA9AtHjkRvu+021dfXh9u7774b7lu9erXefvtt7dixQ+Xl5Tpz5oweeughJ8oAAMclOjJpYqK8Xm+X5U1NTXr55Ze1bds2ff3rX5ckbdmyRbfeeqsOHTqkP/mTP3GiHABwjCNHoidPnpTP59OECRO0ZMkS1dXVSZKOHDmi9vZ25efnh8dOnjxZY8aMUWVlZY/zBYNBBQKBiAYAA0HUQzQ3N1dbt27V7t27tXnzZtXW1uprX/uaLly4IL/fr+TkZKWnp0esk5WVJb/f3+OcZWVlSktLC7ecnJxolw0AfRL1l/OFhYXhn6dOnarc3FyNHTtWP/vZzzR06NA+zVlaWqqSkpLw40AgQJACGBAcv8UpPT1dt9xyi2pqauT1etXW1qbGxsaIMQ0NDd2eQ73M4/EoNTU1ogHAQOB4iDY3N+vUqVPKzs7WzJkzlZSUpH379oX7q6urVVdXp7y8PKdLAYCoi/rL+e9///tasGCBxo4dqzNnzmjt2rVKSEjQ4sWLlZaWpmXLlqmkpEQZGRlKTU3VqlWrlJeXx5V5AHEp6iH62WefafHixTp37pxGjx6tu+++W4cOHdLo0aMlSf/wD/8gt9uthQsXKhgMat68efrHf/zHaJcBAP0i6iG6ffv2K/YPGTJEmzZt0qZNm6K9aQDod7x3HgAsEKIAYIEQBQALjrx3fqAakpCgpRMmqH3EiFiXAqCfJGVkyJOQ4Nj8gypEk9xuFfh8GpaWFutSAPSTluHD9ZHLpU6H5uflPABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4PqZntJUqKRSQzFugoA/SXBSC7nph9cIeo2CmVdkmlriXUlAPqJSU4kRKMqwUiJJtZVAOgvDr/y5JwoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALg+tme5cUTOqQy9Ue60oA9JNgUqeMy7k32AyqEDUyavW0yyQSosBgEUxw9u+dl/MAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKAhaiH6Lhx4+Ryubq04uJiSdKsWbO69K1YsSLaZQBAv4j6zfbvv/++Ojs7w48/+ugj/emf/qn+/M//PLxs+fLl2rBhQ/jxsGHDol1Gj4xLjr57AcDAYhx+vR31EB09enTE440bN2rixIm69957w8uGDRsmr9cb7U1flXFLLb4OBd0d/b5tALHR0dkhc8m5+R1922dbW5t++tOfqqSkRC7X779u79VXX9VPf/pTeb1eLViwQE899dQVj0aDwaCCwWD4cSAQ6FtBLqkz2cjFF9UBg0Znh5FaJTn0Z+9oiO7cuVONjY169NFHw8u++c1vauzYsfL5fDp+/LieeOIJVVdX64033uhxnrKyMq1fv97JUgGgTxwN0ZdfflmFhYXy+XzhZUVFReGfp0yZouzsbM2ZM0enTp3SxIkTu52ntLRUJSUl4ceBQEA5OTnOFQ4A18ixEP3000+1d+/eKx5hSlJubq4kqaampscQ9Xg88ng8Ua8RAGw5dt1qy5YtyszM1H333XfFcceOHZMkZWdnO1UKADjGkSPRUCikLVu2aOnSpUpM/P0mTp06pW3btmn+/PkaOXKkjh8/rtWrV+uee+7R1KlTnSgFABzlSIju3btXdXV1+qu/+quI5cnJydq7d6+ee+45tbS0KCcnRwsXLtSTTz7pRBkA4DhHQnTu3Lkypuv9BDk5OSovL3dikwAQE7x3HgAsDKrvWArJJb+GyJihsS4FQD9xmSHySHJddWTfDKoQ7ZBLR0Mj1OxOinUpAPrJcJOiO+WSU3/1gypEpcvv/HLq/yQAgw3nRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALg+4+UcklY7hPFBg8nP17H1wh2pGszqOF6ggmxLoSAP2k09MpjQ9ICc58ydLgCtGQW6GG8TIt/fcVzQBiKzS8RRr7kZTQefXBfcA5UQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFgbVzfbGhNTSfEqBAO9YAgYLtzpljDM32kuDLEQ7Oi7qxC+fk7+hIdalAOgn2V6vZn+tSNIQR+YfVCEqGXV2tirU2RrrQgD0k1AoqMtfUekEzokCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALDQ6xCtqKjQggUL5PP55HK5tHPnzoh+Y4zWrFmj7OxsDR06VPn5+Tp58mTEmPPnz2vJkiVKTU1Venq6li1bpubmZqsnAgCx0OsQbWlp0bRp07Rp06Zu+59++mk9//zzeumll1RVVaUbbrhB8+bNU2vr798ltGTJEn388cfas2ePdu3apYqKChUVFfX9WQBAjPT6bZ+FhYUqLCzsts8Yo+eee05PPvmkHnjgAUnSK6+8oqysLO3cuVOLFi3SiRMntHv3br3//vu64447JEkvvPCC5s+frx/96Efy+XwWTwcA+ldUz4nW1tbK7/crPz8/vCwtLU25ubmqrKyUJFVWVio9PT0coJKUn58vt9utqqqqbucNBoMKBAIRDQAGgqiGqN/vlyRlZWVFLM/Kygr3+f1+ZWZmRvQnJiYqIyMjPOaPlZWVKS0tLdxycnKiWTYA9FlcXJ0vLS1VU1NTuJ0+fTrWJQGApCiHqNfrlSQ1/NHndTY0NIT7vF6vzp49G9Hf0dGh8+fPh8f8MY/Ho9TU1IgGAANBVEN0/Pjx8nq92rdvX3hZIBBQVVWV8vLyJEl5eXlqbGzUkSNHwmP279+vUCik3NzcaJYDAI7r9dX55uZm1dTUhB/X1tbq2LFjysjI0JgxY/T444/rb//2b3XzzTdr/Pjxeuqpp+Tz+fTggw9Kkm699VYVFBRo+fLleumll9Te3q6VK1dq0aJFXJkHEHd6HaKHDx/W7Nmzw49LSkokSUuXLtXWrVv1gx/8QC0tLSoqKlJjY6Puvvtu7d69W0OG/P6j+V999VWtXLlSc+bMkdvt1sKFC/X8889H4ekAQP/qdYjOmjVLxvT8Ufsul0sbNmzQhg0behyTkZGhbdu29XbTADDgxMXVeQAYqAhRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWeh2iFRUVWrBggXw+n1wul3bu3Bnua29v1xNPPKEpU6bohhtukM/n01/+5V/qzJkzEXOMGzdOLpcrom3cuNH6yQBAf+t1iLa0tGjatGnatGlTl76LFy/q6NGjeuqpp3T06FG98cYbqq6u1v33399l7IYNG1RfXx9uq1at6tszAIAYSuztCoWFhSosLOy2Ly0tTXv27IlY9uKLL+quu+5SXV2dxowZE16ekpIir9fb280DwIDi+DnRpqYmuVwupaenRyzfuHGjRo4cqenTp+uZZ55RR0dHj3MEg0EFAoGIBgADQa+PRHujtbVVTzzxhBYvXqzU1NTw8u9+97uaMWOGMjIydPDgQZWWlqq+vl7PPvtst/OUlZVp/fr1TpYKAH3iWIi2t7frL/7iL2SM0ebNmyP6SkpKwj9PnTpVycnJ+va3v62ysjJ5PJ4uc5WWlkasEwgElJOT41TpAHDNHAnRywH66aefav/+/RFHod3Jzc1VR0eHPvnkE02aNKlLv8fj6TZcASDWoh6ilwP05MmTeueddzRy5MirrnPs2DG53W5lZmZGuxwAcFSvQ7S5uVk1NTXhx7W1tTp27JgyMjKUnZ2tP/uzP9PRo0e1a9cudXZ2yu/3S5IyMjKUnJysyspKVVVVafbs2UpJSVFlZaVWr16tRx55RCNGjIjeMwOAftDrED18+LBmz54dfnz5XOXSpUu1bt06/du//Zsk6Stf+UrEeu+8845mzZolj8ej7du3a926dQoGgxo/frxWr14dcc4TAOJFr0N01qxZMsb02H+lPkmaMWOGDh061NvNAsCAxHvnAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwEKvQ7SiokILFiyQz+eTy+XSzp07I/offfRRuVyuiFZQUBAx5vz581qyZIlSU1OVnp6uZcuWqbm52eqJAEAs9DpEW1paNG3aNG3atKnHMQUFBaqvrw+31157LaJ/yZIl+vjjj7Vnzx7t2rVLFRUVKioq6n31ABBjib1dobCwUIWFhVcc4/F45PV6u+07ceKEdu/erffff1933HGHJOmFF17Q/Pnz9aMf/Ug+n6+3JQFAzDhyTvTAgQPKzMzUpEmT9Nhjj+ncuXPhvsrKSqWnp4cDVJLy8/PldrtVVVXV7XzBYFCBQCCiAcBAEPUQLSgo0CuvvKJ9+/bphz/8ocrLy1VYWKjOzk5Jkt/vV2ZmZsQ6iYmJysjIkN/v73bOsrIypaWlhVtOTk60ywaAPun1y/mrWbRoUfjnKVOmaOrUqZo4caIOHDigOXPm9GnO0tJSlZSUhB8HAgGCFMCA4PgtThMmTNCoUaNUU1MjSfJ6vTp79mzEmI6ODp0/f77H86gej0epqakRDQAGAsdD9LPPPtO5c+eUnZ0tScrLy1NjY6OOHDkSHrN//36FQiHl5uY6XQ4ARFWvX843NzeHjyolqba2VseOHVNGRoYyMjK0fv16LVy4UF6vV6dOndIPfvADfelLX9K8efMkSbfeeqsKCgq0fPlyvfTSS2pvb9fKlSu1aNEirswDiDu9PhI9fPiwpk+frunTp0uSSkpKNH36dK1Zs0YJCQk6fvy47r//ft1yyy1atmyZZs6cqV/84hfyeDzhOV599VVNnjxZc+bM0fz583X33Xfrn//5n6P3rACgn/T6SHTWrFkyxvTY/1//9V9XnSMjI0Pbtm3r7aYBYMDhvfMAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBgodchWlFRoQULFsjn88nlcmnnzp0R/S6Xq9v2zDPPhMeMGzeuS//GjRutnwwA9Ldeh2hLS4umTZumTZs2ddtfX18f0X784x/L5XJp4cKFEeM2bNgQMW7VqlV9ewYAEEOJvV2hsLBQhYWFPfZ7vd6Ix2+99ZZmz56tCRMmRCxPSUnpMhYA4o2j50QbGhr07//+71q2bFmXvo0bN2rkyJGaPn26nnnmGXV0dPQ4TzAYVCAQiGgAMBD0+ki0N37yk58oJSVFDz30UMTy7373u5oxY4YyMjJ08OBBlZaWqr6+Xs8++2y385SVlWn9+vVOlgoAfeJoiP74xz/WkiVLNGTIkIjlJSUl4Z+nTp2q5ORkffvb31ZZWZk8Hk+XeUpLSyPWCQQCysnJca5wALhGjoXoL37xC1VXV+v111+/6tjc3Fx1dHTok08+0aRJk7r0ezyebsMVAGLNsXOiL7/8smbOnKlp06ZddeyxY8fkdruVmZnpVDkA4IheH4k2NzerpqYm/Li2tlbHjh1TRkaGxowZI+l3L7d37Nihv//7v++yfmVlpaqqqjR79mylpKSosrJSq1ev1iOPPKIRI0ZYPBUA6H+9DtHDhw9r9uzZ4ceXz1UuXbpUW7dulSRt375dxhgtXry4y/oej0fbt2/XunXrFAwGNX78eK1evTrinCcAxIteh+isWbNkjLnimKKiIhUVFXXbN2PGDB06dKi3mwWAAYn3zgOABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYc/aI6p11yhWRcoWse3+o2Mi4HCwIcMjQhQSlJSf22vbZQSI1tbf22PSe5QiElB4NKdvXuj7+ztfWaxsV1iB4afklJQ6/8AdF/qD3hki66r308MFDM8XpVdPPN/ba94198obXHj6vzKh/AHg+GXLqk2w4f1g29/E+opb39msbFdYgG3UadvQjFdpeRUfz/UmDwGZqQoMw/+upxJ6UnJ/fbtpx2+UjUE7r2V62S1NHRcU3jOCcKABYIUQCwQIgCgAVCFAAsxPWFJWCwaA2FdD4Y7LftXbjGK9MgRIG4sN/v15Fz5/pte8FQ6Lq4vak/EKJAHGjp6FDLNd5yg/7FOVEAsMCRKIDrWmN7u35eVyePu3fHjMHOzmsaF9chaoyR4bwNgCs4FwzqpZMnHZs/rkP011vekjsx4ZrHhzo61fpFwMGKAAw2cR2ivznyq1iXAGCQ48ISAFggRAHAAiEKABZ6FaJlZWW68847lZKSoszMTD344IOqrq6OGNPa2qri4mKNHDlSw4cP18KFC9XQ0BAxpq6uTvfdd5+GDRumzMxM/fVf//U1f3YfAAwkvQrR8vJyFRcX69ChQ9qzZ4/a29s1d+5ctbS0hMesXr1ab7/9tnbs2KHy8nKdOXNGDz30ULi/s7NT9913n9ra2nTw4EH95Cc/0datW7VmzZroPSsA6C/GwtmzZ40kU15ebowxprGx0SQlJZkdO3aEx5w4ccJIMpWVlcYYY/7jP/7DuN1u4/f7w2M2b95sUlNTTTAYvKbtNjU1GUk0Go3meGtqarpiHlmdE21qapIkZWRkSJKOHDmi9vZ25efnh8dMnjxZY8aMUWVlpSSpsrJSU6ZMUVZWVnjMvHnzFAgE9PHHH3e7nWAwqEAgENEAYCDoc4iGQiE9/vjj+upXv6rbb79dkuT3+5WcnKz09PSIsVlZWfL7/eExfxigl/sv93WnrKxMaWlp4ZaTk9PXsgEgqvocosXFxfroo4+0ffv2aNbTrdLSUjU1NYXb6dOnHd8mAFyLPr1jaeXKldq1a5cqKip00003hZd7vV61tbWpsbEx4mi0oaFBXq83POa9996LmO/y1fvLY/6Yx+ORx+PpS6kA4KzeXEgKhUKmuLjY+Hw+87//+79d+i9fWPr5z38eXvbrX//aSF0vLDU0NITH/NM//ZNJTU01ra2t11QHF5ZoNFp/tatdWOpViD722GMmLS3NHDhwwNTX14fbxYsXw2NWrFhhxowZY/bv328OHz5s8vLyTF5eXri/o6PD3H777Wbu3Lnm2LFjZvfu3Wb06NGmtLT0musgRGk0Wn+1qIZoTxvZsmVLeMylS5fMd77zHTNixAgzbNgw841vfMPU19dHzPPJJ5+YwsJCM3ToUDNq1Cjzve99z7S3txOiNBptwLWrhajr/4djXAkEAkpLS4t1GQAGgaamJqWmpvbYz3vnAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYiMsQjcP3BwCIU1fLm7gM0QsXLsS6BACDxNXyJi7f9hkKhVRdXa0vf/nLOn369BXfkoW+CQQCysnJYf86hP3rrGjsX2OMLly4IJ/PJ7e75+PNPn2eaKy53W7deOONkqTU1FR+CR3E/nUW+9dZtvv3Wj6jIy5fzgPAQEGIAoCFuA1Rj8ejtWvX8rUhDmH/Oov966z+3L9xeWEJAAaKuD0SBYCBgBAFAAuEKABYIEQBwAIhCgAW4jJEN23apHHjxmnIkCHKzc3Ve++9F+uS4tK6devkcrki2uTJk8P9ra2tKi4u1siRIzV8+HAtXLhQDQ0NMax4YKuoqNCCBQvk8/nkcrm0c+fOiH5jjNasWaPs7GwNHTpU+fn5OnnyZMSY8+fPa8mSJUpNTVV6erqWLVum5ubmfnwWA9fV9u+jjz7a5fe5oKAgYowT+zfuQvT1119XSUmJ1q5dq6NHj2ratGmaN2+ezp49G+vS4tJtt92m+vr6cHv33XfDfatXr9bbb7+tHTt2qLy8XGfOnNFDDz0Uw2oHtpaWFk2bNk2bNm3qtv/pp5/W888/r5deeklVVVW64YYbNG/ePLW2tobHLFmyRB9//LH27NmjXbt2qaKiQkVFRf31FAa0q+1fSSooKIj4fX7ttdci+h3Zv1f8VvoB6K677jLFxcXhx52dncbn85mysrIYVhWf1q5da6ZNm9ZtX2Njo0lKSjI7duwILztx4oSRZCorK/upwvglybz55pvhx6FQyHi9XvPMM8+ElzU2NhqPx2Nee+01Y4wxv/rVr4wk8/7774fH/Od//qdxuVzm888/77fa48Ef719jjFm6dKl54IEHelzHqf0bV0eibW1tOnLkiPLz88PL3G638vPzVVlZGcPK4tfJkyfl8/k0YcIELVmyRHV1dZKkI0eOqL29PWJfT548WWPGjGFf90Ftba38fn/E/kxLS1Nubm54f1ZWVio9PV133HFHeEx+fr7cbreqqqr6veZ4dODAAWVmZmrSpEl67LHHdO7cuXCfU/s3rkL0t7/9rTo7O5WVlRWxPCsrS36/P0ZVxa/c3Fxt3bpVu3fv1ubNm1VbW6uvfe1runDhgvx+v5KTk5Wenh6xDvu6by7vsyv97vr9fmVmZkb0JyYmKiMjg31+DQoKCvTKK69o3759+uEPf6jy8nIVFhaqs7NTknP7Ny4/Cg/RUVhYGP556tSpys3N1dixY/Wzn/1MQ4cOjWFlQO8tWrQo/POUKVM0depUTZw4UQcOHNCcOXMc225cHYmOGjVKCQkJXa4QNzQ0yOv1xqiq60d6erpuueUW1dTUyOv1qq2tTY2NjRFj2Nd9c3mfXel31+v1drlA2tHRofPnz7PP+2DChAkaNWqUampqJDm3f+MqRJOTkzVz5kzt27cvvCwUCmnfvn3Ky8uLYWXXh+bmZp06dUrZ2dmaOXOmkpKSIvZ1dXW16urq2Nd9MH78eHm93oj9GQgEVFVVFd6feXl5amxs1JEjR8Jj9u/fr1AopNzc3H6vOd599tlnOnfunLKzsyU5uH/7fEkqRrZv3248Ho/ZunWr+dWvfmWKiopMenq68fv9sS4t7nzve98zBw4cMLW1teZ//ud/TH5+vhk1apQ5e/asMcaYFStWmDFjxpj9+/ebw4cPm7y8PJOXlxfjqgeuCxcumA8++MB88MEHRpJ59tlnzQcffGA+/fRTY4wxGzduNOnp6eatt94yx48fNw888IAZP368uXTpUniOgoICM336dFNVVWXeffddc/PNN5vFixfH6ikNKFfavxcuXDDf//73TWVlpamtrTV79+41M2bMMDfffLNpbW0Nz+HE/o27EDXGmBdeeMGMGTPGJCcnm7vuusscOnQo1iXFpYcffthkZ2eb5ORkc+ONN5qHH37Y1NTUhPsvXbpkvvOd75gRI0aYYcOGmW984xumvr4+hhUPbO+8846R1KUtXbrUGPO725yeeuopk5WVZTwej5kzZ46prq6OmOPcuXNm8eLFZvjw4SY1NdV861vfMhcuXIjBsxl4rrR/L168aObOnWtGjx5tkpKSzNixY83y5cu7HFw5sX/5PFEAsBBX50QBYKAhRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFv4f8VIb0sVYfIMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the observation\n",
    "import matplotlib.pyplot as plt\n",
    "print(obs.shape)\n",
    "plt.imshow(obs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "import ale_py\n",
    "\n",
    "gym.register_envs(ale_py)\n",
    "\n",
    "env = gym.make('ALE/Asteroids-v5')\n",
    "obs, info = env.reset()\n",
    "obs, reward, terminated, truncated, info = env.step(env.action_space.sample())\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(210, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVEAAAGhCAYAAADY5IdbAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAii0lEQVR4nO3df3TU9Z3v8deEJEOAzIQkJJMg4ZcIWCRV0GmuLVuWLCRyqAq7K1y6m1oPXDTYK1jbzT1XqZ7eBuWs2+OWVc65Ftrjb/YWPLCVPTT8SNUQNZjlKjYlnGgCZIJCM5ME8vtz/1id2zE/IPnMZDLyfJzzPifz/XzmO+/5GF988/3OD4cxxggAMCxx0W4AAGIZIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWohqi27dv17Rp0zR27Fh5vV6988470WwHAIYsaiH66quvavPmzdqyZYuOHz+u3NxcLVu2TOfPn49WSwAwZI5ofQCJ1+vVrbfeql/84heSpN7eXk2ZMkUPPvig/uEf/mHQ+/b29urcuXNKTk6Ww+EYiXYBXGOMMWppaVF2drbi4gY+3owfwZ6COjs7VVVVpZKSkuC2uLg45efnq6Kios/8jo4OdXR0BG+fPXtWN95444j0CuDa1tDQoOuuu27A8aj8Of/ZZ5+pp6dHmZmZIdszMzPl8/n6zC8tLZXb7Q4WAQpgpCQnJw86HhNX50tKSuT3+4PV0NAQ7ZYAXCOudMowKn/Op6ena8yYMWpqagrZ3tTUJI/H02e+0+mU0+kcqfYA4KpF5Ug0MTFRCxYsUFlZWXBbb2+vysrKlJeXF42WAGBYonIkKkmbN29WUVGRFi5cqNtuu00///nP1dbWpnvvvTdaLQHAkEUtRO+55x59+umneuyxx+Tz+fT1r39dBw4c6HOxCQBGs6i9TtRGIBCQ2+2OdhuIkKS4OE1PSlJcGF8D3NXbq9OXL6s79n7dY1JKfLyuGzu23zF/d7ca2tv7HUseM0ZTx46V+vlv39bTo48vX9ZI/xf0+/1yuVwDjkftSBQYyHVjx+qJmTOVOMgLnIfqs64u/fCPf9TFrq6w7RMDu8Xl0kM5Of2OvdXcrCc//rjfsdnjx+t/zpjR78Wa/9vaqi2nT4+6fwgJUYw6cZLi4+LCGqIJDod4b9vIidPna97PEeWYQf7CcHx+v/7mxI/SdycSoogpV3P2ibcCYyQRoogpn3Z16cXGRnX09vYZm5iQoL/LytK4MWOi0BmuVYQoYkpbT49+/6c/6VI/IZrtdGp1ZqZEiGIExcTbPgFgtOJIFMCISh4zRjeMG9fv2GSnM+YuABKiAEbU/ORkPTVrVr9jcTH4KgpCFMCIGuNwaMxX6Lw1IQogYobzhshYe4kaIQog7E62tenn9fVDvt/0pCR9Z9KksL7lN9IIUQBhd66jQ+f+7Ct9rtatLpdWTJoUgY4ih5c4AYAFjkQRUxIcDuUkJelyT0+fsczExJj6MxBfDYQoYkqW06n/df31/V6wiHM4lBTGDy0BrgYhipgyxuHQ+K/Qy2MQ+/hnGwAscCSKUaeps1P/+8yZQT93cqgu9/aqrZ/zqBhd6tvbtePMmX7ftfRZV5d6RtkHMkt8PQgADOpKXw/Cn/MAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsBD2EC0tLdWtt96q5ORkZWRk6K677lJNTU3InG9/+9tyOBwhtWHDhnC3AgARF/YQPXr0qIqLi3Xs2DEdPHhQXV1dWrp0qdra2kLmrVu3To2NjcF66qmnwt0KAERc2D/Z/sCBAyG3d+3apYyMDFVVVWnRokXB7ePGjZPH4wn3wwPAiIr4OVG/3y9JSk1NDdn+4osvKj09XfPmzVNJSYkuXbo04D46OjoUCARCCgBGBRNBPT09Zvny5eb2228P2b5jxw5z4MABc+LECfPCCy+YyZMnm7vvvnvA/WzZssVIoiiKGvHy+/2D5lxEQ3TDhg1m6tSppqGhYdB5ZWVlRpKpra3td7y9vd34/f5gNTQ0RH1hKYq6NupKIRqxb/vcuHGj9u/fr/Lycl133XWDzvV6vZKk2tpazZw5s8+40+mU0+mMSJ8AYCPsIWqM0YMPPqg9e/boyJEjmj59+hXvU11dLUnKysoKdzsAEFFhD9Hi4mK99NJLev3115WcnCyfzydJcrvdSkpK0unTp/XSSy/pjjvuUFpamk6cOKFNmzZp0aJFmj9/frjbAYDIGu75zoFogPMKO3fuNMYYU19fbxYtWmRSU1ON0+k0119/vXnkkUeueN7hz/n9/qifJ6Eo6tqoK2WT4/PgiymBQEButzvabQC4Bvj9frlcrgHHee88AFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWAh7iP7kJz+Rw+EIqTlz5gTH29vbVVxcrLS0NE2YMEGrVq1SU1NTuNsAgBERkSPRr33ta2psbAzWm2++GRzbtGmT9u3bp927d+vo0aM6d+6cVq5cGYk2ACDi4iOy0/h4eTyePtv9fr+ef/55vfTSS/rLv/xLSdLOnTs1d+5cHTt2TN/4xjci0Q4ARExEjkRPnTql7OxszZgxQ2vXrlV9fb0kqaqqSl1dXcrPzw/OnTNnjnJyclRRUTHg/jo6OhQIBEIKAEaDsIeo1+vVrl27dODAAT377LOqq6vTt771LbW0tMjn8ykxMVEpKSkh98nMzJTP5xtwn6WlpXK73cGaMmVKuNsGgGEJ+5/zhYWFwZ/nz58vr9erqVOn6rXXXlNSUtKw9llSUqLNmzcHbwcCAYIUwKgQ8Zc4paSk6IYbblBtba08Ho86OzvV3NwcMqepqanfc6hfcDqdcrlcIQUAo0HEQ7S1tVWnT59WVlaWFixYoISEBJWVlQXHa2pqVF9fr7y8vEi3AgDhZ8Ls4YcfNkeOHDF1dXXmrbfeMvn5+SY9Pd2cP3/eGGPMhg0bTE5Ojjl06JB57733TF5ensnLyxvSY/j9fiOJoigq4uX3+wfNo7CfEz1z5ozWrFmjCxcuaNKkSfrmN7+pY8eOadKkSZKkf/qnf1JcXJxWrVqljo4OLVu2TP/yL/8S7jYAYEQ4jDEm2k0MVSAQkNvtjnYbAK4Bfr9/0OswvHceACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALIQ9RKdNmyaHw9GniouLJUnf/va3+4xt2LAh3G0AwIiID/cO3333XfX09ARvf/DBB/qrv/or/c3f/E1w27p16/TEE08Eb48bNy7cbQDAiAh7iE6aNCnk9tatWzVz5kz9xV/8RXDbuHHj5PF4wv3QADDiInpOtLOzUy+88IK+//3vy+FwBLe/+OKLSk9P17x581RSUqJLly4Nup+Ojg4FAoGQAoBRwUTQq6++asaMGWPOnj0b3LZjxw5z4MABc+LECfPCCy+YyZMnm7vvvnvQ/WzZssVIoiiKGvHy+/2D5pPDGGMUIcuWLVNiYqL27ds34JxDhw5pyZIlqq2t1cyZM/ud09HRoY6OjuDtQCCgKVOmhL1fAPgyv98vl8s14HjYz4l+4ZNPPtHvfvc7/eY3vxl0ntfrlaRBQ9TpdMrpdIa9RwCwFbFzojt37lRGRoaWL18+6Lzq6mpJUlZWVqRaAYCIiciRaG9vr3bu3KmioiLFx///hzh9+rReeukl3XHHHUpLS9OJEye0adMmLVq0SPPnz49EKwAQWWG4ftTHv//7vxtJpqamJmR7fX29WbRokUlNTTVOp9Ncf/315pFHHrniidsv8/v9UT/ZTFHUtVFRvbAUKYFAQG63O9ptALgGXOnCEu+dBwALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiwOcSx7k0+cZvyO2ZFu1WEEMIUeBzk6bdqFWPv6av3/H9aLeCGEKIAp9zOOI0JiFRcWMi9iW4+AoiRIEgI5kvvloHuDr8kwt87sKZU/rt0w/oT+dOR7sVxBC+qA4ABsEX1QFABBGiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWBhyiJaXl2vFihXKzs6Ww+HQ3r17Q8aNMXrssceUlZWlpKQk5efn69SpUyFzLl68qLVr18rlciklJUX33XefWltbrZ4IEC63T52q0oIC5WZlRbsVxIAhh2hbW5tyc3O1ffv2fsefeuopPfPMM3ruuedUWVmp8ePHa9myZWpvbw/OWbt2rT788EMdPHhQ+/fvV3l5udavXz/8ZwGE0fysLG3wenV9Wlq0W0EsMBYkmT179gRv9/b2Go/HY7Zt2xbc1tzcbJxOp3n55ZeNMcacPHnSSDLvvvtucM4bb7xhHA6HOXv27FU9rt/v/+KTcykq7HX/N75h/rRli1k1b17Ue6GiX36/f9A8Cus50bq6Ovl8PuXn5we3ud1ueb1eVVRUSJIqKiqUkpKihQsXBufk5+crLi5OlZWV/e63o6NDgUAgpABgNAhriPp8PklSZmZmyPbMzMzgmM/nU0ZGRsh4fHy8UlNTg3O+rLS0VG63O1hTpkwJZ9sAMGwxcXW+pKREfr8/WA0NDdFuCQAkhTlEPR6PJKmpqSlke1NTU3DM4/Ho/PnzIePd3d26ePFicM6XOZ1OuVyukAKA0SCsITp9+nR5PB6VlZUFtwUCAVVWViovL0+SlJeXp+bmZlVVVQXnHDp0SL29vfJ6veFsBxiSnJQU/Y/Fi7V01ixJ0sp58/TIokWaNH58lDvDaDbkb/tsbW1VbW1t8HZdXZ2qq6uVmpqqnJwcPfTQQ/rpT3+qWbNmafr06Xr00UeVnZ2tu+66S5I0d+5cFRQUaN26dXruuefU1dWljRs3avXq1crOzg7bEwOG6jq3W//99tvljP/P/y2Wz5mj26dO1b6PPtKnbW1R7g6j1ZBD9L333tPixYuDtzdv3ixJKioq0q5du/SjH/1IbW1tWr9+vZqbm/XNb35TBw4c0NixY4P3efHFF7Vx40YtWbJEcXFxWrVqlZ555pkwPB0AGFl8ZTLwuf8ydar2/N3fBY9EJan58mUV/PKX+sOnn0axM0QTX5kMABFEiAKABUIUACwM+cIS8FUzLiFB//XrX9f8rCyNiQs9rhgbH697Fy7UBz6fXvmP/1BXb2+UusRoRYjimjc+MVEP5OVpRmpqn7GxCQn6b16v3j93Tv/ngw8IUfTBn/MAYIEjUVzzeo3RxUuXNCExccA5f7p8WTH3WkCMCF4nimtenMOhrORkJYwZM+Ccju5u+VpaCNJr0JVeJ8qRKK55vcboLJ9Ri2HinCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALQw7R8vJyrVixQtnZ2XI4HNq7d29wrKurSz/+8Y910003afz48crOztbf//3f69y5cyH7mDZtmhwOR0ht3brV+skAwEgbcoi2tbUpNzdX27dv7zN26dIlHT9+XI8++qiOHz+u3/zmN6qpqdF3vvOdPnOfeOIJNTY2BuvBBx8c3jMAgCiKH+odCgsLVVhY2O+Y2+3WwYMHQ7b94he/0G233ab6+nrl5OQEtycnJ8vj8Qz14QFgVIn4OVG/3y+Hw6GUlJSQ7Vu3blVaWppuvvlmbdu2Td3d3QPuo6OjQ4FAIKQAYDQY8pHoULS3t+vHP/6x1qxZI5fLFdz+gx/8QLfccotSU1P19ttvq6SkRI2NjXr66af73U9paakef/zxSLYKAMNjLEgye/bs6Xess7PTrFixwtx8883G7/cPup/nn3/exMfHm/b29n7H29vbjd/vD1ZDQ4ORRFEUFfG6Un5F5Ei0q6tLf/u3f6tPPvlEhw4dCjkK7Y/X61V3d7c+/vhjzZ49u8+40+mU0+mMRKsAYCXsIfpFgJ46dUqHDx9WWlraFe9TXV2tuLg4ZWRkhLsdAIioIYdoa2uramtrg7fr6upUXV2t1NRUZWVl6a//+q91/Phx7d+/Xz09PfL5fJKk1NRUJSYmqqKiQpWVlVq8eLGSk5NVUVGhTZs26bvf/a4mTpwYvmcGACPhqk5+/pnDhw/3e96gqKjI1NXVDXhe4fDhw8YYY6qqqozX6zVut9uMHTvWzJ071/zsZz8b8Hxof/x+f9TPk1AUdW3Ulc6JOowxRjEmEAjI7XZHuw0A1wC/3z/odR3eOw8AFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWhhyi5eXlWrFihbKzs+VwOLR3796Q8e9973tyOBwhVVBQEDLn4sWLWrt2rVwul1JSUnTfffeptbXV6okAQDQMOUTb2tqUm5ur7du3DzinoKBAjY2NwXr55ZdDxteuXasPP/xQBw8e1P79+1VeXq7169cPvXsAiDZjQZLZs2dPyLaioiJz5513DnifkydPGknm3XffDW574403jMPhMGfPnr2qx/X7/UYSRVFUxMvv9w+aRxE5J3rkyBFlZGRo9uzZuv/++3XhwoXgWEVFhVJSUrRw4cLgtvz8fMXFxamysrLf/XV0dCgQCIQUAIwGYQ/RgoIC/frXv1ZZWZmefPJJHT16VIWFherp6ZEk+Xw+ZWRkhNwnPj5eqamp8vl8/e6ztLRUbrc7WFOmTAl32wAwLPHh3uHq1auDP990002aP3++Zs6cqSNHjmjJkiXD2mdJSYk2b94cvB0IBAhSAKNCxF/iNGPGDKWnp6u2tlaS5PF4dP78+ZA53d3dunjxojweT7/7cDqdcrlcIQUAo0HEQ/TMmTO6cOGCsrKyJEl5eXlqbm5WVVVVcM6hQ4fU29srr9cb6XYAIKyG/Od8a2tr8KhSkurq6lRdXa3U1FSlpqbq8ccf16pVq+TxeHT69Gn96Ec/0vXXX69ly5ZJkubOnauCggKtW7dOzz33nLq6urRx40atXr1a2dnZ4XtmADASruo1RX/m8OHD/b4MoKioyFy6dMksXbrUTJo0ySQkJJipU6eadevWGZ/PF7KPCxcumDVr1pgJEyYYl8tl7r33XtPS0nLVPfASJ4qiRqqu9BInhzHGKMYEAgG53e5otwHgGuD3+we9DsN75wHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcDCkEO0vLxcK1asUHZ2thwOh/bu3Rsy7nA4+q1t27YF50ybNq3P+NatW62fDACMtCGHaFtbm3Jzc7V9+/Z+xxsbG0Pql7/8pRwOh1atWhUy74knngiZ9+CDDw7vGQBAFMUP9Q6FhYUqLCwccNzj8YTcfv3117V48WLNmDEjZHtycnKfuQAQayJ6TrSpqUn/9m//pvvuu6/P2NatW5WWlqabb75Z27ZtU3d394D76ejoUCAQCCkAGA2GfCQ6FL/61a+UnJyslStXhmz/wQ9+oFtuuUWpqal6++23VVJSosbGRj399NP97qe0tFSPP/54JFsFgOExFiSZPXv2DDg+e/Zss3Hjxivu5/nnnzfx8fGmvb293/H29nbj9/uD1dDQYCRRFEVFvPx+/6D5FbEj0d///veqqanRq6++esW5Xq9X3d3d+vjjjzV79uw+406nU06nMxJtAoCViJ0Tff7557VgwQLl5uZecW51dbXi4uKUkZERqXYAICKGfCTa2tqq2tra4O26ujpVV1crNTVVOTk5kqRAIKDdu3frH//xH/vcv6KiQpWVlVq8eLGSk5NVUVGhTZs26bvf/a4mTpxo8VQAIAqueMLySw4fPtzveYOioqLgnB07dpikpCTT3Nzc5/5VVVXG6/Uat9ttxo4da+bOnWt+9rOfDXg+tD9+vz/q50koiro26krnRB3GGKMYEwgE5Ha7o90GgGuA3++Xy+UacJz3zgOABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoAFQhQALBCiAGCBEAUAC4QoAFggRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKABUIUACwQogBggRAFAAuEKABYIEQBwAIhCgAWCFEAsECIAoCFIYVoaWmpbr31ViUnJysjI0N33XWXampqQua0t7eruLhYaWlpmjBhglatWqWmpqaQOfX19Vq+fLnGjRunjIwMPfLII+ru7rZ/NgAwwoYUokePHlVxcbGOHTumgwcPqqurS0uXLlVbW1twzqZNm7Rv3z7t3r1bR48e1blz57Ry5crgeE9Pj5YvX67Ozk69/fbb+tWvfqVdu3bpscceC9+zAoCRYiycP3/eSDJHjx41xhjT3NxsEhISzO7du4NzPvroIyPJVFRUGGOM+e1vf2vi4uKMz+cLznn22WeNy+UyHR0dV/W4fr/fSKIoiop4+f3+QfPI6pyo3++XJKWmpkqSqqqq1NXVpfz8/OCcOXPmKCcnRxUVFZKkiooK3XTTTcrMzAzOWbZsmQKBgD788MN+H6ejo0OBQCCkAGA0GHaI9vb26qGHHtLtt9+uefPmSZJ8Pp8SExOVkpISMjczM1M+ny84588D9IvxL8b6U1paKrfbHawpU6YMt20ACKthh2hxcbE++OADvfLKK+Hsp18lJSXy+/3BamhoiPhjAsDViB/OnTZu3Kj9+/ervLxc1113XXC7x+NRZ2enmpubQ45Gm5qa5PF4gnPeeeedkP19cfX+izlf5nQ65XQ6h9MqAETWUC4k9fb2muLiYpOdnW3++Mc/9hn/4sLSv/7rvwa3/eEPfzBS3wtLTU1NwTk7duwwLpfLtLe3X1UfXFiiKGqk6koXloYUovfff79xu93myJEjprGxMViXLl0KztmwYYPJyckxhw4dMu+9957Jy8szeXl5wfHu7m4zb948s3TpUlNdXW0OHDhgJk2aZEpKSq66D0KUoqiRqrCG6EAPsnPnzuCcy5cvmwceeMBMnDjRjBs3ztx9992msbExZD8ff/yxKSwsNElJSSY9Pd08/PDDpqurixClKGrU1ZVC1PF5OMaUQCAgt9sd7TYAXAP8fr9cLteA47x3HgAsEKIAYIEQBQALhCgAWCBEAcACIQoAFghRALBAiAKAhZgM0Rh8fwCAGHWlvInJEG1paYl2CwCuEVfKm5h822dvb69qamp04403qqGhYdC3ZGF4AoGApkyZwvpGCOsbWeFYX2OMWlpalJ2drbi4gY83h/V5otEWFxenyZMnS5JcLhe/hBHE+kYW6xtZtut7NZ/REZN/zgPAaEGIAoCFmA1Rp9OpLVu28LUhEcL6RhbrG1kjub4xeWEJAEaLmD0SBYDRgBAFAAuEKABYIEQBwAIhCgAWYjJEt2/frmnTpmns2LHyer165513ot1STPrJT34ih8MRUnPmzAmOt7e3q7i4WGlpaZowYYJWrVqlpqamKHY8upWXl2vFihXKzs6Ww+HQ3r17Q8aNMXrssceUlZWlpKQk5efn69SpUyFzLl68qLVr18rlciklJUX33XefWltbR/BZjF5XWt/vfe97fX6fCwoKQuZEYn1jLkRfffVVbd68WVu2bNHx48eVm5urZcuW6fz589FuLSZ97WtfU2NjY7DefPPN4NimTZu0b98+7d69W0ePHtW5c+e0cuXKKHY7urW1tSk3N1fbt2/vd/ypp57SM888o+eee06VlZUaP368li1bpvb29uCctWvX6sMPP9TBgwe1f/9+lZeXa/369SP1FEa1K62vJBUUFIT8Pr/88ssh4xFZ30G/lX4Uuu2220xxcXHwdk9Pj8nOzjalpaVR7Co2bdmyxeTm5vY71tzcbBISEszu3buD2z766CMjyVRUVIxQh7FLktmzZ0/wdm9vr/F4PGbbtm3Bbc3NzcbpdJqXX37ZGGPMyZMnjSTz7rvvBue88cYbxuFwmLNnz45Y77Hgy+trjDFFRUXmzjvvHPA+kVrfmDoS7ezsVFVVlfLz84Pb4uLilJ+fr4qKiih2FrtOnTql7OxszZgxQ2vXrlV9fb0kqaqqSl1dXSFrPWfOHOXk5LDWw1BXVyefzxeynm63W16vN7ieFRUVSklJ0cKFC4Nz8vPzFRcXp8rKyhHvORYdOXJEGRkZmj17tu6//35duHAhOBap9Y2pEP3ss8/U09OjzMzMkO2ZmZny+XxR6ip2eb1e7dq1SwcOHNCzzz6ruro6fetb31JLS4t8Pp8SExOVkpISch/Weni+WLPBfnd9Pp8yMjJCxuPj45WamsqaX4WCggL9+te/VllZmZ588kkdPXpUhYWF6unpkRS59Y3Jj8JDeBQWFgZ/nj9/vrxer6ZOnarXXntNSUlJUewMGLrVq1cHf77ppps0f/58zZw5U0eOHNGSJUsi9rgxdSSanp6uMWPG9LlC3NTUJI/HE6WuvjpSUlJ0ww03qLa2Vh6PR52dnWpubg6Zw1oPzxdrNtjvrsfj6XOBtLu7WxcvXmTNh2HGjBlKT09XbW2tpMitb0yFaGJiohYsWKCysrLgtt7eXpWVlSkvLy+KnX01tLa26vTp08rKytKCBQuUkJAQstY1NTWqr69nrYdh+vTp8ng8IesZCARUWVkZXM+8vDw1NzerqqoqOOfQoUPq7e2V1+sd8Z5j3ZkzZ3ThwgVlZWVJiuD6DvuSVJS88sorxul0ml27dpmTJ0+a9evXm5SUFOPz+aLdWsx5+OGHzZEjR0xdXZ156623TH5+vklPTzfnz583xhizYcMGk5OTYw4dOmTee+89k5eXZ/Ly8qLc9ejV0tJi3n//ffP+++8bSebpp58277//vvnkk0+MMcZs3brVpKSkmNdff92cOHHC3HnnnWb69Onm8uXLwX0UFBSYm2++2VRWVpo333zTzJo1y6xZsyZaT2lUGWx9W1pazA9/+ENTUVFh6urqzO9+9ztzyy23mFmzZpn29vbgPiKxvjEXosYY88///M8mJyfHJCYmmttuu80cO3Ys2i3FpHvuucdkZWWZxMREM3nyZHPPPfeY2tra4Pjly5fNAw88YCZOnGjGjRtn7r77btPY2BjFjke3w4cPG0l9qqioyBjzny9zevTRR01mZqZxOp1myZIlpqamJmQfFy5cMGvWrDETJkwwLpfL3HvvvaalpSUKz2b0GWx9L126ZJYuXWomTZpkEhISzNSpU826dev6HFxFYn35PFEAsBBT50QBYLQhRAHAAiEKABYIUQCwQIgCgAVCFAAsEKIAYIEQBQALhCgAWCBEAcACIQoAFv4fdZ8poQ60l9gAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "print(obs.shape)\n",
    "plt.imshow(obs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchrl.envs import GymWrapper\n",
    "\n",
    "env = GymWrapper(env, from_pixels=True, pixels_only=False, device='cpu')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Could not find observation in the reset data.'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[25], line 8\u001b[0m\n\u001b[1;32m      5\u001b[0m policy \u001b[38;5;241m=\u001b[39m RandomPolicy(env\u001b[38;5;241m.\u001b[39maction_spec)\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# Define a SyncDataCollector and collect some data\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m collector \u001b[38;5;241m=\u001b[39m \u001b[43mSyncDataCollector\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcreate_env_fn\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43menv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpolicy\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mpolicy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mframes_per_batch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m data \u001b[38;5;129;01min\u001b[39;00m collector:\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;28mprint\u001b[39m(data)\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/collectors/collectors.py:633\u001b[0m, in \u001b[0;36mSyncDataCollector.__init__\u001b[0;34m(self, create_env_fn, policy, frames_per_batch, total_frames, device, storing_device, policy_device, env_device, create_env_kwargs, max_frames_per_traj, init_random_frames, reset_at_each_iter, postproc, split_trajs, exploration_type, exploration_mode, return_same_td, reset_when_done, interruptor, set_truncated)\u001b[0m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;66;03m# Shuttle is a deviceless tensordict that just carried data from env to policy and policy to env\u001b[39;00m\n\u001b[1;32m    632\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m--> 633\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_shuttle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    634\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy_device \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv_device \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv_device \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    635\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_shuttle_has_no_device \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:2120\u001b[0m, in \u001b[0;36mEnvBase.reset\u001b[0;34m(self, tensordict, **kwargs)\u001b[0m\n\u001b[1;32m   2117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensordict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   2118\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_assert_tensordict_shape(tensordict)\n\u001b[0;32m-> 2120\u001b[0m tensordict_reset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensordict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2121\u001b[0m \u001b[38;5;66;03m#        We assume that this is done properly\u001b[39;00m\n\u001b[1;32m   2122\u001b[0m \u001b[38;5;66;03m#        if reset.device != self.device:\u001b[39;00m\n\u001b[1;32m   2123\u001b[0m \u001b[38;5;66;03m#            reset = reset.to(self.device, non_blocking=True)\u001b[39;00m\n\u001b[1;32m   2124\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensordict_reset \u001b[38;5;129;01mis\u001b[39;00m tensordict:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/transforms/transforms.py:814\u001b[0m, in \u001b[0;36mTransformedEnv._reset\u001b[0;34m(self, tensordict, **kwargs)\u001b[0m\n\u001b[1;32m    812\u001b[0m     tensordict \u001b[38;5;241m=\u001b[39m tensordict_reset\u001b[38;5;241m.\u001b[39mempty()\n\u001b[1;32m    813\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_env\u001b[38;5;241m.\u001b[39m_complete_done(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_env\u001b[38;5;241m.\u001b[39mfull_done_spec, tensordict_reset)\n\u001b[0;32m--> 814\u001b[0m tensordict_reset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransform\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensordict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensordict_reset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    815\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m tensordict_reset\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/transforms/transforms.py:1129\u001b[0m, in \u001b[0;36mCompose._reset\u001b[0;34m(self, tensordict, tensordict_reset)\u001b[0m\n\u001b[1;32m   1125\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_reset\u001b[39m(\n\u001b[1;32m   1126\u001b[0m     \u001b[38;5;28mself\u001b[39m, tensordict: TensorDictBase, tensordict_reset: TensorDictBase\n\u001b[1;32m   1127\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m TensorDictBase:\n\u001b[1;32m   1128\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransforms:\n\u001b[0;32m-> 1129\u001b[0m         tensordict_reset \u001b[38;5;241m=\u001b[39m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensordict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtensordict_reset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1130\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tensordict_reset\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/transforms/transforms.py:6111\u001b[0m, in \u001b[0;36mTimeMaxPool._reset\u001b[0;34m(self, tensordict, tensordict_reset)\u001b[0m\n\u001b[1;32m   6109\u001b[0m         tensordict_reset\u001b[38;5;241m.\u001b[39mset(in_key, val_prev)\n\u001b[1;32m   6110\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m val_prev \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m val_reset \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 6111\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCould not find \u001b[39m\u001b[38;5;132;01m{\u001b[39;00min_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in the reset data.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   6112\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(tensordict_reset, _reset\u001b[38;5;241m=\u001b[39m_reset)\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Could not find observation in the reset data.'"
     ]
    }
   ],
   "source": [
    "# Define a random policy\n",
    "from torchrl.collectors import SyncDataCollector\n",
    "from torchrl.envs.utils import RandomPolicy\n",
    "    \n",
    "policy = RandomPolicy(env.action_spec)\n",
    "\n",
    "# Define a SyncDataCollector and collect some data\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn = env,\n",
    "    policy = policy,\n",
    "    frames_per_batch=10\n",
    "    ) \n",
    "\n",
    "\n",
    "for data in collector:\n",
    "    print(data)\n",
    "    break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7YAAADeCAYAAAAJtZwyAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAMeElEQVR4nO3dS28bBdvH4XvssR1bOThtaIvSk6joAgnWLPgG7NjwLdkg8QFYwQ4VIdhxSp9WtKQHJ7Hjw/hZPLLR8+p9Rd5mmslNrkvKomoqOU7/jn6T8UyxXC6XAQAAAEm1mn4AAAAAcB7CFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBq5Vk/sSiKt/k4rrQ3fW6Xy2XNjyS31fP4d8/LWT/vTV3274stvz22XA9b/nt2/PbYcT3s+Gxs+e2x5Xpk2vKZw5Z6tVqtaLVa0W63o9frRbvdPtO/Wy6X64/pdBrT6XT956uoLMvo9XpRFEW02+1otVoxm81iNptFVVUxn89juVxGWZZRlmW02+3odDoRETGfz9efM51OG/5KyMqW62HLNMmO62HHNM2W65F1y8K2Ia1WK8qyjE6nE5ubm1GWZ/tWLJfLqKoqlstlHB8fr//zXNXhtdvt2NjYiHa7Hd1uN1qtVkwmkzg5OYnFYhGLxWI9vF6vF2VZRr/fj6Io4vT0NObzeZyenvohyhuz5XrYMk2y43rYMU2z5Xpk3bKwbUiv14utra3Y3t6Ohw8fxubmZlRVtT7CMZlMoqqqaLVaURRFdLvd6PV66yNJi8Uifv755/j111/X/3mu4vi2t7fjwYMH0e/3YzgcRq/Xi6dPn8bBwUGMx+N49uxZzGazGA6Hsbe3F5ubm7G/vx+tViueP38eo9Eonj9/HuPxOKqqavrLISFbroct0yQ7rocd0zRbrkfWLQvbhuzs7MTdu3fj/v378fnnn8f+/n6cnp7GbDaLo6OjePr0aczn8+h0OtFut2M4HMbNmzejqqoYjUYxmUziq6++ipcvX8ZkMonZbBaLxaLpL+vC3blzJz799NPY29uLBw8exHA4jG+++Sa+/vrrePbsWRwdHcVisYj79+/Hhx9+GHfv3o1PPvkkyrKMR48excHBQXz33Xfxr3/9yxFi3ogt18OWaZId18OOaZot1yPrloVtQ1a/uu/3+7G3txfvvvtujMfjmEwm66MdVVWtz13f2tqKGzduxHK5jH6/H+PxeH2Kxeqo01XU7XZjOBzG9evX4+bNm7G7uxvvvPNO7OzsxHQ6jcFgEFVVxfb2duzu7sbe3l7cunUryrKMg4ODODo6io2NjSv7/HF+tlwPW6ZJdlwPO6ZptlyPrFsWtg1ptVrR6XRiY2MjhsNhDIfDmM1mcXp6Gr/99lt8+eWXMRqNYnd3N/r9fnz88cfx/vvvr9+Y3e121+eyt1pX965Nqxelra2tGA6Hsbu7G7dv344PPvggDg8Po91ux8nJSXz00Ufx8OHD2N/fj729vSiKIjY3N2MwGESv12v6yyAxW66HLdMkO66HHdM0W65H1i0L2watriA2GAxiMBhEq9WKxWIRh4eH8f3338fh4WHcuHEjtre347333lu/gbuqqiiKYn0axVU9mhQR66vebWxsxGAwiM3Nzbh27VrcunUr+v1+nJycxGQyidu3b8eNGzfi+vXrsbm5uT4yt3rD+1V+Djk/Wz4/W6Zpdnx+dsxlYMvnl3XLwpbUXr16FT/99FP88ccfcXR0FDs7O/HkyZN48uRJjMfjmM/nTT9E4AxsGfKzY/hnyLplYUtqz549i2+//Tb6/X788MMPsbGxEVVVrd/ov7ryHXC52TLkZ8fwz5B1y8K2Qat7Zs1ms/WNjouiWL/hvdvtxrVr12JraysGg0Esl8tYLBYxn8//6/5aV/Ey5CvT6XR95bqqqv7X8/nb7Xa8fv06Tk5OYjwex3Q6jaIoYj6fr+/FBedhy+dnyzTNjs/PjrkMbPn8sm5Z2DZkNbjJZBIvX76Mw8PDmM/n0ev14t69e/HZZ5/FZDKJwWAQ3W437t27F9PpNGazWYxGoxiPxzEej2OxWFzpe729evUqfvzxx2i32+sr2P1PZVnGZDKJ+Xwe4/E49vf3oyzLGI1GcXx8HLPZ7Eq/eHE+tlwPW6ZJdlwPO6ZptlyPrFsWtg1ZHR2azWYxHo/j+Pg4ptNpLJfLGAwGcefOnf+6z9bW1tb671ejW/35Kv8AmE6nf3t/rLIs488//4wXL17EcDiMo6Oj6HQ6MZlM4vT09NK+T4AcbLketkyT7LgedkzTbLkeWbcsbBsyGo3i8ePHMZlM4osvvojd3d2YzWZRVVVMp9N4/fp1VFW1virb6opkVVWtbxj96NGjGI1GMZ1Or/RRpb9TVVU8fvw4iqKI33//PZ4+fRqtVisODg7ixYsX8csvvzj1iTdmyxfHlnlb7Pji2DFvky1fnMu45WJ5xsMRV/mS129DWZbR6XSi0+nEzs5OtNvtM//b1bn/o9Eojo6O1ken+L+tLj3e7XZjMBhERKxPn5hMJnF8fFzbkbnLfoTPlutlyxfLlv/DjutlxxfLjv9iy/Wy5Yt12bbsN7YNWY2lKIo4OTmJsjzbt2L1hvjlcrk+d/2yv2hfBqvTUlbPXVEU64sKrC4sAG/Cli+WLfM22PHFsmPeFlu+WJdty35j26CiKKIoije6XPZqcE6ROJvVc736iPjrOaz7xeuyvxDacv1s+eLY8n/Ycf3s+OLY8V9suX62fHEu25aFLdTMD1H4Z7jMW7ZjOJvLvOMIW4azOsuWL9+ddQEAAOD/QdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKQmbAEAAEhN2AIAAJCasAUAACA1YQsAAEBqwhYAAIDUhC0AAACpCVsAAABSE7YAAACkJmwBAABITdgCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkJqwBQAAIDVhCwAAQGrCFgAAgNSELQAAAKkJWwAAAFITtgAAAKRWLJfLZdMPAgAAAN6U39gCAACQmrAFAAAgNWELAABAasIWAACA1IQtAAAAqQlbAAAAUhO2AAAApCZsAQAASE3YAgAAkNq/AQZ77YdxGbVSAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1200x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAEr0lEQVR4nO3cP2tTewDG8SelTUM4kVaqaBVCB3EScRMU6Xtw6djX4O5b6XtwcxLcOmgHQVx0UIqKJW01Ne0JqecOcgVxuEew9zT18xnTP/kFHg5fEnJaVVVVAQD+ajNNHwAAaJ4gAAAEAQAgCACACAIAIIIAAIggAAAiCACAJLN1f7HVap3kOfjDTuJ+UzYwXU7qnmN2MF1cC6i7gdpBMI3a7XZ6vV5mZ399mVVV5fj4OPv7+zk+Pm7gdD+bnZ1NURSZn5/P169fU5ZlJpNJut1uut1uJpNJyrLMaDRq+qhTxQZI7AAbqPW8f/S/nSKtViuXL1/OvXv3sri4mPF4nL29vczMzKTX62Vubi4HBwd59OhRhsNh4yNYWFjI3bt30+/38/z587x58yaDwSA3btzIrVu3MhgM8vr162xtbTV6zmliAyR2gA3UdaaDYGVlJevr6+n3+9nd3c3m5maKosj169eztLSUT58+5enTpxmNRo0PYGlpKffv38/q6mo2NjZ+PL66upr19fW8evUqjx8/dhH4DTZAYgfYQF1nNgiSpNPp5OLFiymKIi9fvszDhw+zsrKSBw8e5Nq1a0m+vzVzGj4Pa7fbuXDhQpaXl3P79u10Op28f/8+d+7cSb/fz87OToqiaPqYU8cGSOwAG6jjTAfBNPnw4UM2Njby5MmTlGWZ8Xicdrvd9LH4H9kAiR3Q3AbOdBCUZZn9/f30er0sLi5mbW0t58+fz6VLl3JwcJDPnz/n27dvTR8zSfLly5dsbm7mxYsXPx4riiJXr17NzZs3MxwOc3R01OAJp5MNkNgBNlDHmQ2Cqqqyt7eXra2tfPz4MaPRKFeuXMn8/Hy2t7czGAyyu7ubw8PDUzGCsizz7t27nx7rdrt59uxZlpeX8/bt22xvbzd0uulkAyR2gA3U1apqfkHxNHyu8rs6nU4WFhYyNzf3y8/+/ZrJzs5OJpNJA6f7b61WK71eL+fOnct4PM7h4WGGw2Gtv/Xd4+9s4GTONG3s4M+fZ9rYQI3nOMtB8DdzEUAQkLgWUH8Dbl0MAAgCAEAQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAACRpVVVVNX0IAKBZ3iEAAAQBACAIAIAIAgAgggAAiCAAACIIAIAIAgAgggAASPIP7TierBE+SL8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAEr0lEQVR4nO3cP2tTewDG8SelTUM4kVaqaBVCB3EScRMU6Xtw6djX4O5b6XtwcxLcOmgHQVx0UIqKJW01Ne0JqecOcgVxuEew9zT18xnTP/kFHg5fEnJaVVVVAQD+ajNNHwAAaJ4gAAAEAQAgCACACAIAIIIAAIggAAAiCACAJLN1f7HVap3kOfjDTuJ+UzYwXU7qnmN2MF1cC6i7gdpBMI3a7XZ6vV5mZ399mVVV5fj4OPv7+zk+Pm7gdD+bnZ1NURSZn5/P169fU5ZlJpNJut1uut1uJpNJyrLMaDRq+qhTxQZI7AAbqPW8f/S/nSKtViuXL1/OvXv3sri4mPF4nL29vczMzKTX62Vubi4HBwd59OhRhsNh4yNYWFjI3bt30+/38/z587x58yaDwSA3btzIrVu3MhgM8vr162xtbTV6zmliAyR2gA3UdaaDYGVlJevr6+n3+9nd3c3m5maKosj169eztLSUT58+5enTpxmNRo0PYGlpKffv38/q6mo2NjZ+PL66upr19fW8evUqjx8/dhH4DTZAYgfYQF1nNgiSpNPp5OLFiymKIi9fvszDhw+zsrKSBw8e5Nq1a0m+vzVzGj4Pa7fbuXDhQpaXl3P79u10Op28f/8+d+7cSb/fz87OToqiaPqYU8cGSOwAG6jjTAfBNPnw4UM2Njby5MmTlGWZ8Xicdrvd9LH4H9kAiR3Q3AbOdBCUZZn9/f30er0sLi5mbW0t58+fz6VLl3JwcJDPnz/n27dvTR8zSfLly5dsbm7mxYsXPx4riiJXr17NzZs3MxwOc3R01OAJp5MNkNgBNlDHmQ2Cqqqyt7eXra2tfPz4MaPRKFeuXMn8/Hy2t7czGAyyu7ubw8PDUzGCsizz7t27nx7rdrt59uxZlpeX8/bt22xvbzd0uulkAyR2gA3U1apqfkHxNHyu8rs6nU4WFhYyNzf3y8/+/ZrJzs5OJpNJA6f7b61WK71eL+fOnct4PM7h4WGGw2Gtv/Xd4+9s4GTONG3s4M+fZ9rYQI3nOMtB8DdzEUAQkLgWUH8Dbl0MAAgCAEAQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAACRpVVVVNX0IAKBZ3iEAAAQBACAIAIAIAgAgggAAiCAAACIIAIAIAgAgggAASPIP7TierBE+SL8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAHiUlEQVR4nO3cz2vbdRzH8Vean8sSSdd2004bpp1DQUUQUSsy8OAf4MXj/gbv/iv7G/TmSRAUdtAJwhRFZZVtykqb1jTNkiaNh+FAVrGD1rTz8TimLd93yZtPnyT5tjCZTCYBAP7XZqY9AAAwfYIAABAEAIAgAAAiCACACAIAIIIAAIggAACSlA76jYVC4Sjn4JAdxf+bsgMny1H9zzF7cLI4CzjoDhw4CE6iSqWSZrOZUunhX3MymWQ8HmdzczPj8XgK0/1dqVRKo9FItVpNr9fLYDDIaDRKvV5PvV7PaDTKYDDIzs7OtEc9UewAiT0gKRaLqdVqKRaLD31tMplkMplkZ2cne3t7U5ju72ZmZlKr1VIqlR48/+PxONVqNZVKJePxOKPRKMPh8FCv+9gGQaFQyFNPPZV33nkns7OzGQ6H6XQ6mZmZSbPZTLlczvb2dj755JN0u92pHwStVitvv/122u12vv766/z8889ZX1/PSy+9lFdffTXr6+v56aefcv369anOeZLYARJ7wH2tVisXL17MqVOnsre3l16vl2KxmGq1mmKxmMFgkG+++SaDwWDqUVCv17O8vJy5ubmsrq5mbW0tvV4v58+fz9LSUra3t3P37t38+uuvh3rdxzoILly4kCtXrqTdbmdjYyPXrl1Lo9HIpUuXMj8/n7t37+bzzz/Pzs7O1A+B+fn5vP/++7l8+XKuXr364PHLly/nypUr+f777/Ppp586BB6BHSCxB9zfgfn5+aysrGRubi7dbjc3b95MtVrNuXPn0mg00u128+OPP2Z3d3fqQdBsNvPaa6/l+eefzxdffJHvvvsuSXLp0qW89dZbuXPnTm7cuCEIHkWtVsvZs2fTaDRy48aNfPTRR7lw4UI+/PDDXLx4Mcn9l+eOw/thlUolCwsLWVxczBtvvJFarZY7d+5kZWUl7XY7a2traTQa0x7zxLEDJPaA+89vs9lMtVrN7du38/HHH2dhYSHvvvtuzp07l+T+S/XHQbFYzOnTp9NqtfLss8+mXC5nc3Mzzz33XObm5rK9vZ1arXbo132sg+Ak+e2333L16tV89tlnGQwGGQ6HqVQq0x6L/5AdILEHJFtbW/nyyy/zww8/ZHd3N+PxeN/PPhy2xzoIBoNBNjc302w2Mzs7mw8++CBnzpzJk08+me3t7WxtbU39paG//PHHH7l27Vq+/fbbB481Go08/fTTeeWVV9LtdnPv3r0pTngy2QESe0AyHo/T7/dTq9VSr9fz+uuvp16vp9VqZTAY5N69e0d2Z86j6vf7+eWXX3L79u0Hj1Wr1czOzuaZZ55Jv9/P7u7uoV/3sQ2CyWSSTqeT69ev5/fff8/Ozk7Onz+farWaW7duZX19PRsbG+n3+8fiIBgMBg+9H1Sv1/PVV19lcXExq6uruXXr1pSmO5nsAIk9IA/uIFhdXc3W1lYGg0FarVZKpVI6nU56vV56vV6Gw+GxiILRaJSNjY2/PVapVHLz5s20Wq2sr6+n0+kc+nULkwP+9sfhvbVHVavV0mq1Ui6XH/raX7cara2tZTQaTWG6f1coFNJsNvPEE09kOBym3++n2+0e6Gfde3yfHTiamU4ae3D485w05XI5p06d+sdbT/f29tLtdo9FFO6nUCikVqulVqtlNBpld3f3wK8UHXQHHusg+D9zCCAISJwFHHwHjsdHKgGAqRIEAIAgAAAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAEAEAQD/gUKhkHK5nJkZf3aOK88MAEeuXC5ndnY2tVpt2qPwDwQBAEeqWCxmYWEh7733XpaWlqY9Dv9AEABwpMrlcl588cWsrKxkeXk5jUZj2iOxD0EAwJGqVCp5+eWX8+abb2Z5eTmtVmvaI7EPQQDAkSkWizl9+nReeOGFtNvttNvtnD17dtpjsQ9BAMCRqVarOXPmTJaWltLtdjM/P5/l5eVpj8U+CpPJZHKgbywUjnoWDtEBn9ZHYgdOlqPYgcQenDTTPgtmZmZSqVSyuLiYUqmUfr+f7e3tdDqdQ5+L/R10BwTBY2rahwDTJwhIjsdZUCgUUqlUUigUsre3l/F4nPF4fOhzsb+D7kDpiOcA4H9uMplkMBhMewz+hc8QAACCAAAQBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAAEQQAAARBABABAEAEEEAACQpTCaTybSHAACmyysEAIAgAAAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAAEn+BBXT5KsOATQiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAJIElEQVR4nO3cy24bBRuA4Xd8nLq2sROnhQTitqQUkAAhIVQIQpUQgh0LNkhseg3suQEuotcAO1ZInKQsIEhUAQW1pWnTgxoSJ9iOMz7NvwiN/lIqUgl3nPR9ltNU/eJ8nXnlGSeI4zhGkiQ91lJJDyBJkpJnEEiSJINAkiQZBJIkCYNAkiRhEEiSJAwCSZKEQSBJkoDMfr8wCIJRzqH/2Ch+35Q7cLCM6neOuQcHi+cC7XcH9h0EB1Eul6NUKpHJ3P9txnHMYDBgc3OTwWCQwHT3ymQyFItF8vk87XabKIro9/sUCgUKhQL9fp8oitje3k561APFHRBAOp0mDEPS6fR9fxbHMXEcs729zXA4TGC6e6VSKcIwJJPJ7O3AYDAgn8+Ty+UYDAb0+3263W7Soybu/1+rf9Lv9/deK3fg3x3aIAiCgKeeeoq3336barVKt9ul0WiQSqUolUpks1larRZffPEFzWYz8QtCpVLhrbfeol6v8+OPP3L58mXW19d56aWXePXVV1lfX+fSpUssLi4mOudB4g7orkqlwunTpzly5AjD4ZB2u006nSafz5NOp4miiJ9++okoihK/IBQKBebm5picnGRlZYW1tTXa7TYzMzPMzs7SarW4c+cO165dS3TOcZDP55menqZSqdxzPAxD8vk8W1tbe6+VO/DvDnUQnDx5kvPnz1Ov19nY2GBhYYFisciZM2eo1WrcuXOHr7/+mu3t7cQvBrVajQ8//JBz585x4cKFvePnzp3j/Pnz/Prrr3z55ZdeDB6COyDY3YNarcb8/DyTk5M0m02uXr1KPp/n+PHjFItFms0mv/32G71eL/GLQalU4rXXXuO5557ju+++45dffgHgzJkzvPnmm9y8eZOlpSWDgN0L/4kTJ5iZmdk7lkqlqNVqexfTpaUlrl+/7g7sw6ENAthdlmPHjlEsFllaWuLTTz/l5MmTfPLJJ5w+fRrYfZt2HO6H5XI5pqammJ6e5uzZs4RhyM2bN5mfn6der7O2tkaxWEx6zAPHHRDs/oxLpRL5fJ4bN27w+eefMzU1xTvvvMPx48eB3QvJOEin0xw9epRKpcKpU6fIZrNsbm7y7LPPMjk5SavVIgzDpMccC71ejz/++IM4jvf+D2ezWZ544gnK5TLlcnnvtXrQDrz77rvuwF8OdRAcJLdu3eLChQt89dVXRFFEt9sll8slPZYeIXdAAFtbW3z//fcsLy/T6/UYDAb/eN9bUC6XOXv2LM8//zxXrlzh9u3bNBoNlpeXuXz5Mo1Gg1u3biU95kNLagcOdRBEUcTm5ialUolqtcpHH33ExMQETz75JK1Wi62trcTfGrrrzz//ZGFhgZ9//nnvWLFY5Omnn+aVV16h2Wyys7OT4IQHkzsggMFgQKfTIQxDCoUCr7/+OoVCgUqlQhRF7OzsjOxTGQ+r0+lw5coVbty4sXcsn89TrVZ55pln6HQ69Hq9BCccP3EcE4Yh5XL5nuOtVmvvgcMH7UC5XHYH/nJogyCOYxqNBouLi9y+fZvt7W1mZmbI5/Osrq6yvr7OxsYGnU5nLC4IURTddz+oUCjwww8/MD09zcrKCqurqwlNdzC5AwL2nh5fWVlha2uLKIqoVCpkMhkajQbtdpt2u0232x2LC0K/32djY+OeY7lcjqtXr1KpVFhfX6fRaCQ03XiJoojV1VWCIKDdbtPpdO558r7X69Hv992BfQrifX7343CP9WGFYUilUiGbzd73Z3c/cra2tka/309gun8XBAGlUolyuUy326XT6dBsNvf1d/3s8S53YDQzHTTZbJYjR4488OOnw+GQZrM5FmH4T4IgIAxDwjCk3+/T6/X2/W7RYT4XpNNpCoXCA58D6vV6e6+VO7CPf+MwB8Hj7DCfBLQ/BoHAc4H2vwPj8UilJElKlEEgSZIMAkmSZBBIkiQMAkmShEEgSZIwCCRJEgaBJEnCIJAkSRgEkiQJg0CSJGEQSJIkDAJJkoRBIEmSMAgkSRIGgSRJwiCQJEkYBJIkCYNAkiRhEEh6BIIgIJvNkkp5ynlc+fMff/50JI1cNpulWq0ShmHSoygBqVSKiYkJSqUSuVwu6XH0AAaBpJFKp9NMTU3x3nvvMTs7m/Q4SkChUOCDDz7g/fff59SpU4bhmMokPYCkwy2bzfLiiy8yPz9Po9FgdXWVVquV9Fh6hDKZDCdOnODjjz9mYWGBb7/9losXL/LNN98Qx3HS4+kvvkMgaaRyuRwvv/wyb7zxBnNzc1QqlaRHUgIymQzVapVKpUKpVPJdgjFkEEgamXQ6zdGjR3nhhReo1+vU63WOHTuW9Fh6xOI4ptPpsLy8zKVLl7h+/Trr6+tJj6W/8ZaBpJHJ5/NMTEwwOztLs9mkVqsxNzfH4uJi0qPpERoMBly7do3PPvuMixcv8vvvvxNFUdJj6W+CeJ83cIIgGPUs+g+N4r6cO3CwjOre7MPsQSqVIpfLMT09TSaTodPp0Gq1aDQaI5lN9xuHc0EQBJTLZXZ2duj1egyHw/98Jj3YfnfAIDikxuEkoGSNQxDc/fpcLkcQBAyHQwaDAYPBYCSz6X7jci5Ip9MMh0MfIkyAQfCYG5eTgJIzLkGgZHku0H53wIcKJUmSQSBJkgwCSZKEQSBJkjAIJEkSBoEkScIgkCRJGASSJAmDQJIkYRBIkiQMAkmShEEgSZIwCCRJEgaBJEnCIJAkSRgEkiQJg0CSJGEQSJIkDAJJkoRBIEmSMAgkSRIGgSRJwiCQJEkYBJIkCYNAkiRhEEiSJAwCSZKEQSBJkjAIJEkSBoEkScIgkCRJGASSJAmDQJIkYRBIkiQMAkmShEEgSZIwCCRJEgaBJEnCIJAkSRgEkiQJg0CSJGEQSJIkDAJJkoRBIEmSMAgkSRIGgSRJwiCQJEkYBJIkCYNAkiRhEEiSJAwCSZKEQSBJkjAIJEkSBoEkScIgkCRJGASSJAmDQJIkYRBIkiQMAkmShEEgSZIwCCRJEgaBJEnCIJAkSRgEkiQJg0CSJGEQSJIkDAJJkoRBIEmSMAgkSRIQxHEcJz2EJElKlu8QSJIkg0CSJBkEkiQJg0CSJGEQSJIkDAJJkoRBIEmSMAgkSRIGgSRJAv4HpG+XoSMpT4oAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAL5UlEQVR4nO3czU8chR/H8c/M7s4My+4yLAutS1tKS6maqGlSFcWYxsazBy8mXnr07N1/wD+iif+BxosnjU+RREWlAaXyzLIQKCzLPs4+ze9Q2V9rS0ITYBZ8v47bkn53+Xb2zcwshu/7vgAAwH+aGfQAAAAgeAQBAAAgCAAAAEEAAABEEAAAABEEAABABAEAABBBAAAAJIUP+xcNwzjOOXDEjuP3TbEDp8tx/c4x9uB04ViAw+7AoYPgNLIsS/F4XOHwk0/T9301m03t7u6q2WwGMN3jwuGwYrGYbNtWqVSS53lqNBqKRqOKRqNqNBryPE/lcjnoUU+VUCgkx3EUCoWe+DPf9+X7vsrlslqtVgDTPc40TTmOo3A43P7+N5tN2bYty7LUbDbVaDRUq9WCHrUjPPp6PU2j0Wi/XuzB2WSapmzbPnAHHn2t9nfANJ9+YrzVaqlcLh9bSD+LR59XrVZr74BlWbIsS61W61h24MwGgWEYeu655/T222+rt7dXtVpNuVxOpmkqHo8rEomoWCzqiy++UKFQCDwKXNfVW2+9paGhIf3666+an5/X9va2XnrpJd24cUPb29uam5vT5ORkoHOeNq7r6tq1a+rq6lKr1VKpVFIoFJJt2wqFQvI8T7/99ps8zwv8zSAajWpkZER9fX1aXl7W1taWSqWSBgcHdenSJRWLRW1ubmplZSXQOTuFbdtKp9NyXfexxx3HkW3byufz7deLPTibbNvW4OCgEolE+zHDMNrxtLe3p62tLa2urqqnp0cjIyPtH7DK5XI7vkzTlOd5mpqaUrVaDTwKurq6NDIyomQyqZWVFT148EDFYlHpdFoXLlxQuVxuP6+jdKaDYHh4WHfu3NHQ0JB2dnY0MTGhWCym69evK5VKaXNzU99++63K5XLgQZBKpfT+++/r1q1bunv3bvvxW7du6c6dO/rzzz/11VdfEQTPwDAMpVIpjY+Pq6+vT4VCQUtLS7JtW+fOnVMsFlOhUND9+/dVr9cDfyOIx+O6efOmRkdH9cMPP2hmZkaSdP36db355pvKZrOanp7mjeAfjuPo8uXLGhwcbD9mmqZSqVT7zXR6elqrq6vswRll27YuXryodDrdvoxhmqaSyaT6+vq0urqqmZkZZTIZJZNJjY2Nqa+vT6VSSYuLi+0d6O7uVqFQ0NzcnOr1uhqNRqDPKx6P68aNG7p69aomJiY0Ozsr3/c1OjqqsbExbWxsaGZmhiB4Fo7jaGBgQLFYTNPT0/rkk080PDysjz/+WNeuXZP08FR9J1wPsyxL/f39SqfTGhsbk+M4ymazGh8f19DQkLa2thSLxYIe89QJh8OKx+OybVtra2v6/PPP1d/fr9u3b+vcuXOSdOApxJMWCoXU3d0t13V15coVRSIR7e7u6urVq+rr61OxWJTjOEGP2THq9boePHgg3/fb/4cjkYh6enqUSCSUSCTar9dBe/Duu++yB6dYvV7Xzs7OY49FIhHF43HF43F1d3fLtm1J/7+EbFmWstmsvvzyS6VSKd2+fVsDAwMyDKNjdsA0zfYODA8PKxKJKJ/P68qVK0omkyoUCu3ndZTOdBCcJuvr67p7966+/vpreZ6nWq0my7KCHgsnKJ/P68cff9Ts7Kzq9bqazeZTr3njoUQiobGxMT3//PNaWFjQxsaGcrmcZmdnNT8/r1wup/X19aDHfGbsweElEgm9+uqrGh0d1eLiojY2NrS7u6u///5bCwsLyuVy2tzcDHrMZ7a3t6effvpJ9+/fb98/cBI7cKaDwPM87e7uKh6Pq7e3Vx988IGSyaTOnz+vYrGofD4f+OnBfXt7e5qYmNDU1FT7sVgspgsXLuiVV15RoVBQtVoNcMLTqdlsqlKpyHEcRaNRvfbaa4pGo3JdV57ndcT1wn2VSkULCwtaW1trP2bbtnp7e3Xx4kVVKhXV6/UAJ+xMvu/LcZzHriNLUrFYbN9sdtAeJBIJ9uCU278p1HEc9fT0PPZT/v59AtLDHSiXy+0duHnzprq7uxWPx+V5niqVSsfsQLVa1eLiorLZbPsx27bluq4GBwdVrVaPZQfObBD4vq9cLqfJyUltbGyoXC5rcHBQtm0rk8loe3tbOzs7qlQqHREFnuc9cU0wGo3ql19+UTqd1vLysjKZTEDTnU77d44vLy8rn8/L8zy5rqtwOKxcLqdSqaRSqaRardYRB4JGo/HE6U/LsrS0tCTXdbW9va1cLhfQdJ3H8zxlMhkZhqFSqaRKpfLYXdf714LZg7PL8zxls1mZpqlisahqtfrEDrRaLfm+r1KppEwmo0KhoFqt1t6B3d1dlUollcvljtqBf3+PLcvSysqKenp6tLOzo93d3SP/dw3/kM++E66zPyvHceS6riKRyBN/tv+xw62trcBvIDmIYRiKx+NKJBKq1WqqVCoqFAqH+lo+e/xQJBJRV1fXgR89bbVaKhQKHRGFT2MYhhzHkeM4ajQaqtfrhz5TdNZ/D0EoFFI0Gj3wPqB6vd5+vdiDo5+nE4RCIXV1dT31GC+p/dE8z/MUDocVjUYP/Ohpq9VSsVjs6B2wbVuO46jZbLaf12EcdgfOdBD8l53lgwAO56wHAQ6HYwEOuwOdcUslAAAIFEEAAAAIAgAAQBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQ4JgZhqFIJCLTZNX+y9gBoPOFgx4AZ1skEpHruioWiyqXy0GPgwCYpqlkMqlqtapKpaJarRb0SAiAaZqKxWIKh8PyfV+1Wk2lUinosfAIggDHJhQKqb+/X++8845+/vln/fXXX0GPhABEo1G99957yufz+uOPP7S0tKRqtRr0WDhBpmkqkUjoo48+0sWLF9VoNPT777/rs88+U6vVCno8/IMgwLGJRCJ68cUXNT4+rlwup0wmo2KxGPRYOGHhcFiXL1/Whx9+qImJCX3//fe6d++evvvuO/m+H/R4OAG+76taraqrq0uvv/66JOmbb74JeCr8Gxf1cGwsy9LLL7+sN954QyMjI3JdN+iREJBwOKze3l65rqt4PC7HcYIeCSdo/xLB+vq6dnZ2lM/nlc1mCcIOwxkCHItQKKTu7m698MILGhoa0tDQkAYGBpTJZIIeDSfM931VKhXNzs5qbm5Oq6ur2t7eDnosnLBWq6X5+XnNzs7KsiyCoAMRBDgWtm0rmUzq0qVLKhQKSqVSGhkZ0eTkZNCj4YQ1m02trKzo008/1b1797S4uCjP84IeCwGYmppSLBaT4zhaW1sLehz8i+EfMtEMwzjuWXCEjqO8n2UHTNOUZVlKp9MKh8OqVCoqFovK5XJHPhee7rh++nrWY4FhGEokEqpWq6rX69xEdsKCPhY8ynEcnT9/XuFwWHNzc0c8FQ5y2B0gCM6oTjgIGIYhy7JkGIZarZaazaaazeaRz4Wn65QgkB5eQmq1WpwiDkAnHAse/br9YwKfNDk5BMF/XCcdBBCMTgoCBIdjAQ67A3zKAAAAEAQAAIAgAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACDJ8H3fD3oIAAAQLM4QAAAAggAAABAEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAACT9D0NpYCzKtnpVAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAMXklEQVR4nO3cS28b9d+G8XvG9szEp0wc50DSNEmbpqWLIKRSCkUIUbFmwQaJDUvW7HkDvAgQ7wDEhhWIk4gESkqqVqRJm5NzUNLEcX0cn+ZZlFgtTfQkf9UdN70+Syfy/Mb9dnx5ZhzD931fAADgpWYGvQAAABA8ggAAABAEAACAIAAAACIIAACACAIAACCCAAAAiCAAAACSwsf9RcMw2rkOPGPt+HtTzMCLpV1/c4w5eLFwLMBxZ+DYQfAiCoVCchxHoVDoqZ/5vi/f91UqldRsNgNY3ZNM05TjOAqHw/I8T/V6XY1GQ7Zty7IsNRoN1et1VavVoJcauMdfq8PU6/XWa8UMnF6macq27SPn4PHX62AOTPPwk6LNZlOlUqltEXUSj+9XtVptzYFlWbIsS81mkzn4VzgcVjwel23bh/68Wq3K8zyVSiVZlqVEInHovPi+r0ajof39fTUajXYv+/91kv16ptt9ps/WYVzX1YULF9TV1aVms6lisahQKCTbthUKheR5nmZnZ+V5XuBvCNFoVBMTE+rt7dXKyop2dnZULBY1PDyss2fPqlAoaHt7W6urq4GusxPYtq2hoSG5rvvE447jyLZt5XK51mvFDJxetm1reHhYyWSy9ZhhGK2AevjwoXZ2drS2tqbu7m5NTEwoGo2qXq+rVCq1Asw0TXmep7m5OVUqlcCjoKurSxMTE0qlUlpdXdWDBw9UKBQ0NDSkM2fOqFQqtfbrZee6rt555x2Njo6qWCyqUqmoXq8rFospHo9rc3NTi4uLmp2d1SuvvKJ3331XPT09qlarymazMk1TiURCkUhEhUJB3333nfL5fOBREIvFNDU1pcHBwdZjpmk+tV8zMzPPdLunNggMw1A6ndb169fV29urfD6v5eVl2batgYEBxeNx5fN53b17V7VaLfA3g0QioStXrmhyclK//fab7ty5I0m6ePGi3n77bW1sbOj27du8GejRG//Y2JiGh4dbj5mmqXQ63XozvX37ttbW1piBU8y2bY2MjGhoaKh1Cts0TaVSKfX29mptbU137txRJpNRKpXStWvX1Nvbq2KxqKWlpdYcxGIx5fN5LS4uqlarqV6vB7pfiURCr7/+us6fP6/p6WnNz8/L931NTk7q2rVr2tra0p07dwgCSel0Wh999JHee+89LS8va3t7W4VCQWNjYxodHdXs7Kx++OEH3bx5U+Pj4/r00081Ojqqvb09TU9PKx6P6+LFi0qn09re3tbPP/+sUqnUEUFw+fJlXbp0SZZlKRwOy7btp/aLIDiBcDisRCIh27a1vr6ub7/9Vn19fbpx44YGBgYk6chTiM9bKBRSLBaT67o6d+6cIpGI9vf3df78efX29qpQKMhxnKCX2RFqtZoePHgg3/dbbwSRSETd3d1KJpNKJpOt1+qoGfjggw+YgRdcrVbT3t7eE49FIhElEgklEgnFYrHWKdeD08WWZWljY0Pff/+90um0bty4of7+fhmG0TFzcPBJ0HVdjY+PKxKJKJfL6dy5c0qlUsrn80eeSn7ZWJalvr4+DQ0NaWFhQZlMRltbWxoYGNDAwIDS6bTi8bikRx8k+vv7FY/Hdfv2bX3xxRcaHx/X559/rgsXLkh6dLzohPsjNjc39fXXXysUCmlqakrj4+MaGxs7dL+epVMdBC+SXC6n33//XfPz86rVamo0Gode94aUTCZ17do1Xbp0Sffv39fW1pay2azm5+d17949ZbNZbW5uBr3ME2MGTiaZTOqNN97Q5OSklpaWtLW1pf39fS0sLOj+/fvKZrPa3t4Oepkn9vDhQ/3xxx+6e/du6/4B5uBwm5ub+uqrr/Tjjz+qWq2qVqspEomoVqsFvbQX0qkOgkajoXK5LMdxFI1GdfXqVUWjUbmuK8/zOuJ64YFyuaz79+9rfX299Zht2+rp6dHIyIjK5TJD/h++78txnCeuIUtSoVBo3Th01Awkk0lm4BQ4uDHUcRx1d3c/8Sn/4D4B6dEclEql1hxcuXJFsVhMiURCnuepXC53zBxUKhUtLS1pY2Oj9Zht23JdV8PDw6pUKszBvx4+fKjp6WnNzc21HovH4xoeHtZrr72mfD6vSqUiSfI8T/v7+0okEurp6dHHH3+sVCqlwcFBFQoF5XK5wC8bHujp6dHVq1d15swZpdNpJRIJJZNJWZalXC73xH49S6c2CA7uHl9ZWVEul5PneXJdV+FwWNlsVsViUcViUdVqtSMOBPV6/anTn5ZlaXl5Wa7rand3V9lsNqDVdRbP85TJZGQYhorFosrl8hN3XB9cB2YGTjfP87SxsSHTNFUoFFSpVJ6ag2azKd/3VSwWlclklM/nVa1WW3Owv7+vYrGoUqnUUXPw339ny7K0urqq7u5u7e3taX9/P5jFdRjP8566pyYajeqvv/7S0NCQVlZWlMlk5Pu+stmsZmZmtLW1pVKppOHhYdm2rUwmo93dXe3t7alcLndEFEQiEfX19WlkZESGYbS+AbGwsNA6G5rJZJ75dg3/mP8DOuG6yklFIhF1dXUd+TWTZrOpfD7fEQNwGMMw5DiOHMdRvV5XrVY7dhWe5u8eh0IhRaPRI6/31Wq11mvFDLRnTZ0gFAqpq6tLkUjk0J8ffDXP8zyFw2FFo9Ejv37abDZVKBQ6eg5s25bjOGo0Gq39Oo7TfCw4jGEYrU/U1WpV5XJZ+XxejuPIdd1D5+Xga4c7OzuB31QqPXrvcl33yHuGDs5q5fP5Yz3fcWfgVAfBy+xlOwjgaac9CHA8HAtw3BnojNtqAQBAoAgCAABAEAAAAIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAAKA2B4FhGIpEIjJNuuNlxb8/ALwYwu188kgkItd1VSgUVCqV2rkpdCDTNJVKpVSpVFQul1WtVoNeEgJimqbi8bjC4bB831e1WlWxWAx6WQAe07YgCIVC6uvr0/vvv68///xT//zzT7s2hQ4VjUb14YcfKpfL6e+//9by8rIqlUrQy8JzZpqmksmkPvvsM42MjKher+vmzZv65ptv1Gw2g14egH+1LQgikYguX76s69evK5vNKpPJqFAotGtz6EDhcFhjY2P65JNPND09rV9//VW3bt3SL7/8It/3g14enhPf91WpVNTV1aU333xTkvTTTz8FvCoA/9W2i7uWZWlqakpvvfWWJiYm5LpuuzaFDhYOh9XT0yPXdZVIJOQ4TtBLwnN2cIlgc3NTe3t7yuVy2tjYIAqBDtOWMwShUEixWEyvvvqqRkdHNTo6qv7+fmUymXZsDh3K932Vy2XNz89rcXFRa2tr2t3dDXpZCECz2dS9e/c0Pz8vy7IIAqADtSUIbNtWKpXS2bNnlc/nlU6nNTExoZmZmXZsDh2q0WhodXVVX375pW7duqWlpSV5nhf0shCQubk5xeNxOY6j9fX1oJcD4D8M/5iZbhjGsZ/UNE1ZlqWhoSGFw2GVy2UVCgVls9n/eaE4mXZ8+jrJDBz8fjKZVKVSUa1W4way56xdn8BPOgcHHMfR4OCgwuGwFhcXn/GqcJROOBYgWMedgbYEwcHvW5YlwzDUbDbVaDTUaDRO9Bz433XKQSAUCqnZbHJ6OACdFgSPHxP4tsnz0ynHAgQn8CBAsDgIoNOCAMHgWIDjzgB/Qg4AABAEAACAIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAACIIAAAACIIAACACAIAACCCAAAAiCAAAAAiCAAAgAgCAAAgyfB93w96EQAAIFicIQAAAAQBAAAgCAAAgAgCAAAgggAAAIggAAAAIggAAIAIAgAAIIIAAABI+j9tRKb3GutOOQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAALk0lEQVR4nO3cS08b9x6H8e+M7ZnB9hhjTKA4FEgIibpIVSlNUVtVlaquu+imUjdddt1930BfRKu+g1bddNWqN5VFFVIikAiEq7kIArbxdXybs+BgJQVUcoRjc3g+ywHh/yQ/zTyeGdvwfd8XAAC40sxOLwAAAHQeQQAAAAgCAABAEAAAABEEAABABAEAABBBAAAARBAAAABJwfP+omEY7VwHLlg7vm+KGbhc2vWdY8zB5cKxAOedgXMHQbcwTVOO4ygYPH3p9Xpd9Xpd1WpVgUBAjuMoEAic+D3f9+X7vkqlkprNZruX/a+e3S/P81Sv19VoNGTbtizLUqPRaO3XVWeapmzbPnMGnv23Op4B0zz9Yliz2VSpVGrbyfNFPLtf1Wq1NQOWZcmyLDWbTWbgGcFgUNFoVLZtn/rzarUqz/NUKpVkWZZc1z11ZnzfV6PRUDabVaPRaPey/9WL7NdVxwxc7AxcuiCwbVvDw8OKx+PPbXccR7ZtK5fLaXd3V+vr64rH47p165Z6enrUbDZVLBYVCARk27YCgYA8z9PMzIw8z+t4FITDYU1MTKi/v19ra2va29tTsVhUKpXSq6++qkKh0Nqvq862baVSKcVisdY2wzBa8XR4eKi9vT1tbGyot7dXExMTCofDqtfrKpVKrfgyTVOe52l2dlaVSqXjUdDT06OJiQklEgmtr6/r6dOnKhQKGh4e1vXr11UqlVr7BSkej+vdd9/V6OioisWiKpWK6vW6IpGIotGotre3tbS0pJmZGb3yyit677331NfXp2q1qkwmI9M05bquQqGQCoWCvv/+e+Xz+Y6fECKRiO7evauhoaHWNtM0T+zXgwcPOrjK7sAMXOwMXLogcBxHY2NjSqVSrW2maSqZTLZOpnNzc9rY2FAymdQ777yj/v5+5fN5ra6uyrZtDQ4OKhqNKp/P6/Hjx6rVah0PAtd1de/ePU1OTur333/X/Py8JOn27dt6++23tbW1pbm5OYJAR0EwMjKi4eHh1qVL0zSVSCTU39+vjY0Nzc/PK51OK5FIaGpqSv39/SoWi1pZWWnNQCQSUT6f19LSkmq1mur1ekf3y3VdvfHGG7p586amp6e1sLAg3/c1OTmpqakp7ezsaH5+niD4r2QyqY8//ljvv/++VldXtbu7q0KhoLGxMY2OjmpmZkY//vijHj58qPHxcX322WcaHR3VwcGBpqenFY1Gdfv2bSWTSe3u7uqXX35RqVTqipPBa6+9pjt37siyLAWDQdm2fWK/CAJm4MoHQa1W09OnT+X7futkEAqF1Nvbq1gsplgsJsdxJB1ddnFdV7Zta3NzU999950GBgb04YcfanBwUJLOvJT8sgUCAUUiEcXjcd24cUOhUEjZbFY3b95Uf3+/CoVCa7+uulqtpoODg+e2hUIhua4r13UViURal9qOLxNalqWtrS398MMPSiaT+uCDD3Tt2jUZhtE1M3D8DiAej2t8fFyhUEi5XE43btxQIpFQPp8/8xLiVWRZlgYGBjQ8PKzFxUWl02nt7OxocHBQg4ODSiaTikajko7eSFy7dk3RaFRzc3P68ssvNT4+ri+++EK3bt2SdHS86IZ749vb2/rmm28UCAR09+5djY+Pa2xs7NT9uuqYgYt16YIgFotpampKd+7c0fLysnZ2dpTJZLSwsKAnT54ok8loe3u708t8YblcTn/88YcWFhZUq9XUaDROffYBRzPw5ptvanJyUisrK9rZ2VE2m9Xi4qKWl5eVyWS0u7vb6WW+sMPDQ/355596/Phx6/kBZuBs29vb+vrrr/XTTz+pWq2qVqspFAqpVqt1eml4SZiBi3XpguCY7/tyHOe5+8iSVCgUWg+NNBoNlctlOY6jcDis+/fvKxwOKxaLyfO8rrhvfKxcLmt5eVmbm5utbbZtq6+vTyMjIyqXywz5M44fCnUcR729vc+9yz9+TkA6moFSqdSagXv37ikSich1XXmep3K53DUzUKlUtLKyoq2trdY227YVj8eVSqVUqVSYgWccHh5qenpas7OzrW3RaFSpVEqvv/668vm8KpWKJMnzPGWzWbmuq76+Pn3yySdKJBIaGhpSoVBQLpfr+G3DY319fbp//76uX7+uZDIp13UVi8VkWZZyudxz+3XVMQMX69IFged5SqfTMgxDxWJR5XL5uaeuj+8FH3+CYG1tTblcTp7nKR6PKxgMKpPJqFgsqlgsqlqtdsUJoV6vn7gMblmWVldXFY/Htb+/r0wm06HVdRfP87S1tSXTNFUoFFSpVE7MQLPZlO/7KhaLSqfTyufzqlarrRnIZrMqFosqlUpdNQP//D+2LEvr6+vq7e3VwcGBstlsZxbXhTzPO/FMTTgc1l9//aXh4WGtra0pnU7L931lMhk9ePBAOzs7KpVKSqVSsm1b6XRa+/v7Ojg4ULlc7ooTQigU0sDAgEZGRmQYRuvp98XFxdbV0HQ63elldgVm4GIZ/jmPhN1wX0U6utceDofPvNdTq9VUq9VUqVQUCoXU09Nz5sdMms2m8vl8VwzAaQzDkOM4chxH9Xq9tV/n8f/82eNAIKCenh6FQqFTf3780TzP8xQMBhUOh8/86Gmz2VShUOjqGbBtW47jqNFotPbrPK7i9xAYhtF6N1WtVlUul5XP5+U4juLx+Kkzc/yRs729vY4/WCodnQzi8fiZzwwdX9nK5/Pn+nv/z8eC0zADJ513Bi5dEOB8rtpBACddxSDASRwLcN4Z6I7HqwEAQEcRBAAAgCAAAAAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABAbQ6CUCgk06Q5AADodsF2/WHTNJVIJFSpVFQul1WtVtv1UuhipmkqGo0qGAzK931Vq1UVi8VOLwsA8A9tC4JwOKyPPvpIuVxOf//9t1ZXV1WpVNr1cuhCpmkqFovp888/18jIiOr1uh4+fKhvv/1WzWaz08sDADyjbUEQDAY1NjamTz/9VNPT0/rtt9/06NEj/frrr/J9v10viy7i+74qlYp6enr01ltvSZJ+/vnnDq8KAHCatgWBdBQFfX19isfjcl1XjuO08+XQZY5vEWxvb+vg4ECBQEBbW1sEIQB0obYFge/7KpfLWlhY0NLSkjY2NrS/v9+ul0OXajabevLkiRYWFmRZFkEAAF2qbUHQaDS0vr6ur776So8ePdLKyoo8z2vXy6GLzc7OKhqNynEcbW5udno5AIBTGP45364ZhvFif9gwFIvFVKlUVKvVeIjsJWvHu/AXnYFjjuNoaGhIwWBQS0tLF7wqnKVdV2L+1zlAZ3TTsQCdcd4ZaFsQSFIgEFCz2eQScQd000HAMAxZliXDMPikyUtEEEDqrmMBOqMrggCdw0EABAEkjgU4/wzwNYIAAIAgAAAABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAAEQQAAEAEAQAAEEEAAABEEAAAABEEAABABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAASYbv+36nFwEAADqLKwQAAIAgAAAABAEAABBBAAAARBAAAAARBAAAQAQBAAAQQQAAAEQQAAAASf8BUsOz3QJBrbQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAIsUlEQVR4nO3czW9U9R7H8c9MpzNDO1NKKQ+2EKgiGhcYEx9YGGPin+DGpUvX7v1XNP4Jxo0rjStZGCAkkiAoCrVtQPvAtMPMdDrnLrg0l9CbW5G5LeH1Wp4hc36/5Jszb07PTKkoiiIAwHOtvNcLAAD2niAAAAQBACAIAIAIAgAgggAAiCAAACIIAIAkld3+w1KpNMx18JQN4/emzMCzZVi/OWYOni2uBex2BnYdBPtFuVxOrVZLpbLz0re2ttLv99Pr9TIyMpJ6vZ5yeecbIYPBIO12e2gXzr/jP/fV6/XS7/eztbWVarWaarWawWCwva/nXaVSSaPRSK1W2/H1Xq+XbrebdrudarWaZrO547wURZGtra2srq5ma2tr2Mv+n/7OvjAH5sAMPO0ZeOaCoFarZXZ2NhMTE9vHSqVSarVaqtVq7t27l7t37+b27ds5ePBgzpw5k7GxsfT7/bTb7ZTL5e1I6Ha7uXLlSjqdzp5HwYEDB3LmzJlMTU3l1q1b+fPPP7O+vp6ZmZmcOHEi7XZ7e1/Pu8nJybz77rs5depUNjY20ul00u/3Mz4+nkajkcXFxdy4cSOXLl3KCy+8kPfeey+HDh1Kr9fLyspKyuVyms1mRkdHs76+nq+++iqtVmvPLwTj4+M5d+5cjh8/vn2sXC4/tq+LFy/u4Sr3D3NgDszA052BZzIITp48mZmZme3bVuVyOVNTUzl8+HBu376dq1evZn5+PlNTUzl//nwOHz6cjY2N3Lx5M7VaLceOHcv4+HharVZu3LiRzc3N9Pv9Pd1Xs9nMG2+8kZdeeikXLlzItWvXUhRFzp49m/Pnz2dpaSlXr14VBEmmp6fz4Ycf5v33389vv/2WO3fuZH19PadPn86pU6dy6dKlfPPNN7l8+XLm5uby8ccf59SpU1leXs6FCxfSaDTyyiuvZHp6Onfu3Mn333+fdru9Ly4Cr732Wl599dVUq9VUKpXUarXH9uWD4AFzYA7MwHMeBJubm1leXn7k2OjoaJrNZprNZsbHx7dvszy8RVStVrOwsJCvv/4609PT+eCDD3L06NGUSqX/+ueE/7eH9Tc5OZm5ubmMjo5mbW0tL774YqamptJqtf7r7aPnTbVazZEjRzIzM5Pr169nfn4+S0tLOXbsWI4dO5bp6ek0Go0kSb1ez9GjR9NoNPLTTz/ls88+y9zcXD799NO8/PLLSR7cntsPfxNdXFzMF198kZGRkZw7dy5zc3M5ffr0jvvCHGAGnrZnLggmJiby1ltv5ezZs7l582aWlpayurqa69ev59dff83Kykru3Lmz18v82+7du5cffvghP//88/bzAyMjI3u9rH1pcXExn3/+eb799tv0er1sbm5mdHQ0m5ube700/o/MAWbg6XrmgiB58ABIURSp1+s5ePDgI//Lf/icQPLgAcN2u516vZ6xsbG8+eabGR8fT7PZTLfbzf379/f82YGHOp1Obt68mYWFhe1jtVotk5OTmZ2dTafTMeT/du/evVy4cCFXrlzZPtZoNDI7O5vXX389rVYrnU4nSdLtdrO6uppms5lDhw7lo48+ytTUVI4fP5719fWsra1lMBjs1VYecejQobz99ts5ceJEpqen02w2MzExkWq1mrW1tUf2hTnADDxtz1wQdLvdLCwspFwuZ319PZ1O55En7zc3NzMYDFIURTY2NjI/P59Wq5Ver5fJyclUKpWsrq5mY2Mj7XY7vV5vX0RBv9/PysrKI8eq1Wpu3bqVgwcPZnl5Oaurq3uzuH2m2+3m1q1bjxwbGxvLjz/+mJmZmfz++++Zn59PURRZWVnJxYsXs7S0lHa7ndnZ2dRqtczPz+evv/7K8vJy7t+/vy8uBKOjozly5EhOnjyZUqm0/dTz9evXs7KykmvXrmV+fn6vl7lvmAPMwNNVKnb5abgf/q6SJCMjIzlw4EBGR0d3fP3hV/O63W4qlUrGxsZ2vPVeFEUGg0HW19f3xQDs5OG3J+r1era2trb3tRvP23ePS6XSdkX3er3cv38/rVYr9Xo9k5OTO87Lw68a3b17d88fKk0eXAQmJydTr9d3fP3hXa1Wq7Wr93sef4fAHDzOtcAM7HYGnrkgYHeet4sAj3seg4DHuRaw2xnYH4/YAwB7ShAAAIIAABAEAEAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAEAEAQAQQQAARBAAABEEAECSyjDfvFwup9FopFKppCiK9Hq9bGxsDPOUAMATGFoQlMvlTExM5JNPPsnJkyfT7/dz+fLlfPnllxkMBsM6LQDwBIYWBEVRpNPp5MCBA3nnnXeSJN99992wTgcA/ANDDYJer5fFxcUsLy9nZGQkCwsLKYpiWKcEAJ7QUJ8hGAwG+eWXX3Lt2rVUq1VBAAD71FCDIEmuXLmSRqORer2eP/74Y9inAwCewNCDYG1tLZcuXdr+pgEAsP+Uil1+SpdKpSc7QamUarWaUqmUTqfzRO/B3zeM+HrSGWBvDCvAzcGzxbWA3c7A0O8QFEWRbrc77NMAAP+AXyoEAAQBACAIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAAiCAAACIIAIAIAgAgggAASFIqiqLY60UAAHvLHQIAQBAAAIIAAIggAAAiCACACAIAIIIAAIggAAAiCACAJP8CC8o1StccI6gAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAB/CAYAAACQeNq9AAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFPklEQVR4nO3cP0+TewPG8avYFiKtKaYIgqYwGI0Dbk7G+B5cHH0N7r4VfQ1uTiZuDAaMiYPBQU0TCEb+2FpoAXuGk0Ni6snDk+BTePr5jAWS+5dcufNNS+9Cv9/vBwAYaWPDvgAAYPgEAQAgCAAAQQAARBAAABEEAEAEAQAQQQAAJCme9BcLhcKfvA5O2Z943pQNnC9/6pljdnC+uBdw0g2cOAjOimKxmEqlkvHx8d/+vNfrpdvtptPppFwup1qtplgcPGa/38/R0VF2dnZydHT0py/7P/pvzjXqbMAGEjuwAxs47Q2cuyCo1Wq5d+9eGo1Gfvz4kf39/RweHmZycjKVSiXr6+v5+PFjVldXc/Xq1dy/fz9TU1Pp9XrZ3t7O2NhYqtVqSqVS2u12Xrx4kVarNfQRTE5OZmlpKbOzs8evjY2NDZxrZWVliFd5NtiADSR2YAc2cNobOHdBUK/X8/Dhwzx48CCfPn3K5uZm2u12FhYW0mg0srq6mpcvX+bt27dZXFzM48eP02g0srW1leXl5VQqldy8eTP1ej2bm5t5/fp1Op3OmRjA7du3c+vWrZTL5RSLxYyPjw+cy03ABmzgb3ZgBzYw4kFQLpczPT2dubm5rK2tpdlsZmNjIzMzM5mZmUm9Xk+lUkmSTExM5MqVK6lUKnn//n2ePn2axcXFPHnyJDdu3Ejy91szZ+HzsPX19Tx//jwXLlzI0tJSFhcXs7Cw8NtzjTobILEDbOC0nbsgWF9fz7Nnz/Lq1av0er0cHBykVCrl4OBg2JfG/4gNkNgBNnDazl0QfP/+PcvLy3n37t3xa5VKJfPz87lz505arVb29/eTJN1uNzs7O6lWq5mamsqjR49y+fLlzM7Opt1uZ3d3Nz9//hzWUX4xNTWVu3fv5tq1a6nX66lWq7l06VLK5XJ2d3d/OdeoswESO8AGTtu5C4Jut5svX7788trFixfz5s2bzM3N5fPnz2k2m+n3+9ne3s7Kyko2NjbS6XQyPz+f8fHxNJvNfPv2LVtbW9nb2zsTIyiVSpmens7169dTKBSO/+N1bW0t29vb+fDhQ5rN5rAv80ywARI7wAZOW6F/wi8onoXPVf5NoVA4Lqher5e9vb20Wq1MTEykVqulVCoN/M0/XzP5+vVrDg8Ph3DVvyqVSqnVapmYmPjtz7vd7vG5TmLUvntsA4NG8TkEdjDIvcAGTrqB/4sgYNCo3QQYNIpBwCD3Ak66AY8uBgAEAQAgCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAAAiCACACAIAIIIAAIggAACSFPr9fn/YFwEADJd3CAAAQQAACAIAIIIAAIggAAAiCACACAIAIIIAAIggAACS/AUN2LnnIEaHRQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the first 4 frames of the first rollout\n",
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(12, 12))\n",
    "\n",
    "for j in range(10):\n",
    "    for i in range(4):\n",
    "        plt.subplot(1, 4, i+1)\n",
    "        plt.imshow(data[\"pixels\"][j, i].cpu().numpy(), cmap=\"gray\")\n",
    "        plt.axis(\"off\")\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchrl.envs import (\n",
    "    CatFrames,\n",
    "    DoubleToFloat,\n",
    "    EndOfLifeTransform,\n",
    "    GrayScale,\n",
    "    GymEnv,\n",
    "    NoopResetEnv,\n",
    "    Resize,\n",
    "    RewardSum,\n",
    "    SignTransform,\n",
    "    StepCounter,\n",
    "    ToTensorImage,\n",
    "    TransformedEnv,\n",
    "    VecNorm,\n",
    "    ObservationNorm\n",
    ")\n",
    "\n",
    "def make_env(env_name=\"LunarLander-v2\", \n",
    "             frame_skip = 4, \n",
    "             device=\"cpu\", \n",
    "             seed = 0, \n",
    "             obs_norm_sd = None):\n",
    "\n",
    "    if obs_norm_sd is None:\n",
    "        obs_norm_sd = {\"standard_normal\": True}\n",
    "\n",
    "    env = GymEnv(\n",
    "        env_name,\n",
    "        frame_skip=frame_skip,\n",
    "        from_pixels=True,\n",
    "        pixels_only=False,\n",
    "        device=device,\n",
    "    )\n",
    "    env = TransformedEnv(env)\n",
    "\n",
    "    env.append_transform(ToTensorImage()) \n",
    "    env.append_transform(GrayScale())\n",
    "    env.append_transform(Resize(84, 84))\n",
    "    env.append_transform(RewardSum())\n",
    "    env.append_transform(StepCounter()) # NOTE: GridWordv0 has a max of 200 steps\n",
    "    env.append_transform(DoubleToFloat())\n",
    "    env.append_transform(ObservationNorm(in_keys=[\"pixels\"], **obs_norm_sd))\n",
    "    env.set_seed(seed)\n",
    "\n",
    "    # NOTE: a rollout will be take a trajectory of frames and group the frames by N=4 sequentially\n",
    "    # so that the output will be 7x4x84x84 because with a rollout of 10 steps we will have 7 groups of\n",
    "    # 4 frames each\n",
    "    return env\n",
    "\n",
    "def get_norm_stats(num_iter = 250):\n",
    "    # NOTE: num_iter depends of the complexity of the input images (set to 1000 is a good approach)\n",
    "    test_env = make_env()\n",
    "    test_env.set_seed(0)\n",
    "    test_env.transform[-1].init_stats(\n",
    "        num_iter=num_iter, cat_dim=0, reduce_dim=[-1, -2, -4], keep_dims=(-1, -2)\n",
    "    )\n",
    "    obs_norm_sd = test_env.transform[-1].state_dict()\n",
    "    # let's check that normalizing constants have a size of ``[C, 1, 1]`` where\n",
    "    # ``C=4`` (because of :class:`~torchrl.envs.CatFrames`).\n",
    "    print(\"state dict of the observation norm:\", obs_norm_sd)\n",
    "    test_env.close()\n",
    "    del test_env\n",
    "    return obs_norm_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n",
      "state dict of the observation norm: OrderedDict([('standard_normal', tensor(True)), ('loc', tensor([[[0.2449]]])), ('scale', tensor([[[0.4242]]]))])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OrderedDict([('standard_normal', tensor(True)),\n",
       "             ('loc', tensor([[[0.2449]]])),\n",
       "             ('scale', tensor([[[0.4242]]]))])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = make_env(env_name=\"LunarLander-v2\")\n",
    "\n",
    "obs_norm_sd = get_norm_stats(num_iter = 1000)\n",
    "obs_norm_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state dict of the observation norm: OrderedDict([('standard_normal', tensor(True)), ('loc', tensor([[[0.2449]]])), ('scale', tensor([[[0.4242]]]))])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OrderedDict([('standard_normal', tensor(True)),\n",
       "             ('loc', tensor([[[0.2449]]])),\n",
       "             ('scale', tensor([[[0.4242]]]))])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs_norm_sd = get_norm_stats(num_iter = 1000)\n",
    "obs_norm_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchrl.data import LazyTensorStorage, TensorDictReplayBuffer, LazyMemmapStorage\n",
    "import tempfile\n",
    "\n",
    "tempdir = tempfile.TemporaryDirectory()\n",
    "scratch_dir = tempdir.name\n",
    "\n",
    "replay_buffer = TensorDictReplayBuffer(\n",
    "    pin_memory=False,\n",
    "    prefetch=8,\n",
    "    storage=LazyMemmapStorage( # NOTE: additional line\n",
    "        max_size=1000,\n",
    "        scratch_dir=scratch_dir,\n",
    "    ),\n",
    "    batch_size=100,\n",
    "    priority_key=\"total_priority\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([10, 11, 12, 13, 14, 15, 16, 17, 18, 19])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "replay_buffer.extend(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 4])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "replay_buffer['observation'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acrobot-v1',\n",
       " 'Ant-v2',\n",
       " 'Ant-v3',\n",
       " 'Ant-v4',\n",
       " 'BipedalWalker-v3',\n",
       " 'BipedalWalkerHardcore-v3',\n",
       " 'Blackjack-v1',\n",
       " 'CarRacing-v2',\n",
       " 'CartPole-v0',\n",
       " 'CartPole-v1',\n",
       " 'CliffWalking-v0',\n",
       " 'FrozenLake-v1',\n",
       " 'FrozenLake8x8-v1',\n",
       " 'GymV21Environment-v0',\n",
       " 'GymV26Environment-v0',\n",
       " 'HalfCheetah-v2',\n",
       " 'HalfCheetah-v3',\n",
       " 'HalfCheetah-v4',\n",
       " 'Hopper-v2',\n",
       " 'Hopper-v3',\n",
       " 'Hopper-v4',\n",
       " 'Humanoid-v2',\n",
       " 'Humanoid-v3',\n",
       " 'Humanoid-v4',\n",
       " 'HumanoidStandup-v2',\n",
       " 'HumanoidStandup-v4',\n",
       " 'InvertedDoublePendulum-v2',\n",
       " 'InvertedDoublePendulum-v4',\n",
       " 'InvertedPendulum-v2',\n",
       " 'InvertedPendulum-v4',\n",
       " 'LunarLander-v2',\n",
       " 'LunarLanderContinuous-v2',\n",
       " 'MountainCar-v0',\n",
       " 'MountainCarContinuous-v0',\n",
       " 'Pendulum-v1',\n",
       " 'Pusher-v2',\n",
       " 'Pusher-v4',\n",
       " 'Reacher-v2',\n",
       " 'Reacher-v4',\n",
       " 'Swimmer-v2',\n",
       " 'Swimmer-v3',\n",
       " 'Swimmer-v4',\n",
       " 'Taxi-v3',\n",
       " 'Walker2d-v2',\n",
       " 'Walker2d-v3',\n",
       " 'Walker2d-v4',\n",
       " 'phys2d/CartPole-v0',\n",
       " 'phys2d/CartPole-v1',\n",
       " 'phys2d/Pendulum-v0',\n",
       " 'tabular/Blackjack-v0',\n",
       " 'tabular/CliffWalking-v0']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.available_envs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 1, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 1, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 2, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 6, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 4, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 5, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n",
      "Action: 4, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "dict_keys(['image', 'direction', 'mission'])\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "env = gym.make(\"MiniGrid-Empty-5x5-v0\") #, render_mode=\"human\")\n",
    "observation, info = env.reset(seed=42)\n",
    "for _ in range(10):\n",
    "   # Random action\n",
    "   action = np.random.randint(env.action_space.n)\n",
    "   observation, reward, terminated, truncated, info = env.step(action)\n",
    "\n",
    "   print(f\"Action: {action}, Reward: {reward}\")\n",
    "   print(f\"Observation shape: {observation['image'].shape}\")\n",
    "   print(observation.keys())\n",
    "\n",
    "   if terminated or truncated:\n",
    "      observation, info = env.reset()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action Space: Discrete(7)\n",
      "Action Space Type: <class 'gymnasium.spaces.discrete.Discrete'>\n",
      "Observation Space: Dict('direction': Discrete(4), 'image': Box(0, 255, (7, 7, 3), uint8), 'mission': MissionSpace(<function EmptyEnv._gen_mission at 0x7f9da0186980>, None))\n",
      "Observation Space Type: <class 'gymnasium.spaces.dict.Dict'>\n",
      "Number of actions: 7\n",
      "Action: 2, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 6, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 6, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 6, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 0, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 4, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 4, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n",
      "Action: 4, Reward: 0\n",
      "Observation shape: (7, 7, 3)\n",
      "Observation keys: dict_keys(['image', 'direction', 'mission'])\n",
      "Mission: get to the green goal square\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "import numpy as np\n",
    "\n",
    "# Create the environment\n",
    "env = gym.make(\"MiniGrid-Empty-5x5-v0\") #, render_mode=\"human\")\n",
    "\n",
    "# Reset the environment\n",
    "observation, info = env.reset(seed=42)\n",
    "\n",
    "# Print action space details\n",
    "print(\"Action Space:\", env.action_space)\n",
    "print(\"Action Space Type:\", type(env.action_space))\n",
    "\n",
    "# Print observation space details\n",
    "print(\"Observation Space:\", env.observation_space)\n",
    "print(\"Observation Space Type:\", type(env.observation_space))\n",
    "\n",
    "# Print more detailed information if it's a Box or Discrete space\n",
    "if isinstance(env.action_space, gym.spaces.Discrete):\n",
    "    print(\"Number of actions:\", env.action_space.n)\n",
    "\n",
    "if isinstance(env.observation_space, gym.spaces.Box):\n",
    "    print(\"Observation Space Shape:\", env.observation_space.shape)\n",
    "    print(\"Observation Space High:\", env.observation_space.high)\n",
    "    print(\"Observation Space Low:\", env.observation_space.low)\n",
    "\n",
    "# Step through the environment\n",
    "for _ in range(10):\n",
    "    # Random action\n",
    "    action = np.random.randint(env.action_space.n)\n",
    "    observation, reward, terminated, truncated, info = env.step(action)\n",
    "\n",
    "    print(f\"Action: {action}, Reward: {reward}\")\n",
    "    print(f\"Observation shape: {observation['image'].shape}\")\n",
    "    print(\"Observation keys:\", observation.keys())\n",
    "    print(\"Mission:\", observation['mission'])\n",
    "\n",
    "    if terminated or truncated:\n",
    "        observation, info = env.reset()\n",
    "\n",
    "# Close the environment\n",
    "env.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n",
      "(2,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "display display: Unable to load font (-*-helvetica-medium-r-normal--12-*-*-*-*-*-iso8859-1) [Resource temporarily unavailable].\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "from gymnasium import spaces\n",
    "import numpy as np\n",
    "from PIL import Image, ImageDraw\n",
    "\n",
    "class SimpleGridEnv(gym.Env):\n",
    "    def __init__(self, grid_size=5):\n",
    "        super(SimpleGridEnv, self).__init__()\n",
    "        self.grid_size = grid_size\n",
    "        self.action_space = spaces.Discrete(4)  # 4 actions: left, right, up, down\n",
    "        self.observation_space = spaces.Box(low=0, high=grid_size-1, shape=(2,), dtype=np.int32)\n",
    "        self.state = None\n",
    "        self.goal = np.array([grid_size-1, grid_size-1])  # Goal position\n",
    "    \n",
    "    def reset(self):\n",
    "        self.state = np.array([0, 0])  # Starting position\n",
    "        return self.state\n",
    "    \n",
    "    def step(self, action):\n",
    "        if action == 0:   # Left\n",
    "            self.state[1] = max(0, self.state[1] - 1)\n",
    "        elif action == 1: # Right\n",
    "            self.state[1] = min(self.grid_size - 1, self.state[1] + 1)\n",
    "        elif action == 2: # Up\n",
    "            self.state[0] = max(0, self.state[0] - 1)\n",
    "        elif action == 3: # Down\n",
    "            self.state[0] = min(self.grid_size - 1, self.state[0] + 1)\n",
    "        \n",
    "        done = np.array_equal(self.state, self.goal)\n",
    "        reward = 1 if done else -0.1  # Reward for reaching the goal\n",
    "        \n",
    "        return self.state, reward, done, {}\n",
    "    \n",
    "    def render(self, mode='human'):\n",
    "        cell_size = 84 // self.grid_size\n",
    "        img = Image.new('RGB', (84, 84), color='white')\n",
    "        draw = ImageDraw.Draw(img)\n",
    "        \n",
    "        for i in range(self.grid_size):\n",
    "            for j in range(self.grid_size):\n",
    "                top_left = (j * cell_size, i * cell_size)\n",
    "                bottom_right = ((j + 1) * cell_size, (i + 1) * cell_size)\n",
    "                draw.rectangle([top_left, bottom_right], outline='black')\n",
    "        \n",
    "        # Draw agent\n",
    "        agent_top_left = (self.state[1] * cell_size, self.state[0] * cell_size)\n",
    "        agent_bottom_right = ((self.state[1] + 1) * cell_size, (self.state[0] + 1) * cell_size)\n",
    "        draw.rectangle([agent_top_left, agent_bottom_right], fill='blue')\n",
    "        \n",
    "        # Draw goal\n",
    "        goal_top_left = (self.goal[1] * cell_size, self.goal[0] * cell_size)\n",
    "        goal_bottom_right = ((self.goal[1] + 1) * cell_size, (self.goal[0] + 1) * cell_size)\n",
    "        draw.rectangle([goal_top_left, goal_bottom_right], fill='green')\n",
    "        \n",
    "        if mode == 'human':\n",
    "            img.show()\n",
    "        elif mode == 'rgb_array':\n",
    "            return np.array(img)\n",
    "\n",
    "# Example usage\n",
    "env = SimpleGridEnv(grid_size=5)\n",
    "observation = env.reset()\n",
    "env.render(mode='human')\n",
    "\n",
    "for _ in range(10):\n",
    "    action = env.action_space.sample()  # Random action\n",
    "    observation, reward, done, info = env.step(action)\n",
    "    print(observation.shape)\n",
    "    # env.render(mode='human')\n",
    "    if done:\n",
    "        print(\"Goal reached!\")\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action Space: Discrete(2)\n",
      "Action Space Type: <class 'gymnasium.spaces.discrete.Discrete'>\n",
      "Observation Space: Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)\n",
      "Observation Space Type: <class 'gymnasium.spaces.box.Box'>\n",
      "Number of actions: 2\n",
      "Observation Space Shape: (4,)\n",
      "Observation Space High: [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\n",
      "Observation Space Low: [-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "\n",
    "# Create the CartPole environment\n",
    "env = gym.make(\"CartPole-v1\")\n",
    "\n",
    "# Print action space details\n",
    "print(\"Action Space:\", env.action_space)\n",
    "print(\"Action Space Type:\", type(env.action_space))\n",
    "\n",
    "# Print observation space details\n",
    "print(\"Observation Space:\", env.observation_space)\n",
    "print(\"Observation Space Type:\", type(env.observation_space))\n",
    "\n",
    "# Print more detailed information if it's a Box or Discrete space\n",
    "if isinstance(env.action_space, gym.spaces.Discrete):\n",
    "    print(\"Number of actions:\", env.action_space.n)\n",
    "\n",
    "if isinstance(env.observation_space, gym.spaces.Box):\n",
    "    print(\"Observation Space Shape:\", env.observation_space.shape)\n",
    "    print(\"Observation Space High:\", env.observation_space.high)\n",
    "    print(\"Observation Space Low:\", env.observation_space.low)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzElEQVR4nO3c0WrjMBRAwWjJh/fPtQ+CA5t1QA44TWDm2aHiInxQ1HbMOecNAG6325/fXgAAn0MUAIgoABBRACCiAEBEAYCIAgARBQBy331wjHHlOgA44aq/O3ZSACCiAEBEAYBs3yk88n/0luO7FrNZ/p2NPbMc7RmzWR5nYy7LO+90nRQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADImHPOrQfHuHotAGzafHWf5qQAQEQBgIgCABEFAHJ/9YNXXXJ8m6MLeLNZHmdjLos985w9c+ydv+jjpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEDGnHNuPTjG1WsBYNPmq/s0JwUAIgoARBQAyP3VD171fda3ObprMZvlcTbmstgzz9kzx955p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMacc249OMbVawFg0+ar+zQnBQAiCgBEFACIKACQ+6sfvOqS49scXcCbzfI4G3NZ7Jnn7Jlj7/xFHycFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMuacc+vBMa5eCwCbNl/dpzkpABBRACCiAEDur37wqu+zvs3RXYvZLI+zMZfFnnnuv9n8/MoyPs/P+36UkwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgAZc8659eAYV68FgE2br+7TnBQAiCgAEFEAIKIAQO67D151qQHA53BSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAOQvUgt2+H5btgsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFy0lEQVR4nO3cwYrqMABAUfPoh8+f5y0Cd+FUiEIdhXPWKRNC6CXGccw55w0Abrfbv7+eAACfQxQAiCgAEFEAIKIAQEQBgIgCABEFAHLsDhxjXDkPAJ5w1f8dOykAEFEAIKIAQLbvFH7zO3rL77sWvzG43N9DWZfl7H7O2iz2zLl33uk6KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoAJAx55xbA8e4ei4AbNp8dT/NSQGAiAIAEQUAIgoA5Hj1wasuOb7N2QW8tVnu18a6LPbMY/bMuXd+0cdJAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIw559waOMbVcwFg0+ar+2lOCgBEFACIKACQ49UHr/o869uc3bVYm+V+bazLYs88Zs+ce+edrpMCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAGXPOuTVwjKvnAsCmzVf305wUAIgoABBRACCiAECOVx+86pLj25xdwFub5X5trMtizzxmz5x75xd9nBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADImHPOrYFjXD0XADZtvrqf5qQAQEQBgIgCADleffCqz7O+zdldi7VZ7tfGuiz2zGO/1ubnT6bxeX7e96ecFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMiYc86tgWNcPRcANm2+up/mpABARAGAiAIAEQUAcuwOvOpSA4DP4aQAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAyH8Z7Xb4gofuFwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzElEQVR4nO3c0WrjMBRAwWjJh/fPtQ+CA5t1QA44TWDm2aHiInxQ1HbMOecNAG6325/fXgAAn0MUAIgoABBRACCiAEBEAYCIAgARBQBy331wjHHlOgA44aq/O3ZSACCiAEBEAYBs3yk88n/0luO7FrNZ/p2NPbMc7RmzWR5nYy7LO+90nRQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADImHPOrQfHuHotAGzafHWf5qQAQEQBgIgCABEFAHJ/9YNXXXJ8m6MLeLNZHmdjLos985w9c+ydv+jjpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEDGnHNuPTjG1WsBYNPmq/s0JwUAIgoARBQAyP3VD171fda3ObprMZvlcTbmstgzz9kzx955p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMacc249OMbVawFg0+ar+zQnBQAiCgBEFACIKACQ+6sfvOqS49scXcCbzfI4G3NZ7Jnn7Jlj7/xFHycFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMuacc+vBMa5eCwCbNl/dpzkpABBRACCiAEDur37wqu+zvs3RXYvZLI+zMZfFnnnuv9n8/MoyPs/P+36UkwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgAZc8659eAYV68FgE2br+7TnBQAiCgAEFEAIKIAQO67D151qQHA53BSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAOQvUgt2+H5btgsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFy0lEQVR4nO3cwYrqMABAUfPoh8+f5y0Cd+FUiEIdhXPWKRNC6CXGccw55w0Abrfbv7+eAACfQxQAiCgAEFEAIKIAQEQBgIgCABEFAHLsDhxjXDkPAJ5w1f8dOykAEFEAIKIAQLbvFH7zO3rL77sWvzG43N9DWZfl7H7O2iz2zLl33uk6KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoAJAx55xbA8e4ei4AbNp8dT/NSQGAiAIAEQUAIgoA5Hj1wasuOb7N2QW8tVnu18a6LPbMY/bMuXd+0cdJAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIw559waOMbVcwFg0+ar+2lOCgBEFACIKACQ49UHr/o869uc3bVYm+V+bazLYs88Zs+ce+edrpMCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAGXPOuTVwjKvnAsCmzVf305wUAIgoABBRACCiAECOVx+86pLj25xdwFub5X5trMtizzxmz5x75xd9nBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADImHPOrYFjXD0XADZtvrqf5qQAQEQBgIgCADleffCqz7O+zdldi7VZ7tfGuiz2zGO/1ubnT6bxeX7e96ecFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMiYc86tgWNcPRcANm2+up/mpABARAGAiAIAEQUAcuwOvOpSA4DP4aQAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAyH8Z7Xb4gofuFwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzUlEQVR4nO3cwWrrMAAAweiRD++f6x0Me3AdUAJKE5g5y1QI4UWV2zHnnDcAuN1u//56AgB8DlEAIKIAQEQBgIgCABEFACIKAEQUAMh9deAYY+c8AHjCrr87dlIAIKIAQEQBgCzfKZz5P3qHq7sWa3M4r411Odgzj9kz1955p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMaccy4NHGP3XABYtPjqfpqTAgARBQAiCgBEFADI/fVH91xyfJ/fF/C7LoC+zfnjBOtyuPpow9oc7Jlr7/zQx0kBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAjDnnXBo4xu65ALBo8dX9NCcFACIKAEQUAMj91Qd3/T7r21zdtVibw3ltrMvBnnnMnrn2zjtdJwUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAy5pxzaeAYu+cCwKLFV/fTnBQAiCgAEFEAIKIAQO6vPrjrkuPbXF3AW5vDeW2sy8GeecyeufbOD32cFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMiYc86lgWPsngsAixZf3U9zUgAgogBARAGA3F99cNfvs77N1V2LtTmc18a6HOyZx36tzc+fTOPz/LzvRzkpABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAkDHnnEsDx9g9FwAWLb66n+akAEBEAYCIAgARBQByXx2461IDgM/hpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADIf/J+dvjBnOyiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzUlEQVR4nO3cwWrjMBRA0WjIh/fPNQvDXbgOKAGlCZyzdqh4CF8UtR1zznkDgNvt9u+vFwDA5xAFACIKAEQUAIgoABBRACCiAEBEAYDcVx8cY+xcBwBP2PV3x04KAEQUAIgoAJDlO4Uz/0fvcHXXYjaH82zM5WDPPGbPXHvnna6TAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABlzzrn04Bi71wLAosVX99OcFACIKAAQUQAgogBA7q9+cNclx7e5uoA3m8N5NuZysGces2euvfMXfZwUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAyJhzzqUHx9i9FgAWLb66n+akAEBEAYCIAgC5v/7RPd9nfZ/fdy27vuv7Nud7KHM5XN3Pmc3Bnrn2zjtdJwUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAy5pxz6cExdq8FgEWLr+6nOSkAEFEAIKIAQEQBgNxf/eCuS45vc3UBbzaH82zM5WDPPGbPXHvnL/o4KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoAJAx55xLD46xey0ALFp8dT/NSQGAiAIAEQUAcn/1g7u+z/o2V3ctZnM4z8ZcDvbMY79m8/Mny/g8P+/7UU4KAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoAZMw559KDY+xeCwCLFl/dT3NSACCiAEBEAYCIAgC5rz6461IDgM/hpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADIf6BqdvgXBfkuAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzUlEQVR4nO3cwWrjMBRA0WjIh/fPNQvDXbgOKAGlCZyzdqh4CF8UtR1zznkDgNvt9u+vFwDA5xAFACIKAEQUAIgoABBRACCiAEBEAYDcVx8cY+xcBwBP2PV3x04KAEQUAIgoAJDlO4Uz/0fvcHXXYjaH82zM5WDPPGbPXHvnna6TAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABlzzrn04Bi71wLAosVX99OcFACIKAAQUQAgogBA7q9+cNclx7e5uoA3m8N5NuZysGces2euvfMXfZwUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAyJhzzqUHx9i9FgAWLb66n+akAEBEAYCIAgC5v/7RPd9nfZ/fdy27vuv7Nud7KHM5XN3Pmc3Bnrn2zjtdJwUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAy5pxz6cExdq8FgEWLr+6nOSkAEFEAIKIAQEQBgNxf/eCuS45vc3UBbzaH82zM5WDPPGbPXHvnL/o4KQAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoAJAx55xLD46xey0ALFp8dT/NSQGAiAIAEQUAcn/1g7u+z/o2V3ctZnM4z8ZcDvbMY79m8/Mny/g8P+/7UU4KAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoAZMw559KDY+xeCwCLFl/dT3NSACCiAEBEAYCIAgC5rz6461IDgM/hpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADIf6BqdvgXBfkuAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzElEQVR4nO3c0WrjMBRAwWjJh/fPtQ+CA5t1QA44TWDm2aHiInxQ1HbMOecNAG6325/fXgAAn0MUAIgoABBRACCiAEBEAYCIAgARBQBy331wjHHlOgA44aq/O3ZSACCiAEBEAYBs3yk88n/0lqO7FrNZHmdjLos985w9c+ydd7pOCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAGTMOefWg2NcvRYANm2+uk9zUgAgogBARAGAiAIAub/6wasuOb7N0QW82SyPszGXxZ55zp459s5f9HFSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIGPOObceHOPqtQCwafPVfZqTAgARBQAiCgDk/uoHr/o+69sc37WYzfLvbOyZ5WjPmM3yOBtzWd55p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMacc249OMbVawFg0+ar+zQnBQAiCgBEFACIKACQ+6sfvOqS49scXcCbzfI4G3NZ7Jnn7Jlj7/xFHycFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMuacc+vBMa5eCwCbNl/dpzkpABBRACCiAEDur37wqu+zvs3RXYvZLI+zMZfFnnnuv9n8/MoyPs/P+36UkwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgAZc8659eAYV68FgE2br+7TnBQAiCgAEFEAIKIAQO67D151qQHA53BSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAOQv2Ih2+Hjr70UAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzElEQVR4nO3c0WrjMBRAwWjJh/fPtQ+CA5t1QA44TWDm2aHiInxQ1HbMOecNAG6325/fXgAAn0MUAIgoABBRACCiAEBEAYCIAgARBQBy331wjHHlOgA44aq/O3ZSACCiAEBEAYBs3yk88n/0lqO7FrNZHmdjLos985w9c+ydd7pOCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAGTMOefWg2NcvRYANm2+uk9zUgAgogBARAGAiAIAub/6wasuOb7N8QW82Sz/zsaeWY72jNksj7Mxl+Wdv+jjpABARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEDGnHNuPTjG1WsBYNPmq/s0JwUAIgoARBQAyP3VD171fda3ObprMZvlcTbmstgzz9kzx955p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMacc249OMbVawFg0+ar+zQnBQAiCgBEFACIKACQ+6sfvOqS49scXcCbzfI4G3NZ7Jnn7Jlj7/xFHycFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMuacc+vBMa5eCwCbNl/dpzkpABBRACCiAEDur37wqu+zvs3RXYvZLI+zMZfFnnnuv9n8/MoyPs/P+36UkwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgAZc8659eAYV68FgE2br+7TnBQAiCgAEFEAIKIAQO67D151qQHA53BSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAOQvKqt2+MFvAgQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAFzElEQVR4nO3c0WrjMBRAwWjJh/fPtQ+CA5t1QA44TWDm2aHiInxQ1HbMOecNAG6325/fXgAAn0MUAIgoABBRACCiAEBEAYCIAgARBQBy331wjHHlOgA44aq/O3ZSACCiAEBEAYBs3yk88n/0lqO7FrNZHmdjLos985w9c+ydd7pOCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAGTMOefWg2NcvRYANm2+uk9zUgAgogBARAGAiAIAub/6wasuOb7N0QW82SyPszGXxZ55zp459s5f9HFSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIGPOObceHOPqtQCwafPVfZqTAgARBQAiCgDk/uoHr/o+69sc37WYzfLvbOyZ5WjPmM3yOBtzWd55p+ukAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQMacc249OMbVawFg0+ar+zQnBQAiCgBEFACIKACQ+6sfvOqS49scXcCbzfI4G3NZ7Jnn7Jlj7/xFHycFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMuacc+vBMa5eCwCbNl/dpzkpABBRACCiAEDur37wqu+zvs3RXYvZLI+zMZfFnnnuv9n8/MoyPs/P+36UkwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgAZc8659eAYV68FgE2br+7TnBQAiCgAEFEAIKIAQO67D151qQHA53BSACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAOQv2Ih2+Hjr70UAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "from gymnasium import spaces\n",
    "import numpy as np\n",
    "from PIL import Image, ImageDraw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "class SimpleGridEnv(gym.Env):\n",
    "    def __init__(self, grid_size=5, render_mode = None):\n",
    "        super(SimpleGridEnv, self).__init__()\n",
    "        self.render_mode = render_mode\n",
    "        self.grid_size = grid_size\n",
    "        self.action_space = spaces.Discrete(4)  # 4 actions: left, right, up, down\n",
    "        self.observation_space = spaces.Box(low=0, high=grid_size-1, shape=(2,), dtype=np.int32)\n",
    "        self.state = None\n",
    "        self.goal = np.array([grid_size-1, grid_size-1])  # Goal position\n",
    "    \n",
    "    def reset(self):\n",
    "        self.state = np.array([0, 0])  # Starting position\n",
    "        return self.state\n",
    "    \n",
    "    def step(self, action):\n",
    "        if action == 0:   # Left\n",
    "            self.state[1] = max(0, self.state[1] - 1)\n",
    "        elif action == 1: # Right\n",
    "            self.state[1] = min(self.grid_size - 1, self.state[1] + 1)\n",
    "        elif action == 2: # Up\n",
    "            self.state[0] = max(0, self.state[0] - 1)\n",
    "        elif action == 3: # Down\n",
    "            self.state[0] = min(self.grid_size - 1, self.state[0] + 1)\n",
    "        \n",
    "        done = np.array_equal(self.state, self.goal)\n",
    "        reward = 1 if done else -0.1  # Reward for reaching the goal\n",
    "        \n",
    "        return self.state, reward, done, {}\n",
    "    \n",
    "    def render(self, mode='human'):\n",
    "        cell_size = 84 // self.grid_size\n",
    "        img = Image.new('RGB', (84, 84), color='white')\n",
    "        draw = ImageDraw.Draw(img)\n",
    "        \n",
    "        # Draw agent\n",
    "        agent_top_left = (self.state[1] * cell_size, self.state[0] * cell_size)\n",
    "        agent_bottom_right = ((self.state[1] + 1) * cell_size, (self.state[0] + 1) * cell_size)\n",
    "        draw.rectangle([agent_top_left, agent_bottom_right], fill='blue')\n",
    "        \n",
    "        # Draw goal\n",
    "        goal_top_left = (self.goal[1] * cell_size, self.goal[0] * cell_size)\n",
    "        goal_bottom_right = ((self.goal[1] + 1) * cell_size, (self.goal[0] + 1) * cell_size)\n",
    "        draw.rectangle([goal_top_left, goal_bottom_right], fill='green')\n",
    "\n",
    "        for i in range(self.grid_size):\n",
    "            for j in range(self.grid_size):\n",
    "                top_left = (j * cell_size, i * cell_size)\n",
    "                bottom_right = ((j + 1) * cell_size, (i + 1) * cell_size)\n",
    "                draw.rectangle([top_left, bottom_right], outline='black')\n",
    "        \n",
    "        if self.render_mode == 'human':\n",
    "            img.show()\n",
    "        elif self.render_mode == 'rgb_array':\n",
    "            return np.array(img)\n",
    "\n",
    "# Example usage\n",
    "env = SimpleGridEnv(grid_size=5, render_mode='rgb_array')\n",
    "observation = env.reset()\n",
    "\n",
    "for _ in range(10):\n",
    "    action = env.action_space.sample()  # Random action\n",
    "    observation, reward, done, info = env.step(action)\n",
    "\n",
    "    pixels = env.render()\n",
    "\n",
    "    print(pixels.shape)\n",
    "\n",
    "    # Convert numpy array to PIL image and resize to 84x84 pixels\n",
    "    img = Image.fromarray(pixels)\n",
    "\n",
    "    # Display the resized image using matplotlib\n",
    "    plt.imshow(img)\n",
    "    plt.axis('off')  # Hide axis\n",
    "    plt.show()\n",
    "\n",
    "    # Close the environment\n",
    "    if done:\n",
    "        print(\"Goal reached!\")\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "7\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([['x', 'x', 'x', 'x', 'x'],\n",
       "       ['x', '.', '.', 'G', 'x'],\n",
       "       ['x', '.', '.', '.', 'x'],\n",
       "       ['x', 'x', '.', 'x', 'x'],\n",
       "       ['x', '.', '.', 'G', 'x'],\n",
       "       ['x', '.', '.', '.', 'x'],\n",
       "       ['x', 'x', 'x', 'x', 'x']], dtype='<U1')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"grid.txt\", 'r') as file:\n",
    "    lines = file.readlines()\n",
    "    print(len(lines[0].split()))\n",
    "    print(len(lines))\n",
    "\n",
    "np.array([line.split() for line in lines])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/gymnasium/envs/registration.py:694: UserWarning: \u001b[33mWARN: Overriding environment SimpleGridEnv-v0 already in registry.\u001b[0m\n",
      "  logger.warn(f\"Overriding environment {new_spec.id} already in registry.\")\n",
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:2989: DeprecationWarning: Your wrapper was not given a device. Currently, this value will default to 'cpu'. From v0.5 it will default to `None`. With a device of None, no device casting is performed and the resulting tensordicts are deviceless. Please set your device accordingly.\n",
      "  warnings.warn(\n",
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/gymnasium/utils/passive_env_checker.py:189: UserWarning: \u001b[33mWARN: The result returned by `env.reset()` was not a tuple of the form `(obs, info)`, where `obs` is a observation and `info` is a dictionary containing additional information. Actual type: `<class 'numpy.ndarray'>`\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Shape mismatch: the value has shape torch.Size([]) which is incompatible with the spec shape torch.Size([2]).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/data/tensor_specs.py:579\u001b[0m, in \u001b[0;36mTensorSpec.encode\u001b[0;34m(self, val, ignore_device)\u001b[0m\n\u001b[1;32m    578\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 579\u001b[0m     val \u001b[38;5;241m=\u001b[39m \u001b[43mval\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreshape\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    580\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "\u001b[0;31mRuntimeError\u001b[0m: shape '[2]' is invalid for input of size 1",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[26], line 135\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[38;5;66;03m# Create and wrap the environment\u001b[39;00m\n\u001b[1;32m    134\u001b[0m env \u001b[38;5;241m=\u001b[39m gym\u001b[38;5;241m.\u001b[39mmake(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSimpleGridEnv-v0\u001b[39m\u001b[38;5;124m'\u001b[39m, grid_file\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcustom_envs/grid_envs/grid_world2.txt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 135\u001b[0m env_torchrl \u001b[38;5;241m=\u001b[39m \u001b[43mGymWrapper\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    137\u001b[0m \u001b[38;5;66;03m# Example usage\u001b[39;00m\n\u001b[1;32m    138\u001b[0m observation \u001b[38;5;241m=\u001b[39m env_torchrl\u001b[38;5;241m.\u001b[39mreset()\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:543\u001b[0m, in \u001b[0;36m_AsyncMeta.__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    542\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 543\u001b[0m     instance: GymWrapper \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    545\u001b[0m     \u001b[38;5;66;03m# before gym 0.22, there was no final_observation\u001b[39;00m\n\u001b[1;32m    546\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m instance\u001b[38;5;241m.\u001b[39m_is_batched:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:175\u001b[0m, in \u001b[0;36m_EnvPostInit.__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    173\u001b[0m auto_reset \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_reset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m    174\u001b[0m auto_reset_replace \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_reset_replace\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 175\u001b[0m instance: EnvBase \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    176\u001b[0m \u001b[38;5;66;03m# we create the done spec by adding a done/terminated entry if one is missing\u001b[39;00m\n\u001b[1;32m    177\u001b[0m instance\u001b[38;5;241m.\u001b[39m_create_done_specs()\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:729\u001b[0m, in \u001b[0;36mGymWrapper.__init__\u001b[0;34m(self, env, categorical_action_encoding, **kwargs)\u001b[0m\n\u001b[1;32m    727\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m set_gym_backend(libname):\n\u001b[1;32m    728\u001b[0m         kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m env\n\u001b[0;32m--> 729\u001b[0m         \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    730\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    731\u001b[0m     \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:3021\u001b[0m, in \u001b[0;36m_EnvWrapper.__init__\u001b[0;34m(self, device, batch_size, allow_done_after_reset, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3019\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_specs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_env)  \u001b[38;5;66;03m# writes the self._env attribute\u001b[39;00m\n\u001b[1;32m   3020\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m-> 3021\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_init_env\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:1139\u001b[0m, in \u001b[0;36mGymWrapper._init_env\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1138\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_init_env\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m-> 1139\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:2120\u001b[0m, in \u001b[0;36mEnvBase.reset\u001b[0;34m(self, tensordict, **kwargs)\u001b[0m\n\u001b[1;32m   2117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensordict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   2118\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_assert_tensordict_shape(tensordict)\n\u001b[0;32m-> 2120\u001b[0m tensordict_reset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensordict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2121\u001b[0m \u001b[38;5;66;03m#        We assume that this is done properly\u001b[39;00m\n\u001b[1;32m   2122\u001b[0m \u001b[38;5;66;03m#        if reset.device != self.device:\u001b[39;00m\n\u001b[1;32m   2123\u001b[0m \u001b[38;5;66;03m#            reset = reset.to(self.device, non_blocking=True)\u001b[39;00m\n\u001b[1;32m   2124\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensordict_reset \u001b[38;5;129;01mis\u001b[39;00m tensordict:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:1165\u001b[0m, in \u001b[0;36mGymWrapper._reset\u001b[0;34m(self, tensordict, **kwargs)\u001b[0m\n\u001b[1;32m   1163\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m reset \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1164\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m tensordict\u001b[38;5;241m.\u001b[39mexclude(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_reset\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_reset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtensordict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/gym_like.py:360\u001b[0m, in \u001b[0;36mGymLikeEnv._reset\u001b[0;34m(self, tensordict, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_reset\u001b[39m(\n\u001b[1;32m    356\u001b[0m     \u001b[38;5;28mself\u001b[39m, tensordict: Optional[TensorDictBase] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m    357\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m TensorDictBase:\n\u001b[1;32m    358\u001b[0m     obs, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset_output_transform(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_env\u001b[38;5;241m.\u001b[39mreset(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n\u001b[0;32m--> 360\u001b[0m     source \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_obs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    362\u001b[0m     tensordict_out \u001b[38;5;241m=\u001b[39m TensorDict(\n\u001b[1;32m    363\u001b[0m         source\u001b[38;5;241m=\u001b[39msource,\n\u001b[1;32m    364\u001b[0m         batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size,\n\u001b[1;32m    365\u001b[0m         _run_checks\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalidated,\n\u001b[1;32m    366\u001b[0m     )\n\u001b[1;32m    367\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minfo_dict_reader \u001b[38;5;129;01mand\u001b[39;00m info \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/gym_like.py:267\u001b[0m, in \u001b[0;36mGymLikeEnv.read_obs\u001b[0;34m(self, observations)\u001b[0m\n\u001b[1;32m    265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, spec \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobservation_spec\u001b[38;5;241m.\u001b[39mitems(\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m    266\u001b[0m     observations_dict \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m--> 267\u001b[0m     observations_dict[key] \u001b[38;5;241m=\u001b[39m \u001b[43mspec\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobservations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mignore_device\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    268\u001b[0m     \u001b[38;5;66;03m# we don't check that there is only one spec because obs spec also\u001b[39;00m\n\u001b[1;32m    269\u001b[0m     \u001b[38;5;66;03m# contains the data spec of the info dict.\u001b[39;00m\n\u001b[1;32m    270\u001b[0m     \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/data/tensor_specs.py:581\u001b[0m, in \u001b[0;36mTensorSpec.encode\u001b[0;34m(self, val, ignore_device)\u001b[0m\n\u001b[1;32m    579\u001b[0m                 val \u001b[38;5;241m=\u001b[39m val\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m    580\u001b[0m             \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 581\u001b[0m                 \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m    582\u001b[0m                     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShape mismatch: the value has shape \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mval\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m which \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    583\u001b[0m                     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mis incompatible with the spec shape \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    584\u001b[0m                 ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m    585\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _CHECK_SPEC_ENCODE:\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39massert_is_in(val)\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Shape mismatch: the value has shape torch.Size([]) which is incompatible with the spec shape torch.Size([2])."
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "from gymnasium import spaces\n",
    "import numpy as np\n",
    "from PIL import Image, ImageDraw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class SimpleGridEnv(gym.Env):\n",
    "    def __init__(self, render_mode=None, grid_file=None):\n",
    "        super(SimpleGridEnv, self).__init__()\n",
    "        self.render_mode = render_mode\n",
    "        self.goal_positions = []\n",
    "        self.load_grid_from_file(grid_file)\n",
    "        self.action_space = spaces.Discrete(4)  # 4 actions: left, right, up, down\n",
    "        self.observation_space = spaces.Box(low=0, high=self.grid_size-1, shape=(2,), dtype=np.int32)\n",
    "        self.state = None\n",
    "        \n",
    "        \n",
    "\n",
    "    def load_grid_from_file(self, file_path):\n",
    "        with open(file_path, 'r') as file:\n",
    "            lines = file.readlines()\n",
    "            self.height = len(lines)\n",
    "            self.width = len(lines[0].split())\n",
    "            self.grid_size = max(self.height, self.width)\n",
    "\n",
    "            if self.height != self.width:\n",
    "                print(\"Warning: The grid is not square. Padding with 'x' to make it square.\")\n",
    "            self.grid = np.array([line.split() for line in lines])\n",
    "\n",
    "        def make_square(grid):\n",
    "            # Get the dimensions of the grid\n",
    "            rows, cols = grid.shape\n",
    "            \n",
    "            # Determine the size of the square matrix\n",
    "            size = max(rows, cols)\n",
    "            \n",
    "            # Calculate padding for rows and columns\n",
    "            row_padding_top = (size - rows) // 2\n",
    "            row_padding_bottom = size - rows - row_padding_top\n",
    "            col_padding_left = (size - cols) // 2\n",
    "            col_padding_right = size - cols - col_padding_left\n",
    "            \n",
    "            # Pad the grid with 'x' to make it square\n",
    "            square_grid = np.pad(grid, \n",
    "                                pad_width=((row_padding_top, row_padding_bottom), (col_padding_left, col_padding_right)),\n",
    "                                mode='constant', \n",
    "                                constant_values='x')\n",
    "            \n",
    "            return square_grid\n",
    "           \n",
    "        self.grid = make_square(self.grid)\n",
    "\n",
    "            # Add walls in the width or height to make the grid square, but keep it centered\n",
    "\n",
    "        for i in range(self.grid_size):\n",
    "            for j in range(self.grid_size):\n",
    "                if self.grid[i, j] == 'G':\n",
    "                    self.goal_positions.append((i, j))\n",
    "        \n",
    "\n",
    "\n",
    "    def reset(self, seed=None, return_info=False, options=None):\n",
    "        super().reset(seed=seed)\n",
    "        # Find a starting position randomly among the '.' positions\n",
    "        start_positions = np.argwhere(self.grid == '.')\n",
    "        start_idx = np.random.choice(len(start_positions))\n",
    "        self.state = start_positions[start_idx]\n",
    "        if return_info:\n",
    "            return self.state, {}\n",
    "        else:\n",
    "            return self.state\n",
    "\n",
    "    def step(self, action):\n",
    "        if action == 0:   # Left\n",
    "            self.state[1] = max(0, self.state[1] - 1)\n",
    "        elif action == 1: # Right\n",
    "            self.state[1] = min(self.grid_size - 1, self.state[1] + 1)\n",
    "        elif action == 2: # Up\n",
    "            self.state[0] = max(0, self.state[0] - 1)\n",
    "        elif action == 3: # Down\n",
    "            self.state[0] = min(self.grid_size - 1, self.state[0] + 1)\n",
    "        \n",
    "        # Check for wall collisions\n",
    "        if self.grid[self.state[0], self.state[1]] == 'x':\n",
    "            if action == 0:   # Move back to the right\n",
    "                self.state[1] += 1\n",
    "            elif action == 1: # Move back to the left\n",
    "                self.state[1] -= 1\n",
    "            elif action == 2: # Move back down\n",
    "                self.state[0] += 1\n",
    "            elif action == 3: # Move back up\n",
    "                self.state[0] -= 1\n",
    "        \n",
    "        done = tuple(self.state) in self.goal_positions\n",
    "        reward = 1 if done else -0.1  # Reward for reaching the goal\n",
    "        \n",
    "        return self.state, reward, done, False, {}\n",
    "\n",
    "    def render(self, mode='human'):\n",
    "        cell_size = 84 // self.grid_size\n",
    "        img = Image.new('RGB', (84, 84), color='white')\n",
    "        draw = ImageDraw.Draw(img)\n",
    "        \n",
    "        # Draw grid\n",
    "        for i in range(self.grid_size):\n",
    "            for j in range(self.grid_size):\n",
    "                top_left = (j * cell_size, i * cell_size)\n",
    "                bottom_right = ((j + 1) * cell_size, (i + 1) * cell_size)\n",
    "                if self.grid[i, j] == 'x':\n",
    "                    draw.rectangle([top_left, bottom_right], fill='#646464ff', outline='#646464ff')\n",
    "                elif self.grid[i, j] == 'G':\n",
    "                    draw.rectangle([top_left, bottom_right], fill='#00FF00', outline='#646464ff')\n",
    "                else:\n",
    "                    draw.rectangle([top_left, bottom_right], fill='#000000', outline='#646464ff')\n",
    "        \n",
    "        # Draw agent\n",
    "        agent_top_left = (self.state[1] * cell_size, self.state[0] * cell_size)\n",
    "        agent_bottom_right = ((self.state[1] + 1) * cell_size, (self.state[0] + 1) * cell_size)\n",
    "        draw.rectangle([agent_top_left, agent_bottom_right], fill='#0000FF', outline='#646464ff')\n",
    "\n",
    "        if self.render_mode == 'human':\n",
    "            img.show()\n",
    "        elif self.render_mode == 'rgb_array':\n",
    "            return np.array(img)\n",
    "\n",
    "# Example usage\n",
    "grid_file_content = \"\"\"\\\n",
    "x x x x x\n",
    "x . . G x\n",
    "x . . . x\n",
    "x x . x x\n",
    "x . . G x\n",
    "x . . . x\n",
    "x x x x x\n",
    "\"\"\"\n",
    "\n",
    "# with open('grid.txt', 'w') as file:\n",
    "#     file.write(grid_file_content)\n",
    "\n",
    "env = SimpleGridEnv(grid_file='custom_envs/grid_envs/grid_world2.txt', render_mode='rgb_array')\n",
    "observation = env.reset()\n",
    "\n",
    "for _ in range(3):\n",
    "    action = env.action_space.sample()  # Random action\n",
    "    observation, reward, done, truncated, info = env.step(action)\n",
    "\n",
    "    print(observation)\n",
    "\n",
    "    pixels = env.render()\n",
    "\n",
    "    print(pixels.shape)\n",
    "\n",
    "    # Convert numpy array to PIL image and resize to 84x84 pixels\n",
    "    img = Image.fromarray(pixels)\n",
    "\n",
    "    # Display the resized image using matplotlib\n",
    "    plt.imshow(img)\n",
    "    plt.axis('off')  # Hide axis\n",
    "    plt.show()\n",
    "\n",
    "    # Convert to gray scale and plot\n",
    "    # gray_img = img.convert('L')\n",
    "    # plt.imshow(gray_img, cmap='gray')\n",
    "    # plt.axis('off')  # Hide axis\n",
    "    # plt.show()\n",
    "\n",
    "    if done:\n",
    "        print(\"Goal reached!\")\n",
    "        break\n",
    "\n",
    "# Close the environment\n",
    "env.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pygame 2.6.0 (SDL 2.28.4, Python 3.11.9)\n",
      "Hello from the pygame community. https://www.pygame.org/contribute.html\n"
     ]
    }
   ],
   "source": [
    "import gymnasium as gym\n",
    "from gymnasium import spaces\n",
    "import pygame\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class GridWorldEnv(gym.Env):\n",
    "    metadata = {\"render_modes\": [\"human\", \"rgb_array\"], \"render_fps\": 4}\n",
    "\n",
    "    def __init__(self, render_mode=None, grid_file = None):\n",
    "\n",
    "        # Load from a file\n",
    "        if grid_file is None:\n",
    "            raise ValueError(\"You must provide a grid file. As an example, you can use the grid.txt file.\")\n",
    "        \n",
    "        self._grid = self.load_grid_from_file(grid_file)\n",
    "\n",
    "        self.size = self._grid.shape[0] # if self._grid else size  # The size of the square grid\n",
    "        self.window_size = 512  # The size of the PyGame window\n",
    "\n",
    "        # Set walls and target\n",
    "        self._set_components()\n",
    "\n",
    "        # Observations are dictionaries with the agent's and the target's location.\n",
    "        # Each location is encoded as an element of {0, ..., `size`}^2, i.e. MultiDiscrete([size, size]).\n",
    "        # self.observation_space = spaces.Dict(\n",
    "        #     {\n",
    "        #         \"agent\": spaces.Box(0, self.size - 1, shape=(2,), dtype=int),\n",
    "        #         \"target\": spaces.Box(0, self.size - 1, shape=(2,), dtype=int),\n",
    "        #     }\n",
    "        # )\n",
    "        self.observation_space = spaces.Box(low=0, high=self.size-1, shape=(2,), dtype=np.int64)\n",
    "\n",
    "\n",
    "        # We have 4 actions, corresponding to \"down\", \"right\", \"up\" and \"left\".\n",
    "        # Action 0: Down\n",
    "        # Action 1: Right\n",
    "        # Action 2: Up\n",
    "        # Action 3: Left\n",
    "        self.action_space = spaces.Discrete(4)\n",
    "\n",
    "        \"\"\"\n",
    "        The following dictionary maps abstract actions from `self.action_space` to \n",
    "        the direction we will walk in if that action is taken.\n",
    "        I.e. 0 corresponds to \"right\", 1 to \"up\" etc.\n",
    "        \"\"\"\n",
    "        self._action_to_direction = {\n",
    "            0: np.array([1, 0]),\n",
    "            1: np.array([0, 1]),\n",
    "            2: np.array([-1, 0]),\n",
    "            3: np.array([0, -1]),\n",
    "        }\n",
    "\n",
    "\n",
    "        assert render_mode is None or render_mode in self.metadata[\"render_modes\"]\n",
    "        self.render_mode = render_mode\n",
    "\n",
    "        \"\"\"\n",
    "        If human-rendering is used, `self.window` will be a reference\n",
    "        to the window that we draw to. `self.clock` will be a clock that is used\n",
    "        to ensure that the environment is rendered at the correct framerate in\n",
    "        human-mode. They will remain `None` until human-mode is used for the\n",
    "        first time.\n",
    "        \"\"\"\n",
    "        self.window = None\n",
    "        self.clock = None\n",
    "\n",
    "    def load_grid_from_file(self, file_path):\n",
    "        with open(file_path, 'r') as file:\n",
    "            lines = file.readlines()\n",
    "            self.height = len(lines)\n",
    "            self.width = len(lines[0].split())\n",
    "            self.grid_size = self.height\n",
    "\n",
    "            if self.height != self.width:\n",
    "                # Raise a error message\n",
    "                raise ValueError(\"Invalid shape of widht and height must be equal, add extra walls to make it square\")\n",
    "\n",
    "            return np.array([line.split() for line in lines])\n",
    "        \n",
    "    def _set_components(self):\n",
    "        self._walls_location = []\n",
    "        self._targets_location = []\n",
    "        for i in range(self.size):\n",
    "            for j in range(self.size):\n",
    "                if self._grid[i, j] == 'x':\n",
    "                    self._walls_location.append((i, j))\n",
    "                elif self._grid[i, j] == 'G':\n",
    "                    self._targets_location.append((i, j))\n",
    "        self._walls_location = set(self._walls_location)\n",
    "        self._targets_location = set(self._targets_location)\n",
    "\n",
    "    def _get_obs(self):\n",
    "        # return {\"agent\": self._agent_location, \"target\": self._targets_location}\n",
    "        return self._agent_location\n",
    "\n",
    "    def _get_info(self):\n",
    "        # return {\n",
    "        #     \"distance\": np.linalg.norm(\n",
    "        #         self._agent_location - self._target_location, ord=1\n",
    "        #     )\n",
    "        # }\n",
    "        return {}\n",
    "\n",
    "    def reset(self, seed=None, options=None):\n",
    "        # We need the following line to seed self.np_random\n",
    "        super().reset(seed=seed)\n",
    "\n",
    "        # We will sample the agent's location randomly until it does not coincide with the target or walls's location\n",
    "        self._agent_location = self.np_random.integers(0, self.size, size=2, dtype=int)\n",
    "\n",
    "        while tuple(self._agent_location) in self._walls_location or tuple(self._agent_location) in self._targets_location:\n",
    "            self._agent_location = self.np_random.integers(\n",
    "                0, self.size, size=2, dtype=int\n",
    "            )\n",
    "\n",
    "        # while np.array_equal(self._target_location, self._agent_location) or tuple(self._agent_location) in self._walls_location:\n",
    "        #     self._agent_location = self.np_random.integers(\n",
    "        #         0, self.size, size=2, dtype=int\n",
    "        #     )\n",
    "\n",
    "        observation = self._get_obs()\n",
    "        info = self._get_info()\n",
    "\n",
    "        if self.render_mode == \"human\":\n",
    "            self._render_frame()\n",
    "\n",
    "        return observation, info\n",
    "\n",
    "    def step(self, action):\n",
    "        # Map the action (element of {0,1,2,3}) to the direction we walk in\n",
    "        direction = self._action_to_direction[action]\n",
    "        \n",
    "        # if to make sure we don't walk into a wall\n",
    "        if tuple(self._agent_location + direction) not in self._walls_location:\n",
    "            # We use `np.clip` to make sure we don't leave the grid\n",
    "            self._agent_location = np.clip(\n",
    "                self._agent_location + direction, 0, self.size - 1\n",
    "            )\n",
    "\n",
    "\n",
    "        # An episode is done iff the agent has reached the target\n",
    "        # terminated = np.array_equal(self._agent_location, self._target_location)\n",
    "        terminated = tuple(self._agent_location) in self._targets_location\n",
    "        reward = 1 if terminated else 0  # Binary sparse rewards\n",
    "        observation = self._get_obs()\n",
    "        info = self._get_info()\n",
    "\n",
    "        if self.render_mode == \"human\":\n",
    "            self._render_frame()\n",
    "\n",
    "        return observation, reward, terminated, False, info\n",
    "\n",
    "    def render(self):\n",
    "        if self.render_mode == \"rgb_array\":\n",
    "            return self._render_frame()\n",
    "\n",
    "    def _render_frame(self):\n",
    "        if self.window is None and self.render_mode == \"human\":\n",
    "            pygame.init()\n",
    "            pygame.display.init()\n",
    "            self.window = pygame.display.set_mode((self.window_size, self.window_size))\n",
    "        if self.clock is None and self.render_mode == \"human\":\n",
    "            self.clock = pygame.time.Clock()\n",
    "\n",
    "        canvas = pygame.Surface((self.window_size, self.window_size))\n",
    "        canvas.fill((255, 255, 255))\n",
    "        pix_square_size = (\n",
    "            self.window_size / self.size\n",
    "        )  # The size of a single grid square in pixels\n",
    "\n",
    "        # First we draw the walls\n",
    "        for wall_location in self._walls_location:\n",
    "            pygame.draw.rect(\n",
    "                canvas,\n",
    "                (0, 0, 0),\n",
    "                pygame.Rect(\n",
    "                    pix_square_size * np.array(wall_location)[::-1], # I have to switch to display in order (rows, cols)\n",
    "                    (pix_square_size, pix_square_size),\n",
    "                ),\n",
    "            )\n",
    "\n",
    "        # Second we draw the targets\n",
    "        for target_location in self._targets_location:\n",
    "            pygame.draw.rect(\n",
    "                canvas,\n",
    "                (255, 0, 0),\n",
    "                pygame.Rect(\n",
    "                    pix_square_size * np.array(target_location)[::-1], # I have to switch to display in order (rows, cols)\n",
    "                    (pix_square_size, pix_square_size),\n",
    "                ),\n",
    "            )\n",
    "        \n",
    "        # Now we draw the agent\n",
    "        pygame.draw.circle(\n",
    "            canvas,\n",
    "            (0, 0, 255),\n",
    "            (self._agent_location[::-1] + 0.5) * pix_square_size,\n",
    "            pix_square_size / 3,\n",
    "        )\n",
    "\n",
    "        # Finally, add some gridlines\n",
    "        for x in range(self.size + 1):\n",
    "            pygame.draw.line(\n",
    "                canvas,\n",
    "                0,\n",
    "                (0, pix_square_size * x),\n",
    "                (self.window_size, pix_square_size * x),\n",
    "                width=3,\n",
    "            )\n",
    "            pygame.draw.line(\n",
    "                canvas,\n",
    "                0,\n",
    "                (pix_square_size * x, 0),\n",
    "                (pix_square_size * x, self.window_size),\n",
    "                width=3,\n",
    "            )\n",
    "\n",
    "        if self.render_mode == \"human\":\n",
    "            # The following line copies our drawings from `canvas` to the visible window\n",
    "            self.window.blit(canvas, canvas.get_rect())\n",
    "            pygame.event.pump()\n",
    "            pygame.display.update()\n",
    "\n",
    "            # We need to ensure that human-rendering occurs at the predefined framerate.\n",
    "            # The following line will automatically add a delay to keep the framerate stable.\n",
    "            self.clock.tick(self.metadata[\"render_fps\"])\n",
    "        else:  # rgb_array\n",
    "            return np.transpose(\n",
    "                np.array(pygame.surfarray.pixels3d(canvas)), axes=(1, 0, 2)\n",
    "            )\n",
    "        \n",
    "\n",
    "    def close(self):\n",
    "        if self.window is not None:\n",
    "            pygame.display.quit()\n",
    "            pygame.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "walls = set([(1,2),(3,4)])\n",
    "arr = np.array([1,2])\n",
    "\n",
    "# Check if wall in arr\n",
    "tuple(arr) in walls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2]\n",
      "[3 4]\n"
     ]
    }
   ],
   "source": [
    "walls = set([(1,2),(3,4)])\n",
    "for wall in walls:\n",
    "    print(np.array(wall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "sys.path.insert(0, os.path.abspath('../'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/gymnasium/envs/registration.py:694: UserWarning: \u001b[33mWARN: Overriding environment GridWorldEnv-v0 already in registry.\u001b[0m\n",
      "  logger.warn(f\"Overriding environment {new_spec.id} already in registry.\")\n"
     ]
    }
   ],
   "source": [
    "from gymnasium.envs.registration import register\n",
    "\n",
    "register(\n",
    "    id='GridWorldEnv-v0',\n",
    "    entry_point='custom_envs.grid_world:GridWorldEnv',  # Adjust the path to the actual module and class\n",
    "    max_episode_steps=200\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from gymnasium.envs.registration import register\n",
    "\n",
    "# register(\n",
    "#     id='GridWorldEnv-v0',\n",
    "#     entry_point='__main__:GridWorldEnv',\n",
    "#     max_episode_steps=200\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action Space: Discrete(4)\n",
      "Action Space Type: <class 'gymnasium.spaces.discrete.Discrete'>\n",
      "Observation Space: Box(0, 6, (2,), int64)\n",
      "Observation Space Type: <class 'gymnasium.spaces.box.Box'>\n",
      "Number of actions: 4\n",
      "Observation Space Shape: (2,)\n",
      "Observation Space High: [6 6]\n",
      "Observation Space Low: [0 0]\n"
     ]
    }
   ],
   "source": [
    "# Check the cartpole environment\n",
    "env = gym.make(\"GridWorldEnv-v0\", grid_file='custom_envs/grid_envs/grid_world2.txt')\n",
    "\n",
    "# Print action space details\n",
    "print(\"Action Space:\", env.action_space)\n",
    "print(\"Action Space Type:\", type(env.action_space))\n",
    "\n",
    "# Print observation space details\n",
    "print(\"Observation Space:\", env.observation_space)\n",
    "print(\"Observation Space Type:\", type(env.observation_space))\n",
    "\n",
    "# Print more detailed information if it's a Box or Discrete space\n",
    "if isinstance(env.action_space, gym.spaces.Discrete):\n",
    "    print(\"Number of actions:\", env.action_space.n)\n",
    "\n",
    "if isinstance(env.observation_space, gym.spaces.Box):\n",
    "    print(\"Observation Space Shape:\", env.observation_space.shape)\n",
    "    print(\"Observation Space High:\", env.observation_space.high)\n",
    "    print(\"Observation Space Low:\", env.observation_space.low)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check GridWorldEnv\n",
    "env = gym.make(\"GridWorldEnv-v0\", render_mode=\"rgb_array\", grid_file='custom_envs/grid_envs/grid_world2.txt')\n",
    "\n",
    "# Reset the environment\n",
    "observation, info = env.reset(seed=42)\n",
    "print(\"Initial Observation:\", observation)\n",
    "print(\"Initial Info:\", info)\n",
    "\n",
    "# Step through the environment\n",
    "\n",
    "for _ in range(10):\n",
    "    # Random action\n",
    "    action = np.random.randint(env.action_space.n)\n",
    "    observation, reward, terminated, truncated, info = env.step(action)\n",
    "\n",
    "    print(f\"Action: {action}, Reward: {reward}\")\n",
    "    print(f\"Observation: {observation}\")\n",
    "    print(f\"Info: {info}\")\n",
    "\n",
    "    # Render the environment\n",
    "    pixels = env.render()\n",
    "    print(f\"Pixels shape: {pixels.shape}\")\n",
    "    plt.imshow(pixels)\n",
    "    plt.axis('off')\n",
    "    plt.show()\n",
    "\n",
    "    if terminated or truncated:\n",
    "        observation, info = env.reset()\n",
    "\n",
    "# Action 0: Down\n",
    "# Action 1: Right\n",
    "# Action 2: Up\n",
    "# Action 3: Left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "from torchrl.envs import GymWrapper\n",
    "import gymnasium as gym\n",
    "\n",
    "env = gym.make(\"GridWorldEnv-v0\", render_mode = \"rgb_array\", grid_file='custom_envs/grid_envs/grid_world2.txt')\n",
    "env = GymWrapper(env, from_pixels=True, pixels_only=False, device=\"cpu\")\n",
    "print(env.spec.max_episode_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim\n",
    "from torchrl.data import CompositeSpec\n",
    "from torchrl.envs.libs.gym import GymEnv\n",
    "from torchrl.modules import ConvNet, MLP, QValueActor\n",
    "from torchrl.record import VideoRecorder\n",
    "from tensordict.nn import TensorDictModule\n",
    "import gymnasium as gym\n",
    "from torchrl.envs import GymWrapper\n",
    "\n",
    "from torchrl.envs import (\n",
    "    CatFrames,\n",
    "    DoubleToFloat,\n",
    "    EndOfLifeTransform,\n",
    "    GrayScale,\n",
    "    CenterCrop,\n",
    "    GymEnv,\n",
    "    NoopResetEnv,\n",
    "    Resize,\n",
    "    RewardSum,\n",
    "    SignTransform,\n",
    "    StepCounter,\n",
    "    ToTensorImage,\n",
    "    TransformedEnv,\n",
    "    VecNorm,\n",
    "    ObservationNorm\n",
    ")\n",
    "\n",
    "# from utils_modules import DQNNetwork\n",
    "\n",
    "# ====================================================================\n",
    "# Environment utils\n",
    "# --------------------------------------------------------------------\n",
    "\n",
    "\n",
    "def make_env(env_name=\"CartPole-v1\", frame_skip = 4, \n",
    "             device=\"cpu\", seed = 0, obs_norm_sd = None):#, is_test=False):\n",
    "    \n",
    "    if obs_norm_sd is None:\n",
    "        obs_norm_sd = {\"standard_normal\": True}\n",
    "\n",
    "    env = gym.make(\"GridWorldEnv-v0\", render_mode = \"rgb_array\", grid_file='../custom_envs/grid_envs/grid_world2.txt')\n",
    "    env = GymWrapper(env, from_pixels=True, pixels_only=False, device=\"cpu\")\n",
    "    # env = GymEnv(\n",
    "    #     env_name,\n",
    "    #     frame_skip=frame_skip,\n",
    "    #     from_pixels=True,\n",
    "    #     pixels_only=True,\n",
    "    #     device=device,\n",
    "    # )\n",
    "    env = TransformedEnv(env)\n",
    "    # env.append_transform(NoopResetEnv(noops=30, random=True)) # NOTE: Cartpole with no noops will fall into reset in the begining\n",
    "                                                                # I could use an small noop reset to avoid this, but I think is not necesary\n",
    "                                                                # in this case. Analyze this later\n",
    "    # if not is_test:\n",
    "        # env.append_transform(EndOfLifeTransform()) # NOTE: Check my environment is not based on lives (so not important)\n",
    "        # env.append_transform(SignTransform(in_keys=[\"reward\"])) #NOTE: cartpole has no negative rewards\n",
    "    env.append_transform(ToTensorImage()) \n",
    "    env.append_transform(GrayScale())\n",
    "    env.append_transform(Resize(64, 64))\n",
    "    env.append_transform(RewardSum())\n",
    "    env.append_transform(StepCounter()) # NOTE: GridWordv0 has a max of 200 steps\n",
    "    env.append_transform(DoubleToFloat())\n",
    "    env.append_transform(ObservationNorm(in_keys=[\"pixels\"], **obs_norm_sd))\n",
    "    env.set_seed(seed)\n",
    "\n",
    "    # NOTE: a rollout will be take a trajectory of frames and group the frames by N=4 sequentially\n",
    "    # so that the output will be 7x4x84x84 because with a rollout of 10 steps we will have 7 groups of\n",
    "    # 4 frames each\n",
    "    return env\n",
    "\n",
    "def get_norm_stats(num_iter = 250):\n",
    "    # NOTE: num_iter depends of the complexity of the input images (set to 1000 is a good approach)\n",
    "    test_env = make_env()\n",
    "    test_env.set_seed(0)\n",
    "    test_env.transform[-1].init_stats(\n",
    "        num_iter=num_iter, cat_dim=0, reduce_dim=[-1, -2, -4], keep_dims=(-1, -2)\n",
    "    )\n",
    "    obs_norm_sd = test_env.transform[-1].state_dict()\n",
    "    # let's check that normalizing constants have a size of ``[C, 1, 1]`` where\n",
    "    # ``C=4`` (because of :class:`~torchrl.envs.CatFrames`).\n",
    "    print(\"state dict of the observation norm:\", obs_norm_sd)\n",
    "    test_env.close()\n",
    "    del test_env\n",
    "    return obs_norm_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state dict of the observation norm: OrderedDict([('standard_normal', tensor(True)), ('loc', tensor([[[0.3603]]])), ('scale', tensor([[[0.4399]]]))])\n"
     ]
    }
   ],
   "source": [
    "obs_norm_sd = get_norm_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "state dict of the observation norm: OrderedDict([('standard_normal', tensor(True)), ('loc', tensor([[[0.3603]]])), ('scale', tensor([[[0.4399]]]))])\n"
     ]
    }
   ],
   "source": [
    "obs_norm_sd = get_norm_stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = make_env(obs_norm_sd=obs_norm_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 1, 64, 64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 1, 64, 64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.rollout(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim\n",
    "from torchrl.data import CompositeSpec\n",
    "from torchrl.envs.libs.gym import GymEnv\n",
    "from torchrl.modules import ConvNet, MLP, QValueActor\n",
    "from torchrl.record import VideoRecorder\n",
    "from tensordict.nn import TensorDictModule\n",
    "import gymnasium as gym\n",
    "from torchrl.envs import GymWrapper\n",
    "\n",
    "from torchrl.envs import (\n",
    "    CatFrames,\n",
    "    DoubleToFloat,\n",
    "    EndOfLifeTransform,\n",
    "    GrayScale,\n",
    "    CenterCrop,\n",
    "    GymEnv,\n",
    "    NoopResetEnv,\n",
    "    Resize,\n",
    "    RewardSum,\n",
    "    SignTransform,\n",
    "    StepCounter,\n",
    "    ToTensorImage,\n",
    "    TransformedEnv,\n",
    "    VecNorm,\n",
    "    ObservationNorm\n",
    ")\n",
    "\n",
    "# from utils_modules import DQNNetwork\n",
    "\n",
    "# ====================================================================\n",
    "# Environment utils\n",
    "# --------------------------------------------------------------------\n",
    "\n",
    "\n",
    "def make_env(env_name=\"CartPole-v1\", frame_skip = 4, \n",
    "             device=\"cpu\", seed = 0):\n",
    "\n",
    "    env = gym.make(\"GridWorldEnv-v0\", render_mode = \"rgb_array\", grid_file='../custom_envs/grid_envs/grid_world2.txt')\n",
    "    env = GymWrapper(env, from_pixels=True, pixels_only=False, device=\"cpu\")\n",
    "    # env = GymEnv(\n",
    "    #     env_name,\n",
    "    #     frame_skip=frame_skip,\n",
    "    #     from_pixels=True,\n",
    "    #     pixels_only=True,\n",
    "    #     device=device,\n",
    "    # )\n",
    "    env = TransformedEnv(env)\n",
    "    # env.append_transform(NoopResetEnv(noops=30, random=True)) # NOTE: Cartpole with no noops will fall into reset in the begining\n",
    "                                                                # I could use an small noop reset to avoid this, but I think is not necesary\n",
    "                                                                # in this case. Analyze this later\n",
    "    # if not is_test:\n",
    "        # env.append_transform(EndOfLifeTransform()) # NOTE: Check my environment is not based on lives (so not important)\n",
    "        # env.append_transform(SignTransform(in_keys=[\"reward\"])) #NOTE: cartpole has no negative rewards\n",
    "    env.append_transform(ToTensorImage()) \n",
    "    env.append_transform(GrayScale())\n",
    "    env.append_transform(Resize(64, 64))\n",
    "    env.append_transform(RewardSum())\n",
    "    env.append_transform(StepCounter()) # NOTE: GridWordv0 has a max of 200 steps\n",
    "    env.append_transform(DoubleToFloat())\n",
    "    env.append_transform(ObservationNorm(in_keys=[\"pixels\"]))\n",
    "    env.transform[-1].init_stats(\n",
    "        num_iter=300, cat_dim=0, reduce_dim=[-1, -2, -4], keep_dims=(-1, -2)\n",
    "    )\n",
    "    env.set_seed(seed)\n",
    "\n",
    "    # NOTE: a rollout will be take a trajectory of frames and group the frames by N=4 sequentially\n",
    "    # so that the output will be 7x4x84x84 because with a rollout of 10 steps we will have 7 groups of\n",
    "    # 4 frames each\n",
    "    return env\n",
    "\n",
    "def get_norm_stats(num_iter = 250):\n",
    "    # NOTE: num_iter depends of the complexity of the input images (set to 1000 is a good approach)\n",
    "    test_env = make_env()\n",
    "    test_env.set_seed(0)\n",
    "    test_env.transform[-1].init_stats(\n",
    "        num_iter=num_iter, cat_dim=0, reduce_dim=[-1, -2, -4], keep_dims=(-1, -2)\n",
    "    )\n",
    "    obs_norm_sd = test_env.transform[-1].state_dict()\n",
    "    # let's check that normalizing constants have a size of ``[C, 1, 1]`` where\n",
    "    # ``C=4`` (because of :class:`~torchrl.envs.CatFrames`).\n",
    "    print(\"state dict of the observation norm:\", obs_norm_sd)\n",
    "    test_env.close()\n",
    "    del test_env\n",
    "    return obs_norm_sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = make_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 1, 64, 64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 1, 64, 64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env.rollout(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent: tensor([5, 3])\n",
      "Done main: tensor([False])\n",
      "Done info: tensor([False])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAANo0lEQVR4nO3dX4xcdd3H8c90V7YBi6iVlnCBijQ2Gqr8MSLEXqDyT4OXmBD/3ZCY6KOCQkwUoon8MyYNXnhhUokXjdwYJSLSXrRRaChUuChSUEsMPAQMoFRKOuzsnudiwveR5DFZn5ntr3v29Uo2mWxn5/c9OzvnPefMdHfQdV0XAEiypvUAABw/RAGAIgoAFFEAoIgCAEUUACiiAEARBQDK7FKvOBgMlnMOAJbZUv6vsiMFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgzLYeoLW77rormzZtaj3GxA4cOJCrr74669aty+7duzMzM9N6pIlt374927Zty0UXXZQf/ehHrceZii996Ut54IEH8tWvfjWf//znW48zsdFolK1bt+bIkSPZkWRz64Gm4PEkn2k9REOrPgqbNm3Kli1bWo8xseFwmCSZmZnJ2WefndnZlX/Xbty4MUmybt26XtxHyXhbkuS0007rxTbNz8/XE5Czkqz8LUoWWg/QmNNHABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgDLbeoDWDhw4kOFw2HqMif3xj39MkoxGozz00EOZmZlpPNHknnnmmSTJyy+/nH379jWeZjpefvnlJMnTTz/di20ajUZZWFhIkjyWZKHtOFPxROsBGht0Xdct6YqDwXLPAsAyWsruftUfKaxbt64Xz6pHo1FeeeWVDAaDvOUtb2k9zlQcPXo0R48ezezsbN785je3HmcqXnnllYxGo6xduzZr165tPc5U/OMf/0jSn8fSwsJC/vnPf7Yeo5lVf6Swf//+nH322a3HmNhDDz2Uj3zkIznllFPy3HPP9eLBefvtt+db3/pWLr300tx9992tx5mKK664Ivfdd19uueWWXHvtta3HmdhoNMqGDRty+PDhPPjggznnnHNajzSxRx55JB/60Idaj7EsHCkswczMTGZnV/634V8j0Jdtev2JyGAw6MX2JP3bpn/dyfTl564PT6gm4d1HABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgDLbeoDWtm/fno0bN7YeY2LPPPNMkuTo0aO5/fbbMxgMGk80uT179iRJDh06lFtuuaXxNNPx1FNPJUl2796dxcXFxtNMbnFxMcPhMEly5513ZufOnY0nmtyzzz7beoSmBl3XdUu6Yg92MgCr2VJ296v+SIHj3zve8Y6cf/75rceYin379uWFF15oPQb8W6LAce+8887Lr3/969ZjTMWll16a3/72t63HgH/LC80AFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAlEHXdd2SrjgYLPcs8H9av359zj333NZjTMX+/fvzwgsvtB6DVWopu3tRAFgllrK7nz0GcxzXvva1r+W0005rPcbEnn766dxxxx1Zu3ZtbrrppqxZs/LPDO7evTv33HNP3vve9+aLX/xi63Gm4ic/+UmefPLJXHHFFdm6dWvrcSa2uLiYG2+8McPhMP+V5PTWA03BfyfZ1nqIlrolStLLj0cffXSp34Lj2oMPPtgl6U455ZRufn6+9ThTcfPNN3dJussuu6z1KFNzySWXdEm6W2+9tfUoU/Haa691J598cpekezjpuh587D8O9kvL9bEUK//pJABTIwoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQZlsP0NpoNMr8/HzrMSY2Go3ecLnruobTTMfi4mKSpOu6XtxHSep+WVxc7MU2veHnLsnK36Lxdqxmg26Je4/BYLDcszRx0kknZWZmpvUYE1tYWMiRI0eSJCeffHLjaaZjOBxmOBxmdnY2J554YutxpuLVV1/NaDTK3Nxc5ubmWo8zFYcPH06SnJRk5T+SkoUkR1oPsUyWsrtf9VEAWC2Wsrtf9aePduzYkbPOOqv1GBN77LHH8rnPfS7r1q3Lrl27enH0c+edd+aOO+7IhRdemG3btrUeZyq+/OUvZ+/evfnKV76Sz372s63HmdhoNMrFF1+cI0eO5Gc/+1k2b97ceqSJHTx4MFdffXXrMZpZ9VHYvHlztmzZ0nqMiS0sLCRJZmZmcs4552R2duXftTt37kwyPh127rnnNp5mOl4/tXf66af3Ypvm5+frCcjmzZt7sU2r/ayIdx8BUEQBgCIKABRRAKCIAgBFFAAoK/99i/Afeu655LXXxpevuip58cU3/vv69cmOHePLc3PJhg3Hdj5oSRRYNZ5/PvnlL5Pvfz/561///fWefDI544zx5Xe9K7nhhuTTn05OPfWYjAlNiQK9t7iY3HZbcu+9yZ49/9nXPvVUcs014yOHyy9Prr02WeOkKz0mCvTao48mv/hFcvPNySS/lHT37uT++5NXX02uvDL5wAemNCAcZzznobfuuSe5+OLku9+dLAivm59Pbrop+djHkt/8ZvLbg+ORKNA7XZfcd1/yhS8kL700/dt/8cXxbe/aNV4L+kQU6J2dO5NPfSr529+Wb43nn08++clxGKBPRIFe6brk1lv/9y2ny2k4HL+A7WiBPhEFemM4HL9+8PvfH7s1f/e75HvfG68NfSAK9Maf/jR+IfhYHCW8bjhMbrwx+ctfjt2asJxEgV7ouuT669utf/31TiPRD6JALxw4kOzd2279Bx5IHnus3fowLaJAL/zgB8nf/95u/ZdeSn74w3brw7SIAgBFFFjxDh1K/vzn1lOMf5HeoUOtp4DJiAIr3r5943P6rd1/f/Lww62ngMmIAgBFFAAoogBAEQUAiiiw4p1/fnLBBa2nSC68MDnvvNZTwGREgRXvzDOTs85qPUWyaVPy7ne3ngImIwoAFFGgF667LnnrW9ut/7a3JV//erv1YVpEgV54//uTD3+43foXXJC8733t1odpEQV6YTAY/xW0Vm67bTwDrHSiQG+85z3jP3hzwgnHbs0TThj/YZ8zzzx2a8JyEgV6Y+3acRQuuujYrfnRjybf+U4yN3fs1oTlJAr0ymCQfPObx+ZoYW4u+cY3nDaiX0SB3vnEJ5Jf/So59dTlW+PUU5O7704+/vHlWwNaEAV6ZzBILrkk2b59/FbRaXv725Of/nQcBEcJ9I0o0FuXX57s2pV8+9vJ7Ozkt/emN41fs9i5M7nssslvD45HU3iowPHrgx9MtmxJTjwxuffeZM+e/9/tbN06jsx11yVrPJWix/x403tr1iQ33JD8/OfJj3+cnHHG0r/2ne8cf81dd41fwBYE+s6RAqvGhg3JNdckV16ZDIfjz111VfLii2+83vr1yY4d48tzc8nGjcd2TmhJFFh1/nUnv3dvuzngeORgGIAiCgAUUQCgiAIARRQAKKIAQFn1b0l9/PHHs7Cw0HqMiT3xxBNJkoWFhTzyyCOZmZlpPNHknn322STJ4cOH84c//KHxNNNx+PDhJONt68M2jUajevwcPHgwgx78MqiDBw+2HqGpQdd13ZKu2IM7G2A1W8ru3ukjAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgzC71il3XLeccABwHHCkAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUP4Hkrq8C+8nUh0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action: tensor([0, 0, 1, 0])\n",
      "Agent: tensor([4, 3])\n",
      "Done main: tensor([False])\n",
      "Done info: tensor([False])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAANtklEQVR4nO3da4hd9b3H4e/ODHVajR003hAptJg20Cp4K1WpUGm9hEK1FCqkF6QvesFbpfTypvaVEcES4qtiG0RQsKUUNILRQnyjJDnRQrUmaVHQ1Eaw1MSMZpK9Z50X+/g7zeGUM6d7T9bM2s8Dg6NZ7v9vZc+sz1577zXTa5qmCQAkWdX2AAAsH6IAQBEFAIooAFBEAYAiCgAUUQCgiAIAZXqxG/Z6vaWcA4AltphrlZ0pAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIAZbrtAdr26KOPZu3atW2PMbIXX3wxGzZsyOrVq7N9+/ZMTU21PdLItmzZkk2bNuXKK6/M/fff3/Y4Y/Hd7343zz77bG6//fZ885vfbHuckfX7/Vx11VWZm5vLI0nWtT3QGLyc5Ka2h2jRxEdh7dq1ufDCC9seY2Tz8/NJkqmpqVxwwQWZnl75d+3ZZ5+dJFm9enUn7qNkuC9Jcs4553Rin44dO1YPQM5PsvL3KBm0PUDLPH0EQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKNNtD9C2F198MfPz822PMbI//elPSZJ+v59du3Zlamqq5YlGt3///iTJwYMHs3PnzpanGY+DBw8mSV5//fVO7FO/389gMEiSvJRk0O44Y7G37QFa1muaplnUhr3eUs8CwBJazOF+4s8UVq9e3YlH1f1+P4cPH06v18uHP/zhtscZiyNHjuTIkSOZnp7OKaec0vY4Y3H48OH0+/3MzMxkZmam7XHG4u23307Sne+lwWCQd955p+0xWjPxZwq7d+/OBRdc0PYYI9u1a1cuv/zyzM7O5sCBA5345rz33nvzk5/8JNdee20ee+yxtscZi/Xr12fbtm3ZuHFj7rzzzrbHGVm/389ZZ52VQ4cOZceOHbnooovaHmlkL7zwQi677LK2x1gSzhQWYWpqKtPTK/+v4Z8j0JV9ev+BSK/X68T+JN3bp38+yHTl664LD6hG4d1HABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgDLd9gBt27JlS84+++y2xxjZ/v37kyRHjhzJvffem16v1/JEo3vmmWeSJK+88ko2btzY8jTj8eqrryZJtm/fnoWFhZanGd3CwkLm5+eTJA8++GCeeuqplica3RtvvNH2CK3qNU3TLGrDDhxkACbZYg73E3+mwPJ3xhln5NJLL217jLHYuXNn3nrrrbbHgH9JFFj2LrnkkmzdurXtMcbi2muvzZNPPtn2GPAveaEZgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgNJrmqZZ1Ia93lLPAv+rNWvW5OKLL257jLHYvXt33nrrrbbHYEIt5nAvCgATYjGH++kTMMeydscdd+Scc85pe4yRvf7669m8eXNmZmZy1113ZdWqlf/M4Pbt2/PEE0/kE5/4RG6++ea2xxmLBx54IPv27cv69etz1VVXtT3OyBYWFvLTn/408/PzuS3JuW0PNAZ/TbKp7SHa1CxSkk5+/OEPf1jsX8GytmPHjiZJMzs72xw7dqztccbi7rvvbpI01113XdujjM0111zTJGnuueeetkcZi6NHjzannnpqk6T5j6RpOvCxexkcl5bqYzFW/sNJAMZGFAAoogBAEQUAiigAUEQBgDLx1ykwef75+p0f/Sg5ePD4P5+dTe6++7//3XWbTBJRYGK8916yf3/yve8le/cO/9tf/5oMBsdvNzWVPPLI8PN165LNm5PzzktmZk7svNAGUWAibN2a/P73yc9//n9vOxgkr702/Py115K1a5Pvfz+5+urk+uuXdk5omyjQaUePJtu2Jd/6VvLmm//+7dx3X/Lww8kDDySf/3zygQ+Mb0ZYTrzQTGf9+c/JJz+ZfPnLowXhfQcOJDfemHzqU8lf/jL67cFyJAp00t69yVe/OgzD0aPju92jR5N9+4a3vW/f+G4XlgtRoHP+9rfki19Mnn9+6dbYvXu4xoEDS7cGtEEU6JwNG4ZnCEtt377ka19b+nXgRBIFOqNpksceS1566cSt+cc/Jo8/fvy1D7CSiQKdsWdPctNN43lRebHefNPrC3SLKNAJTZM89FAyN3fi156bG67tbIEuEAU64b33kl/9qr31f/nL5MiR9taHcREFOuGWW07s00b/04EDyW23tbc+jIso0An9ftsTLI8ZYFSiwIp3+PDwo23vvLM85oBRiAIr3uOPJ7/9bdtTJL/5TfLEE21PAaMRBQCKKABQRAGAIgoAFFFgxfvIR5KPfrTtKZKPfWw4C6xkosCK95nPJFde2fYUyWc/m3z6021PAaMRBQCKKNAJ69cnJ5/c3vqnnJJcf31768O4iAKd8JWvJGee2d76Z545/F3QsNKJAp3R5lXNy+GKahgHUaATer3kvPOSL33pxK99ww3DtXu9E782jJso0Bmnn55s2TJ8N9KJcsUVw9/jcNppJ25NWEqiQKfMziY33pisOgFf2VNTw7VmZ5d+LThRRIHOuf325K67ljYMq1YlP/uZX6xD94gCnTM9nfz4x8nGjcnMzPhv/4MfTO65J/nhD4dnC9Al020PAEthejr5wQ+G1w9s25b87nfjud0bbki+8IXk298ez+3BciMKdNp3vjO8hqFpku3bk4MH/73bmZ1NPve55Be/GL6gDV3l6SM6b82a4XUEO3YkN930/3un0OmnD/+fnTuTX/9aEOg+ZwpMhFWrko9/PHn44WTr1uTvfx+ePdxxR/KPfxy/7WmnJffdN7zuYM0aP76CySIKTJz164f/bJrkuuuShYXj/3zVquSMM1yMxmQSBSZWr9fuz0uC5chrCgAUUQCgiAIARRQAKKIAQJn4dx/1+/0cO3as7TFG1u/3j/u8aZoWpxmPhf96r2jTNJ24j5LU/bKwsNCJfTru6y7Jyt+j4X5Msl6zyKNHr6Nv2j755JMz1YGfajYYDDI3N5ckOfXUU1ueZjzm5+czPz+f6enpfOhDH2p7nLF499130+/3c9JJJ+Wkk05qe5yxOHToUJLk5CQr/zspGSSZa3uIJbKYw/3ERwFgUizmcD/xTx898sgjOf/889seY2QvvfRSvvGNb2T16tV5+umnO3H28+CDD2bz5s254oorsmnTprbHGYtbbrklzz33XG699dZ8/etfb3uckfX7/Vx99dWZm5vLQw89lHXr1rU90sj27NmTDRs2tD1GayY+CuvWrcuFF17Y9hgjGwwGSZKpqalcdNFFmZ5e+XftU089lWT4dNjFF1/c8jTj8f5Te+eee24n9unYsWP1AGTdunWd2KdJf1bEu48AKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAIooAFBEAYAiCgAUUQCgiAIAZbrtAdr28ssvZzAYtD3GyPbu3ZskGQwGeeGFFzI1NdXyRKN74403kiSHDh3K888/3/I043Ho0KEkw33rwj71+/36/tmzZ096vV7LE41uz549bY/Qql7TNM2iNuzAnQ0wyRZzuPf0EQBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUKYXu2HTNEs5BwDLgDMFAIooAFBEAYAiCgAUUQCgiAIARRQAKKIAQBEFAMp/AqHIyBmYOWG7AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action: tensor([0, 1, 0, 0])\n",
      "Agent: tensor([4, 4])\n",
      "Done main: tensor([False])\n",
      "Done info: tensor([True])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAANuklEQVR4nO3dX4hchd3H4e+4oy5qbMCIsd4UQTFWEkg0pSp4oaKSUrAXBUGMFaRFqq82tLTSqr0y4kWbKkWoRWyg0mAvrDWg8SLeKGqjVvwTvVAweVMpQTQmaTY7u+e9GPxpUV8WZ3bP7OzzwMJIxjm/k8mez5wz58x0mqZpAgBJjml7AABGhygAUEQBgCIKABRRAKCIAgBFFAAoogBA6c71jp1OZz7nAGCezeVaZXsKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBA6bY9QNu2bduWs88+u+0xBvbaa6/l2muvzbJly7Jz585MTEy0PdLAHnrooWzZsiUXX3xx7r///rbHGYqbbropzz77bG699dZcf/31bY8zsF6vl0suuSSHDh3KI0lWtT3QELyZ5Jq2h2jRko/C2WefnTVr1rQ9xsCmpqaSJBMTE1m9enW63cX/1K5cuTJJsmzZsrF4jpL+uiTJ6aefPhbrND09XS9Azkqy+NcomWl7gJY5fARAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAo3bYHaNtrr72WqamptscY2BtvvJEk6fV6efHFFzMxMdHyRIPbu3dvkuSjjz7KCy+80PI0w/HRRx8lSfbs2TMW69Tr9TIzM5MkeT3JTLvjDMVbbQ/Qsk7TNM2c7tjpzPcsAMyjuWzul/yewrJly8biVXWv18vBgwfT6XTyta99re1xhuLIkSM5cuRIut1uTjrppLbHGYqDBw+m1+tlcnIyk5OTbY8zFB9++GGS8fldmpmZyccff9z2GK1Z8nsKu3btyurVq9seY2AvvvhiLrzwwixfvjzvv//+WPxy3nvvvbn99ttz5ZVX5vHHH297nKHYsGFDnnrqqWzevDmbNm1qe5yB9Xq9nHbaaTlw4ECef/75rF27tu2RBvbyyy9n/fr1bY8xL+wpzMHExES63cX/1/DZCIzLOn3yQqTT6YzF+iTjt06f3ciMy7+7cXhBNQhnHwFQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQBKt+0B2vbQQw9l5cqVbY8xsL179yZJjhw5knvvvTedTqfliQb3zDPPJEneeeedbN68ueVphuPdd99NkuzcuTOzs7MtTzO42dnZTE1NJUkefvjh7Nixo+WJBrdv3762R2hVp2maZk53HIONDMBSNpfN/ZLfU2D0nXrqqbngggvaHmMoXnjhhezfv7/tMeBLiQIj7/zzz88TTzzR9hhDceWVV+bJJ59sewz4Ut5oBqCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKB0mqZp5nTHTme+Z4EvtGLFiqxbt67tMYZi165d2b9/f9tjsETNZXMvCgBLxFw2990FmGOk3XbbbTn99NPbHmNge/bsyX333ZfJycncddddOeaYxX9kcOfOndm+fXvOOeec3HDDDW2PMxQPPvhg3n777WzYsCGXXHJJ2+MMbHZ2NnfeeWempqbyP0nOaHugIfjfJFvaHqJNzRwlGcufV155Za5/BSPt+eefb5I0y5cvb6anp9seZyjuvvvuJklz1VVXtT3K0FxxxRVNkuaee+5pe5ShOHr0aHPyySc3SZp/JE0zBj+7RmC7NF8/c7H4X04CMDSiAEARBQCKKABQRAGAIgoAlCV/nQKMg0+uSdq/P/nVrz7/5+vXJz/4waf/7VpUvowowCL2wQfJ3r3J1VcnvV7/Z9++z99v69bk17/u377mmuTGG5MzzxQHPs/hI1iEmib5wx+SjRuTNWuSd95J3nvvi4OQJIcP9//8vfeSe+5Jzj03+e1vk1dfXdCxWQTsKcAi0jTJ0aPJb36T3HFHMj391R7n6NHkJz/px2H79uSMM5KurQGxpwCLyo4dySmnJL/85VcPwme98UayalX/kJIPbyWxpwCLxvbt/cNFhw4N93H/85/k0Uf7t7duTSYnh/v4LC72FGDENU1/D2Hjxvl9Nf/oo/1lHDw4f8tg9IkCjLijR5PvfW9hDu9s25b87nfzvxxGlyjACGua/pvKR44s3DL/8pf+2Uxz+/otxo0owAh78MHkzjv71x8slFdfTS6/XBSWKlGAEfXBB8nf/tY/fLTQ3n8/eeyxhV8u7RMFGFF79yZ//3s7yz58OPnzn4dz2iuLiyjACGqa/kdXtOmvf+2/v8DSIgowohbyfYQv0jTJ7Gy7M7DwRAFG0P797UchSf7979GYg4UjCjCC7rjjyz/cbiH99Kc+/mKpEQUAiigAUEQBgCIKABRRgBG0fn1y4oltT5FceKGP0l5qRAFG0PXX979Mp20//GGyfHnbU7CQRAGAIgowoq65pt3ln3tuct557c7AwhMFGEGdTnLjjclxx7U3w7e+laxd297yaYcowIg688xk8+Z2ln3KKckDD7SzbNolCjCiOp3k0kuTb35z4Zf94x8nxx678MulfaIAI2z16mT79uSEExZumZs2Jb/4RT9KLD2iACPu619PNmxYmGWtXJlcdlly/PELszxGT7ftAYD/X7eb/P73/e83ePTR+VvOccclf/pT//uZWbrsKcAisGJFsnVr8v3vz8/jr1zZ/+rPyy6bn8dn8bCnAIvE5GTyxz8ma9Yk27Yl//zncB5306b+G9r2EEhEARaVk05Kbr+9f2Hb5Zcn//pXcvjwV3usFSv6Zxn9/OfeQ+BTogCL0De+kbz9dvLYY8kjj/Tfa2iauf2/557bvzDtgQf6p506y4jPEgVYhDqd/s/VVyff+U7y3e8ms7P971T+2c8+H4hvfzv50Y/6t887z5XKfDlRgEXu2GOTa6/t3+71Pr39WZOTPu2UuREFGCPdbv9MIviqnJIKQBEFAIooAFBEAYAiCgCUJX/2Ua/Xy/T0dNtjDKzX6/3X7WauVzKNsNnZ2SRJ0zRj8RwlqedldnZ2LNbpv/7dJVn8a9Rfj6Ws08xx69EZ08seTzzxxExMTLQ9xsBmZmZy6NChJMnJJ5/c8jTDMTU1lampqXS73ZywkF8oMI8OHz6cXq+X448/PsePyWdLHDhwIElyYpLF/5uUzCQ51PYQ82Qum/slHwWApWIum/slf/jokUceyVlnndX2GAN7/fXXs3HjxixbtixPP/30WOz9PPzww7nvvvty0UUXZcuWLW2PMxQ333xznnvuudxyyy257rrr2h5nYL1eL5deemkOHTqUrVu3ZtWqVW2PNLDdu3fn2i+6LHyJWPJRWLVqVdasWdP2GAObmZlJkkxMTGTt2rXpdhf/U7tjx44k/cNh69ata3ma4fjk0N4ZZ5wxFus0PT1dL0BWrVo1Fuu01I+KOPsIgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAULptD9C2N998MzMzM22PMbC33norSTIzM5OXX345ExMTLU80uH379iVJDhw4kJdeeqnlaYbjwIEDSfrrNg7r1Ov16vdn9+7d6XQ6LU80uN27d7c9Qqs6TdM0c7rjGDzZAEvZXDb3Dh8BUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgClO9c7Nk0zn3MAMALsKQBQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQ/g8SH9e59LGQ6QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action: tensor([0, 1, 0, 0])\n"
     ]
    }
   ],
   "source": [
    "# Plot the first pixels image\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "for i in range(len(data)):\n",
    "    print(f\"Agent: {data['observation'][i]}\")\n",
    "    print(f\"Done main: {data['done'][i]}\")\n",
    "    print(f\"Done info: {data['next', 'done'][i]}\")\n",
    "    plt.imshow(data[\"pixels\"][i])\n",
    "    plt.axis('off')\n",
    "    plt.show()\n",
    "    print(f\"Action: {data['action'][i]}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAN+0lEQVR4nO3db6zdhV3H8c/puYxauKVuENrVJ5rQ9WYJ3SjwAHDdxISyOo1xM7rcAGE8wsBAEkWWSGfmKBLFCvqIjXU0w7CZqAiOgVuZbgRK15kAbXkAJpRuJIjrXQu93HPvzwds3zCR5W7n3PO799zXK7lJSQ7n9/n19J73+dN72mmapgkAJFnR9gAAFg9RAKCIAgBFFAAoogBAEQUAiigAUEQBgDI23wt2Op2F3AHAApvPzyp7pgBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKAJSxtge07b777suGDRvantG3p556KpOTkxkfH8+ePXvS7XbbntS3u+++Ozt37sxFF12UO++8s+05A3H11Vfn29/+dq677rpcccUVbc/pW6/Xy5YtW3L8+PHcm2Si7UEDcCDJ77c9okXLPgobNmzIpk2b2p7Rt+np6SRJt9vN2WefnbGxpX/Trl27NkkyPj4+ErdR8sa5JMm6detG4pxmZmbqAchZSZb+GSWzbQ9omZePACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAGWs7QFte+qppzI9Pd32jL4988wzSZJer5e9e/em2+22vKh/hw8fTpIcPXo0TzzxRMtrBuPo0aNJkhdeeGEkzqnX62V2djZJ8nSS2XbnDMShtge0rNM0TTOvC3Y6C70FgAU0n7v7Zf9MYXx8fCQeVfd6vRw7diydTiennXZa23MG4sSJEzlx4kTGxsZy6qmntj1nII4dO5Zer5eVK1dm5cqVbc8ZiB/84AdJRud7aXZ2Nj/84Q/bntGaZf9MYd++fTn77LPbntG3vXv35oILLsiaNWvy/e9/fyS+OW+77bbcdNNN2bp1a+6///625wzEtm3b8rWvfS07duzIDTfc0PacvvV6vZx55pmZmprK448/nnPOOaftSX3bv39/zj///LZnLAjPFOah2+1mbGzp/za8OQKjck4/fiDS6XRG4nyS0TunN9/JjMqfu1F4QNUPf/sIgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUMbaHtC2u+++O2vXrm17Rt8OHz6cJDlx4kRuu+22dDqdlhf179FHH02SPPfcc9mxY0fLawbj+eefT5Ls2bMnc3NzLa/p39zcXKanp5Mku3btysMPP9zyov4dOXKk7Qmt6jRN08zrgiNwJwOwnM3n7n7ZP1Ng8TvjjDNy3nnntT1jIJ544om8/PLLbc+AtyUKLHrnnntuHnjggbZnDMTWrVvz0EMPtT0D3pY3mgEoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAoogBAEQUAiigAUEQBgCIKABRRAKCIAgBFFAAonaZpmnldsNNZ6C3w/zr99NOzefPmtmcMxL59+/Lyyy+3PYNlaj5396IAsEzM5+5+bAg7FrXrr78+69ata3tG31544YXccccdWblyZbZv354VK5b+K4N79uzJgw8+mI0bN+bKK69se85A3HXXXXn22Wezbdu2bNmype05fZubm8vNN9+c6enpfDLJ+rYHDcCLSXa2PaJNzTwlGcmv7373u/P9LVjUHn/88SZJs2bNmmZmZqbtOQNxyy23NEmaSy+9tO0pA3PJJZc0SZpbb7217SkD8frrrzerV69ukjRPJk0zAl/7FsH90kJ9zcfSfzgJwMCIAgBFFAAoogBAEQUAiigAUJb9zykAi0/zpl9/KxdmdybfcpnJ7M6F+Vb9tx+vHQxRABaVF/Pu/Ecuyh/lL5Ikx3JqXsm73nK5L+djOTXHkiQ7cmM+kG9mfY4MdesoEgVgUXglv5jdmczf5eocysZ5XP5dFYuP595syKH8Qf42k9mdd+Z/FnruyBIFoFWzWZFexvKJfC7/mN/+ua/n2bwnn8zf5Ov5tfx9fi8nZSbdzA1w6fLgjWagNTMZyy35k7wzr+Sf8lsDuc5/zm/mXfnvfDY3Zcbj3p+ZKACt+etclz/Nn+XVnJJmQHdHTVbk1ZySm/Pp3J7rB3Kdy4mMAkP3ek7K7bk+n87NA4vB/9VkRbZne5p0cn1uzzsysyDHGTWeKQBDtz/vz43ZkdeyakGP81pW5cbsyJM5d0GPM0pEARiqXrrZkRszvJ8s6OTW/HF66Q7peEubKABDdVXuGtibyvN1fz6ST+RzQz3mUiUKwNDsz/uyN+ct2PsIb6fJijyZc7M/7xvqcZciUQCGYi6d/Ht+Nc/kva0c/5m8N9/MBzLnAzF+KlEAhuLFrM8f5q9a3XBD/jKH80utbljsRAEYiiadzLb8Zu9sumk8U/ipRAEYiu9lXdsTkiyeHYuVKABDcVm+mPY/4Lrzox28HVEAoIgCAEUUACiiAEARBWAoLs6/5Sf/9eU2ND/awdsRBWAobsyOtickWTw7FitRAKCIAjAUqzOVrflqqxu25qs5LUdb3bDYiQIwFGtyNL+b+9JNr5Xjd9PLx/LlrBGFn0oUgKG5LF/MR3J/K8f+jfxLLs+uVo69lIgCMDTdzOWq3JVfyKtDPe7KvJarcle6mRvqcZciUQCG6sN5MF/Kx4d6zN2ZzLY8MNRjLlWiAAxVJ8mm/Gc258mhHG9znsz7s7/1j+JbKkQBGLpfzn9ldyazMQeycD/Q1uQ9OZjdmcyv5PkFOsboEQWgFRtzKN/IhzKRAwty/RM5kD35YDbm0IJc/6gSBaA1a/NS/iG/k8/kUxnP1ECuczxT+Uw+la/ko1mblwZyncvJWNsDgOVtIgdzUz6b92d/Ls+uHM1pmck7fubrOSmv57QczRdyRT6cB72H8HPyTAFoXSfJpfnXvJQzsz3b86F8/Wf6/z+Yb+TmfDov5UxB6JNnCsCi0EnSSZObckuuyBfySH49SfJotuTzufItl78yn8+WPJrkjU9gXZ8jw5w7skQBWHTene/lstyTJPlovpI/z6fecpnVmcqqvDbsaSNPFIBFbVVec+c/RN5TAKCIAgBFFAAoogBAEQUAyrL/20e9Xi8zMzNtz+hbr9f7iV83zUJ9yNjwzM298dn3TdOMxG2UpG6Xubm5kTinn/hzl2Tpn1Fa+nfhFo9OM897j05nNH9G8JRTTkm32217Rt9mZ2dz/PjxJMnq1atbXjMY09PTmZ6eztjYWFatWtX2nIF49dVX0+v1cvLJJ+fkk09ue85ATE298ZlFpyRZ+t9JyWyS422PWCDzubtf9lEAWC7mc3e/7F8+uvfee3PWWWe1PaNvTz/9dC6//PKMj4/nkUceGYlnP7t27codd9yRCy+8MDt37mx7zkBcc801eeyxx3Lttdfmsssua3tO33q9Xi6++OIcP34899xzTyYmJtqe1LeDBw9mcnKy7RmtWfZRmJiYyKZNm9qe0bfZ2dkkSbfbzTnnnJOxsaV/0z788MNJ3ng5bPPmzS2vGYwfv7S3fv36kTinmZmZegAyMTExEue03F8V8bePACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAGWs7QFtO3DgQGZnZ9ue0bdDhw4lSWZnZ7N///50u92WF/XvyJEjSZKpqal85zvfaXnNYExNTSV549xG4Zx6vV59/xw8eDCdTqflRf07ePBg2xNa1WmappnXBUfgxgZYzuZzd+/lIwCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACiiAEARBQCKKABQRAGAIgoAFFEAoIzN94JN0yzkDgAWAc8UACiiAEARBQCKKABQRAGAIgoAFFEAoIgCAEUUACj/CyN1wzjWVjNCAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAP3RFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMS5wb3N0MSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8kixA/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAANh0lEQVR4nO3dzW9Uhf7H8e/MFFroAwUpD2oqRiMGk8YgLoxGjSuifwIJIXFlXBgiKxcmLkyUnRh3hpC4MHFtJIqPG2OswgYUNmraFKwl0pa2dOw83AXh+zM/783tZdo5Zfp6Lc3Y8zm2PW/OMDOWms1mMwAgIspFDwBg7RAFAJIoAJBEAYAkCgAkUQAgiQIASRQASF3LfWCpVFrNHQCssuW8V9mdAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkLqKHlC0jz76KPbu3Vv0jJadP38+Dh06FP39/fHNN99EpVIpelLLTp48Ge+880489dRT8d577xU9Z0W89NJL8e2338bRo0fjyJEjRc9pWa1Wi6effjrm5+fj0UcfjYGBgaIntWxubi7Onj1b9IzCrPso7N27N0ZGRoqe0bJqtRoREZVKJUZGRjoiCrt3746IiP7+/o74HkXcPJeIm+fWCedUq9XyZ62vr68jotBoNIqeUChPHwGQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQBSV9EDinb+/PmoVqtFz2jZhQsXIiKiXq/H6OhoVCqVghe1bnx8PCIiZmZmYnR0tOA1K2NmZiYibp5bJ5xTrVaLer0eERFzc3MFr1kZnXIet6vUbDaby3pgqbTaWwBYRcu53K/7O4X+/v6O+FN1vV6P69evR6lUii1bthQ9Z0UsLi7G4uJidHV1RV9fX9FzVsTc3FzUarXo6emJnp6eouesiOnp6YjovN+l9Wrd3ymcPXs2RkZGip7RstHR0XjiiSdicHAwJicnO+KX8/jx4/Haa6/FwYMH4+OPPy56zop4/vnn47PPPou33norjh07VvScltVqtdixY0fMzs7G999/H/v37y96UsvOnTsXjz/+eNEzVoU7hWWoVCodcQH9+zl0yjmVyzdfB1EqlTrifCL+7w9X5XK5I87p7xeZTvm564RzaIVXHwGQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQCSKACQRAGAJAoAJFEAIIkCAEkUAEiiAEASBQBSV9EDinby5MnYvXt30TNaNj4+HhERi4uLcfz48SiX7/zef/311xER8csvv8Tbb79d7JgV8uuvv0ZExFdffRWNRqPgNa1rNBpRrVYjIuLUqVNx5syZghe1bmJiougJhSo1m83msh5YKq32FgBW0XIu9+v+ToG1b2hoKA4cOFD0jBXxww8/xNTUVNEz4D8SBda8AwcOxCeffFL0jBVx8ODB+PTTT4ueAf/Rnf/EMwArRhQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIAqdRsNpvLemCptNpb4N/avn17PPbYY0XPWBE//vhjXL16tegZrFPLudyLAsA6sZzLfVcbdqxpR48ejbvvvrvoGS0bHx+PEydORE9PT7zxxhtRLt/5zwx++eWXcfr06Xj44YfjxRdfLHrOinj//ffj0qVL8cILL8Szzz5b9JyWNRqNeP3116NarcaePXuip6en6EktW1xcjN9++63oGYVZ91E4cuRIjIyMFD2jZaOjoxmFV199NSqVStGTWlav1+P06dNx//33x7Fjx4qesyI+//zzuHTpUjzzzDMdcU61Wi3efPPNqFarce+998bg4GDRk1o2PT29rqNw5/9xEoAVIwoAJFEAIIkCAEkUAEiiAEBa9y9JBda2+fn5uHbt2j/++datW6O3t7eARZ1NFIA1ZWlpKebn5+Py5csRcfMNcvV6/R+Pm56ezjdp7t69O/r6+mLDhg1t3dqJRAFYE+r1ely7di2uXr0a1Wp1WY+/FYuxsbHo7u6O7du3x+DgYHR1ubTdLv/lgELd+jyesbGxmJ2dve2vU61WY2JiIq5fvx7Dw8NRLpd9ZtttEAWgMM1mMyYnJ2NqaioajcaKfM3Z2dn46aefYmhoKHbu3CkM/yOvPgIKMzU1FZOTkysWhFsajUbGhv+NOwWg7ZrNZkxNTcXvv/++qse59fWHhobcMSyTOwWg7RYWFuLKlSvL+nz/VjSbzbhy5UosLCys6nE6iSgAbXXrLqGd/vjjj1UPUKcQBaCtxsfHY2Zmpq3HnJ2djfHx8bYe804lCkDb3Lhxo7CnchYWFuLGjRuFHPtOIgpA28zPzy/rjWmroVqtxtzcXCHHvpOIAtAWS0tLMTExUeiGy5cvx19//VXohrVOFIC28Be9dwZRANpiaWmp6AkRsXZ2rFWiALTFWnn1z9jYWNET1jRRACCJAgBJFABIogBAEgWgLfr6+oqeEBER/f39RU9Y00QBaIsdO3YUPSEi1s6OtUoUAEiiALRFpVIp/Kmb/v7+qFQqhW5Y60QBaItKpRKDg4OFbhgcHBSF/0IUgLbZunVrDAwMFHLsgYGB2Lp1ayHHvpOIAtA2pVIp7rrrrrb//5KLOu6dSBSAthoYGIjh4eG2HnN4eLiwO5Q7jSgAbbd58+bYtGlTW461adOm2Lx5c1uO1QlEAWi7jRs3xn333Rfd3d2repzu7u4YHh6OjRs3rupxOokoAIXo7u6OBx98cNXC0N3dHQ888ED09PSsytfvVF1FDwDWr66urtizZ09MT0/H1NRUNBqNlr9muVyOoaGhGBwcjA0bNqzAyvVFFIBC9fT0xK5du2Lz5s0xNjYWjUbjtv7XnaVSKcrlsr9UbpEoAGvCwMBAPPLIIzE5ORlzc3MxPz+/7H+3t7c3+vr6YufOnV522iJRANaMUqkUu3btiqWlpbh+/XpERMzPz8eff/75j8du27Ytent7I+Lmx1d4qmhliAKw5mzYsCG2bdsWERFbtmyJXbt2/eMx5XLZR1asAlEA1rRKpeLi30ZekgpAEgUAkigAkEQBgCQKAKR1/+qjWq0WtVqt6Bkt+/s51Gq123pH6Fpz6yMPms1mR3yPIiK/L41GoyPO6f+fQyf83K13peYyv4ud+i7B3t7ejni5W71ez3eAdspb/KvValSr1ejq6uqYjz5eWFiIWq0W3d3dq/4Joe0yOzsbETdfOtoJ14lmsxn1er3oGatiOZf7dR8FgPViOZf7df/00YcffhgPPfRQ0TNaduHChTh8+HD09/fHF1980RF3P6dOnYp33303nnzyyThx4kTRc1bEyy+/HN9991288sorcfjw4aLntKxWq8Vzzz0X8/Pz8cEHH8S+ffuKntSyixcvxqFDh4qeUZh1H4V9+/bFyMhI0TNadut2t1KpxP79+zsiCmfOnImIm0+H7d+/v+A1K2PLli0REXHPPfd0xDnVarX8Wdu3b19HnNN6f1bEq48ASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAgiQIAqavoAUX7+eefo16vFz2jZRcvXoyIiHq9HufOnYtKpVLwotZNTExERMTs7GycO3eu4DUrY2ZmJiJunlsnnFOtVsvfn4sXL0apVCp4Uetu/S6tV6Vms9lc1gM74JsNsJ4t53Lv6SMAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACCJAgBJFABIogBAEgUAkigAkEQBgCQKACRRACB1LfeBzWZzNXcAsAa4UwAgiQIASRQASKIAQBIFAJIoAJBEAYAkCgAkUQAg/QvHQt/6iylOBwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data['next','pixels'][-1].numpy())\n",
    "plt.axis('off')\n",
    "plt.show()\n",
    "\n",
    "# convert a numpy to a gray scale image\n",
    "# with PIL\n",
    "from PIL import Image\n",
    "import numpy\n",
    "\n",
    "# Create a numpy array\n",
    "\n",
    "# Create a numpy array\n",
    "numpy_array = data['next','pixels'][-1].numpy()\n",
    "\n",
    "# Create a PIL Image from the numpy array\n",
    "image = Image.fromarray(numpy_array)\n",
    "\n",
    "# Convert the PIL image to a gray scale image\n",
    "gray_image = image.convert('L')\n",
    "\n",
    "# Display the gray scale image\n",
    "plt.imshow(gray_image, cmap='gray')\n",
    "plt.axis('off')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        agent: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                agent: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 512, 512, 3]), device=cpu, dtype=torch.uint8, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                target: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 512, 512, 3]), device=cpu, dtype=torch.uint8, is_shared=False),\n",
       "        target: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'key \"observation\" not found in TensorDict with keys [\\'action\\', \\'agent\\', \\'done\\', \\'next\\', \\'pixels\\', \\'target\\', \\'terminated\\', \\'truncated\\']'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdata\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mobservation\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/tensordict/base.py:323\u001b[0m, in \u001b[0;36mTensorDictBase.__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    321\u001b[0m idx_unravel \u001b[38;5;241m=\u001b[39m _unravel_key_to_tuple(index)\n\u001b[1;32m    322\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m idx_unravel:\n\u001b[0;32m--> 323\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_tuple\u001b[49m\u001b[43m(\u001b[49m\u001b[43midx_unravel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mNO_DEFAULT\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    324\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m is_non_tensor(result):\n\u001b[1;32m    325\u001b[0m         result_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(result, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata\u001b[39m\u001b[38;5;124m\"\u001b[39m, NO_DEFAULT)\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/tensordict/_td.py:2078\u001b[0m, in \u001b[0;36mTensorDict._get_tuple\u001b[0;34m(self, key, default)\u001b[0m\n\u001b[1;32m   2077\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_tuple\u001b[39m(\u001b[38;5;28mself\u001b[39m, key, default):\n\u001b[0;32m-> 2078\u001b[0m     first \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_str\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdefault\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2079\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(key) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m first \u001b[38;5;129;01mis\u001b[39;00m default:\n\u001b[1;32m   2080\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m first\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/tensordict/_td.py:2074\u001b[0m, in \u001b[0;36mTensorDict._get_str\u001b[0;34m(self, key, default)\u001b[0m\n\u001b[1;32m   2072\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tensordict\u001b[38;5;241m.\u001b[39mget(first_key, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m   2073\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m out \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 2074\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_default_get\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfirst_key\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdefault\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2075\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/tensordict/base.py:2915\u001b[0m, in \u001b[0;36mTensorDictBase._default_get\u001b[0;34m(self, key, default)\u001b[0m\n\u001b[1;32m   2912\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m default\n\u001b[1;32m   2913\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   2914\u001b[0m     \u001b[38;5;66;03m# raise KeyError\u001b[39;00m\n\u001b[0;32m-> 2915\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\n\u001b[1;32m   2916\u001b[0m         _KEY_ERROR\u001b[38;5;241m.\u001b[39mformat(key, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, \u001b[38;5;28msorted\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeys()))\n\u001b[1;32m   2917\u001b[0m     )\n",
      "\u001b[0;31mKeyError\u001b[0m: 'key \"observation\" not found in TensorDict with keys [\\'action\\', \\'agent\\', \\'done\\', \\'next\\', \\'pixels\\', \\'target\\', \\'terminated\\', \\'truncated\\']'"
     ]
    }
   ],
   "source": [
    "data[\"observation\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.29.1\n"
     ]
    }
   ],
   "source": [
    "import gymnasium\n",
    "print(gym.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorDict(\n",
      "    fields={\n",
      "        action: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        collector: TensorDict(\n",
      "            fields={\n",
      "                traj_ids: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
      "            batch_size=torch.Size([4]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        next: TensorDict(\n",
      "            fields={\n",
      "                done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "            batch_size=torch.Size([4]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "    batch_size=torch.Size([4]),\n",
      "    device=cpu,\n",
      "    is_shared=False)\n"
     ]
    }
   ],
   "source": [
    "from torchrl.envs.utils import RandomPolicy\n",
    "    \n",
    "policy = RandomPolicy(env.action_spec)\n",
    "\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn=env,\n",
    "    policy=policy,\n",
    "    frames_per_batch=4,\n",
    "    total_frames=16000,\n",
    "    device=\"cpu\",\n",
    "    storing_device=\"cpu\",\n",
    "    max_frames_per_traj=-1,\n",
    "    init_random_frames=1000,\n",
    ")\n",
    "\n",
    "for data in collector:\n",
    "    print(data)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([4]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([4]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([4]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNNetwork(torch.nn.Module):\n",
    "    \"\"\"The convolutional network used to compute the agent's Q-values.\"\"\"\n",
    "    def __init__(self, \n",
    "                 input_shape,\n",
    "                 num_outputs,\n",
    "                 num_cells_cnn, \n",
    "                 kernel_sizes, \n",
    "                 strides, \n",
    "                 num_cells_mlp,\n",
    "                 activation_class,\n",
    "                 use_batch_norm=False):\n",
    "        super(DQNNetwork, self).__init__()\n",
    "\n",
    "        self.activation_class = activation_class()\n",
    "        self.use_batch_norm = use_batch_norm\n",
    "      \n",
    "        # Input shape example: (10, 4, 84, 84)\n",
    "        _, channels, width, height = input_shape\n",
    "        self.num_outputs = num_outputs\n",
    "\n",
    "        # Xavier (Glorot) uniform initialization\n",
    "        self.initializer = torch.nn.init.xavier_uniform_\n",
    "\n",
    "        # Convolutional layers\n",
    "        self.conv_layers = torch.nn.ModuleList()\n",
    "        self.batch_norm_layers = torch.nn.ModuleList()\n",
    "        in_channels = channels\n",
    "        for out_channels, kernel_size, stride in zip(num_cells_cnn, kernel_sizes, strides):\n",
    "            conv_layer = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride)\n",
    "            self.conv_layers.append(conv_layer)\n",
    "            if self.use_batch_norm:\n",
    "                batch_norm_layer = torch.nn.BatchNorm2d(out_channels)\n",
    "                self.batch_norm_layers.append(batch_norm_layer)\n",
    "            in_channels = out_channels\n",
    "\n",
    "        # Number of Linear input connections depends on output of conv2d layers\n",
    "        # and therefore the input image size, so compute it.\n",
    "        def conv2d_size_out(size, kernel_size, stride):\n",
    "            return (size - kernel_size) // stride  + 1\n",
    "        \n",
    "        # Compute the output shape of the conv layers\n",
    "        width_output = width\n",
    "        height_output = height\n",
    "        for kernel_size, stride in zip(kernel_sizes, strides):\n",
    "            width_output = conv2d_size_out(width_output, kernel_size, stride)\n",
    "            height_output = conv2d_size_out(height_output, kernel_size, stride)\n",
    "\n",
    "        cnn_output = width_output * height_output * num_cells_cnn[-1]\n",
    "\n",
    "        # Fully connected layers\n",
    "        input_size = cnn_output\n",
    "\n",
    "        if len(num_cells_mlp) != 0:\n",
    "            self.fc_layers = torch.nn.ModuleList()\n",
    "            for units in num_cells_mlp:\n",
    "                fc_layer = torch.nn.Linear(input_size, units)\n",
    "                self.fc_layers.append(fc_layer)\n",
    "                input_size = units\n",
    "        else:\n",
    "            self.fc_layers = None\n",
    "        \n",
    "        # Final output layer\n",
    "        self.output_layer = torch.nn.Linear(input_size, self.num_outputs)\n",
    "\n",
    "        self._initialize_weights()\n",
    "\n",
    "    def _initialize_weights(self):\n",
    "        for layer in self.conv_layers:\n",
    "            self.initializer(layer.weight)\n",
    "            if layer.bias is not None:\n",
    "                torch.nn.init.zeros_(layer.bias)\n",
    "        if self.fc_layers is not None:\n",
    "            for layer in self.fc_layers:\n",
    "                self.initializer(layer.weight)\n",
    "                if layer.bias is not None:\n",
    "                    torch.nn.init.zeros_(layer.bias)\n",
    "        self.initializer(self.output_layer.weight)\n",
    "        if self.output_layer.bias is not None:\n",
    "            torch.nn.init.zeros_(self.output_layer.bias)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x = x.float() / 255.0 # Already normalized by VecNorm\n",
    "        for i, conv_layer in enumerate(self.conv_layers):\n",
    "            x = conv_layer(x)\n",
    "            if self.use_batch_norm:\n",
    "                x = self.batch_norm_layers[i](x)\n",
    "            x = self.activation_class(x)\n",
    "        x = x.view(x.size(0), -1)  # Flatten the tensor\n",
    "\n",
    "        if self.fc_layers is not None:\n",
    "            for fc_layer in self.fc_layers:\n",
    "                x = self.activation_class(fc_layer(x))\n",
    "        q_values = self.output_layer(x)\n",
    "        return q_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MICODQNNetwork(torch.nn.Module):\n",
    "    \"\"\"The convolutional network used to compute the agent's Q-values.\"\"\"\n",
    "    def __init__(self, \n",
    "                 input_shape,\n",
    "                 num_outputs,\n",
    "                 num_cells_cnn, \n",
    "                 kernel_sizes, \n",
    "                 strides, \n",
    "                 num_cells_mlp,\n",
    "                 activation_class,\n",
    "                 use_batch_norm=False):\n",
    "        super(MICODQNNetwork, self).__init__()\n",
    "\n",
    "        self.activation_class = activation_class()\n",
    "        self.use_batch_norm = use_batch_norm\n",
    "      \n",
    "        # Input shape example: (4, 84, 84)\n",
    "        channels, width, height = input_shape\n",
    "        self.num_outputs = num_outputs\n",
    "\n",
    "        # Xavier (Glorot) uniform initialization\n",
    "        self.initializer = torch.nn.init.xavier_uniform_\n",
    "\n",
    "        # Convolutional layers\n",
    "        self.conv_layers = torch.nn.ModuleList()\n",
    "        self.batch_norm_layers = torch.nn.ModuleList()\n",
    "        in_channels = channels\n",
    "        for out_channels, kernel_size, stride in zip(num_cells_cnn, kernel_sizes, strides):\n",
    "            conv_layer = torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride)\n",
    "            self.conv_layers.append(conv_layer)\n",
    "            if self.use_batch_norm:\n",
    "                batch_norm_layer = torch.nn.BatchNorm2d(out_channels)\n",
    "                self.batch_norm_layers.append(batch_norm_layer)\n",
    "            in_channels = out_channels\n",
    "\n",
    "        # Number of Linear input connections depends on output of conv2d layers\n",
    "        # and therefore the input image size, so compute it.\n",
    "        def conv2d_size_out(size, kernel_size, stride):\n",
    "            return (size - kernel_size) // stride  + 1\n",
    "        \n",
    "        # Compute the output shape of the conv layers\n",
    "        width_output = width\n",
    "        height_output = height\n",
    "        for kernel_size, stride in zip(kernel_sizes, strides):\n",
    "            width_output = conv2d_size_out(width_output, kernel_size, stride)\n",
    "            height_output = conv2d_size_out(height_output, kernel_size, stride)\n",
    "\n",
    "        cnn_output = width_output * height_output * num_cells_cnn[-1]\n",
    "\n",
    "        # Fully connected layers\n",
    "        input_size = cnn_output\n",
    "\n",
    "        if len(num_cells_mlp) != 0:\n",
    "            self.fc_layers = torch.nn.ModuleList()\n",
    "            for units in num_cells_mlp:\n",
    "                fc_layer = torch.nn.Linear(input_size, units)\n",
    "                self.fc_layers.append(fc_layer)\n",
    "                input_size = units\n",
    "        else:\n",
    "            self.fc_layers = None\n",
    "        \n",
    "        # Final output layer\n",
    "        self.output_layer = torch.nn.Linear(input_size, self.num_outputs)\n",
    "\n",
    "        self._initialize_weights()\n",
    "\n",
    "    def _initialize_weights(self):\n",
    "        for layer in self.conv_layers:\n",
    "            self.initializer(layer.weight)\n",
    "            if layer.bias is not None:\n",
    "                torch.nn.init.zeros_(layer.bias)\n",
    "        if self.fc_layers is not None:\n",
    "            for layer in self.fc_layers:\n",
    "                self.initializer(layer.weight)\n",
    "                if layer.bias is not None:\n",
    "                    torch.nn.init.zeros_(layer.bias)\n",
    "        self.initializer(self.output_layer.weight)\n",
    "        if self.output_layer.bias is not None:\n",
    "            torch.nn.init.zeros_(self.output_layer.bias)\n",
    "\n",
    "    def forward(self, input):\n",
    "\n",
    "        x = input\n",
    "        for i, conv_layer in enumerate(self.conv_layers):\n",
    "            x = conv_layer(x)\n",
    "\n",
    "            # NOTE: The collector uses a tensor for checking something\n",
    "            # but this tensor is not in batch format, so we need to\n",
    "            # check if the tensor is in batch format to apply batch norm\n",
    "            if self.use_batch_norm and len(input.shape) == 4:\n",
    "                x = self.batch_norm_layers[i](x)\n",
    "            x = self.activation_class(x)\n",
    "\n",
    "        if len(input.shape) == 4:\n",
    "            x = x.view(x.size(0), -1)  # Flatten the tensor\n",
    "        else:\n",
    "            x = x.view(-1)\n",
    "        \n",
    "        representation = x\n",
    "\n",
    "        if self.fc_layers is not None:\n",
    "            for fc_layer in self.fc_layers:\n",
    "                x = self.activation_class(fc_layer(x))\n",
    "        q_values = self.output_layer(x)\n",
    "        return q_values, representation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the network with a random input\n",
    "tensor = torch.rand(10, 4, 84, 84)\n",
    "num_actions = 2\n",
    "num_cells_cnn = [32, 64, 64]\n",
    "kernel_sizes = [8, 4, 3]\n",
    "strides = [4, 2, 1]\n",
    "num_cells_mlp = [512]\n",
    "activation_class = torch.nn.ReLU\n",
    "\n",
    "network = MICODQNNetwork(\n",
    "    input_shape=tensor.shape[-3:],\n",
    "    num_outputs=num_actions,\n",
    "    num_cells_cnn=num_cells_cnn,\n",
    "    kernel_sizes=kernel_sizes,\n",
    "    strides=strides,\n",
    "    num_cells_mlp=num_cells_mlp,\n",
    "    activation_class=activation_class,\n",
    "    use_batch_norm=True\n",
    ")\n",
    "q_values, representation = network(tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MICODQNNetwork(\n",
       "  (activation_class): ReLU()\n",
       "  (conv_layers): ModuleList(\n",
       "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "    (1): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "  )\n",
       "  (batch_norm_layers): ModuleList(\n",
       "    (0): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1-2): 2 x BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (fc_layers): ModuleList(\n",
       "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
       "  )\n",
       "  (output_layer): Linear(in_features=512, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dqn_modules(proof_environment, policy_cfg):\n",
    "\n",
    "    # Define input shape\n",
    "    input_shape = proof_environment.observation_spec[\"pixels\"].shape\n",
    "    env_specs = proof_environment.specs\n",
    "\n",
    "    # NOTE: I think I can change the next two lines by\n",
    "    # num_outputs = proof_environment.action_spec.shape[-1]\n",
    "    # action_spec = proof_environment.action_spec.space\n",
    "    num_outputs = env_specs[\"input_spec\", \"full_action_spec\", \"action\"].space.n\n",
    "    action_spec = env_specs[\"input_spec\", \"full_action_spec\", \"action\"]\n",
    "\n",
    "    # Define Q-Value Module and Representations (for MICO)\n",
    "\n",
    "    activation_class = torch.nn.ReLU\n",
    "\n",
    "    module = MICODQNNetwork(\n",
    "        input_shape=input_shape,\n",
    "        num_outputs=num_outputs,\n",
    "        num_cells_cnn=policy_cfg.cnn_net.num_cells,\n",
    "        kernel_sizes=policy_cfg.cnn_net.kernel_sizes,\n",
    "        strides=policy_cfg.cnn_net.strides,\n",
    "        num_cells_mlp=policy_cfg.mlp_net.num_cells,\n",
    "        activation_class=activation_class,\n",
    "        use_batch_norm=policy_cfg.use_batch_norm\n",
    "    )\n",
    "\n",
    "    module = TensorDictModule(module,\n",
    "            in_keys=[\"pixels\"], \n",
    "            out_keys=[\"action_value\", \"representation\"])\n",
    "\n",
    "    # NOTE: Do I need CompositeSpec here?\n",
    "    # I think I only need proof_environment.action_spec\n",
    "    qvalue_module = QValueActor(\n",
    "        module=module,\n",
    "        spec=CompositeSpec(action=action_spec),\n",
    "        in_keys=[\"pixels\"],\n",
    "    )\n",
    "\n",
    "    return qvalue_module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'device': None, 'exp_name': 'DQN_pixels', 'env': {'env_name': 'CartPole-v1', 'seed': 118398}, 'collector': {'total_frames': 5000016, 'frames_per_batch': 16, 'eps_start': 1.0, 'eps_end': 0.05, 'annealing_frames': 200000, 'init_random_frames': 50000, 'frame_skip': 2}, 'policy': {'type': 'CNN_MLP', 'cnn_net': {'num_cells': [32, 64, 64], 'kernel_sizes': [8, 4, 3], 'strides': [4, 2, 1]}, 'mlp_net': {'num_cells': [512]}, 'activation': 'ReLU', 'use_batch_norm': False}, 'buffer': {'buffer_size': 50000, 'batch_size': 128, 'scratch_dir': None, 'prioritized_replay': False, 'alpha': 0.7, 'beta': 0.4}, 'logger': {'backend': 'wandb', 'project_name': 'dqn_pixels_cartpole', 'group_name': None, 'test_interval': 50000, 'num_test_episodes': 3, 'video': False}, 'optim': {'lr': 0.001, 'max_grad_norm': 10}, 'loss': {'gamma': 0.99, 'hard_update_freq': 50, 'num_updates': 1}}"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dqn_mico_er/config_cartpole.yaml\n",
    "import yaml\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "with open(\"dqn_pixels/config_cartpole.yaml\") as f:\n",
    "    cfg = OmegaConf.create(yaml.safe_load(f))\n",
    "\n",
    "cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorDict(\n",
      "    fields={\n",
      "        action: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        action_value: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        chosen_action_value: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        collector: TensorDict(\n",
      "            fields={\n",
      "                traj_ids: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
      "            batch_size=torch.Size([4]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        next: TensorDict(\n",
      "            fields={\n",
      "                done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "            batch_size=torch.Size([4]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        representation: Tensor(shape=torch.Size([4, 3136]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "    batch_size=torch.Size([4]),\n",
      "    device=cpu,\n",
      "    is_shared=False)\n"
     ]
    }
   ],
   "source": [
    "from torchrl.envs.utils import RandomPolicy\n",
    "from tensordict.nn import TensorDictModule, TensorDictSequential\n",
    "\n",
    "module = make_dqn_modules(env, cfg.policy)\n",
    "\n",
    "greedy_module = EGreedyModule(\n",
    "    annealing_num_steps=1000,\n",
    "    eps_init=0.9,\n",
    "    eps_end=0.1,\n",
    "    spec=module.spec,\n",
    ")\n",
    "model_explore = TensorDictSequential(\n",
    "    module,\n",
    "    greedy_module,\n",
    ")\n",
    "\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn=env,\n",
    "    policy=model_explore,\n",
    "    frames_per_batch=4,\n",
    "    total_frames=16000,\n",
    "    device=\"cpu\",\n",
    "    storing_device=\"cpu\",\n",
    "    max_frames_per_traj=-1,\n",
    ")\n",
    "\n",
    "for data in collector:\n",
    "    print(data)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([4, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([4]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([4]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([4, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([4, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([4, 3136]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([4, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([4]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[False],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False]])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','done']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','pixels'][2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]],\n",
       "\n",
       "        [[1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05],\n",
       "         [1.7881e-05, 1.7881e-05, 1.7881e-05,  ..., 1.7881e-05,\n",
       "          1.1921e-05, 1.7881e-05],\n",
       "         ...,\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [5.9605e-06, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.1921e-05, 1.1921e-05],\n",
       "         [1.1921e-05, 1.1921e-05, 1.1921e-05,  ..., 1.1921e-05,\n",
       "          1.7881e-05, 1.1921e-05]]])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','pixels'][2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.0000, 0.0000,  ..., 0.0704, 0.0711, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000,  ..., 0.0824, 0.0610, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0000,  ..., 0.0814, 0.0059, 0.0000],\n",
       "        [0.0000, 0.0000, 0.0010,  ..., 0.0142, 0.0028, 0.0000]])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10, 2]), torch.Size([10, 3136]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_values.shape, representation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the network with a random input\n",
    "tensor = torch.rand(10, 4, 84, 84)\n",
    "num_actions = 2\n",
    "num_cells_cnn = [64, 64, 32]\n",
    "kernel_sizes = [5, 5, 5]\n",
    "strides = [2, 2, 2]\n",
    "num_cells_mlp = []\n",
    "activation_class = torch.nn.ReLU\n",
    "\n",
    "network = DQNNetwork(\n",
    "    input_shape=tensor.shape,\n",
    "    num_outputs=num_actions,\n",
    "    num_cells_cnn=num_cells_cnn,\n",
    "    kernel_sizes=kernel_sizes,\n",
    "    strides=strides,\n",
    "    num_cells_mlp=num_cells_mlp,\n",
    "    activation_class=activation_class,\n",
    "    use_batch_norm=True\n",
    ")\n",
    "q_values = network(tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DQNNetwork(\n",
       "  (activation_class): ReLU()\n",
       "  (conv_layers): ModuleList(\n",
       "    (0): Conv2d(4, 64, kernel_size=(5, 5), stride=(2, 2))\n",
       "    (1): Conv2d(64, 64, kernel_size=(5, 5), stride=(2, 2))\n",
       "    (2): Conv2d(64, 32, kernel_size=(5, 5), stride=(2, 2))\n",
       "  )\n",
       "  (batch_norm_layers): ModuleList(\n",
       "    (0-1): 2 x BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  )\n",
       "  (output_layer): Linear(in_features=1568, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10, 2]), torch.Size([10, 3136]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_values.shape, representation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torch/nn/modules/lazy.py:181: UserWarning: Lazy modules are a new feature under heavy development so changes to the API or functionality can happen at any moment.\n",
      "  warnings.warn('Lazy modules are a new feature under heavy development '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ConvNet(\n",
       "  (0): LazyConv2d(0, 32, kernel_size=(8, 8), stride=(4, 4))\n",
       "  (1): ReLU()\n",
       "  (2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
       "  (3): ReLU()\n",
       "  (4): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "  (5): ReLU()\n",
       "  (6): SquashDims()\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchrl.modules import ConvNet\n",
    "\n",
    "cnn = ConvNet(\n",
    "    activation_class=torch.nn.ReLU,\n",
    "    num_cells=[32, 64, 64],\n",
    "    kernel_sizes=[8, 4, 3],\n",
    "    strides=[4, 2, 1])\n",
    "cnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3136])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the network with a toy input\n",
    "x = torch.randn(1, 4, 84, 84)\n",
    "y = cnn(x)\n",
    "y.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28224"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "84 * 84 * 4\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1 2 3 4 5 6 7 8 9 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([3, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([3, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([3, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([3]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([3, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([3, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([3, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([3]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# NOTE: a rollout will be take a trajectory of frame of 10 steps and will concat them by N = 4\n",
    "# so the output will be 7x4x84x84\n",
    "env.rollout(max_steps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:2989: DeprecationWarning: Your wrapper was not given a device. Currently, this value will default to 'cpu'. From v0.5 it will default to `None`. With a device of None, no device casting is performed and the resulting tensordicts are deviceless. Please set your device accordingly.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TransformedEnv(\n",
       "    env=GymEnv(env=CartPole-v1, batch_size=torch.Size([]), device=cpu),\n",
       "    transform=StepCounter(keys=[]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = TransformedEnv(GymEnv(\"CartPole-v1\"), StepCounter())\n",
    "env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rollout = env.rollout(max_steps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # considering rollout['pixels'].shape equal torch.Size([10, 400, 600, 3]), plot the 10 images\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "# fig, axs = plt.subplots(1, 10, figsize=(20, 2))\n",
    "# for i, ax in enumerate(axs):\n",
    "#     ax.imshow(rollout['pixels'][i])\n",
    "#     ax.axis('off')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MICOMLPNetwork(torch.nn.Module):\n",
    "    def __init__(self,\n",
    "                 in_features,\n",
    "                 activation_class, \n",
    "                 encoder_out_features,\n",
    "                 mlp_out_features,\n",
    "                 encoder_num_cells = None,\n",
    "                 mlp_num_cells = None):\n",
    "        super(MICOMLPNetwork, self).__init__()\n",
    "\n",
    "        self.activation = activation_class()\n",
    "\n",
    "        if encoder_num_cells is None:\n",
    "            encoder_num_cells = []\n",
    "        layers_sizes = [in_features] + encoder_num_cells + [encoder_out_features]\n",
    "\n",
    "        self.encoder = torch.nn.ModuleList()\n",
    "        for i in range(len(layers_sizes) - 1):\n",
    "            self.encoder.append(torch.nn.Linear(layers_sizes[i], layers_sizes[i+1]))\n",
    "\n",
    "        if mlp_num_cells is None:\n",
    "            mlp_num_cells = []\n",
    "\n",
    "        layers_sizes = [encoder_out_features] + mlp_num_cells + [mlp_out_features.item()]\n",
    "\n",
    "        self.q_net = torch.nn.ModuleList()\n",
    "        for i in range(len(layers_sizes) - 1):\n",
    "            self.q_net.append(torch.nn.Linear(layers_sizes[i], layers_sizes[i+1]))\n",
    "        \n",
    "    \n",
    "    def forward(self, x):\n",
    "        for i in range(len(self.encoder)):\n",
    "            x = self.activation(self.encoder[i](x))\n",
    "\n",
    "        representation = x\n",
    "\n",
    "        for i in range(len(self.q_net)-1):\n",
    "            x = self.activation(self.q_net[i](x))\n",
    "\n",
    "        return self.q_net[-1](x), representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MICOMLPNetwork(\n",
       "  (activation): ReLU()\n",
       "  (encoder): ModuleList(\n",
       "    (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "    (1): Linear(in_features=64, out_features=3, bias=True)\n",
       "  )\n",
       "  (q_net): ModuleList(\n",
       "    (0): Linear(in_features=3, out_features=64, bias=True)\n",
       "    (1): Linear(in_features=64, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_mlp = MICOMLPNetwork(\n",
    "    in_features=4,\n",
    "    activation_class=torch.nn.ReLU,\n",
    "    encoder_out_features=3,\n",
    "    mlp_out_features=env.action_spec.shape[-1],\n",
    "    encoder_num_cells=[64],\n",
    "    mlp_num_cells=[64]\n",
    ")\n",
    "\n",
    "value_mlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-2.4385e-01,  1.6361e+01,  1.6819e-01,  1.6592e+01],\n",
      "        [-2.4529e+00,  6.2587e+00,  2.1316e-01, -1.7105e+01],\n",
      "        [ 1.0673e+00,  6.0321e-01,  4.1905e-02, -6.5497e+00],\n",
      "        [-2.2201e+00, -5.3833e+00, -5.6525e-02, -2.0494e+01],\n",
      "        [ 2.4468e+00,  1.3530e-01,  8.5826e-02, -7.4412e+00],\n",
      "        [-1.0956e+00, -1.7736e+01, -2.9005e-01,  5.1246e+00],\n",
      "        [-8.3562e-02,  5.9217e+00, -3.8831e-01, -1.0923e+01],\n",
      "        [-9.6649e-02, -1.7004e+01,  1.8255e-01,  5.0339e+00],\n",
      "        [-1.9215e+00, -6.0012e+00, -3.3985e-01, -1.9400e+00],\n",
      "        [-7.9192e-01, -8.4512e+00, -1.1339e-01, -2.4514e+00],\n",
      "        [-3.8455e+00, -4.1582e+00, -6.5327e-02, -6.5401e+00],\n",
      "        [ 2.5188e+00, -2.6075e-01, -4.0146e-01, -4.9766e+00],\n",
      "        [ 3.4100e+00, -5.7687e+00, -1.3475e-01, -4.9081e+00],\n",
      "        [-3.0499e+00,  1.7682e+01,  2.6956e-01,  6.2869e+00],\n",
      "        [ 3.5047e+00, -1.3430e+01,  3.0075e-01, -9.5426e+00],\n",
      "        [ 3.7697e+00,  1.1834e+01,  1.7286e-01,  1.1398e+01],\n",
      "        [ 4.7634e+00, -1.6998e+01, -2.9006e-01,  9.7743e+00],\n",
      "        [ 1.2804e+00, -1.6293e+01, -1.0586e-01,  1.2830e+01],\n",
      "        [-1.3542e+00,  1.2150e+00,  3.0854e-01, -6.3442e-01],\n",
      "        [ 1.3793e+00, -1.2606e+00, -2.7460e-01, -7.9568e+00],\n",
      "        [-3.4166e+00,  4.0322e+00,  3.5936e-01,  7.5585e-01],\n",
      "        [ 1.6275e+00, -1.2598e+01, -2.7653e-01,  8.0609e+00],\n",
      "        [-2.8654e+00, -1.2266e+01,  3.2490e-01,  3.5293e+00],\n",
      "        [ 2.2782e+00, -1.9660e+01,  1.3434e-01,  6.5428e+00],\n",
      "        [-3.0847e+00, -5.8907e+00,  9.9763e-02, -1.0073e+01],\n",
      "        [-4.4755e+00, -6.6018e+00,  2.9518e-01, -7.1580e+00],\n",
      "        [-7.7387e-01, -5.6805e+00, -3.8991e-01,  2.2664e+01],\n",
      "        [ 4.1145e+00,  3.5917e+00,  1.1061e-01,  3.8655e+00],\n",
      "        [ 3.6731e+00, -3.3140e+00, -2.8392e-01, -7.2439e+00],\n",
      "        [ 3.3574e+00,  1.0757e+01,  1.1015e-01,  1.0531e+01],\n",
      "        [ 9.3583e-01,  3.1470e+00, -9.9956e-02, -3.7179e+00],\n",
      "        [ 2.6877e+00,  4.0512e+00,  1.3872e-01, -1.1658e+00],\n",
      "        [-3.6225e+00, -1.5832e+01,  1.2115e-01,  9.4262e-01],\n",
      "        [-3.4231e+00, -5.0136e-01, -1.6577e-01,  4.5416e+00],\n",
      "        [ 3.6540e-01,  2.4872e+01,  3.1235e-01, -7.4388e+00],\n",
      "        [ 4.1537e+00, -5.6400e+00,  2.7341e-01,  1.0245e+01],\n",
      "        [ 2.9621e+00,  4.7071e+00, -2.7539e-01, -3.1666e+00],\n",
      "        [ 3.9641e-01, -1.6041e+01, -2.6614e-01,  8.3124e+00],\n",
      "        [-2.6162e+00, -1.0861e+01, -4.0242e-01, -1.0668e+00],\n",
      "        [-3.5202e-01, -1.6549e+01,  2.0079e-01,  3.8322e-01],\n",
      "        [ 6.5740e-01, -9.0891e+00,  3.0693e-01,  1.8954e+01],\n",
      "        [ 1.9418e+00, -7.5173e+00, -2.6882e-01,  4.5016e+00],\n",
      "        [-2.6346e+00, -1.1080e+01, -1.6767e-01, -5.9871e-01],\n",
      "        [-4.6178e+00, -4.3420e+00,  2.9479e-01,  1.5252e+00],\n",
      "        [ 2.5610e+00, -1.9293e+00, -1.5271e-01,  6.9512e+00],\n",
      "        [ 3.6868e+00,  1.1232e+01, -4.0983e-01, -1.1306e+01],\n",
      "        [-1.9918e+00,  1.6648e+01, -2.2109e-01,  6.8823e-02],\n",
      "        [-2.2255e+00, -2.6932e+00,  1.3542e-01,  1.6596e+01],\n",
      "        [ 2.1815e+00,  2.1407e+01, -2.7794e-01, -2.0183e+01],\n",
      "        [ 1.2551e+00,  9.4587e+00, -5.1251e-02,  9.4529e+00],\n",
      "        [-3.3348e+00,  8.8511e+00, -3.1012e-01, -5.5034e+00],\n",
      "        [ 1.5573e+00,  6.4087e-01,  2.1983e-01,  7.8851e-01],\n",
      "        [ 1.6549e+00,  4.2646e+00, -2.0338e-01,  2.1678e+00],\n",
      "        [ 1.4192e+00, -3.4515e+00, -5.2174e-02, -7.3042e-02],\n",
      "        [ 3.0520e+00,  1.3845e-01,  3.0907e-01,  9.0296e+00],\n",
      "        [-3.1411e+00, -5.8337e+00,  1.2755e-01,  7.2418e+00],\n",
      "        [-2.3204e+00,  9.7580e+00,  2.0980e-03, -8.5886e-01],\n",
      "        [ 1.8602e+00,  1.3757e+01,  9.6058e-02, -1.1331e+00],\n",
      "        [ 4.4212e+00,  4.5136e+00,  2.0828e-01,  4.6422e+00],\n",
      "        [ 4.1317e+00,  9.3499e+00,  3.6456e-01, -1.4507e+01],\n",
      "        [ 2.1573e+00,  2.4650e+00, -2.9339e-01, -1.0865e+01],\n",
      "        [-3.8495e+00, -1.3594e+01,  1.2703e-02,  1.2632e+01],\n",
      "        [-6.8884e-01,  2.1425e+00,  1.0611e-01, -4.7899e-01],\n",
      "        [-3.2792e+00, -8.4699e+00,  2.5777e-01, -2.2777e+01],\n",
      "        [-3.2588e-02,  1.3808e+00, -4.0830e-02,  2.8451e+00],\n",
      "        [-5.5408e-01,  7.8820e+00, -2.5214e-01, -5.9751e+00],\n",
      "        [ 4.3313e+00,  1.9255e+01, -1.0267e-01,  4.5766e+00],\n",
      "        [-2.8965e+00, -1.7932e+01,  1.1841e-01, -3.2316e-01],\n",
      "        [ 2.2868e+00,  1.1169e+01,  8.2241e-02,  2.0722e+01],\n",
      "        [-3.8481e+00,  6.2149e-01, -2.4441e-01,  1.4349e+01],\n",
      "        [ 3.9882e+00, -2.4354e+01,  2.7695e-01, -4.6339e-02],\n",
      "        [-1.6930e+00, -5.0639e+00, -1.8364e-02, -5.8502e+00],\n",
      "        [ 3.6190e+00,  1.2486e+01,  2.2415e-01, -1.0536e+01],\n",
      "        [-2.6681e+00, -2.9938e+00, -1.7717e-01, -3.6205e-01],\n",
      "        [ 1.2115e+00, -8.4943e+00,  4.0222e-01, -5.2072e+00],\n",
      "        [-3.7875e+00, -6.7369e+00, -3.9171e-01,  4.1235e+00],\n",
      "        [-4.2319e+00,  7.2685e+00,  1.9693e-01, -5.7202e+00],\n",
      "        [-4.2699e+00,  1.5016e+01, -5.4718e-02,  4.1169e+00],\n",
      "        [ 1.6500e+00, -1.3882e+01,  1.9879e-01, -9.4679e+00],\n",
      "        [-4.7901e+00, -1.9096e-01, -1.1157e-01, -8.3775e+00],\n",
      "        [-2.5481e+00, -1.3673e+01, -7.5641e-02,  3.8938e+00],\n",
      "        [-4.2958e+00, -1.1290e+01, -8.5735e-02,  5.4854e+00],\n",
      "        [-4.6203e+00, -2.3623e+01, -2.8453e-01,  4.9888e+00],\n",
      "        [ 1.5685e+00,  1.0291e+01, -3.7911e-01, -1.9327e+01],\n",
      "        [-2.0667e+00, -6.1631e+00, -1.7566e-01,  6.6134e+00],\n",
      "        [-4.5113e+00,  1.1305e+01,  7.1680e-02, -1.4278e+01],\n",
      "        [-2.0119e+00,  1.3298e+01, -5.8542e-02, -2.6829e+00],\n",
      "        [-4.0401e-01,  1.4928e+01,  1.6699e-01,  4.1037e+00],\n",
      "        [ 2.1938e+00, -1.9031e+00, -4.9020e-02,  5.3712e+00],\n",
      "        [-4.2902e+00, -3.9736e+00,  1.4393e-02, -2.5382e+00],\n",
      "        [-3.6693e+00,  8.4035e+00, -2.2838e-01,  2.7123e+00],\n",
      "        [ 7.9573e-01, -1.5453e+01,  3.8860e-01, -2.3633e+00],\n",
      "        [-1.8259e+00,  2.4629e+00, -1.8037e-02, -6.0044e+00],\n",
      "        [ 2.9725e+00,  9.7075e+00, -1.9743e-01, -3.5357e+00],\n",
      "        [-4.5811e-01,  5.0699e+00,  3.8326e-01, -4.4813e+00],\n",
      "        [ 3.7240e+00, -2.7199e+00,  1.9372e-01, -1.5675e+01],\n",
      "        [-3.8979e+00, -1.8111e+01, -2.1873e-01, -1.5584e+01],\n",
      "        [ 1.1599e+00,  1.8423e+01, -5.4413e-02, -1.0687e+01],\n",
      "        [-4.0736e+00,  1.2573e+01, -2.3828e-01, -2.8207e-01],\n",
      "        [-4.9072e-01,  3.8657e+00,  2.7469e-01, -2.0025e-01]])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Specifications\n",
    "num_observations = 4\n",
    "cart_position_min = -4.8\n",
    "cart_position_max = 4.8\n",
    "cart_velocity_min = -np.inf\n",
    "cart_velocity_max = np.inf\n",
    "pole_angle_min = -0.418\n",
    "pole_angle_max = 0.418\n",
    "pole_angular_velocity_min = -np.inf\n",
    "pole_angular_velocity_max = np.inf\n",
    "\n",
    "def create_batched_random_tensor(n):\n",
    "    # Creating the batched random tensor\n",
    "    cart_position = np.random.uniform(cart_position_min, cart_position_max, size=n)\n",
    "    cart_velocity = np.random.normal(loc=0.0, scale=10.0, size=n)  # Assuming normal distribution with large std deviation\n",
    "    pole_angle = np.random.uniform(pole_angle_min, pole_angle_max, size=n)\n",
    "    pole_angular_velocity = np.random.normal(loc=0.0, scale=10.0, size=n)  # Assuming normal distribution with large std deviation\n",
    "\n",
    "    # Combining into a single tensor of shape (n, 4)\n",
    "    batched_tensor = np.stack((cart_position, cart_velocity, pole_angle, pole_angular_velocity), axis=-1)\n",
    "    \n",
    "    return batched_tensor\n",
    "\n",
    "# Example usage with batch size n = 5\n",
    "n = 100\n",
    "batched_tensor = torch.tensor(create_batched_random_tensor(n), dtype=torch.float32)\n",
    "print(batched_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# Initialize the network\n",
    "# network = MICOMLPNetwork(in_features=4,\n",
    "#                          activation_class=torch.nn.ReLU, \n",
    "#                          encoder_out_features=8,\n",
    "#                          mlp_out_features=1,\n",
    "#                          encoder_num_cells=[16],\n",
    "#                          mlp_num_cells=[8])\n",
    "\n",
    "network = MICOMLPNetwork(\n",
    "    in_features=4,\n",
    "    activation_class=torch.nn.ReLU,\n",
    "    encoder_out_features=3,\n",
    "    mlp_out_features=env.action_spec.shape[-1],\n",
    "    encoder_num_cells=[128],\n",
    "    mlp_num_cells=[64]\n",
    ")\n",
    "\n",
    "# Define dummy target tensors for losses\n",
    "target_representation = torch.randn(100, 2)  # Assuming the representation has 8 features\n",
    "target_q_values = torch.randn(100, 2)  # Assuming the Q-values have 1 feature\n",
    "\n",
    "# Define loss functions\n",
    "criterion_representation = nn.MSELoss()\n",
    "criterion_q_values = nn.MSELoss()\n",
    "\n",
    "# Define an optimizer\n",
    "optimizer = optim.Adam(network.parameters(), lr=0.001)\n",
    "\n",
    "# Forward pass\n",
    "q_values, representation = network(batched_tensor)\n",
    "\n",
    "# Compute the losses\n",
    "# loss_representation = criterion_representation(representation, target_representation)\n",
    "# loss_q_values = criterion_q_values(q_values, target_q_values)\n",
    "\n",
    "# # Sum the losses\n",
    "# total_loss = loss_representation + loss_q_values\n",
    "\n",
    "# # Backward pass and optimization\n",
    "# optimizer.zero_grad()\n",
    "# total_loss.backward()\n",
    "# optimizer.step()\n",
    "\n",
    "# # Print losses\n",
    "# print(f\"Total Loss: {total_loss.item()}, Loss Representation: {loss_representation.item()}, Loss Q-Values: {loss_q_values.item()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Print the gradients of the total_loss\n",
    "# for name, param in network.named_parameters():\n",
    "#     print(name, param.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# toy_example = torch.tensor(batched_tensor, dtype=torch.float32)\n",
    "# q_values, representation = value_mlp(toy_example)\n",
    "# print(q_values)\n",
    "# print(representation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictModule(\n",
       "    module=MICOMLPNetwork(\n",
       "      (activation): ReLU()\n",
       "      (encoder): ModuleList(\n",
       "        (0): Linear(in_features=4, out_features=128, bias=True)\n",
       "        (1): Linear(in_features=128, out_features=3, bias=True)\n",
       "      )\n",
       "      (q_net): ModuleList(\n",
       "        (0): Linear(in_features=3, out_features=64, bias=True)\n",
       "        (1): Linear(in_features=64, out_features=2, bias=True)\n",
       "      )\n",
       "    ),\n",
       "    device=cpu,\n",
       "    in_keys=['observation'],\n",
       "    out_keys=['action_value', 'representation'])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_net = Mod(network, \n",
    "                in_keys=[\"observation\"], \n",
    "                out_keys=[\"action_value\", \"representation\"])\n",
    "value_net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "QValueActor(\n",
       "    module=ModuleList(\n",
       "      (0): TensorDictModule(\n",
       "          module=MICOMLPNetwork(\n",
       "            (activation): ReLU()\n",
       "            (encoder): ModuleList(\n",
       "              (0): Linear(in_features=4, out_features=128, bias=True)\n",
       "              (1): Linear(in_features=128, out_features=3, bias=True)\n",
       "            )\n",
       "            (q_net): ModuleList(\n",
       "              (0): Linear(in_features=3, out_features=64, bias=True)\n",
       "              (1): Linear(in_features=64, out_features=2, bias=True)\n",
       "            )\n",
       "          ),\n",
       "          device=cpu,\n",
       "          in_keys=['observation'],\n",
       "          out_keys=['action_value', 'representation'])\n",
       "      (1): QValueModule()\n",
       "    ),\n",
       "    device=cpu,\n",
       "    in_keys=['observation'],\n",
       "    out_keys=['representation', 'action', 'action_value', 'chosen_action_value'])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# policy = Seq(value_net, \n",
    "#              QValueModule(spec=env.action_spec))\n",
    "# policy\n",
    "\n",
    "policy = QValueActor(\n",
    "    module=value_net,\n",
    "    spec=CompositeSpec(action= env.specs[\"input_spec\", \"full_action_spec\", \"action\"]),\n",
    "    in_keys=[\"observation\"],\n",
    ")\n",
    "policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictSequential(\n",
       "    module=ModuleList(\n",
       "      (0): QValueActor(\n",
       "          module=ModuleList(\n",
       "            (0): TensorDictModule(\n",
       "                module=MICOMLPNetwork(\n",
       "                  (activation): ReLU()\n",
       "                  (encoder): ModuleList(\n",
       "                    (0): Linear(in_features=4, out_features=128, bias=True)\n",
       "                    (1): Linear(in_features=128, out_features=3, bias=True)\n",
       "                  )\n",
       "                  (q_net): ModuleList(\n",
       "                    (0): Linear(in_features=3, out_features=64, bias=True)\n",
       "                    (1): Linear(in_features=64, out_features=2, bias=True)\n",
       "                  )\n",
       "                ),\n",
       "                device=cpu,\n",
       "                in_keys=['observation'],\n",
       "                out_keys=['action_value', 'representation'])\n",
       "            (1): QValueModule()\n",
       "          ),\n",
       "          device=cpu,\n",
       "          in_keys=['observation'],\n",
       "          out_keys=['representation', 'action', 'action_value', 'chosen_action_value'])\n",
       "      (1): EGreedyModule()\n",
       "    ),\n",
       "    device=cpu,\n",
       "    in_keys=['observation'],\n",
       "    out_keys=['representation', 'action_value', 'chosen_action_value', 'action'])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the exploration step (e-greedy policy)\n",
    "exploration_module = EGreedyModule(\n",
    "    env.action_spec, \n",
    "    annealing_num_steps=100_000, \n",
    "    eps_init=0.1,\n",
    ")\n",
    "policy_explore = Seq(policy, \n",
    "                     exploration_module)\n",
    "policy_explore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define how to collect the data (experiences)\n",
    "init_rand_steps = 5000 # warm-up steps\n",
    "frames_per_batch = 100\n",
    "optim_steps = 10\n",
    "replay_capacity = 100_000\n",
    "\n",
    "# NOTE: collector will gather rollouts continously\n",
    "# If the current trajectory ends, it will start a new one\n",
    "# NOTE: the rollout gotten from the collector is a dictionary\n",
    "# that defines the sate and next state as a tensor with a batch dimension in the begining\n",
    "# for example a rollout of 10 steps will have a tensor of observation of 10 in the batch dimension\n",
    "# and the next will also have 10 which are all the tensors of the next state\n",
    "# Practically, next is as you will shift the tensor of observation by one step\n",
    "# collector = SyncDataCollector(\n",
    "#     env,\n",
    "#     policy_explore,\n",
    "#     frames_per_batch=frames_per_batch,\n",
    "#     total_frames=500_100,\n",
    "#     init_random_frames=init_rand_steps,\n",
    "# )\n",
    "# rb = ReplayBuffer(storage=LazyTensorStorage(replay_capacity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:2989: DeprecationWarning: Your wrapper was not given a device. Currently, this value will default to 'cpu'. From v0.5 it will default to `None`. With a device of None, no device casting is performed and the resulting tensordicts are deviceless. Please set your device accordingly.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Define the recording and logging\n",
    "path = \"./training_loop\"\n",
    "logger = CSVLogger(exp_name=\"dqn\", log_dir=path, video_format=\"mp4\")\n",
    "video_recorder = VideoRecorder(logger, tag=\"video\")\n",
    "record_env = TransformedEnv(\n",
    "    GymEnv(\"CartPole-v1\", from_pixels=True, pixels_only=False), video_recorder\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000, 0.0873, 0.2517],\n",
      "        [0.0000, 0.0238, 0.1848],\n",
      "        [0.0000, 0.0000, 0.3682],\n",
      "        [0.0000, 0.0874, 0.2489],\n",
      "        [0.0000, 0.0206, 0.1842],\n",
      "        [0.0000, 0.0000, 0.3667],\n",
      "        [0.0000, 0.0905, 0.2513],\n",
      "        [0.0000, 0.0148, 0.1964],\n",
      "        [0.0000, 0.0970, 0.2523],\n",
      "        [0.0000, 0.0000, 0.2323]])\n"
     ]
    }
   ],
   "source": [
    "# collector = SyncDataCollector(\n",
    "#     env,\n",
    "#     policy_explore,\n",
    "#     frames_per_batch=10,\n",
    "#     total_frames=500_100,\n",
    "#     init_random_frames=10000,\n",
    "# )\n",
    "\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn=env,\n",
    "    policy=policy_explore,\n",
    "    frames_per_batch=10,\n",
    "    total_frames=100,\n",
    "    device=\"cpu\",\n",
    "    storing_device=\"cpu\",\n",
    "    max_frames_per_traj=-1\n",
    ")\n",
    "# NOTE: IMPORTANTISIMO en las primeras iteraciones no se usa la policy, entonces representation se configura\n",
    "# a zero, por lo que el primer batch de datos no tiene representation\n",
    "# Tengo que hacer el warm-up de otra manera (ojo con esto)\n",
    "\n",
    "for data in collector:\n",
    "    print(data['representation'])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[False],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False],\n",
       "        [False],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False],\n",
       "        [False],\n",
       "        [ True]])"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','done']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.]])"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','reward']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_next_rewards(rewards, dones):\n",
    "    next_next_rewards = rewards.clone().roll(-1, dims=0)\n",
    "    next_next_rewards[dones] = 0\n",
    "    return next_next_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [0.]])"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next_next_rewards = get_next_next_rewards(data['next','reward'], data['next','done'])\n",
    "\n",
    "next_next_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                next_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if data.device is not None:\n",
    "    next_next_rewards = next_next_rewards.to(data.device)\n",
    "\n",
    "data.set(\n",
    "    (\"next\", \"next_reward\"),\n",
    "    next_next_rewards,\n",
    "    inplace=True,\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3, 4])"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = torch.tensor([1, 2, 3, 4, 5])\n",
    "arr[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not data['next','done'][-1]:\n",
    "    # next_next_rewards[-1] = 0 # Set according the reward of the env \n",
    "                              # in cartpole will be one so I don't need to do anything\n",
    "    # NOTE: Better remove that last step to avoid problems\n",
    "    data = data[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_tensor_dict_next_next_reward(tensordict):\n",
    "\n",
    "    def get_next_next_rewards(rewards, dones):\n",
    "        next_next_rewards = rewards.clone().roll(-1, dims=0)\n",
    "        next_next_rewards[dones] = 0\n",
    "        return next_next_rewards\n",
    "\n",
    "    next_next_rewards = get_next_next_rewards(tensordict['next','reward'], tensordict['next','done'])\n",
    "\n",
    "    if tensordict.device is not None:\n",
    "        next_next_rewards = next_next_rewards.to(tensordict.device)\n",
    "\n",
    "    tensordict.set(\n",
    "        (\"next\", \"next_reward\"),\n",
    "        next_next_rewards,\n",
    "        inplace=True,\n",
    "    )\n",
    "\n",
    "    # Side effect of the last step. As we cannot get the next_next_reward of the last step\n",
    "    # we need to remove it or assign it to a value that we know for sure like in cartpole\n",
    "    # will be always be one, and zero if it is done True.\n",
    "\n",
    "    # NOTE: We comment this part, because we are using the cartpole environment, but \n",
    "    # uncomment and update properly depending the case\n",
    "    # if not tensordict['next','done'][-1]:\n",
    "    #     # Two options\n",
    "    #     # 1. Assign a value\n",
    "    #     # next_next_rewards[-1] = 1 # Set according the reward of the env \n",
    "    #                                # in cartpole will be one so I don't need to do anything\n",
    "    #                                # in cartpole I don't need even to assign because all are 1\n",
    "    #     # 2. Remove that last step to avoid problems\n",
    "    #     data = data[:-1] \n",
    "\n",
    "    return tensordict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                episode_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                next_reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        pixels: Tensor(shape=torch.Size([10, 4, 84, 84]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = update_tensor_dict_next_next_reward(data)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.],\n",
       "        [1.]])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_next_next_reward = data['next','reward'].clone()\n",
    "tmp_next_next_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[False],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False],\n",
       "        [False],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False],\n",
       "        [ True],\n",
       "        [False]])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','done']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "next_next_mask = data['next','done'].clone().roll(1, dims=0)\n",
    "next_next_mask[0] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  1,   2,   3,  10,  20, 100, 200])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_temp = torch.tensor([1, 2, 3, 10, 20, 100, 200])\n",
    "reward_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([False, False,  True, False,  True, False,  True])"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = torch.tensor([False, False, True, False, True, False, True])\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  2,   3,  10,  20, 100, 200,   1])"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_temp.roll(-1, dims=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 10, 100,   1])"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_temp.roll(-1, dims=0)[mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   1,    2,    3,   10,   20,  100,  200, 1000])"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_temp = torch.tensor([1, 2, 3, 10, 20, 100, 200, 1000])\n",
    "reward_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([False, False,  True, False,  True, False,  True, False])"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = torch.tensor([False, False, True, False, True, False, True, False])\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  2,   3,   0,  20,   0, 200,   0,   1])"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_next_rewards(reward_temp, mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   3,   10,   20,  100,  200, 1000])"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_temp = torch.tensor([3, 10, 20, 100, 200, 1000])\n",
    "reward_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ True, False,  True, False,  True, False])"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = torch.tensor([ True, False, True, False, True, False])\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([  0,  20,   0, 200,   0,   3])"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_next_rewards(reward_temp, mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.1196, -1.7470,  0.2361,  2.8161])"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','observation'][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0037, -0.7668,  0.0538,  1.1900],\n",
       "        [-0.0885, -1.5509,  0.1867,  2.4725],\n",
       "        [-0.1196, -1.7470,  0.2361,  2.8161],\n",
       "        [-0.0581, -0.7588,  0.0363,  1.1615],\n",
       "        [-0.0955,  0.0178,  0.0960,  0.0764],\n",
       "        [-0.1176, -0.7678,  0.1408,  1.3712],\n",
       "        [-0.1755, -1.3575,  0.2432,  2.3923],\n",
       "        [-0.0218, -0.7441,  0.0790,  1.2167],\n",
       "        [-0.1049, -1.5290,  0.2149,  2.5275],\n",
       "        [-0.0177, -0.7991,  0.0436,  1.1731]])"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['next','observation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 1, 1, 1, 1, 2, 2, 3])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['collector','traj_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.8554e-02,  1.5046e-02,  1.8845e-02, -1.1108e-02],\n",
       "        [-3.6897e-03, -7.6677e-01,  5.3815e-02,  1.1900e+00],\n",
       "        [-8.8540e-02, -1.5509e+00,  1.8668e-01,  2.4725e+00],\n",
       "        [-3.6490e-02,  2.2149e-02,  2.5354e-03, -1.8976e-02],\n",
       "        [-5.8139e-02, -7.5877e-01,  3.6262e-02,  1.1615e+00],\n",
       "        [-9.5512e-02,  1.7841e-02,  9.6036e-02,  7.6353e-02],\n",
       "        [-1.1765e-01, -7.6778e-01,  1.4078e-01,  1.3712e+00],\n",
       "        [-1.4165e-03,  3.8949e-02,  4.3404e-02, -1.4220e-02],\n",
       "        [-2.1787e-02, -7.4414e-01,  7.9022e-02,  1.2167e+00],\n",
       "        [ 7.1530e-03, -1.7813e-02,  9.4300e-03, -1.6209e-02]])"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['observation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.8554e-02,  1.5046e-02,  1.8845e-02, -1.1108e-02],\n",
       "        [-3.6897e-03, -7.6677e-01,  5.3815e-02,  1.1900e+00],\n",
       "        [-8.8540e-02, -1.5509e+00,  1.8668e-01,  2.4725e+00],\n",
       "        [-3.6490e-02,  2.2149e-02,  2.5354e-03, -1.8976e-02],\n",
       "        [-5.8139e-02, -7.5877e-01,  3.6262e-02,  1.1615e+00],\n",
       "        [-9.5512e-02,  1.7841e-02,  9.6036e-02,  7.6353e-02],\n",
       "        [-1.1765e-01, -7.6778e-01,  1.4078e-01,  1.3712e+00],\n",
       "        [-1.4165e-03,  3.8949e-02,  4.3404e-02, -1.4220e-02],\n",
       "        [-2.1787e-02, -7.4414e-01,  7.9022e-02,  1.2167e+00],\n",
       "        [ 7.1530e-03, -1.7813e-02,  9.4300e-03, -1.6209e-02]])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['observation']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictReplayBuffer(\n",
       "    storage=LazyTensorStorage(\n",
       "        data=<empty>, \n",
       "        shape=None, \n",
       "        len=0, \n",
       "        max_size=100), \n",
       "    sampler=SliceSampler(num_slices=None, slice_len=2, end_key=('next', 'done'), traj_key=('collector', 'traj_ids'), truncated_key=('next', 'truncated'), strict_length=True), \n",
       "    writer=TensorDictRoundRobinWriter(cursor=0, full_storage=False), \n",
       "    batch_size=10, \n",
       "    collate_fn=<function _collate_id at 0x7fe4f157fba0>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchrl.data import SliceSampler\n",
    "from torchrl.data import TensorDictReplayBuffer\n",
    "\n",
    "size = 100\n",
    "rb = TensorDictReplayBuffer(\n",
    "    storage=LazyTensorStorage(size),\n",
    "    sampler=SliceSampler(traj_key=(\"collector\",\"traj_ids\"), slice_len=2),\n",
    "    batch_size=10,\n",
    ")\n",
    "rb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['collector','traj_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rb.extend(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = rb.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        index: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[7],\n",
       "        [8],\n",
       "        [2],\n",
       "        [3],\n",
       "        [5],\n",
       "        [6],\n",
       "        [1],\n",
       "        [2],\n",
       "        [0],\n",
       "        [1]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample['step_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 4.0251e-02,  1.3341e+00, -9.1547e-02, -2.0319e+00],\n",
       "        [ 6.6933e-02,  1.5300e+00, -1.3218e-01, -2.3514e+00],\n",
       "        [-3.4538e-02,  3.5789e-01,  2.1562e-02, -5.5068e-01],\n",
       "        [-2.7381e-02,  5.5270e-01,  1.0548e-02, -8.3650e-01],\n",
       "        [-1.3732e-03,  9.4288e-01, -2.8698e-02, -1.4205e+00],\n",
       "        [ 1.7484e-02,  1.1383e+00, -5.7108e-02, -1.7220e+00],\n",
       "        [-3.7802e-02,  1.6316e-01,  2.6894e-02, -2.6660e-01],\n",
       "        [-3.4538e-02,  3.5789e-01,  2.1562e-02, -5.5068e-01],\n",
       "        [-3.7170e-02, -3.1572e-02,  2.6542e-02,  1.7587e-02],\n",
       "        [-3.7802e-02,  1.6316e-01,  2.6894e-02, -2.6660e-01]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample[\"observation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0669,  1.5300, -0.1322, -2.3514],\n",
       "        [ 0.0975,  1.7260, -0.1792, -2.6817],\n",
       "        [-0.0274,  0.5527,  0.0105, -0.8365],\n",
       "        [-0.0163,  0.7477, -0.0062, -1.1258],\n",
       "        [ 0.0175,  1.1383, -0.0571, -1.7220],\n",
       "        [ 0.0403,  1.3341, -0.0915, -2.0319],\n",
       "        [-0.0345,  0.3579,  0.0216, -0.5507],\n",
       "        [-0.0274,  0.5527,  0.0105, -0.8365],\n",
       "        [-0.0378,  0.1632,  0.0269, -0.2666],\n",
       "        [-0.0345,  0.3579,  0.0216, -0.5507]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample[\"next\", \"observation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5099, 0.0000],\n",
       "        [0.0000, 0.5961, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.2237, 0.0000],\n",
       "        [0.0000, 0.3417, 0.0000],\n",
       "        [0.0000, 0.4248, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.0813, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([10, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        index: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([10, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([10, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([10, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([10]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorDict(\n",
      "    fields={\n",
      "        action: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        action_value: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        chosen_action_value: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        collector: TensorDict(\n",
      "            fields={\n",
      "                traj_ids: Tensor(shape=torch.Size([5]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
      "            batch_size=torch.Size([5]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        index: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        next: TensorDict(\n",
      "            fields={\n",
      "                done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                reward: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "            batch_size=torch.Size([5]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        representation: Tensor(shape=torch.Size([5, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "    batch_size=torch.Size([5]),\n",
      "    device=cpu,\n",
      "    is_shared=False)\n",
      "TensorDict(\n",
      "    fields={\n",
      "        action: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        action_value: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        chosen_action_value: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        collector: TensorDict(\n",
      "            fields={\n",
      "                traj_ids: Tensor(shape=torch.Size([5]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
      "            batch_size=torch.Size([5]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        index: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        next: TensorDict(\n",
      "            fields={\n",
      "                done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                reward: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "                step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "                terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "                truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "            batch_size=torch.Size([5]),\n",
      "            device=cpu,\n",
      "            is_shared=False),\n",
      "        observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        representation: Tensor(shape=torch.Size([5, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
      "        step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
      "        terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
      "        truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
      "    batch_size=torch.Size([5]),\n",
      "    device=cpu,\n",
      "    is_shared=False)\n"
     ]
    }
   ],
   "source": [
    "sample['representation']\n",
    "\n",
    "first_states = sample[0::2] # even rows\n",
    "second_states = sample[1::2] # odd rows (or next states)\n",
    "\n",
    "print(first_states)\n",
    "print(second_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5099, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.3417, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000],\n",
       "        [0.0000, 0.0813, 0.0000]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_states['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5961, 0.0000],\n",
       "        [0.0000, 0.2237, 0.0000],\n",
       "        [0.0000, 0.4248, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_states['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3, 1, 2, 3, 1, 2, 3],\n",
      "        [1, 2, 3, 1, 2, 3, 1, 2, 3]])\n",
      "tensor([[1, 2, 3, 1, 2, 3, 1, 2, 3],\n",
      "        [1, 2, 3, 1, 2, 3, 1, 2, 3]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "a = torch.tensor([1, 2, 3])\n",
    "repeated_a = torch.Tensor.repeat(a, (2, 3))\n",
    "\n",
    "print(repeated_a)\n",
    "\n",
    "a = torch.tensor([1, 2, 3])\n",
    "tiled_a = torch.tile(a, (2, 3))\n",
    "print(tiled_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5961, 0.0000],\n",
       "        [0.0000, 0.2237, 0.0000],\n",
       "        [0.0000, 0.4248, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_states['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000]]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_states['representation'].shape # batch, rep_dim\n",
    "\n",
    "repeated_rep = torch.tile(second_states['representation'], (1,1,5)).view(5,5,3)\n",
    "repeated_rep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 3])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_states['representation'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squarify(x):\n",
    "    # Squarify will take the input and adds a new dimension between the batch and the representation\n",
    "    # so that the representation is repeated along the new dimension\n",
    "    # To visualize thing of x as a matrix of batch_size x representation_dim\n",
    "    # and squarify will place that matrix in a lateral way and repeat it along the new dimension j\n",
    "\n",
    "    # NOTE: after squarify if you pick a i-th row all the elements (j-th index) in that row will be the same\n",
    "    batch_size = x.shape[0]\n",
    "    if len(x.shape) > 1:\n",
    "        representation_dim = x.shape[-1]\n",
    "        return x.tile((batch_size,)).view(batch_size, batch_size, representation_dim)\n",
    "    return x.tile((batch_size,)).view(batch_size, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.],\n",
       "        [1., 1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squarify(second_states['next','reward']).squeeze(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000],\n",
       "         [0.0000, 0.5961, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000],\n",
       "         [0.0000, 0.2237, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000],\n",
       "         [0.0000, 0.4248, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000],\n",
       "         [0.0000, 0.1776, 0.0000]],\n",
       "\n",
       "        [[0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000],\n",
       "         [0.0000, 0.1231, 0.0000]]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squarify(second_states['representation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25])\n",
      "torch.Size([25])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.3077, 0.1550, 0.2202, 0.1458, 0.1376, 0.1935, 0.0408, 0.1060, 0.0316,\n",
       "        0.0234, 0.2360, 0.0834, 0.1486, 0.0741, 0.0659, 0.1853, 0.0326, 0.0978,\n",
       "        0.0234, 0.0152, 0.1810, 0.0284, 0.0935, 0.0191, 0.0109])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def representation_distances(first_representations, second_representations,\n",
    "                             distance_fn, beta=0.1,\n",
    "                             return_distance_components=False):\n",
    "  \"\"\"Compute distances between representations.\n",
    "     In the paper, it corresponds to the calculation of the U term\n",
    "     for each pair of representations in the batch (all-vs-all).\n",
    "\n",
    "  This will compute the distances between two representations.\n",
    "\n",
    "  Args:\n",
    "    first_representations: first set of representations to use.\n",
    "    second_representations: second set of representations to use.\n",
    "    distance_fn: function to use for computing representation distances.\n",
    "    beta: float, weight given to cosine distance between representations.\n",
    "    return_distance_components: bool, whether to return the components used for\n",
    "      computing the distance.\n",
    "\n",
    "  Returns:\n",
    "    The distances between representations, combining the average of the norm of\n",
    "    the representations and the distance given by distance_fn.\n",
    "  \"\"\"\n",
    "  batch_size = first_representations.shape[0]\n",
    "  representation_dim = first_representations.shape[-1]\n",
    "\n",
    "  # Squarify the representations and reshape them to make a pair-waise comparison with vmap\n",
    "  first_squared_reps = squarify(first_representations)\n",
    "  first_squared_reps = torch.reshape(first_squared_reps,\n",
    "                                   [batch_size**2, representation_dim])\n",
    "  \n",
    "  # Squarify the representations and reshape them to make a pair-waise comparison with vmap\n",
    "  # However, we now need to permute (transpose) the dimension 0, 1 to alternate the values\n",
    "  # so that we have the pair-wise comparisons of all-vs-all\n",
    "  second_squared_reps = squarify(second_representations)\n",
    "  second_squared_reps = torch.permute(second_squared_reps, dims=(1, 0, 2))\n",
    "  second_squared_reps = torch.reshape(second_squared_reps,\n",
    "                                    [batch_size**2, representation_dim])\n",
    "  \n",
    "  # vmap will calculate the pairwise distance_fn along the dimension specified\n",
    "  # in in_axes. In this case, will take the dim 0 of the first_squared_reps and\n",
    "  # the dim 0 of the second_squared_reps and apply the distance\n",
    "  # It vertorize the process of calculating the distance between all the pairs\n",
    "\n",
    "  # NOTE: base distance corresponds to the second term in the U calculation in the paper\n",
    "  # It calculates the angle between the representations in the paper\n",
    "  # Check what function is using\n",
    "  base_distances = torch.vmap(distance_fn, in_dims=(0, 0))(first_squared_reps,\n",
    "                                                         second_squared_reps)\n",
    "  base_distances = base_distances\n",
    "  print(base_distances.shape)\n",
    "  # Sum along the second dimension and normalize the distance\n",
    "  # NOTE: this is practically the first term of U in the paper\n",
    "  norm_average = 0.5 * (torch.sum(torch.square(first_squared_reps), -1) +\n",
    "                        torch.sum(torch.square(second_squared_reps), -1))\n",
    "  \n",
    "  print(norm_average.shape)\n",
    "  if return_distance_components:\n",
    "    return norm_average + beta * base_distances, norm_average, base_distances\n",
    "  return norm_average + beta * base_distances\n",
    "\n",
    "EPSILON = 1e-9\n",
    "\n",
    "def _sqrt(x):\n",
    "  # zeros like instead of zeros\n",
    "  # It is because vmap works with a weird way of broadcasting\n",
    "  # and a weird structure based on tensors\n",
    "  tol = torch.zeros_like(x)\n",
    "  return torch.sqrt(torch.maximum(x, tol))\n",
    "\n",
    "\n",
    "def cosine_distance(x, y):\n",
    "  # NOTE: the cosine similarity is not calculate directly for \n",
    "  # instabilities observed when using `jnp.arccos`, but I'm using torch\n",
    "  # so I don't know if I will need to do this\n",
    "  numerator = torch.sum(x * y)\n",
    "  denominator = torch.sqrt(torch.sum(x**2)) * torch.sqrt(torch.sum(y**2))\n",
    "  cos_similarity = numerator / (denominator + EPSILON)\n",
    "\n",
    "  # cos_similarity = cos(theta)\n",
    "\n",
    "  # NOTE: From, the Pythagorean trigometric identity\n",
    "  # sin^2(theta) + cos^2(theta) = 1\n",
    "  # you can get sin(theta) = sqrt(1 - cos^2(theta))\n",
    "  # and the arctan2(sin(theta), cos(theta)) = theta\n",
    "  return torch.arctan2(_sqrt(1. - cos_similarity**2), cos_similarity)\n",
    "\n",
    "distances = representation_distances(first_states['representation'], \n",
    "                                     second_states['representation'], \n",
    "                                     cosine_distance)\n",
    "distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25])\n",
      "torch.Size([25])\n"
     ]
    }
   ],
   "source": [
    "# NOTE: check in the main code if the output of this requires grad and the \n",
    "# other output must require grad\n",
    "# @torch.no_grad()\n",
    "def target_distances(representations, rewards, distance_fn, cumulative_gamma):\n",
    "  \"\"\"Target distance using the metric operator. This is the T in the paper :D\"\"\"\n",
    "  next_state_similarities = representation_distances(\n",
    "      representations, representations, distance_fn)\n",
    "  squared_rews = squarify(rewards).squeeze(-1)\n",
    "  squared_rews_transp = squared_rews.T\n",
    "  squared_rews = squared_rews.reshape((squared_rews.shape[0]**2))\n",
    "  squared_rews_transp = squared_rews_transp.reshape(\n",
    "      (squared_rews_transp.shape[0]**2))\n",
    "  reward_diffs = torch.abs(squared_rews - squared_rews_transp)\n",
    "  return reward_diffs + cumulative_gamma * next_state_similarities\n",
    "\n",
    "t_distances = target_distances(first_states['representation'], first_states['next','reward'], cosine_distance, cumulative_gamma = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        action: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([5, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([5]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([5]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        index: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([5]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([5, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([5, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([5, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([5]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25])\n",
      "torch.Size([25])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.3077, 0.1550, 0.2202, 0.1458, 0.1376, 0.1935, 0.0408, 0.1060, 0.0316,\n",
       "        0.0234, 0.2360, 0.0834, 0.1486, 0.0741, 0.0659, 0.1853, 0.0326, 0.0978,\n",
       "        0.0234, 0.0152, 0.1810, 0.0284, 0.0935, 0.0191, 0.0109])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mico_distance = representation_distances(\n",
    "    first_states['representation'], second_states['representation'], cosine_distance)\n",
    "mico_distance\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([25])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mico_distance.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = first_states['representation'].shape[0]\n",
    "batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_vs_all_mico_priorities(batch_online_representation, batch_target_representation, mico_beta):\n",
    "\n",
    "    all_vs_all_mico_distances = representation_distances(\n",
    "        batch_online_representation, batch_target_representation, cosine_distance)\n",
    "    \n",
    "    batch_size = batch_online_representation.shape[0]\n",
    "\n",
    "    # NOTE: Mico distance is a unidimensional tensor with the distances of all the pairs\n",
    "    # Apply the reshape to get the distances of all the pairs, and get the mean\n",
    "    # of the distances of the current states\n",
    "    all_vs_all_mico_distances = all_vs_all_mico_distances.reshape((batch_size,batch_size))\n",
    "    return all_vs_all_mico_distances.mean(-1).detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([25])\n",
      "torch.Size([25])\n"
     ]
    }
   ],
   "source": [
    "mico_priorities = all_vs_all_mico_priorities(\n",
    "        batch_online_representation = first_states['representation'],\n",
    "        batch_target_representation = second_states['representation'],\n",
    "        mico_beta = 0.1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.1932, 0.0790, 0.1216, 0.0709, 0.0666])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mico_priorities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.3077, 0.1550, 0.2202, 0.1458, 0.1376],\n",
      "        [0.1935, 0.0408, 0.1060, 0.0316, 0.0234],\n",
      "        [0.2360, 0.0834, 0.1486, 0.0741, 0.0659],\n",
      "        [0.1853, 0.0326, 0.0978, 0.0234, 0.0152],\n",
      "        [0.1810, 0.0284, 0.0935, 0.0191, 0.0109]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.1932, 0.0790, 0.1216, 0.0709, 0.0666])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Mico distance is a unidimensional tensor with the distances of all the pairs\n",
    "# Apply the reshape to get the distances of all the pairs, and get the mean\n",
    "# of the distances of the current states\n",
    "mico_priority = mico_distance.reshape((batch_size,batch_size))\n",
    "print(mico_priority)\n",
    "mico_priority = mico_priority.mean(-1)\n",
    "mico_priority"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0790)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mico_distance.reshape((batch_size,batch_size))[1].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_distances.requires_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def current_vs_next_mico_priorities(\n",
    "        current_state_representations,\n",
    "        next_state_representations,\n",
    "        mico_beta,\n",
    "        return_distance_components = False):\n",
    "\n",
    "  base_distances = torch.vmap(cosine_distance, in_dims=(0, 0))(current_state_representations,\n",
    "                                                         next_state_representations)\n",
    "\n",
    "  # Sum along the second dimension and normalize the distance\n",
    "  # NOTE: this is practically the first term of U in the paper\n",
    "  norm_average = 0.5 * (torch.sum(torch.square(current_state_representations), -1) +\n",
    "                        torch.sum(torch.square(next_state_representations), -1))\n",
    "  \n",
    "  mico_distance = norm_average + mico_beta * base_distances\n",
    "  \n",
    "  if return_distance_components:\n",
    "    return mico_distance, norm_average, base_distances\n",
    "  \n",
    "  # Repeat the priority to assign the same priority to both\n",
    "  # the current and next state\n",
    "  return mico_distance.repeat_interleave(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.3077, 0.1550, 0.2202, 0.1458, 0.1376, 0.1935, 0.0408, 0.1060, 0.0316,\n",
       "        0.0234, 0.2360, 0.0834, 0.1486, 0.0741, 0.0659, 0.1853, 0.0326, 0.0978,\n",
       "        0.0234, 0.0152, 0.1810, 0.0284, 0.0935, 0.0191, 0.0109])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mico_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5099, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.3417, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000],\n",
       "        [0.0000, 0.0813, 0.0000]])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_states['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.5961, 0.0000],\n",
       "        [0.0000, 0.2237, 0.0000],\n",
       "        [0.0000, 0.4248, 0.0000],\n",
       "        [0.0000, 0.1776, 0.0000],\n",
       "        [0.0000, 0.1231, 0.0000]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_states['representation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.3077, 0.3077, 0.0408, 0.0408, 0.1486, 0.1486, 0.0234, 0.0234, 0.0109,\n",
       "        0.0109])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current_vs_next_mico_priorities(\n",
    "        current_state_representations = first_states['representation'],\n",
    "        next_state_representations = second_states['representation'],\n",
    "        mico_beta = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0.1717, 0.1063, 0.0000],\n",
       "         [0.1832, 0.0750, 0.4068],\n",
       "         [0.0490, 0.0752, 0.0896],\n",
       "         [0.0857, 0.0904, 0.1998],\n",
       "         [0.0490, 0.0752, 0.0896]]),\n",
       " tensor([[0.0836, 0.2269],\n",
       "         [0.0057, 0.2379],\n",
       "         [0.0791, 0.2383],\n",
       "         [0.0543, 0.2416],\n",
       "         [0.0791, 0.2383]]),\n",
       " tensor([[0.2269],\n",
       "         [0.2379],\n",
       "         [0.2383],\n",
       "         [0.2416],\n",
       "         [0.2383]]),\n",
       " tensor([[0, 1],\n",
       "         [0, 1],\n",
       "         [0, 1],\n",
       "         [0, 1],\n",
       "         [0, 1]]))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# NOTE: Checar la distancia euclidean entre la representacion target y la representacion con\n",
    "# la politica actual\n",
    "\n",
    "# NOTE: en el repositorio de MICO, la distancia target es calculada con\n",
    "# la target network (que es una copia de la politica actual) osea mis representaciones guardadas\n",
    "# la distancia online por otra parte es calculada con una representacion con red actual\n",
    "# y una representacion target\n",
    "\n",
    "\n",
    "collector.policy(first_states['observation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CNN_MLP for q-net architecture\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA error: invalid device ordinal\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 72\u001b[0m\n\u001b[1;32m     63\u001b[0m model_explore \u001b[38;5;241m=\u001b[39m TensorDictSequential(\n\u001b[1;32m     64\u001b[0m     model,\n\u001b[1;32m     65\u001b[0m     greedy_module,\n\u001b[1;32m     66\u001b[0m ) \u001b[38;5;66;03m#.to(device)\u001b[39;00m\n\u001b[1;32m     68\u001b[0m \u001b[38;5;66;03m# Create the collector\u001b[39;00m\n\u001b[1;32m     69\u001b[0m \u001b[38;5;66;03m# NOTE: init_random_frames: Number of frames \u001b[39;00m\n\u001b[1;32m     70\u001b[0m \u001b[38;5;66;03m# for which the policy is ignored before it is called.\u001b[39;00m\n\u001b[1;32m     71\u001b[0m collector \u001b[38;5;241m=\u001b[39m SyncDataCollector(\n\u001b[0;32m---> 72\u001b[0m     create_env_fn\u001b[38;5;241m=\u001b[39m\u001b[43mmake_env\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcfg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcpu\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcfg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseed\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m     73\u001b[0m     policy\u001b[38;5;241m=\u001b[39mmodel_explore,\n\u001b[1;32m     74\u001b[0m     frames_per_batch\u001b[38;5;241m=\u001b[39mcfg\u001b[38;5;241m.\u001b[39mcollector\u001b[38;5;241m.\u001b[39mframes_per_batch,\n\u001b[1;32m     75\u001b[0m     total_frames\u001b[38;5;241m=\u001b[39mcfg\u001b[38;5;241m.\u001b[39mcollector\u001b[38;5;241m.\u001b[39mtotal_frames,\n\u001b[1;32m     76\u001b[0m     device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     77\u001b[0m     storing_device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     78\u001b[0m     max_frames_per_traj\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m     79\u001b[0m     init_random_frames\u001b[38;5;241m=\u001b[39mcfg\u001b[38;5;241m.\u001b[39mcollector\u001b[38;5;241m.\u001b[39minit_random_frames,\n\u001b[1;32m     80\u001b[0m )\n\u001b[1;32m     82\u001b[0m \u001b[38;5;66;03m# Create the replay buffer\u001b[39;00m\n\u001b[1;32m     83\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cfg\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mprioritized_replay:\n",
      "File \u001b[0;32m~/workspace/final_project/dqn_pixels_mico/utils_cartpole.py:41\u001b[0m, in \u001b[0;36mmake_env\u001b[0;34m(env_name, frame_skip, device, seed, cropping)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake_env\u001b[39m(env_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCartPole-v1\u001b[39m\u001b[38;5;124m\"\u001b[39m, frame_skip \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m4\u001b[39m, \n\u001b[1;32m     39\u001b[0m              device\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m, seed \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m, cropping \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m):\u001b[38;5;66;03m#, is_test=False):\u001b[39;00m\n\u001b[0;32m---> 41\u001b[0m     env \u001b[38;5;241m=\u001b[39m \u001b[43mGymEnv\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     42\u001b[0m \u001b[43m        \u001b[49m\u001b[43menv_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     43\u001b[0m \u001b[43m        \u001b[49m\u001b[43mframe_skip\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mframe_skip\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     44\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfrom_pixels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     45\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpixels_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m     46\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     47\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     48\u001b[0m     env \u001b[38;5;241m=\u001b[39m TransformedEnv(env)\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;66;03m# env.append_transform(NoopResetEnv(noops=30, random=True)) # NOTE: Cartpole with no noops will fall into reset in the begining\u001b[39;00m\n\u001b[1;32m     50\u001b[0m                                                                 \u001b[38;5;66;03m# I could use an small noop reset to avoid this, but I think is not necesary\u001b[39;00m\n\u001b[1;32m     51\u001b[0m                                                                 \u001b[38;5;66;03m# in this case. Analyze this later\u001b[39;00m\n\u001b[1;32m     52\u001b[0m     \u001b[38;5;66;03m# if not is_test:\u001b[39;00m\n\u001b[1;32m     53\u001b[0m         \u001b[38;5;66;03m# env.append_transform(EndOfLifeTransform()) # NOTE: Check my environment is not based on lives (so not important)\u001b[39;00m\n\u001b[1;32m     54\u001b[0m         \u001b[38;5;66;03m# env.append_transform(SignTransform(in_keys=[\"reward\"])) #NOTE: cartpole has no negative rewards\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:543\u001b[0m, in \u001b[0;36m_AsyncMeta.__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    542\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mcls\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 543\u001b[0m     instance: GymWrapper \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    545\u001b[0m     \u001b[38;5;66;03m# before gym 0.22, there was no final_observation\u001b[39;00m\n\u001b[1;32m    546\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m instance\u001b[38;5;241m.\u001b[39m_is_batched:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:175\u001b[0m, in \u001b[0;36m_EnvPostInit.__call__\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m    173\u001b[0m auto_reset \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_reset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m    174\u001b[0m auto_reset_replace \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_reset_replace\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 175\u001b[0m instance: EnvBase \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    176\u001b[0m \u001b[38;5;66;03m# we create the done spec by adding a done/terminated entry if one is missing\u001b[39;00m\n\u001b[1;32m    177\u001b[0m instance\u001b[38;5;241m.\u001b[39m_create_done_specs()\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:1277\u001b[0m, in \u001b[0;36mGymEnv.__init__\u001b[0;34m(self, env_name, **kwargs)\u001b[0m\n\u001b[1;32m   1275\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv_name\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m env_name\n\u001b[1;32m   1276\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_gym_args(kwargs)\n\u001b[0;32m-> 1277\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:731\u001b[0m, in \u001b[0;36mGymWrapper.__init__\u001b[0;34m(self, env, categorical_action_encoding, **kwargs)\u001b[0m\n\u001b[1;32m    729\u001b[0m         \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    730\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 731\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    732\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_post_init()\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/common.py:3019\u001b[0m, in \u001b[0;36m_EnvWrapper.__init__\u001b[0;34m(self, device, batch_size, allow_done_after_reset, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3017\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_kwargs(kwargs)\n\u001b[1;32m   3018\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_env \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_env(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# writes the self._env attribute\u001b[39;00m\n\u001b[0;32m-> 3019\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_specs\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_env\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# writes the self._env attribute\u001b[39;00m\n\u001b[1;32m   3020\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m   3021\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_env()\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:951\u001b[0m, in \u001b[0;36mGymWrapper._make_specs\u001b[0;34m(self, env, batch_size)\u001b[0m\n\u001b[1;32m    945\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_specs\u001b[39m(\u001b[38;5;28mself\u001b[39m, env: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgym.Env\u001b[39m\u001b[38;5;124m\"\u001b[39m, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:  \u001b[38;5;66;03m# noqa: F821\u001b[39;00m\n\u001b[1;32m    946\u001b[0m     action_spec \u001b[38;5;241m=\u001b[39m _gym_to_torchrl_spec_transform(\n\u001b[1;32m    947\u001b[0m         env\u001b[38;5;241m.\u001b[39maction_space,\n\u001b[1;32m    948\u001b[0m         device\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice,\n\u001b[1;32m    949\u001b[0m         categorical_action_encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_categorical_action_encoding,\n\u001b[1;32m    950\u001b[0m     )\n\u001b[0;32m--> 951\u001b[0m     observation_spec \u001b[38;5;241m=\u001b[39m \u001b[43m_gym_to_torchrl_spec_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    952\u001b[0m \u001b[43m        \u001b[49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mobservation_space\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    953\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    954\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcategorical_action_encoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_categorical_action_encoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    955\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    956\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(observation_spec, CompositeSpec):\n\u001b[1;32m    957\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfrom_pixels:\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:354\u001b[0m, in \u001b[0;36m_gym_to_torchrl_spec_transform\u001b[0;34m(spec, dtype, device, categorical_action_encoding, remap_state_to_observation, batch_size)\u001b[0m\n\u001b[1;32m    352\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m CompositeSpec(spec_out, device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[1;32m    353\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(spec, gym_spaces\u001b[38;5;241m.\u001b[39mdict\u001b[38;5;241m.\u001b[39mDict):\n\u001b[0;32m--> 354\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_gym_to_torchrl_spec_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    355\u001b[0m \u001b[43m        \u001b[49m\u001b[43mspec\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspaces\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    356\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    357\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcategorical_action_encoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcategorical_action_encoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    358\u001b[0m \u001b[43m        \u001b[49m\u001b[43mremap_state_to_observation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremap_state_to_observation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    359\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    360\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    361\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\n\u001b[1;32m    362\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspec of type \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(spec)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is currently unaccounted for\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    363\u001b[0m     )\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:345\u001b[0m, in \u001b[0;36m_gym_to_torchrl_spec_transform\u001b[0;34m(spec, dtype, device, categorical_action_encoding, remap_state_to_observation, batch_size)\u001b[0m\n\u001b[1;32m    335\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m    336\u001b[0m         remap_state_to_observation\n\u001b[1;32m    337\u001b[0m         \u001b[38;5;129;01mand\u001b[39;00m k \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstate\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    342\u001b[0m         \u001b[38;5;66;03m# naming it 'state' will result in envs that have a different name for the state vector\u001b[39;00m\n\u001b[1;32m    343\u001b[0m         \u001b[38;5;66;03m# when queried with and without pixels\u001b[39;00m\n\u001b[1;32m    344\u001b[0m         key \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mobservation\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 345\u001b[0m     spec_out[key] \u001b[38;5;241m=\u001b[39m \u001b[43m_gym_to_torchrl_spec_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    346\u001b[0m \u001b[43m        \u001b[49m\u001b[43mspec\u001b[49m\u001b[43m[\u001b[49m\u001b[43mk\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    347\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    348\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcategorical_action_encoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcategorical_action_encoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    349\u001b[0m \u001b[43m        \u001b[49m\u001b[43mremap_state_to_observation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremap_state_to_observation\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    350\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;66;03m# the batch-size must be set later\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m CompositeSpec(spec_out, device\u001b[38;5;241m=\u001b[39mdevice)\n",
      "File \u001b[0;32m~/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/envs/libs/gym.py:309\u001b[0m, in \u001b[0;36m_gym_to_torchrl_spec_transform\u001b[0;34m(spec, dtype, device, categorical_action_encoding, remap_state_to_observation, batch_size)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dtype \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    308\u001b[0m     dtype \u001b[38;5;241m=\u001b[39m numpy_to_torch_dtype_dict[spec\u001b[38;5;241m.\u001b[39mdtype]\n\u001b[0;32m--> 309\u001b[0m low \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_tensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mspec\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlow\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    310\u001b[0m high \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mas_tensor(spec\u001b[38;5;241m.\u001b[39mhigh, device\u001b[38;5;241m=\u001b[39mdevice, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[1;32m    311\u001b[0m is_unbounded \u001b[38;5;241m=\u001b[39m low\u001b[38;5;241m.\u001b[39misinf()\u001b[38;5;241m.\u001b[39mall() \u001b[38;5;129;01mand\u001b[39;00m high\u001b[38;5;241m.\u001b[39misinf()\u001b[38;5;241m.\u001b[39mall()\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA error: invalid device ordinal\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Copyright (c) Meta Platforms, Inc. and affiliates.\n",
    "#\n",
    "# This source code is licensed under the MIT license found in the\n",
    "# LICENSE file in the root directory of this source tree.\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import hydra\n",
    "import torch.nn\n",
    "import torch.optim\n",
    "import tqdm\n",
    "import wandb\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "from tensordict.nn import TensorDictSequential\n",
    "# from torchrl._utils import logger as torchrl_logger\n",
    "from torchrl.collectors import SyncDataCollector\n",
    "from torchrl.data import LazyTensorStorage, TensorDictReplayBuffer, LazyMemmapStorage\n",
    "from torchrl.envs import ExplorationType, set_exploration_type\n",
    "from torchrl.modules import EGreedyModule\n",
    "from torchrl.objectives import DQNLoss, HardUpdate, SoftUpdate\n",
    "from torchrl.record import VideoRecorder\n",
    "from torchrl.data.replay_buffers.samplers import RandomSampler, PrioritizedSampler\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR\n",
    "\n",
    "# from torchrl.record.loggers import generate_exp_name, get_logger\n",
    "from utils_cartpole import (\n",
    "    eval_model,\n",
    "    make_dqn_model,\n",
    "    make_env,\n",
    "    print_hyperparameters,\n",
    "    update_tensor_dict_next_next_rewards\n",
    ")\n",
    "from utils_modules import MICODQNLoss\n",
    "\n",
    "import tempfile\n",
    "\n",
    "from utils_cartpole import make_dqn_model, make_env\n",
    "from utils_modules import MICODQNLoss\n",
    "\n",
    "from tensordict.nn import TensorDictSequential\n",
    "from torchrl.data.replay_buffers.samplers import RandomSampler, PrioritizedSampler, PrioritizedSliceSampler\n",
    "from torchrl.objectives import DQNLoss, HardUpdate\n",
    "\n",
    "# load condig_cartpole.yaml\n",
    "import yaml\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "with open(\"config_cartpole.yaml\") as f:\n",
    "    cfg = OmegaConf.create(yaml.safe_load(f))\n",
    "\n",
    "model = make_dqn_model(\"CartPole-v1\", cfg.policy, frame_skip=cfg.collector.frame_skip)\n",
    "\n",
    "greedy_module = EGreedyModule(\n",
    "    annealing_num_steps=cfg.collector.annealing_frames,\n",
    "    eps_init=cfg.collector.eps_start,\n",
    "    eps_end=cfg.collector.eps_end,\n",
    "    spec=model.spec,\n",
    ")\n",
    "model_explore = TensorDictSequential(\n",
    "    model,\n",
    "    greedy_module,\n",
    ") #.to(device)\n",
    "\n",
    "# Create the collector\n",
    "# NOTE: init_random_frames: Number of frames \n",
    "# for which the policy is ignored before it is called.\n",
    "collector = SyncDataCollector(\n",
    "    create_env_fn=make_env(cfg.env.env_name, \"cpu\", cfg.env.seed),\n",
    "    policy=model_explore,\n",
    "    frames_per_batch=cfg.collector.frames_per_batch,\n",
    "    total_frames=cfg.collector.total_frames,\n",
    "    device=\"cpu\",\n",
    "    storing_device=\"cpu\",\n",
    "    max_frames_per_traj=-1,\n",
    "    init_random_frames=cfg.collector.init_random_frames,\n",
    ")\n",
    "\n",
    "# Create the replay buffer\n",
    "if cfg.buffer.prioritized_replay:\n",
    "    print(\"Using Prioritized Replay Buffer\")\n",
    "    sampler = PrioritizedSliceSampler(\n",
    "        max_capacity=cfg.buffer.buffer_size, \n",
    "        alpha=cfg.buffer.alpha, \n",
    "        beta=cfg.buffer.beta, \n",
    "        traj_key=(\"collector\",\"traj_ids\"), \n",
    "        slice_len=2)\n",
    "else:\n",
    "    sampler = SliceSampler(\n",
    "        traj_key=(\"collector\",\"traj_ids\"), \n",
    "        slice_len=2)\n",
    "    \n",
    "replay_buffer = TensorDictReplayBuffer(\n",
    "    pin_memory=False,\n",
    "    prefetch=10,\n",
    "    storage=LazyTensorStorage(\n",
    "        max_size=cfg.buffer.buffer_size,\n",
    "        device=\"cpu\",\n",
    "    ),\n",
    "    batch_size=cfg.buffer.batch_size,\n",
    "    sampler = sampler\n",
    ")\n",
    "\n",
    "# Create the loss module\n",
    "loss_module = MICODQNLoss(\n",
    "    value_network=model,\n",
    "    loss_function=\"l2\", \n",
    "    delay_value=True, # delay_value=True means we will use a target network\n",
    "    mico_gamma=cfg.loss.mico_gamma,\n",
    "    mico_beta=cfg.loss.mico_beta,\n",
    "    mico_weight=cfg.loss.mico_weight,\n",
    ")\n",
    "\n",
    "loss_module.make_value_estimator(gamma=cfg.loss.gamma) # only to change the gamma value\n",
    "loss_module = loss_module #.to(device)\n",
    "target_net_updater = HardUpdate(\n",
    "    loss_module, value_network_update_interval=cfg.loss.hard_update_freq\n",
    ")\n",
    "\n",
    "# Create the optimizer\n",
    "optimizer = torch.optim.Adam(loss_module.parameters(), lr=cfg.optim.lr)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum of the model parameters per layer\n",
      "tensor(0.4998)\n",
      "tensor(0.4991)\n",
      "tensor(0.0913)\n",
      "tensor(0.0879)\n",
      "tensor(0.1083)\n",
      "tensor(-0.0046)\n",
      "tensor(0.5714)\n",
      "tensor(0.5667)\n",
      "tensor(0.1090)\n",
      "tensor(0.0080)\n",
      "Maximum of the model parameters per layer\n",
      "tensor(0.4998, grad_fn=<MaxBackward1>)\n",
      "tensor(0.4991, grad_fn=<MaxBackward1>)\n",
      "tensor(0.0913, grad_fn=<MaxBackward1>)\n",
      "tensor(0.0879, grad_fn=<MaxBackward1>)\n",
      "tensor(0.1083, grad_fn=<MaxBackward1>)\n",
      "tensor(-0.0046, grad_fn=<MaxBackward1>)\n",
      "tensor(0.5714, grad_fn=<MaxBackward1>)\n",
      "tensor(0.5667, grad_fn=<MaxBackward1>)\n",
      "tensor(0.1090, grad_fn=<MaxBackward1>)\n",
      "tensor(0.0080, grad_fn=<MaxBackward1>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[tensor(0.4998, grad_fn=<MaxBackward1>),\n",
       " tensor(0.4991, grad_fn=<MaxBackward1>),\n",
       " tensor(0.0913, grad_fn=<MaxBackward1>),\n",
       " tensor(0.0879, grad_fn=<MaxBackward1>),\n",
       " tensor(0.1083, grad_fn=<MaxBackward1>),\n",
       " tensor(-0.0046, grad_fn=<MaxBackward1>),\n",
       " tensor(0.5714, grad_fn=<MaxBackward1>),\n",
       " tensor(0.5667, grad_fn=<MaxBackward1>),\n",
       " tensor(0.1090, grad_fn=<MaxBackward1>),\n",
       " tensor(0.0080, grad_fn=<MaxBackward1>)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the maximum of the model parameters\n",
    "def print_maximum_weights(model):\n",
    "    print(\"Maximum of the model parameters per layer\")#\n",
    "    weights = []\n",
    "    for p in model.parameters():\n",
    "        print(torch.max(p))\n",
    "        weights.append(torch.max(p))\n",
    "    return weights\n",
    "\n",
    "def print_maximum_grads(model):\n",
    "    print(\"Maximum of the model gradients per layer\")#\n",
    "    max_grads = []\n",
    "    for p in model.parameters():\n",
    "        if p.grad is not None:\n",
    "            print(torch.max(p.grad))\n",
    "            max_grads.append(torch.max(p.grad))\n",
    "    return max_grads\n",
    "\n",
    "def print_target_value_weights(loss_module):\n",
    "    with loss_module.target_value_network_params.to_module(loss_module.value_network):\n",
    "        return print_maximum_weights(loss_module.value_network)    \n",
    "\n",
    "def print_value_weights(loss_module):\n",
    "    with loss_module.value_network_params.to_module(loss_module.value_network):\n",
    "        return print_maximum_weights(loss_module.value_network)\n",
    "        # print_maximum_grads(loss_module.value_network)    \n",
    "\n",
    "def print_value_grads(loss_module):\n",
    "    with loss_module.value_network_params.to_module(loss_module.value_network):\n",
    "        # print_maximum_weights(loss_module.value_network)\n",
    "        return print_maximum_grads(loss_module.value_network)   \n",
    "\n",
    "# Print the maximum of the model parameters\n",
    "print_target_value_weights(loss_module)\n",
    "\n",
    "print_value_weights(loss_module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torchviz/dot.py:65: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if LooseVersion(torch.__version__) < LooseVersion(\"1.9\") and \\\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"340pt\" height=\"468pt\"\n",
       " viewBox=\"0.00 0.00 340.00 468.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 464)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-464 336,-464 336,4 -4,4\"/>\n",
       "<!-- 139726566625584 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>139726566625584</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"193.5,-31 139.5,-31 139.5,0 193.5,0 193.5,-31\"/>\n",
       "<text text-anchor=\"middle\" x=\"166.5\" y=\"-7\" font-family=\"monospace\" font-size=\"10.00\"> ()</text>\n",
       "</g>\n",
       "<!-- 139726561649728 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>139726561649728</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"223,-86 110,-86 110,-67 223,-67 223,-86\"/>\n",
       "<text text-anchor=\"middle\" x=\"166.5\" y=\"-74\" font-family=\"monospace\" font-size=\"10.00\">MseLossBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649728&#45;&gt;139726566625584 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>139726561649728&#45;&gt;139726566625584</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M166.5,-66.79C166.5,-60.07 166.5,-50.4 166.5,-41.34\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"170,-41.19 166.5,-31.19 163,-41.19 170,-41.19\"/>\n",
       "</g>\n",
       "<!-- 139726561649920 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>139726561649920</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"217,-141 116,-141 116,-122 217,-122 217,-141\"/>\n",
       "<text text-anchor=\"middle\" x=\"166.5\" y=\"-129\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649920&#45;&gt;139726561649728 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>139726561649920&#45;&gt;139726561649728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M166.5,-121.75C166.5,-114.8 166.5,-104.85 166.5,-96.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"170,-96.09 166.5,-86.09 163,-96.09 170,-96.09\"/>\n",
       "</g>\n",
       "<!-- 139726561649680 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>139726561649680</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"101,-196 0,-196 0,-177 101,-177 101,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 139726561649680&#45;&gt;139726561649920 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>139726561649680&#45;&gt;139726561649920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.14,-176.98C87.8,-168.46 116.75,-155.23 138.24,-145.41\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"139.88,-148.51 147.52,-141.17 136.97,-142.14 139.88,-148.51\"/>\n",
       "</g>\n",
       "<!-- 139726566625200 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>139726566625200</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"83,-262 18,-262 18,-232 83,-232 83,-262\"/>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-250\" font-family=\"monospace\" font-size=\"10.00\">fc2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"50.5\" y=\"-239\" font-family=\"monospace\" font-size=\"10.00\"> (2)</text>\n",
       "</g>\n",
       "<!-- 139726566625200&#45;&gt;139726561649680 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>139726566625200&#45;&gt;139726561649680</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.5,-231.84C50.5,-224.21 50.5,-214.7 50.5,-206.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54,-206.27 50.5,-196.27 47,-206.27 54,-206.27\"/>\n",
       "</g>\n",
       "<!-- 139726561649824 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>139726561649824</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"214,-196 119,-196 119,-177 214,-177 214,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"166.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">ReluBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649824&#45;&gt;139726561649920 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>139726561649824&#45;&gt;139726561649920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M166.5,-176.75C166.5,-169.8 166.5,-159.85 166.5,-151.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"170,-151.09 166.5,-141.09 163,-151.09 170,-151.09\"/>\n",
       "</g>\n",
       "<!-- 139726561649632 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>139726561649632</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"213,-256.5 112,-256.5 112,-237.5 213,-237.5 213,-256.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"162.5\" y=\"-244.5\" font-family=\"monospace\" font-size=\"10.00\">AddmmBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649632&#45;&gt;139726561649824 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>139726561649632&#45;&gt;139726561649824</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.09,-237.37C163.65,-229.25 164.5,-216.81 165.21,-206.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"168.72,-206.38 165.91,-196.17 161.73,-205.91 168.72,-206.38\"/>\n",
       "</g>\n",
       "<!-- 139726561649296 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>139726561649296</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"118,-322.5 17,-322.5 17,-303.5 118,-303.5 118,-322.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-310.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 139726561649296&#45;&gt;139726561649632 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>139726561649296&#45;&gt;139726561649632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M80.31,-303.37C95.89,-292.87 122.43,-275 141.12,-262.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"143.25,-265.19 149.59,-256.7 139.34,-259.38 143.25,-265.19\"/>\n",
       "</g>\n",
       "<!-- 139726566625008 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>139726566625008</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"100,-394 35,-394 35,-364 100,-364 100,-394\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-382\" font-family=\"monospace\" font-size=\"10.00\">fc1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-371\" font-family=\"monospace\" font-size=\"10.00\"> (5)</text>\n",
       "</g>\n",
       "<!-- 139726566625008&#45;&gt;139726561649296 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>139726566625008&#45;&gt;139726561649296</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M67.5,-363.8C67.5,-354.7 67.5,-342.79 67.5,-332.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"71,-332.84 67.5,-322.84 64,-332.84 71,-332.84\"/>\n",
       "</g>\n",
       "<!-- 139726561649344 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>139726561649344</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"213,-322.5 136,-322.5 136,-303.5 213,-303.5 213,-322.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"174.5\" y=\"-310.5\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649344&#45;&gt;139726561649632 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>139726561649344&#45;&gt;139726561649632</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M172.88,-303.37C171.14,-294.07 168.31,-278.98 166.04,-266.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.45,-266.09 164.17,-256.91 162.57,-267.38 169.45,-266.09\"/>\n",
       "</g>\n",
       "<!-- 139726561649248 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>139726561649248</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"225,-388.5 124,-388.5 124,-369.5 225,-369.5 225,-388.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"174.5\" y=\"-376.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 139726561649248&#45;&gt;139726561649344 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>139726561649248&#45;&gt;139726561649344</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M174.5,-369.37C174.5,-360.16 174.5,-345.29 174.5,-333.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"178,-332.91 174.5,-322.91 171,-332.91 178,-332.91\"/>\n",
       "</g>\n",
       "<!-- 139726566624912 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>139726566624912</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"213,-460 136,-460 136,-430 213,-430 213,-460\"/>\n",
       "<text text-anchor=\"middle\" x=\"174.5\" y=\"-448\" font-family=\"monospace\" font-size=\"10.00\">fc1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"174.5\" y=\"-437\" font-family=\"monospace\" font-size=\"10.00\"> (5, 10)</text>\n",
       "</g>\n",
       "<!-- 139726566624912&#45;&gt;139726561649248 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>139726566624912&#45;&gt;139726561649248</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M174.5,-429.8C174.5,-420.7 174.5,-408.79 174.5,-398.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"178,-398.84 174.5,-388.84 171,-398.84 178,-398.84\"/>\n",
       "</g>\n",
       "<!-- 139726561649968 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>139726561649968</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"313,-196 236,-196 236,-177 313,-177 313,-196\"/>\n",
       "<text text-anchor=\"middle\" x=\"274.5\" y=\"-184\" font-family=\"monospace\" font-size=\"10.00\">TBackward0</text>\n",
       "</g>\n",
       "<!-- 139726561649968&#45;&gt;139726561649920 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>139726561649968&#45;&gt;139726561649920</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M257.15,-176.98C239.93,-168.54 213.3,-155.47 193.35,-145.68\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.69,-142.43 184.17,-141.17 191.6,-148.72 194.69,-142.43\"/>\n",
       "</g>\n",
       "<!-- 139726561649200 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>139726561649200</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"332,-256.5 231,-256.5 231,-237.5 332,-237.5 332,-256.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"281.5\" y=\"-244.5\" font-family=\"monospace\" font-size=\"10.00\">AccumulateGrad</text>\n",
       "</g>\n",
       "<!-- 139726561649200&#45;&gt;139726561649968 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>139726561649200&#45;&gt;139726561649968</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M280.47,-237.37C279.5,-229.25 278.01,-216.81 276.76,-206.39\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"280.2,-205.68 275.54,-196.17 273.25,-206.51 280.2,-205.68\"/>\n",
       "</g>\n",
       "<!-- 139726566625104 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>139726566625104</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"320,-328 243,-328 243,-298 320,-298 320,-328\"/>\n",
       "<text text-anchor=\"middle\" x=\"281.5\" y=\"-316\" font-family=\"monospace\" font-size=\"10.00\">fc2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"281.5\" y=\"-305\" font-family=\"monospace\" font-size=\"10.00\"> (2, 5)</text>\n",
       "</g>\n",
       "<!-- 139726566625104&#45;&gt;139726561649200 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>139726566625104&#45;&gt;139726561649200</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M281.5,-297.8C281.5,-288.7 281.5,-276.79 281.5,-266.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"285,-266.84 281.5,-256.84 278,-266.84 285,-266.84\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f14a0127d50>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchviz import make_dot\n",
    "\n",
    "# Define a simple neural network with two outputs\n",
    "class SimpleNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(10, 5)\n",
    "        self.fc2 = nn.Linear(5, 2)  # Two outputs\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "# Create the network and some input data\n",
    "net = SimpleNet()\n",
    "x = torch.randn(1, 10)\n",
    "target = torch.randn(1, 2)  # Dummy target for the loss function\n",
    "\n",
    "# Perform the forward pass\n",
    "outputs = net(x)\n",
    "\n",
    "# Define a loss function\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "# Compute the loss\n",
    "loss = criterion(outputs, target)\n",
    "\n",
    "# Perform backpropagation\n",
    "loss.backward()\n",
    "\n",
    "# Visualize the computational graph using torchviz\n",
    "make_dot(loss, params=dict(net.named_parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "Maximum of the model gradients per layer\n",
      "tensor(0.0024)\n",
      "tensor(0.0056)\n",
      "tensor(0.0090)\n",
      "tensor(0.0161)\n",
      "tensor(0.0317)\n",
      "tensor(0.0813)\n",
      "tensor(0.0212)\n",
      "tensor(0.1943)\n",
      "tensor(0.)\n",
      "tensor(-1.0666)\n",
      "Maximum of the model gradients per layer\n",
      "tensor(0.0024)\n",
      "tensor(0.0056)\n",
      "tensor(0.0090)\n",
      "tensor(0.0161)\n",
      "tensor(0.0317)\n",
      "tensor(0.0813)\n",
      "tensor(0.0212)\n",
      "tensor(0.1943)\n",
      "tensor(0.)\n",
      "tensor(-1.0666)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Main loop\n",
    "collected_frames = 0\n",
    "total_episodes = 0\n",
    "start_time = time.time()\n",
    "num_updates = cfg.loss.num_updates\n",
    "batch_size = cfg.buffer.batch_size\n",
    "test_interval = cfg.logger.test_interval\n",
    "num_test_episodes = cfg.logger.num_test_episodes\n",
    "frames_per_batch = cfg.collector.frames_per_batch\n",
    "# pbar = tqdm.tqdm(total=cfg.collector.total_frames)\n",
    "init_random_frames = cfg.collector.init_random_frames\n",
    "sampling_start = time.time()\n",
    "q_losses = torch.zeros(num_updates) #, device=device)\n",
    "\n",
    "for i, data in enumerate(collector):\n",
    "\n",
    "        # NOTE: This reshape must be for frame data (maybe)\n",
    "        data = data.reshape(-1)\n",
    "        current_frames = data.numel()\n",
    "        replay_buffer.extend(data)\n",
    "        collected_frames += current_frames\n",
    "        greedy_module.step(current_frames)\n",
    "\n",
    "        # Get the number of episodes\n",
    "        total_episodes += data[\"next\", \"done\"].sum()\n",
    "\n",
    "        # Get and log training rewards and episode lengths\n",
    "        # Collect the episode rewards and lengths in average over the\n",
    "        # transitions in the current data batch\n",
    "        episode_rewards = data[\"next\", \"episode_reward\"][data[\"next\", \"done\"]]\n",
    "\n",
    "\n",
    "        # Warmup phase (due to the continue statement)\n",
    "        # Additionally This help us to keep a track of the collected_frames\n",
    "        # after the init_random_frames\n",
    "        if collected_frames < init_random_frames:\n",
    "            continue\n",
    "\n",
    "        # optimization steps\n",
    "        training_start = time.time()\n",
    "        for j in range(num_updates):\n",
    "            sampled_tensordict = replay_buffer.sample(batch_size)\n",
    "            # TODO: check if the sample is already in the device\n",
    "            sampled_tensordict = sampled_tensordict #.to(device)\n",
    "\n",
    "            # Also the loss module will use the current and target model to get the q-values\n",
    "            loss_td = loss_module(sampled_tensordict)\n",
    "            q_loss = loss_td[\"loss\"]\n",
    "\n",
    "            # with loss_module.value_network_params.to_module(loss_module.value_network):\n",
    "            #     dot = make_dot(q_loss, params=dict(loss_module.value_network.named_parameters()), show_attrs=True, show_saved=True)\n",
    "            #     dot.render(\"computational_graph_with_loss\", format=\"png\")\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            q_loss.backward()\n",
    "\n",
    "            # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=2.0)\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "            # Update the priorities\n",
    "            if cfg.buffer.prioritized_replay:\n",
    "                replay_buffer.update_priority(index=sampled_tensordict['index'], priority = sampled_tensordict['td_error'])\n",
    "\n",
    "            # NOTE: This is only one step (after n-updated steps defined before)\n",
    "            # the target will update\n",
    "            target_net_updater.step()\n",
    "            q_losses[j].copy_(q_loss.detach())\n",
    "        training_time = time.time() - training_start\n",
    "\n",
    "        # Get and log evaluation rewards and eval time\n",
    "        # NOTE: As I'm using only the model and not the model_explore that will deterministic I think\n",
    "        # with torch.no_grad(): #, set_exploration_type(ExplorationType.DETERMINISTIC):\n",
    "\n",
    "        #     # NOTE: Check how we are using the frames here because it seems that I am dividing \n",
    "        #     # 10 for 50000\n",
    "        #     prev_test_frame = ((i - 1) * frames_per_batch) // test_interval\n",
    "        #     cur_test_frame = (i * frames_per_batch) // test_interval\n",
    "        #     final = current_frames >= collector.total_frames\n",
    "\n",
    "        #     # compara prev_test_frame < cur_test_frame is the same as current_frames % test_interval == 0\n",
    "        #     if (i >= 1 and (prev_test_frame < cur_test_frame)) or final:\n",
    "        #         model.eval()\n",
    "        #         eval_start = time.time()\n",
    "        #         test_rewards = eval_model(model, test_env, num_test_episodes)\n",
    "        #         eval_time = time.time() - eval_start\n",
    "        #         model.train()\n",
    "        #         log_info.update(\n",
    "        #             {\n",
    "        #                 \"eval/reward\": test_rewards,\n",
    "        #                 \"eval/eval_time\": eval_time,\n",
    "        #             }\n",
    "        #         )\n",
    "\n",
    "        # Log all the information\n",
    "\n",
    "        # update weights of the inference policy\n",
    "        # NOTE: Updates the policy weights if the policy of the data \n",
    "        # collector and the trained policy live on different devices.\n",
    "        collector.update_policy_weights_()\n",
    "        sampling_start = time.time()\n",
    "\n",
    "collector.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = DQNLoss(value_network=policy, \n",
    "               action_space=env.action_spec, \n",
    "               delay_value=True) # delay_value=True means we will use a target network\n",
    "optim = Adam(loss.parameters(), lr=0.02)\n",
    "\n",
    "# eps: will be used to update the target network as \n",
    "# \\theta_t = \\theta_{t-1} * \\epsilon + \\theta_t * (1-\\epsilon)\n",
    "# where eps = 1 is hard update\n",
    "updater = SoftUpdate(loss, eps=0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictParams(params=TensorDict(\n",
       "    fields={\n",
       "        module: TensorDict(\n",
       "            fields={\n",
       "                0: TensorDict(\n",
       "                    fields={\n",
       "                        module: TensorDict(\n",
       "                            fields={\n",
       "                                activation: TensorDict(\n",
       "                                    fields={\n",
       "                                    },\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False),\n",
       "                                encoder: TensorDict(\n",
       "                                    fields={\n",
       "                                        0: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([64, 4]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False),\n",
       "                                        1: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([3, 64]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False)},\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False),\n",
       "                                q_net: TensorDict(\n",
       "                                    fields={\n",
       "                                        0: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([64, 3]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False),\n",
       "                                        1: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([2, 64]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False)},\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False)},\n",
       "                            batch_size=torch.Size([]),\n",
       "                            device=None,\n",
       "                            is_shared=False)},\n",
       "                    batch_size=torch.Size([]),\n",
       "                    device=None,\n",
       "                    is_shared=False),\n",
       "                1: TensorDict(\n",
       "                    fields={\n",
       "                    },\n",
       "                    batch_size=torch.Size([]),\n",
       "                    device=None,\n",
       "                    is_shared=False)},\n",
       "            batch_size=torch.Size([]),\n",
       "            device=None,\n",
       "            is_shared=False)},\n",
       "    batch_size=torch.Size([]),\n",
       "    device=None,\n",
       "    is_shared=False))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.value_network_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictParams(params=TensorDict(\n",
       "    fields={\n",
       "        module: TensorDict(\n",
       "            fields={\n",
       "                0: TensorDict(\n",
       "                    fields={\n",
       "                        module: TensorDict(\n",
       "                            fields={\n",
       "                                activation: TensorDict(\n",
       "                                    fields={\n",
       "                                    },\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False),\n",
       "                                encoder: TensorDict(\n",
       "                                    fields={\n",
       "                                        0: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([64, 4]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False),\n",
       "                                        1: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([3, 64]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False)},\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False),\n",
       "                                q_net: TensorDict(\n",
       "                                    fields={\n",
       "                                        0: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([64]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([64, 3]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False),\n",
       "                                        1: TensorDict(\n",
       "                                            fields={\n",
       "                                                bias: Parameter(shape=torch.Size([2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                                                weight: Parameter(shape=torch.Size([2, 64]), device=cpu, dtype=torch.float32, is_shared=False)},\n",
       "                                            batch_size=torch.Size([]),\n",
       "                                            device=None,\n",
       "                                            is_shared=False)},\n",
       "                                    batch_size=torch.Size([]),\n",
       "                                    device=None,\n",
       "                                    is_shared=False)},\n",
       "                            batch_size=torch.Size([]),\n",
       "                            device=None,\n",
       "                            is_shared=False)},\n",
       "                    batch_size=torch.Size([]),\n",
       "                    device=None,\n",
       "                    is_shared=False),\n",
       "                1: TensorDict(\n",
       "                    fields={\n",
       "                    },\n",
       "                    batch_size=torch.Size([]),\n",
       "                    device=None,\n",
       "                    is_shared=False)},\n",
       "            batch_size=torch.Size([]),\n",
       "            device=None,\n",
       "            is_shared=False)},\n",
       "    batch_size=torch.Size([]),\n",
       "    device=None,\n",
       "    is_shared=False))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.target_value_network_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode are grouped tensor([3, 3, 4, 4, 2, 2, 1, 1], dtype=torch.int32)\n",
      "steps are successive tensor([0, 1, 0, 1, 0, 1, 0, 1])\n"
     ]
    }
   ],
   "source": [
    "from tensordict import TensorDict\n",
    "from torchrl.data import SliceSampler\n",
    "from torchrl.data import LazyMemmapStorage\n",
    "\n",
    "rb = TensorDictReplayBuffer(\n",
    "    storage=LazyMemmapStorage(size),\n",
    "    sampler=SliceSampler(traj_key=\"episode\", num_slices=4),\n",
    "    batch_size=8,\n",
    ")\n",
    "episode = torch.zeros(10, dtype=torch.int)\n",
    "episode[:3] = 1\n",
    "episode[3:5] = 2\n",
    "episode[5:7] = 3\n",
    "episode[7:] = 4\n",
    "steps = torch.cat([torch.arange(3), torch.arange(2), torch.arange(2), torch.arange(3)])\n",
    "obs = torch.randn((3, 4, 5)).expand(10, 3, 4, 5)\n",
    "data = TensorDict(\n",
    "    {\n",
    "        \"episode\": episode,\n",
    "        \"obs\": obs,\n",
    "        \"act\": torch.randn((20,)).expand(10, 20),\n",
    "        \"other\": torch.randn((20, 50)).expand(10, 20, 50),\n",
    "        \"steps\": steps,\n",
    "    },\n",
    "    [10],\n",
    ")\n",
    "rb.extend(data)\n",
    "sample = rb.sample()\n",
    "print(\"episode are grouped\", sample[\"episode\"])\n",
    "print(\"steps are successive\", sample[\"steps\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 1, 1, 2, 2, 3, 3, 4, 4, 4], dtype=torch.int32)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDictReplayBuffer(\n",
       "    storage=LazyMemmapStorage(\n",
       "        data=TensorDict(\n",
       "            fields={\n",
       "                act: MemoryMappedTensor(shape=torch.Size([10, 20]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                episode: MemoryMappedTensor(shape=torch.Size([10]), device=cpu, dtype=torch.int32, is_shared=False),\n",
       "                index: MemoryMappedTensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                obs: MemoryMappedTensor(shape=torch.Size([10, 3, 4, 5]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                other: MemoryMappedTensor(shape=torch.Size([10, 20, 50]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                steps: MemoryMappedTensor(shape=torch.Size([10]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([10]),\n",
       "            device=cpu,\n",
       "            is_shared=False), \n",
       "        shape=torch.Size([10]), \n",
       "        len=10, \n",
       "        max_size=100), \n",
       "    sampler=SliceSampler(num_slices=4, slice_len=None, end_key=('next', 'done'), traj_key=episode, truncated_key=('next', 'truncated'), strict_length=True), \n",
       "    writer=TensorDictRoundRobinWriter(cursor=10, full_storage=False), \n",
       "    batch_size=8, \n",
       "    collate_fn=<function _collate_id at 0x7f7435503c40>)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2024_07_23-17_34_50'"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import datetime\n",
    "\n",
    "current_date = datetime.datetime.now()\n",
    "date_str = current_date.strftime(\"%Y_%m_%d-%H_%M_%S\")  # Includes date and time\n",
    "date_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zosov/anaconda3/envs/final-project/lib/python3.11/site-packages/torchrl/data/replay_buffers/replay_buffers.py:585: UserWarning: Got conflicting batch_sizes in constructor (32) and `sample` (128). Refer to the ReplayBuffer documentation for a proper usage of the batch-size arguments. The batch-size provided to the sample method will prevail.\n",
      "  warnings.warn(\n",
      "2024-07-23 17:34:53,433 [torchrl][INFO] solved after 0 steps, 0 episodes and in 2.57519268989563s.\n"
     ]
    }
   ],
   "source": [
    "total_count = 0\n",
    "total_episodes = 0\n",
    "t0 = time.time()\n",
    "for i, data in enumerate(collector):\n",
    "    # Write data in replay buffer\n",
    "    rb.extend(data)\n",
    "    max_length = rb[:][\"next\", \"step_count\"].max() # From all the next steps get the max step count\n",
    "    if len(rb) > init_rand_steps: # wam-up steps\n",
    "        # Optim loop (we do several optim steps\n",
    "        # per batch collected for efficiency)\n",
    "        for _ in range(optim_steps):\n",
    "            sample = rb.sample(128) # sample a batch of 128 (repetition is allowed)\n",
    "            # print(sample)\n",
    "            break\n",
    "            loss_vals = loss(sample)\n",
    "            loss_vals[\"loss\"].backward()\n",
    "            optim.step()\n",
    "            optim.zero_grad()\n",
    "            # Update exploration factor\n",
    "            # NOTE: Why I am updating the exploration factor here? \n",
    "            # I'm considering practically that I did 100 (or n) iteractions in the environment time optim_steps\n",
    "            exploration_module.step(data.numel()) # data.numel() returns the number of elements in the data\n",
    "            # Update target params each optimisation step\n",
    "            updater.step()\n",
    "            if i % 10:\n",
    "                torchrl_logger.info(f\"Max num steps: {max_length}, rb length {len(rb)}\")\n",
    "            total_count += data.numel()\n",
    "            total_episodes += data[\"next\", \"done\"].sum() # sum the number of done episodes\n",
    "    \n",
    "    if max_length > 200:\n",
    "        break\n",
    "\n",
    "t1 = time.time()\n",
    "\n",
    "torchrl_logger.info(\n",
    "    f\"solved after {total_count} steps, {total_episodes} episodes and in {t1-t0}s.\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        _weight: Tensor(shape=torch.Size([]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        action: Tensor(shape=torch.Size([2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        index: Tensor(shape=torch.Size([]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorDict(\n",
       "    fields={\n",
       "        _weight: Tensor(shape=torch.Size([128]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        action: Tensor(shape=torch.Size([128, 2]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        action_value: Tensor(shape=torch.Size([128, 2]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        chosen_action_value: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        collector: TensorDict(\n",
       "            fields={\n",
       "                traj_ids: Tensor(shape=torch.Size([128]), device=cpu, dtype=torch.int64, is_shared=False)},\n",
       "            batch_size=torch.Size([128]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        done: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        index: Tensor(shape=torch.Size([128]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        next: TensorDict(\n",
       "            fields={\n",
       "                done: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                observation: Tensor(shape=torch.Size([128, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                reward: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "                step_count: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "                terminated: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "                truncated: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "            batch_size=torch.Size([128]),\n",
       "            device=cpu,\n",
       "            is_shared=False),\n",
       "        observation: Tensor(shape=torch.Size([128, 4]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        representation: Tensor(shape=torch.Size([128, 3]), device=cpu, dtype=torch.float32, is_shared=False),\n",
       "        step_count: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.int64, is_shared=False),\n",
       "        terminated: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False),\n",
       "        truncated: Tensor(shape=torch.Size([128, 1]), device=cpu, dtype=torch.bool, is_shared=False)},\n",
       "    batch_size=torch.Size([128]),\n",
       "    device=cpu,\n",
       "    is_shared=False)"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "record_env.rollout(max_steps=1000, policy=policy)\n",
    "video_recorder.dump()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[118398, 676190, 786456, 171936, 887739, 919409, 711872, 442081, 189061, 117840]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "# Generate and print 10 random seeds\n",
    "random_seeds = [random.randint(0, 1000000) for _ in range(10)]\n",
    "print(random_seeds)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "final-project",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}