test_dqn_mlp_total.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from test_dqn_mlp_total import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ft06\n",
      "la01\n",
      "la06\n",
      "la11\n",
      "la21\n",
      "la31\n",
      "la36\n",
      "orb01\n",
      "swv01\n",
      "swv06\n",
      "swv11\n",
      "yn1\n"
     ]
    }
   ],
   "source": [
    "all_makespans = []\n",
    "for instance in instances:\n",
    "    print(instance)\n",
    "    env = HeuristicJsspEnv(instance)\n",
    "    agent = load_agent(instance=instance)\n",
    "    makespans = []\n",
    "    makespans.append(get_makespan_of_agent_policy(env=env,agent=agent))\n",
    "    makespans.append(get_makespan_of_random_policy(env=env,num_simulations=10))\n",
    "    for rule in range(8):\n",
    "        makespans.append(get_makespan_of_heuristic_rule(env=env,rule=rule))\n",
    "    all_makespans.append(makespans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "policies = ['agent', 'random', 'fifo', 'lifo', 'lpt', 'spt', 'ltpt', 'stpt','mor','lor']\n",
    "df = pd.DataFrame(data = all_makespans,\n",
    "\t\t\t\tindex = instances,\n",
    "\t\t\t\tcolumns = policies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>agent</th>\n",
       "      <th>random</th>\n",
       "      <th>fifo</th>\n",
       "      <th>lifo</th>\n",
       "      <th>lpt</th>\n",
       "      <th>spt</th>\n",
       "      <th>ltpt</th>\n",
       "      <th>stpt</th>\n",
       "      <th>mor</th>\n",
       "      <th>lor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ft06</th>\n",
       "      <td>65</td>\n",
       "      <td>68.4</td>\n",
       "      <td>68</td>\n",
       "      <td>70</td>\n",
       "      <td>77</td>\n",
       "      <td>88</td>\n",
       "      <td>68</td>\n",
       "      <td>83</td>\n",
       "      <td>59</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la01</th>\n",
       "      <td>675</td>\n",
       "      <td>801.1</td>\n",
       "      <td>830</td>\n",
       "      <td>764</td>\n",
       "      <td>822</td>\n",
       "      <td>751</td>\n",
       "      <td>835</td>\n",
       "      <td>933</td>\n",
       "      <td>763</td>\n",
       "      <td>941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la06</th>\n",
       "      <td>967</td>\n",
       "      <td>1054.0</td>\n",
       "      <td>1078</td>\n",
       "      <td>1031</td>\n",
       "      <td>1125</td>\n",
       "      <td>1200</td>\n",
       "      <td>1098</td>\n",
       "      <td>1012</td>\n",
       "      <td>926</td>\n",
       "      <td>1095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la11</th>\n",
       "      <td>1259</td>\n",
       "      <td>1361.6</td>\n",
       "      <td>1577</td>\n",
       "      <td>1580</td>\n",
       "      <td>1467</td>\n",
       "      <td>1473</td>\n",
       "      <td>1416</td>\n",
       "      <td>1446</td>\n",
       "      <td>1222</td>\n",
       "      <td>1586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la21</th>\n",
       "      <td>1234</td>\n",
       "      <td>1325.8</td>\n",
       "      <td>1417</td>\n",
       "      <td>1479</td>\n",
       "      <td>1451</td>\n",
       "      <td>1324</td>\n",
       "      <td>1278</td>\n",
       "      <td>1541</td>\n",
       "      <td>1251</td>\n",
       "      <td>1547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la31</th>\n",
       "      <td>1819</td>\n",
       "      <td>2031.0</td>\n",
       "      <td>2148</td>\n",
       "      <td>2256</td>\n",
       "      <td>2245</td>\n",
       "      <td>1951</td>\n",
       "      <td>2083</td>\n",
       "      <td>2270</td>\n",
       "      <td>1836</td>\n",
       "      <td>2129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>la36</th>\n",
       "      <td>1453</td>\n",
       "      <td>1690.5</td>\n",
       "      <td>1676</td>\n",
       "      <td>1831</td>\n",
       "      <td>1737</td>\n",
       "      <td>1799</td>\n",
       "      <td>1750</td>\n",
       "      <td>1894</td>\n",
       "      <td>1492</td>\n",
       "      <td>1932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orb01</th>\n",
       "      <td>1139</td>\n",
       "      <td>1375.4</td>\n",
       "      <td>1456</td>\n",
       "      <td>1495</td>\n",
       "      <td>1410</td>\n",
       "      <td>1478</td>\n",
       "      <td>1308</td>\n",
       "      <td>1458</td>\n",
       "      <td>1307</td>\n",
       "      <td>1410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swv01</th>\n",
       "      <td>1635</td>\n",
       "      <td>1980.4</td>\n",
       "      <td>1889</td>\n",
       "      <td>2123</td>\n",
       "      <td>2145</td>\n",
       "      <td>1737</td>\n",
       "      <td>1961</td>\n",
       "      <td>1751</td>\n",
       "      <td>1971</td>\n",
       "      <td>1838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swv06</th>\n",
       "      <td>2077</td>\n",
       "      <td>2369.8</td>\n",
       "      <td>2243</td>\n",
       "      <td>2331</td>\n",
       "      <td>2542</td>\n",
       "      <td>2140</td>\n",
       "      <td>2327</td>\n",
       "      <td>2360</td>\n",
       "      <td>2287</td>\n",
       "      <td>2383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swv11</th>\n",
       "      <td>3504</td>\n",
       "      <td>4133.2</td>\n",
       "      <td>3808</td>\n",
       "      <td>3744</td>\n",
       "      <td>4763</td>\n",
       "      <td>3714</td>\n",
       "      <td>3928</td>\n",
       "      <td>3834</td>\n",
       "      <td>4642</td>\n",
       "      <td>4029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yn1</th>\n",
       "      <td>1017</td>\n",
       "      <td>1142.4</td>\n",
       "      <td>1123</td>\n",
       "      <td>1177</td>\n",
       "      <td>1115</td>\n",
       "      <td>1196</td>\n",
       "      <td>1177</td>\n",
       "      <td>1188</td>\n",
       "      <td>1045</td>\n",
       "      <td>1205</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       agent  random  fifo  lifo   lpt   spt  ltpt  stpt   mor   lor\n",
       "ft06      65    68.4    68    70    77    88    68    83    59    68\n",
       "la01     675   801.1   830   764   822   751   835   933   763   941\n",
       "la06     967  1054.0  1078  1031  1125  1200  1098  1012   926  1095\n",
       "la11    1259  1361.6  1577  1580  1467  1473  1416  1446  1222  1586\n",
       "la21    1234  1325.8  1417  1479  1451  1324  1278  1541  1251  1547\n",
       "la31    1819  2031.0  2148  2256  2245  1951  2083  2270  1836  2129\n",
       "la36    1453  1690.5  1676  1831  1737  1799  1750  1894  1492  1932\n",
       "orb01   1139  1375.4  1456  1495  1410  1478  1308  1458  1307  1410\n",
       "swv01   1635  1980.4  1889  2123  2145  1737  1961  1751  1971  1838\n",
       "swv06   2077  2369.8  2243  2331  2542  2140  2327  2360  2287  2383\n",
       "swv11   3504  4133.2  3808  3744  4763  3714  3928  3834  4642  4029\n",
       "yn1     1017  1142.4  1123  1177  1115  1196  1177  1188  1045  1205"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"test_dqn_staic.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from rl.agent import DQNAgent\n",
    "from gymjsp.jsspenv import HeuristicJsspEnv\n",
    "from ortools_scheduler import ORtools_scheduler\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from copy import deepcopy\n",
    "\n",
    "num_episodes = 1000\n",
    "memory_size = 100000\n",
    "batch_size = 64\n",
    "target_update = 100\n",
    "noisy = False\n",
    "plotting_inteval = 10\n",
    "\n",
    "instances = [\"ft06\", \"la01\", \"la06\", \"la11\", \"la21\", \"la31\", \"la36\", \"orb01\", \"swv01\", \"swv06\", \"swv11\", \"yn1\"]\n",
    "# num_episodes of swv11 is actually 100, changed to 1000 for implementation simplicity\n",
    "\n",
    "def load_agent(instance):\n",
    "    policy_file = f\"policies/dqn_mlp/{instance}_num_episodes={num_episodes}_memory_size={memory_size}_target_update={target_update}_noisy={noisy}_cycle=8.pth\"\n",
    "    env = HeuristicJsspEnv(instance)\n",
    "    agent = DQNAgent(env, memory_size, batch_size, target_update, noisy=noisy)\n",
    "    agent.load_dqn(policy_file)\n",
    "    return agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59\n",
      "675\n",
      "938\n",
      "1222\n",
      "1221\n",
      "1817\n",
      "1424\n",
      "1134\n",
      "1668\n",
      "2077\n",
      "3403\n",
      "1045\n"
     ]
    }
   ],
   "source": [
    "l = []\n",
    "for instance in instances:\n",
    "    agent = load_agent(instance=instance)\n",
    "    x = agent.test()\n",
    "    print(x)\n",
    "    l.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"test_dqn_staic.csv\")\n",
    "l_ = df[\"agent\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0       65\n",
      "1      675\n",
      "2      967\n",
      "3     1259\n",
      "4     1234\n",
      "5     1819\n",
      "6     1453\n",
      "7     1139\n",
      "8     1635\n",
      "9     2077\n",
      "10    3504\n",
      "11    1017\n",
      "Name: agent, dtype: int64\n",
      "[59, 675, 938, 1222, 1221, 1817, 1424, 1134, 1668, 2077, 3403, 1045]\n"
     ]
    }
   ],
   "source": [
    "print(l_)\n",
    "print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrMUlEQVR4nO3dd3hUZd7G8e+ZSSckoaXRe+hVMRZERSIGbNgREVFXN7gCrvCyuthWUewNFV0FV1HRFVdBwUhTqhANndBJKEmAkEx6mTnvH0NGoqAEEk4muT/XNZfMOc/M3DMB55fzNMM0TRMRERERL2KzOoCIiIhIZamAEREREa+jAkZERES8jgoYERER8ToqYERERMTrqIARERERr6MCRkRERLyOChgRERHxOj5WB6guLpeLAwcOUL9+fQzDsDqOiIiInALTNMnNzSU6Ohqb7eTXWWptAXPgwAGaN29udQwRERE5DWlpaTRr1uyk52ttAVO/fn3A/QGEhIRYnEZEREROhcPhoHnz5p7v8ZOptQVMebdRSEiIChgREREv82fDPzSIV0RERLyOChgRERHxOipgRERExOvU2jEwp8I0TcrKynA6nVZHqZPsdjs+Pj6a5i4iIpVWZwuYkpISDh48SEFBgdVR6rSgoCCioqLw8/OzOoqIiHiROlnAuFwudu/ejd1uJzo6Gj8/P10FOMtM06SkpIRDhw6xe/du2rdv/4cLFomIiByvThYwJSUluFwumjdvTlBQkNVx6qzAwEB8fX3Zu3cvJSUlBAQEWB1JRES8RJ3+lVe/8VtPPwMRETkd+vYQERERr6MCRkRERLxOpQqYN998k+7du3uW54+NjeXbb7/1nB8wYACGYVS43XvvvRWeIzU1lfj4eIKCgggPD+ehhx6irKysQpslS5bQu3dv/P39adeuHTNmzDj9dyjVwjAMvvzyywrHDh48yK233kqHDh2w2WyMHTvWkmwiIlL7VaqAadasGc888wxJSUmsXbuWSy+9lKuvvppNmzZ52tx9990cPHjQc5s6darnnNPpJD4+npKSElasWMHMmTOZMWMGkydP9rTZvXs38fHxXHLJJSQnJzN27FjuuusuFixYUAVvV6pTcXExTZo04ZFHHqFHjx5WxxERkdrMPEMNGjQw3333XdM0TfPiiy82H3jggZO2/eabb0ybzWamp6d7jr355ptmSEiIWVxcbJqmaU6YMMHs0qVLhcfddNNNZlxcXKVy5eTkmICZk5Pzu3OFhYXm5s2bzcLCwko9Z03w7bffmhdccIEZGhpqNmzY0IyPjzd37NjhOb98+XKzR48epr+/v9mnTx9zzpw5JmD+8ssvnjYbNmwwr7jiCrNevXpmeHi4edttt5mHDh3ynL/44ovN+++/33zooYfMBg0amBEREeajjz7qOd+yZUsT8Nxatmz5u5x/9nehnDf/LEREqsPOzFzzvWW7zDKny+oolvij7+/jnfYYGKfTySeffEJ+fj6xsbGe4x999BGNGzema9euTJo0qcJCcStXrqRbt25ERER4jsXFxeFwODxXcVauXMnAgQMrvFZcXBwrV678wzzFxcU4HI4Kt1NlmiYFJWWW3EzTPOWcAPn5+YwfP561a9eycOFCbDYb1157LS6XC4fDwdChQ+nWrRs///wzTz75JBMnTqzw+OzsbC699FJ69erF2rVrmT9/PhkZGdx4440V2s2cOZN69eqxevVqpk6dyhNPPEFiYiIAa9asAeD999/n4MGDnvsiInLm/v7ZOh7/ejNz1x+wOkqNVul1YDZs2EBsbCxFRUUEBwczZ84cOnfuDMCtt95Ky5YtiY6OZv369UycOJGUlBS++OILANLT0ysUL4Dnfnp6+h+2cTgcFBYWEhgYeMJcU6ZM4fHHH6/s2wGgsNRJ58nWdFFtfiKOIL9T/zEMGzaswv333nuPJk2asHnzZpYtW4ZhGLzzzjsEBATQuXNn9u/fz9133+1p//rrr9OrVy+efvrpCs/RvHlztm3bRocOHQDo3r07jz76KADt27fn9ddfZ+HChVx++eU0adIEgLCwMCIjI0/7vYuISEWH84r5JS0bgM0HHFzds6m1gWqwShcwHTt2JDk5mZycHD7//HNGjhzJ0qVL6dy5M/fcc4+nXbdu3YiKiuKyyy5j586dtG3btkqD/9akSZMYP368577D4aB58+bV+ppW2L59O5MnT2b16tUcPnwYl8sFuAdHp6Sk0L179woLwp177rkVHr9u3ToWL15McHDw7557586dFQqY40VFRZGZmVnVb0dERI6zJOUQ5Rfmt2XkWhumhqt0AePn50e7du0A6NOnD2vWrOGVV17h7bff/l3bfv36AbBjxw7atm1LZGQkP/30U4U2GRkZAJ7f5CMjIz3Hjm8TEhJy0qsvAP7+/vj7+1f27QAQ6Gtn8xNxp/XYMxXoa69U+6FDh9KyZUveeecdoqOjcblcdO3alZKSklN6fF5eHkOHDuXZZ5/93bmoqCjPn319fSucMwzDUyyJiEj1WLT11++/bRl5Fiap+c54KwGXy0VxcfEJzyUnJwO/fjHGxsby1FNPkZmZSXh4OACJiYmEhIR4uqFiY2P55ptvKjxPYmJihXE2Vc0wjEp141jlyJEjpKSk8M4773DRRRcBsGzZMs/5jh078uGHH1JcXOwp5n47PqV3797897//pVWrVvj4nP579vX11S7eIiJVqKTMxY/bDnvu788uJK+4jGD/mv/9ZIVKDeKdNGkSP/zwA3v27GHDhg1MmjSJJUuWMHz4cHbu3MmTTz5JUlISe/bs4auvvuL222+nf//+nu6IQYMG0blzZ0aMGMG6detYsGABjzzyCAkJCZ4v3HvvvZddu3YxYcIEtm7dyrRp05g9ezbjxo2r+nfvZRo0aECjRo2YPn06O3bsYNGiRRW6zW699VZcLhf33HMPW7ZsYcGCBTz//PMAns0qExISyMrK4pZbbmHNmjXs3LmTBQsWMGrUqEoVJK1atWLhwoWkp6dz9OhRz/Hk5GSSk5PJy8vj0KFDJCcns3nz5ir6BEREaq+1e7LILS6jcbAfjYPd34nb1Y10UpUqYDIzM7n99tvp2LEjl112GWvWrGHBggVcfvnl+Pn58f333zNo0CBiYmJ48MEHGTZsGF9//bXn8Xa7nblz52K324mNjeW2227j9ttv54knnvC0ad26NfPmzSMxMZEePXrwwgsv8O677xIXZ00XT01is9n45JNPSEpKomvXrowbN47nnnvOcz4kJISvv/6a5ORkevbsycMPP+xZY6d8XEx0dDTLly/H6XQyaNAgunXrxtixYwkLC6vUvkQvvPACiYmJNG/enF69enmO9+rVi169epGUlMSsWbPo1asXV155ZRV9AiIitdeire5xhgM6hhMTWR+A7epGOinDrOw8Xi/hcDgIDQ0lJyeHkJCQCueKiorYvXs3rVu3rvU7IH/00UeMGjWKnJycPxxDZJW69LMQEfkjl76whF2H8pk2vDdr9mTx/vI93HVhax4Z0tnqaGfVH31/H08da7XMBx98QJs2bWjatCnr1q1j4sSJ3HjjjTWyeBEREbc9h/PZdSgfH5vBhe0bk1NYCsC2TF2BORkVMLVMeno6kydPJj09naioKG644Qaeeuopq2OJiMgfKO8+Ord1Q0LSf+KyXbMI4DK2pWsMzMmogKllJkyYwIQJE6yOISIilVBewFwaEw5zbyL88DausQfwieNScgpLCQ30/ZNnqHtOeysBEREROXN5xWWs3n0EgLjwHDi8DYC+/vsB2JGpqzAnogJGRETEQsu2H6bUadKqURDNMxZ6jnf13QdASrrGwZyIChgRERELla++e0lMOGyd6znesmwPYGpLgZNQASMiImIRl8tkccohAAY3d8KBXwADDDuBzlwiyWK7upBOSAWMiIiIRTYeyOFQbjH1/Oz0LlzhPti8HzR2b6wbY0vVnkgnoQJGRETEIuWzjy5q3wSflGPdR52GQEQX9x+NNA7lFnM0/9Q27K1LVMCIiIhYZPGxAuaKNn6w99gVmJghEOFefbdXwAEAjYM5ARUwcloMw+DLL7+scOyLL77g8ssvp0mTJoSEhBAbG8uCBQusCSgiUsNl5haxbl8OAJfaksB0QkQ3aNgaIroC0NmWCmhF3hNRASNV5ocffuDyyy/nm2++ISkpiUsuuYShQ4fyyy+/WB1NRKTGWXJs8G73ZqGE7Dn2y16nIe7/HutCiipLw5cy7Up9AipgvMz8+fO58MILCQsLo1GjRgwZMoSdO3d6zq9YsYKePXsSEBBA3759+fLLLzEMg+TkZE+bjRs3MnjwYIKDg4mIiGDEiBEcPnzYc37AgAH87W9/Y8KECTRs2JDIyEgee+wxz/lWrVoBcO2112IYhuf+yy+/zIQJEzjnnHNo3749Tz/9NO3bt6+wI7mIiLgt2uLuPhrULhh2HFv/JeZYARPSFPxDsZtO2hoHSNGWAr+jAgbANKEk35pbJTcDz8/PZ/z48axdu5aFCxdis9m49tprcblcOBwOhg4dSrdu3fj555958sknmThxYoXHZ2dnc+mll9KrVy/Wrl3L/PnzycjI4MYbb6zQbubMmdSrV4/Vq1czdepUnnjiCRITEwFYs2YNAO+//z4HDx703P8tl8tFbm4uDRs2rNR7FBGp7UrKXCzb4f7FcWi9TeAshgatPFdeMAzPn2OMVLarC+l3tBcSQGkBPB1tzWv/4wD41Tvl5sOGDatw/7333qNJkyZs3ryZZcuWYRgG77zzDgEBAXTu3Jn9+/dz9913e9q//vrr9OrVi6effrrCczRv3pxt27bRoYN76l737t159NFHAWjfvj2vv/46Cxcu9IxxAQgLCyMyMvKkWZ9//nny8vJ+VxyJiNR1a/ZkkVdcRuNgf1pkLHYfjBniLlzKRXSB1BXE2NL4Mr+Ew3nFNA72tyZwDaQrMF5m+/bt3HLLLbRp04aQkBBP901qaiopKSl0796dgIAAT/tzzz23wuPXrVvH4sWLCQ4O9txiYmIAKnRFde/evcLjoqKiyMzMPOWcs2bN4vHHH2f27NmEh4dX9m2KiNRqC491Hw1sH4ax/Tv3wU5DKzYqn4l0bE8kzUSqSFdgAHyD3FdCrHrtShg6dCgtW7bknXfeITo6GpfLRdeuXSkpObU1AvLy8hg6dCjPPvvs785FRUX9Gsu34s6nhmHgcrlO6TU++eQT7rrrLj777DMGDhx4So8REalLFqe4C5jrG+2CLTkQHAHNKv7CWT4TqQN7AdiWnsv5bRuf1Zw1mQoYcF+yq0Q3jlWOHDlCSkoK77zzDhdddBEAy5Yt85zv2LEjH374IcXFxfj7uy8z/nZ8Su/evfnvf/9Lq1at8PE5/R+/r68vTqfzd8c//vhj7rzzTj755BPi4+NP+/lFRGqrXYfy2H04H1+7QY/85e6DHa8E2286RcI7AdDAeYQwcjWV+jfUheRFGjRoQKNGjZg+fTo7duxg0aJFjB8/3nP+1ltvxeVycc8997BlyxYWLFjA888/D7ivoAAkJCSQlZXFLbfcwpo1a9i5cycLFixg1KhRJyxITqZVq1YsXLiQ9PR0jh49Cri7jW6//XZeeOEF+vXrR3p6Ounp6eTk5FThpyAi4t3KV989r1UYvtu+cR8snz59PP/6ENYSgBhbmqZS/4YKGC9is9n45JNPSEpKomvXrowbN47nnnvOcz4kJISvv/6a5ORkevbsycMPP8zkyZMBPONioqOjWb58OU6nk0GDBtGtWzfGjh1LWFgYtt9W/3/ghRdeIDExkebNm9OrVy8Apk+fTllZGQkJCURFRXluDzzwQBV+CiIi3q28++jGqHTIzwT/UGjV/8SNj3UjxRjuPZHMSs5crc3UheRlBg4cyObNmyscO/4v9Pnnn8+6des89z/66CN8fX1p0aKF51j79u354osvTvoaS5Ys+d2x3666O3ToUIYOrTjg7ESPExGRX+UWlbJ6VxYA/ctWuQ92GAQ+fid+QERnSJlHjC2NnMJSMnOLiQgJOHHbOkYFTC3zwQcf0KZNG5o2bcq6deuYOHEiN954I4GBgVZHExGp85ZtP0yZy6RNoyBC9x5bfTfmBN1H5Y6tBdPDdx+UumciqYBxUwFTy6SnpzN58mTS09OJiorihhtu4KmnnrI6loiIAAuPjX+5qWUubN4Ddn9o9wezNY91IbUx0zBwsS0jj4vaNzkLSWs+FTC1zIQJE5gwYYLVMURE5DdcLpMlx8a/DPZZ6z7Y7jLwDz75gxq2AZ8A/MuKaGFksk1bCnhoEK+IiMhZsGF/DofzSgj296FZxm/2PjoZmx2auBcbjTFS2ZapAqacChgREZGzoLz76NpWJdgyNoJhh46D//yBnj2R0tihmUgedbqA0V8C6+lnICJ1xeJjBcz1QcnuAy3Ph6BT2Oz2WAHTyZ5GbnEZB3OKqimhd6mTBUz5MvkFBQUWJ5Hyn8Fvty4QEalNMh1FbNjvXtSzc84P7oO/3fvoZI4VMN180gBI0YJ2QB0dxGu32wkLC/NsThgUFORZqVbODtM0KSgoIDMzk7CwMOx2u9WRRESqTfnidf2bmvgeOLbFS8wpbrcS7i5golzpBFLE9oxcLumoTXLrZAEDEBkZCVCpHZal6oWFhXl+FiIitVX59gEjG2yCIyZE94LQZqf24OAmUC8cW34mHYx9bMtoV41JvUedLWAMwyAqKorw8HBKS0utjlMn+fr66sqLiNR6xWVOftx+GIBzi1a4D55q91G5iC6wK5MYWxpb1YUE1OECppzdbteXqIiIVJufdmdRUOKkdXAZwQeO7T4dczoFzGJijFS+ysjD5TKx2er20Ic6OYhXRETkbFm4xd19dHfkTgxXKTTuAE06VO5Jymci2dIoLHWyP7uwqmN6HRUwIiIi1cQ0Tc8A3ktZ7T74Z4vXnUh4Z8A9lRpMtqkbSQWMiIhIddl1OJ+9Rwqoby8jIuNH98FOp1HANIkBw0aomUs42WzLyKvaoF5IBYyIiEg1WXSs+2hU1G6M0nwIaQrRvSv/RL4B0Kg9AJ1sqboCgwoYERGRalM+fXqI38/uAzHxcLrrjkW4u5E6GipgoJIFzJtvvkn37t0JCQkhJCSE2NhYvv32W8/5oqIiEhISaNSoEcHBwQwbNoyMjIwKz5Gamkp8fDxBQUGEh4fz0EMPUVZWVqHNkiVL6N27N/7+/rRr144ZM2ac/jsUERGxgKOolDV7srDjpG3WsdV3T2f8S7nyPZFsaezIzMPpqttbsVSqgGnWrBnPPPMMSUlJrF27lksvvZSrr76aTZs2ATBu3Di+/vprPvvsM5YuXcqBAwe47rrrPI93Op3Ex8dTUlLCihUrmDlzJjNmzGDy5MmeNrt37yY+Pp5LLrmE5ORkxo4dy1133cWCBQuq6C2LiIhUvx+3HabMZXJ1g73Yi45CYANoecHpP2FEVwA621IpLnORmlW3t8MxzDPcTa9hw4Y899xzXH/99TRp0oRZs2Zx/fXXA7B161Y6derEypUrOe+88/j2228ZMmQIBw4cICIiAoC33nqLiRMncujQIfz8/Jg4cSLz5s1j48aNnte4+eabyc7OZv78+aecy+FwEBoaSk5ODiEhIWfyFkVERCrtwdnr+O/P+5jd4kvOzZwNPYfDNdNO/wmP7oVXulOKD52K3uONEf2I61L7VjI/1e/v0x4D43Q6+eSTT8jPzyc2NpakpCRKS0sZOHCgp01MTAwtWrRg5cqVAKxcuZJu3bp5iheAuLg4HA6H5yrOypUrKzxHeZvy5xAREanpXC6TJSmZgEmPvGOzj86k+wggrAX41ceXMtoYB9lex8fBVHol3g0bNhAbG0tRURHBwcHMmTOHzp07k5ycjJ+fH2FhYRXaR0REkJ6eDkB6enqF4qX8fPm5P2rjcDgoLCwkMDDwhLmKi4spLi723Hc4HJV9ayIiIlVi3b5sjuSX0C8gFf+Cg+BbD9pecmZPahjucTBpq4gx0ur8VOpKX4Hp2LEjycnJrF69mvvuu4+RI0eyefPm6shWKVOmTCE0NNRza968udWRRESkjiqffTSqwbHhEO0uA98T/wJeKcdmIsVoKnXlCxg/Pz/atWtHnz59mDJlCj169OCVV14hMjKSkpISsrOzK7TPyMjw7DYcGRn5u1lJ5ff/rE1ISMhJr74ATJo0iZycHM8tLS2tsm9NRESkSpQXMOeXHhv+UNnNG0+mfCaSkcquQ/mUOV1V87xe6IzXgXG5XBQXF9OnTx98fX1ZuHCh51xKSgqpqanExsYCEBsby4YNG8jMzPS0SUxMJCQkhM6dO3vaHP8c5W3Kn+Nk/P39PdO7y28iIiJnW3pOEZsOOGhrO0BI3i6w+UD7QVXz5OG/7olU4nSx50jdnYlUqTEwkyZNYvDgwbRo0YLc3FxmzZrFkiVLWLBgAaGhoYwePZrx48fTsGFDQkJCuP/++4mNjeW8884DYNCgQXTu3JkRI0YwdepU0tPTeeSRR0hISMDf3x+Ae++9l9dff50JEyZw5513smjRImbPns28efOq/t2LiIhUsfK9j+5osBHygdYXQ2BY1Tz5sS6kKOMIIeSxPSOXduHBVfPcXqZSBUxmZia33347Bw8eJDQ0lO7du7NgwQIuv/xyAF566SVsNhvDhg2juLiYuLg4pk37dcqY3W5n7ty53HfffcTGxlKvXj1GjhzJE0884WnTunVr5s2bx7hx43jllVdo1qwZ7777LnFxcVX0lkVERKpPeffR5bY17gOns/fRyQSEQmgLyEklxkgjJSOXwd2iqu75vcgZrwNTU2kdGBEROduKSp30eiKR0NJMVgXcDxjwYArUj/jTx56yWTfBtvn8s/QOsjqP5I3hp7G3Ug1W7evAiIiISEWrd2dRWOpkWL1k94Hm51Zt8QKegbyd6vieSCpgREREqsiiLe5ZtNcG/uI+cKaL152IZ0+kVHYfzqekrG7ORFIBIyIiUgVM02RRSiZh5NImf537YFWOfyl3bCZSR2MfTpeT3Yfzq/41vIAKGBERkSqw81AeaVmFxPkmYzOd7kKjYZuqf6FG7cDuRz2jiGbGoTrbjaQCRkREpAos3OKefXRjcPnVlypavO637D7QpCMAMUZand0TSQWMiIhIFVi0NZNAiuhRnOQ+UB3dR+UiugLuFXlTVMCIiIjI6cgpKGXt3qP0t63Hx1UMYS09RUa1OG4g7/Y6uqmjChgREZEz9MP2QzhdJjeUT5/uNNS9e3R1CT+2qaORxp4j+RSVOqvvtWooFTAiIiJnaPHWTHwp40LXWveB6pg+fbxjV3da2dLxM4vZdajuzURSASMiInIGnC6TxSmZnGfbTIAzD+o1cS9gV52CwyGoMXZM2hv76+RMJBUwIiIiZyA5LZujBaUM9Ts2eLfjlWCzV++LGoZnY8cYW91ckVcFjIiIyBlYtDUDAxdx9p/dBzpddXZe2DMTKY1tdXAgrwoYERGRM7Bo6yF6GTsIdR4B/xBo3f/svPCxgbwdjVS2Z+oKjIiIiJyigzmFbDnoIM5+bPBu+0Hg43d2Xrx8U0dbKqlZ+RSW1K2ZSCpgRERETtOirZmAyVD/8u6jap59dLwmMWDYaGTk0tjMYUdm3epGUgEjIiJymhZvzaSjkUa08wDY/aHd5Wfvxf2CPHstdbSl1bmBvCpgRERETkNRqZNlOw4TZzvWfdT2EvAPPrshylfkNVLZVsfGwaiAEREROQ0rdx2hqNTFkPLp09W9eN2JHJuJ1MmWyrZ0FTAiIiLyJxZvzaSZkUkHczcYNvf6L2ebZyZS3ZtKrQJGRESkkkzTZOGWzF+7j1peAPUanf0gx7qQ2hv7SM/OI7+47OxnsIgKGBERkUranpnH/uxCrvA5S3sfnUxYS/ALxt8oo7VxkO11aCaSChgREZFKWrglk8bk0MdIcR+IibcmiM0G4Z3cEYy0OjUORgWMiIhIJS3emslAexI2TIjqCWHNrQtTPhOpju2JpAJGRESkErILSkhKPUqcbY37wNlcvO5Ewt0FTEcjjW3qQhIREZETWbrtEIGufC6wb3IfiBlqbaDjthRQF5KIiIic0OKtmVxiS8aPMmjUHpp0tDZQhHsqdTPjMPmOLHIKS63Nc5aogBERETlFTpfJkm2Hft28sdMQMAxrQwU2gJCmAHQw0thRR1bkVQEjIiJyin5JPUphQT6X2JPdB6zuPip3fDdSHVnQTgWMiIjIKVq0NZPzbZuoRxHUj4boXlZHcjtuT6SUOjIORgWMiIjIKVq0NZMrymcfxcS712GpCcpnItnS2K4uJBERESm3P7uQ7enZDLQf27zR6unTx4s4biq1rsCIiIhIuUVbM+lrbKORkQsBYe79j2qKxu0xbb6EGIX45e3naH6J1YmqnQoYERGRU7B4ayZx9mPdRx2vBLuvtYGOZ/fFODadu66syKsCRkRE5E8UljhZvuMQg46fPl3TeAby1o0VeVXAiIiI/ImVuw7TzrmLZsZhTN8gaHup1ZF+L9y9oF2MLZXtugIjIiIii47rPjLaXQa+gRYnOoGIrsCxKzAqYEREROo20zRZtCWTONux7qOasnjdbx3rQmptHGRPepbFYaqfChgREZE/kJKRi79jNx1t+zBtPtBhkNWRTqx+JGZgA3wMF40K93A4r9jqRNWqUgXMlClTOOecc6hfvz7h4eFcc801pKSkVGgzYMAADMOocLv33nsrtElNTSU+Pp6goCDCw8N56KGHKCsrq9BmyZIl9O7dG39/f9q1a8eMGTNO7x2KiIicgUVbM4k7tnid0eoi995DNZFhYHi6kWr/TKRKFTBLly4lISGBVatWkZiYSGlpKYMGDSI/P79Cu7vvvpuDBw96blOnTvWcczqdxMfHU1JSwooVK5g5cyYzZsxg8uTJnja7d+8mPj6eSy65hOTkZMaOHctdd93FggULzvDtioiIVM6iLZnHbd5YQ7uPyh0byNvRVvsXtPOpTOP58+dXuD9jxgzCw8NJSkqif//+nuNBQUFERkae8Dm+++47Nm/ezPfff09ERAQ9e/bkySefZOLEiTz22GP4+fnx1ltv0bp1a1544QUAOnXqxLJly3jppZeIi4ur7HsUERE5LUfzS9ifupNe/jswMTBi4q2O9MeO2xNpfi2fSn1GY2BycnIAaNiwYYXjH330EY0bN6Zr165MmjSJgoICz7mVK1fSrVs3IiIiPMfi4uJwOBxs2rTJ02bgwIEVnjMuLo6VK1eeNEtxcTEOh6PCTURE5Ews3XaIy2zurQOMZudA/RP/cl5jHOtC6lQHplJX6grM8VwuF2PHjuWCCy6ga9eunuO33norLVu2JDo6mvXr1zNx4kRSUlL44osvAEhPT69QvACe++np6X/YxuFwUFhYSGDg76evTZkyhccff/x0346IiMjvLNqayQ3lmzfWxMXrfis8BhODJkYOh9P3Y5omhmFYnapanHYBk5CQwMaNG1m2bFmF4/fcc4/nz926dSMqKorLLruMnTt30rZt29NP+icmTZrE+PHjPfcdDgfNmzevttcTEZHarczp4peUXbxg2+I+EOMFBYxfPcwGrTGO7iKqZBeZucVEhARYnapanFYX0pgxY5g7dy6LFy+mWbNmf9i2X79+AOzYsQOAyMhIMjIyKrQpv18+buZkbUJCQk549QXA39+fkJCQCjcREZHT9XNqNueU/ISv4cQM7wyNqu+X8Kpki3SPg+lUy2ciVaqAMU2TMWPGMGfOHBYtWkTr1q3/9DHJyckAREVFARAbG8uGDRvIzMz0tElMTCQkJITOnTt72ixcuLDC8yQmJhIbG1uZuCIiIqfNvfque/aR4Q1XX8qFuwuYjkYa2zJq70DeShUwCQkJfPjhh8yaNYv69euTnp5Oeno6hYWFAOzcuZMnn3ySpKQk9uzZw1dffcXtt99O//796d69OwCDBg2ic+fOjBgxgnXr1rFgwQIeeeQREhIS8Pf3B+Dee+9l165dTJgwga1btzJt2jRmz57NuHHjqvjti4iInNjyLan0t61336np06ePVz4TqZYP5K1UAfPmm2+Sk5PDgAEDiIqK8tw+/fRTAPz8/Pj+++8ZNGgQMTExPPjggwwbNoyvv/7a8xx2u525c+dit9uJjY3ltttu4/bbb+eJJ57wtGndujXz5s0jMTGRHj168MILL/Duu+9qCrWIiJwV+44WEH14BYFGCc7QFhDZzepIp+5YAdPB2Mf29Gxrs1SjSg3iNU3zD883b96cpUuX/unztGzZkm+++eYP2wwYMIBffvmlMvFERESqxOKtmQw6tnmjvdNQ8KaZPA1a4/IJJKCskJLMnZjmRbVyJpL2QhIREfmNJVsOMND2s/uON0yfPp7N5lmRt3npLg7mFFkcqHqogBERETlOQUkZzt3LCDUKKAtoBM37WR2p0mwRx20pUEvHwaiAEREROc6KHUe4zFwNgL1TPNjsFic6DeUr8tbiqdQqYERERI6zeGs6g8qnT3vT7KPjHbcnUm2dSq0CRkRE5BjTNMnYspwII5syn3rQ5mKrI52eYwVMC9sh9h3M+JPG3kkFjIiIyDFbDubSt3AFAEaHOPDxtzjRaQpqSFk99+r2xqGtuFx/PIvYG6mAEREROWbx1gwGHdu80d7ZS7uPjinfUqC1aw/7swstTlP1VMCIiIgcs23jGtrY0nHa/KD95VbHOSO2iPItBWrnQF4VMCIiIkBWfgktMhcBUNqyP/jXtzjRGTo2EynGVjv3RFIBIyIiAizdlunpPgrodrXFaarAsbVgYoxUtqU7LA5T9VTAiIiIAL+sX0832x5c2KDjlVbHOXONO+AyfAg1Csg6uNvqNFVOBYyIiNR5pU4XwbvnA5Af0RfqNbY4URXw8aesQTsA/LO24KxlM5FUwIiISJ2XtPcoF7vcq+/W63mtxWmqjk+0exxMW1cqaVkFFqepWipgRESkzlu1IYW+RgoANm/bvPEPlM9EirGlklLLZiKpgBERkTrPuWUedsMkO6wzhLWwOk7VKZ+JZKSyXQWMiIhI7ZGWVUDP/OVALZl9dLxjM5HaGAfZdTDL4jBVSwWMiIjUaT9s3M2Ftg1ALSxgQppS6huCr+GkMH2r1WmqlAoYERGp07LXzcPfKCM7sAU0ibE6TtUyDJxNOgEQfHQrZU6XxYGqjgoYERGpswpKymh1yL36rhkTD4ZhcaKq59+0OwBtSWXPkdozE0kFjIiI1FkrUg7S30gGIKz3MGvDVBPj2EykTrVsIK8KGBERqbP2JX1DfaMQh29jjKZ9rI5TPco3daxleyKpgBERkTrJNE3CUr8DILdVHNhq6VdiuHsMTKRxlP0H9lkcpurU0p+WiIjIH9u07ygXOn8CoMk5tbP7CAD/+hQGNwfAlb7R4jBVRwWMiIjUSSlrvqex4SDfVh+/tv2tjlO9wt3dSCGO7ZSU1Y6ZSCpgRESkTvLdPg+AzKgBYPe1Nkw1C2jWDYAO7GX34XyL01QNFTAiIlLnHMktonfBMgDCel9ncZrqZ5RvKWBLZVstmYmkAkZEROqcX9b8SDPjMEX406DbFVbHqX7HZiJ1MPazIz3b2ixVRAWMiIjUOSUb/wdAWsNY8AuyOM1Z0LANZTZ/goxiju7bbnWaKqECRkRE6pRSp4v2R5YA4Nv1KmvDnC02O4Vh7QEwDm22OEzVUAEjIiJ1ysb1v9DeSKMMOy361f7xL+V8otzjYBrmbaOo1GlxmjOnAkZEROqUrKT/ArCzXk9s9RpYnObsCWjm3hOpo5HGrkPePxNJBYyIiNQpkQe+B6C0XbzFSc6u8j2ROhqpbM/0/plIKmBERKTO2Ld3J11cKQC0vPAGi9OcZcemUrc0Mtm1P8PiMGdOBYyIiNQZqSs/B2CbbyfqN2lhcZqzrF5jCvwaYzNMCvZtsjrNGVMBIyIidcKhvVtpmzIdgKMtBlmcxholjWIA8D2yxeIkZ04FjIiI1HpHUzdhzBhMhHmYvUYzOg5OsDqSJfyaurcUCC/YQWGJd89EUgEjIiK1Wm7qBsz3r6SxmcVOowW+o78hrHGE1bEsEXTcTKQdmXkWpzkzKmBERKTWKkhNxvX+lTQ0s0mhNfZR84hu1tLqWNY5NhMpxpbKtnSHxWHOTKUKmClTpnDOOedQv359wsPDueaaa0hJSanQpqioiISEBBo1akRwcDDDhg0jI6PiaOfU1FTi4+MJCgoiPDychx56iLKysgptlixZQu/evfH396ddu3bMmDHj9N6hiIjUScWpSbjejyfUdLCJthh3/I9WLerYwN3fatwRJ3YaGHkc2L/L6jRnpFIFzNKlS0lISGDVqlUkJiZSWlrKoEGDyM//dUGccePG8fXXX/PZZ5+xdOlSDhw4wHXX/brSodPpJD4+npKSElasWMHMmTOZMWMGkydP9rTZvXs38fHxXHLJJSQnJzN27FjuuusuFixYUAVvWUREarvSPatxvj+UYDOPZLMDrhFf0qFVHb7yUs43gNx67s+hZP9Gi8OcGcM0TfN0H3zo0CHCw8NZunQp/fv3JycnhyZNmjBr1iyuv/56ALZu3UqnTp1YuXIl5513Ht9++y1DhgzhwIEDRES4+yDfeustJk6cyKFDh/Dz82PixInMmzePjRt//XBvvvlmsrOzmT9//illczgchIaGkpOTQ0hIyOm+RRER8TJlu5dT9sEwAsxC1pid4NZPOaejipdyR2YMp9GeubzhczsJj7xmdZzfOdXv7zMaA5OTkwNAw4YNAUhKSqK0tJSBAwd62sTExNCiRQtWrlwJwMqVK+nWrZuneAGIi4vD4XCwadMmT5vjn6O8TflznEhxcTEOh6PCTURE6hbXzqU4P7iOALOQla4uFN2k4uW3gpq7B/JGF+8kv7jsT1rXXKddwLhcLsaOHcsFF1xA167u1f3S09Px8/MjLCysQtuIiAjS09M9bY4vXsrPl5/7ozYOh4PCwsIT5pkyZQqhoaGeW/PmzU/3rYmIiBcyt3+P88Pr8TeLWOrqQe6wWVzUWcXLbwUem4kUY6Sx3YtnIp12AZOQkMDGjRv55JNPqjLPaZs0aRI5OTmeW1pamtWRRETkLDFTvsU562Z8zRK+d/Ym56qZDOrRyupYNdOxmUhtjf3sOJBlcZjTd1oFzJgxY5g7dy6LFy+mWbNmnuORkZGUlJSQnZ1doX1GRgaRkZGeNr+dlVR+/8/ahISEEBgYeMJM/v7+hISEVLiJiEgdsOVrXJ/cho9ZyjfOczl85btc1be11alqrtDmFNnq4Wc4ObLXewfyVqqAMU2TMWPGMGfOHBYtWkTr1hX/gvTp0wdfX18WLlzoOZaSkkJqaiqxsbEAxMbGsmHDBjIzMz1tEhMTCQkJoXPnzp42xz9HeZvy5xAREQFg439xzR6J3SzjK2csBwe+wc2xba1OVbMZBrmhHQBwZWy2OMzpq1QBk5CQwIcffsisWbOoX78+6enppKene8alhIaGMnr0aMaPH8/ixYtJSkpi1KhRxMbGct555wEwaNAgOnfuzIgRI1i3bh0LFizgkUceISEhAX9/fwDuvfdedu3axYQJE9i6dSvTpk1j9uzZjBs3rorfvoiIeK11n+D6/C5sppP/Oi9iT/+XGX1xB6tTeQXzWDdSveytFic5fZUqYN58801ycnIYMGAAUVFRntunn37qafPSSy8xZMgQhg0bRv/+/YmMjOSLL77wnLfb7cydOxe73U5sbCy33XYbt99+O0888YSnTevWrZk3bx6JiYn06NGDF154gXfffZe4uLgqeMsiIuL1fv4P5px7seHi47JLSDnvWe4f2NHqVF6jfgv3QN5mJbvJKSy1OM3pOaN1YGoyrQMjIlJLrXkX5j0IwAdll7Ot9z958truGIZhcTAvkroK3ovjgNmQg3cm0adlQ6sTeZyVdWBERETOqlVveoqXd8sGk9z1YZ64RsVLpYV3AiDayGL3vv0Whzk9KmBERMQ7LHsZ5v8fANPKrmJth78z9YYe2GwqXiotIJRsP/fM39y96ywOc3pUwIiISM23dCp8/ygAL5UOY3XrMbxyay987PoaO135YTEAGJmbLE5yevSTFxGRmss0YdG/YPFTAEwtvYmVLe7mrRF98fexWxzOu9mj3Kvoh+ZsszjJ6VEBIyIiNZNpQuJk+OE5AJ4sHc7y6JG8d8c5BPqpeDlTYa16AtDSuYej+SXWhjkNKmBERKTmMU2YPwlWvArA5NKRLG9yMzNHnUOwv4/F4WqHgGN7InU00tiWnmNxmspTASMiIjWLywXzxsPqN3FhMKl0NMsaXMd/RvcjLMjP6nS1R8O2lOJLPaOYA3u9rxtJBYyIiNQcLid8dT+sfQ8XBhNK7+GH+kP48K5+NKnvb3W62sXuw5Eg95ZAhWneNxNJBYyIiNQMzjKYcy8kf4gTG2NL/soPQYOYdXc/osNOvJGvnJmihu71YHwOe9+eSCpgRKRauVwmB7MLrI4hNZ2zFP47GjbMpgw7Y0ru58eAAXx0Vz9aNqpndbpayz+6GwANcrdbnKTyVMCISLUxTZOvXxuLz0sd+e/Ml3C6auXOJXKmyorhsztg85eU4cN9JQ+wzPcC/jO6H+0j6ludrlZr2LYXAG1cezmcV2xxmspRASMi1Sbx61lcfXQGTQwH1+56nE+mPUZhidPqWFKTlBbBp7fB1rmUGr7cVTKeZfZ+zLjzHLo2DbU6Xa3n39Q9E6m1kc6O/ZkWp6kcFTAiUi22795Dz6R/AHDItyk2w2T44Zf57JXxXvebnlSTkgL4+GbY/h0lhj93FP+dFbbevDuyb43aXLBWCw7HYQvDZpgc3rXe6jSVogJGRKpcUUkZGR/9hXAjm/2+LWj80BoOdP8rALfnz2DBy/eyMzPX4pRiqeI8mHUj7FpMsS2QEUUTWE13pt3amwvaNbY6XZ2SFdwegOL9KmBEpI779sPnubBsFaX4EHTz+xh+9Yi+bgpHYh8BYHjZF/w8bRQ/7TxkcVKxRJEDPhwGe36k2BbErYUTWEMnXrqpJwM7R1idrs4pa+yeiRSQtcXiJJWjAkZEqtTy1T8xaO+LAKT1HE+Dtn095xrFPUTeoBdwYXADiRyaOYK5v+yxKKlYojAb/nMtpK2iyF6fGwv/jySzI89c152hPaKtTlcnBTZ3j4NpXLAT0/SegfYqYESkymRk51H/279Szyhmb3Av2lz1f79rE3z+XZRd+y5l+BBvW0m9L27nnYUbvep/nHKaCrLgg6tg/1qKfEIZVvB/rDPb8djQztx4TnOr09VZjdv2BqCduZdDjiKL05w6FTAiUiVcLpPl/55Id7aTZ9QjctRMsJ14wz2/Htdj3PoJpTZ/LrGvo/vS0Tzx31WUOV1nObWcNXmHYOZQOLiOIr8GXJ0/iU1max6K68gdF7S2Ol2d5h/VGSc2Ghm57Nq72+o4p0wFjIhUia/mzeEqxywA8gZOxb9Ryz9sb+9wOb4j/0eJTzD9bFu5bv29jHv/e/KKy85GXDmbctNh5hDI2EiRf2OG5k0ixWzBXwe0JeGSdlanE99ADvk2BSB79y8Whzl1KmBE5Ixt2rWf3msn4mO42B0dT+QFt53aA1vG4jf6G0r8G9LNtocHUh8gYdpXZHjRZWz5Ezn7YUY8HNpKUWAE8bmT2O5qxh3nt+KhuI5Wp5NjckI6AOA8uNHiJKdOBYyInJH84jJSZ91PCyOTIz7htBrxRuWeIKoHfnd/R0m9aNrZDvBU9kPc/9pnpKRrmrXXy06FGVfCkR0U1WvKlY5J7HRFcUOfZkwe0hnDMKxOKMeY4Z0BCMpOsTjJqVMBIyJn5IsPpzG4bCEuDPyufwcjsEHln6Rxe/zu/o7SsLY0Mw7zRsk/+Mebs1i+43DVB5azI2sXvH8lHN1DUf2WXJEziV3OcOK7R/HMsO7YbCpeapL6LXoCEFnoPTORVMCIyGn7fnUyQ1KfBeBAl79QP2bA6T9ZWHN871pAWXhXmhgO3uMxXn7/Qz5P2lc1YeXsObwd3o+HnDSKQtsyOOf/2FPWkMtiwnnpxp7YVbzUOE3auWcitWEfB7O84+qnChgROS37j+YT9O39NDDySK/XkWbXPnnmTxrcBJ9R83A160eoUcBMn6f533//w0uJ27zmt8I6L3Or+8pL7gGKG3TgypyJ7C4J5fy2jXhjeG/8fPS1UxP5NWpFAYH4G2Xs27nB6jinRH+TRKTSnC6TxPcf53zWU4w/jW7/AHz8qubJA8Ow3f4lZtvLCDKK+bfvc2xb/CF//2w9JWWaZl2jZWxyD9jNz6S4UWfiHRPZVRRM7xZhvHN7XwJ8TzytXmoAm430gDYA5O5dZ3GYU6MCRkQqbfa8+dyS8x4AeRc/hm9ETNW+gF8Qxi2fQOdr8DOcvO77KrZ1H3LH+z/hKCqt2teSqpF3CD66AQoOUxLenatyJ7IjP5Au0SG8P+pc6vn7WJ1Q/kR+mHsmkpmx2eIkp0YFjIhUyi+7DtJrzUP4G6UcDL+YRgPuq54X8vGD69+DXiOwGybP+U6n057/cP2bK9ifXVg9rymnx1kGn48Cx37KGrTlmtwJpDh8aRcezAd3nktooK/VCeUU2CK7AhCS4x0zkVTAiMgpyy0qJWXWBGJsaeTaw4gc8Q5U51RYmx2ueg3Ovx+Af/p+SPyR97n29WVs3J9Tfa8rlbPwcdjzI6ZvEH8pGcfmozZaNAziw9H9aBTsb3U6OUWhrXoB0KxkFy5XzR9zpgJGRE7Zh7NmcnPZVwDYr30Do/5Z2DnYMODyJ+HSfwLwgM8c7it6h5veXs7irZnV//ryxzbNgRWvAvBc4AMsPNKQyJAAPrqrH5GhARaHk8qIbOcuYKKNwxzIyLA4zZ9TASMip2Tu6o1cu/dfABzqOJygrkPO3osbBvT/O1z5PACjfBbwhPkGf/lgNR+t3nv2ckhFmVvhywQAEhvcxLTMboQF+fLhXf1o3jDI4nBSWT7BDck0GgOQvj3J4jR/TgWMiPyp1MP5+H7zIJHGUbICW9Jk2PPWBDn3brh2OqZhZ5h9Ga/bX+KJOT/zzLdbveKSd61SlAOfDofSfFJD+nDvwSH42g3euq0P7cKDrU4npykzyL03VUHaeouT/DkVMCLyh0qdLubMfJ44YxVl2AkZPgP8LPztusdNGDd9iGn3Z5A9ifd8n+M/SzfywKfJFJU6rctVl7hcMOc+OLKDgsBIrsm8Gyd2nr62G+e1aWR1OjkDxQ3dMwrth2r+TCQVMCLyh2bOW8JoxzQA8s+fgE+z3hYnAmKuxLjtc/AL5gL7Jmb5Pc0P67Yx4t+rOZpfYnW62m/Zi5AyD5fNjxG595NFCPcNaMsNfZtbnUzOkE90NwAa5G23OMmfUwEjIie1ekcGPddOJNgo4kijPoQOfMjqSL9q3R9GfgWBDehh28nnAU+yd88uhr25gtQjBVanq712LIRF7rFQ/zLvJKmsNYO7RvLQIO0sXRs0auMeyNuidA9OZ81eOFIFjIicUE5BKes+nkxf2zYKbfVodNv77mnNNUnTPjDqW6gfRXvSmBP4BKVHdnHttOX8knrU6nS1z9G98N/RgMk838t5r7A/3ZuF8uKNPbU5Yy0R1aYbJaadYKOQg3u3WR3nD6mAEZHfMU2Ttz7+lDvLZgNgxD8PDVpanOokwjvBnfOhQSuamhl8GfAkDQt2ccs7q5i/Md3qdLVHaSF8ehsUHmWnbwfG5w4nKjSAd2/vS6BfDSts5bTZff3Y59MCgEM7a/ZMJBUwIvI7c1Zt48a9T+BjuDja5ioCet9idaQ/1qAV3LkAwjvTyMxiTuCTdCzbxn0fJfHvZbutTuf9TBPmjof09eTZw7gt937sfoH8e+Q5hIdorZfaJqueeyZS0f6avamjChgRqWDnoTzKvp1Ea1sGuf6RNLjh1epdbbeq1I+EO+ZB074Eu3KZHTiF84xNPDl3M499tQmnplmfvrX/hnWzcGHj7sIE0o1GvHpzLzpHh1idTKpBSePOAPgf2Wpxkj9W6QLmhx9+YOjQoURHR2MYBl9++WWF83fccQeGYVS4XXHFFRXaZGVlMXz4cEJCQggLC2P06NHk5eVVaLN+/XouuugiAgICaN68OVOnTq38uxORSikpc/HxB9O40bYQFwb1bpwOgQ2sjnXqghrC7f+D1hfj7yrkPwHPMdCWxIwVe7jvwyQKSzTNutLSfoJv/w+AZ8tuYqWrCw9f2YmBnc/CKsxiicBm3QFonF+zZyJVuoDJz8+nR48evPHGGydtc8UVV3Dw4EHP7eOPP65wfvjw4WzatInExETmzp3LDz/8wD333OM573A4GDRoEC1btiQpKYnnnnuOxx57jOnTp1c2rohUwltzl/NXxysAFPT5K7a2F1uc6DT4B8OtsyFmCD6uEqb7v8z1vsv5bnMGN7+zisN5xVYn9B65GTD7dnCVssDsx9tlQ7i1XwtGX9ja6mRSjcLbuZdKaOrcT1lxzZ3RV+n9zQcPHszgwYP/sI2/vz+RkZEnPLdlyxbmz5/PmjVr6Nu3LwCvvfYaV155Jc8//zzR0dF89NFHlJSU8N577+Hn50eXLl1ITk7mxRdfrFDoiEjVWbbtEN2THqahPQ9HWCdCBj9qdaTT5xsAN8yEr8ZgW/cxz9vfoJFPIW+nDeTaacuZMepc2jbRarF/yFnq3mE69yC7jWaML7qHC9s14fGrumB4Q5einLaopq04atangZFL2o51NO8Sa3WkE6qWMTBLliwhPDycjh07ct9993HkyBHPuZUrVxIWFuYpXgAGDhyIzWZj9erVnjb9+/fHz8/P0yYuLo6UlBSOHj3x1Mji4mIcDkeFm4icmiN5xaz6ZAoD7OsoNfwIuXUG+Hj5LsJ2H7h6Gpz7FwAmme/xcP15pGUVcN20Ffy0O8vigDVc4mTYu5wCI5DRRWOJbNKYN4b3xteuoZO1nc1uI82vFQBHd/1sbZg/UOV/E6+44go++OADFi5cyLPPPsvSpUsZPHgwTqe77zk9PZ3w8PAKj/Hx8aFhw4akp6d72kREVOxfLb9f3ua3pkyZQmhoqOfWvLlWhBQ5FaZp8srHXzHG+YH7/uVPQniMxamqiM0Gg5+FiycCcHfpR7zc4L/kFJZw27ur+WrdAYsD1lAbPodV7tWXxxbfy9HAlrx3xzmEBvpaHEzOlpz6HQAoO7jJ4iQnV+UFzM0338xVV11Ft27duOaaa5g7dy5r1qxhyZIlVf1SFUyaNImcnBzPLS0trVpfT6S2mLViOzenPUmAUUpe8wH4xf7F6khVyzDgkn9A3BQArin8gg+bfESZs4y/ffwL05bswDQ1Q8kjYxN8dT8Ar5ddzRKjH9Nv70vLRvUsDiZnkyu8CwCBR2vuTKRqvxbYpk0bGjduzI4dOwCIjIwkMzOzQpuysjKysrI842YiIyPJyMio0Kb8/snG1vj7+xMSElLhJiJ/bFtGLoULnqCzbS+Fvg0IvnG6d0yZPh2xf4Wr3wDDxoW53zA36t/4UcrU+Sn8Y85Gymr4sulnRWE2fDIcSgv4wdmNF8tu4Nnru3FOq4ZWJ5OzrF4L90ykiKKdFic5uWovYPbt28eRI0eIiooCIDY2luzsbJKSfl3hb9GiRbhcLvr16+dp88MPP1BaWuppk5iYSMeOHWnQwIumdIrUYEWlTt79YCZ3GnMB8L/udahfy6fG9rrNPbjX5kvno4tZ3PRtgowiPv4plbs+WIujqPTPn6O2crlgzl/g6G72mY35W+kYEi7twLW9mlmdTCwQ3b4XLtOgoZlNSXbNXNG60gVMXl4eycnJJCcnA7B7926Sk5NJTU0lLy+Phx56iFWrVrFnzx4WLlzI1VdfTbt27YiLiwOgU6dOXHHFFdx999389NNPLF++nDFjxnDzzTcTHR0NwK233oqfnx+jR49m06ZNfPrpp7zyyiuMHz++6t65SB338tc/MTbvBWyGSWG3Edg6DbE60tnR+Sq49VPwDaLpkRUsj36dJr6FLEk5xJBXl7FhX47VCa3xw3OwbT5F+PKXknFc0L0D4wZ2sDqVWCSqcSNScfd4ZO6smQN5K13ArF27ll69etGrl3vHyvHjx9OrVy8mT56M3W5n/fr1XHXVVXTo0IHRo0fTp08ffvzxR/z9f53R8NFHHxETE8Nll13GlVdeyYUXXlhhjZfQ0FC+++47du/eTZ8+fXjwwQeZPHmyplCLVJFFW9Lp8stjRBtZFAS3JHDIM1ZHOrvaXQYjvoSAUBoc+Zkfwl+gW1gxqVkFDHtzBTNX7Klb42K2fYe5xD1G6OGS0fg268ULN/TQBo11mGEYHAxwr/eTsyfZ2jAnYZi19F+pw+EgNDSUnJwcjYcROU5mbhGvvvgU/zJfxYUd212J0KyP1bGskb4R/nMt5GfibNCGx+s9wgc73Hv7XNElkmev7177Z95k7cKcPgCjKIf/lA3kreAEvky4gCb1vXwavZyxxGnjuDzzPTY2GULXhI/O2uue6ve3JvSL1CEul8mUWQuY4HoHAGf/CXW3eAGI7OreyTq0Bfaju3g8/a982uMX/Owm8zelM+S1H1mXlm11yupTUoD56W0YRTn87GrHC/Y7+fcdfVW8CABGhHtPpPo5KRYnOTEVMCJ1yPvLdnDL/qcIMQopjOyL78V/tzqS9Rq1hbsSoe1lGGVF9Et5jqQWb9AnLI+0rEKuf2sF7y3bXfu6lEwTvv4bRsYmDpkhJJSO5aVbzyEmUlesxS20dU8AIkv2grPM2jAnoAJGpI7YuD+H7MTnONeWQom9HoE3/du9Wq24d7K+7b8Q/wL4BlH/4Ao+dz3Ioy3WU+p08cTczfzlP0nkFNSiWUqr34YNn1Fm2kgoeYC/DLmQSzqG//njpM5o0bYLBaY//pRQnFnzNnZUASNSBxSUlPHGR7P5m+1zAHyHPAcNWlkbqqYxDDjnLrh3GTTti1GSy6jMZ1ja4j0i7Hl8tzmDK1/9kV9ST7ydiVfZuxJzwcMAPF02nJjzruCOC7RBo1QUHhLITsO9qn3mzl8sTvN7KmBE6oBnvvqZv+e9gK/hpLjjVRg9b7U6Us3VqC3cuQAufQRsPrTMXMiykIe5OXQT+7MLueGtlbz74y7v7VJyHMT56e0YZhn/c57PjjYjmDyks9WppAYyDIOMwHYAFKSuszjN76mAEanl5m88SIfkZ2hrO0hxUCT+V79Se1fbrSp2H+j/ENy1EJrE4Ft4iGeKn+Kj8I/wdxXwr3lbuPuDtWQXlFidtHLKSnB+ejv2gky2uJrzbthYXh/eGx9t0CgnUdjAvS+a7VDN2xNJf2tFarGDOYV88/n73OazEAD/69+GIC0Lf8qie8I9SyF2DGBwgWMeqxpMJtZnG99vyeTKV34kaa/3dCm5FvwD+/6fcJhB/J/PBKaNuoiQgFo+TVzOiG+Ue0+ksNwdFif5PRUwIrWU02Xy+KwlTDbfdN/vlwBtBlgbyhv5BkDcUzDyawhtTv3C/czyeZwp9T/ncE4uN729kreX7sTlquFdSus+wbbGPX3+764EJo8cSvOGQRaHkpquYRv3orVNyg5CkcPiNBWpgBGppd5euoMbDzxDY8NBcaNO2C9/1OpI3q31RXDfcug5HAOTW0q/YHHoE7Qz9zLl262MnrmGrPwa2qV0cB1l//sbAC+XXUf8sFH0aal95eTPtWnZgoOm+6pt8YGNFqepSAWMSC2UnJZN+sJpXGpPxmnzw//G98BHi5OdsYBQuGYa3PQhBDWiafFO5gU8whjfr1maksGVr/zImj1ZVqesqCCLoo+G4+MqZpGzJ1w8kat7NrU6lXiJxsH+7DJaAHC4hs1EUgEjUsvkFZfxwqyvmWT/EADb5Y9DhGaZVKlOQ+Gvq6DDYOxmGX+3f8z/6j2Fb+5ebp6+imlLdtSMLiWXk/yPRxGQl8ZeVzjfxTzJAwM7Wp1KvMzheu0BKNq/3uIkFamAEallnvgymQn5zxNolFDaagBGv3utjlQ7BYfDLR/DVa+DXzDdnFv4PuAf3GAsZOr8rYyasYYjecWWRixM/Bf10pZQaPrxSuPHeOzGCzA0A00qqbRRJwD8jmyxOElFKmBEapH/Je+n1YZX6GbbQ5l/GL7XvQU2/TOvNoYBvUe4x8a0vAB/s5BnfN9lhv/zbN62nStf/ZHVu45YEq1081wCV74IwFS/v/KPO68nwNduSRbxbv7NuwPQKG+HewuKGkL/ZxOpJdKyCvhyzmzutX8NgM/Vr0FIlMWp6ogGrdyzlC5/Eux+DDB+4fvAifTK+5Fb3lnF64u2n9UuJfPwdso+vweAD83B3HrX32kcrDFQcnoiWnWl1LQTZOZDzj6r43iogBGpBcqcLv7x8TKe5HVshomr523Q+SqrY9UtNjtc8Df3ujER3Qg1c3nL72We85nG9O9+YeT7P3Eo9yx0KRXnkfXeTQS68vnJFUPLW16kfUT96n9dqbXaRzdipxkNQNH+DRan+ZV2chM5ibV7spi7/iB+PjZCAnyoH+BL/WP/Pf5+SIAvwQE+2G3WjS14bdEOhqW/RDP7YUpDW+E7+BnLstR5EZ3h7kWwZAosf5lh9mXE2rbw951/4cpXc3nl5p6c37Zx9by2aXLgP3cRXbCTDDOMvZdN44aY6Op5LakzGtTzY5W9FTFmGlm7fyG6y5VWRwJUwIj8TnGZkxe/28b0H3dVqrs32N/nWIHjLmrqH1/kBFYsfn57vn6AD8H+Pqc1wHLNniz2LpnBON8VuAw7vte/C/76jdtSPn4w8FHocAXM+QvRR3czy+9p3iu8gtHv3sy9l3VlzKXtqrzo3f/t8zTd9y0lpp25HZ9l9MV9qvT5pe7Krt8eHD9SdkBXYERqpK3pDsZ+kszW9FwA4rtHERUSQG5RGbnFpeQWleEoPPbfojJyi0opLnMB7unLecVlHMw5vde2GeVF0K9Fz2+v9Py2KAryszPl4++Y4fO++zn6PwTNz6mSz0KqQIt+7t2tv3sEkt7nTp/59LetZ9zCv7J69zm8fHNPwusHVMlLHdrwPRE/PQ3Ap43u446bb6qS5xUBcDXpDA4IyEqxOoqHChgR3Mvu/3vZLp5fsI0Sp4uG9fyYcl034rpE/ulji8uc7gLnWEFTscip+N/c3913Hyt1mrhMcBwrjE6VDRez/F4mxFaAM7oP9v4PncnHINXBPxiGvgwx8fC/BNrlHeALv0d5bc+1DH35Bl68pS8XtDuzLqX8Q3vx/eJOfHCR6Hsp197zmKVdmlL7BDXvATuhUdEeKCuuEQtjGqbX7gn/xxwOB6GhoeTk5BASEmJ1HKnB9h0t4MHZ61i9272C6mUx4Twb35LGyx+H9Z+CqxQ49mXg6eIxKv7Zc+5kfz75Y8r/AZoY7ptpHvuziWkauHDPXHT/133ehYHLNDFMFw1w4PQJwn7fMmjUtko/G6liBVkwdxxs/hKAZFcbHiz7K/GXXMwDl7U/raLDWVLE3ucvpk3JVlJoTfCYRTRtrA07pWqt3X2E9jO6EmoUwF9+hKju1fZap/r9rSswUmeZpskXP+/nsa82kVtcRpCfnclDOnNTgxSMD26H3APHty5/0O8OnanflDSnxR7/vIoXbxDUEG6YARs+x/zmQXoW7WKu7z+YsuQWbtt1Ey/f0oeIkMp1Kf0y/S/0LdlKtlkP500fqHiRatE+MoStZgv6GVsp3L+BwGosYE6VChipk7LyS3h4zga+3ZgOQO8WYbx8bTtarHkKvpnpbtSwDQx9FZp0PFa4/LaIMSv+2XPuZH/+7WM4jcecoJ1fMDRsXdmPQKxiGND9BoyW58P//krgriU84TuTH/clccfL9zPp5svo36HJKT3Vys9fJvbwl7hMg20XvMy5na3/UpHaKTTQl1Sf1vRzbSVnzy8E9h1udSQVMFL3LE7JZMLn6zmUW4yPzWDc5R24t8V+7J8MhJxUd6N+98Jlj4JfkLVhpfYKbQq3zYE17+JK/CcXsZFPneOYPPMOfrpoJGMv74CP/eRLdf28chG9N/wLDPip1V84b9CNZzG81EW5oR3g6LeY6ZutjgJoITupQwpKynh4zgZGvb+GQ7nFtAsP5n/39CKh8G3s/7nKXbyEtYCRc2HwsypepPrZbNDvHmz3LsMV1ZsQo4CXfacRs/xv3PN2Iuk5RSd82M49e4mcfzf+Rikbgy+g38inz3JwqYuMiK4ABOfUjJlIKmCkTvgl9Sjxry7jo9XuKyyjLmjFvGt86PLVlfDTdHejPqPgvhXQ+iILk0qd1Lg9trsSYcA/cBl2hthXMyXjHv71yissScms0PSIo4CsD0YQbRzmgL0p7e/9CMOmPY6k+oW27AFA/dLDkG/NHl/HUwEjtVqp08WLidu4/q2V7D6cT1RoALNG9uBRv1n4fxAPWbsgpCnc9oV7qqsWgBOr2H1gwERsd31PSYN2RBjZvO56mn3/uZcX5/1MmdNFUamTpW89wDmudRQQQL0RH+Mf3MDq5FJHtG4azl5XuPtO5iZrw6AxMFKL7cjMY/zsZNbvc68sd3XPaJ46p4Tgb6+Fw9vcjXoOh7inITDMuqAix2vaG7+/LqPsu0fxWfM2t/ksZM/qjUzeMZFWAQ7uKZgNgGPQS0S26mFxWKlL2kfUZ7nZgpZkUpC2jqDW/S3NowJGah3TNPlg5V6e/mYLxWUuQgJ8ePqqjgw5+gF8+DKYTgiOcM8w6niF1XFFfs83EJ/4qdDpSgo/+wutCtN5MutBSvAFA/Z1Gk2z82+1OqXUMcH+Phzwaw3OteSnrsfqUYIqYKRWSc8p4qHP1/Hj9sMAXNS+MS/1t9M48dZfL3l2vR6ufM69JodITdZmAIF/W03elw8SnPI5gZSQ0fAcml0/1epkUkcVNIiBw2Acsn4mkgoYqTXmrj/Aw3M2klNYir+PjX/EtWVE2RfYPn4OXGUQ1AiGvASdr7Y6qsipCwwj+JZ/49pyLcU7fiDi0onu8TIiFvCJ6gaHIcSxHVxOsHAAuf4ViNfLKSzl0f9t5Mtk98q53ZqG8sblgbRYejccTHY36jQU4l+C4FNbIEykprF1GkJgpyFWx5A6rlHLGArX+xFIMWTthsbtLMuiAka82oodh3nws3UczCnCZsD9A1rzt8D52D+bAs4SCAiDK5+Hbtcftw+RiIicjo6RYWwzm9HD2OXullcBI1I5RaVOnluQwr+X7QagVaMg3rgihC6r/wb71rgbtY+Doa9ASJSFSUVEao924cF8ZbagB7vcM5Es7JJXASNeZ+P+HMZ9msz2zDwAhp/bjMlNfsT/f09CWRH4h8AVU9xTpHXVRUSkygT62ckMaAOlSyjct8HSmUgqYMRrOF0mby3dycvfb6PUadI42J9Xrwjl/A2TYP1yd6M2l8BVr0FYc2vDiojUUsWNOkE6+B7eYmkOFTDiFVKPFDB+djJr9x4FIK5zOC+0SSJ4wRNQmg++9WDQk9D3Tl11ERGpRv5Nu0E6hBSmQXEe+AdbkkMFjNRopmkye20aT3y9mfwSJ8H+Pjw7sAFX7v4XxvdL3I1aXghXvw4NW1uaVUSkLmjerAWZa8MIN7Lh0FZo1teSHCpgpMY6lFvMpC/W8/0W92Z257ZqwFtdttBw2V1Q7ACfQBj4KJz7F/euviIiUu3aRwSz1dWccHs2ZvpGDIsKmEr/X/+HH35g6NChREdHYxgGX375ZYXzpmkyefJkoqKiCAwMZODAgWzfvr1Cm6ysLIYPH05ISAhhYWGMHj2avLy8Cm3Wr1/PRRddREBAAM2bN2fqVK08WZckbs7gipd/4PstmfjZbfzrskZ8Wv9lGi4c7y5emp0L9y6D8+5T8SIicha1bRLMe84r+WvJ3zgSPcCyHJX+P39+fj49evTgjTfeOOH5qVOn8uqrr/LWW2+xevVq6tWrR1xcHEVFRZ42w4cPZ9OmTSQmJjJ37lx++OEH7rnnHs95h8PBoEGDaNmyJUlJSTz33HM89thjTJ8+/TTeoniTvOIyJn6+nrs/WMuR/BJiIoJZHJfJbUk3YWxfAHY/GPg43Dnf0vUHRETqqgBfO6kNz+cb13lsya9nWY5KdyENHjyYwYMHn/CcaZq8/PLLPPLII1x9tXtu+AcffEBERARffvklN998M1u2bGH+/PmsWbOGvn3dl51ee+01rrzySp5//nmio6P56KOPKCkp4b333sPPz48uXbqQnJzMiy++WKHQkdpl7Z4sxs1OJi2rEMOAseeFMaZgGvZFc90NonrCtW9BeCdLc4qI1HXtI4LZdTifvUcKuKi9NRmq9Nr77t27SU9PZ+DAgZ5joaGh9OvXj5UrVwKwcuVKwsLCPMULwMCBA7HZbKxevdrTpn///vj5+XnaxMXFkZKSwtGjR0/42sXFxTgcjgo38Q4lZS6enb+VG99eSVpWIU3DAlkwKIsHUkZgT5kLNh+45GG463sVLyIiNcCjQ7uwbvIgbjuvpWUZqnQQb3p6OgAREREVjkdERHjOpaenEx4eXjGEjw8NGzas0KZ169a/e47ycw0aNPjda0+ZMoXHH3+8at6InDXbMnIZ+0kymw+6C84R3esz2f4evkvnuBtEdIVr3oSo7hamFBGR40WHBVodofbMQpo0aRLjx4/33Hc4HDRvrsXMaiqXy+S95buZuiCFkjIXDYJ8efe8Q/RZPw7yMsCww4Xj4OKJ4OP3508oIiJ1SpUWMJGRkQBkZGQQFfXr/jMZGRn07NnT0yYzM7PC48rKysjKyvI8PjIykoyMjAptyu+Xt/ktf39//P39q+R9SPXan13IQ5+tY8XOIwBc2T6QF0I+IXDFp+4GjTvCtW9C0z4WphQRkZqsSsfAtG7dmsjISBYuXOg55nA4WL16NbGxsQDExsaSnZ1NUlKSp82iRYtwuVz069fP0+aHH36gtLTU0yYxMZGOHTuesPtIvINpmnz5y36uePkHVuw8QqCvnfcuzOGNnDEEbvoUMOD8v8FfflDxIiIif6jSV2Dy8vLYsWOH5/7u3btJTk6mYcOGtGjRgrFjx/Kvf/2L9u3b07p1a/75z38SHR3NNddcA0CnTp244ooruPvuu3nrrbcoLS1lzJgx3HzzzURHRwNw66238vjjjzN69GgmTpzIxo0beeWVV3jppZeq5l3LWZddUMLDX25k3vqDAJzX1I/pkV8SsvZDd4OGbdxjXVqcZ2FKERHxFoZpmmZlHrBkyRIuueSS3x0fOXIkM2bMwDRNHn30UaZPn052djYXXngh06ZNo0OHDp62WVlZjBkzhq+//hqbzcawYcN49dVXCQ7+dT+F9evXk5CQwJo1a2jcuDH3338/EydOPOWcDoeD0NBQcnJyCAkJqcxblCq2dNshHvpsHZm5xfjYDF7uuZ/4fS9iOA64G5z7F/eKun7WrScgIiI1w6l+f1e6gPEWKmCsV1jiZMq3W/hg5V4A+jUqYHrjzwjdu8DdoEEr987RrftbF1JERGqUU/3+rjWzkKRmWZeWzbhPk9l1OB8bLt5ot4YrMv+NsTfPva7L+X+DiyeAr/VT8URExPuogJEqVeZ08cbinby6aDtOl0n/4P28ETKT+vs2uhs07wdDXoaIzpbmFBER76YCRqrMrkN5jJu9jnVp2QRRxOuR33BJzhcYWS7wD4XLH4feI7X5ooiInDEVMHLGTNPkw9WpPD1vC4WlToYGJPNs4AcEZbtXVqbrMIibAvUj/viJRERETpEKGDkjmY4iJvx3PUtSDhFBFjMbfMy5hcuhEAhrCfEvQvuBf/o8IiIilaECRk7btxsO8o85G8gpKGa07/dM9PsMv8L8Y4N074f+E8AvyOqYIiJSC6mAkUpzFJXy2Feb+OLn/XQx9vBpvffp4NwOTqDZuTD0ZYjoYnVMERGpxVTASKWs2nWEB2ev42j2UR7x+Zw7feZjcx4bpDvwUegzSoN0RUSk2qmAkVNSVOrkxcRtvPPjLi4xfubpwJlEmofcJ7tcB1dMgfon3mhTRESkqqmAkT+15aCDcZ8mczR9L2/4fsCV9p/ABMJaHBuke7nVEUVEpI5RASMn5XSZvPPjLl76bgs38R0TAmYTTCEYdjh/DFz8fxqkKyIillABIyeUllXAg7PXkbf3Zz71/Tc9bTvdJ5r2haGvQGRXawOKiEidpgJGKjBNk8+T9jH165+5y/kpo/2+xcdwYfqHYHgG6dqtjikiInWcChjxOJJXzD/mbKBky3zm+L5PM5/D7hNdrsW44hkN0hURkRpDBYwAsHBLBs99vpT7S94h3u8nAMzQ5hjxL0KHQRanExERqUgFTB2XX1zGU3M3Yvt5BrN9PiHEXohp2DFiEzAG/B/41bM6ooiIyO+ogKnDkvYe5Y2P53B/wRv08t0BgCu6N7arXoXIbhanExEROTkVMHVQqdPFtO/WE7D8Oabbv8HH5qLMNxifyx/D1vdODdIVEZEaTwVMHbMjM5f//Odd7sp5g+Y+7pV0SztehW/8VAiJsjidiIjIqVEBU0e4XCazl6whZMk/edy2CmxQEBhN0DUv4dvxCqvjiYiIVIoKmDrgYHY+82dMYdjRdwmxFeLERlGfe6kX94gG6YqIiFdSAVPLLf5hCQ0X/Z1RbAcDDod0odEtb1IvqofV0URERE6bCphaKicnh59mTmTAkdn4Gk4KjEDyL3yYJpf8VYN0RUTE66mAqYVSfl5K/a/u4nIywYDtjS6l1W2v0aRBM6ujiYiIVAkVMLVMTtZhGnx1B+FkkWE0Jn/gs7S/4HqrY4mIiFQpFTC1zLYZ93EOWewzomgwbgURIQ2tjiQiIlLlbFYHkKqz9tsPOMfxHU7TID/+deqpeBERkVpKBUwtcSg9jTarHwYgqdntdOw70OJEIiIi1UcFTC1gulykzvwLDXGw296KXrc/a3UkERGRaqUCphZY+eWb9ClcTolpx3bd2/j6B1odSUREpFqpgPFy+/Zup+u6fwGwvu29tOxynsWJREREqp8KGC/mdLo4/NE9hBgF7PDtSO9bH7c6koiIyFmhAsaLLfvkOXqW/EyR6Uvwze9g8/G1OpKIiMhZoQLGS+3Yup6+214AYEuX8US21d5GIiJSd6iA8ULFJSUUf3YP9Yxitgb0oOf1E62OJCIiclapgPFCyz98gi7OLeQRSPiIf2Noc0YREaljVMB4mQ2/rOKCvW8CsLfvwzRs2t7iRCIiImdflRcwjz32GIZhVLjFxMR4zhcVFZGQkECjRo0IDg5m2LBhZGRkVHiO1NRU4uPjCQoKIjw8nIceeoiysrKqjup18goK8fvqPvyNMjYHx9IlfozVkURERCxRLZs5dunShe+///7XF/H59WXGjRvHvHnz+OyzzwgNDWXMmDFcd911LF++HACn00l8fDyRkZGsWLGCgwcPcvvtt+Pr68vTTz9dHXG9xqoZkxho7iKHYFrc8Q4YhtWRRERELFEtBYyPjw+RkZG/O56Tk8O///1vZs2axaWXXgrA+++/T6dOnVi1ahXnnXce3333HZs3b+b7778nIiKCnj178uSTTzJx4kQee+wx/Pz8qiNyjbd2xUIGZMwEAzIuepoOjZtbHUlERMQy1TIGZvv27URHR9OmTRuGDx9OamoqAElJSZSWljJw4K8bDcbExNCiRQtWrlwJwMqVK+nWrRsRERGeNnFxcTgcDjZt2nTS1ywuLsbhcFS41RZZ2Tk0/O5v+BguNjUYSIfLRlodSURExFJVXsD069ePGTNmMH/+fN588012797NRRddRG5uLunp6fj5+REWFlbhMREREaSnpwOQnp5eoXgpP19+7mSmTJlCaGio59a8ee24QmGaJj/PeJA27OOI0YC2d7xldSQRERHLVXkX0uDBgz1/7t69O/369aNly5bMnj2bwMDq22Rw0qRJjB8/3nPf4XDUiiJm2fdfcenRz8GA3EEv0ii0idWRRERELFft06jDwsLo0KEDO3bsIDIykpKSErKzsyu0ycjI8IyZiYyM/N2spPL7JxpXU87f35+QkJAKN2+XfugQrZf/HZthsinialrFXmd1JBERkRqh2guYvLw8du7cSVRUFH369MHX15eFCxd6zqekpJCamkpsbCwAsbGxbNiwgczMTE+bxMREQkJC6Ny5c3XHrTFcLpNNMx6gGZlk2sLpOPJVqyOJiIjUGFXehfT3v/+doUOH0rJlSw4cOMCjjz6K3W7nlltuITQ0lNGjRzN+/HgaNmxISEgI999/P7GxsZx33nkADBo0iM6dOzNixAimTp1Keno6jzzyCAkJCfj7+1d13Bpr4dxZXJ4/D4Cyoa/jExRmbSAREZEapMoLmH379nHLLbdw5MgRmjRpwoUXXsiqVato0sQ9duOll17CZrMxbNgwiouLiYuLY9q0aZ7H2+125s6dy3333UdsbCz16tVj5MiRPPHEE1UdtcbanbaP7kkPgwFbWtxKp15xVkcSERGpUQzTNE2rQ1QHh8NBaGgoOTk5XjUepszpYtnU6xhQvJiDPs2IeOgnbP71rI4lIiJyVpzq97f2QqphFnw+nQHFi3Fi4Hv9dBUvIiIiJ6ACpgbZsm0HsZufBGBnh7tpHHOBxYlERERqJhUwNURRSRlHZ99HQyOPfX5taX/jv6yOJCIiUmOpgKkhFsx6ifPLfqIUH0Ju+TeGT92ZcSUiIlJZKmBqgLXr13PJ7hcASO3+ACGte1mcSEREpGZTAWMxR2Ex5pdjCDEKSQ3qQtur/2F1JBERkRpPBYzFvp/5NOe41lGEH41vew/sVb40j4iISK2jAsZCP65axeCD7kX8MvtNIig6xuJEIiIi3kEFjEUOOwoImX8/gUYJe0L60iJurNWRREREvIYKGAuYpsnSGf+kB9vIJ5Coke+BTT8KERGRU6VvTQt8t3gRQ47MACDn4ifxb9TS2kAiIiJeRgXMWZZ2KJsWS8fjb5Sxp1F/ogfcZXUkERERr6MC5ixyuUx+mvl/dDL24DBCaD7yHTAMq2OJiIh4HRUwZ9HX33zN1bmfAlAc9zz2kEiLE4mIiHgnFTBnyfb9mXRbMxEfw8WeqME0Oe8mqyOJiIh4LRUwZ0FJmYtNH/ydNsYBsu2NaDlimtWRREREvJoKmLNgzpxPuKb4f+47V72KEdTQ2kAiIiJeTgVMNUvemcYFG/8JQGqrGwjrMcTiRCIiIt5PBUw1KigpY9/H42hmHOaIbyQtbnnJ6kgiIiK1ggqYavTFJ+8xpCwRFwb+178N/vWtjiQiIlIrqICpJis2bGPQzqcAOBgziuCOA6wNJCIiUouogKkGOQWl5M0ZS7iRzSH/ljQd9rTVkURERGoVFTDV4IsPX2OQazlObNS/5d/gG2h1JBERkVpFBUwVS/xpHdfsfwGAQz3HENDqHIsTiYiI1D4qYKpQRk4hft+Mo4GRR0a9jkQO+afVkURERGolFTBVxDRN5n7wHBeTRCk+NLztPfDxszqWiIhIraQCpor8b+kqbjzs3iIg57wJ+EZ1tTiRiIhI7aUCpgrsOZRL5OIHqW8UkhHag8aD/m51JBERkVpNBcwZcrpMFn7wL84zNlFk+NNkxPtgs1sdS0REpFZTAXOGPp2/kOGOfwNQNOAxbI3bWpxIRESk9lMBcwY278ui86qJBBilZDSOJeyie62OJCIiUieogDlNxWVOVn84mZ62HRTY6hF+2ztg08cpIiJyNugb9zR9+OU8hhfOAsAZ9wxGWHOLE4mIiNQdKmBOw9qdBzl//cP4GU4yoy+j/rkjrI4kIiJSp6iAqaS84jK2fPwwnWyp5NlDCb/1bTAMq2OJiIjUKSpgKumD2Z9xa+kXANivegWCm1icSEREpO5RAVMJpsvFsAPPYzdMDrW+msAe11odSUREpE5SAVMJhs1GxKiPKGw9iCY3vmJ1HBERkTqrRhcwb7zxBq1atSIgIIB+/frx008/WR0JwmMIHPkZBDawOomIiEidVWMLmE8//ZTx48fz6KOP8vPPP9OjRw/i4uLIzMy0OpqIiIhYrMYWMC+++CJ33303o0aNonPnzrz11lsEBQXx3nvvWR1NRERELFYjC5iSkhKSkpIYOHCg55jNZmPgwIGsXLnyhI8pLi7G4XBUuImIiEjtVCMLmMOHD+N0OomIiKhwPCIigvT09BM+ZsqUKYSGhnpuzZtrZVwREZHaqkYWMKdj0qRJ5OTkeG5paWlWRxIREZFq4mN1gBNp3LgxdrudjIyMCsczMjKIjIw84WP8/f3x9/c/G/FERETEYjXyCoyfnx99+vRh4cKFnmMul4uFCxcSGxtrYTIRERGpCWrkFRiA8ePHM3LkSPr27cu5557Lyy+/TH5+PqNGjbI6moiIiFisxhYwN910E4cOHWLy5Mmkp6fTs2dP5s+f/7uBvSIiIlL3GKZpmlaHqA4Oh4PQ0FBycnIICQmxOo6IiIicglP9/q6RY2BERERE/ogKGBEREfE6KmBERETE69TYQbxnqnxoj7YUEBER8R7l39t/NkS31hYwubm5ANpSQERExAvl5uYSGhp60vO1dhaSy+XiwIED1K9fH8Mwqux5HQ4HzZs3Jy0tTbObzoA+x6qhz7Fq6HOsGvocq0Zd/xxN0yQ3N5fo6GhstpOPdKm1V2BsNhvNmjWrtucPCQmpk3+xqpo+x6qhz7Fq6HOsGvocq0Zd/hz/6MpLOQ3iFREREa+jAkZERES8jgqYSvL39+fRRx/VztdnSJ9j1dDnWDX0OVYNfY5VQ5/jqam1g3hFRESk9tIVGBEREfE6KmBERETE66iAEREREa+jAkZERES8jgqYSnrjjTdo1aoVAQEB9OvXj59++snqSF5lypQpnHPOOdSvX5/w8HCuueYaUlJSrI7l9Z555hkMw2Ds2LFWR/E6+/fv57bbbqNRo0YEBgbSrVs31q5da3Usr+J0OvnnP/9J69atCQwMpG3btjz55JN/updNXffDDz8wdOhQoqOjMQyDL7/8ssJ50zSZPHkyUVFRBAYGMnDgQLZv325N2BpIBUwlfPrpp4wfP55HH32Un3/+mR49ehAXF0dmZqbV0bzG0qVLSUhIYNWqVSQmJlJaWsqgQYPIz8+3OprXWrNmDW+//Tbdu3e3OorXOXr0KBdccAG+vr58++23bN68mRdeeIEGDRpYHc2rPPvss7z55pu8/vrrbNmyhWeffZapU6fy2muvWR2tRsvPz6dHjx688cYbJzw/depUXn31Vd566y1Wr15NvXr1iIuLo6io6CwnraFMOWXnnnuumZCQ4LnvdDrN6Ohoc8qUKRam8m6ZmZkmYC5dutTqKF4pNzfXbN++vZmYmGhefPHF5gMPPGB1JK8yceJE88ILL7Q6hteLj48377zzzgrHrrvuOnP48OEWJfI+gDlnzhzPfZfLZUZGRprPPfec51h2drbp7+9vfvzxxxYkrHl0BeYUlZSUkJSUxMCBAz3HbDYbAwcOZOXKlRYm8245OTkANGzY0OIk3ikhIYH4+PgKfy/l1H311Vf07duXG264gfDwcHr16sU777xjdSyvc/7557Nw4UK2bdsGwLp161i2bBmDBw+2OJn32r17N+np6RX+bYeGhtKvXz995xxTazdzrGqHDx/G6XQSERFR4XhERARbt261KJV3c7lcjB07lgsuuICuXbtaHcfrfPLJJ/z888+sWbPG6ihea9euXbz55puMHz+ef/zjH6xZs4a//e1v+Pn5MXLkSKvjeY3/+7//w+FwEBMTg91ux+l08tRTTzF8+HCro3mt9PR0gBN+55Sfq+tUwIhlEhIS2LhxI8uWLbM6itdJS0vjgQceIDExkYCAAKvjeC2Xy0Xfvn15+umnAejVqxcbN27krbfeUgFTCbNnz+ajjz5i1qxZdOnSheTkZMaOHUt0dLQ+R6k26kI6RY0bN8Zut5ORkVHheEZGBpGRkRal8l5jxoxh7ty5LF68mGbNmlkdx+skJSWRmZlJ79698fHxwcfHh6VLl/Lqq6/i4+OD0+m0OqJXiIqKonPnzhWOderUidTUVIsSeaeHHnqI//u//+Pmm2+mW7dujBgxgnHjxjFlyhSro3mt8u8VfeecnAqYU+Tn50efPn1YuHCh55jL5WLhwoXExsZamMy7mKbJmDFjmDNnDosWLaJ169ZWR/JKl112GRs2bCA5Odlz69u3L8OHDyc5ORm73W51RK9wwQUX/G4a/7Zt22jZsqVFibxTQUEBNlvFrxO73Y7L5bIokfdr3bo1kZGRFb5zHA4Hq1ev1nfOMepCqoTx48czcuRI+vbty7nnnsvLL79Mfn4+o0aNsjqa10hISGDWrFn873//o379+p6+3NDQUAIDAy1O5z3q16//u3FD9erVo1GjRhpPVAnjxo3j/PPP5+mnn+bGG2/kp59+Yvr06UyfPt3qaF5l6NChPPXUU7Ro0YIuXbrwyy+/8OKLL3LnnXdaHa1Gy8vLY8eOHZ77u3fvJjk5mYYNG9KiRQvGjh3Lv/71L9q3b0/r1q355z//SXR0NNdcc411oWsSq6dBeZvXXnvNbNGihenn52eee+655qpVq6yO5FWAE97ef/99q6N5PU2jPj1ff/212bVrV9Pf39+MiYkxp0+fbnUkr+NwOMwHHnjAbNGihRkQEGC2adPGfPjhh83i4mKro9VoixcvPuH/D0eOHGmapnsq9T//+U8zIiLC9Pf3Ny+77DIzJSXF2tA1iGGaWipRREREvIvGwIiIiIjXUQEjIiIiXkcFjIiIiHgdFTAiIiLidVTAiIiIiNdRASMiIiJeRwWMiIiIeB0VMCIiIuJ1VMCIiIiI11EBIyIiIl5HBYyIiIh4HRUwIiIi4nX+H3qNWNdP82KHAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df2 = pd.DataFrame()\n",
    "df2[\"agent1\"] = l_\n",
    "df2[\"agent2\"] = l\n",
    "df2.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   number  squares  cubes\n",
      "0       1        1      1\n",
      "1       2        4      8\n",
      "2       3        9     27\n",
      "3       4       16     64\n",
      "4       5       25    125\n",
      "5       6       36    216\n",
      "6       7       49    343\n"
     ]
    }
   ],
   "source": [
    "# importiong the modules\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# creating the Numpy array\n",
    "array = np.array([[1, 1, 1], [2, 4, 8], [3, 9, 27],\n",
    "\t\t\t\t[4, 16, 64], [5, 25, 125], [6, 36, 216],\n",
    "\t\t\t\t[7, 49, 343]])\n",
    "\n",
    "# creating a list of index names\n",
    "index_values = ['first', 'second', 'third',\n",
    "\t\t\t\t'fourth', 'fifth', 'sixth', 'seventh']\n",
    "\n",
    "# creating a list of column names\n",
    "column_values = ['number', 'squares', 'cubes']\n",
    "\n",
    "# creating the dataframe\n",
    "df = pd.DataFrame(data = array,\n",
    "\t\t\t\tindex = index_values,\n",
    "\t\t\t\tcolumns = column_values)\n",
    "\n",
    "# displaying the dataframe\n",
    "print(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = [\"ft06\", \"la01\", \"la06\", \"la11\", \"la21\", \"la31\", \"la36\", \"orb01\", \"swv01\", \"swv06\", \"swv11\", \"yn1\"]\n",
    "# instances = [\"swv06\"]\n",
    "num_episodes = 1000\n",
    "memory_size = 100000\n",
    "batch_size = 64\n",
    "target_update = 100\n",
    "noisy = False\n",
    "plotting_inteval = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 随机环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_rate = 0.5\n",
    "cv = 0.2\n",
    "n = 10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance = \"swv06\"\n",
    "policy_file = f\"policies/dqn_mlp/{instance}_num_episodes={num_episodes}_memory_size={memory_size}_target_update={target_update}_noisy={noisy}.pth\"\n",
    "env = HeuristicJsspEnv(instance)\n",
    "agent = DQNAgent(env, memory_size, batch_size, target_update, noisy=noisy)\n",
    "agent.load_dqn(policy_file)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对比随机动作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action, makespan = 2077\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "while not done:\n",
    "    action = agent.select_action(state, determine=True)\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action, makespan = {makespan}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random action, makespan = 2375.10\n"
     ]
    }
   ],
   "source": [
    "makespans = []\n",
    "for _ in range(10):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = np.random.randint(0, 8)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "\n",
    "    makespan = info[\"makespan\"]\n",
    "    makespans.append(makespan)\n",
    "print(f\"Random action, makespan = {np.mean(makespans):.2f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对比 heuristic rule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action=6, makespan = 2287\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "while not done:\n",
    "    action = 6\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action=6, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### agent train on swv06, test on swv07-swv09"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On instance swv07, makespan = 1914\n",
      "On instance swv08, makespan = 2366\n",
      "On instance swv09, makespan = 2074\n"
     ]
    }
   ],
   "source": [
    "test_instances = [f\"swv0{x}\" for x in range(7,10)]\n",
    "for instance in test_instances:\n",
    "    test_env = HeuristicJsspEnv(instance)\n",
    "    state = test_env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = test_env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"On instance {instance}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On instance swv07, makespan = 2173\n",
      "On instance swv08, makespan = 2518\n",
      "On instance swv09, makespan = 2472\n"
     ]
    }
   ],
   "source": [
    "test_instances = [f\"swv0{x}\" for x in range(7,10)]\n",
    "for instance in test_instances:\n",
    "    test_env = HeuristicJsspEnv(instance)\n",
    "    state = test_env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = np.random.randint(0, 8)\n",
    "        next_state, reward, done, info = test_env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"On instance {instance}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 哪些动作做的多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent choose action 3 for 49 times\n",
      "Agent choose action 6 for 154 times\n",
      "Agent choose action 7 for 38 times\n",
      "Agent choose action 5 for 19 times\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "for i in Counter(agent_actions).keys():\n",
    "    print(f\"Agent choose action {i} for {Counter(agent_actions)[i]} times\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "action | rule  \n",
    "3        shortest processing time  \n",
    "6        most operations remaining  \n",
    "7        least operations remaining  \n",
    "5        shortest processing time remained  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 哪些state重要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = agent._get_dqn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.0000,  0.0000, -1.0000,  ...,  0.0000,  0.0000,  0.7333],\n",
      "       device='cuda:0', requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "state = torch.FloatTensor(state).to(agent.device)\n",
    "state.requires_grad_(True)\n",
    "print(state)\n",
    "model.eval()\n",
    "Q = model(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "external_grad = torch.tensor(np.ones(8)).to(agent.device)\n",
    "Q.backward(gradient=external_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0558,  0.0433, -0.0338,  ..., -0.6092, -1.2825, -0.1516],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_grad = state.grad.detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15.880271"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_grad.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([   7,   62,   97,  247,  307,  347,  397,  427,  477,  527,  537,\n",
       "         587,  677,  687,  737,  867,  977, 1297, 1447, 1507, 1517, 1577,\n",
       "        1597, 1607, 1907, 1917, 2012, 2057, 2077, 2097, 2187, 2357, 2376,\n",
       "        2467, 2537, 2547, 2677, 2707, 2817, 2846, 2847, 2976], dtype=int64),)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(state_grad>3)              # 阈值为 3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "显然agent考虑的主要是 7 结尾的特征，即工件的剩余加工时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent consider state 7 for 100 times\n",
      "Agent consider state 2 for 78 times\n",
      "Agent consider state 8 for 43 times\n",
      "Agent consider state 6 for 64 times\n"
     ]
    }
   ],
   "source": [
    "a = np.where(state_grad>1)[0]           # 阈值为 1\n",
    "a = [x%10 for x in a]\n",
    "for i in Counter(a).keys():\n",
    "    print(f\"Agent consider state {i} for {Counter(a)[i]} times\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2 - type  \n",
    "6 - waiting_time  \n",
    "7 - remain_time  \n",
    "8 - doable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12.657236, 15.880271, 10.319552, 14.00865 , 11.86575 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_grad[np.where(state_grad>10)]   # 阈值为 10"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第一个动作，有多重要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent first action is 0, makespan = 2049\n",
      "Agent first action is 1, makespan = 2052\n",
      "Agent first action is 2, makespan = 2085\n",
      "Agent first action is 3, makespan = 2077\n",
      "Agent first action is 4, makespan = 1951\n",
      "Agent first action is 5, makespan = 2241\n",
      "Agent first action is 6, makespan = 2049\n",
      "Agent first action is 7, makespan = 2049\n"
     ]
    }
   ],
   "source": [
    "for first_action in range(8):\n",
    "    state = env.reset()\n",
    "    next_state, reward, done, info = env.step(first_action)\n",
    "    state = next_state\n",
    "    while not done:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"Agent first action is {first_action}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实际agent选择action=3，可以看出不是最好。agent应该学到了根据这些工件的剩余加工时间来调度，"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### enhanced policy by greedy search\n",
    "greedy can be substituted by MCTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_greedy_action(env_start):\n",
    "    \"\"\"Given env_start at some status, simulate the process with first action [0,1,2,3,4,5,6,7]\n",
    "    and return the action with the lowest makespan.\n",
    "    \"\"\"\n",
    "    makespans = []\n",
    "    for a in range(8):\n",
    "        env = deepcopy(env_start)\n",
    "        next_state, reward, done, info = env.step(a)\n",
    "        state = next_state\n",
    "        while not done:\n",
    "            action = agent.select_action(state, determine=True)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            state = next_state\n",
    "        makespans.append(info[\"makespan\"])\n",
    "    return np.argmin(makespans), makespans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2049, 2052, 2085, 2077, 1951, 2241, 2049, 2049]\n",
      "[1981, 1984, 1984, 1951, 1981, 1984, 1951, 1981]\n",
      "[2009, 1951, 2009, 1951, 1951, 2009, 1951, 2125]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1993, 1996, 1951, 1993, 1951, 2013, 1951, 1996]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n"
     ]
    }
   ],
   "source": [
    "step_size =30\n",
    "\n",
    "done = False\n",
    "state = env.reset()\n",
    "while not done:\n",
    "    a, makespans = get_greedy_action(env)\n",
    "    next_state, reward, done, info = env.step(a)\n",
    "    state = next_state\n",
    "    print(makespans)\n",
    "    steps = 0\n",
    "    while steps < step_size:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "        if done:\n",
    "            break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "只有 first action 能优化makespan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2049, 2052, 2085, 2077, 1951, 2241, 2049, 2049]\n",
      "[2082, 2142, 2109, 1951, 2058, 2045, 2000, 2082]\n",
      "[1951, 2026, 2026, 1951, 1951, 2026, 1951, 1951]\n",
      "[1951, 2025, 2025, 1951, 2025, 1951, 2025, 1951]\n",
      "[1951, 2045, 2045, 1951, 2045, 1981, 2045, 1981]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_8940\\1018890013.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmakespans\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_greedy_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_8940\\3722718511.py\u001b[0m in \u001b[0;36mget_greedy_action\u001b[1;34m(env_start)\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdetermine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m             \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m             \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[0mmakespans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"makespan\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\桌面文件夹\\桌面\\毕设\\github\\DRL-for-Job-Shop-Scheduling\\gymjsp\\jsspenv.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m   1127\u001b[0m         \u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobserve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1128\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1129\u001b[1;33m         \u001b[0mobservation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mg2s\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1130\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m         info = {\n",
      "\u001b[1;32md:\\桌面文件夹\\桌面\\毕设\\github\\DRL-for-Job-Shop-Scheduling\\gymjsp\\jsspenv.py\u001b[0m in \u001b[0;36mg2s\u001b[1;34m(self, g, state_len)\u001b[0m\n\u001b[0;32m    570\u001b[0m             \u001b[0mjob_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstep_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    571\u001b[0m             \u001b[0mt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprior_processing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstep_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 572\u001b[1;33m             \u001b[0mchanged_t\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprocessing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstep_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    573\u001b[0m             \u001b[0msum_t\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprior_processing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    574\u001b[0m             \u001b[0mstate\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m10\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_jobs\u001b[0m  \u001b[1;31m# job id\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "step_size =10\n",
    "\n",
    "done = False\n",
    "state = env.reset()\n",
    "while not done:\n",
    "    a, makespans = get_greedy_action(env)\n",
    "    next_state, reward, done, info = env.step(a)\n",
    "    state = next_state\n",
    "    print(makespans)\n",
    "    steps = 0\n",
    "    while steps < step_size:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "        if done:\n",
    "            break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "dce9ce0b8994960befdaf1abc06919148bccb19973be1d090d69590bd56698c8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}