Hill_Climbing-Steepest_Hill-with_Adaptive_noise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hill Climbing - Steepest Hill - with Adaptive Noise\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train hill climbing with adaptive noise scaling with OpenAI Gym's Cartpole environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#!python -m pip install pyvirtualdisplay\n",
    "#from pyvirtualdisplay import Display\n",
    "#display = Display(visible=0, size=(1400, 900))\n",
    "#display.start()\n",
    "\n",
    "#is_ipython = 'inline' in plt.get_backend()\n",
    "#if is_ipython:\n",
    "#    from IPython import display\n",
    "\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define the Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)\n",
      "action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "\n",
    "class Policy():\n",
    "    def __init__(self, s_size=4, a_size=2):\n",
    "        self.w = 1e-4*np.random.rand(s_size, a_size)  # weights for simple linear policy: state_space x action_space\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = np.dot(state, self.w)\n",
    "        x = self.stable_softmax(x)\n",
    "        return x\n",
    "    \n",
    "    def pick_action(self, state):\n",
    "        probs = self.forward(state)\n",
    "        action = np.random.choice(2, p=probs) # option 1: stochastic policy\n",
    "        #action = np.argmax(probs)              # option 2: deterministic policy\n",
    "        return action\n",
    "    \n",
    "    def stable_softmax(self, x):\n",
    "        z = x - max(x)\n",
    "        numerator = np.exp(z)\n",
    "        denominator = np.sum(numerator)\n",
    "        softmax = numerator/denominator\n",
    "\n",
    "        return softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with Stochastic Policy Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 21.23\n",
      "Episode 200\tAverage Score: 22.78\n",
      "Episode 300\tAverage Score: 21.56\n",
      "Episode 400\tAverage Score: 23.41\n",
      "Episode 500\tAverage Score: 22.62\n",
      "Episode 600\tAverage Score: 24.02\n",
      "Episode 700\tAverage Score: 124.87\n",
      "Episode 800\tAverage Score: 191.00\n",
      "Environment solved in 820 episodes!\tAverage Score: 195.66\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)       # 0\n",
    "np.random.seed(0) # 0\n",
    "\n",
    "policy = Policy()\n",
    "\n",
    "def hill_climbing(n_episodes=1000, n_cand=10, max_t=1000, gamma=1.0, print_every=100, noise_scale=1e-2, noise_decay=0.99, verbose=True):\n",
    "    \"\"\"Implementation of Steepest hill climbing with adaptive noise scaling.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        noise_scale (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_R = -np.Inf\n",
    "    #best_cand_R = best_R\n",
    "    best_w = policy.w\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        #breakpoint()\n",
    "        for c in range(n_cand):\n",
    "            best_cand_R = -np.Inf\n",
    "            policy.w = best_w + noise_scale * np.random.rand(*policy.w.shape)\n",
    "            rewards = []\n",
    "            state = env.reset()\n",
    "            for t in range(max_t):\n",
    "                action = policy.pick_action(state)\n",
    "                state, reward, done, _ = env.step(action)\n",
    "                rewards.append(reward)\n",
    "                if done:\n",
    "                    break\n",
    "\n",
    "            discounts = [gamma**i for i in range(len(rewards)+1)]\n",
    "            R = sum([a*b for a,b in zip(discounts, rewards)])\n",
    "\n",
    "            if R > best_cand_R: # found better weights\n",
    "                #breakpoint()\n",
    "                best_cand_R = R\n",
    "                best_cand_w = policy.w\n",
    "        \n",
    "        if best_cand_R >= best_R:\n",
    "            #breakpoint()\n",
    "            best_R = best_cand_R\n",
    "            best_w = best_cand_w\n",
    "            policy.w = best_w\n",
    "            noise_scale *= noise_decay                # Adaptive\n",
    "        else: # did not find better weights\n",
    "            noise_scale /= noise_decay                # Adaptive\n",
    "            #policy.w += noise_scale * np.random.rand(*policy.w.shape) # generuojamas kandidatas\n",
    "            \n",
    "        scores_deque.append(sum(rewards))\n",
    "        scores.append(sum(rewards))\n",
    "        if verbose & (i_episode % print_every == 0):\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "        if (np.mean(scores_deque)>=195.0) & (len(scores_deque)==100):\n",
    "            if verbose:\n",
    "                print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(len(scores), np.mean(scores_deque)))\n",
    "            policy.w = best_w\n",
    "            break\n",
    "        \n",
    "    return scores\n",
    "            \n",
    "scores = hill_climbing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot the Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABFlklEQVR4nO2dd5wU9f3/X++rXKMfHTyaIogeelYUFTV2UVPUJGpsqLHEdEu+0Zho/JmoMRoTsUWNQY0aNWJDRLHjUaQK0tsBR4eDa7vv3x87Mzsz+5nZmb2d3bnb9/PxgNud+tnPzr7fn3f5vD/EzBAEQRAEAMjLdgMEQRCE8CBKQRAEQTAQpSAIgiAYiFIQBEEQDEQpCIIgCAYF2W5AW+jZsydXVVVluxmCIAjtilmzZm1h5krVvnatFKqqqlBbW5vtZgiCILQriGi10z5xHwmCIAgGohQEQRAEA1EKgiAIgoEoBUEQBMFAlIIgCIJgEJhSIKKBRDSdiBYT0UIi+om2vTsRTSWib7S/3Uzn3EJEy4hoCRGdGlTbBEEQBDVBWgqtAH7OzAcCOArAdUQ0EsDNAKYx83AA07T30PZdCGAUgNMAPEJE+QG2TxAEQbAR2DwFZq4DUKe93k1EiwH0BzABwAnaYU8D+ADAr7XtzzNzE4CVRLQMwBEAPguqjYIghJuFG3aiqTWKAd1KMGfNDuQTYfSALnh34UYU5ufhwiMGAQAWrN+Jz1dsxVkH98Mrc9ahe2kRvt64Gwf174JDB3XFV+t2gECIRBmrtzYAALY2NOOUkb2xbPMe7NrXAgDYuKsRvTt3AgHY2xxBa5TRv2sJIsxYu20vepQVWdq3taE5Ydu+lgiYgdKi5GParQ3NAJBwDS/s36cCZx3cz/d5ycjI5DUiqgIwBsAXAHprCgPMXEdEvbTD+gP43HTaOm2b/VoTAUwEgEGDBgXYakEQss2Zf/0YADCkZxlWbGlI2F89qCtG9OmMGybPwcotDZi1ejveWrDR8/Wf+2KN7zYRxf6al6JRbTNvV+HnWBVnHdyvfSoFIioH8DKAm5h5Fzl/ctWOhBWAmHkSgEkAUFNTIysECUIOoFIIALCvOQIAWLNtLwCgQXvvl0cvPgxXPzvLeH/+mP54Zc76hOP+/oNDcfrovgCAic/U4t1Fm3DVcYNx25kjAQB1O/fh6D++DwD4+NcnYkC3Usd7XvbUTExfUg8AmFDdDw9eOCaltqebQLOPiKgQMYXwHDO/om3eRER9tf19AWzWtq8DMNB0+gAAG4JsnyAI7Rv7IDNtK0k6jF0L8+MiU7+1uQ15Dq/bE0FmHxGAJwAsZub7TbteB3Cp9vpSAK+Ztl9IRMVENBjAcAAzg2qfIAgdj1R1gl18k4NWKCzISzjGfKTldRKdYFYmYVIfQbqPxgK4GMB8IpqrbbsVwD0AXiSiKwCsAfBdAGDmhUT0IoBFiGUuXcfMqdmCgiDkFLpQ5USPs7fzbRLcSaAX5lPiMaZjnawG5T1d7p9Ngsw++hjOCvAkh3PuAnBXUG0SBKFjE7D3SO0+glkRIGG/4z3CowcsyIxmQRA6DOlyHzmN8i1KQXcfOVgKTi4o1V3DpB9EKQiCkPPYdYAX9xHI8geA1VLI82MphEgriFIQBEGw4aQUiiyWQuKxZusgWZzAqhPCoxVEKQiC0G6xi9LUA83JrhzDGlPQs49MisAkUX1ZCiFClIIgCB2G1GMKVgntJNALTO4j/Riry8i7pWC5f4gUhCgFQRDaPUaZiTRfz47KfQSH+QZ+BH2IdIIoBUEQOhCpagWPUlntPoqT6oxmsRQEQRDSgF2YRlP0H6Uyo1nVBrK4klJqStYRpSAIQs5DRPj3VUcmPa5AIemtGUfq7cp7WuYphEeDiFIQBCGU+ClupwvVtsQUepYXx6/noSCe6lhroNn7vcV9JAiCkISWiH8Rn2qVVIK3sEK+0lJweC1KQRAEIX20RqNJj4lbCKz9TY2EGc2KYx644JCk5/oJNKc6pyJoRCkIghBKUrMUUr+fUw0jnWOHVTqcp3YZ+VtPITymgigFQRBCSSSaQkwhZfdRcgeSynUEuBXE83H/8OgEUQqCIIQTP0pBJ4VTAHgTyk4ppk6ZQzJ5TRAEIY2k4nP3Ok+hU6F7FpEKp7IVTueFaeEcP4hSEAQhlHiR7/bVz7xaCvbRvd15pJLnzpZCajjNb8g2Qa7R/CQRbSaiBaZtLxDRXO3fKn2ZTiKqIqJ9pn3/CKpdgiC0D1IJD0Q9aoUEAU/JR/ZOgeN0CPQwTV4Lco3mfwJ4GMAz+gZmvkB/TUT3AdhpOn45M1cH2B5BENoRqZSs8HpOwprMCZZDopB2VAppEOhhshSCXKN5BhFVqfZR7Bv5HoDxQd1fEIT2TSoxY89KweM2y37H2IGnWyYg8xSsHAdgEzN/Y9o2mIjmENGHRHSc04lENJGIaomotr6+PviWCoKQFby6gsx4Ni4Uy286FbbT8TfvwB8hMhSyphQuAjDZ9L4OwCBmHgPgZwD+TUSdVScy8yRmrmHmmspK9WQSQRBykxR1gieh7BhoToOyCFOmUsaVAhEVADgfwAv6NmZuYuat2utZAJYD2D/TbRMEITxkMqYA2KuWJuJkKbTXEtlOZMNSOBnA18y8Tt9ARJVElK+9HgJgOIAVWWibIAghIaXsI89Kwf6ePMxTcNju6Y7thyBTUicD+AzAAUS0joiu0HZdCKvrCADGAZhHRF8BeAnANcy8Lai2CYIQfrwIeF1Q64LZQw09y/H26zi9j21zSklNTS2EdZ5CkNlHFzls/5Fi28sAXg6qLYIgtD9Syc3xWvsonT78jjZPQWY0C4IQSlIpbud9RnPi+2RVUr1eKxXCZCmIUhAEIZT40Qm6UI2kOnnNNqPZl4xOUaLLPAVBEAQfeBn1290u9bubPF07MWMo9aF6iAb5aUGUgiAIoSTIkbQykOz4xv+1/BImxSJKQRCEUOI1kygVEmodUerCPR1B4jA5kkQpCIIQSrxYCtOXbE4pIJ1sTWY/gj7VyWthyjgyE2SVVEEQhJTxIuv/9M4S7Nej1Pe11QXxUpxvEE7ZnjJiKQiCEEq8GgAbdza2+V72Gc3+ltJMQ+2jNl8hfYhSEAQhlKRS+8griesptEEwp0GiS0xBEAQhCX4Epd/RetI6R77unRoyT0EQBMEHbal4mvwcxftUA8YdLKggSkEQhFASoPdIufxmqgXq0lLmIg3XSBeiFARBCCWppJp6xZ5G2pbBfjoMhTA5kkQpCIIQSoIUlMpFdszZRxkYu4d1noIoBUEQQkmw7iNv23IRUQqCIISSYFNS/b0PmjApJFEKgiCEEq86wb4WgqdzlO6jFEtnp4GciCkQ0ZNEtJmIFpi23UFE64lorvbvDNO+W4hoGREtIaJTg2qXIAjtgyADzapaR5lXBGFSBXGCtBT+CeA0xfYHmLla+/cmABDRSMTWbh6lnfMIEeUH2DZBEEJOJkVmQpXUDjb3wA+BKQVmngFgm8fDJwB4npmbmHklgGUAjgiqbYIghJ9MxhSyTZiak42YwvVENE9zL3XTtvUHsNZ0zDptWwJENJGIaomotr6+Pui2CoKQJbzqhDvfWIS9zZE23YvImiIqMYXM8XcAQwFUA6gDcJ+2XfUdKPuJmScxcw0z11RWVgbSSEEQsk+QloIdAmVcE8g8BQDMvImZI8wcBfAY4i6idQAGmg4dAGBDJtsmCEK4yOboOWzupUySUaVARH1Nb88DoGcmvQ7gQiIqJqLBAIYDmJnJtgmCEC6CzT5K33Kc6SBMOiiwldeIaDKAEwD0JKJ1AG4HcAIRVSM2CFgF4GoAYOaFRPQigEUAWgFcx8xtcxIKgtCuyaD3KGE9hUy7dsIUUwhMKTDzRYrNT7gcfxeAu4JqjyAI7YtoJpVCFobquThPQRAEIWWCdB+psMxozuEyF4FZCoIgCG0hk5ZCOh1Gj158WMYVWjoRpSAIQkjJYEqqLdDcFgVx6qg+vs8JkwoR95EgCKEkSEshDP58macgCILgg8xnHyXGFM46uK/6hADuHxZEKQiCEEoyOqOZyOo+0t6oSmx3dEQpCIIQSrLv4MncCD4Mn1VHlIIgCKEkozOanY4LUCuEIa6hQpSCIAihJKMxBYcyF5myFMLkpBKlIAhCKMl0lVRVNpDEFARBELLMa3PXY8uepoxaCk4EqRLMSigEH9VAlIIgCKGhfncTfvL8XEx8pjbD2UcO8YPcMxREKQiCEB6aI1EAQN3OxoyPntU6ITNaIUy6R5SCIAhZYd32vZi3bodlm55xRMh8QTwVORhSkNpHgiBkh2P/33QAwKp7zkzYR0RZyD5SBJozdP/sq784YikIghAazIogk7WPiNSOIpmnkEaI6Eki2kxEC0zb/kREXxPRPCL6LxF11bZXEdE+Ipqr/ftHUO0SBKF9EAahKTGF9PJPAKfZtk0FcBAzHwxgKYBbTPuWM3O19u+aANslCEJI0S0FomAtBdWMZuXktTBJ6wwRmFJg5hkAttm2vcvMrdrbzwEMCOr+giC0XygWac7o/ZQxhQCVgsxTSORyAG+Z3g8mojlE9CERHed0EhFNJKJaIqqtr68PvpWCIGSFjK7R7Fz9KHONCAlZUQpEdBuAVgDPaZvqAAxi5jEAfgbg30TUWXUuM09i5hpmrqmsrMxMgwVByAjmCWu5lJIaJtWTcaVARJcCOAvAD1j71pm5iZm3aq9nAVgOYP9Mt00QhOwSNeYpUGYtBQepHCZhnSkyqhSI6DQAvwZwDjPvNW2vJKJ87fUQAMMBrMhk2wRByD5RU6A5k3ZCNkpnm8m+TRQnsMlrRDQZwAkAehLROgC3I5ZtVAxgqhbU+VzLNBoH4E4iagUQAXANM29TXlgQhA4Lh819lIO2QmBKgZkvUmx+wuHYlwG8HFRbBEFoH0QsZS4yeGMn91GGJq+FSfXIjGZBEEJDNFYPD0QUWJXU6b84IWGbk0UQJmGdKUQpCIIQGqImSyGoQPPgnmWeZ0sHuciOzFMQBEFIgrX2UWYnr/llQLcSAEC3sqI0tya7SJVUQRBCQ8SkCKIB5qSqylwoj3NRFjeeNByj+nXBCfu3fb5UmNxUYikIghAadOtgxZYGrNm2N8nRat772fFJj+mvjfKT4ZZ9VJifh9MO6tPh1nH2rBSIqISIDgiyMYIg5CYL1u9E1c1TsHprg7Ht0+VbU7rWsF7l+PCXJ+BXpzmLqwe+V42HLhpjvHcS7Hk5OE/Bk1IgorMBzAXwtva+moheD7BdgiDkEC98uRYAMG3xZmNbW+Yp7NejDMcNc3brdCktxNmH9DPeJ3MfdTBjwBWvlsIdAI4AsAMAmHkugKogGiQIQu6hCiq3NaTgR5A7lrnQdgShE9r7PIVWZt4ZaEsEQchZVAogk9lHyeha2rEyjNzwmn20gIi+DyCfiIYDuBHAp8E1SxCEXEKVaZTZCc3qsXp+HuH3E0bhuOHpr8jc3ucp3ABgFIAmAP8GsBPATQG1SRCEHMMob2Hy4zAzigpST5D0FQdwOfbio6tQ1bMs5Xa0N5JaClr10teZ+WQAtwXfJEEQcg2Vq2jLnmZUFBeguTWa0jXbUzG7MLU0qRpm5giAvUTUJQPtEQQhB3EKH+Tnpy4u0xFozkW8xhQaAcwnoqkAjERiZr4xkFYJgpBTRBxSjfLbIK19KYWU75IewhRT8KoUpmj/BEEQ0o65EJ6ZjjZbuD3gSSkw89NEVIT4EplLmLkluGYJgpBL6O4jVQ7SDeOH4aH3l/m+pp+YQraVT5hUnyelQEQnAHgawCrE2j+QiC5l5hmBtUwQhJzBaU5CJMopC8z25D4KE17zve4D8C1mPp6ZxwE4FcADbicQ0ZNEtJmIFpi2dSeiqUT0jfa3m2nfLUS0jIiWENGpqXwYQRDaJ04xhdYoW6R7vo9iRL4yUkUrGHhVCoXMvER/w8xLARQmOeefAE6zbbsZwDRmHg5gmvYeRDQSwIWIzYU4DcAjWiqsIAg5gK4T7LI5EmVLUTo/BepE0KeGV6VQS0RPENEJ2r/HAMxyO0FzLW2zbZ6AmBsK2t9zTdufZ+YmZl4JYBlitZYEQejgMLNR/M5uL8TcR8FL9/Y0pyFovCqFawEsRKy8xU8ALAJwTQr3683MdQCg/e2lbe8PYK3puHXatgSIaCIR1RJRbX19fQpNEAQhTDBbF9cxE2W2jPj9CW8/gWYfl+3geE1JLQDwIDPfDxiznIvT2A7VV6J8Sph5EoBJAFBTUxOm9F5BEFKA4ew+as1QoFmI49VSmAbAvFRRCYD3UrjfJiLqCwDaX714+joAA03HDQCwIYXrC4LQznhrQR32NbcCSBwJMtuEu2QUBY5XpdCJmffob7TXpSnc73UAl2qvLwXwmmn7hURUTESDAQwHMDOF6wuC0M64/t9z8OWq7QDU1VL9zCH40TFVKZ0nVkUcr0qhgYgO1d8QUQ2AfW4nENFkAJ8BOICI1hHRFQDuAXAKEX0D4BTtPZh5IYAXEYtVvA3gOq3mkiAIHQhmxodL65XCH1CnplpjCs4cXtUNd5wzqo0tFLzGFG4C8B8i2oCYhdcPwAVuJzDzRQ67TnI4/i4Ad3lsjyAI7ZD/zavDjZPn4M4JauGtCjibg8tuI3q7PvEXkhZTQcfVUiCiw4moDzN/CWAEgBcAtCI2ml+ZgfYJgpBldu5rwbLNu9Nyrc27GgEAK7c0KPer3Uem1y7C276mc9irpJ4+uk/mb+qBZO6jRwE0a6+PBnArgL8B2A4tA0gQhI7N+Y98gpPvT09Fm8L8mMhpjTi4j5SWgjfakoqYDTthQnV//PLUA7JwZ3eSuY/ymVmfgHYBgEnM/DKAl4lobqAtEwQhFCyvV4/qU6FAWx+hNdWYgg/3kZHmSs7rNWSbovzUV5YLimQtyiciXXGcBOB90z6v8QhBEAQAQIFWp6I1ol5NrUmxytpB/b2t72V3H+lF9grzkgvebFdJDRPJemsygA+J6DXEso0+AgAiGobYOs2CIAieKdAEtJOloFIKxwztiQcvrAbg7uaxWwO6kvBSRE9UQhzX0T4z30VE0wD0BfAux1VxHoAbgm6cIAgdC9191OJkKbSoM9F7VXQC4D6it5ff1vVOgZ8qekJyFxAzf67YtjSY5giC0JHRA81OSqFZYSmY8WMp6ErCyzrP4j2KE74ohyAIHZZk2Ucq9xHgTWgnWArapbxYChJTiCNKQRCEjGG4jxxjCkkKGbjIbidLocBDoFmII70lCELGyCP/2UeAt0Cw3VLQ3/pZrU0QpSAIQgbRc1X8uo903MS7XSmUFccWbxzRp8J7AwVRCoIgZA5dbM9cZV+UMYY90PzRr04EEPf5u/n+7WpmSGU5/nXFkfjjt0en1NZcRZSCIAiZw+fM4oHdrRX6XePBimsfO7wnyopknq0fRCkIguAJ+4zhVLC7eLySSvaRTp5kFvlClEI75M7/LcJZD32U7WYIOUY66ge19RruMQWHc0Qn+ELsqnbIk59I1XIh86Sjplyq1/Ai19nh6pJ95A+xFARB8IQf99HWPU049PdTsWC9tURaqi4oT+4jh8QlcR/5I+NKgYgOIKK5pn+7iOgmIrqDiNabtp+R6bYJguCMH3H+0TdbsK2hGZNmrEj5GipSmXkshoI/Mq4UmHkJM1czczWAwwDsBfBfbfcD+j5mfjPTbROCIRJl/ODxz/Hpsi2B32vttr0466GPsHVPU+D3yjX8BIl12W0/I/WYApn+V+PUvjCXsCjUZngXFoTHaZPtlpwEYDkzr85yO4QA2drQhE+WbcWNz88N/F6PfbQCC9bvwv++2hD4vXKNVAR6orso+UXKi51Dne6L7IR0JR0XLjpyEK45fiiuP3FYtptikG2lcCFiazboXE9E84joSSLqpjqBiCYSUS0R1dbX12emlULbMK2AFfitjHuFd3SYC+j9n4qlMH5EL8X1kp+X3w6/8+KCfNx8+giUuSjCTJM1pUBERQDOAfAfbdPfAQwFUA2gDsB9qvOYeRIz1zBzTWVlZSaa2i5Yumk3bn9tQVpyydNNNloUJvnw+EcrMG3xpmw3o834ebSM7rcvfOPh3GJXV4rzF/vMFUd4uLqQjGxaCqcDmM3MmwCAmTcxc4SZowAeAyDfsA8ue+pLPP3Zaqzbvi/bTUnAGL1n4l5ZUUHu/GHKYlzxdK3rMY0tEdzyynzs2NucoVb5J7WYgrpInRudCvMTr+fhnsN6SY2jdJBNpXARTK4jIupr2ncegAUZb1EHIEwjZB1dMGQiNTCTCiidvDx7HSbPXIM/v7sk201xxI+6JYdvwIvSdrMUwvh8dzSy4sgiolIApwC42rT5XiKqRuzZW2XbJ7RjopmMKegv2pn0iGqdFELvn4Ef16RhKSSscZD8XKWlQMmzj4T0kBWlwMx7AfSwbbs4G20RgkcXJhlxH7VTS6E94M9S0M6xxxQ8KBaxFLJLeELeQocls6NfTQH5FB6vzV2PYb3KMapflwDa1DFIKSU1hRhPcWGiUsiULvjuYQMwpLK8Xaa3potsp6TmLGu37cUrs9dluxkZIRLVBXUmYwr+7vWT5+fizL9+HECLOg7pcB+lGmg2rhuwevjTdw/BtScMDfQeYUeUQpY492+f4GcvfpXtZmSECKc2ek8FzmD8ItfQ+3bTrka8Omd9kqMd5imkGGgO6vv87VkjZWU2G+I+yhJbG2Kph8zc4SdaRaMZVArIXPwi19DF+SVPzMSSTbsx/sBe6Nyp0HLMrNXbUVyQZ9QbSrulkOYv9vJjB+PyYwen96LtHLEUsoyXbAwvhHHSmk66PqOQXfRnbNPuRgBARLHO8rf//inOeuhj00DH/zyFwnxVTMFdG5wxuk/yCwueEKWQZXIhoKXHFDI6T0FMhbSjP6l613p5dhMsBQ/3cVv/wGnPIz84zMOVBS+IUsgy6VIKRq2ZEOqYaCZTUrW/QQckvTBjaT32Nrdmuxlpw/geHeoamVHbCd4s2gKFUtCVfKqu1rIiZ5eUYEWUQpZJtxAPo1LIRvZRtnXCyi0NuOTJmbj1lfnZbUg60fpWl9luA5p49pHNfWR63b2sSHluXgALICy887S0X7OjIkohy6RdKYSw9k8ko5ZCOALNu/a1AABWbGlIemzdzn1YvXVv0E1qM/EnK9a7TiudAc7uO7OScJL9KktByBySfZRl0h1TCGNQl+OTB3IGu6vFjaP/+H7QzUkZsxC3x2sibpaCU0qqaYNTjMk1ppBDz1C2EEshy6RbKYQxCymijSgz8nsOyXoKUZurpb3y9KerjNf6s2q4jzyMQNwCzU7CvyAvc/MUhEREKWSZdKekhtFSSFdMYf2OfUkDtyEJKaRc7ylswm/Jpt3Ga3sQv9XlYdMViJul4KQUFBmpBmHrn46IKIUsk/6Rffi0gn2EmSpj73kfP3z8C0/HprsXolHG2m3e/f5xS6F9SjHVZzUUne4+clEK+mOdGGiOv3dWCs7zFMKQVdbREaWQZZx+V5t3N7r+6PxeL5sYlkIaftCz1+xw3R+3mLx3hBfF/I8Zy3HcvdOxbPPupMea75+qUmiJRLFlT1NK57aVN+ZtwHH3TseHS63L3bJN0bn1sdMei6Xg0Dd6oPmw/eIr8rZT3douEaUAYE9TK5paIxm7304tMwVQ/7C2NzTjiLum4Y9vLvZ97RCGFNISN/HivwbiwsiPBebl0M+WbwUAzyvbGUohxV/Yr16ah5o/vJcwMNjWEPzKbB9/swUA8HXdLmXf6AK6VTGjWcdwH7nEFJxST/OIMPO2k/DclUc63lsIDlEKAA66/R1MePiTjN3vkN+9a7xWCcwdmtKY6mNdX/IwessWUZvbIRWaIy75jyb0j+/HYvLSZ14mbKnakaqlMGVeHYCYxaDz1vw6HPr7qahdtS2la3qhNRLF81+uVe6zf4+uloLuPkpYpNnkPnKyFPIJvSo6WWogiTLIHKIUNL7e6M0t0FYSfKyqkZjLPjOtCkEZRqVgZB/5/GVHomwIxRavSkH766cfvCgQY1BrO9bJwmyr+8gYjZsa9/mKmLUyf/3OlK7pBfP97E23Kzr3mELqlkIqZS6E9JEVpUBEq4hoPhHNJaJabVt3IppKRN9of7slu057xP5DcjPPk3HJkzMTtmVKJzCzZxdNPKbgj7Mf+hjDb3sLANDcqlCACqGUShaWJ0tBcezKLQ044DdvK9fFaOsSpIbgdXHRBIG5KwhkeW/P7HLLPoq78Zyv72fymgSYM0c2LYUTmbmamWu09zcDmMbMwwFM0953OOwTftyDde4C4VPNzw04j8yC4qcvzMXgW970dKw9a8Uri+p2Ga/t7qO5a3dgyK1v4tPlW6z3st3TW/uSH6OqLbVkY6x97yzcmHC8n8lrKvTRcovbtOEAcHcJWT+T27HxlFTn593JInCzrrI9/yQXCJP7aAKAp7XXTwM4N3tNCQ77b1z1wzJmg6Yg4DNV5uLVuRs8H+s289UrLa1WxfKF5kqZ/vVm64F6TMGHqeDHUjAfaVgDqlGssS81dHlpDuZmQiCavysi6/PEpu32ttlx6lIv8xQK8l0K4jneUUgX2VIKDOBdIppFRBO1bb2ZuQ4AtL+9VCcS0UQiqiWi2vr6etUhvlD55YPELiDTPbIPc0pqW3L2myMx332hls4T92tbj9OFWKruo6qbp2BPU+IEOVWBN7cy3fEJe97bYUb3t6tiKUFag+zyc7BPyPNkKbjEFJwCzW4xBZVWaO+zxsNGtpTCWGY+FMDpAK4jonFeT2TmScxcw8w1lZWVbW5Ik8JX7ZfNuxtRdfMUTF20CRt27EPVzVMwY6laYdljCqofli7YUrIUMhxonrG0HlU3T8GGHc6pmmnJPmq1pnjqQtNJMLUl0Lx5V2PCMarsI/17Uim7tlpHusB089sHgVu7/QSa7efE38c3OAaaFf3p9uh8/OvxSdsheCcrSoGZN2h/NwP4L4AjAGwior4AoP3d7HyF9KEKYPpl0YaYb/mZz1Zh9prtAIDnv1yjPNbu1lD9rvxMWrOPbFWnXvbUTGVQOh38+4vY55y7dofjMemofaTHFPS6OLqHwS784zNpvV337QV1lhRhQC3k4xlh8Qsbfe1mKXhrRuL9dKWQZkt27ba9qLp5imNaa8SSfWRtvd195GlGs4s7s8ihnoWq9pHRJsW2fl1LHI8X/JNxpUBEZURUob8G8C0ACwC8DuBS7bBLAbyWifakw1Iw/3jiLgW1OPASaI4ags2bZLOWJEg8Z/qSekfLpa3ER8vOx0Td/Cwe0ZV3fh7hpufn4I7/LbJeW2+PHlPw2Hd/nbYsYZtSKajCBi71jVoN91Fqn1nvz5Y0Zx99siwWmH+xVj0Xwe2Zixe7TW4pOLqPTO8LFbEDAMh32C5khmxYCr0BfExEXwGYCWAKM78N4B4ApxDRNwBO0d4HTjpmMpvnFSTLT7dbCqofoZFJ5PH+zOaMEI8npQnzHIQvV23D6NvfwY691lm30TaOmoG4b70gjyxB7rbGFFoV2T1uctySomlzp5iJtDFryM1F05avOG7ceBu0mN/a3YBurian/jdbDqq1mAH39RQk+yh4Mq4UmHkFMx+i/RvFzHdp27cy80nMPFz7G9y0TRPpcB8ZQciEZUgSsf+QznjwY2P2qtMxfvCTdZMOzErw4feXYXdTK+bY6hNFbMIkFcyWghmnyYBeLQW3DBozqvUBjAV9FJ/L63WdUKWkplMeOl3L/PgQbJ/XZgVf/ewsvDwrcY4GEH8O7b1gsRQK1OJHpWRdPHVCmglTSmpWSIv7yJRCmsxTYh/5NUeiuH7ybMs2XQ541Q2qtMFU2byrEeP//IHniqCtRmZRfIRnD46mw1JoNlkKZhImA+p/vSoFhRK9/J9f4j8290o8+8h0LxdLoa0zy3W3ukq5tCWZINmpboMK1ap2P//PV8pjdWXmthync0xBoRTa7oEUPCJKIR3uI5PAsC9S/4c3FuGJj1cax6q8CvYfqtPEHydiyih27N0pFNEz88qc9VixpQHPfr7a0/F6IDSPyBjd2l0n6UhJ1YWj3X3g5O7w7D5SBHK/2bwHv3xpnmWbqjJo3MceZ9KM5fjjW4vjMQVvzUhAv1+6A81O1s1Vz9TivUWbLJ/PscyFB6mhf19uloKTUlBlJYVxmdmOSs4rhXQE8szuI/vo8fGPV+L3bywyjvXiGmrLKHPeup0pn2u+t1f5HRfW8UlH9tF3axqUgpMLKnFkG3uvK6Lb/jsf7y1yLizY4lV7GN+x/U6wSP673/waj364QhkLuOypmZ4HIXpaZkuUMWNpPX72wlxv7UwC2xrd3BrFlU9/iamLNuHKZ2qt2UcO1/BSckKPAa3bvg+XPDkTd01ZhBe+XGN5tgsL/D8PUu4ieHJeKaSyZoEd/UGNslpQ+L3fozNWAPDhPkrjIMqeYZIMPVAbsxRij5P9M/qdyHX/1KWYahPkToFbe3fGU1JjL577Yg2ufKbW8V5ev391SqqzsjMrS53pS+rx1VpvSjs+aziKS56ciVfmrA9EIC6q24X3Fsezv52yucz7nL5Hc9/oA4H63U2YsbQej320Er9+eb7leKdAs/rang8V2kjOKwWv1Tf3NUfwi/98ha2KhU+MH4nJjeP0A/ZiBeiBZ6+/g1RNa2bGHa8vxFLTkotRU4zAC2aBb8QUbNaXX0vhr9O+wVU2QR7PcrLd39af+nuvut7r96/KenGLlbR1sKH3ldmSTYcLxQjYKmZoA7ZAM5Hlnpt3NeFXL33lmJxhPtfJ7WW+X5FDoNkNiSkET0G2G5Bt3H68s9dsx2fLt+K6E4fhlTnr8NKsdSjMz0PfLp0wbv9KVA/sCsAaA4i7j/zfz0797ibMXrMdhw7qlvJncGPd9n3456erMHXRJnxy83hs3t2I+6YuBeBdgJuFVr4RaLYKBEPRmGTA5yu2Yv66nbhq3BBP94kLYPfsI70v0p99ZL0+EHc9KbOPHGa2eQ0Sq1JSjYyetugGW8zLfinr5DXrvjvfWIQ1LgkI5j53cst6iSkkO08Ilpy3FNzKCJz/yKf40ztLbFsZ909dinP/Fl+Ux5wtlMzE9ivAz3/k06THJJbj9naPV+est7y//bWFxmu/lkIkyoalYM/oUlkKF076HHf5CIo7xWLM979ryiJs3hWz5KLsra89u48Us3j10bDKKowH4D1dPoE8hYL1k6r83BerjdXiVOif50XbgjpuyjRZX0Ut7iMHS8H02pdSMALkYioEjVgKnn3KzpOJIoalkHxyUBCL4ESibPmxNLVGLatWOWFYBdpv0zK682opaD/+KLNhKTS2WIOpEY8j3HcXbnT80bc6jMr1r+OTZVvw2Ecrje3M7Mk15LU0tcpScIuV6Cm0qcqwfMV3kuzjLNu8B9MWb8LVxw/Fbf9dAABYdc+ZlmPsX4F9lTVzd9ibnmywYT73mc/U2WvmSxSIpRBKcl4pOAkbM9bRYeLTGTfr4+6jdFkKXnjhy7VYbypI19gS8aQUdOLzLJJnntiJj9RhUgpqSyGZQpz47CzHfXofr966V7k9UVmwsdawG16Fja6sIgoXCRFhcd0uzFu3w9inW0t2Qe51pKsH7c3++2QTE7/36GfY1tCMS4+pcjwmWSJBwndkepvMUjGfu7sxsdJs7HLxY5zKXLghdkLw5Lz76DktH9/tYTOPOFVrBRsjYSQ3c4OwFF6xuYH2NqvTHv2Y/15jCrqSjDIb59vTLiMmayIVmNmx7bqgss90jjJcs478YrYUJs1Yju0NzcbnIgJOf/AjS3ZNs6EUrM9L4gxs9ecq1kbRZqsrbpHGXWZ/m74Me5tjAnintra3l8qqjoOWFEpXxPcnv69ZsRWnEGgWgienLYVdjS34YmWsmoabEDQrAlXmhfFjZS8zmlNsrA/2taiVQkskivy8RAtCl6fmH71XX7huoUSjbCgI+2eMWwrermnnsY9WYF+zuuM+WFKPHXubEyprpl35av3xxcptmDKvDrNWb8eIPp3NuywYloJLM1bU73Fcb1nP4W80KVi7pfD6V+vxp3eWYOueZvz27JGG4nSb8JbcBWTOdvJ5rocuNz+bZcU5LX5CS05/K0s2xlMx3ZSC2WWkXvTE5D7StjldLRX30Zw129EaZRxe1d3T8Xafvo7TCFK1vKJTrXsnIsyGO8U+OjYEjeL2r81djwnV/V2vffebX7vuv2HyHNx08v6WbenXCZqQ1qywXftajWCqqlv1wYNbddqT7v8woZ36fXQl19icaCno6IqywbYokNuEzGTP5+KN1vRk85WSWpoenm1z23pVdEp6vB2JMwdPTttv67ebFoZxedic3EdvL4jNJ9A3RTmuIByrpKYgrc575FN89x+feT7ebM2YR3dOAsrrD239jn2Ytlg9OzjK8YwTpxnNqs/+k+fn4tU5610X6UnG1yZBFm9PerWCriPNLhz9c6km1qncjLHzTK9dmqjvajR9l+b4DRD/3uzfn/l5fWt+neOiQSoh/3+vLjBe23f7cT86YW5bz4qipMfrSO2jzJHTSsEsvNyetebWqPEwmgXuNf+ajX3NEYugSDZ7N4hAsx3zaMx8ux8/N1txtLX0t86HSxIVyNkPfYwrnlb76aPReLaP04xmJ6Fx0wtzLSm+fqnf3ZRwz3SsWLZk4258pS0epE5JVbvLAKDZoZxFNMqYv24nFtftcr23rszNVp/ef8mEr9myvfa52bjKFMA3d0uyCsFRZuXyo87Hu++3t61HWXHyE/R7G8X4rD8sURLpJ6fdR2Zz1+3hsloK1id/d1OLZVKRecUtlTntFshrjUTx9sKNXpruivnH7pQvrsIsbGau2obm1qhl1um2htg6Ccq03GjcfeRsKTjfe/PuxJnifqhdba207uRXn7t2B7qVFmK/HmVJr3nqX2YAiKV1mivh6rgpOyeBG2HG2Q9/nPTe+rUtgWbTnAw3vjJlQQHAtoZ435qfyWR1mJwKNTof78FSMD2P3cv8WwoA8Itv7Y8/v+tvkqXgHbEUNOwPl7l0tPm4FtuPfXdjqyUPPz55jZQKwM3v+tf3l+H6f8/x8QnUmJWYY9aO4rPbf/ROQsAp2K4L4ohNcZqzs4Li3retkwydhPK5f/sEx//pA9/Xt1sKzPF+VlklTu4jr5Zi1LAUEr9Lp1XNdG6YbH2GupXGha85BTtZ2fjE58H52PU79nkqxtgaYRTmE1bdc2ZCxpgXiIDrxw/Hkj+cBiD1yYGCMxm3FIhoIIBnAPQBEAUwiZkfJKI7AFwFQPdb3MrMbwbZFrMv2K4Ujrt3uvG6uTVqGK32H/u8dTssk9fMPzr7RKf8PLUfF4gpi6+TuBS80mSxFNT3MysO/aPbD22JqCfBqUaYC9fvTGoptGUdAL9s39sSyHUts3b1dFxFH3+yTD2b2LNS0L4ec7ZOi8v93Pq2S0mh6f6xC89evR1NLcmUgvW9m5U79p73Xa+l0xqNplTYz35nv4UbBe9kw1JoBfBzZj4QwFEAriOikdq+B5i5WvsXqEIAvMcUzALUnn300xe+skxeW7Rhl3Y9qwLQz3MafUeYsasxuSDzsvhNcySK2Wu2o7Elgk8dhJNFKSgmrwHOAsw+OQ0Anv5stSmmYF9PoW3zFFLhQ0VQ3fz5/K64p5+qf0bzAMBPnMi/pRBXCnPWbI9dQ1FK5QNFDEins0UpxP5+tW4nNu9ODEDb22BurZOV60fZt0S4TTPQ7PEv8R6ln4xbCsxcB6BOe72biBYDcM9JDIA1W/fi/a83K/fNt5nBza1RfKVts7uPAGDFlgYAwNaGZnytVTglsiqdup2NiDI7zlNYs20vltc3JG232YJx4rU56zHt680oLy7AHi1dsaQwXzkbF4jN11hevyfBHeGU2ujki9Y/r91S8FrmImgaTOmdd/xvIe4+b7Tnc3Uhvcs0U9cp28rLdZKhf1fm0bx+b9UztGRTYgaWTqHJx2KuiLu7ST3r2GhrlC3fmVPb/axJ0hqJpqQT2KYF9LU7fnjUfilcLTlHDo6lf48d2jOQ64eZrAaaiagKwBgAXwAYC+B6IroEQC1i1sR2xTkTAUwEgEGDBqV873F/sgpX/Ue4YP3OhEDgIx8sN0aeKl/xJG39g3pTsNQcdAaAE//8AQDgp7Z8ep2T7vswaZt1KyQZ0zRlt8f0oy8rzjcCxYDVUqjb2YiT7vsQhw7qarlOJMrY29yKTbuaMLhnPDDr5Is2Zx+1RqJYsaUB+/euMPnCPTU/MHabLDH7OtLJ0Nu+w+SWMs/m9sqC9d6+Q/1+jQoFrLrfPW85z+Uwr3MxZX58PfAmh/ks9jY4vdfxYym1Rjml0b1+B/3Uwvw8LP3D6cqlO50oys/zXFqjpqo7lvzhNBQXeC8X01HIWqCZiMoBvAzgJmbeBeDvAIYCqEbMkrhPdR4zT2LmGmauqaysTFt79Ad7485Ek3r2mrhusgvEoZXqLJaoQ2mGB95bmnIbz/jrRymfW1yQjyjHXQAq90lDk1VI7NjXjHMe/gQn/vkDi4vAaXLcPm0k3hpl3PvOEnzrgRn4euMuo15RJmMKKsz1eLY1NKFup7e5EZt3NWLnvphC1f+u27YXDVp5Ca9rMgDAw9OXeTpO76t9ipIl9hLaybpVl5v2std7mpIpBW/f1+KN3mNhLZEUYwqKphQV5PmaZDn/d9/C7N+e4vn4XFQIQJYsBSIqREwhPMfMrwAAM28y7X8MwBuZbFN8TYREzD9MuwBQ+df162ViToJXyopjD3hzJIqtu5qVFo9dSJ72l7gSMitDJ0tht+HeYNSu2pZwjXTMHUiVaJQtlsKmXU04+o/egqNH3D3NeK27SjbsbMQGbQCxy6H4Wyrs3NeCHXub4ympCkshvpBQ7G+ySq96r89abTW8LZM3Ved5VArJyrt3KswzficrtzSklDHUp0ts9vMJB6Q+EMxVIe+XjFsKFKup8ASAxcx8v2l7X9Nh5wFYYD83XagCZkYlBsUPwSzM7EpgvcNMXN31EhZKi2L6/563vsbYe97Hmq2JAWs34WYupeBkKehCtzXKyvxxr+sTB8G67fsSLKF0oU9wM+NnrQAzr3+1AdV3To27jxSDDv351RXHjiSZVvpx05dYY2jJrNZ06fA+nePlLDbtarLEdrzSv2sJZt52Em4cPzw9jRIcyYalMBbAxQDmE9FcbdutAC4iomrEBjarAFwdVAP2KoRaunPpowzH2b/ZoFwrPvaeVqZief0eALElNL2M4M3xCH0hGzv6jz0SjSr9xipXiB+6lhZiZN/O+NRl8RgzY4f1wNDKcjzz2WqM+9N0VFZ4n0HbVooK8hznKtjpUlJoVDjV0QV/vWJSnz2g75QwobO1oRnNrVHs2NuMUf06Y6HH2JQ9+yhVKjoVJj/IA6nUShL8k43so4+hTkoLPAVVZ4/DiLg1Ek1bhkw0yli5JXk2UabQ3Uc7tVGlHjvpWV6MjYraOHZOeWCG4Qa4O8mKaTv2tqCsKPHRaqtSKCsq8CXYK4oLMcQUIFcJ2KDwEtAc0acCX2/cjd6dixOVgsuDqLvv3KqhmpmxtB6/fOkr7G5s9dV/5lpebaGiU04XTmh35OSM5j22VLzLxlYBiAVa0xUHyGROvp3CfMLFtlQ9XUjraYiPf7wSAHwJCd0NsNVkNahYuGEXZq7alrC9oblt/ZufR77KGpR3KkChj5r9z1x+RCrNUlLowX00rFc5ALWv222imO6+8xOjeW3uBuxpbDUsRi+kKzGgVDFAEMJLTioFe6nhA/vG6uLvamxJm987mzHm2t+cgjMP7mvZ5lS7Xg/geaElklo6IQD88KhY+vCPn3NeXS0Z+1oivvJWKjoVeBLOQEzh6EI6HRR5UEa6gC5QWBVO1iwQizMwszKD7PFLapTnlBTmY3dTq69Re0skPe4j3UoV2gc5qRQGdCvBn797iPG+s/ZDOe7e6bjzjUXG9rMP6Yf+XUtSuodXS+EvF1SndH0zPzkpHnyrrChGl5LChHY7KYUePoqSNbVGfVW21Kn9zckY1L0UAPDOQnXpbS/YlXkyKooLPAd8vVghI/pUeL63l/z5kqKYsFQpru17na2xxpYIzv/7p3hw2jcJ+2qquinP2dcScbQUnFxdT36yEm8vaHuBRrEU2hc5qRR6lBfjO4cNMN53NgXCzJkcp47qbSkR4If/aktk9ix3F7r2JQmHVpbhoP6dLW1Ihm7pAPGcdLsFUFaUOFo7dVRv4wfbtVT9Oc3n7WlqQf+u/oN9PcuL05L50+qzREJZsXdL4aB+nZGX5FA/Bdy8pKnqikO1LKVqlvCg7qXoWV6MxpaI4+Q7t7W597VElEHfzi6B4HS4U7uXpSfQLGSGnFQKdpyyI7qVFuGYoT2Snj+yb2cMd3A9nDG6r3K7jl3QjOrXBZOvOsoQFIftpx75mSkujH+N+mjXLgxVlsLRQ3oYpr05bdDMtJ+fYEzQa2yJon+31CynZOm540f0SnqNZp8Tn5wWj7dz9bgheOpHRyS1FPwohW1J4i5mnBTXxHFD8P0j47P2B3YvwSEDuqDWNt/ATDLLqLy4wGIlA3GL5aaT05/u+dyVR+Kgfl0c9+/Xo9Ty3s8MZSEYRCkA6FyiNm+7lhZiQnW/pOf3rCjGyH6dlfuS1Yy3+5P3NreiolMhjhseq7lS1aPMEh949bqxxuszRvfB1J+Os4w0nQSbym1QUpRvCKReDkqhT5dOOPPgeB/06+JPKehuI3OlzqqbpyQcN8qh/8wQJRZAe+qywx2PL8zP87SexJkH90WX0kLkJ1EK5COgMsCD8tSTh5wEeWlRPq49fqjxfmC3UldL4PoTh1lm+L50zdEJx5R3KsDo/lYhrV/z1FF9krbZL2OH9cRpB/XBr047QLn/pWuOwT9+eKjxfsavTkx7GwR/iFIA0M1BcHctLcLo/l3w0EVjkl5Dd498a6TV3dOz3N0Hn2/zWeipifqoNBJlfPvQWL3Azp0KUD2wKw7oHfNtj+rXBcN7V1iEilMMRGUpdCrMx14tTbSXSxaS2YWkCkw/c/kROG+Muqbhy9ceAwCYOG6oq09ed5dUVhTjt2eNNLbfMH6Y8douPC8bW4UTD+iVEFztXlaEBy44BFcfP8Ti/nDKLtK/I12hOo1WzZvv/94hymN0Xrz6aLx543E4wmVd7XItltXXwSXXGmHL93n72aNci9iN0WpXvfezcXj2iiNQo7h3RXFBQgyhk2Zp+inX4YVnr4j1NxHh7IPVg6vKimIcrRWdqyguQL8UY3hC+hClgNjDqKJ3RXHsgT4k/kC/ccOxCeb3pp2NhnvEXrWxsqIYb9xwrGXUZrYe7EJcVwp6qminwnxDaOn56eMPtLpa9CyUksJ8/OPiw5SfxSkDRG/3qH6dEz6XTqmpf6p6lOGhi8ag9jcnG9vG7V+JA/smCvyhlWXG5ygqyMOlx1Qpr6/vB4Dhvcpx+bGDDUX0vZqB+MO5BxnHmLNyThoRU8An2xRxr4pinDdmADoV5lvSNkf0qTAEoJketrjPsF7lhuI10xKJGm62c6vjSvA+W7/t37sc/bqWYGS/zoag1jnfpDzHj+iF//ft0fj1aSMS7gXELNi8PMLTlx+B9352PEqK8l2LIuofdVivChw3XF0OoqJTocUNVlaUb6TEJlt0R8Xd541G9cCuyn3mNqgyrHT09vT2kQknBIekBcDZLVCgMOsP6t8FB/Xvgl/85yvT+XH3jD1gW9GpAAdp5vodZ4/EHf9bhP17l+PqcUPRr2sJqmw+VT3we+sZB+KA3hU44YBK1GkTzfQfrb2W/FFDeuB354zCd2sGWDI9pv50HD5fsRUlRQU4eEBXAEC30kKMH9EbL89eh6aWKEo010H3siKcc0g/y+fSGWhyhVR0KjCE8L+uONJwvemzTffvXY7dja2o29mY0K9OAc3fn3sQRvatwF+nfWO4mUb07WzU6jl+/5hwKS7IM7JyzhjdB2OHxeM9FZ0KsLuxFQ9eWI0jB8e3m11rhfl5qKwoxtpt+/B/Z43Et0b2xpy1Owyh2KW0EPd99xAct39PgIGzH/4Ym3Y1YUJ1P7w2dwN6VXTCoxfXYPbq7cjLIzxxaQ3261GKoZXl+Lmp38zWiS4Mj9+/EjeMH4bD9uuGV7QkhJLCfFxweCxmMO3nxydUyv3RMYMtnx8AimzCtW+XTsbz4SkNtlOBZYJml5JCSzzpP9ccjTwCtje04MpnrDPyX772GFz33GzLZMfvHzkIH5jKZ+SROh3bLV5dXlyAv1xQjaOGJI/fCcEjSkHBJUfvlxAgfuCCQzCyb2LA7N7vHIwjB3dHeXEBxg7rafHX3nz6CIv7QB8pn1Pd32It/PWiMRjSswzz1+80Cn6VFhXg4qNjx+sjWf2cUm0UrQv0vDxSjsKH967AcNOI96GLxmBwzzL8b94G47yfnDwcPcuLcdbB/RyVozm7aZBJiR07PF5rXg9AD+tVjiuPG4LzH/nUUm4bSJzZ+tglNehVUYxDBnYFM+OOs0fivDGxrLBHLz4MU+bVYUC3EkMI9e7cyVAK15843NLeN244FrPXbMeEaqsb66yD+2Heup0Y2K0E3cqKMPmqo/Dp8q34Xs1AAMDA7lal/G1TVtp7PzseL3y5FpceU4VR/Trj3Or+6NW5k2HdnXSgOjPMnLiguwfHDOpquHNqf3My3ppfhwNM7rShlfFEhQcuOATVA7sphfyzVx6JdxZuxL1vL8Ho/l3AYNTtbMRVxw3GONP34URpUb4l7lRaXIC7zx+Nw/brhpr9urnGTQ7brxu+c9gAPDx9GY4d1hPXnhCLd9z7nYPR+92lePbz1Y7Cv3/XEtw5YRSGVpZbChPqnOvgfhQyjygFjccvqcFjH63AyH6d8ctTD0jIrdaFlc7ff3Ao+nTphDGD4tlBumB+8kc1KCkswNG2zCUiwo/GDk649zmae+qg/uosjeKCfPzx/NE4Qlv4Y+K4Icgj4AdH+ltgRHeDDaksQ3lRASZU90Nhfh6uGjdEebyeGmuONzhlKdXs1w23nXEgzh3THz3KivDLUw/AhYcPtBxjT+89xeT2sfdNz/Jioz/7dinB7WePxGkH9cFlT30JIDGAv1+PMuzXI7GMeX4e4f9MMYoB3UrxvZrShONUVHQqxJXHxfpm4rihSY6OcfvZI3H6QfEBhe4GM8cpepYXGwrfzKMXH4YeZUXKWIDO0Mpy/PiEYSgpzMepo/ogyoypizbhR8dUKQX6Kz8+Bht3NuLHz8022jGoRyl+e9ZI7NjbjDMP7ocuJYW4/NjE5/KpHx2O4sI8fP+xL4xteqbS6AFdMHZYTAl1LS3C7WePxLOfrwYQsza27knMvrpE8ZmF8EHZrnHfFmpqari2NvWic3oWzKp7zkxXk9o9//p8NQ6v6o53Fm7EhYcPNLKSPlm2BRt3NlpG0n5pao3g7imLQUSoqeqGsxyCj26s3tqAqYs2GcI6LLz45VqM7Nc5QbE3tkTwt+nLcN2Jw1wzh4Jmw459eHXuelx7/FBfWVSA9bM989kq/Pa1hbj2hKEJsZAXa9dieK9yy0BJCCdENIuZldPfc1opvP/1JjS1RHF6krkEgiDEaGqN4L53l+KG8cPSVv1UyDxuSiGn3UfjRySfLSwIQpzignzcesaB2W6GECCSkioIgiAYiFIQBEEQDEQpCIIgCAahUwpEdBoRLSGiZUR0c7bbIwiCkEuESikQUT6AvwE4HcBIxNZtHul+liAIgpAuQqUUABwBYBkzr2DmZgDPA5iQ5TYJgiDkDGFTCv0BrDW9X6dtMyCiiURUS0S19fX1GW2cIAhCRydsSkE11dIyu46ZJzFzDTPXVFaqK0EKgiAIqRG2yWvrAJgL5gwAsMHp4FmzZm0hotUp3qsngC0pnptLSD95R/rKG9JP3giynxwLp4WqzAURFQBYCuAkAOsBfAng+8y8MIB71TpN8xbiSD95R/rKG9JP3shWP4XKUmDmViK6HsA7APIBPBmEQhAEQRDUhEopAAAzvwngzWy3QxAEIRcJW6A5k0zKdgPaCdJP3pG+8ob0kzey0k+hiikIgiAI2SWXLQVBEATBhigFQRAEwSAnlYIU3YtDRAOJaDoRLSaihUT0E217dyKaSkTfaH+7mc65Reu7JUR0avZan3mIKJ+I5hDRG9p76ScbRNSViF4ioq+15+po6Sc1RPRT7Xe3gIgmE1GnbPdVzikFKbqXQCuAnzPzgQCOAnCd1h83A5jGzMMBTNPeQ9t3IYBRAE4D8IjWp7nCTwAsNr2XfkrkQQBvM/MIAIcg1l/STzaIqD+AGwHUMPNBiKXhX4gs91XOKQVI0T0LzFzHzLO117sR+wH3R6xPntYOexrAudrrCQCeZ+YmZl4JYBlifdrhIaIBAM4E8Lhps/STCSLqDGAcgCcAgJmbmXkHpJ+cKABQok3cLUWsgkNW+yoXlULSonu5ChFVARgD4AsAvZm5DogpDgC9tMNyuf/+AuBXAKKmbdJPVoYAqAfwlOZme5yIyiD9lAAzrwfwZwBrANQB2MnM7yLLfZWLSiFp0b1chIjKAbwM4CZm3uV2qGJbh+8/IjoLwGZmnuX1FMW2Dt9PiI18DwXwd2YeA6ABmvvDgVztJ2ixggkABgPoB6CMiH7odopiW9r7KheVgq+ie7kAERUiphCeY+ZXtM2biKivtr8vgM3a9lztv7EAziGiVYi5HMcT0b8g/WRnHYB1zPyF9v4lxJSE9FMiJwNYycz1zNwC4BUAxyDLfZWLSuFLAMOJaDARFSEWuHk9y23KGkREiPl/FzPz/aZdrwO4VHt9KYDXTNsvJKJiIhoMYDiAmZlqb7Zg5luYeQAzVyH2zLzPzD+E9JMFZt4IYC0RHaBtOgnAIkg/qVgD4CgiKtV+hychFtPLal+FrvZR0EjRvQTGArgYwHwimqttuxXAPQBeJKIrEHt4vwsAzLyQiF5E7IfeCuA6Zo5kvNXhQfopkRsAPKcNulYAuAyxAaj0kwlm/oKIXgIwG7HPPgex0hblyGJfSZkLQRAEwSAX3UeCIAiCA6IUBEEQBANRCoIgCIKBKAVBEATBQJSCIAiCYCBKQchJiChCRHNN/1yr5RLRNUR0SRruu4qIeqZw3qlEdAcRdSMiWa5WCIycm6cgCBr7mLna68HM/I8A2+KF4wBMR6zY3CdZbovQgRGlIAgmtDIWLwA4Udv0fWZeRkR3ANjDzH8mohsBXIPYBKJFzHwhEXUH8CRiBeH2ApjIzPOIqAeAyQAqEZt9SqZ7/RCx0slFiBUh/LF9MhIRXQDgFu26EwD0BrCLiI5k5nOC6AMhtxH3kZCrlNjcRxeY9u1i5iMAPIxYZVQ7NwMYw8wHI6YcAOB3AOZo224F8Iy2/XYAH2vF4V4HMAgAiOhAABcAGKtZLBEAP7DfiJlfQKx20AJmHg1ggXZvUQhCIIilIOQqbu6jyaa/Dyj2z0OsjMOrAF7Vth0L4NsAwMzvE1EPIuqCmLvnfG37FCLarh1/EoDDAHwZK3uDEsQLn9kZDmC59rpUW/dCEAJBlIIgJMIOr3XOREzYnwPg/4hoFNzLGquuQQCeZuZb3BpCRLUAegIoIKJFAPpqNapuYOaPXD+FIKSAuI8EIZELTH8/M+8gojwAA5l5OmIL7nRFrIDZDGjuHyI6AcAWbV0K8/bTAejr7U4D8B0i6qXt605E+9kbwsw1AKYgFk+4F8BtzFwtCkEICrEUhFylxFQVFoitKaynpRYT0ReIDZousp2XD+BfmmuIADzAzDu0QPRTRDQPsUCzXvr4dwAmE9FsAB8iVvUSzLyIiH4D4F1N0bQAuA7AakVbD0UsIP1jAPcr9gtC2pAqqYJgQss+qmHmLdluiyBkA3EfCYIgCAZiKQiCIAgGYikIgiAIBqIUBEEQBANRCoIgCIKBKAVBEATBQJSCIAiCYPD/ASpCfJeqotOGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n    Observation:\\n        Type: Box(4)\\n        Num     Observation               Min                     Max\\n        0       Cart Position             -4.8                    4.8\\n        1       Cart Velocity             -Inf                    Inf\\n        2       Pole Angle                -0.418 rad (-24 deg)    0.418 rad (24 deg)\\n        3       Pole Angular Velocity     -Inf                    Inf\\n    Actions:\\n        Type: Discrete(2)\\n        Num   Action\\n        0     Push cart to the left\\n        1     Push cart to the right\\n        \\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "    Observation:\n",
    "        Type: Box(4)\n",
    "        Num     Observation               Min                     Max\n",
    "        0       Cart Position             -4.8                    4.8\n",
    "        1       Cart Velocity             -Inf                    Inf\n",
    "        2       Pole Angle                -0.418 rad (-24 deg)    0.418 rad (24 deg)\n",
    "        3       Pole Angular Velocity     -Inf                    Inf\n",
    "    Actions:\n",
    "        Type: Discrete(2)\n",
    "        Num   Action\n",
    "        0     Push cart to the left\n",
    "        1     Push cart to the right\n",
    "        \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAADnCAYAAABBu67aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFsUlEQVR4nO3dzW5UdRzH4d9AKW0poImW+kKMiElT0ahrXWli4oob4Abccwls5Aq8AgxuupXEHcTERNCNqaBipOGtVCRt6Yttxw0uYEo7wpeZIzzPrv85yfktTj45/zmT01a73S4AHt+Ofg8A8LQQVIAQQQUIEVSAEEEFCBnY5nM/AQDo1Nps0R0qQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKv8L7Y2NWrj+S92dm+n3KPBQA/0eAB7myrdf1fLtq1VV1W63a+Haxdp38Egd/uSzPk8GmxNUGmvx5uVavPFrv8eArtnyA4QIKkCIoAKECCpAiKDSWC9MfNCxtnjjt1q69+QfmkZQaayh/Qc61taW52t9ZakP08D2BBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQAUIEFSBEUAFCBBUgRFABQgQVIERQaaydu4drYGhvx/rynRt9mAa2J6g01vDzL9eeA4c61m9Nn+3DNLA9QQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUGq3Vam263m63ezwJbE9QabSxIx9V1f1RXZq7UgvXLvZnINiCoNJou0b2PdjT2lhbrfXV5f4MBFsQVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEGl0XYODtfQ/gMd64uzv3trP40jqDTarpH9NTp+uGP9r8vnq0pQaRZBBQhpbbNtcgvAE3H69Ok6depUV8d+emRvvX9w+L61Wwtr9cXZP6ubXf/k5GSdOHHiUcaEh9n0v0cO9HoKqKqanp6uqampro59a/TDeueVt2u9vauqqlqtjZqfv11TU1NdBXVubu4xJoXuCSqNt7qxu76//XHNrb5UVVVDOxbrxbUv+zwVdBJUGu/S/Hs1snKw/t1lLa4/Vyt337z3t2+laA4PpWi8tfZgPfiV1bXlQ/0ZBrYgqDTe8M6FevBO9LU9P/VnGNiCLT+N98boj/X6nnfr5vKrdfXWfA3UnRr5+4ey3adpBJXG+/q76fr5j8+r3a765vzlWlhaqap2V0/4oZe2DOrJkyd7NQfPmHPnznV97IVL1+vCpeuPfK6ZmRnXMlHHjx/fdH3LoB47duyJDAOzs7N15syZnpxrbGzMtUxPbBnU8fHxXs3BM2Z0dLRn5xocHHQt0xOe8gOECCpAiKAChAgqQIigAoT4YT99MTExUUePHu3JuSYnJ3tyHvCCaYD/btMXTNvyA4QIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKEDGzzeasnUwA8BdyhAoQIKkCIoAKECCpAiKAChAgqQMg/rWm8i4mVV2sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "state = env.reset()\n",
    "img = plt.imshow(env.render(mode='rgb_array'))\n",
    "for t in range(200):\n",
    "    #breakpoint()\n",
    "    action = policy.pick_action(state)\n",
    "    img.set_data(env.render(mode='rgb_array')) \n",
    "    plt.axis('off')\n",
    "    #display.display(plt.gcf())\n",
    "    #display.clear_output(wait=True)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        break \n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[257.44824045, 252.62111642],\n",
       "       [263.80890831, 274.86493239],\n",
       "       [255.99231243, 265.24264096],\n",
       "       [259.69445774, 278.85406808]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "policy.w\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Get distribution of episodes needed to solve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17:36:44 - Run nr#0\n",
      "17:58:12 - Run nr#50\n",
      "18:18:27 - Run nr#100\n",
      "18:38:10 - Run nr#150\n",
      "18:59:15 - Run nr#200\n",
      "19:19:26 - Run nr#250\n",
      "19:38:44 - Run nr#300\n",
      "19:58:45 - Run nr#350\n",
      "20:19:13 - Run nr#400\n",
      "20:39:38 - Run nr#450\n",
      "20:58:53 - Run nr#500\n",
      "21:20:03 - Run nr#550\n",
      "21:39:34 - Run nr#600\n",
      "21:59:40 - Run nr#650\n",
      "22:20:34 - Run nr#700\n",
      "22:41:11 - Run nr#750\n",
      "23:00:49 - Run nr#800\n",
      "23:21:16 - Run nr#850\n",
      "23:40:36 - Run nr#900\n",
      "00:01:10 - Run nr#950\n",
      "Wall time: 6h 42min 48s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from datetime import datetime\n",
    "\n",
    "distribution = []\n",
    "weights_history = []\n",
    "for i in range(1000):\n",
    "    env = gym.make('CartPole-v0')\n",
    "    env.seed(i)\n",
    "    np.random.seed(i)\n",
    "\n",
    "    policy = Policy()\n",
    "    scores = hill_climbing(verbose=False)\n",
    "    distribution.append(len(scores))\n",
    "    weights_history.append(policy.w)\n",
    "    if i % 50 == 0:\n",
    "        print(\"{} - Run nr#{}\".format(datetime.now().strftime(\"%H:%M:%S\"), i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAALiklEQVR4nO3dX4idd17H8ffHRHuhsrRkWkKaOkWDbCpYYYji3lSqu5EVU4WuqSIRwkal6x/wJl2E9SbQvVBhwbpk2dBcaGtQ10a6upawS1kQ23QRbRrrhm1sx4Rm1oquN5XErxd5iqfJTGbmnDk5M1/fLyhzzu88z3O+w5R3Hh7mPJOqQpLUy7fNegBJ0sYz7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNbZ/1AAA7duyo+fn5WY8hSVvKK6+88s2qmlvutU0R9/n5ec6ePTvrMSRpS0nyLyu95mUZSWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNbYoPMUlSd/NHn192/eKTH53K+3nmLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1NCqcU+yO8mXk5xPci7JbwzrdyV5IcnXh693juzzRJILSV5P8pFpfgOSpJut5cz9KvBbVfVB4EeAx5PsBY4CZ6pqD3BmeM7w2kHgAWA/8FSSbdMYXpK0vFXjXlWXq+prw+NvAeeBXcAB4OSw2UngkeHxAeDZqnq3qt4ALgD7NnhuSdItrOuae5J54IeAvwPuqarLcP0fAODuYbNdwFsjuy0Oazce60iSs0nOLi0tjTG6JGkla457ku8C/gz4zar6z1ttusxa3bRQdbyqFqpqYW5ubq1jSJLWYE1xT/LtXA/7H1XVnw/LbyfZOby+E7gyrC8Cu0d2vxe4tDHjSpLWYi2/LRPg88D5qvq9kZdOA4eGx4eA50bWDya5I8n9wB7gpY0bWZK0mrX8gewPAb8I/GOSvx/WPgk8CZxKchh4E3gUoKrOJTkFvMb137R5vKqubfTgkqSVrRr3qvoqy19HB3h4hX2OAccmmEuSNAE/oSpJDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhravtkGSE8BPAVeq6geGtd8BPg4sDZt9sqq+OLz2BHAYuAb8elV9aQpzS9LUzR99ftn1i09+9DZPsn5rOXN/Gti/zPrvV9WDw3/vhX0vcBB4YNjnqSTbNmpYSdLarBr3qnoReGeNxzsAPFtV71bVG8AFYN8E80mSxjDJNfdPJPmHJCeS3Dms7QLeGtlmcViTJN1G48b9D4HvBR4ELgO/O6xnmW1ruQMkOZLkbJKzS0tLy20iSRrTWHGvqrer6lpV/Q/wOf7v0ssisHtk03uBSysc43hVLVTVwtzc3DhjSJJWMFbck+wcefozwKvD49PAwSR3JLkf2AO8NNmIkqT1WsuvQj4DPATsSLIIfAp4KMmDXL/kchH4ZYCqOpfkFPAacBV4vKquTWVySdKKVo17VT22zPLnb7H9MeDYJENJkibjJ1QlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDW06oeYJKm7lf4ox1bmmbskNeSZuySt01b483ueuUtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyD/WIen/jY5/Tm8lnrlLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLU0KpxT3IiyZUkr46s3ZXkhSRfH77eOfLaE0kuJHk9yUemNbgkaWVrOXN/Gth/w9pR4ExV7QHODM9Jshc4CDww7PNUkm0bNq0kaU1WjXtVvQi8c8PyAeDk8Pgk8MjI+rNV9W5VvQFcAPZtzKiSpLUa95r7PVV1GWD4evewvgt4a2S7xWHtJkmOJDmb5OzS0tKYY0iSlrPR93PPMmu13IZVdRw4DrCwsLDsNpK0lWym+8WPe+b+dpKdAMPXK8P6IrB7ZLt7gUvjjydJGse4cT8NHBoeHwKeG1k/mOSOJPcDe4CXJhtRkrReq16WSfIM8BCwI8ki8CngSeBUksPAm8CjAFV1Lskp4DXgKvB4VV2b0uySpBWsGveqemyFlx5eYftjwLFJhpIkTcZPqEpSQ8Zdkhoy7pLUkHGXpIY2+kNMkjRTm+mDRLPkmbskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1tH2SnZNcBL4FXAOuVtVCkruAPwHmgYvAx6rq3ycbU5K0Hhtx5v5jVfVgVS0Mz48CZ6pqD3BmeC5Juo2mcVnmAHByeHwSeGQK7yFJuoVJ417A3yR5JcmRYe2eqroMMHy9e8L3kCSt00TX3IEPVdWlJHcDLyT5p7XuOPxjcATgvvvum3AMSdKoic7cq+rS8PUK8AVgH/B2kp0Aw9crK+x7vKoWqmphbm5ukjEkSTcYO+5JvjPJd7/3GPgw8CpwGjg0bHYIeG7SISVJ6zPJZZl7gC8kee84f1xVf53kZeBUksPAm8Cjk48pSVqPseNeVd8AfnCZ9X8DHp5kKEnSZPyEqiQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGpr09gOSNBPzR5+f9QibmmfuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJamj7rAeQpFuZP/r8rEfYkjxzl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIamFvck+5O8nuRCkqPTeh9J0s2mcvuBJNuAPwB+AlgEXk5yuqpem8b7Sdr6vM3AxprWmfs+4EJVfaOq/ht4FjgwpfeSJN1gWjcO2wW8NfJ8Efjh0Q2SHAGODE//K8nrwAeA/xjj/XYA3xxjP63fuD+jzW6zfl+zmGva7zmN42/EMSc9xlj759MT9et7VnphWnHPMmv1vidVx4Hj79spOV5VR1inJGeramG9+2n9xv0ZbXab9fuaxVzTfs9pHH8jjjnpMTZbv6Z1WWYR2D3y/F7g0hr2+8vpjKMN1PVntFm/r1nMNe33nMbxN+KYkx5jU/0/lKpafav1HjTZDvwz8DDwr8DLwM9X1bkNfzM8c5e0dU2rX1O5LFNVV5N8AvgSsA04Ma2wD46vvokkbUpT6ddUztwlSbPlJ1QlqSHjLkkNGXdJaqhd3JM8kuRzSZ5L8uFZzyNJ65Hkg0k+m+RPk/zquMfZEnFPciLJlSSv3rB+083JquovqurjwC8BPzeDcSXpfdbZsPNV9SvAx4Cxf0VyS8QdeBrYP7owcnOynwT2Ao8l2TuyyW8Pr0vSrD3NOhqW5KeBrwJnxn3DLRH3qnoReOeG5WVvTpbrPg38VVV97XbPKkk3Wk/Dhu1PV9WPAr8w7ntO694yt8NKNyf7NeDHgQ8k+b6q+uwshpOkVSzbsCQPAT8L3AF8cdyDb+W4L3tzsqr6DPCZ2z2MJK3TSg37CvCVSQ++JS7LrGDcm5NJ0mYw1YZt5bi/DOxJcn+S7wAOAqdnPJMkrdVUG7Yl4p7kGeBvge9PspjkcFVdBd67Odl54NSUb04mSWOZRcO8cZgkNbQlztwlSetj3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNfS/cvC4k+xySr8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ep_needed = np.array(distribution)\n",
    "bins=np.logspace(np.log10(100),np.log10(1000), 50)\n",
    "\n",
    "plt.figure()\n",
    "plt.hist(ep_needed, bins)\n",
    "plt.gca().set_xscale(\"log\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.839"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(ep_needed < 1000) / len(ep_needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save history of Episodes needed\n",
    "pd.Series(distribution).to_csv('SHC_AN_S.csv')\n",
    "\n",
    "# Save weights\n",
    "arr = np.array(weights_history)\n",
    "indx = pd.MultiIndex.from_product([range(s)for s in arr.shape], names=['run', 'x', 'y'])\n",
    "df = pd.Series(arr.flatten(), index=indx).unstack()\n",
    "df.to_pickle('SHC_AN_w_S.df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}