TennisFinal.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Collaboration and Competition\n",
    "\n",
    "---\n",
    "\n",
    "You are welcome to use this coding environment to train your agent for the project.  Follow the instructions below to get started!\n",
    "\n",
    "### 1. Start the Environment\n",
    "\n",
    "Run the next code cell to install a few packages.  This line will take a few minutes to run!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mtensorflow 1.7.1 has requirement numpy>=1.13.3, but you'll have numpy 1.12.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mipython 6.5.0 has requirement prompt-toolkit<2.0.0,>=1.0.15, but you'll have prompt-toolkit 3.0.36 which is incompatible.\u001b[0m\n",
      "\u001b[31mjupyter-console 6.4.3 has requirement jupyter-client>=7.0.0, but you'll have jupyter-client 5.2.4 which is incompatible.\u001b[0m\n",
      "Requirement already satisfied: unityagents in /opt/conda/lib/python3.6/site-packages (0.4.0)\n",
      "Requirement already satisfied: numpy>=1.11.0 in /opt/conda/lib/python3.6/site-packages (from unityagents) (1.12.1)\n",
      "Requirement already satisfied: protobuf==3.5.2 in /opt/conda/lib/python3.6/site-packages (from unityagents) (3.5.2)\n",
      "Requirement already satisfied: grpcio==1.11.0 in /opt/conda/lib/python3.6/site-packages (from unityagents) (1.11.0)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.6/site-packages (from unityagents) (3.12)\n",
      "Requirement already satisfied: jupyter in /opt/conda/lib/python3.6/site-packages (from unityagents) (1.0.0)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.6/site-packages (from unityagents) (1.2.1)\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.6/site-packages (from unityagents) (2.1.0)\n",
      "Requirement already satisfied: tensorflow==1.7.1 in /opt/conda/lib/python3.6/site-packages (from unityagents) (1.7.1)\n",
      "Requirement already satisfied: pytest>=3.2.2 in /opt/conda/lib/python3.6/site-packages (from unityagents) (4.5.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.6/site-packages (from unityagents) (0.23.3)\n",
      "Requirement already satisfied: Pillow>=4.2.1 in /opt/conda/lib/python3.6/site-packages (from unityagents) (5.2.0)\n",
      "Requirement already satisfied: docopt in /opt/conda/lib/python3.6/site-packages (from unityagents) (0.6.2)\n",
      "Requirement already satisfied: torch==0.4.0 in /opt/conda/lib/python3.6/site-packages (from unityagents) (0.4.0)\n",
      "Requirement already satisfied: ipykernel in /opt/conda/lib/python3.6/site-packages (from unityagents) (4.9.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from protobuf==3.5.2->unityagents) (38.4.0)\n",
      "Requirement already satisfied: six>=1.9 in /opt/conda/lib/python3.6/site-packages (from protobuf==3.5.2->unityagents) (1.11.0)\n",
      "Requirement already satisfied: nbconvert in /opt/conda/lib/python3.6/site-packages (from jupyter->unityagents) (5.4.0)\n",
      "Requirement already satisfied: qtconsole in /opt/conda/lib/python3.6/site-packages (from jupyter->unityagents) (5.2.2)\n",
      "Requirement already satisfied: jupyter-console in /opt/conda/lib/python3.6/site-packages (from jupyter->unityagents) (6.4.3)\n",
      "Requirement already satisfied: notebook in /opt/conda/lib/python3.6/site-packages (from jupyter->unityagents) (5.7.0)\n",
      "Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.6/site-packages (from jupyter->unityagents) (7.0.5)\n",
      "Requirement already satisfied: python-dateutil>=2.0 in /opt/conda/lib/python3.6/site-packages (from matplotlib->unityagents) (2.6.1)\n",
      "Requirement already satisfied: pytz in /opt/conda/lib/python3.6/site-packages (from matplotlib->unityagents) (2017.3)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.6/site-packages/cycler-0.10.0-py3.6.egg (from matplotlib->unityagents) (0.10.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from matplotlib->unityagents) (2.2.0)\n",
      "Requirement already satisfied: gast>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (0.5.3)\n",
      "Requirement already satisfied: tensorboard<1.8.0,>=1.7.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (1.7.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (0.8.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (1.1.0)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (1.3.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.7.1->unityagents) (0.30.0)\n",
      "Requirement already satisfied: py>=1.5.0 in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (1.8.0)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (1.3.0)\n",
      "Requirement already satisfied: pluggy!=0.10,<1.0,>=0.9 in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (0.11.0)\n",
      "Requirement already satisfied: more-itertools>=4.0.0; python_version > \"2.7\" in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (7.0.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (0.1.7)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.6/site-packages (from pytest>=3.2.2->unityagents) (19.1.0)\n",
      "Requirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.6/site-packages (from ipykernel->unityagents) (6.5.0)\n",
      "Requirement already satisfied: traitlets>=4.1.0 in /opt/conda/lib/python3.6/site-packages (from ipykernel->unityagents) (4.3.2)\n",
      "Requirement already satisfied: jupyter_client in /opt/conda/lib/python3.6/site-packages (from ipykernel->unityagents) (5.2.4)\n",
      "Requirement already satisfied: tornado>=4.0 in /opt/conda/lib/python3.6/site-packages (from ipykernel->unityagents) (4.5.3)\n",
      "Requirement already satisfied: mistune>=0.8.1 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (0.8.3)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (2.10)\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (2.2.0)\n",
      "Requirement already satisfied: jupyter_core in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (4.4.0)\n",
      "Requirement already satisfied: nbformat>=4.4 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (4.4.0)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (0.2.3)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (1.5.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (1.4.1)\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (0.3.1)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.6/site-packages (from nbconvert->jupyter->unityagents) (0.5.0)\n",
      "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.6/site-packages (from qtconsole->jupyter->unityagents) (0.2.0)\n",
      "Requirement already satisfied: qtpy in /opt/conda/lib/python3.6/site-packages (from qtconsole->jupyter->unityagents) (2.0.1)\n",
      "Requirement already satisfied: pyzmq>=17.1 in /opt/conda/lib/python3.6/site-packages (from qtconsole->jupyter->unityagents) (17.1.2)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from jupyter-console->jupyter->unityagents) (3.0.36)\n",
      "Requirement already satisfied: Send2Trash in /opt/conda/lib/python3.6/site-packages (from notebook->jupyter->unityagents) (1.5.0)\n",
      "Requirement already satisfied: terminado>=0.8.1 in /opt/conda/lib/python3.6/site-packages (from notebook->jupyter->unityagents) (0.8.1)\n",
      "Requirement already satisfied: prometheus_client in /opt/conda/lib/python3.6/site-packages (from notebook->jupyter->unityagents) (0.3.1)\n",
      "Requirement already satisfied: widgetsnbextension~=3.0.0 in /opt/conda/lib/python3.6/site-packages (from ipywidgets->jupyter->unityagents) (3.0.8)\n",
      "Requirement already satisfied: html5lib==0.9999999 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.8.0,>=1.7.0->tensorflow==1.7.1->unityagents) (0.9999999)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.8.0,>=1.7.0->tensorflow==1.7.1->unityagents) (2.6.9)\n",
      "Requirement already satisfied: werkzeug>=0.11.10 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.8.0,>=1.7.0->tensorflow==1.7.1->unityagents) (0.14.1)\n",
      "Requirement already satisfied: jedi>=0.10 in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (0.10.2)\n",
      "Requirement already satisfied: simplegeneric>0.8 in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (0.8.1)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (0.1.0)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (4.3.1)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (0.7.4)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.6/site-packages (from ipython>=4.0.0->ipykernel->unityagents) (4.0.11)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /opt/conda/lib/python3.6/site-packages (from jinja2->nbconvert->jupyter->unityagents) (1.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.4->nbconvert->jupyter->unityagents) (2.6.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.6/site-packages (from qtpy->qtconsole->jupyter->unityagents) (16.8)\r\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.6/site-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0->ipykernel->unityagents) (0.5.2)\r\n"
     ]
    }
   ],
   "source": [
    "!pip -q install ./python\n",
    "!pip install unityagents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The environment is already saved in the Workspace and can be accessed at the file path provided below. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:unityagents:\n",
      "'Academy' started successfully!\n",
      "Unity Academy name: Academy\n",
      "        Number of Brains: 1\n",
      "        Number of External Brains : 1\n",
      "        Lesson number : 0\n",
      "        Reset Parameters :\n",
      "\t\t\n",
      "Unity brain name: TennisBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 8\n",
      "        Number of stacked Vector Observation: 3\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 2\n",
      "        Vector Action descriptions: , \n"
     ]
    }
   ],
   "source": [
    "#from UnityEnvironment import UnityEnvironment\n",
    "import numpy as np\n",
    "from unityagents import UnityEnvironment\n",
    "\n",
    "env = UnityEnvironment(file_name=\"/data/Tennis_Linux_NoVis/Tennis\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Environments contain **_brains_** which are responsible for deciding the actions of their associated agents. Here we check for the first brain available, and set it as the default brain we will be controlling from Python."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Examine the State and Action Spaces\n",
    "\n",
    "Run the code cell below to print some information about the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 2\n",
      "Size of each action: 2\n",
      "There are 2 agents. Each observes a state with length: 24\n",
      "The state for the first agent looks like: [ 0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.         -6.65278625 -1.5        -0.          0.\n",
      "  6.83172083  6.         -0.          0.        ]\n"
     ]
    }
   ],
   "source": [
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "\n",
    "# number of agents \n",
    "num_agents = len(env_info.agents)\n",
    "print('Number of agents:', num_agents)\n",
    "\n",
    "# size of each action\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Size of each action:', action_size)\n",
    "\n",
    "# examine the state space \n",
    "states = env_info.vector_observations\n",
    "state_size = states.shape[1]\n",
    "print('There are {} agents. Each observes a state with length: {}'.format(states.shape[0], state_size))\n",
    "print('The state for the first agent looks like:', states[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Take Random Actions in the Environment\n",
    "\n",
    "In the next code cell, you will learn how to use the Python API to control the agent and receive feedback from the environment.\n",
    "\n",
    "Note that **in this coding environment, you will not be able to watch the agents while they are training**, and you should set `train_mode=True` to restart the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total score (averaged over agents) this episode: -0.004999999888241291\n",
      "Total score (averaged over agents) this episode: -0.004999999888241291\n",
      "Total score (averaged over agents) this episode: -0.004999999888241291\n",
      "Total score (averaged over agents) this episode: 0.04500000085681677\n",
      "Total score (averaged over agents) this episode: -0.004999999888241291\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):                                         # play game for 5 episodes\n",
    "    env_info = env.reset(train_mode=False)[brain_name]     # reset the environment    \n",
    "    states = env_info.vector_observations                  # get the current state (for each agent)\n",
    "    scores = np.zeros(num_agents)                          # initialize the score (for each agent)\n",
    "    while True:\n",
    "        actions = np.random.randn(num_agents, action_size) # select an action (for each agent)\n",
    "        actions = np.clip(actions, -1, 1)                  # all actions between -1 and 1\n",
    "        env_info = env.step(actions)[brain_name]           # send all actions to tne environment\n",
    "        next_states = env_info.vector_observations         # get next state (for each agent)\n",
    "        rewards = env_info.rewards                         # get reward (for each agent)\n",
    "        dones = env_info.local_done                        # see if episode finished\n",
    "        scores += env_info.rewards                         # update the score (for each agent)\n",
    "        states = next_states                               # roll over states to next time step\n",
    "        if np.any(dones):                                  # exit loop if episode finished\n",
    "            break\n",
    "    print('Total score (averaged over agents) this episode: {}'.format(np.mean(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When finished, you can close the environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. It's Your Turn!\n",
    "\n",
    "Now it's your turn to train your own agent to solve the environment!  A few **important notes**:\n",
    "- When training the environment, set `train_mode=True`, so that the line for resetting the environment looks like the following:\n",
    "```python\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "```\n",
    "- To structure your work, you're welcome to work directly in this Jupyter notebook, or you might like to start over with a new file!  You can see the list of files in the workspace by clicking on **_Jupyter_** in the top left corner of the notebook.\n",
    "- In this coding environment, you will not be able to watch the agents while they are training.  However, **_after training the agents_**, you can download the saved model weights to watch the agents on your own machine! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import torch\n",
    "from collections import deque\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "def hidden_init(layer):\n",
    "    fan_in = layer.weight.data.size()[0]\n",
    "    lim = 1. / np.sqrt(fan_in)\n",
    "    return (-lim, lim)\n",
    "\n",
    "#Actor network\n",
    "class Actor(nn.Module):\n",
    "    \"\"\"Actor (Policy) Model.\"\"\"\n",
    "\n",
    "    def __init__(self, input_size, output_size, seed, fc1_units=256, fc2_units=256):\n",
    "         \"\"\"Initialize parameters and build actor model.\n",
    "        Params\n",
    "        ======\n",
    "            input_size (int):  number of dimensions for input layer\n",
    "            output_size (int): number of dimensions for output layer\n",
    "            seed (int): random seed\n",
    "            fc1_units (int): number of nodes in first hidden layer\n",
    "            fc2_units (int): number of nodes in second hidden layer\n",
    "        \"\"\"\n",
    "        super(Actor, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fc1 = nn.Linear(input_size, fc1_units)\n",
    "        self.fc2 = nn.Linear(fc1_units, fc2_units)\n",
    "        self.fc3 = nn.Linear(fc2_units, output_size)\n",
    "        self.bn = nn.BatchNorm1d(fc1_units)\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        \"\"\"Initialize weights with near zero values.\"\"\"\n",
    "        self.fc1.weight.data.uniform_(*hidden_init(self.fc1))\n",
    "        self.fc2.weight.data.uniform_(*hidden_init(self.fc2))\n",
    "        self.fc3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "\n",
    "    def forward(self, state):\n",
    "        \"\"\"Build an actor network that maps states to actions.\"\"\"\n",
    "        if state.dim() == 1:\n",
    "            state = torch.unsqueeze(state,0)\n",
    "        x = F.leaky_relu(self.fc1(state))\n",
    "        x = self.bn(x)\n",
    "        x = F.leaky_relu(self.fc2(x))\n",
    "        x = F.tanh(self.fc3(x))\n",
    "        return x\n",
    "\n",
    "#Critic network \n",
    "class Critic(nn.Module):\n",
    "\n",
    "    def __init__(self, input_size, seed, fc1_units=256, fc2_units=256):\n",
    "        \"\"\"Initialize parameters and build model.\n",
    "        Params\n",
    "            input_size (int): number of dimensions for input layer\n",
    "            seed (int): random seed\n",
    "            fc1_units (int): number of nodes in the first hidden layer\n",
    "            fc2_units (int): number of nodes in the second hidden layer\n",
    "        \"\"\"\n",
    "        super(Critic, self).__init__()\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.fc1 = nn.Linear(input_size, fc1_units)\n",
    "        self.fc2 = nn.Linear(fc1_units, fc2_units)\n",
    "        self.fc3 = nn.Linear(fc2_units, 1)\n",
    "        self.bn1 = nn.BatchNorm1d(fc1_units)\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        \"\"\"Initialize weights with near zero values.\"\"\"\n",
    "        self.fc1.weight.data.uniform_(*hidden_init(self.fc1))\n",
    "        self.fc2.weight.data.uniform_(*hidden_init(self.fc2))\n",
    "        self.fc3.weight.data.uniform_(-3e-3, 3e-3)\n",
    "\n",
    "    def forward(self, states, actions):\n",
    "        \"\"\"Build a critic network that maps (states, actions) pairs to Q-values.\"\"\"\n",
    "        xs = torch.cat((states, actions), dim=1)\n",
    "        x = F.leaky_relu(self.fc1(xs))\n",
    "        x = self.bn1(x)\n",
    "        x = F.leaky_relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "#Creation of the 4 networks \n",
    "class Actor_Critic_Models():\n",
    "\n",
    "    def __init__(self, n_agents, state_size=24, action_size=2, seed=0):\n",
    "        \"\"\"\n",
    "        Params\n",
    "        ======\n",
    "            n_agents (int): number of agents\n",
    "            state_size (int): number of state dimensions for a single agent\n",
    "            action_size (int): number of action dimensions for a single agent\n",
    "            seed (int): random seed\n",
    "        \"\"\"\n",
    "        self.actor_local = Actor(state_size, action_size, seed).to(device)\n",
    "        self.actor_target = Actor(state_size, action_size, seed).to(device)\n",
    "        critic_input_size = (state_size+action_size)*n_agents\n",
    "        self.critic_local = Critic(critic_input_size, seed).to(device)\n",
    "        self.critic_target = Critic(critic_input_size, seed).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import copy\n",
    "from collections import namedtuple, deque\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "class MADDPG():\n",
    "    \"\"\"Meta agent that contains the two DDPG agents and shared replay buffer.\"\"\"\n",
    "\n",
    "    def __init__(self, action_size=2, seed=0, \n",
    "                 n_agents=2,\n",
    "                 buffer_size=10000,\n",
    "                 batch_size=256,\n",
    "                 gamma=0.99,\n",
    "                 update_every=2,\n",
    "                 noise_start=1.0,\n",
    "                 noise_decay=1.0,\n",
    "                 t_stop_noise=30000):\n",
    "        \n",
    "        \"\"\"\n",
    "        Params\n",
    "        ======\n",
    "            action_size (int): dimension of each action\n",
    "            seed (int): Random seed\n",
    "            n_agents (int): number of distinct agents\n",
    "            buffer_size (int): replay buffer size\n",
    "            batch_size (int): minibatch size\n",
    "            gamma (float): discount factor\n",
    "            noise_start (float): initial noise weighting factor\n",
    "            noise_decay (float): noise decay rate\n",
    "            update_every (int): how often to update the network\n",
    "            t_stop_noise (int): max number of timesteps with noise applied in training\n",
    "        \"\"\"\n",
    "\n",
    "        self.buffer_size = buffer_size\n",
    "        self.batch_size = batch_size\n",
    "        self.update_every = update_every\n",
    "        self.gamma = gamma\n",
    "        self.n_agents = n_agents\n",
    "        self.noise_weight = noise_start\n",
    "        self.noise_decay = noise_decay\n",
    "        self.t_step = 0\n",
    "        self.noise_on = True\n",
    "        self.t_stop_noise = t_stop_noise\n",
    "\n",
    "        # create two agents, each with their own actor and critic\n",
    "        models = [Actor_Critic_Models(n_agents=n_agents) for _ in range(n_agents)]\n",
    "        self.agents = [DDPG(i, models[i]) for i in range(n_agents)]\n",
    "        \n",
    "        # create shared replay buffer\n",
    "        self.memory = ReplayBuffer(action_size, self.buffer_size, self.batch_size, seed)\n",
    "\n",
    "    def step(self, all_states, all_actions, all_rewards, all_next_states, all_dones):\n",
    "        all_states = all_states.reshape(1, -1)  # reshape 2x24 into 1x48 dim vector\n",
    "        all_next_states = all_next_states.reshape(1, -1)  # reshape 2x24 into 1x48 dim vector\n",
    "        self.memory.add(all_states, all_actions, all_rewards, all_next_states, all_dones)\n",
    "        \n",
    "        # if t_stop_noise time steps are achieved turn off noise\n",
    "        if self.t_step > self.t_stop_noise:\n",
    "            self.noise_on = False\n",
    "        \n",
    "        self.t_step = self.t_step + 1     \n",
    "        # Learn every update_every time steps.\n",
    "        if self.t_step % self.update_every == 0:\n",
    "            # If enough samples are available in memory, get random subset and learn\n",
    "            if len(self.memory) > self.batch_size:\n",
    "                # sample from the replay buffer for each agent\n",
    "                experiences = [self.memory.sample() for _ in range(self.n_agents)]\n",
    "                self.learn(experiences, self.gamma)\n",
    "\n",
    "    def act(self, all_states,currepisode, add_noise=True):\n",
    "        # pass each agent's state from the environment and calculate its action\n",
    "        all_actions = []\n",
    "        for agent, state in zip(self.agents, all_states):\n",
    "            action = agent.act(state,currepisode, noise_weight=self.noise_weight, add_noise=self.noise_on)\n",
    "            self.noise_weight *= self.noise_decay\n",
    "            all_actions.append(action)\n",
    "        return np.array(all_actions).reshape(1, -1) # reshape 2x2 into 1x4 dim vector\n",
    "\n",
    "    def learn(self, experiences, gamma):\n",
    "        # each agent uses its own actor to calculate next_actions\n",
    "        all_next_actions = []\n",
    "        all_actions = []\n",
    "        for i, agent in enumerate(self.agents):\n",
    "            states, _, _, next_states, _ = experiences[i]\n",
    "            agent_id = torch.tensor([i]).to(device)\n",
    "            # extract agent i's state and get action via actor network\n",
    "            state = states.reshape(-1, 2, 24).index_select(1, agent_id).squeeze(1)\n",
    "            action = agent.actor_local(state)\n",
    "            all_actions.append(action)\n",
    "            # extract agent i's next state and get action via target actor network\n",
    "            next_state = next_states.reshape(-1, 2, 24).index_select(1, agent_id).squeeze(1)\n",
    "            next_action = agent.actor_target(next_state)\n",
    "            all_next_actions.append(next_action)\n",
    "                       \n",
    "        # each agent learns from its experience sample\n",
    "        for i, agent in enumerate(self.agents):\n",
    "            agent.learn(i, experiences[i], gamma, all_next_actions, all_actions)\n",
    "            \n",
    "    def save_agents(self):\n",
    "        # save models for local actor and critic of each agent\n",
    "        for i, agent in enumerate(self.agents):\n",
    "            torch.save(agent.actor_local.state_dict(),  f\"checkpoint_actor_agent_{i}.pth\")\n",
    "            torch.save(agent.critic_local.state_dict(), f\"checkpoint_critic_agent_{i}.pth\")\n",
    "\n",
    "\n",
    "class DDPG():\n",
    "    \"\"\"DDPG agent with own actor and critic.\"\"\"\n",
    "\n",
    "    def __init__(self, agent_id, model, action_size=2, seed=0,\n",
    "                 tau=1e-3,\n",
    "                 lr_actor=1e-4,\n",
    "                 lr_critic=1e-3,\n",
    "                 weight_decay=0.0):\n",
    "        \n",
    "        \"\"\"\n",
    "        Params\n",
    "        ======\n",
    "            model: model object\n",
    "            action_size (int): dimension of each action\n",
    "            seed (int): Random seed\n",
    "            tau (float): for soft update of target parameters\n",
    "            lr_actor (float): learning rate for actor\n",
    "            lr_critic (float): learning rate for critic\n",
    "            weight_decay (float): L2 weight decay\n",
    "        \"\"\"\n",
    "        random.seed(seed)\n",
    "        self.id = agent_id\n",
    "        self.action_size = action_size\n",
    "        self.tau = tau\n",
    "        self.lr_actor = lr_actor\n",
    "        self.lr_critic = lr_critic\n",
    "       \n",
    "        # Actor Network\n",
    "        self.actor_local = model.actor_local\n",
    "        self.actor_target = model.actor_target\n",
    "        self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=lr_actor)\n",
    "        \n",
    "        # Critic Network\n",
    "        self.critic_local = model.critic_local\n",
    "        self.critic_target = model.critic_target\n",
    "        self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=lr_critic, weight_decay=weight_decay)\n",
    "        \n",
    "        # Set weights for local and target actor, respectively, critic the same\n",
    "        self.hard_copy_weights(self.actor_target, self.actor_local)\n",
    "        self.hard_copy_weights(self.critic_target, self.critic_local)\n",
    "        \n",
    "        # Noise process\n",
    "        self.noise = OUNoise(action_size, seed)\n",
    "\n",
    "    def hard_copy_weights(self, target, source):\n",
    "        \"\"\" copy weights from source to target network (part of initialization)\"\"\"\n",
    "        for target_param, param in zip(target.parameters(), source.parameters()):\n",
    "            target_param.data.copy_(param.data)\n",
    "    \n",
    "    def act(self, state,numepisode, noise_weight=1.0, add_noise=True):\n",
    "        \"\"\"Returns actions for given state as per current policy.\"\"\"\n",
    "        state = torch.from_numpy(state).float().to(device)\n",
    "        # calculate action values\n",
    "        self.actor_local.eval()\n",
    "        with torch.no_grad():\n",
    "            action = self.actor_local(state).cpu().data.numpy()\n",
    "        self.actor_local.train()\n",
    "        if add_noise:\n",
    "            #self.noise_val = self.noise.sample() * noise_weight           \n",
    "            self.noise_val = self.noise.sample() * noise_weight\n",
    "            action += self.noise_val\n",
    "            #action += (self.noise.sample()/np.sqrt(numepisode))  \n",
    "        return np.clip(action, -1, 1)\n",
    "\n",
    "    def reset(self):\n",
    "        self.noise.reset()\n",
    "\n",
    "    def learn(self, agent_id, experiences, gamma, all_next_actions, all_actions):\n",
    "        \"\"\"Update policy and value parameters using given batch of experience tuples.\n",
    "        Params\n",
    "        ======\n",
    "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples\n",
    "            gamma (float): discount factor\n",
    "            all_next_actions (list): each agent's next_action (as calculated by its actor)\n",
    "            all_actions (list): each agent's action (as calculated by its actor)\n",
    "        \"\"\"\n",
    "\n",
    "        states, actions, rewards, next_states, dones = experiences\n",
    "\n",
    "        # ---------------------------- update critic ---------------------------- #\n",
    "        # get predicted next-state actions and Q values from target models\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        agent_id = torch.tensor([agent_id]).to(device)\n",
    "        actions_next = torch.cat(all_next_actions, dim=1).to(device)\n",
    "        with torch.no_grad():\n",
    "            q_targets_next = self.critic_target(next_states, actions_next)\n",
    "        # compute Q targets for current states (y_i)\n",
    "        q_expected = self.critic_local(states, actions)\n",
    "        # q_targets = reward of this timestep + discount * Q(st+1,at+1) from target network\n",
    "        q_targets = rewards.index_select(1, agent_id) + (gamma * q_targets_next * (1 - dones.index_select(1, agent_id)))\n",
    "        # compute critic loss\n",
    "        critic_loss = F.mse_loss(q_expected, q_targets.detach())\n",
    "        # minimize loss\n",
    "        critic_loss.backward()\n",
    "        \n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "        # ---------------------------- update actor ---------------------------- #\n",
    "        # compute actor loss\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        # detach actions from other agents\n",
    "        actions_pred = [actions if i == self.id else actions.detach() for i, actions in enumerate(all_actions)]\n",
    "        actions_pred = torch.cat(actions_pred, dim=1).to(device)\n",
    "        actor_loss = -self.critic_local(states, actions_pred).mean()\n",
    "        # minimize loss\n",
    "        actor_loss.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        # ----------------------- update target networks ----------------------- #\n",
    "        self.soft_update(self.critic_local, self.critic_target, self.tau)\n",
    "        self.soft_update(self.actor_local, self.actor_target, self.tau)\n",
    "\n",
    "\n",
    "    def soft_update(self, local_model, target_model, tau):\n",
    "        \"\"\"Soft update model parameters.\n",
    "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
    "        Params\n",
    "        ======\n",
    "            local_model (PyTorch model): weights will be copied from\n",
    "            target_model (PyTorch model): weights will be copied to\n",
    "            tau (float): interpolation parameter\n",
    "        \"\"\"\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
    "\n",
    "\n",
    "class OUNoise:\n",
    "    \"\"\"Ornstein-Uhlenbeck process.\"\"\"\n",
    "\n",
    "    def __init__(self, size, seed, mu=0., theta=0.15, sigma=0.2):\n",
    "        \"\"\"Initialize parameters and noise process.\"\"\"\n",
    "        random.seed(seed)\n",
    "        np.random.seed(seed)\n",
    "        self.size = size\n",
    "        self.mu = mu * np.ones(size)\n",
    "        self.theta = theta\n",
    "        self.sigma = sigma\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        \"\"\"Reset the internal state (= noise) to mean (mu).\"\"\"\n",
    "        self.state = copy.copy(self.mu)\n",
    "\n",
    "    def sample(self):\n",
    "        \"\"\"Update internal state and return it as a noise sample.\"\"\"\n",
    "        x = self.state\n",
    "        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.size)\n",
    "        self.state = x + dx\n",
    "        return self.state\n",
    "\n",
    "\n",
    "class ReplayBuffer():\n",
    "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
    "\n",
    "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
    "        \"\"\"Initialize a ReplayBuffer object.\n",
    "        Params\n",
    "        ======\n",
    "            action_size (int): dimension of each action\n",
    "            buffer_size (int): maximum size of buffer\n",
    "            batch_size (int): size of each training batch\n",
    "            seed (int): Random seed\n",
    "        \"\"\"\n",
    "        random.seed(seed)\n",
    "        np.random.seed(seed)\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=buffer_size)\n",
    "        self.batch_size = batch_size\n",
    "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
    "\n",
    "\n",
    "    def add(self, state, action, reward, next_state, done):\n",
    "        \"\"\"Add a new experience to memory.\"\"\"\n",
    "        e = self.experience(state, action, reward, next_state, done)\n",
    "        self.memory.append(e)\n",
    "\n",
    "    def sample(self):\n",
    "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
    "        experiences = random.sample(self.memory, k=self.batch_size)\n",
    "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
    "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).float().to(device)\n",
    "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
    "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
    "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
    "        return (states, actions, rewards, next_states, dones)\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Return the current size of internal memory.\"\"\"\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Train_MADDPGNetwork(env,brain_name,agent,num_agents,num_episodes=5000,max_timesteps=1000,print_every=100,train_mode=True):\n",
    "    \n",
    "    scores = []\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_avg = []\n",
    "    \n",
    "    \n",
    "    for episode in range(1, num_episodes+1):\n",
    "        scorestab = np.zeros(num_agents)  # initialize the score (for each agent)\n",
    "        rewardslist = []\n",
    "        env_info = env.reset(train_mode=False)[brain_name]  # reset the environment    \n",
    "        states = env_info.vector_observations      # get the current state (for each agent)\n",
    "\n",
    "        # loop over steps\n",
    "        for t in range(max_timesteps):\n",
    "            # select an action\n",
    "            actions = agent.act(states,episode)\n",
    "            # take action in environment and set parameters to new values\n",
    "            env_info = env.step(actions)[brain_name]\n",
    "            next_states = env_info.vector_observations\n",
    "            rewards = env_info.rewards\n",
    "            dones = env_info.local_done\n",
    "            scorestab += env_info.rewards    # update the score (for each agent)\n",
    "            # update and train agent with returned information\n",
    "            agent.step(states, actions, rewards, next_states, dones)\n",
    "            states = next_states\n",
    "            rewardslist.append(rewards)\n",
    "            if np.any(dones):\n",
    "                break\n",
    "\n",
    "        # calculate episode reward as maximum of individually collected rewards of agents\n",
    "        episode_reward = np.max(np.sum(np.array(rewardslist),axis=0))\n",
    "        \n",
    "        scores.append(episode_reward)             # save most recent score to overall score array\n",
    "        scores_deque.append(episode_reward)       # save most recent score to running window of 100 last scores\n",
    "        current_avg_score = np.mean(scores_deque)\n",
    "        scores_avg.append(current_avg_score)      # save average of last 100 scores to average score array\n",
    "    \n",
    "        print('\\rEpisode {}\\tAverage Score: {:.3f}'.format(episode, current_avg_score),end=\"\")\n",
    "        \n",
    "        # log average score every 100 episodes\n",
    "        if episode % print_every == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.3f}'.format(episode, current_avg_score))\n",
    "            print(\"\\n\")\n",
    "            print('Score (max over agents) from episode {}: {}'.format(episode, np.max(scorestab)))\n",
    "            print(\"\\n\")\n",
    "       \n",
    "        # break and report success if environment is solved\n",
    "        if np.mean(scores_deque)>=.5:\n",
    "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.3f}'.format(episode, np.mean(scores_deque)))\n",
    "            agent.save_agents()\n",
    "            break\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'unityagents.environment.UnityEnvironment'>\n",
      "<class 'str'>\n",
      "TennisBrain\n",
      "Unity brain name: TennisBrain\n",
      "        Number of Visual Observations (per agent): 0\n",
      "        Vector Observation space type: continuous\n",
      "        Vector Observation space size (per agent): 8\n",
      "        Number of stacked Vector Observation: 3\n",
      "        Vector Action space type: continuous\n",
      "        Vector Action space size (per agent): 2\n",
      "        Vector Action descriptions: , \n",
      "Number of agents: 2\n",
      "Size of each action: 2\n",
      "There are 2 agents. Each observes a state with length: 24\n",
      "The state for the first agent looks like: [ 0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.         -7.0517621  -1.5        -0.          0.\n",
      " -7.11741829  5.94114017 -0.          0.        ]\n",
      "<class '__main__.MADDPG'>\n",
      "Episode 100\tAverage Score: 0.008\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 100: 0.0\n",
      "\n",
      "\n",
      "Episode 200\tAverage Score: 0.019\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 200: 0.0\n",
      "\n",
      "\n",
      "Episode 300\tAverage Score: 0.027\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 300: 0.0\n",
      "\n",
      "\n",
      "Episode 400\tAverage Score: 0.021\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 400: 0.0\n",
      "\n",
      "\n",
      "Episode 500\tAverage Score: 0.014\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 500: 0.0\n",
      "\n",
      "\n",
      "Episode 600\tAverage Score: 0.020\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 600: 0.10000000149011612\n",
      "\n",
      "\n",
      "Episode 700\tAverage Score: 0.041\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 700: 0.0\n",
      "\n",
      "\n",
      "Episode 800\tAverage Score: 0.045\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 800: 0.0\n",
      "\n",
      "\n",
      "Episode 900\tAverage Score: 0.057\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 900: 0.0\n",
      "\n",
      "\n",
      "Episode 1000\tAverage Score: 0.056\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1000: 0.0\n",
      "\n",
      "\n",
      "Episode 1100\tAverage Score: 0.070\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1100: 0.20000000298023224\n",
      "\n",
      "\n",
      "Episode 1200\tAverage Score: 0.079\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1200: 0.10000000149011612\n",
      "\n",
      "\n",
      "Episode 1300\tAverage Score: 0.128\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1300: 1.3000000193715096\n",
      "\n",
      "\n",
      "Episode 1400\tAverage Score: 0.168\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1400: 0.20000000298023224\n",
      "\n",
      "\n",
      "Episode 1500\tAverage Score: 0.156\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1500: 0.0\n",
      "\n",
      "\n",
      "Episode 1600\tAverage Score: 0.198\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1600: 0.09000000171363354\n",
      "\n",
      "\n",
      "Episode 1700\tAverage Score: 0.227\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1700: 0.10000000149011612\n",
      "\n",
      "\n",
      "Episode 1800\tAverage Score: 0.152\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1800: 0.20000000298023224\n",
      "\n",
      "\n",
      "Episode 1900\tAverage Score: 0.158\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 1900: 0.20000000298023224\n",
      "\n",
      "\n",
      "Episode 2000\tAverage Score: 0.216\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2000: 0.7000000104308128\n",
      "\n",
      "\n",
      "Episode 2100\tAverage Score: 0.406\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2100: 2.600000038743019\n",
      "\n",
      "\n",
      "Episode 2200\tAverage Score: 0.312\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2200: 1.9000000283122063\n",
      "\n",
      "\n",
      "Episode 2300\tAverage Score: 0.173\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2300: 0.0\n",
      "\n",
      "\n",
      "Episode 2400\tAverage Score: 0.150\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2400: 1.5000000223517418\n",
      "\n",
      "\n",
      "Episode 2500\tAverage Score: 0.249\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2500: 0.20000000298023224\n",
      "\n",
      "\n",
      "Episode 2600\tAverage Score: 0.372\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2600: 1.1000000163912773\n",
      "\n",
      "\n",
      "Episode 2700\tAverage Score: 0.390\n",
      "\n",
      "\n",
      "Score (max over agents) from episode 2700: 0.6000000089406967\n",
      "\n",
      "\n",
      "Episode 2731\tAverage Score: 0.506\n",
      "Environment solved in 2731 episodes!\tAverage Score: 0.506\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(type(env))\n",
    "# get the default brain\n",
    "brain_name = env.brain_names[0]\n",
    "brain = env.brains[brain_name]  \n",
    "print(type(brain_name))\n",
    "print(brain_name)    \n",
    "print(brain)\n",
    "# reset the environment\n",
    "env_info = env.reset(train_mode=True)[brain_name]\n",
    "# number of agents\n",
    "num_agents = len(env_info.agents)\n",
    "print('Number of agents:', num_agents)\n",
    "# size of each action\n",
    "action_size = brain.vector_action_space_size\n",
    "print('Size of each action:', action_size)\n",
    "# examine the state space \n",
    "states = env_info.vector_observations\n",
    "state_size = states.shape[1]\n",
    "print('There are {} agents. Each observes a state with length: {}'.format(states.shape[0], state_size))\n",
    "print('The state for the first agent looks like:', states[0])\n",
    "    \n",
    "MADDPG_Agent = MADDPG(seed=2, noise_start=0.5, update_every=2, gamma=0.99, t_stop_noise=30000)   \n",
    "print(type(MADDPG_Agent))\n",
    "training_scores = Train_MADDPGNetwork(env,brain_name,MADDPG_Agent,num_agents)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Results################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XecXHW9//HXZ3eTUIJSEqSYEJColyItBmNBLCDFH1ivqD+wcEW92O7Vx71B/CkggliQK6AQFMSGXKUIJgRIDIQW0ggppBfIpm7aZlM2ye5+fn/MmdnZ2ek7Z+ac2ffz8djHzpw5c+b7nfL9nG895u6IiIgANNQ6ASIiEh0KCiIikqKgICIiKQoKIiKSoqAgIiIpCgoiIpKioCAiIikKCiIikqKgICIiKU21TkCphgwZ4iNGjKh1MkREYmXWrFmb3H1oof1iFxRGjBjBzJkza50MEZFYMbNXi9lPzUciIpKioCAiIikKCiIikqKgICIiKQoKIiKSoqAgIiIpCgoiIpKioCAiNTdn9TYefmkNzyxt4a8zV5N+meDZr23l4tufY27zttQ2d+dvs5pZtH47zy/bVIskl231ll1MXdJS62TkFLvJayJSfz5y+3M97h98wEDOOeENAHzsV88DcNFtz7HqxxcC8NTiFr7z15dT+ye3x8F7fzqFLo9umlVTEJHIaWvfl/fx7QUej7IuL7xPLSkoiIhIioKCiIikhBYUzGyYmU0xs4VmtsDMvplln7PNrNXM5gR/3w8rPSIiUliYHc0dwLfdfbaZHQTMMrMn3f2VjP2ecfcPh5gOEREpUmg1BXdf5+6zg9ttwELg6LBeT0RE+q4qfQpmNgI4DXgxy8NjzOxlM3vMzE6sRnpERCS70OcpmNlg4AHgW+6+PePh2cAx7r7DzC4AHgZGZjnGFcAVAMOHDw85xSIi/VeoNQUzG0AiIPzJ3R/MfNzdt7v7juD2BGCAmQ3Jst84dx/l7qOGDi14NTkRESlTmKOPDPgtsNDdb86xzxHBfpjZ6CA9m8NKk4iI5Bdm89G7gEuBeWY2J9j2XWA4gLvfAXwC+KqZdQC7gUs8fdETERGpqtCCgrs/C1iBfW4DbgsrDSIiUhrNaBaRyLG8p5MSJgUFEYkcNSLXjoKCiIikKCiIiEiKgoKIiKQoKIiISIqCgoiIpCgoiEjkaEhq7SgoiEjkaEhq7SgoiIiE7JZJS1i6oa3WySiKgoKISIja2vdxy6Sl/OudL9Q6KUVRUBARqYK9HV21TkJRFBREREIUXB2AuHSTKCiISOTU0+ijhiAvcek8V1AQEQmRkawpxCMqKCiISOTE5ay6FHHJk4KCiEiIkk1hMYkJCgoiUjsjxo7n8t/N6PNx2vd1MmLseG6dvLQCqQpJTKKCgoKI1NTkRRv7fIztu/cB8Ptpr/b5WGHpikn7kYKCiEROPY0+ikksSFFQEBGpgrjEBgUFEZEq8JhUGRQURCRyYlJ+lqQrJnlSUBARkRQFBRGREMVlJnOSgoKIRE49jT6KGwUFERFJUVAQEZEUBQURqRv1OGqp2kILCmY2zMymmNlCM1tgZt/Mso+Z2S/NbJmZzTWz08NKj4jERz0V7nHLS1OIx+4Avu3us83sIGCWmT3p7q+k7XM+MDL4OxP4dfBfRKRkceqgdvfUVdmiJLSagruvc/fZwe02YCFwdMZuFwO/94RpwMFmdmRYaRIRkfyq0qdgZiOA04AXMx46Gliddr+Z3oFDRPqZCJ5A9xuhBwUzGww8AHzL3bdnPpzlKb1a4MzsCjObaWYzW1pawkimiEhVRbWvIdSgYGYDSASEP7n7g1l2aQaGpd1/I7A2cyd3H+fuo9x91NChQ8NJrIhICCJa9ucU5ugjA34LLHT3m3Ps9ghwWTAK6R1Aq7uvCytNIiJREdVgEeboo3cBlwLzzGxOsO27wHAAd78DmABcACwDdgFfCDE9IhITUW1a6Q9CCwru/izZ+wzS93HgyrDSICISVYniL3o96prRLCKRo9FHtaOgICJ1I4rNTulXXGvf11lw/6Ub2mp6lTYFBRGRKvmP++ekbmcr9p9Z2sI5v5jKX2c2Vy9RGRQURKRuRL3ZaeqS/POslm/cAcCCta3VSE5WCgoiIjUQxaYuUFAQEZE0CgoiIiHKVSGI6rWbFRREpG5EtUkmThQUREQkRUFBRGKvK6ghVGL0kbvzs8cXs2bb7r4fLO/rFN5n4vx13PzkEu6auiLUtKQLc+0jEZGqmLN6W8WOtXBdG7dNWcYzyzbx9yvfVbHjluMrf5yduv2ls46rymuqpiAidaBynQldwSn8vo6uihyvlH6OKHSJKCiISN1QR3PfKSiIiNRAtgAWhQnZCgoiEntxqSFY1NfhQEFBROpI1Mvc9NVPNXlNRCQk0SxeA+poFpFqmbZiMyPGjmdec+1W1axXYdQ61HwkIqH656KNADy/fFONUxINlexbCLufQh3NIiIhiUJhWoxaXlGtWAoKIhJ7YRS1lWrpydWhHNXwoKAgIrHnFVz7KEyF+hSiECgUFERE0lSrhSeqTUkKCiJSNyJazhYtChUdBQWROhDzsjBSot4EFTYFBRGJHKvhOXOlaxu5jhfVQK6gIFIH6u3kttQlIJL7V/Isv7/WGBQURESyqMXktShQUBARkZTQgoKZ3W1mG81sfo7HzzazVjObE/x9P6y0iEj/UMmz79CbjyJaUwjzGs2/A24Dfp9nn2fc/cMhpkFEpKZKKfujECdCqym4+1RgS1jHF5FuUShMaimq7fP56HoK2Y0xs5fN7DEzO7HGaRGRiCh1SGqyeK1kk8/8Ndsrd7AiJZP/4Etr+PS4aT0eGzF2PLdPWRZ6GsJsPipkNnCMu+8wswuAh4GR2XY0syuAKwCGDx9evRSKxES9jZ6M6ll0tbS1d/DCis29ts9cFX7jS81qCu6+3d13BLcnAAPMbEiOfce5+yh3HzV06NCqplNEoi8ZFOPUjFROWt/31sMrn5AMNQsKZnaEBUsGmtnoIC29Q6OIFBSjsjDyKl1LSV/4rlCNrtArV6NGWHTzkZm9Gxjp7veY2VBgsLuvzLP/fcDZwBAzawZ+AAwAcPc7gE8AXzWzDmA3cIlHddlAEZEK8By3i1aFadZFBQUz+wEwCngLcA+Jwv2PwLtyPcfdP53vmO5+G4khqyLSR/XWp1CqSp5N1nLdpUKvXI2UFdt89FHgImAngLuvBQ4KK1EiUpp6q2KXPPrIK7f2UZid3OnJi2rDSLFBYW/QtOMAZnZgeEkSEclt5aadtU5CRbXu3kdL256i9q3GIn3F9in8r5ndCRxsZl8CvgjcFV6yRKQU/aX5aPZrW/nYr57njGMOyfp4FE++C/UjjLlxMrv2drLqxxcW0dEckT4Fd/+ZmZ0DbCfRr/B9d38y1JSJSL+VqwlnVVBLmLN6WzWTE4pkDnft7Sz6OZGoKZhZI/C4u38QUCAQiaAIniCHKqrt8YVEoSO5kIJ9Cu7eCewys9dXIT0iIjklz5TjGRJ6KieuRWmeQjswz8yeJBiBBODu3wglVSJSkiicYVZSuW3nFRl9VMOIU7BPIQrNR4HxwZ+IRFA9nDlXQhRblXJfo7n0xEapo/leMxsIvDnYtNjd94WXLBEpRr3VEAqpRqHYX6/NnFTUPAUzOxtYCtwO/ApYYmZnhZgukchZvL6Nu6auAGDD9nZ+/sTimnd4RvDEOPYq8ZHePmVZWfMpCsajKgSsYiev/Rw4193f6+5nAR8CfhFeskSi58O3PsOPJiwE4Ft/mcOt/1wWmaGR9XZyG+els7fs3MtPH1/MZ++aln/HiHY0FxsUBrj74uQdd19CsLidSH+xr7P7V9zekRhb3hWRsisiyaiazLP55P0oNP10BYlp7+gq+bmFO5oj0qcAzDSz3wJ/CO5/FpgVTpJEpL/L1XeQq0yMUs0iFaCS93OkLTop7qnYoPBV4ErgGyTyOpVE34KIREAETpAjIUqjj8I4qY/SPIUm4H/c/WZIzXIeFFqqRCKsZ+dyNEqhaKRCoPhaS7YAVnDGc4Q6micD+6fd3x+YVPnkiEgpVEOIoFRhX/lPJ0pBYb/k9ZQBgtsHhJMkkWjreYZX22JZNYSEKDUbJRUqwLPVKKKQjWKDwk4zOz15x8xGkbiEpojUUPu+xCioccH8iXpRasdxlEYf9Up5BUv6yMxoBr4F/NXM1pLI4lHAp0JLlUiEReFsLmn77sTCAlt27q1xSqIhCjWGzNFHlVTz5iMze7uZHeHuM4C3AvcDHcBEYGX4yROJnih1NFdj3Hot5B6SGn5+K/WJFmw+KqOjuRoKNR/dCSRPQcYA3yWx1MVWYFyI6RIRiaUozZkoR6Hmo0Z33xLc/hQwzt0fAB4wsznhJk0kmqL0k6/12ktREaV3obv5KMt5f9qmbGmOwozmQjWFRjNLBo4PAP9Me6zY/giRuuIejWo+1G/zUS65chul4JhMSRgXBIrC5LX7gKfNbBOJ0UbPAJjZ8UBryGkTiazoFEGSLkoxMmtS0r445QSymnc0u/uPgG8DvwPe7d25aAC+Hm7SRKIpSm3GUTpDzqZ9XycT56/vsW3mqi00b90VyutF4e1I/0ymr9zC2m3do/fb9nTkfW4UYlrBJiB377X+a7BKqki/FYUfL0BDlE6Ns7juH6/w5xdf48F/fyenDz8EgE/c8QIAq358YS2TFpruORPGv975QmnPLfB4NeYpFDt5TUQCUTgbjYvVWxI1guR8ir7KvUpq/JTzPap585GISF/0t47wsEXpIjsiImUL/Uy+gi8Q9X6asIUWFMzsbjPbaGbzczxuZvZLM1tmZnPT11YSibJ+XmbUVDXa1CulnEpSnJbOLsfvgPPyPH4+MDL4uwL4dYhpEamYKI0+ilEZ2W8Ue9KQbb/CT41xR7O7TwW25NnlYuD3njANONjMjgwrPSJ9Fcnm8QjFp2yq9ZZFKVAn0xLKlddiXlMo5Ghgddr95mCbSKSp+agMFXrPIhmYc4hTU1e6WgaFbO9Y1q+OmV1hZjPNbGZLS0vIyRKJkYiXO9UqxKMUqItuPspS3BXsUyg9OSWrZVBoBoal3X8jsDbbju4+zt1HufuooUOHViVxIrlEqPyJjWo170ShJpG59lElRWFBvDA9AlwWjEJ6B9Dq7utqmB6RvJI/x0gNWYxQUrKpXp9C8D8C70fy+1Eo7+V0NEdhQbyymdl9wNnAEDNrBn4ADABw9zuACcAFwDJgF/CFsNIiUrcicGZcTXHKblwn7oUWFNz90wUed+DKsF5fJCwROBlNiUtnZthn8FGoISQVm5Ryklzvo49EYiV55hetAihCickirmfLfdGXazTX++Q1kdhYvWUXnV3RLmAzvbp5Z6QCVD6VSme1Y8ymHXvYkWO56w3b22nf15n7yQWv0dz7TdEqqSIRsK51N+/5yRR+MnFRcU/ocaGUcNJUyLzmVt7706d46KU1tUlAkeI+eW3U9ZP44M+fzvrYmTdM5rLfTs+amjhTUJB+b1PbXgCeW76p6OfUullk1eadNX39Sti6c2+tk5BVZpG+fnt7zn2nr+q9aENfmo8KUvORSPQ43j3ssEaxoR6a6vd1ddU6CaEqdOJQVkdzeUkpiYKC9HvFNj10z1NIe268WwqqpnJvUx1EwzwKdzSrT0EkkmrdfBSXoajlvk2lPi9KwbnoIallrZIaPgUFkRJF4YcbN1lnged5I0st5KP0mYTZp6DmI5EISZ69RmGZi/j0KQRzOyp1tFz5jsBnklT8SKjS06x5CiL9wOotu7h9yrKSgk2+suEfc9fy7NLiR1Lls2j9dn733Mo+H+emiYuYuiS6Kxy7O7dMWsKGPCON0i3d0JZ1+59efJV5za1ASAviVaGuENoyFyJxU+wPrtLnpJffO4MlG3Zw0SlHMezQA/p8vK/9+SUAVv34wj4f67xbngHg8+86tk/HWdGyk8vunt4jTRV9H/tYAs9b08otk5bywvLN/Pf5by24/4W3Ppt1+9UPdV99uJwCPAoVQNUURIoU1lnazj15ZsXGXL6yuqItPn08WHI2e94Zymn2dhQeTlsoTpW1Sqqaj0TCV3KnZnSar2OtK88bmavwq8aZdJQ/X3U0i1RRsWdhUViELi4dzdW+nkKclJVm1RREIiQmBXFc5CsU66H2Vuu5LOVSUBApVSQKoHgWOOm6yliVNq4FbTbZAlnhazRrRrNIZKSWuahpKmojjLkZUTy7r6Ryim91NIvE1JqtuwGYtHBjxY75np9M4d//NKuofR+c3Vz0ca96cC4jxo4vN1lAaQX4utbdjBg7nhFjx7OxbU+Px9LTka+juVTJoLVpx54Ce+b3cnMrX//z7Lz73D/jtaKOVagA//ucnsuejxg7nukre6+62uOYRb1y3ygoSL9XatHk3r2ccimFczEmzFtf1H5PvLKh6GPeN311uclJKeU9Wrhue+p2rkleUN7oo2pY25p/Atu9z79akde5b3rv4DJxQXGff5gUFET6oI6auPMqt/koXx9AORe6i9PbXc53o9DbrFVSRSIofUhqXFYr7atyG3ryvTuV7KeIYvdEoe9GWddTUJ+CSHR0L4jXe1u9K6X8LnbffDWFeu+ELpf6FEQkEsKYsFdOR3OuINzXIBJGs0w5y1xEgYKCSKDYYiGiv+VQlV2A5XlTKzn6qL9Q85FIBLg71z36Cu37EougvfTa1tRj61rbWbVpZ0nHe37ZJjbv2MOkVzZwz3Mr2V1gEbbVW3bx48cWsa8zftc0bmvvyPnY5h17cz62c2/u5xVj6869RS8fvreji1snL827z+zXtnLn08vZuaeDV9JGV+VTqPwur09FS2eL1NxTS1q4O+2aAsmlqZPO/tlTRS9T3dXlfOY3L/LWIw5i0frcwzXTvecnUwBoajC+86G3VOxaCaUI46T+srun53zs6ofm89kzjyn6WJnJ+9w905nb3MqiH57HfgMa8z73Z08sZvKi/PNNPvar5wEYP29d0WkKg2oKIlVQ6Ixt687cZ7Qlv1bwf3Ge8fu5rN2WmDC3dVfl0lOsUpp6wmwVKrZQXBK8v8Wku3nrrqJf/5W1xdUSgIKJjWrjmYKCSA2UU3B6xv9qikoBlmuYZ3pgd3caggK50gGqsaH4U3Vdo1mkTlWyyt6Xsfm1vDZ0FK5LXazOLk8VnpXuzG4qISgUVM7kvbhPXjOz88xssZktM7OxWR7/vJm1mNmc4O/fwkyPSF4Rn3SQLENqkcr4hITE/IewCs+GEo4bzjWawxdaR7OZNQK3A+cAzcAMM3vE3V/J2PV+d/9aWOkQ6atKzlqOU+GaLkYVhR61g0onu6GCzUdRfUvDrCmMBpa5+wp33wv8Bbg4xNcTKUs1f5x9KVyTz61JYRLVEiyQ/r52dnnW2eeVUNHmozLEffTR0UD68ozNwbZMHzezuWb2NzMbFmJ6RLJKjlFv272v7GPcMGFhpZLTw8T53atmFirfpizayGd/My2UdPz0iUWhHLeQmavyLyWdTad39ylUui8kW03h6ofmcd4tU2nJWCZ89mvb8h5rx54Ojv/uhJJeP+4X2cmW+sxP6FFghLu/DZgE3Jv1QGZXmNlMM5vZ0tJS4WRKfzdlceI7tSLHJLRizs7GTV1R1GuVulzEV/7YfX2FZAGXKzlf/sMsnlu2Ofdr96GA/OO04q4hUGlXPTiv54YiPouuLk/1KZSzEms+2SoKf3rxNRatb+t1fYRidJSYwLjXFJqB9DP/NwJr03dw983ungyvdwFnZDuQu49z91HuPmro0KGhJFYkl0qebIbaNh+RtXZqmEUg2dGcvB3xdq8ICjMozABGmtmxZjYQuAR4JH0HMzsy7e5FQDh1cJE6UGieQlw7Niuts6t7nkK9BYVq1BRCG33k7h1m9jXgcaARuNvdF5jZdcBMd38E+IaZXQR0AFuAz4eVHpHYK3gBlgJPdydel6kpvhBMb5br6tGnUPk05VKVOQRVeI1Q1z5y9wnAhIxt30+7fRVwVZhpEOmrSi4b3afRR+TvUyjUCVnp9vUoyTX6qJo1hWqE27j3KYhIFRWsKcSwAamc0TZdaTWiSgfCWl9pL+6jj0Rqqq19X6/F7DZub2dvR+4lqFuzDEst92SzrX0f89e09jxWCQVzZlqat+5m6869bMwY+pi0a2/+Jbhbd+1je3txw26Xt+xg4/aeF7Bv3b2P5q27cPfU/2xqsSTGlrTPuaure5RQS9sedhd4XyplT57vVaVUY5qEls6WunXmDZPZtbcztax1V5cz+obJXHDyEfzqs1kHunHKtU8UvQx2ISdf8wQA4y49g3NPPAKAqUuKW/Z66pKWXktLz21u5bQfPll2ekbfMJnGBmP5DRfk3W/Gqi188o4Xem0/5dpEfj49ehj3TV/NdRefyGVjRtDR2cWjc9dy8SlHlzTjt5AtWVanzdVuf2fakOBO724++sjtz3Hy0a/n0a+/O+frrNm6u+g05auN3TQx/Lkcaj4S6YPMM+fk+Wv6hLBqmJ+23HKxF2jJd62Bvugsoj1lSYFlvR+YlRiP/+LKxMSyu59byX/c/zJ/m93c9wSm2Z6l1lZMmZiZx3kZtbVMG7Znr3lFk5qPRCqm3GaNys5TiF+7fqbU2WqQlY1BobqtBtd5yKYe3uNcVFMQqaBaFRXpv+N6GDefulZB8I56xvZKKfdwnSG+x7Ue0FuNIakKCtJvlFtWVLKIiUNMKDTCJXmhmWRewgx05ZSBiesphFN4VmMuQt7Xr8JrKChIv1HukMxKNkfUw1yBhow5AMm3p9IFZrkFe1f4g4BqRs1HIhUUjZpC4aNFvU28IaOmkJTR1VAzpdZcIn5tpR40TyHG5q9p5f4ZtVlZsr95ZmkLE+ev4/LfzeCuLKuVtu9LjEJKXqC9y2FjW3uv/XIqsowZc+Nk3v6jSezc09Fj+93PruT55ZsYN3V5j6GTuTy9pDIrAY8YO547nl7ea/tfpuf+Xs5YtYVHX16b83GAbbsSo4KeeGUDN05YyEurE0tET1ywnimLN/LlP8zK9/SiZSusixmVla9P4fnlm5gwb13ZaVqzrfjhq2GI9dpH/d2Hb30WgE+9fXiNU1L/Lv1td0ExedFGvnTWcT0eHzd1Bd/4wMge+/3n/S/zx387s6jjF9vstK41EWgmLdzAxad2XzqkbU8Hn7nrxaKOAfD5e2YUvW8hP36s99j5sQ/O45LRvb+Xi9e3ZZ2fkE96kJu+cgvTV5Z+/YNc9nZ2sWh9/uGx2XzhnhnsP6Ax62PJzyF9LkqMKgpqPhKphGRNoa29+wy+LeNsvpI6OmvdgFKebLWKWsqcjV6sbLPS84nTp1WVRfdCfwWRGkt27qa31XeV0OMb8Sb+ulXqBWikMhQUpO5la/6ph/kC9a6UwJ0pTp3HpejLe1IsBQWpe8kfUvrPKd9vK/OHV+rPMK7hJmqjnqpVU4hYtvOqxsmMgoLUvWTZ0lVk81HmDy9OhUY9KWadpv6mGu+JgoLEWq7CvUf/QcYkK8g/bLHTHXcvu6pepy0XVac+hd6q8ZZY1KqMhYwaNcpnzpxZ62SkfOauacxZvS3nWvaVWoa5Uu59fhU/eGQBC687j70dXZxy3RP89BNv45OjhrGiZQfv//nTAMy95lxet9+Asl+nq8tZtXknxw0d3Oc0v7Z5F2f9dAr7D2hk8H5NtOS4nkClvGfkEJ5Zuon9BzSye1911uKvhY+cehQPz8k/J6FeJH+HI8aO73Ef4B03TGb99hLmrdTQM//1PoYdekBZzzWzWe4+qtB+qin00fPLNxe8uEmUJIcdbt21l9XBZK57nlsFdC+FDLC+tW8/kl89tYz3//xpFpcxzjzTjFWJdO3e1xl6QAB4Zumm1OvVs/4SEAqJcqf0n7/Ucy5NuQGhFAoK/VjyxxBGXXHWq1uB7lnEfRGvuqzETVQbS0YdcwjvfNOQqr+ugkI/lloCOYRfRXIlzUq0gWr4qPRHtfrWKyj0M+nlazIohDGiwSp47Lj1e0m8RLX5qFbfewWFfsqs9xLIlZQ8diW+2BqEImGKaEyoGQWFfsq9+2ze07aR5XY5Ktl8pPHq0h/V6lvf71ZJfX7ZJh56aQ1fOus4xj4wlzsuPYPX7TeAx+av4/yTjmS/YHXFZ5a2cNrwQxg8qIndezt5ceVmzn7L4Xz3oXk8sWADhx44gE9nWWmyFFMWb+RNQwbz+IL1/GjCQt4zcgjnnvAGZr66lcXr22hsMNa3trO3o4uGBuPgAwYw8Ztnsf/A7CtAZrNo/Xb27Ovi2kcXMPu1bant5/5iKjuCReFWtOzk4tue5eXm7gucd3Y5f5z2Ktc+uoABjQ384fIzuWXSktTIHICBjQ0MaDSOOnh/Th12MFOXtrB11z4OGtTE5mAxsyv/PJtxU1/PJaOH8/BLa1i0vo3Bg5rY2NbOAQObcHe2BwvVnXnsoSze0Ma2XftSw0HPP+kIHpu/vk/vs8iIseM5+uD9U/dvfmIxv/znshqmqLBatZr2u3kKyXHK6a7/yEl87+H53PDRk/nMmcNZ39rOO26czAf/5XB+87m385/3z+HBl9bw16+MKXlp4ZU3XpBzZcMRY8eXPBb+/334BC5/97FF758tv8V431uGMmVxZdb1F5HSnTrsYB6+8l1c/rsZTF60kaYGY9kNF5R9vGLnKfS7mkI2G4OJK8kx8MlCetnGHQAs37QTgM07Sh8jn2im6b09OVu21LHwa7ZW5yIfCgjV96lRw7j+oyext6MLB076weMALLn+fJZsaOPmJ5fwi389lc/8ZhoL1m7n2+e8mVe37OJvs5p7HeuFq97Pkg07+FyWi9LMuPqDDGxq4JRrn+Dz7xzB195/PKOun1RWmh/46hhumriYM445hF8/1XPp7bcecVDqegj3fnE073zTYVz36CuMGHIg73vLUK56cB7jLh3FKdc9UdRrHXbgwFQNtBomfOM9HH/4YB6es4b/+ttcAL7+/uP5jw++mZ17O2hqaKCxwXjz9x4DYP61H2K/pgb2dHQxoLEBM2hqMDq7nOOvfizvay390fmpJdcXrt/Ox371fKr56Leff3toecxGQYHebXe9ak99qE3lema5U/g63gvoAAAL10lEQVQ76vkCtP1cQ4MxoLGBAY0NPfpRBjY1cNLRr+fuoHAY/4339HheZlBIztZdvnFn1tcxg9fvP6DHrN6LTjmKRwpccS2b44YM5n+/PAaA/z7vrUB37XTit87qtf8PP3JS6vb9wfNy+djpR/Pg7DWp+wft11QwKKQHor7ab0ADA5saGNTU3fV64KAmGhqMg7LM9h88KFGcNjX27KptaizclZ343BO3a93xrY7mPCpxQYtczXPljvjRejD1K/3r1hBiyZDt0OV+1RuLKPDK1ZCRqMYi3pRiCuBSXz/9p9pYhfGrqXKnHoekmtl5ZrbYzJaZ2dgsjw8ys/uDx180sxFhpqdUmQV6OeVxrueUW7hXYz11qY304qYiJyQljF8pt/wJs5DMjAFNDYWLq8Yi9ilV+glcQ5jROpB8hbqbvGZmjcDtwPnACcCnzeyEjN0uB7a6+/HAL4CbwkpPKSrZ5JPrh9lZ5iUbVVOoX5UuX6txolnM2Xu5jNJrCgNCSE9Xj5pCxQ+f83Ov1RigMGsKo4Fl7r7C3fcCfwEuztjnYuDe4PbfgA9YNS5CWkDybDwzKeWcpef6YPMt3ZyPxuxLsUr5ppT7q2sKMShknvQPKKJErmSQ6m7F6X4nwwyCma9bK2F2NB8NrE673wycmWsfd+8ws1bgMGATFfb0khau/8crWR+7NRiv/D+Tl/CPuWvZ05HozF25aSfn3Pw0S4NRSDc+trDk173wl8/0ahuF8gv3h15aw/w1rYV3lNjZr6n4+SfFKKXATu9MLUWYheSgjPdj8H6Fi6sDB1WuSEvWVNLzOLDM9ymfAwf2THOyvCj3M+mrMINCtm9LZklYzD6Y2RXAFQDDh5c3YWzwoCZGvmFwqoAfPKiJHXs6OPGo1zH80AN4bP56PnTiEako/dqWXYw+9lCGDB7I0IMG8fzyzZxxzCFMmFfcRKqjD96fwYOaeNPhB+bcZ9XmnZxw1OuYv2Z70fk4/6QjSjqTaN66u+Rhr4OCYXXpku9XvZr0n+/lgzc/nfWx9PfjrDcPZeqSFr581nHcOXVFap8hgwexqcCQ5YFNDexNe18vOPkI2to7UhMCLxszosf+3//wCbzjuMMKpv1/LjmV+6a/xrQVW/jrV7pH9Iw57rDU5L+Pn/5GFqxt5cKTj+SwwYN6HePqC05gyYYdfObM4TSY8Z2/vswBAxt7LQv/zQ+M5N4XVnH/FWOYuqQla9/Hg//+ThatK34E0E0fP5lHX17HF989ghUtO1m7rZ09HZ18+9w3c95JR/D3OWt4YPYafnPZ2/mX70/kpo+fzP/ObGZAozFtxZYex/r5J0/hw7c+y5pt5Q/dvvPSM1i8vo1hhyYmu/2fU45i5qtbcYePnvbGXvv/7StjWN6yI+8xx116Btc8soCRbziIAY0NTFq4AUiMsPrqe9/UY98Tj3od3/zASC4ZPazsPPRFaJPXzGwMcI27fyi4fxWAu9+Yts/jwT4vmFkTsB4Y6nkSFbWL7IiIxEEULrIzAxhpZsea2UDgEuCRjH0eAT4X3P4E8M98AUFERMIVWvNR0EfwNeBxoBG4290XmNl1wEx3fwT4LfAHM1sGbCEROEREpEZCndHs7hOACRnbvp92ux34ZJhpEBGR4mlGs4iIpCgoiIhIioKCiIikKCiIiEiKgoKIiKTE7sprZtYCvFrm04cQwhIaEaL8xVs956+e8wbxyN8x7j600E6xCwp9YWYzi5nRF1fKX7zVc/7qOW9QX/lT85GIiKQoKIiISEp/Cwrjap2AkCl/8VbP+avnvEEd5a9f9SmIiEh+/a2mICIiefSboGBm55nZYjNbZmZja52ecpjZKjObZ2ZzzGxmsO1QM3vSzJYG/w8JtpuZ/TLI71wzO722qe/NzO42s41mNj9tW8n5MbPPBfsvNbPPZXutWsiRv2vMbE3wGc4xswvSHrsqyN9iM/tQ2vbIfXfNbJiZTTGzhWa2wMy+GWyvi88vT/7q4vPLy93r/o/E0t3LgeOAgcDLwAm1TlcZ+VgFDMnY9hNgbHB7LHBTcPsC4DESV7d7B/BirdOfJT9nAacD88vND3AosCL4f0hw+5Ba5y1P/q4BvpNl3xOC7+Ug4Njg+9oY1e8ucCRwenD7IGBJkIe6+Pzy5K8uPr98f/2lpjAaWObuK9x9L/AX4OIap6lSLgbuDW7fC3wkbfvvPWEacLCZHVmLBObi7lNJXEcjXan5+RDwpLtvcfetwJPAeeGnvrAc+cvlYuAv7r7H3VcCy0h8byP53XX3de4+O7jdBiwkcc31uvj88uQvl1h9fvn0l6BwNLA67X4z+T/gqHLgCTObFVy3GuAN7r4OEl9k4PBge1zzXGp+4pjPrwVNKHcnm1eIcf7MbARwGvAidfj5ZeQP6uzzy9RfgkK2S93HcdjVu9z9dOB84EozOyvPvvWS56Rc+YlbPn8NvAk4FVgH/DzYHsv8mdlg4AHgW+6+Pd+uWbbFMX919fll01+CQjMwLO3+G4G1NUpL2dx9bfB/I/AQiarphmSzUPB/Y7B7XPNcan5ilU933+Dune7eBdxF4jOEGObPzAaQKDD/5O4PBpvr5vPLlr96+vxy6S9BYQYw0syONbOBJK4F/UiN01QSMzvQzA5K3gbOBeaTyEdyxMbngL8Htx8BLgtGfbwDaE1W6yOu1Pw8DpxrZocEVflzg22RlNGv81ESnyEk8neJmQ0ys2OBkcB0IvrdNTMjcY31he5+c9pDdfH55cpfvXx+edW6p7tafyRGPywhMRLg6lqnp4z0H0di5MLLwIJkHoDDgMnA0uD/ocF2A24P8jsPGFXrPGTJ030kquD7SJxRXV5OfoAvkujYWwZ8odb5KpC/PwTpn0uicDgybf+rg/wtBs6P8ncXeDeJZpC5wJzg74J6+fzy5K8uPr98f5rRLCIiKf2l+UhERIqgoCAiIikKCiIikqKgICIiKQoKIiKSoqAgsWFmnWmrU84ptOKkmX3FzC6rwOuuMrMhJez/lAWr2Ab3R5nZU31NR3Csz5vZbZU4lkg2TbVOgEgJdrv7qcXu7O53hJmYAg43s/Pd/bEapqEXM2t0985ap0OiSzUFib3gTP4mM5se/B0fbL/GzL4T3P6Gmb0SLGT2l2DboWb2cLBtmpm9Ldh+mJk9YWYvmdmdpK1fY2b/N3iNOWZ2p5k15kjWT4HvZUlrjzN9M/uHmZ0d3N4R5GOWmU0ys9FBrWOFmV2UdphhZjYxWKP/B4XSFhz3OjN7ERhTznss/YeCgsTJ/hnNR59Ke2y7u48GbgNuyfLcscBp7v424CvBtmuBl4Jt3wV+H2z/AfCsu59GYtbqcAAz+xfgUyQWJjwV6AQ+myOtLwB7zOx9JeTvQOApdz8DaAOuB84hsZzCdWn7jQ5e91Tgk0HzVL60HUjimg5nuvuzJaRH+iE1H0mc5Gs+ui/t/y+yPD4X+JOZPQw8HGx7N/BxAHf/Z1BDeD2Ji+N8LNg+3sy2Bvt/ADgDmJFYGof96V7wLZvrSdQW/ruIvAHsBSYGt+cBe9x9n5nNA0ak7feku28GMLMHg3x05ElbJ4mF3UQKUlCQeuE5biddSKKwvwj4f2Z2IvmXNc52DAPudferikpQItD8kMSVxpI66FlD3y/t9j7vXnemC9gTHKfLzNJ/q5lpSy7RnCtt7epHkGKp+UjqxafS/r+Q/oCZNQDD3H0K8F/AwcBgYCpBE0vQrr/JE2vmp28/n8RlIiGxwNsnzOzw4LFDzeyYAun6UfCaSauAU82swcyG0b30cinOCV57fxJXNnuuzLSJ9KKagsTJ/mY2J+3+RHdPDksdFHSkNgCfznheI/DHoGnIgF+4+zYzuwa4x8zmArvoXvL5WuA+M5sNPA28BuDur5jZ90hc/a6BxOqnVwKv5kqwu08ws5a0Tc8BK0k0D80HZpf0DiQ8S2K1zuOBP7v7TIBS0yaSjVZJldgzs1UklmLeVOu0iMSdmo9ERCRFNQUREUlRTUFERFIUFEREJEVBQUREUhQUREQkRUFBRERSFBRERCTl/wPcoAS+0DViXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f86faea4a90>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "#plotting the scores\n",
    "fig = plt.figure()\n",
    "ax= fig.add_subplot()\n",
    "plt.plot(np.arange(1,len(training_scores) + 1),training_scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode Number')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}