MineRLNavigate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from MinerlNavigate_utils import *\n",
    "from MinerlNavigate_models import Actor, Critic\n",
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "import gym \n",
    "import minerl\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "buffer_size = 100000\n",
    "batch_size = 128\n",
    "gamma = 0.99\n",
    "tau = 0.001\n",
    "lr_actor = 0.0001\n",
    "lr_critic = 0.001\n",
    "    \n",
    "action_size = 9\n",
    "state_size = (64, 64, 3)\n",
    "    \n",
    "rate = 0.2\n",
    "epsilon = 0.01\n",
    "render = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_from_memory(mission, buff, actor, critic, batch_size = 128, sequence = False):\n",
    "    \n",
    "    gamma = 0.99\n",
    "    \n",
    "    batch = buff.getBatch(batch_size, sequence)\n",
    "    #batch_size -= 1\n",
    "    states = np.asarray([e[0] for e in batch])\n",
    "    states_pov = np.stack([s['pov'][0] for s in states])\n",
    "    states_cA = np.stack([s['compassAngle'][0] for s in states])\n",
    "    actions = np.asarray([e[1] for e in batch])\n",
    "    #actions = np.stack([a[0] for a in actions])\n",
    "    \n",
    "    rewards = np.asarray([e[2] for e in batch])\n",
    "\n",
    "    next_states = np.asarray([e[3] for e in batch])\n",
    "    next_states_pov = np.stack([ns['pov'][0] for ns in next_states])\n",
    "    next_states_cA = np.stack([ns['compassAngle'][0] for ns in next_states])\n",
    "\n",
    "    dones = np.asarray([e[4] for e in batch])\n",
    "    \n",
    "    target_actions = actor.target_predict([next_states_pov, next_states_cA])\n",
    "    target_q_values = critic.target_predict([next_states_pov, next_states_cA, target_actions])\n",
    "     \n",
    "    critic_target = np.asarray(target_q_values)\n",
    "    for i in range(target_q_values.shape[0]):\n",
    "        if dones[i]:\n",
    "            critic_target[i] = rewards[i]\n",
    "        else:\n",
    "            critic_target[i] = rewards[i] + gamma * target_q_values[i]\n",
    "\n",
    "    critic.train_on_batch(states_pov, states_cA, actions, critic_target) \n",
    "   \n",
    "    action_for_grads = actor.predict([states_pov, states_cA])\n",
    "    grads = critic.gradients(states_pov, states_cA, action_for_grads)\n",
    "    actor.train(states_pov, states_cA, np.array(grads).reshape(-1, 9))\n",
    "    actor.transfer_weights()\n",
    "    critic.transfer_weights()\n",
    "    \n",
    "    return buff, actor, critic\n",
    "\n",
    "\n",
    "def train_on_batch(state, action, reward, next_state, done, actor, critic):\n",
    "    \n",
    "    q_values = critic.target_predict([next_state['pov'], next_state['compassAngle'], action])\n",
    "    \n",
    "    critic_target = np.asarray(q_values)\n",
    "    for i in range(q_values.shape[0]):\n",
    "        if done[i]:\n",
    "            critic_target[i] = reward[i]\n",
    "        else:\n",
    "            critic_target[i] = reward[i] + gamma * q_values[i]\n",
    "    \n",
    "    critic.train_on_batch(state['pov'], state['compassAngle'], action, critic_target)\n",
    "        \n",
    "    actions = actor.predict([state['pov'], state['compassAngle']])\n",
    "    #print(actions)\n",
    "    gradients = critic.gradients(state['pov'], state['compassAngle'], actions)\n",
    "    actor.pre_train(state['pov'], state['compassAngle'], action)\n",
    "    \n",
    "    actor.transfer_weights()\n",
    "    critic.transfer_weights()\n",
    "    #print('Done!')\n",
    "    \n",
    "def pretrain2(mission, buff, actor, critic, batch_size = 128):\n",
    "    max_iterations = 1\n",
    "    outerstepsize0 = 0.1\n",
    "    \n",
    "    data = minerl.data.make(mission)\n",
    "        \n",
    "    for iteration in range(1, max_iterations+1):\n",
    "        print('--------------------------------------------------------------')\n",
    "        t1 = time()\n",
    "        print('Starting outer loop...iteration ',iteration, '/',max_iterations)\n",
    "        old_weights = actor.model.get_weights()\n",
    "        #print(old_weights)\n",
    "        print('Starting inner loop....')\n",
    "        for state, action, reward, next_state, done in data.sarsd_iter(num_epochs=1, max_sequence_len=batch_size, sample_no = sample_no):\n",
    "            batch_size = state['pov'].shape[0]\n",
    "            if(state['pov'].shape != (batch_size,64,64,3)):\n",
    "                print('skipping batch....found unexpected array size', state['pov'].shape)\n",
    "            else:\n",
    "                state = observation_wrapper(state, batch_size)\n",
    "                action = map_from_actionset(action, batch_size)\n",
    "                #print(action.shape)\n",
    "                #print('Batch ready!!...Beginning training....')        \n",
    "                train_on_batch(state, action, reward, next_state, done, actor, critic)\n",
    "        print('Done!')\n",
    "        new_weights = actor.model.get_weights()\n",
    "\n",
    "        outerstepsize = outerstepsize0 * (1 - iteration/max_iterations)\n",
    "        print('Updating weights...')        \n",
    "        modified_weights = []\n",
    "        for i in range(len(old_weights)):\n",
    "            value = old_weights[i] + (new_weights[i] - old_weights[i])*outerstepsize\n",
    "            modified_weights.append(value)  \n",
    "        print('Setting new weights to model...')            \n",
    "        actor.model.set_weights(modified_weights)\n",
    "        print('End of OuterLoop....')\n",
    "        print('Time taken:', (time()-t1)/3600, 'hrs')\n",
    "        actor.save('navigatedense')\n",
    "        critic.save('navigatedense')\n",
    "    return buff, actor, critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Mrugank_pc\\Softwares\\Python\\Anaconda3_5.1.0\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Mrugank_pc\\Softwares\\Python\\Anaconda3_5.1.0\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "actor = Actor(state_size, action_size, lr_actor, tau)\n",
    "critic = Critic(state_size, action_size, lr_critic, tau)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "buff = ReplayBuffer(buffer_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mission = 'MineRLNavigateDense-v0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "buff, actor, critic = pretrain2(mission, buff, actor, critic, batch_size)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    " env = gym.make(mission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully loaded previously trained weights!\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    actor.load_weights(\"navigatedense_actor.h5\")\n",
    "    critic.load_weights(\"navigatedense_critic.h5\")\n",
    "    actor.load_targetweights(\"navigatedense_actortarget.h5\")\n",
    "    critic.load_targetweights(\"navigatedense_critictarget.h5\")\n",
    "    print(\"Successfully loaded previously trained weights!\")\n",
    "except:\n",
    "    print(\"Saved model not found...starting training from scratch\")\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode : 0 Replay Buffer 0\n",
      "episode: 0/1, mean reward: -0.00026065572102864585, e: 0.01\n",
      "Time taken: 7.419208641846975 mins\n",
      "Training completed!!!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2979d3ec358>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAF69JREFUeJzt3X/wXXV95/Hn6/sNCSoqQQQikAa2WSvddYK9Q6Vs3Ypg0ekKOtji9Efa6mamrbNt3bbCsuN03TqD7ra4O2urKWrpjvV3XTKiRUDQ6YyA31TkV4yJCJImkiiiHfmV5PveP+75wj3f3O/3m+Te7+/nY+bOPedzPueczyffk/u653POvTdVhSRJE0bmuwGSpIXFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpZcUwNpLkIuB/AaPANVV11aTlbwPeAhwA9gG/XVUPNssOAnc3Vb9TVa+baX8nnnhirVu3bhhNl6RlY+vWrd+rqhfOVG/gYEgyCrwPuBDYBXw1yZaquq+n2teATlU9luR3gPcAv9Ise7yqNhzJPtetW8fY2NigTZekZSXJg4dTbxhDSecAO6vq/qp6CvgYcHFvhaq6paoea2ZvA04bwn4lSbNgGMFwKvBQz/yupmwqbwY+3zN/bJKxJLcluWSqlZJsauqN7du3b7AWS5KmNIxrDOlT1vcrW5P8GtAB/n1P8dqq2p3kTOCLSe6uqm8dssGqzcBmgE6n41fCStIsGcYZwy7g9J7504DdkysluQC4EnhdVT05UV5Vu5vn+4FbgbOH0CZJ0lEaRjB8FVif5IwkK4HLgC29FZKcDXyAbijs7SlfnWRVM30icB7Qe9FakjTHBh5KqqoDSd4K3ED3dtUPVdW9Sd4JjFXVFuB/AMcBn0wCz9yW+hLgA0nG6YbUVZPuZpIkzbEsxl9w63Q65e2qknRkkmytqs5M9ZbVJ59v3vYwf3nrzvluhiQtaMsqGG7dvo+//vL9890MSVrQllUwjI6Eg+OLb+hMkubSsgqGkQRzQZKmt6yCYXQEzxgkaQZD+XbVxWJkJBxchHdhHYmnDozzpW/uY3QEMulD6QfGixUjoSZ9MH18HEYmvUWoguTQMji0fKptTC4L4cB4MTryzHxvuybrV364ZYOuP99t2n9wnGNGR2asN1ttOpz9F0XI03Un5vutXxQHx3n6uJy8bu/2JtZfMZq+dWfaz8RxNrmtB8c5pGx8vDvEPDEP/Y9lOLL/E/3qTt527/E/+f/rVH+/V/zrE1m1YvTQDQ/ZsgqG0YTxJX7G8J5/+AbX/OO357sZkmbBr7/8J/jvl/ybWd/PMhtKWvpnDA98/7GZK0lalHbs/Zc52c+yCoaRhCpYjB/qO1yjy+ovKi0vczXgsaxeRibGEpfyBejRPmPIkpaGuXrtWpbBcGAJB8NIvytekpYEg2EWTATD+JIeSjIYpKVqrl67llcwxKEkSYvXgYMGw9CNTJwxjM9zQ2bRqENJ0pLlGcMsGG1eM5fyLaueMUhL11xdH11eweBdSZIWMS8+z4KRZXDxud9XHkhaGgyGWbDULz5XFbds3zffzZA0SwyGWTCyxIeSbrzvYb7ziF+JIS1Viy4YklyUZHuSnUku77N8VZKPN8tvT7KuZ9kVTfn2JL84rDZNttTPGH7w2FPz3QRJs2iubpwZSjAkGQXeB7wGOAt4U5KzJlV7M/CDqvpJ4Grg3c26ZwGXAT8NXAT8ZbO9obtp28MA/O1XHpyNzc+7eKuqtKTN1ZvaYX3t9jnAzqq6HyDJx4CLgft66lwM/Gkz/Sng/6T7SnYx8LGqehL4dpKdzfa+MqS2Pe3Rx/YD3YD41ZevHfbmpxV4+lcQeqeHae+PnpiFrUpaKB758VM89tQBnr1ydn8xYVhbPxV4qGd+F/CzU9WpqgNJfgi8oCm/bdK6pw6pXS1rT3g2X7n/+3znkcd41Z9/aTZ2IUmz6rb7v8/5P3XyrO5jWMHQbwxj8pviqeoczrok2QRsAli79uje7R97THfkbMPpx/Nb5607qm0cjdvu/z4fveMhXnra87lr1w8B+JOLXsypxz9rqPv54eP7ecd19w51m5IWlhef8rxZ38ewgmEXcHrP/GnA7inq7EqyAng+8MhhrktVbQY2A3Q6naMaidn4c+u44d6Hef+v/QynPP/Yo9nEUXli/0E+esdD/NQpz306GC58ycmsP/m5Q93Pj588YDBIS9ya583+a9ew7kr6KrA+yRlJVtK9mLxlUp0twMZm+lLgi9X9xZwtwGXNXUtnAOuBO4bUrpYzX3gct/2XV81pKMAzv+Xae0PBbHxCefJv9Epaekbm4EOsQzljaK4ZvBW4ARgFPlRV9yZ5JzBWVVuADwL/t7m4/Ajd8KCp9wm6F6oPAL9XVQeH0a6FbDZexI8Z9a4kSYMb2qXtqvoc8LlJZe/omX4CeOMU674LeNew2rIYrJiFF3FvV5U0DI49zJMVI/7TS1qYfHWaJw77SFqoDIY51Hsr1QovFEtaoHx1mgPVRELvOYJfjy1poTIY5sCxx3S/+um4Y5+51u+tpZIWqtn9wg0B8EsvfRH//Ojj/ObPrePU45/Fn12/zV9ak7RgGQxzYHQk/O4v/CQAb/n5M3nLz585zy2SpKk5niFJajEYJEktDiUtUScet4o1fb4T6on9B5++GD5deVE8sX+cZzVlE3dWPbl/nGOPGX16/pn1x5/+9tphlIXw+P6DR7T+099J1bRtct0Qnth/kFU9dSf63q9s5YoRRiZ9mvzJAwc5ZnSkdY3o8ae6ZStGQxUk3fVXjDxTb2IzT+4fZ2QkrbvSnjrQLku6ZQArV4y06kH3xoXeZvWru/9gMT5erDpm+jKAAweLA+Pjrb//5LIqGK9i/8Hm79/8+ZPuj8c8dWD8kONn4ljpV9Z7/Dy5f/yQNk3++0/8bR7ff5BnNev3lh076e8H/Y/1Qcp6/0/M1P5+x+lU5UdS9h9e+qJDtjkbDIYl6sY/fAWrn7NyvpshaRFyKGmJ8muTJB0tg2GJmnxaLUmHy2CQJLUYDEuVJwySjpLBIElqMRiWKC8+SzpaBsMSZS5IOloGwxLjl/NJGtRAwZDkhCQ3JtnRPK/uU2dDkq8kuTfJXUl+pWfZ3yT5dpI7m8eGQdqjZ34Zzt9/lnS0Bj1juBy4uarWAzc385M9BvxGVf00cBHw3iTH9yz/46ra0DzuHLA9y94xzW9JHxyvGWpKUn+DBsPFwLXN9LXAJZMrVNU3q2pHM70b2Au8cMD9agrPXtX9fpcDB8fnuSWSFqtBg+HkqtoD0DyfNF3lJOcAK4Fv9RS/qxliujrJqgHbs+z93X98Of/pVes5we9JknSUUjX9kEOSm4BT+iy6Eri2qo7vqfuDqjrkOkOzbA1wK7Cxqm7rKfsu3bDYDHyrqt45xfqbgE0Aa9eu/ZkHH3xw+p5JklqSbK2qzkz1Zvx21aq6YJqdPJxkTVXtaV7k905R73nA9cB/nQiFZtt7msknk3wY+KNp2rGZbnjQ6XQcQJekWTLoUNIWYGMzvRG4bnKFJCuBzwB/W1WfnLRsTfMcutcn7hmwPZKkAQ0aDFcBFybZAVzYzJOkk+Saps4vA68AfrPPbakfSXI3cDdwIvBnA7ZHkjSgGa8xLESdTqfGxsbmuxmStKgc7jUGP/ksSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqWXgYEhyQpIbk+xonldPUe9gkjubx5ae8jOS3N6s//EkKwdtkyTp6A3jjOFy4OaqWg/c3Mz383hVbWger+spfzdwdbP+D4A3D6FNkqSjNIxguBi4tpm+FrjkcFdMEuB84FNHs74kafiGEQwnV9UegOb5pCnqHZtkLMltSSZe/F8APFpVB5r5XcCpQ2iTJOkorTicSkluAk7ps+jKI9jX2qraneRM4ItJ7gZ+1KdeTdGGTcAmgLVr1x7BbiVJR+KwgqGqLphqWZKHk6ypqj1J1gB7p9jG7ub5/iS3AmcDnwaOT7KiOWs4Ddg9xfqbgc0AnU6nb3hIkgY3jKGkLcDGZnojcN3kCklWJ1nVTJ8InAfcV1UF3AJcOt36kqS5M4xguAq4MMkO4MJmniSdJNc0dV4CjCX5Ot0guKqq7muWvR14W5KddK85fHAIbZIkHaV037QvLp1Op8bGxua7GZK0qCTZWlWdmer5yWdJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQMFQ5ITktyYZEfzvLpPnVcmubPn8USSS5plf5Pk2z3LNgzSHknS4AY9Y7gcuLmq1gM3N/MtVXVLVW2oqg3A+cBjwBd6qvzxxPKqunPA9kiSBjRoMFwMXNtMXwtcMkP9S4HPV9VjA+5XkjRLBg2Gk6tqD0DzfNIM9S8DPjqp7F1J7kpydZJVA7ZHkjSgFTNVSHITcEqfRVceyY6SrAH+LXBDT/EVwHeBlcBm4O3AO6dYfxOwCWDt2rVHsmtJ0hGYMRiq6oKpliV5OMmaqtrTvPDvnWZTvwx8pqr292x7TzP5ZJIPA380TTs20w0POp1OzdRuSdLRGXQoaQuwsZneCFw3Td03MWkYqQkTkoTu9Yl7BmyPJGlAgwbDVcCFSXYAFzbzJOkkuWaiUpJ1wOnAlyat/5EkdwN3AycCfzZgeyRJA5pxKGk6VfV94FV9yseAt/TMPwCc2qfe+YPsX5I0fH7yWZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJahk4GJK8Mcm9ScaTdKapd1GS7Ul2Jrm8p/yMJLcn2ZHk40lWDtomSdLRG8YZwz3AG4AvT1UhySjwPuA1wFnAm5Kc1Sx+N3B1Va0HfgC8eQhtkiQdpYGDoaq2VdX2GaqdA+ysqvur6ingY8DFSQKcD3yqqXctcMmgbZIkHb25usZwKvBQz/yupuwFwKNVdWBS+SGSbEoylmRs3759s9pYSVrOVhxOpSQ3Aaf0WXRlVV13OJvoU1bTlB9aWLUZ2AzQ6XT61pEkDe6wgqGqLhhwP7uA03vmTwN2A98Djk+yojlrmCiXJM2TuRpK+iqwvrkDaSVwGbClqgq4Bbi0qbcROJwzEEnSLBnG7aqvT7ILOBe4PskNTfmLknwOoDkbeCtwA7AN+ERV3dts4u3A25LspHvN4YODtkmSdPTSfdO+uHQ6nRobG5vvZkjSopJka1VN+XmzCX7yWZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVLLQMGQ5I1J7k0ynqTv74gmOT3JLUm2NXV/v2fZnyb55yR3No/XDtIeSdLgVgy4/j3AG4APTFPnAPCfq+qfkjwX2Jrkxqq6r1l+dVX9zwHbIUkakoGCoaq2ASSZrs4eYE8z/S9JtgGnAvdNuZIkad7M6TWGJOuAs4Hbe4rfmuSuJB9Ksnou2yNJOtSMwZDkpiT39HlcfCQ7SnIc8GngD6rqR03xXwH/CthA96ziz6dZf1OSsSRj+/btO5JdS5KOwIxDSVV1waA7SXIM3VD4SFX9fc+2H+6p89fAZ6dpx2ZgM0Cn06lB2yRJ6m/Wh5LSvQDxQWBbVf3FpGVremZfT/ditiRpHg16u+rrk+wCzgWuT3JDU/6iJJ9rqp0H/Dpwfp/bUt+T5O4kdwGvBP5wkPZIkgaXqsU3KtPpdGpsbGy+myFJi0qSrVXV9zNnvfzksySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqSWgYIhyRuT3JtkPMmUvyOa5IEkdye5M8lYT/kJSW5MsqN5Xj1IeyRJgxv0jOEe4A3Alw+j7iurasOkH6K+HLi5qtYDNzfzkqR5NFAwVNW2qto+wCYuBq5tpq8FLhmkPZKkwc3VNYYCvpBka5JNPeUnV9UegOb5pDlqjyRpCitmqpDkJuCUPouurKrrDnM/51XV7iQnATcm+UZVHc7wU287NgGbANauXXskq0qSjsCMwVBVFwy6k6ra3TzvTfIZ4By61yUeTrKmqvYkWQPsnWYbm4HNAJ1OpwZtkySpv1kfSkrynCTPnZgGXk33ojXAFmBjM70RONwzEEnSLBn0dtXXJ9kFnAtcn+SGpvxFST7XVDsZ+MckXwfuAK6vqn9oll0FXJhkB3BhMy9JmkepWnyjMp1Op8bGxmauKEl6WpKtkz4y0JeffJYktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWhbll+gl2Qc8eJSrnwh8b4jNmU/2ZeFZKv0A+7JQDdKXn6iqF85UaVEGwyCSjB3OtwsuBvZl4Vkq/QD7slDNRV8cSpIktRgMkqSW5RgMm+e7AUNkXxaepdIPsC8L1az3ZdldY5AkTW85njFIkqaxrIIhyUVJtifZmeTy+W7PZEk+lGRvknt6yk5IcmOSHc3z6qY8Sf5305e7krysZ52NTf0dSTbOU19OT3JLkm1J7k3y+4u1P0mOTXJHkq83fflvTfkZSW5v2vXxJCub8lXN/M5m+bqebV3RlG9P8otz3ZemDaNJvpbks4u8Hw8kuTvJnUnGmrJFd3w1bTg+yaeSfKP5P3PuvPalqpbFAxgFvgWcCawEvg6cNd/tmtTGVwAvA+7pKXsPcHkzfTnw7mb6tcDngQAvB25vyk8A7m+eVzfTq+ehL2uAlzXTzwW+CZy1GPvTtOm4ZvoY4PamjZ8ALmvK3w/8TjP9u8D7m+nLgI8302c1x90q4IzmeBydh7/N24C/Az7bzC/WfjwAnDipbNEdX007rgXe0kyvBI6fz77Maefn8wGcC9zQM38FcMV8t6tPO9fRDobtwJpmeg2wvZn+APCmyfWANwEf6Clv1ZvHfl0HXLjY+wM8G/gn4GfpfshoxeTjC7gBOLeZXtHUy+RjrrfeHLb/NOBm4Hzgs027Fl0/mv0+wKHBsOiOL+B5wLdprvkuhL4sp6GkU4GHeuZ3NWUL3clVtQegeT6pKZ+qPwuun80QxNl032kvyv40wy93AnuBG+m+S360qg70adfTbW6W/xB4AQujL+8F/gQYb+ZfwOLsB0ABX0iyNcmmpmwxHl9nAvuADzdDfNckeQ7z2JflFAzpU7aYb8maqj8Lqp9JjgM+DfxBVf1ouqp9yhZMf6rqYFVtoPuO+xzgJf2qNc8Lsi9JfgnYW1Vbe4v7VF3Q/ehxXlW9DHgN8HtJXjFN3YXclxV0h5D/qqrOBn5Md+hoKrPel+UUDLuA03vmTwN2z1NbjsTDSdYANM97m/Kp+rNg+pnkGLqh8JGq+vumeNH2B6CqHgVupTu2e3ySFX3a9XSbm+XPBx5h/vtyHvC6JA8AH6M7nPReFl8/AKiq3c3zXuAzdAN7MR5fu4BdVXV7M/8pukExb31ZTsHwVWB9cwfGSroX07bMc5sOxxZg4u6CjXTH6ifKf6O5Q+HlwA+b080bgFcnWd3cxfDqpmxOJQnwQWBbVf1Fz6JF158kL0xyfDP9LOACYBtwC3BpU21yXyb6eCnwxeoO+m4BLmvu9jkDWA/cMTe9gKq6oqpOq6p1dI//L1bVr7LI+gGQ5DlJnjsxTfe4uIdFeHxV1XeBh5K8uCl6FXAf89mXub5gNJ8Pulfzv0l3fPjK+W5Pn/Z9FNgD7Keb/m+mO6Z7M7CjeT6hqRvgfU1f7gY6Pdv5bWBn8/iteerLv6N7GnsXcGfzeO1i7A/wUuBrTV/uAd7RlJ9J9wVxJ/BJYFVTfmwzv7NZfmbPtq5s+rgdeM08Hmu/wDN3JS26fjRt/nrzuHfi//NiPL6aNmwAxppj7P/Rvato3vriJ58lSS3LaShJknQYDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktTy/wETxwkR2hbPQAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train = False\n",
    "track_reward = []\n",
    "max_episodes = 1\n",
    "render = True\n",
    "frames = 1  # DO NOT CHANGE\n",
    "batch_size = 32\n",
    "\n",
    "for episode in range(max_episodes):\n",
    "    t1 = time()\n",
    "    print(\"Episode : \" + str(episode) + \" Replay Buffer \" + str(buff.count()))\n",
    "        \n",
    "    state = env.reset()\n",
    "    state = observation_wrapper(state, frames)\n",
    "    done = False\n",
    "    episode_reward = []\n",
    "        \n",
    "    while not done:\n",
    "        if render:\n",
    "            env.render()\n",
    "        \n",
    "        action_vec = actor.predict([state['pov'],state['compassAngle']])\n",
    "        #print(action_vec)    \n",
    "        if train:\n",
    "            action_vec = add_noise(action_vec[0], epsilon)\n",
    "            #print(action_vec)\n",
    "            epsilon = epsilon*((1+rate)**episode)  \n",
    "        action = map_to_actionset(mission, action_vec[0])\n",
    "        #print(action)    \n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        next_state = observation_wrapper(next_state, frames)\n",
    "        #done_ = 0 if done else 1\n",
    "        episode_reward.append(reward)\n",
    "        \n",
    "        if train:\n",
    "            buff.add(state, action_vec, reward, next_state, done)\n",
    "            if buff.count() > batch_size:\n",
    "                train_from_memory(mission, buff, actor, critic, batch_size)\n",
    "\n",
    "        state = next_state\n",
    "            \n",
    "            \n",
    "        if done:\n",
    "            track_reward.append(np.sum(episode_reward))\n",
    "            print(\"episode: {}/{}, mean reward: {}, e: {:.2}\"\n",
    "                .format(episode, max_episodes, np.mean(episode_reward), epsilon))\n",
    "            plt.plot(range(len(episode_reward)), episode_reward)    \n",
    "                \n",
    "            break\n",
    "    print('Time taken:', (time()-t1)/60, 'mins')\n",
    "    if train:    \n",
    "        if episode % 5 == 0:\n",
    "            actor.model.save_weights('train_navigatedense_actor.h5', overwrite=True)\n",
    "            actor.target_model.save_weights('train_navigatedense_actortarget.h5', overwrite=True)\n",
    "            critic.model.save_weights('train_navigatedense_critic.h5', overwrite=True)\n",
    "            critic.target_model.save_weights('train_navigatedense_critictarget.h5', overwrite=True)\n",
    "\n",
    "print('Training completed!!!')\n",
    "plt.plot(range(max_episodes), track_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to delete the temporary minecraft directory.\n"
     ]
    }
   ],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}