deep_q.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook will be me trying to make the DQN algorithm. \n",
    "Implementation of DQN for the gym environment CartPole-v0. Performance is quite variable, sure somemore hyperparameter tuning would help, but that's a problem for another day. \n",
    "\n",
    "![here](images/dqn-cartpole.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf \n",
    "import numpy as np\n",
    "import gym\n",
    "import matplotlib.pyplot as plot\n",
    "\n",
    "tf.random.set_random_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replay buffer \n",
    "So this is a buffer of length n, which we can get samples from\n",
    "Added in form \n",
    "[s,a,r,s']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MemoryBuffer(object):\n",
    "    \n",
    "    def __init__(self, length):\n",
    "        self.length = length +1\n",
    "        self.first_run = True \n",
    "    \n",
    "    def add_to_buffer(self, e):\n",
    "        \"\"\"\n",
    "        The experience (e) added is in form: \n",
    "        [state, action, reward, next_state, int( not done) ]\n",
    "        The reason for int(not done) is so that if it's a terminal state\n",
    "        it will be stored as 0 and then will easily allow for calculation\n",
    "        of the state value without the need for any if statements. \n",
    "        \"\"\"\n",
    "        if not self.first_run:\n",
    "            for num, i in enumerate(e):\n",
    "                self.experience[num] = np.vstack([i,self.experience[num]])                \n",
    "        else:\n",
    "            for i in e:\n",
    "                self.experience = [np.array(e[0]),np.array(e[1]),\n",
    "                                   np.array(e[2]),np.array(e[3]),\n",
    "                                  np.array(e[4])]\n",
    "\n",
    "            self.first_run = False\n",
    "        self.maintain_length()\n",
    "        \n",
    "    def get_batch(self, number):\n",
    "        number = min(self.experience[0].shape[0],number)\n",
    "        items_to_get = np.random.randint(0,self.experience[0].shape[0], number)\n",
    "        return [self.experience[c][items_to_get,:] for c in range(5)]\n",
    "    \n",
    "    def maintain_length(self):\n",
    "        if self.experience[0].shape[0] >= self.length:\n",
    "            for i in range(len(self.experience)):\n",
    "                self.experience[i] = self.experience[i][0:-1]\n",
    "                \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DQN Agent \n",
    "This is the DQN agent and its architecture.\n",
    "We need both a target network too, don't forget. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class DeepQ(object):\n",
    "    \n",
    "    def __init__(self, state_size, action_size, lr , y ):\n",
    "        \"\"\"\n",
    "        Initialise model which is defined using keras. \n",
    "        We use the RMSPropOptimiser as that is what is specified in the \n",
    "        paper. \n",
    "        The target model is initialised as a clone of the model. Note, \n",
    "        this operation does not clone the weights. \n",
    "        \"\"\"\n",
    "        self.lr = lr\n",
    "        self.model = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(24, activation = tf.keras.activations.relu, \n",
    "                                  input_shape = state_size,\n",
    "                                 kernel_initializer=tf.keras.initializers.RandomUniform(0.0, 0.1)),\n",
    "            tf.keras.layers.Dense(24, activation = tf.keras.activations.relu,\n",
    "                                 kernel_initializer=tf.keras.initializers.RandomUniform(0.0, 0.1)),\n",
    "            tf.keras.layers.Dense(action_size, activation = tf.keras.activations.linear,\n",
    "                                 kernel_initializer=tf.keras.initializers.RandomUniform(0.0, 0.1))\n",
    "        ])\n",
    "        self.model.compile(optimizer = tf.train.RMSPropOptimizer(self.lr),\n",
    "                          loss = tf.losses.huber_loss,\n",
    "                          metrics = ['accuracy'],)\n",
    "        self.target_model = tf.keras.models.clone_model(self.model)\n",
    "        self.y = y #our discount factor\n",
    "        \n",
    "    def train(self, experience):\n",
    "        \"\"\"\n",
    "        state, action, reward, next_state, done. \n",
    "        Observed = r_t + y max(target)\n",
    "        The fit method takes in the current state and then computes the \n",
    "        MSE difference between the current network and the observed value. \n",
    "        \"\"\"\n",
    "        batch_size = experience[0].shape[0]\n",
    "        observed = experience[2].T + (self.y * np.multiply(experience[4].T,self.value_next_state(experience[3]))) #changed from model\n",
    "        update = self.model.predict(experience[0]) \n",
    "        update[list(range(batch_size)),experience[1].flatten()] = observed\n",
    "        self.model.fit(experience[0],update, verbose = 0 , batch_size = batch_size)\n",
    "        return update\n",
    "    \n",
    "    def replace_model(self):\n",
    "        \"\"\"\n",
    "        The target network aids stability. It provides it with a value to \n",
    "        move towards which isn't affected by its own training. \n",
    "        This might not be the most effecient way to do this, but it works. \n",
    "        \"\"\"\n",
    "        #self.model.save_weights('my_model')\n",
    "        #self.target_model.load_weights('my_model')\n",
    "        self.target_model.set_weights(self.model.get_weights())\n",
    "    \n",
    "    def value_next_state(self, next_state):\n",
    "        return np.max(deepQ.target_model.predict(next_state),axis = 1).T\n",
    "                                              \n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average data\n",
    "This makes it easier to see if the mean value of the data is improving. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def running_average(data):\n",
    "    new_data =[]\n",
    "    for i in range(len(data)):\n",
    "        new_data.append(np.average(data[max(0, i - 10):i+10]))\n",
    "    return new_data\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the environment \n",
    "For this example, a constant epsilon is used to specify the exploration. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.002 \n",
    "y = 0.8\n",
    "repl_int = 20\n",
    "deepQ = DeepQ((4,), 2 , lr, y)\n",
    "ExpR = MemoryBuffer(1000)\n",
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "episodes, total_iterations = 0,0 \n",
    "episode_reward = []\n",
    "epsilon = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "while episodes < 2000:\n",
    "    state = np.array([env.reset()])\n",
    "    #epsilon = np.max([0.01, 1 - ((episodes/1950))])\n",
    "    i, reward, done = 0, 0.0, False\n",
    "    while i < 200 and done != True:\n",
    "        if np.random.rand() > epsilon:   \n",
    "            action = np.argmax(deepQ.model.predict(state)[0])\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        next_state, r, done, _ = env.step(action)\n",
    "        reward += r # add reward to monitor episode reward\n",
    "        ExpR.add_to_buffer([state,action,r,next_state, float(not done)])\n",
    "        if total_iterations > 200:\n",
    "            deepQ.train(ExpR.get_batch(20))\n",
    "        state = np.array([next_state])\n",
    "        i += 1\n",
    "        total_iterations += 1     \n",
    "    if episodes % repl_int == 0 and episodes != 0:\n",
    "        deepQ.replace_model()\n",
    "    print('Episodes %i: %i, Epsilon: %f '%(episodes, reward,epsilon))\n",
    "    episode_reward.append(reward)\n",
    "    episodes += 1\n",
    "print(\"There were %i total iterations\"%total_iterations)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot.plot(episode_reward)\n",
    "plot.ylabel('Averaged reward')\n",
    "plot.xlabel('Episodes')\n",
    "plot.plot(running_average(episode_reward))\n",
    "plot.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}