train_mc_simple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ZLX5bfvuvJBg",
    "outputId": "549b447d-bf92-4e93-bfda-6edbbe2c6ff8"
   },
   "outputs": [],
   "source": [
    "# !unzip robohearts.zip\n",
    "#%cd robohearts/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import multiprocessing\n",
    "from gymhearts.Hearts import *\n",
    "from gymhearts.Agent.agent_random import RandomAgent\n",
    "from gymhearts.Agent.agent_mc_simple import MonteCarlo\n",
    "from gymhearts.Agent.utils_env import *\n",
    "from gymhearts.Agent.utils_nn import *\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ----------- TRAINING CONFIGURATION -------------\n",
    "# Number of games to train on \n",
    "TRAINING_ITERS = 10000\n",
    "\n",
    "# Number of episodes to run during model evaluation\n",
    "NUM_EPISODES = 1000\n",
    "\n",
    "# Number of model evaluations to average together\n",
    "NUM_TESTS = 10\n",
    "\n",
    "# Max score for players to win the game\n",
    "MAX_SCORE = 100\n",
    "\n",
    "# Set to false to skip training\n",
    "run_train = False\n",
    "\n",
    "# Set to true to resume training from an existing model\n",
    "continue_train = False\n",
    "\n",
    "# Run testing on a random agent for comparison\n",
    "run_random = False\n",
    "\n",
    "# Name of the file that is saved :: {model_name}.th\n",
    "model_name = 'mc_simple'\n",
    "\n",
    "# Configuration parameters for the model\n",
    "mc_simple_config = {\n",
    "    'print_info' : False,\n",
    "    'epsilon' : 0.01,\n",
    "    'gamma' : 1,\n",
    "    'alpha': 0.01\n",
    "}\n",
    "\n",
    "if continue_train:\n",
    "    mc_simple_config['load_model'] = model_name\n",
    "\n",
    "playersNameList = ['MonteCarlo', 'Rando', 'Randy', 'Randall']\n",
    "agent_list = [0, 0, 0, 0]\n",
    "\n",
    "agent_list[0] = MonteCarlo(playersNameList[0], mc_simple_config)\n",
    "agent_list[1] = RandomAgent(playersNameList[1], {'print_info' : False})\n",
    "agent_list[2] = RandomAgent(playersNameList[2], {'print_info' : False})\n",
    "agent_list[3] = RandomAgent(playersNameList[3], {'print_info' : False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e211c9ebb1564687a3c4ee66adb9ab37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# ------- TRAIN MONTE CARLO SIMPLE AGENT ---------\n",
    "env = gym.make('Hearts_Card_Game-v0')\n",
    "env.__init__(playersNameList, MAX_SCORE)\n",
    "weights = []\n",
    "if run_train:\n",
    "    for trn_episode in tqdm_notebook(range(TRAINING_ITERS)):\n",
    "        # Save the model every 50 steps\n",
    "        if trn_episode % 50 == 0:\n",
    "            save_model(weights, model_name, 'mc_simple')\n",
    "        observation = env.reset()\n",
    "        history = []\n",
    "        while True:\n",
    "            #env.render()\n",
    "            now_event = observation['event_name']\n",
    "            IsBroadcast = observation['broadcast']\n",
    "            action = None\n",
    "            if IsBroadcast == True:\n",
    "                for agent in agent_list:\n",
    "                    agent.Do_Action(observation)\n",
    "            else:\n",
    "                playName = observation['data']['playerName']\n",
    "                for agent in agent_list:\n",
    "                    if agent.name == playName:\n",
    "                        action = agent.Do_Action(observation)\n",
    "\n",
    "            # update my agent\n",
    "            if now_event == 'RoundEnd':\n",
    "                errors = agent_list[0].update_weights(history, -reward['MonteCarlo'])\n",
    "                history = []\n",
    "                weights = agent_list[0].weight_vec\n",
    "            if now_event == 'GameOver':\n",
    "                  break\n",
    "            if not IsBroadcast and observation['data']['playerName'] == 'MonteCarlo':\n",
    "                history.append(observation)\n",
    "            observation, reward, done, info = env.step(action)\n",
    "    save_model(weights, model_name, 'mc_simple')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to test mc simple model with multiprocessing\n",
    "def run_test(num_won):\n",
    "    # Weird hack to make progress bars render properly\n",
    "    print(' ', end='', flush=True)\n",
    "    for i_ep in tqdm_notebook(range(NUM_EPISODES)):\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            now_event = observation['event_name']\n",
    "            IsBroadcast = observation['broadcast']\n",
    "            action = None\n",
    "            if IsBroadcast == True:\n",
    "                for agent in agent_list:\n",
    "                    agent.Do_Action(observation)\n",
    "            else:\n",
    "                playName = observation['data']['playerName']\n",
    "                for agent in agent_list:\n",
    "                    if agent.name == playName:\n",
    "                        action = agent.Do_Action(observation)\n",
    "            if now_event == 'GameOver':\n",
    "                num_won += int(observation['data']['Winner'] == 'MonteCarlo')\n",
    "                break\n",
    "            observation, reward, done, info = env.step(action)\n",
    "    return num_won"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 185
    },
    "colab_type": "code",
    "id": "Qn-MK5D5pBUw",
    "outputId": "2d5fa5ed-87d6-4c10-aaf7-cb7e6e5883c1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "009f902902fe48c39d61a2e7ef32342c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aba4d8ede444c31a8f9c5eb166f4821",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0d7144baf8a4493b1dffc8ab706d9bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83a2c78139e449d7a95b9b2a73036ab4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24d917346c9c4b5ea09d9a000e773636",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5355feb069dd4b2f88f4f3214c8cc314",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "572e1aeca09a46dbb4503386dec025cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce05fd04f09d479084a0f8aaea10b470",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "273ca1304edb4127a6b26e2f8da0126a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9cb3e74e6a14946b4ff8d0d059c2c30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "[298, 351, 357, 312, 331, 347, 337, 314, 348, 344]\n"
     ]
    }
   ],
   "source": [
    "# ---------- EVALUATE MC SIMPLE AGENT --------------\n",
    "    \n",
    "env = gym.make('Hearts_Card_Game-v0')\n",
    "env.__init__(playersNameList, MAX_SCORE)\n",
    "\n",
    "# Evaluation parameters for testing\n",
    "mc_simple_config = {\n",
    "    'print_info' : False,\n",
    "    'load_model' : model_name\n",
    "}\n",
    "\n",
    "agent_list[0] = MonteCarlo(playersNameList[0], mc_simple_config)\n",
    "mc_wins = [0] * NUM_TESTS\n",
    "   \n",
    "pool = multiprocessing.Pool(processes=NUM_TESTS)\n",
    "mc_wins = pool.map(run_test, mc_wins)\n",
    "print(mc_wins)\n",
    "pool.close()\n",
    "pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to test random model with multiprocessing\n",
    "def run_test_random(num_won):\n",
    "    # Weird hack to make progress bars render properly\n",
    "    print(' ', end='', flush=True)\n",
    "    for i_ep in tqdm_notebook(range(NUM_EPISODES)):\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            now_event = observation['event_name']\n",
    "            IsBroadcast = observation['broadcast']\n",
    "            action = None\n",
    "            if IsBroadcast == True:\n",
    "                for agent in agent_list:\n",
    "                    agent.Do_Action(observation)\n",
    "            else:\n",
    "                playName = observation['data']['playerName']\n",
    "                for agent in agent_list:\n",
    "                    if agent.name == playName:\n",
    "                        action = agent.Do_Action(observation)\n",
    "            if now_event == 'GameOver':\n",
    "                num_won += int(observation['data']['Winner'] == 'Randman')\n",
    "                break\n",
    "            observation, reward, done, info = env.step(action)\n",
    "    return num_won"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 185
    },
    "colab_type": "code",
    "id": "-b3qOJsbh9pf",
    "outputId": "1d699944-7e94-4730-8a4e-69c99f3e30a7"
   },
   "outputs": [],
   "source": [
    "# ----------- EVALUTATE RANDOM AGENT ---------------\n",
    "if run_random:\n",
    "    env = gym.make('Hearts_Card_Game-v0')\n",
    "    env.__init__(playersNameList, MAX_SCORE)\n",
    "    playersNameList[0] = 'Randman'\n",
    "    agent_list[0] = RandomAgent(playersNameList[0])\n",
    "    rand_wins = [0] * NUM_TESTS\n",
    "    pool = multiprocessing.Pool(processes=NUM_TESTS)\n",
    "    rand_wins = pool.map(run_test_random, rand_wins)\n",
    "    print(rand_wins)\n",
    "    pool.close()\n",
    "    pool.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "colab_type": "code",
    "id": "3kXj1tPLnR7F",
    "outputId": "6ea21659-4bce-4baf-9e89-b995889ddad2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Monte Carlo won 333.9 times on average :: [298, 351, 357, 312, 331, 347, 337, 314, 348, 344]\n",
      "298\n",
      "351\n",
      "357\n",
      "312\n",
      "331\n",
      "347\n",
      "337\n",
      "314\n",
      "348\n",
      "344\n"
     ]
    }
   ],
   "source": [
    "print(f\"Monte Carlo won {sum(mc_wins)/len(mc_wins)} times on average :: {str(mc_wins)}\")\n",
    "for wins in mc_wins:\n",
    "    print(wins)\n",
    "if run_random:\n",
    "    print(f\"Random won {sum(rand_wins)/len(rand_wins)} times on average :: {str(rand_wins)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "mc_simple.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}