report.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### report of implementations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "from classes2048 import state, tbl\n",
    "from time import sleep\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from functions import mv\n",
    "from pynput.keyboard import Key, Controller\n",
    "import math\n",
    "from IPython.display import clear_output\n",
    "import os\n",
    "keys= [Key.right,Key.down,Key.up,Key.left]\n",
    "from tqdm import tqdm, trange\n",
    "# try:\n",
    "#     from tqdm import tqdm\n",
    "# except ImportError:\n",
    "#     def tqdm(x,*args, **kwargs):\n",
    "#         return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "game=state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'top': {'l0': {'starts': 534, 'ends': 631},\n",
       "  'l1': {'starts': 643, 'ends': 740},\n",
       "  'l2': {'starts': 753, 'ends': 850},\n",
       "  'l3': {'starts': 862, 'ends': 959}},\n",
       " 'left': {'l0': {'starts': 53, 'ends': 150},\n",
       "  'l1': {'starts': 162, 'ends': 259},\n",
       "  'l2': {'starts': 272, 'ends': 369},\n",
       "  'l3': {'starts': 381, 'ends': 478}}}"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.getprops()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table=tbl(game)\n",
    "keyboard = Controller()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0     1     2     3\n",
       "0  2.0   2.0   4.0   2.0\n",
       "1  8.0  16.0  16.0   4.0\n",
       "2  4.0   8.0   8.0  16.0\n",
       "3  NaN   NaN   2.0   8.0"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table.gettbl()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'going'"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.statepredict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_input_neurons = len(env.reset())\n",
    "# num_ouptut_neurons = env.action_space.n\n",
    "\n",
    "num_input_neurons = 16\n",
    "num_output_neurons = 4\n",
    "#num_hidden_layer_neurons_1 = 256\n",
    "\n",
    "gamma = 0.99\n",
    "max_epsilon = 0.9\n",
    "epsilon = max_epsilon\n",
    "min_epsilon = 0.01\n",
    "epsilon_decay_factor = 0.001\n",
    "\n",
    "num_episodes = 100\n",
    "\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training the Q-network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Dense(128, activation='relu', input_dim = num_input_neurons),\n",
    "    tf.keras.layers.Dense(128,activation='relu'),\n",
    "    tf.keras.layers.Dense(num_output_neurons, activation='linear')\n",
    "    \n",
    "])\n",
    "#opt=tf.keras.optimizers.Adam(lr=learning_rate)\n",
    "\n",
    "model.compile(optimizer=opt,\n",
    "              loss='mse',       # mean squared error\n",
    "              metrics=['mae'])\n",
    "\n",
    "\n",
    "# model.compile(loss='mse',\n",
    "#               optimizer='Adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(checkpoint_path)\n",
    "loss,acc = model.evaluate(test_images, test_labels)\n",
    "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "reward_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "1it [01:29, 71.13s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  2.0  NaN NaN  NaN\n",
      "  4.0  NaN NaN  2.0\n",
      " 16.0  2.0 NaN  NaN\n",
      "  8.0  4.0 NaN  NaN \n",
      " reward= -1.0 \n",
      " Key.right\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-197-89e194d6b78e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtablestate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mnext_tablestate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatepredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'lost'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m             \u001b[0mreward\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/2048/classes2048.py\u001b[0m in \u001b[0;36mstatepredict\u001b[0;34m(self, save, folder)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstatepredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfolder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'./'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mmss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msct\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m             \u001b[0;31m# l1,l2 =107, 15\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m             \u001b[0;31m# top, left =335, 581\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.6/site-packages/mss/factory.py\u001b[0m in \u001b[0;36mmss\u001b[0;34m(**kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m         )\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mMSS\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/venv/lib/python3.6/site-packages/mss/linux.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, display)\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXSetErrorHandler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 207\u001b[0;31m         \u001b[0mxrandr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_library\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Xrandr\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    208\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mxrandr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mScreenShotError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No Xrandr extension found.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/ctypes/util.py\u001b[0m in \u001b[0;36mfind_library\u001b[0;34m(name)\u001b[0m\n\u001b[1;32m    311\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mfind_library\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    312\u001b[0m             \u001b[0;31m# See issue #9998\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 313\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0m_findSoname_ldconfig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    314\u001b[0m                    \u001b[0m_get_soname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_findLib_gcc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_findLib_ld\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/ctypes/util.py\u001b[0m in \u001b[0;36m_findSoname_ldconfig\u001b[0;34m(name)\u001b[0m\n\u001b[1;32m    281\u001b[0m                                       \u001b[0mstdout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m                                       env={'LC_ALL': 'C', 'LANG': 'C'}) as p:\n\u001b[0;32m--> 283\u001b[0;31m                     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mregex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    284\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    285\u001b[0m                         \u001b[0;32mreturn\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfsdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/re.py\u001b[0m in \u001b[0;36msearch\u001b[0;34m(pattern, string, flags)\u001b[0m\n\u001b[1;32m    180\u001b[0m     \"\"\"Scan through string looking for a match to the pattern, returning\n\u001b[1;32m    181\u001b[0m     a match object, or None if no match was found.\"\"\"\n\u001b[0;32m--> 182\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_compile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_episodes=2\n",
    "progress = tqdm(num_episodes )\n",
    "sleep(2)\n",
    "for i in range(num_episodes) :\n",
    "    table.gettbl()\n",
    "    tablestate = table.tablestate()\n",
    "    reward_so_far = 0\n",
    "    while game.statepredict()=='going':\n",
    "        clear_output(wait=True)\n",
    "        progress.refresh()\n",
    "        action, q_vals = np.array([np.argmax(model.predict(tablestate))]), model.predict(tablestate)\n",
    "        \n",
    "        if np.random.rand(1) < epsilon:\n",
    "            action[0] = np.random.randint(4)\n",
    "            \n",
    "        mv(keys[action[0]] )\n",
    "        \n",
    "        print(table.tbl.to_string(index=False, header=False),'\\n reward=',reward, '\\n', str(keys[action[0]]))\n",
    "\n",
    "        \n",
    "        sleep(0.35)\n",
    "        table.gettbl()\n",
    "        next_tablestate=table.tablestate()\n",
    "        \n",
    "        if np.array_equal(next_tablestate,tablestate):\n",
    "            reward=-50\n",
    "            sleep(2)\n",
    "        else:\n",
    "            reward = tablestate.sum()-next_tablestate.sum()\n",
    "        \n",
    "        if game.statepredict()=='lost':\n",
    "            reward=-100\n",
    "\n",
    "        max_q=np.max(model.predict(tablestate))\n",
    "        target_q_vals = q_vals\n",
    "        target_q_vals[0, action[0]] = reward + gamma * max_q\n",
    "        \n",
    "        model.fit(tablestate, target_q_vals, epochs=1)\n",
    "\n",
    "        #clear_output(wait=True)\n",
    "        \n",
    "        \n",
    "\n",
    "        tablestate = next_tablestate\n",
    "        reward_so_far += reward\n",
    "        sleep(0.1)\n",
    "        \n",
    "\n",
    "    # epsilon decay\n",
    "    progress.update()\n",
    "    progress.refresh()\n",
    "    reward_list.append([reward_so_far, table.tbl.sum().sum()])\n",
    "    game.statepredict(save=True, folder='./data/lost_games/')\n",
    "    keyboard.press(Key.f5)\n",
    "    sleep(0.02)\n",
    "    keyboard.release(Key.f5)\n",
    "    sleep(3)\n",
    "    epsilon = min_epsilon + (max_epsilon - min_epsilon) * math.exp(-epsilon_decay_factor * i)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[-943.0, 312.0],\n",
       " [-1091.0, 202.0],\n",
       " [-1195.0, 268.0],\n",
       " [-2449.0, 346.0],\n",
       " [-1639.0, 324.0],\n",
       " [-688.0, 214.0],\n",
       " [0, 214.0],\n",
       " [0, 214.0],\n",
       " [0, 214.0],\n",
       " [-1050.0, 350.0],\n",
       " [-1846.0, 298.0],\n",
       " [-741.0, 184.0],\n",
       " [-996.0, 308.0],\n",
       " [-2445.0, 430.0],\n",
       " [-596.0, 228.0],\n",
       " [-1701.0, 408.0],\n",
       " [-795.0, 220.0],\n",
       " [-1742.0, 294.0],\n",
       " [-1749.0, 342.0],\n",
       " [-1342.0, 298.0],\n",
       " [-1446.0, 362.0],\n",
       " [-787.0, 144.0],\n",
       " [-639.0, 178.0],\n",
       " [-927.0, 214.0],\n",
       " [-1493.0, 216.0],\n",
       " [-1299.0, 336.0],\n",
       " [-1697.0, 322.0],\n",
       " [-944.0, 198.0],\n",
       " [-1195.0, 292.0],\n",
       " [-1291.0, 268.0],\n",
       " [-1650.0, 380.0],\n",
       " [-1490.0, 366.0],\n",
       " [-1344.0, 300.0],\n",
       " [-2547.0, 268.0],\n",
       " [-996.0, 266.0],\n",
       " [-1041.0, 214.0],\n",
       " [-1895.0, 338.0],\n",
       " [-843.0, 284.0],\n",
       " [-1795.0, 432.0],\n",
       " [-1190.0, 222.0],\n",
       " [-1086.0, 222.0],\n",
       " [-1494.0, 236.0],\n",
       " [-1343.0, 292.0],\n",
       " [-443.0, 192.0],\n",
       " [-990.0, 240.0],\n",
       " [-1092.0, 234.0],\n",
       " [-840.0, 236.0],\n",
       " [-2201.0, 468.0],\n",
       " [-1291.0, 188.0],\n",
       " [-1398.0, 330.0],\n",
       " [-1442.0, 186.0],\n",
       " [-1395.0, 294.0],\n",
       " [-945.0, 286.0],\n",
       " [-1343.0, 282.0],\n",
       " [-691.0, 206.0],\n",
       " [-1741.0, 320.0],\n",
       " [0, 320.0],\n",
       " [-1448.0, 326.0],\n",
       " [-835.0, 164.0],\n",
       " [-585.0, 156.0],\n",
       " [-386.0, 134.0],\n",
       " [-743.0, 190.0],\n",
       " [-689.0, 178.0],\n",
       " [-1698.0, 430.0],\n",
       " [-797.0, 324.0],\n",
       " [-1087.0, 220.0],\n",
       " [-1043.0, 296.0],\n",
       " [-839.0, 172.0],\n",
       " [-896.0, 312.0],\n",
       " [-1093.0, 246.0],\n",
       " [-843.0, 196.0],\n",
       " [-541.0, 258.0],\n",
       " [-842.0, 252.0],\n",
       " [-343.0, 240.0],\n",
       " [-1348.0, 302.0],\n",
       " [-1400.0, 358.0],\n",
       " [-1088.0, 178.0],\n",
       " [-1594.0, 284.0],\n",
       " [-737.0, 142.0],\n",
       " [-940.0, 186.0],\n",
       " [-1945.0, 426.0],\n",
       " [-242.0, 188.0],\n",
       " [-445.0, 216.0],\n",
       " [-540.0, 240.0],\n",
       " [-838.0, 186.0],\n",
       " [-1496.0, 386.0],\n",
       " [-1041.0, 294.0],\n",
       " [-1347.0, 338.0],\n",
       " [-892.0, 204.0],\n",
       " [-1244.0, 288.0],\n",
       " [-890.0, 186.0],\n",
       " [-639.0, 178.0],\n",
       " [-1195.0, 332.0],\n",
       " [-591.0, 186.0],\n",
       " [-894.0, 216.0],\n",
       " [-993.0, 282.0],\n",
       " [-691.0, 188.0],\n",
       " [-1592.0, 386.0],\n",
       " [-2099.0, 456.0],\n",
       " [-1246.0, 306.0],\n",
       " [-1844.0, 290.0],\n",
       " [-996.0, 284.0],\n",
       " [-1548.0, 336.0],\n",
       " [-590.0, 180.0],\n",
       " [-590.0, 156.0],\n",
       " [-1294.0, 210.0],\n",
       " [-792.0, 250.0],\n",
       " [-697.0, 250.0],\n",
       " [-438.0, 168.0],\n",
       " [-895.0, 244.0],\n",
       " [-1093.0, 250.0],\n",
       " [-492.0, 218.0],\n",
       " [-691.0, 188.0],\n",
       " [-1092.0, 244.0],\n",
       " [-1187.0, 158.0],\n",
       " [-793.0, 196.0],\n",
       " [-990.0, 176.0]]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reward_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "180.0"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path = \"checkpoints/\"\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "\n",
    "# Create checkpoint callback\n",
    "# cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n",
    "#                                                  save_weights_only=True,\n",
    "#                                                  verbose=1)\n",
    "\n",
    "model.save_weights('./checkpoints/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### in case we have wrong gamestate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'going'"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.statepredict(save=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ainur/venv/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "game.fit()\n",
    "game.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## testing the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-152-72f09b1c6322>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32mwhile\u001b[0m \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatepredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'going'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgettbl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mtablestate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtablestate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;31m#         action = sess.run(best_action, feed_dict={input_obs: state})\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/2048/classes2048.py\u001b[0m in \u001b[0;36mgettbl\u001b[0;34m(self, ts, save, folder)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m                     \u001b[0msct_img\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrab\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    215\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0msave\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m                         \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrombytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"RGB\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msct_img\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msct_img\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbgra\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"raw\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"BGRX\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.6/site-packages/mss/linux.py\u001b[0m in \u001b[0;36mgrab\u001b[0;34m(self, monitor)\u001b[0m\n\u001b[1;32m    428\u001b[0m             \u001b[0mmonitor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"height\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    429\u001b[0m             \u001b[0mPLAINMASK\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 430\u001b[0;31m             \u001b[0mZPIXMAP\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    431\u001b[0m         )\n\u001b[1;32m    432\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/venv/lib/python3.6/site-packages/mss/linux.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(retval, func, args)\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m     \u001b[0;31m# type: (int, Any, Tuple[Any, Any]) -> Any\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m     \u001b[0;34m\"\"\" Validate the returned value of a Xlib or XRANDR function. \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_episodes = 2\n",
    "total_reward = []\n",
    "\n",
    "# model = create_model()\n",
    "# model.load_weights(latest)\n",
    "# loss, acc = model.evaluate(test_images, test_labels)\n",
    " \n",
    "sleep(2)\n",
    "table.gettbl()\n",
    "\n",
    "for _ in tqdm(range(num_episodes)):\n",
    "    \n",
    "    while game.statepredict()=='going':\n",
    "        table.gettbl()\n",
    "        tablestate = table.tablestate()\n",
    "#         action = sess.run(best_action, feed_dict={input_obs: state})\n",
    "        action= np.array([np.argmax(model.predict(tablestate))])\n",
    "        mv(keys[action[0]])\n",
    "        sleep(0.35)\n",
    "        \n",
    "        if np.array_equal(next_tablestate,tablestate):\n",
    "            reward=-100\n",
    "        else:\n",
    "            reward = tablestate.sum()-next_tablestate.sum()\n",
    "        \n",
    "        if game.statepredict()=='lost':\n",
    "            reward=-1000\n",
    "            \n",
    "       # state, reward, done, _ = env.step(action[0])\n",
    "        total_reward.append(reward)\n",
    "    keyboard.press(Key.f5)\n",
    "    sleep(0.02)\n",
    "    keyboard.release(Key.f5)\n",
    "    sleep(3)\n",
    "\n",
    "\n",
    "print(\"Average reward per episode: {}\".format(total_reward/float(num_episodes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}