segmentation_step_by_step.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import time\n",
    "import numpy as np\n",
    "import cv2\n",
    "from datetime import timedelta\n",
    "slim = tf.contrib.slim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = 'checkpoints/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def restore_session():\n",
    "    global_step = 0\n",
    "    if not os.path.exists(checkpoint_dir):\n",
    "        raise IOError(checkpoint_dir + ' does not exist.')\n",
    "    else:\n",
    "        path = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "        if path is None:\n",
    "            raise IOError('No checkpoint to restore in ' + self.checkpoint_dir)\n",
    "        else:\n",
    "            saver.restore(session, path.model_checkpoint_path)\n",
    "            global_step = int(path.model_checkpoint_path.split('-')[-1])\n",
    "\n",
    "    return global_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "    \n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv_layer(inputs, W_shape, b_shape, name, padding='SAME'):\n",
    "    W = weight_variable(W_shape)\n",
    "    b = bias_variable([b_shape])\n",
    "    layer = tf.nn.conv2d(inputs, W, strides=[1, 1, 1, 1], padding=padding) + b\n",
    "    layer = slim.batch_norm(layer)\n",
    "    layer = tf.nn.relu(layer)\n",
    "    return layer       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pool_layer(inputs, padding='SAME'):\n",
    "    layer, pool_argmax = tf.nn.max_pool_with_argmax(inputs, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=padding)\n",
    "    return layer, pool_argmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deconv_layer(inputs, W_shape, b_shape, name, padding='SAME'):\n",
    "    W = weight_variable(W_shape)\n",
    "    b = bias_variable([b_shape])\n",
    "\n",
    "    x_shape = tf.shape(inputs)\n",
    "    out_shape = tf.pack([x_shape[0], x_shape[1], x_shape[2], W_shape[2]])\n",
    "\n",
    "    layer = tf.nn.conv2d_transpose(inputs, W, out_shape, [1, 1, 1, 1], padding=padding) + b\n",
    "    layer = slim.batch_norm(layer)\n",
    "    layer = tf.nn.relu(layer)\n",
    "    \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unravel_argmax(argmax, shape):\n",
    "    output_list = []\n",
    "    output_list.append(argmax // (shape[2] * shape[3]))\n",
    "    output_list.append(argmax % (shape[2] * shape[3]) // shape[3])\n",
    "    return tf.pack(output_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unpool_layer2x2(inputs, raveled_argmax, out_shape):\n",
    "    argmax = unravel_argmax(raveled_argmax, tf.to_int64(out_shape))\n",
    "    output = tf.zeros([out_shape[1], out_shape[2], out_shape[3]])\n",
    "\n",
    "    height = tf.shape(output)[0]\n",
    "    width = tf.shape(output)[1]\n",
    "    channels = tf.shape(output)[2]\n",
    "\n",
    "    t1 = tf.to_int64(tf.range(channels))\n",
    "    t1 = tf.tile(t1, [((width + 1) // 2) * ((height + 1) // 2)])\n",
    "    t1 = tf.reshape(t1, [-1, channels])\n",
    "    t1 = tf.transpose(t1, perm=[1, 0])\n",
    "    t1 = tf.reshape(t1, [channels, (height + 1) // 2, (width + 1) // 2, 1])\n",
    "\n",
    "    t2 = tf.squeeze(argmax)\n",
    "    t2 = tf.pack((t2[0], t2[1]), axis=0)\n",
    "    t2 = tf.transpose(t2, perm=[3, 1, 2, 0])\n",
    "\n",
    "    t = tf.concat(3, [t2, t1])\n",
    "    indices = tf.reshape(t, [((height + 1) // 2) * ((width + 1) // 2) * channels, 3])\n",
    "\n",
    "    x1 = tf.squeeze(inputs)\n",
    "    x1 = tf.reshape(x1, [-1, channels])\n",
    "    x1 = tf.transpose(x1, perm=[1, 0])\n",
    "    values = tf.reshape(x1, [-1])\n",
    "\n",
    "    delta = tf.SparseTensor(indices, values, tf.to_int64(tf.shape(output)))\n",
    "    return tf.expand_dims(tf.sparse_tensor_to_dense(tf.sparse_reorder(delta)), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "session = tf.Session()\n",
    "x = tf.placeholder(tf.float32, shape=(1, None, None, 3))\n",
    "y_true = tf.placeholder(tf.int64, shape=(1, None, None))\n",
    "expected = tf.expand_dims(y_true, -1)\n",
    "rate = tf.placeholder(tf.float32, shape=[])\n",
    "\n",
    "conv_1_1 = conv_layer(inputs=x, W_shape=[3, 3, 3, 64], b_shape=64, name='conv_1_1')\n",
    "conv_1_2 = conv_layer(conv_1_1, [3, 3, 64, 64], 64, 'conv_1_2')\n",
    "\n",
    "pool_1, pool_1_argmax = pool_layer(conv_1_2)\n",
    "\n",
    "conv_2_1 = conv_layer(pool_1, [3, 3, 64, 128], 128, 'conv_2_1')\n",
    "conv_2_2 = conv_layer(conv_2_1, [3, 3, 128, 128], 128, 'conv_2_2')\n",
    "\n",
    "pool_2, pool_2_argmax = pool_layer(conv_2_2)\n",
    "\n",
    "conv_3_1 = conv_layer(pool_2, [3, 3, 128, 256], 256, 'conv_3_1')\n",
    "conv_3_2 = conv_layer(conv_3_1, [3, 3, 256, 256], 256, 'conv_3_2')\n",
    "conv_3_3 = conv_layer(conv_3_2, [3, 3, 256, 256], 256, 'conv_3_3')\n",
    "\n",
    "pool_3, pool_3_argmax = pool_layer(conv_3_3)\n",
    "\n",
    "conv_4_1 = conv_layer(pool_3, [3, 3, 256, 512], 512, 'conv_4_1')\n",
    "conv_4_2 = conv_layer(conv_4_1, [3, 3, 512, 512], 512, 'conv_4_2')\n",
    "conv_4_3 = conv_layer(conv_4_2, [3, 3, 512, 512], 512, 'conv_4_3')\n",
    "\n",
    "pool_4, pool_4_argmax = pool_layer(conv_4_3)\n",
    "\n",
    "conv_5_1 = conv_layer(pool_4, [3, 3, 512, 512], 512, 'conv_5_1')\n",
    "conv_5_2 = conv_layer(conv_5_1, [3, 3, 512, 512], 512, 'conv_5_2')\n",
    "conv_5_3 = conv_layer(conv_5_2, [3, 3, 512, 512], 512, 'conv_5_3')\n",
    "\n",
    "pool_5, pool_5_argmax = pool_layer(conv_5_3)\n",
    "\n",
    "fc_6 = conv_layer(pool_5, [7, 7, 512, 4096], 4096, 'fc_6')\n",
    "fc_7 = conv_layer(fc_6, [1, 1, 4096, 4096], 4096, 'fc_7')\n",
    "\n",
    "deconv_fc_6 = deconv_layer(fc_7, [7, 7, 512, 4096], 512, 'fc6_deconv', padding='VALID')\n",
    "\n",
    "unpool_5 = unpool_layer2x2(deconv_fc_6, pool_5_argmax, tf.shape(conv_5_3))\n",
    "\n",
    "deconv_5_3 = deconv_layer(unpool_5, [3, 3, 512, 512], 512, 'deconv_5_3')\n",
    "deconv_5_2 = deconv_layer(deconv_5_3, [3, 3, 512, 512], 512, 'deconv_5_2')\n",
    "deconv_5_1 = deconv_layer(deconv_5_2, [3, 3, 512, 512], 512, 'deconv_5_1')\n",
    "\n",
    "unpool_4 = unpool_layer2x2(deconv_5_1, pool_4_argmax, tf.shape(conv_4_3))\n",
    "\n",
    "deconv_4_3 = deconv_layer(unpool_4, [3, 3, 512, 512], 512, 'deconv_4_3')\n",
    "deconv_4_2 = deconv_layer(deconv_4_3, [3, 3, 512, 512], 512, 'deconv_4_2')\n",
    "deconv_4_1 = deconv_layer(deconv_4_2, [3, 3, 256, 512], 256, 'deconv_4_1')\n",
    "\n",
    "unpool_3 = unpool_layer2x2(deconv_4_1, pool_3_argmax, tf.shape(conv_3_3))\n",
    "\n",
    "deconv_3_3 = deconv_layer(unpool_3, [3, 3, 256, 256], 256, 'deconv_3_3')\n",
    "deconv_3_2 = deconv_layer(deconv_3_3, [3, 3, 256, 256], 256, 'deconv_3_2')\n",
    "deconv_3_1 = deconv_layer(deconv_3_2, [3, 3, 128, 256], 128, 'deconv_3_1')\n",
    "\n",
    "unpool_2 = unpool_layer2x2(deconv_3_1, pool_2_argmax, tf.shape(conv_2_2))\n",
    "\n",
    "deconv_2_2 = deconv_layer(unpool_2, [3, 3, 128, 128], 128, 'deconv_2_2')\n",
    "deconv_2_1 = deconv_layer(deconv_2_2, [3, 3, 64, 128], 64, 'deconv_2_1')\n",
    "\n",
    "unpool_1 = unpool_layer2x2(deconv_2_1, pool_1_argmax, tf.shape(conv_1_2))\n",
    "\n",
    "deconv_1_2 = deconv_layer(unpool_1, [3, 3, 64, 64], 64, 'deconv_1_2')\n",
    "deconv_1_1 = deconv_layer(deconv_1_2, [3, 3, 32, 64], 32, 'deconv_1_1')\n",
    "\n",
    "score_1 = deconv_layer(deconv_1_1, [1, 1, 21, 32], 21, 'score_1')\n",
    "\n",
    "logits = tf.reshape(score_1, (-1, 21))\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, tf.reshape(expected, [-1]), name='x_entropy')\n",
    "loss = tf.reduce_mean(cross_entropy, name='x_entropy_mean')\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(rate).minimize(loss)\n",
    "\n",
    "prediction = tf.argmax(tf.reshape(tf.nn.softmax(logits), tf.shape(score_1)), dimension=3)\n",
    "accuracy = tf.reduce_sum(tf.pow(prediction - expected, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver(max_to_keep = 5, keep_checkpoint_every_n_hours =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimize(train_stage=1, training_steps=1000, restore_session=False, learning_rate=1e-6):\n",
    "    if restore_session:\n",
    "        step_start = restore_session()\n",
    "    else:\n",
    "        step_start = 0\n",
    "\n",
    "    if train_stage == 1:\n",
    "        trainset = open('data/stage_1_train_imgset/train.txt').readlines()\n",
    "    else:\n",
    "        trainset = open('data/stage_2_train_imgset/train.txt').readlines()\n",
    "\n",
    "    for i in range(step_start, step_start+training_steps):\n",
    "        # pick random line from file\n",
    "        random_line = random.choice(trainset)\n",
    "        image_file = random_line.split(' ')[0]\n",
    "        ground_truth_file = random_line.split(' ')[1]\n",
    "        image = np.float32(cv2.imread('data' + image_file))\n",
    "        ground_truth = cv2.imread('data' + ground_truth_file[:-1], cv2.IMREAD_GRAYSCALE)\n",
    "        # norm to 21 classes [0-20] (see paper)\n",
    "        ground_truth = (ground_truth / 255) * 20\n",
    "        print('run train step: '+str(i))\n",
    "        start = time.time()\n",
    "        train_step.run(session=session, feed_dict={x: [image], y_true: [ground_truth], rate: learning_rate})\n",
    "\n",
    "        if i % 10000 == 0:\n",
    "            print('step {} finished in {:.2f} s with loss of {:.6f}'.format(\n",
    "                i, time.time() - start, loss.eval(session=session, feed_dict={x: [image], y_true: [ground_truth]})))\n",
    "            saver.save(session, checkpoint_dir+'model', global_step=i)\n",
    "            print('Model {} saved'.format(i))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run train step: 0\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:\n  <no registered kernels>\n\n\t [[Node: MaxPoolWithArgmax_15 = MaxPoolWithArgmax[T=DT_FLOAT, Targmax=DT_INT64, ksize=[1, 2, 2, 1], padding=\"SAME\", strides=[1, 2, 2, 1]](Relu_62)]]\n\nCaused by op 'MaxPoolWithArgmax_15', defined at:\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-20-ea7a22c7d446>\", line 10, in <module>\n    pool_1, pool_1_argmax = pool_layer(conv_1_2)\n  File \"<ipython-input-8-a5a87bb3bf60>\", line 2, in pool_layer\n    layer, pool_argmax = tf.nn.max_pool_with_argmax(inputs, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=padding)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 1770, in max_pool_with_argmax\n    Targmax=Targmax, name=name)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n    op_def=op_def)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:\n  <no registered kernels>\n\n\t [[Node: MaxPoolWithArgmax_15 = MaxPoolWithArgmax[T=DT_FLOAT, Targmax=DT_INT64, ksize=[1, 2, 2, 1], padding=\"SAME\", strides=[1, 2, 2, 1]](Relu_62)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m    998\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 999\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1000\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1047\u001b[0m           tf_session.TF_ExtendGraph(\n\u001b[0;32m-> 1048\u001b[0;31m               self._session, graph_def.SerializeToString(), status)\n\u001b[0m\u001b[1;32m   1049\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_opened\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m    468\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 469\u001b[0;31m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m    470\u001b[0m   \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:\n  <no registered kernels>\n\n\t [[Node: MaxPoolWithArgmax_15 = MaxPoolWithArgmax[T=DT_FLOAT, Targmax=DT_INT64, ksize=[1, 2, 2, 1], padding=\"SAME\", strides=[1, 2, 2, 1]](Relu_62)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-01293e109ed8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-22-5127687a9f62>\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(train_stage, training_steps, restore_session, learning_rate)\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'run train step: '\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0mtrain_step\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mimage\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mground_truth\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrate\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10000\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, feed_dict, session)\u001b[0m\n\u001b[1;32m   1447\u001b[0m         \u001b[0mnone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdefault\u001b[0m \u001b[0msession\u001b[0m \u001b[0mwill\u001b[0m \u001b[0mbe\u001b[0m \u001b[0mused\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1448\u001b[0m     \"\"\"\n\u001b[0;32m-> 1449\u001b[0;31m     \u001b[0m_run_using_default_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_run_using_default_session\u001b[0;34m(operation, feed_dict, graph, session)\u001b[0m\n\u001b[1;32m   3666\u001b[0m                        \u001b[0;34m\"the operation's graph is different from the session's \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3667\u001b[0m                        \"graph.\")\n\u001b[0;32m-> 3668\u001b[0;31m   \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3669\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3670\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:\n  <no registered kernels>\n\n\t [[Node: MaxPoolWithArgmax_15 = MaxPoolWithArgmax[T=DT_FLOAT, Targmax=DT_INT64, ksize=[1, 2, 2, 1], padding=\"SAME\", strides=[1, 2, 2, 1]](Relu_62)]]\n\nCaused by op 'MaxPoolWithArgmax_15', defined at:\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-20-ea7a22c7d446>\", line 10, in <module>\n    pool_1, pool_1_argmax = pool_layer(conv_1_2)\n  File \"<ipython-input-8-a5a87bb3bf60>\", line 2, in pool_layer\n    layer, pool_argmax = tf.nn.max_pool_with_argmax(inputs, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=padding)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 1770, in max_pool_with_argmax\n    Targmax=Targmax, name=name)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n    op_def=op_def)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/Users/yuravasiliuk/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:\n  <no registered kernels>\n\n\t [[Node: MaxPoolWithArgmax_15 = MaxPoolWithArgmax[T=DT_FLOAT, Targmax=DT_INT64, ksize=[1, 2, 2, 1], padding=\"SAME\", strides=[1, 2, 2, 1]](Relu_62)]]\n"
     ]
    }
   ],
   "source": [
    "optimize(training_steps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}