CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Apr 14 21:15:18 2016\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "#import matplotlib.plot as pyplt\n",
    "import pandas as pd\n",
    "import time\n",
    "localtime = time.asctime( time.localtime(time.time()) )\n",
    "print (localtime)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_file = 'train.csv'\n",
    "test_file = 'test.csv'\n",
    "train = pd.read_csv(train_file)\n",
    "test = pd.read_csv(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'label', u'pixel0', u'pixel1', u'pixel2', u'pixel3', u'pixel4',\n",
      "       u'pixel5', u'pixel6', u'pixel7', u'pixel8',\n",
      "       ...\n",
      "       u'pixel774', u'pixel775', u'pixel776', u'pixel777', u'pixel778',\n",
      "       u'pixel779', u'pixel780', u'pixel781', u'pixel782', u'pixel783'],\n",
      "      dtype='object', length=785)\n",
      "0    1\n",
      "1    0\n",
      "2    1\n",
      "3    4\n",
      "4    0\n",
      "Name: label, dtype: int64\n",
      "(28000, 784)\n"
     ]
    }
   ],
   "source": [
    "print (train.columns)\n",
    "print (train['label'].head())\n",
    "print (test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29399, 784)\n",
      "(29399, 10)\n",
      "<type 'numpy.ndarray'>\n",
      "(28000, 784)\n",
      "[[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "(12601, 784)\n"
     ]
    }
   ],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "if isinstance(train, pd.DataFrame):\n",
    "    train = train.as_matrix()\n",
    "if isinstance(test, pd.DataFrame):\n",
    "    test_dataset = test.as_matrix().astype(np.float32)\n",
    "#print (train.shape)\n",
    "    \n",
    "def reformat(dataset, labels):\n",
    "  dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "  # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n",
    "  labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "  return dataset, labels\n",
    "\n",
    "train_num, img_size= train.shape\n",
    "train_size = int(train_num * 0.7)\n",
    "\n",
    "train_dataset = train[0:train_size,1:]\n",
    "valid_dataset = train[train_size::,1:]\n",
    "\n",
    "train_labels_raw = train[0:train_size, 0]\n",
    "valid_labels_raw = train[train_size::,0]\n",
    "\n",
    "\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels_raw)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels_raw)\n",
    "\n",
    "print (train_dataset.shape)\n",
    "print (train_labels.shape)\n",
    "print (type(train_dataset))\n",
    "print (test_dataset.shape)\n",
    "print (train_labels[0:5,:])\n",
    "print (valid_dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn import svm\n",
    "num = 20000\n",
    "model2 = SVC()\n",
    " \n",
    "model2.fit(train_dataset[0:num,:], train_labels_raw[0:num,])\n",
    "    \n",
    "valid_pred = model2.predict(valid_dataset)\n",
    "print ('Training Nums:',valid_pred.shape[0], 'Training Accuracy:', float(sum(valid_pred==valid_labels_raw)) / float(valid_pred.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "          / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run the model!\n"
     ]
    }
   ],
   "source": [
    "batch_size = 200\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  #tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Variables.\n",
    "  weights = tf.Variable(\n",
    "    tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "  biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "  # Training computation.\n",
    "  logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  #valid_prediction = tf.nn.softmax(\n",
    "  #  tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "  test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)\n",
    "  print ('Run the model!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 3649.566895\n",
      "Minibatch accuracy: 6.5%\n",
      "Minibatch loss at step 500: 824.608765\n",
      "Minibatch accuracy: 87.5%\n",
      "Minibatch loss at step 1000: 588.991577\n",
      "Minibatch accuracy: 91.0%\n",
      "Minibatch loss at step 1500: 924.452087\n",
      "Minibatch accuracy: 85.0%\n",
      "Minibatch loss at step 2000: 741.771851\n",
      "Minibatch accuracy: 91.5%\n",
      "Minibatch loss at step 2500: 895.657043\n",
      "Minibatch accuracy: 85.0%\n",
      "Minibatch loss at step 3000: 562.268738\n",
      "Minibatch accuracy: 93.5%\n",
      "Minibatch loss at step 3500: 830.433228\n",
      "Minibatch accuracy: 89.5%\n",
      "Minibatch loss at step 4000: 6555.941406\n",
      "Minibatch accuracy: 79.0%\n",
      "Minibatch loss at step 4500: 1258.187012\n",
      "Minibatch accuracy: 83.0%\n",
      "Minibatch loss at step 5000: 1354.845581\n",
      "Minibatch accuracy: 83.5%\n",
      "Minibatch loss at step 5500: 256.456177\n",
      "Minibatch accuracy: 93.0%\n",
      "Minibatch loss at step 6000: 456.457642\n",
      "Minibatch accuracy: 91.0%\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "<type 'numpy.ndarray'>\n",
      "(28000, 10)\n",
      "(28000,)\n",
      "done!\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "num_steps = 6001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 500 == 0):\n",
    "      print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "      #print(\"Validation accuracy: %.1f%%\" % accuracy(\n",
    "      #  valid_prediction.eval(), valid_labels))\n",
    "  #print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))\n",
    "  print (type(test_prediction))\n",
    "  test_pred = test_prediction.eval()\n",
    "  print (type(test_pred))\n",
    "  print (test_pred.shape)\n",
    "  out = np.argmax(test_pred, 1)\n",
    "  print out.shape\n",
    "  out = pd.DataFrame(out)\n",
    "  out.to_csv('out.csv')\n",
    "  print ('done!')\n",
    "  print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run the following model!\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "n_hidden = 1024\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  tf_hidden_dataset = tf.placeholder(tf.float32, shape = (batch_size, n_hidden))\n",
    "    \n",
    "  \n",
    "\n",
    "  # Variables.\n",
    "  weights_0 = tf.Variable(\n",
    "    tf.truncated_normal([image_size * image_size, n_hidden]))\n",
    "  biases_0 = tf.Variable(tf.zeros([n_hidden]))\n",
    "    \n",
    "  weights_1 = tf.Variable(tf.truncated_normal([n_hidden, num_labels]))\n",
    "  biases_1 = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "  # Training computation.\n",
    "  hidden_level = tf.matmul(tf_train_dataset, weights_0) + biases_0\n",
    "  tf_hidden_dataset = tf.nn.relu(hidden_level)\n",
    "  logits = tf.matmul(tf_hidden_dataset, weights_1) + biases_1\n",
    "  \n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "    \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.0001).minimize(loss)\n",
    "  \n",
    "\n",
    "  def multilayer_perceptron(x, w_0, b_0, w_1, b_1):\n",
    "        l1 = tf.nn.relu(tf.matmul(x, w_0) + b_0)\n",
    "        pred = tf.matmul(l1, w_1) + b_1\n",
    "        return pred\n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_prediction = tf.nn.softmax(multilayer_perceptron(tf_valid_dataset, weights_0, biases_0, weights_1, biases_1))\n",
    "  test_prediction = tf.nn.softmax(multilayer_perceptron(tf_test_dataset, weights_0, biases_0, weights_1, biases_1))\n",
    "  #valid_prediction = tf.nn.softmax(\n",
    "  #  tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset,weights_0) + biases_0), weights_1) + biases_1)\n",
    "  #test_prediction = tf.nn.softmax(\n",
    "  #  tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset,weights_0) + biases_0), weights_1) + biases_1))\n",
    "  print (r'Run the following model!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 56586.421875\n",
      "Minibatch accuracy: 7.8%\n",
      "Validation accuracy: 30.1%\n",
      "Minibatch loss at step 500: 288.070038\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 93.4%\n",
      "Minibatch loss at step 1000: 73.831123\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 93.9%\n",
      "Minibatch loss at step 1500: 12.505539\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 94.3%\n",
      "Minibatch loss at step 2000: 0.000000\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 94.4%\n",
      "Minibatch loss at step 2500: 11.752121\n",
      "Minibatch accuracy: 97.7%\n",
      "Validation accuracy: 94.4%\n",
      "Minibatch loss at step 3000: 4.557785\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 94.4%\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "<type 'numpy.ndarray'>\n",
      "(28000, 10)\n",
      "(28000,)\n",
      "done!\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "num_steps = 3001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 500 == 0):\n",
    "      print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "      print(\"Validation accuracy: %.1f%%\" % accuracy(\n",
    "        valid_prediction.eval(), valid_labels))\n",
    "  #print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))\n",
    "  print (type(train_prediction))\n",
    "  print (type(test_prediction))\n",
    "  test_pred = test_prediction.eval()\n",
    "  print (type(test_pred))\n",
    "  print (test_pred.shape)\n",
    "  out = np.argmax(test_pred, 1)\n",
    "  print out.shape\n",
    "  out = pd.DataFrame(out)\n",
    "  out.to_csv('out-single level CNN.csv', header = ['Label'])\n",
    "  print ('done!')\n",
    "  print ('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run the following model!\n"
     ]
    }
   ],
   "source": [
    "batch_size = 256\n",
    "n_hidden = [512, 512]\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  \n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  #tf_hidden_dataset = tf.placeholder(tf.float32, shape = (batch_size, n_hidden))\n",
    "    \n",
    "  \n",
    "\n",
    "  # Variables.\n",
    "  #weights_0 = tf.Variable(\n",
    "  #  tf.truncated_normal([image_size * image_size, n_hidden]))\n",
    "  weights = [tf.truncated_normal([image_size * image_size, n_hidden[0]]),\n",
    "             tf.truncated_normal([n_hidden[0], n_hidden[1]]),\n",
    "             tf.truncated_normal([n_hidden[1], num_labels])]\n",
    "  #biases_0 = tf.Variable(tf.zeros([n_hidden]))\n",
    "  #biases_0 = tf.Variable(tf.truncated_normal([n_hidden]))\n",
    "  biases = [tf.Variable(tf.zeros([n_hidden[0]])),\n",
    "            tf.Variable(tf.zeros([n_hidden[1]])), \n",
    "            tf.Variable(tf.zeros([num_labels]))]  \n",
    "  #weights_1 = tf.Variable(tf.truncated_normal([n_hidden, num_labels]))\n",
    "  #biases_1 = tf.Variable(tf.zeros([num_labels]))\n",
    "  #biases_1 = tf.Variable(tf.truncated_normal([num_labels]))\n",
    "  \n",
    "  # Training computation.\n",
    "  def multilayer_perceptron(x, w, b):\n",
    "        h1 = tf.nn.relu(tf.matmul(x,w[0]) + b[0])\n",
    "        h2 = tf.nn.relu(tf.matmul(h1, w[1]) + b[1])\n",
    "        pred = tf.matmul(h2, w[2]) + b[2]\n",
    "        return pred\n",
    "            \n",
    "  #hidden_level_1 = tf.matmul(tf_train_dataset, weights_0) + biases_0\n",
    "  #tf_hidden_dataset = tf.nn.softmax(hidden_level)\n",
    "  #logits = tf.matmul(tf_hidden_dataset, weights_1) + biases_1,\n",
    "  logits = multilayer_perceptron(tf_train_dataset, weights, biases)\n",
    "  loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "    \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.000000001).minimize(loss)\n",
    "  \n",
    "  \n",
    "  \n",
    "  \n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  #valid_prediction = tf.nn.softmax(multilayer_perceptron(tf_valid_dataset, weights_0, biases_0, weights_1, biases_1))\n",
    "  test_prediction = tf.nn.softmax(multilayer_perceptron(tf_test_dataset, weights,biases))\n",
    "  #valid_prediction = tf.nn.softmax(\n",
    "  #  tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset,weights_0) + biases_0), weights_1) + biases_1)\n",
    "  #test_prediction = tf.nn.softmax(\n",
    "  #  tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset,weights_0) + biases_0), weights_1) + biases_1))\n",
    "  print (r'Run the following model!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 606609.625000\n",
      "Minibatch accuracy: 17.2%\n",
      "Minibatch loss at step 100: 525017.500000\n",
      "Minibatch accuracy: 12.5%\n",
      "Minibatch loss at step 200: 643852.437500\n",
      "Minibatch accuracy: 9.0%\n",
      "Minibatch loss at step 300: 860628.750000\n",
      "Minibatch accuracy: 10.9%\n",
      "Minibatch loss at step 400: 496180.062500\n",
      "Minibatch accuracy: 19.5%\n",
      "Minibatch loss at step 500: 530763.000000\n",
      "Minibatch accuracy: 9.8%\n",
      "Minibatch loss at step 600: 739653.000000\n",
      "Minibatch accuracy: 7.8%\n",
      "Minibatch loss at step 700: 798790.875000\n",
      "Minibatch accuracy: 11.3%\n",
      "Minibatch loss at step 800: 733743.750000\n",
      "Minibatch accuracy: 2.7%\n",
      "Minibatch loss at step 900: 703986.625000\n",
      "Minibatch accuracy: 8.2%\n",
      "Minibatch loss at step 1000: 659720.250000\n",
      "Minibatch accuracy: 11.7%\n",
      "Minibatch loss at step 1100: 597081.375000\n",
      "Minibatch accuracy: 10.9%\n",
      "Minibatch loss at step 1200: 512912.906250\n",
      "Minibatch accuracy: 6.6%\n",
      "Minibatch loss at step 1300: 925025.812500\n",
      "Minibatch accuracy: 9.4%\n",
      "Minibatch loss at step 1400: 510685.437500\n",
      "Minibatch accuracy: 11.7%\n",
      "Minibatch loss at step 1500: 692519.375000\n",
      "Minibatch accuracy: 10.5%\n",
      "Minibatch loss at step 1600: 534496.062500\n",
      "Minibatch accuracy: 7.8%\n",
      "Minibatch loss at step 1700: 732072.187500\n",
      "Minibatch accuracy: 12.9%\n",
      "Minibatch loss at step 1800: 558194.000000\n",
      "Minibatch accuracy: 12.1%\n",
      "Minibatch loss at step 1900: 606723.250000\n",
      "Minibatch accuracy: 6.2%\n",
      "Minibatch loss at step 2000: 791240.500000\n",
      "Minibatch accuracy: 10.5%\n",
      "Minibatch loss at step 2100: 564306.250000\n",
      "Minibatch accuracy: 13.3%\n",
      "Minibatch loss at step 2200: 555285.625000\n",
      "Minibatch accuracy: 5.5%\n",
      "Minibatch loss at step 2300: 735237.437500\n",
      "Minibatch accuracy: 8.6%\n",
      "Minibatch loss at step 2400: 791858.375000\n",
      "Minibatch accuracy: 10.5%\n",
      "Minibatch loss at step 2500: 628487.687500\n",
      "Minibatch accuracy: 11.7%\n",
      "Minibatch loss at step 2600: 547912.750000\n",
      "Minibatch accuracy: 9.4%\n",
      "Minibatch loss at step 2700: 651408.500000\n",
      "Minibatch accuracy: 6.6%\n",
      "Minibatch loss at step 2800: 569455.312500\n",
      "Minibatch accuracy: 8.2%\n",
      "Minibatch loss at step 2900: 694442.562500\n",
      "Minibatch accuracy: 10.5%\n",
      "Minibatch loss at step 3000: 585585.000000\n",
      "Minibatch accuracy: 5.5%\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "<type 'numpy.ndarray'>\n",
      "(28000, 10)\n",
      "(28000,)\n",
      "done!\n"
     ]
    }
   ],
   "source": [
    "num_steps = 3001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 100 == 0):\n",
    "      print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "      #print(\"Validation accuracy: %.1f%%\" % accuracy(\n",
    "      #  valid_prediction.eval(), valid_labels))\n",
    "  #print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))\n",
    "  print (type(train_prediction))\n",
    "  print (type(test_prediction))\n",
    "  test_pred = test_prediction.eval()\n",
    "  print (type(test_pred))\n",
    "  print (test_pred.shape)\n",
    "  out = np.argmax(test_pred, 1)\n",
    "  print out.shape\n",
    "  out = pd.DataFrame(out)\n",
    "  out.to_csv('out-two level CNN.csv', header = ['Label'])\n",
    "  print ('done!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}