LeNet_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class LeNet:\n",
    "    \n",
    "    def __init__(self, dataset, ws=None, trainable=True, x_shape=[None,784], y_shape=[None,10]):\n",
    "        '''\n",
    "        \n",
    "        trainable: if true, we will build a new LeNet model and train it; otherwise it means\n",
    "                    that we already have a LeNet model and wanna to rebuild that old model,\n",
    "                    this time, all weights in LeNet are fixed and non-trainable, instead, \n",
    "                    the inputs (xs) are changed into weights, we wanna to see what kind of these\n",
    "                    inputs (xs) could trigger a perticular neuron to have its maximium output. \n",
    "        \n",
    "        '''\n",
    "        self.trainable=trainable\n",
    "        self.x_shape=x_shape\n",
    "        self.y_shape=y_shape\n",
    "        \n",
    "        if self.trainable:\n",
    "            self.ws=[None for i in range(8)]\n",
    "        else:\n",
    "            self.ws=ws\n",
    "        \n",
    "        self.build_up_LeNet()\n",
    "        \n",
    "        if self.trainable:\n",
    "            self.build_up_loss()      \n",
    "            \n",
    "    def build_up_LeNet(self):\n",
    "        '''\n",
    "        \n",
    "        trainable: if true, build the model; if false, weights are non-trainable, and inputs\n",
    "                    (xs) are trainable variables, and are initialized with ws.\n",
    "        ws = [w_conv1, b_conv1, w_conv2, b_conv2, w_fc1, b_fc1, w_fc2, b_fc2]\n",
    "        '''\n",
    "        # build up the LeNet\n",
    "            \n",
    "        # 1st layer\n",
    "        # Output: [-1,14,14,32]\n",
    "        if self.trainable:\n",
    "            self.x=tf.placeholder(tf.float32, self.x_shape)\n",
    "        else:\n",
    "            self.x=tf.Variable(tf.truncated_normal(shape=[1,784], stddev=0.1))\n",
    "            \n",
    "        self.y_=tf.placeholder(tf.float32, self.y_shape)\n",
    "        self.x_img=tf.reshape(self.x, [-1,28,28,1])\n",
    "            \n",
    "        self.w_conv1=self.weight_variable([5,5,1,32], self.ws[0])\n",
    "        self.b_conv1=self.bias_variable([32], self.ws[1])\n",
    "            \n",
    "        self.h_conv1=tf.nn.relu(self.conv2d(self.x_img, self.w_conv1)+self.b_conv1)\n",
    "        self.h_pool1=self.max_pooling_2x2(self.h_conv1)\n",
    "            \n",
    "        # 2nd layer\n",
    "        # Output: [-1,7,7,64]\n",
    "        self.w_conv2=self.weight_variable([5,5,32,64], self.ws[2])\n",
    "        self.b_conv2=self.bias_variable([64], self.ws[3])\n",
    "\n",
    "        self.h_conv2=tf.nn.relu(self.conv2d(self.h_pool1, self.w_conv2)+self.b_conv2)\n",
    "        self.h_pool2=self.max_pooling_2x2(self.h_conv2)\n",
    "            \n",
    "        # fully connected layer\n",
    "        # Output: [-1,1024]\n",
    "        self.w_fc1=self.weight_variable([7*7*64,1024], self.ws[4])\n",
    "        self.b_fc1=self.bias_variable([1024], self.ws[5])\n",
    "\n",
    "        self.h_pool2_flat=tf.reshape(self.h_pool2, [-1,7*7*64])\n",
    "        self.h_fc1=tf.nn.relu(tf.matmul(self.h_pool2_flat, self.w_fc1)+self.b_fc1)\n",
    "            \n",
    "        # drop out\n",
    "        # drop out rate = 1 - keep_prob\n",
    "        # automatically rescall the neurons that are not chosen\n",
    "        self.keep_prob=tf.placeholder(tf.float32)\n",
    "        self.h_fc1_drop=tf.nn.dropout(self.h_fc1, self.keep_prob)\n",
    "            \n",
    "        # output layer\n",
    "        self.w_fc2=self.weight_variable([1024,10], self.ws[6])\n",
    "        self.b_fc2=self.bias_variable([10], self.ws[7])\n",
    "\n",
    "        self.y_conv=tf.nn.softmax(tf.matmul(self.h_fc1_drop, self.w_fc2)+self.b_fc2)\n",
    "    \n",
    "    def build_up_loss(self):\n",
    "        # loss function\n",
    "        self.cross_entropy=tf.reduce_mean(tf.reduce_sum(-self.y_*tf.log(self.y_conv),\n",
    "                                                                reduction_indices=[1]))\n",
    "        self.correct_prediction=tf.equal(tf.arg_max(self.y_, 1), \n",
    "                                                 tf.arg_max(self.y_conv, 1))\n",
    "        self.accuracy=tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))\n",
    "        # train\n",
    "        self.train_step=tf.train.AdamOptimizer(1e-4).minimize(self.cross_entropy)\n",
    "    \n",
    "    def weight_variable(self, shape, w=None):\n",
    "        if self.trainable:\n",
    "            initial=tf.truncated_normal(shape, stddev=0.1)\n",
    "        else:\n",
    "            initial=w\n",
    "        return tf.Variable(initial, trainable=self.trainable)\n",
    "    \n",
    "    def bias_variable(self, shape, b=None):\n",
    "        if self.trainable:\n",
    "            initial=tf.constant(0.1, shape=shape)\n",
    "        else:\n",
    "            initial=b\n",
    "        return tf.Variable(initial, trainable=self.trainable)\n",
    "    \n",
    "    def conv2d(self, x, w):\n",
    "        return tf.nn.conv2d(x, w, strides=[1,1,1,1], padding='SAME')\n",
    "    \n",
    "    def max_pooling_2x2(self, x):\n",
    "        return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')\n",
    "    \n",
    "    def train(self, dataset):\n",
    "        self.close()\n",
    "        self.sess=tf.InteractiveSession()\n",
    "        self.sess.run(tf.initialize_all_variables())\n",
    "        \n",
    "        for i in range(1000):\n",
    "            batch=dataset.train.next_batch(100)\n",
    "            if i%200 == 0:\n",
    "                train_accuracy = self.accuracy.eval(\n",
    "                    feed_dict={self.x:batch[0], self.y_: batch[1], self.keep_prob: 1.0})\n",
    "                print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "            self.train_step.run(feed_dict={self.x: batch[0], \n",
    "                                           self.y_: batch[1], self.keep_prob: 0.5})\n",
    "    \n",
    "    def train_feature(self, max_which_neuron):\n",
    "        '''\n",
    "        \n",
    "        When the trainable is False, and we wanna max_which_neuron node in LeNet to be maximized,\n",
    "        use this. Traing xs (inputs) without changing original weights, in this way we could find\n",
    "        the feature for each layer in LeNet.\n",
    "        '''\n",
    "        if self.trainable:\n",
    "            return\n",
    "        \n",
    "        self.close()\n",
    "        self.sess=tf.InteractiveSession()\n",
    "        self.sess.run(tf.initialize_all_variables())\n",
    "        #\n",
    "        # loss=tf.reduce_sum(max_which_neuron)\n",
    "        # add L2 norm regularization\n",
    "        #\n",
    "        self.train_step=tf.train.GradientDescentOptimizer(0.2).minimize(\\\n",
    "            -max_which_neuron.eval().max()+0.5*tf.sqrt(tf.reduce_sum(tf.square(self.x))))\n",
    "        \n",
    "        for i in range(3000):\n",
    "            self.train_step.run()\n",
    "        \n",
    "        return self.x.eval()\n",
    "    \n",
    "    def close(self):\n",
    "        try:\n",
    "            self.sess.close()\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "# end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.05\n",
      "step 200, training accuracy 0.93\n",
      "step 400, training accuracy 0.9\n",
      "step 600, training accuracy 0.91\n",
      "step 800, training accuracy 0.99\n"
     ]
    }
   ],
   "source": [
    "# First, build the LeNet\n",
    "m1=LeNet(dataset=mnist)\n",
    "m1.train(mnist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# record the model weights\n",
    "ws=[m1.w_conv1.eval(), m1.b_conv1.eval(), m1.w_conv2.eval(), m1.b_conv2.eval()\n",
    "    , m1.w_fc1.eval(), m1.b_fc1.eval(), m1.w_fc2.eval(), m1.b_fc2.eval()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# maximize one activation\n",
    "m2=LeNet(dataset=mnist, ws=ws, trainable=False)\n",
    "a2=m2.train_feature(max_which_neuron=m2.h_pool2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# max another neuron\n",
    "a2=m2.train_feature(max_which_neuron=m2.h_pool2[0,0,0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# visiualize the input which could maximize that activation\n",
    "plt.imshow(m2.x_img.eval()[0,:10,:10,0], cmap='Greys_r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# just visiualize the weights of the first layer\n",
    "fig, axes=plt.subplots(4,8)\n",
    "k=0\n",
    "for i in axes.flat:\n",
    "    i.imshow(m1.w_conv1.eval()[:,:,0,k], cmap='Greys_r')\n",
    "    k=k+1 \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20003392"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1.w_conv1.eval().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}