TestRNN-HWM0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T05:51:40.717295Z",
     "start_time": "2017-12-13T05:51:40.116208Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#Import the libraries we will need.\n",
    "# -*- coding:utf-8 -*-\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.contrib import rnn\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:00:49.855409Z",
     "start_time": "2017-12-13T06:00:49.561219Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n"
     ]
    }
   ],
   "source": [
    "# 设置 GPU 按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "# 首先导入数据，看一下数据的形式\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n",
    "print mnist.train.images.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### 首先设置好模型用到的各个超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:32.911645Z",
     "start_time": "2017-12-13T06:18:32.903675Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "#\n",
    "lr = 1e-3\n",
    "# 在训练和测试的时候，我们想用不同的 batch_size.所以采用占位符的方式\n",
    "#batch_size = tf.placeholder(tf.int32)  # 注意类型必须为 tf.int32\n",
    "# 在 1.0 版本以后请使用 ：\n",
    "keep_prob = tf.placeholder(tf.float32, [])\n",
    "batch_size = tf.placeholder(tf.int32, [])\n",
    "\n",
    "# 每个时刻的输入特征是28维的，就是每个时刻输入一行，一行有 28 个像素\n",
    "input_size = 28\n",
    "# 时序持续长度为28，即每做一次预测，需要先输入28行\n",
    "timestep_size = 28\n",
    "# 每个隐含层的节点数\n",
    "hidden_size = 28*28\n",
    "# LSTM layer 的层数\n",
    "layer_num = 2\n",
    "# 最后输出分类类别数量，如果是回归预测的话应该是 1\n",
    "class_num = 10\n",
    "\n",
    "_X = tf.placeholder(tf.float32, [None, 784])\n",
    "y = tf.placeholder(tf.float32, [None, class_num])\n",
    "keep_prob = tf.placeholder(tf.float32)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### 开始搭建 RNN 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:33.251103Z",
     "start_time": "2017-12-13T06:18:33.179419Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# 把784个点的字符信息还原成 28 * 28 的图片\n",
    "# 下面几个步骤是实现 RNN / LSTM 的关键\n",
    "####################################################################\n",
    "# **步骤1：RNN 的输入shape = (batch_size, timestep_size, input_size) \n",
    "timestep_size=28/28\n",
    "input_size=28*28\n",
    "X = tf.reshape(_X, [-1, timestep_size, input_size])\n",
    "\n",
    "#output = new_state = act(W input + U state + B)\n",
    "Rnn_W = tf.Variable(tf.truncated_normal([input_size,hidden_size], stddev=0.1), dtype=tf.float32)\n",
    "Rnn_U = tf.Variable(tf.truncated_normal([hidden_size,hidden_size], stddev=0.1), dtype=tf.float32)\n",
    "Rnn_bias = tf.Variable(tf.constant(0.1,shape=[hidden_size]), dtype=tf.float32)\n",
    "\n",
    "outputs = list()\n",
    "new_state = tf.matmul(X[:,0,0:1],tf.Variable(tf.constant(0.1,shape=[1,hidden_size]), dtype=tf.float32))\n",
    "\n",
    "#超级无语，竟然batch_size不能作为Variable的入参\n",
    "#print(new_state)\n",
    "\n",
    "for timestep in range(layer_num):\n",
    "    new_state = tf.nn.tanh(tf.matmul(X[:, 0, :], Rnn_W)+ tf.matmul(new_state, Rnn_U)+ Rnn_bias)\n",
    "    outputs.append(new_state)\n",
    "h_state = outputs[-1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### 设置 loss function 和 优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:33.519375Z",
     "start_time": "2017-12-13T06:18:33.427565Z"
    }
   },
   "outputs": [],
   "source": [
    "# 上面 LSTM 部分的输出会是一个 [hidden_size] 的tensor，我们要分类的话，还需要接一个 softmax 层\n",
    "# 首先定义 softmax 的连接权重矩阵和偏置\n",
    "out_W = tf.placeholder(tf.float32, [hidden_size, class_num], name='out_Weights')\n",
    "out_bias = tf.placeholder(tf.float32, [class_num], name='out_bias')\n",
    "# 开始训练和测试\n",
    "W = tf.Variable(tf.truncated_normal([hidden_size, class_num], stddev=0.1), dtype=tf.float32)\n",
    "bias = tf.Variable(tf.constant(0.1,shape=[class_num]), dtype=tf.float32)\n",
    "y_pre = tf.nn.softmax(tf.matmul(h_state, W) + bias)\n",
    "\n",
    "# 损失和评估函数\n",
    "cross_entropy = -tf.reduce_mean(y * tf.log(y_pre))\n",
    "train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy)\n",
    "#train_op = tf.train.GradientDescentOptimizer(1e-3).minimize(cross_entropy)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:37.585726Z",
     "start_time": "2017-12-13T06:18:33.653161Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter65, step 200, training accuracy 0.914062\n",
      "Iter66, step 400, training accuracy 0.96875\n",
      "Iter66, step 600, training accuracy 0.96875\n",
      "Iter67, step 800, training accuracy 1\n",
      "Iter67, step 1000, training accuracy 0.984375\n",
      "Iter67, step 1200, training accuracy 1\n",
      "Iter68, step 1400, training accuracy 0.992188\n",
      "Iter68, step 1600, training accuracy 0.992188\n",
      "Iter69, step 1800, training accuracy 1\n",
      "Iter69, step 2000, training accuracy 0.984375\n",
      "test accuracy 0.9763\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.global_variables_initializer())\n",
    "for i in range(2000):\n",
    "    _batch_size = 128\n",
    "    batch = mnist.train.next_batch(_batch_size)\n",
    "    if (i+1)%200 == 0:\n",
    "        train_accuracy = sess.run(accuracy, feed_dict={\n",
    "            _X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size})\n",
    "        # 已经迭代完成的 epoch 数: mnist.train.epochs_completed\n",
    "        print \"Iter%d, step %d, training accuracy %g\" % ( mnist.train.epochs_completed, (i+1), train_accuracy)\n",
    "    sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size})\n",
    "\n",
    "# 计算测试数据的准确率\n",
    "print \"test accuracy %g\"% sess.run(accuracy, feed_dict={\n",
    "    _X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:37.718753Z",
     "start_time": "2017-12-13T06:18:37.717367Z"
    }
   },
   "outputs": [],
   "source": [
    "#%prun sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### 可视化看看 LSTM 的是怎么做分类的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:37.846617Z",
     "start_time": "2017-12-13T06:18:37.844642Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "print mnist.test.labels[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:38.060974Z",
     "start_time": "2017-12-13T06:18:37.971017Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd07a916c50>"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAC7CAYAAAB1qmWGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEw1JREFUeJzt3XuMVeW5x/Hfw9BRHFBAhCL1gtR6\nKXJopJTaFsWeKtZEa0ybGq2YYtD02FovjcQ01WhOYtKi549zJKHBQNVj1bZeUtsK1SZzkOMF8Arq\nAQFblIsg6IBJR5jn/MEmGX3fxezZe621937n+0nMzDzzzl7PWvPMw3Kv913L3F0AgNY3qNEJAADy\nQUMHgETQ0AEgETR0AEgEDR0AEkFDB4BE0NABIBE0dABIRF0N3cxmmtmbZrbOzObmlRTQaNQ2WpHV\nulLUzNok/Z+kb0naJOkFSZe4+5r80gPKR22jVQ2u42enSlrn7uslycx+K+lCSZlFb2bcZwCFcnfL\n4WX6Xdvt7e1+2GGH5bBpIPTRRx+pu7u7z9qup6GPk/SPXl9vkvSVOl4PaBb9ru3DDjtM06dPLzQp\nDFydnZ1VjaunoVfFzOZImlP0doCy9a7tIUOGNDgboL6Lou9IOqbX15+rxD7B3Re4+xR3n1LHtoAy\n9bu229vbS0sOyFJPQ39B0olmNt7M2iV9X9Lj+aQFNBS1jZZU81su7r7XzK6R9KSkNkn3uPvq3DID\nGoTaRquq6z10d/+TpD/llAvQNKhttCJWigJAImjoAJAIGjoAJKLweegAajd4cPgnOm3atOjYZ599\nNojt3bu37hwGDQrP+4YNGxYd+8EHH9S9vXp8/PHH0Xjs2HzlK/G1YvVOQc065l1dXUFsxIgRdW3r\n0zhDB4BE0NABIBE0dABIBA0dABLBRVGgCbS1tUXjP//5z4PYli1bomOXLVuWa04HXH/99UHslVde\niY5dsmRJITnExC4+bty4MTr22muvDWKrVq3KOyVJ0rvvvhuNn3nmmUFsw4YNuW6bM3QASAQNHQAS\nQUMHgETQ0AEgETR0AEgEs1yAJnDKKadE48cee2wQu/322wvJ4YwzzojGx40bF8TuvPPOQnLoj507\ndwaxGTNmRMe+/PLLheSwenV4m/ys4/j2228XkkNvnKEDQCJo6ACQCBo6ACSChg4AiajroqiZbZTU\nJWmfpL3uPiWPpJrZ6NGjo/GHHnooiC1fvjw6dsGCBUEsa8lyszriiCOC2PTp06Nj//KXvwSxrPtW\nN4uya/snP/lJNP7www8HsX379tW9vbFjxwax2bNnR8fecsstQaynp6fuHOp11FFHBbGs2yLUe8yy\nlvOPHDkyiDXyXvF5zHKZ4e7bc3gdoNlQ22gpvOUCAImot6G7pCVmttLM5uSRENAkqG20nHrfcvm6\nu79jZqMlLTWzN9y9s/eAyh8DfxBoNf2q7SFDhjQiR+AT6jpDd/d3Kh+3SXpE0tTImAXuPmUgXDBF\nOvpb2/U+WBjIQ81n6GbWIWmQu3dVPj9H0m25ZdYEYk/kji31leKzPrZu3Rod20ozWmL7JUkrV64M\nYrFZB5J0+umnB7F169bVl1iBGlHbWbMorrrqqiB29tlnR8e+9957QSxrxsWkSZOC2IoVK6JjN23a\nFI03Wmymztq1a6NjY/EdO3ZEx3Z0dASx0047LTr2i1/8YhArY4l/lnrechkj6REzO/A6/+3u4fw0\noPVQ22hJNTd0d18v6V9yzAVoCtQ2WhXTFgEgETR0AEgE90OXNGrUqGj8wQcfDGKxpb6SdPfddwex\nH//4x/Ul1gRiT52XpPHjxwex2AU8qbkvgDaLrHucx47z1KnBhBtJ0htvvBHE9uzZEx37y1/+Mojd\nd999B0ux6Wzbti2ITZ48OTp2zpxw5nTseEnxyQxZF1C7u7sPlmLpOEMHgETQ0AEgETR0AEgEDR0A\nEkFDB4BEmLuXtzGz8jbWD+ecc040/uc//7nq1/jsZz8bxGJLsZtZbBnzq6++Gh37yCOPBLErrrgi\nOrarq6uuvPrD3a20jfUyfPhwz3rARyNddtll0XjsNg133XVX0em0hPXr1wexL3zhC9Gxe/fuLTod\nSVJnZ6d27drVZ21zhg4AiaChA0AiaOgAkAgaOgAkYsAt/R89enQQu/jii6v++awno7fSBdDYxU9J\n+utf/1r1a8QuipZ58RPVmTlzZjSedasBxHvE8OHDo2O3b2+uZ4hzhg4AiaChA0AiaOgAkAgaOgAk\nos+Gbmb3mNk2M3utV2ykmS01s7WVj+HTlIEmR20jNdXMclkk6T8l/aZXbK6kp9z9DjObW/n6pvzT\ny9+8efOCWNby6NiT7R9++OHccyrbN77xjWh8zJgxQWzRokXRsa32MIQMi5RQbX/1q18NYoMGxc/Z\n1q5dG8QGD463g7KWt5cta2bahAkTgtiWLVuiY3t6eoJYe3t7fYnVoc8zdHfvlPT+p8IXSlpc+Xyx\npO/knBdQOGobqan1PfQx7r658vkWSeGpHdCaqG20rLoXFrm7H+wuimY2R1L4QD+gyfWntocMGVJa\nXkCWWs/Qt5rZWEmqfAyf1lrh7gvcfYq7T6lxW0CZaqrtRr5vChxQ6xn645JmSbqj8vGx3DIqWOz+\n77ELG5L07rvvBrFme8r3AVlniDfffHMQ+9GPfhQdGzs2P/zhD+tLrPW0bG2PGjUqiLW1tUXH3nrr\nrUFs8eLF4UBJb731Vl15NavYPeElacOGDUEsq0ecdtppQSzrAmoZqpm2+ICk/5V0kpltMrPZ2l/s\n3zKztZL+tfI10FKobaSmzzN0d78k41vfzDkXoFTUNlLDSlEASAQNHQASQUMHgEQMuAdc9Mf5558f\nxJYsWRIdu2vXriA2f/783HOSpDPPPDOInXXWWdGx06ZNq/p1f/e739WaEprA008/HcR27NgRHfv8\n888HsVSX+Gfp6OiIxs8444wgFjteUmNntMRwhg4AiaChA0AiaOgAkAgaOgAkwmLLvQvb2EFudFSW\n008/PYg9+uij0bFHH3101a9rZkGsqGNb77bWr18fjceeEN9qy77dPTw4JRg+fLhPnz69EZvGANDZ\n2aldu3b1WducoQNAImjoAJAIGjoAJIKGDgCJGHArRWMPfp40aVJ07OTJk4NY7MKhJP3sZz8LYlkP\noc2673S17r333iD28ssvV/3zy5cvj8Zb7QIogE/iDB0AEkFDB4BE0NABIBE0dABIRDXPFL3HzLaZ\n2Wu9Yrea2Ttm9lLlv28XmyaQP2obqelz6b+ZTZe0W9Jv3H1iJXarpN3u/qt+bawJlv6n4IQTTghi\n69ati4596aWXgti5554bHZs1K6eV9Gfpf561zdJ/FCm3pf/u3inp/VyyApoItY3U1PMe+jVm9krl\nf1tH5JYR0HjUNlpSrQ19vqQJkiZL2ixpXtZAM5tjZivMbEWN2wLKVFNtd3d3l5UfkKmmhu7uW919\nn7v3SPq1pKkHGbvA3ae4+5RakwTKUmttt7e3l5ckkKGmpf9mNtbdN1e+vEjSawcbj3z94he/CGJZ\nF7dvuummIJbCxc+iNFttf/zxx0HsiCOOiI6N/V9CHg9+jr3G8OHDo2N3795d9/bqMXhwvKXFHuKe\n9Y9wvf84DxoUP0+O5XD44YfXta1P67Ohm9kDks6SNMrMNkm6RdJZZjZZkkvaKOmqXLMCSkBtIzV9\nNnR3vyQSXlhALkCpqG2khpWiAJAIGjoAJIKGDgCJGHAPuGgl3/3ud6Pxyy+/PIh1dXVFx+7YsSPX\nnFCMrNkosVs3zJo1Kzp2zZo1ueZ0wLPPPhvEbrzxxqrHFqWtrS2IbdiwITo2NiunqKmmWX9zn//8\n56seWyvO0AEgETR0AEgEDR0AEkFDB4BEcFG0iZ133nlVj/3jH/8Yja9atSqvdFCg9evXR+MXXXRR\nEHvzzTcLyWH16tXR+NVXXx3Enn/++UJy6I9DDz00iGXdAiN2UXTfvn1153D00UcHsQ8++CA6dufO\nnXVvry+coQNAImjoAJAIGjoAJIKGDgCJoKEDQCKY5dLEsma57NmzJ4jNm5f5pDS0gKwHM5x00klB\nbNOmTXVvb+vWrUFs6NCh0bGxBzP09PTUnUO9Yg/TGD9+fHRsvTNaRo8eHY2vXLkyiH35y1+Ojs3j\n99YXztABIBE0dABIBA0dABLRZ0M3s2PM7G9mtsbMVpvZtZX4SDNbamZrKx9HFJ8ukB9qG6mp5qLo\nXkk3uPsqMxsmaaWZLZV0haSn3P0OM5sraa6k8BHzqEpsefWYMWOiY7dt2xbEWOJfk6ap7diFRyl+\nf/EjjzwyOjZ2b+3YhUNJ6ujoCGIzZ86Mjn3xxRej8UaLLefPuq987DYBWfcijx2zrOMYu6d6GRc/\ns/R5hu7um919VeXzLkmvSxon6UJJiyvDFkv6TlFJAkWgtpGafr2HbmbHS/qSpOckjXH3zZVvbZEU\nP50EWgC1jRRU3dDNbKik30v6qbt/2Pt7vv8WZ9HbnJnZHDNbYWYr6soUKEgetd3d3V1CpsDBVdXQ\nzewz2l/w97v7HyrhrWY2tvL9sZLCN3YlufsCd5/i7lPySBjIU161XdTzKYH+qGaWi0laKOl1d7+z\n17cel3TgabWzJD2Wf3pAcahtpKaaWS5fk/QDSa+a2YFHkN8s6Q5JD5nZbElvS/peMSkODLFZLlk3\n63/iiSeqft1hw4YFsREj4rPw/v73v1f9uolomtqeNGlSNL558+YgNnLkyKpfN2vJemzpftZMm2YV\nm6Xy0UcfRcf255hNmDAhiG3cuDE6duLEiUEs9jsrS58N3d2XSbKMb38z33SA8lDbSA0rRQEgETR0\nAEgEDR0AEsH90FtQ7N7Ol156aXTsddddF8Synu4+a9asaBzFy7of+uGHHx7Espa3xzz55JPR+G23\n3RbEli5dWvXrNoPYMYsdL6l/xyx2a41DDjkkOraRF0BjOEMHgETQ0AEgETR0AEgEDR0AEkFDB4BE\nMMulBV155ZVBbPbs2dGxCxcuDGK333577jmhOZ188snR+AC8zUPV3n///SB24oknRsdu37696HT6\nhTN0AEgEDR0AEkFDB4BE0NABIBFcFG0S11xzTRCLLc+WpM7OziA2f/786NidO3cGMR6Xlqbly5cH\nsQsuuCA6Nnb7h0MPPTQ6tj/L5lvJkUceGY1/+OGHQSx2oVSK/y018ulVnKEDQCJo6ACQCBo6ACSi\nmodEH2NmfzOzNWa22syurcRvNbN3zOylyn/fLj5dID/UNlJTzUXRvZJucPdVZjZM0kozO3Dj5Lvc\n/VfFpQcUitpGUqp5SPRmSZsrn3eZ2euSxhWd2ECzbNmyIHb22Wc3IJOBI7Xa3rFjRxDr6emp+ueP\nO+64aPytt96qOadmFjteknT88ccHsWeeeSY6dsaMGUFsy5YtdeVVj369h25mx0v6kqTnKqFrzOwV\nM7vHzEbknBtQGmobKai6oZvZUEm/l/RTd/9Q0nxJEyRN1v6znHkZPzfHzFaY2Yoc8gVyl0dtM7cf\nzaCqhm5mn9H+gr/f3f8gSe6+1d33uXuPpF9Lmhr7WXdf4O5T3H1KXkkDecmrthu5mAQ4oJpZLiZp\noaTX3f3OXvGxvYZdJOm1/NMDikNtIzXVzHL5mqQfSHrVzF6qxG6WdImZTZbkkjZKuqqQDIHiJFXb\nsQt0//znP6NjY0+xT/XiZ5bdu3dH4x0dHUFs4sSJ0bGNvAAaU80sl2WSLPKtP+WfDlAeahupYaUo\nACSChg4AiaChA0AiaOgAkAgecAEkYujQoUFsz549DcikNcSOl9Tax4wzdABIBA0dABJBQweARNDQ\nASAR5u7lbczsPUlvV74cJWl7aRsvD/vVOMe5+1GN2HCv2m6F41SrVPetFfarqtoutaF/YsNmK1K8\nAyP7NbClfJxS3beU9ou3XAAgETR0AEhEIxv6ggZuu0js18CW8nFKdd+S2a+GvYcOAMgXb7kAQCJK\nb+hmNtPM3jSzdWY2t+zt56nyRPhtZvZar9hIM1tqZmsrH1vuifFmdoyZ/c3M1pjZajO7thJv+X0r\nUiq1TV233r4dUGpDN7M2Sf8l6TxJp2r/o75OLTOHnC2SNPNTsbmSnnL3EyU9Vfm61eyVdIO7nypp\nmqR/q/yeUti3QiRW24tEXbekss/Qp0pa5+7r3b1b0m8lXVhyDrlx905J738qfKGkxZXPF0v6TqlJ\n5cDdN7v7qsrnXZJelzROCexbgZKpbeq69fbtgLIb+jhJ/+j19aZKLCVj3H1z5fMtksY0Mpl6mdnx\nkr4k6Tkltm85S722k/rdp1rXXBQtkO+fQtSy04jMbKik30v6qbt/2Pt7rb5vqF2r/+5TruuyG/o7\nko7p9fXnKrGUbDWzsZJU+bitwfnUxMw+o/1Ff7+7/6ESTmLfCpJ6bSfxu0+9rstu6C9IOtHMxptZ\nu6TvS3q85ByK9rikWZXPZ0l6rIG51MTMTNJCSa+7+529vtXy+1ag1Gu75X/3A6GuS19YZGbflvQf\nktok3ePu/15qAjkyswcknaX9d2vbKukWSY9KekjSsdp/973vufunLzA1NTP7uqT/kfSqpJ5K+Gbt\nf7+xpfetSKnUNnXdevt2ACtFASARXBQFgETQ0AEgETR0AEgEDR0AEkFDB4BE0NABIBE0dABIBA0d\nABLx/5t5MoLpg8iDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd07aba5f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X3 = mnist.test.images[6]\n",
    "img3 = X3.reshape([28, 28])\n",
    "plt.subplot(1,2,1)\n",
    "plt.imshow(img3, cmap='gray')\n",
    "\n",
    "import pywt,cv2\n",
    "cA2, (cH2, cV2, cD2)=pywt.wavedec2(np.float32(img3)/255,'haar',level=1);\n",
    "X2=np.vstack((np.hstack((cA2, cH2)),np.hstack((cV2, cD2))))\n",
    "plt.subplot(1,2,2)\n",
    "plt.imshow(X2,cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:38.217719Z",
     "start_time": "2017-12-13T06:18:38.184845Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 1, 784)\n",
      "(2, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "X3.shape = [-1, 784]\n",
    "y_batch = mnist.train.labels[0]\n",
    "y_batch.shape = [-1, class_num]\n",
    "\n",
    "X3_outputs = np.array(sess.run(outputs, feed_dict={\n",
    "            _X: X3, y: y_batch, keep_prob: 1.0, batch_size: 1}))\n",
    "print X3_outputs.shape\n",
    "X3_outputs.shape = [-1,28, 28]\n",
    "print X3_outputs.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "Collapsed": "false",
    "ExecuteTime": {
     "end_time": "2017-12-13T06:18:38.442770Z",
     "start_time": "2017-12-13T06:18:38.335444Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJgAAAA9CAYAAABRE6B8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFA5JREFUeJztm2lUFGfWgJ+qgmZHBUFoEJQdNdqA\nGy4xaqTd4hbcFZNoUMmqSXRitsk4WdS4rzHuxhh13BXtuCVGREUQUQFFBARaVAz72kt9P5hhzvz5\nTjjno4fzWc8vuquh7q1+zlv3vbcQZFlGQaG5EP/bASj8/0YRTKFZUQRTaFYUwRSaFUUwhWZFEUyh\nWVEEU2hWFMEUmhVFMIVmxcqSJ/vi1mj5yvSulC4xUK1rx7uzD3FokIaKnj44pehJ/1s7kAV+e3kV\n096dj31BNVJxORnzPVEHPaXoVjtWj93O14teoyRIpMavnltR65jYfyJyRSVrk48SvWQB4TFpbO+5\nXbBkbv9thraeKR/NuMDakkBOvTWA0z9tZbhXeONxK492nEzRAaD1Cmt4U5Ypnh1J8hcbOVcj8fHi\nWHxmZlE535PTR3fzrr4Hd7sbyP6uN/enbKLHJ3O58vf1WHtm/+lrK1hyVNRL9xfZvNcd1/h7bE45\niq4qgP2dPFmVk0CFbE1qrS8OYh2fXniVOX0vEGpbyOdrXqOssxHB1oT4VEXQD08QKqp4PLIjz7ob\nkRyN7IzcyvRLs1C7l/JTp11My5jOpSFLnyvBhkgT5PxPI0mfuwGTbGbgW3OwO3INydmZ+MyLAFSa\na4n64H2cj9zg28yLJNQEcKyTKzp9Klq1BkQJQZI4nXcNrVqDTp/K8AHjqHjBjfNrN1BirmW6Tz/O\nmPb/6Wtr0RWszXtg2FjAa59e57unL3E0KZyAFw3crMvFhMiOv47iyag6gn+o4kDyy3SIyUJ9/hnt\n1t2jzaU2XKMjWEnkzfDD52ARTvmulPrZMf3ZXLp0y8PRqo5BBz6kS/ccS6bVIvjjeCDp4RsaX1/c\nsBk2QEKtGRDRqjXIfbpRNkDEqb6ehR17kb0nDH9uADTINGQimXNao1XXYxwcweBpEYTvS+FIpjMj\nvSLQ6VORQgKaFJdFazDNvizykrzZOeRFTpzvwZy+F3jzh0OkVbdn++zRPBpgxlyi4rMDu6kfUcqT\naicyZ7dm98NLbPGNx8ahntY/PGXdm5tw3lHKlu9XUhZqwtGnnNt5am4WqcEscCu5oyXTahEY49sy\nIlzL+tL2jbfG4WFR9LUViYqewdaHlziwfxN33tkAssy9zT2wVhn5/EEKAGXmGkx37jKkVxoFH/fB\n6lwyVueTWeZxA7+pN4GGW+ujb6UmxWVRwa4+68D5KcvodiQPB73A9oze/OWXSfx8OZIDu9cRuuoZ\ni4f8g/e+eYuzEVsouOeO3SOJVxZ9yGOTEZcDDiRfCOHjz2Ipn+XKmPULODByLd4LDYR+Xoz3EhGp\nDsx2Zkum1SIw2cLfEo9jkCWyl0Uy1Kc78Td+YWjmCGRJZKZPP1qJdmjVGqQ2bQiKTSK1zza+7j8S\ngFaiHQAxbRPw/uYyOn0qCELD593dANAV3qDdhKbdHSwqWM0PaoavWsCFpX1I+GAFDuccGRuZhNSm\njgrZzN25bnz66zhaPagn0+BA4HvXUV+qQVVlJm78XFqlPEY0CNQ7ClBciu2LxXzWYxiPvxMpi/Dk\n3lwVAd/nEzIvzZJptQi8Tz4h+vTbTHS6zf2pG5GNRkaEaynf3J4z+7Y3ynJ/ZW/i71wAYEzIIIyF\nerRqDQXGSgD62ooMulXFsMC+8M/6/HjK6cbzyHV1TYrLooLZlBpRlcus+3oN/ZbOJ+79w5Qb7dAG\nZjDw0Id4JsiEzE/H9kExMw/MxThQg3TlDuW+Eu/+dADZ1obzM5fiH3OP8DNFFOtbsfz6ccoq7DFb\nCazstw/ZyZ7dWecsmVaLQLa1RrAz4WnlyNoSX3T6VE6m6Di2bDklpmoO5icSczef7ImbGneR5ooK\ndPpUdPpUhq1ZgE6fSp1s4EJYK05lJaDTp1IcG8nIYVNYlnsFg2zicMG1JsVlUcFyx4NNucwn0a+j\n3pvFhvsvkt/PQGKRLzejV1HjIjLrxi1ypnjhd7iK+F2bsD3bhooAEy+oihm9/3daiSqqoq24PrMr\nEaE5nK0KxVSu4mm4QL0s8bS3K/23fmTJtFoEJ+L38CBqa8PPndugVWv+uTJZ4Sza8krGRKY6PWvY\nLcoyOn0qEzKKiBr/Glq1BvV3lwEY5dUD2SyjHTOd/m/PxmgnIOTqcRGNVJrrGDfmjSbFZdE2hdZ+\nuix4eyLrH1PXJ5Qd21bTVlQR3ellpibd4YsT4xG9qrnVfytl5nr67f4Q206lmBPb8PnMPeTUuXP2\nSQiT1En0t89mWVEUv/7aFZ9f6hHrzaiy9Lx09j5nuzhxxnzg+WpTiOP/44t88G0kWTEbGRbcn56X\n/uCKRsWAm9UsansXoEE0/t0f06o1HCy4QmzeMHZ3OIckiHQ8GkvoymeY7mX/x7macm0tuoJNTr1P\njb8rdzeGUDmvnKgrcxk1bQ4PtvjSWaUnaHEG4e0L6HTgHWKyJhCw8j7W8a2pVptZfGcEB5e/jJ9T\nMe2tnzH11uuYZQHRAA9nmkCWGX3hFvm1LhQdCbVkWi0CuU83oKHdANCtbxapdXWcyPyNL93ugCzz\nW9eGQn7w9JnEF6ZQsKgPZX07UGCsRKdPJXrMLFb5nGBFSSAjwrUExSWR+UkrdPpUBGsViBJSoF+T\n4rKoYNvmj8WmuAYrGyNuc2rwi9NT1tEGSTKz7Vk/OpypI+Vhe4I3PEWY34qtyYdxSa/BqlogyicT\naeIT0lZ2I9fghpVk4rffX+DXmGWkD9hK3ttmVt4ezO1F3XDe6WzJtFoE1o9KQZTQqjV8+SCZiv7F\nHC4PRxLExtVK6hxM8Pa52KYXIgkiPqtSubT2e7ytHBv6ZNdv4yra4asqxlj0mMP5V3kwZBvDOw/k\nZO4V5t69iznnYZPisqhgtS4Sjqses6b7Pioi1BSPDKK4lwm3rfb8tqcHaptS1K5lyAWPECpr2F4a\ngWbNTVSlAoeudufZDXceD6vnmwsjOd5lF0FbnjJwx0d0T5pG3AsXWfiCjkd9VNQ7PX8jVmNeAbqC\n5IaiveIFECVulLbnnqEKqVMQOn0queNc6fBJIsZHRWjVGk7dv4zWO4KR94bxc35DDSYJIhMcy4gv\nbOiPFRgrib9zAUkQ2RgYwL7c35sUl0W/idYZFVR8pObzv7/BU40VZYHQOs0K+6vZ1LcCkyySf9uD\nu0tfoLivB/u3DObOIGfOv72Mq6+sxOBuYHDwXUL/komtIFE00A2fU9VIutacierEl7+OwW+XHpup\nRZZMq2VgNtHx1Cy0XmF84ZaO3KsLUzyu8k6Hfhhc7AG4PnsVAFnreoEoMbRjL3QFyZwIOsWk9n04\nUZiMSTbzyFjJI1M1NoIVsWGjCNo1t3Hn6SjYNCksixb53d5ZKR9asJSkOi92jI7i4VcqZgUn8Ki+\nNS5WVVwcFsTSS//gy4KRLGl/lOHX5uLjUkK9WSL/iQvux21Y/NUPxCbGoLIxorrsRLWnjP+BSrYc\n3IiTKLG3PIAjXT35pX7vc1Xkm4sCG7/IUVlDORBwnFFePeh908CnbdMaRz3/ul3GF6YgCSLhf5tL\n0mfrkQSRYlMVK4ojmeN6mdkhUQ0rnFrT+Flo2By02CLfPKSE2PuTGW7/mIx5rXE43FArXSzy50J3\nF0y74PfqAJJu+ZNuaMsSzUEqNntTa7Ti4/BTzPziCLMuvI65wpr3Op/HI7ES59Bn1La1JTZsFC9/\nPh8TIrXaMEum1SKIvPlqozyG4RWM9o1Ep0/lSjdrRnpF8MmDVIJ2zOVYYVLDTFEQ2VPhSsrnGykx\n17Cp1Iu2kgNft0vDx8oRfaym8e+N6NAbrVcYQ326N24i/iwWFUz8pQ1Gs0i0/wAGdc2gwkfg9Gv9\niVJnsjnrHGV1tixJGM7UyERWvT6ZxXdHcnDZd1TVqehtl0N2rTuXo1YRujCTn/J7smHfBtothIJp\nRtqdNBAZd52T0ZGUzy63ZFotglaj8hFtbamTDZzKSkD08208ptOn8vqxOYT0zcFGsG7skU11egZA\nW8mBpPKG+a1WrWHIhNdI+WgdAHvyEzidd40J6Y+QjcYmx2VRwdolVTBOnUruJ+FkfxmKx9U6hFtZ\nvO+SxOgbs3jF6zaYBOI39cNzaTZuU4pwkWwY1eE2C3qPwd/2CVNi57H45llMssDqpwO5/5kd9wdu\n516pG2mfaahcaaAu0dWSabUIZEM9O7PO8dRUh1atwdDOuXEFAsiesIm6AQ3FfatLDdfHJJupkw1o\n1Ro+9Tzd+IjOmf07kAQRK++GVU3rHcH+UA/EriF0uTK1SXFZVLDB2y+zc8Vw2l0zsX3TSnImC+T9\nGEC6wRbxZBuulXTAukSipIeBerMVz0Z3JiQ+DlvRwOORfuwP9SD8q2Sm7XgfL8cyzpzoQSunGrou\nj2N98F4qva2w/8SBGo/nb9iNIOAuOTD5vQ8QbW35atdmBGsV8YUpDH/pVUyyGcFaxdc516iaaNPw\nvlc4NoI12XvCmOPbD8Hm3wX8sIA+HLt6HK13BLqCZADMaZl4jbvTpLAsKtjrrdKwjn6CQ+J9ZmVN\nIWfYFhZ3Pcbr+99i88eryT7hT8CSdII31lLygTdtbpfzZq+LdLV7iOuWRB58G0n8sd6ceGMpFdpa\nDEHV1NRb89LkJCbvmMe5L1YwfvdZrKqeq/oeaHjSIezrOOyPXid3YTg9baw5nXeN4V7hbD27k0VP\nwpGNBhZ17Im5tIzh3hGsyE0EwH96GlZ+HTj5oOH1i3GxmKuricoYg64gmYz6aiQ3N0QHBxCadm0t\nKlhM15G0sa3BXFaOamwJrz/sz7UqP2QBfqsKwefHBxTuUPPgVScKBzgi6p9yvLAL84/MwD/JlsVj\nf6bW08jYNQsQrK2QzQLzQs8xz+0CsggT70XjalVJ4PcFlkyrRdDxSCzjY8+B2URa7FqGh7yI1isM\n0d4eSRBI/ee+50RhMqeyEgA4XB5GpbkWXUEyxge5bCrzRavWYHekYaBtPayIjqdm4SbJ+MWX8+iN\nbo1PWPxZLCpYe10tr3klcHezhp3pp7n0wJ/3XS8xY9gFflqtpby3L6qjrXHs/AezY06y6/phWr2S\nh6pjBTeKvVj590lsH7KFSTPOkbnaH57a4GVdwtu9o+m4JJVFHU6y6r3J6Nc4WDKtFkFwSCGL2t6l\neHYkIgI587qgK7zBqfuXmd6+Lzp9Ku9mZWAtSHQ8+SZSoB8JPVvT9VxcY632Vut8NuVdwjWhDTp9\nKsdzEwmaeZ2p7fuiOx9OXNyRxgbsn8Wigl08Eca50k54eJbw2tA3CFxcxV2DMz8eHkRUXAKVnhLP\nIsx4zHjCistRTJ72NrJZRrzuTP1hd6xrzHS3qebH/YMJXG/g4JjVfPteDBlL1OTN07C56CX+unYr\nVRltLJlWi0AeVEiPlAm025dOrrEany8bOvP/ai1ox8WwJiAErVpDzogfMN3LxlxbS+CMBmGEsM4Y\nZBNxoVHEevwGQNfN7zQcs7EhcNtTlh0fze4KjybFZVHBTPYyRrOEtKUtMYd+IWtGW95Jm8RXk38k\ndVoozvlGVmj38MfwYOxzrOmxKhl6dmbSlPOoxj6h3Ffi1XFv0nHnQ/7o7MiU5JmsX78G919UrH3j\newIdnjBvxRz8ejZtXvb/AdHeHpdXsvg2VYe/tSOSszNl5hpkoxGtWoOYdh8EobHZqtOnEppsxZKc\nqxwrTOL0yT30X/gWGzJ+oa+tAYC2aSYATj5IZP+5H/FbeIW9oV5Ni+v/PNP/BauACvSDZcr8JXZO\nGIo6wYTzT05sjI0mZ7wrYX9NYXNkL2xLTbRNM1JpsuHrvVs4lNuNfZ130m9SCkZHFRkfeuN27Q8C\n3IrJMrjx+EUTb56eRZh9Li6ZdYiTDZZMq0VgrqkBYOyR99F6hWEqL2dyxGjur+qNVXtvSg96IkgS\nw/x6Aw39rszYYJxEA6P9+gFw4pvlRCW8zQePejMsuD/2R68jR3ajTjbiKNpSGd2zZddgNyN3IrZp\nTds0AzVejjydUU3rxAKsi6txv25kiksimct9MThI2OlS+e3HHnz+4jjMZ12Z22ciuVF2rNm2jtNj\nloP+CfVRJXyZPpKUYasJXZxLQmUQzzrZUNH3+funD13hDeILkkl8dTkIIjp9KvcW+BHw/hW+uniQ\nxG4HeRbTA3NtbePvFGucKTQ5Itc19M7SDQ5kvbSD9I9ewFxRQcHCXgiJNxnr3ZOhvj2p9JJYkH2r\nSXFZdBYZMWuFXDG0Er/PanDcWkJyUiBuwcW4vllN9mxf7IoEyoPMuAc/pajABf+fTEzYcJrDER1Y\nnXGGrX/05fTOPrS7VoVVxkOKJobgPTmHu5c64r+9CGxU8LiYR1vduPnK4ueqV/GvWaRWrWHrw0u4\nSTaM9o0kLiOd/rbF5JtEOluruGeoZdsffVnmcQOTbCb415ms7rWXEfb/Fk+r1lAV3YtDK1dQL8tc\nrvViW+dgEAVO51xF9MhqmbNIwQTurSoxeDjx5Bs/Zg6+gMu7Ju4ubcfVN1bgefA+3ufMqDa4Erqk\nmMI4AwfejCJ3gYYRP3/IZ25XqOldyYH9m0DtjqpC5k5KB8YMT2Tz+d3I1hIFMcFs77rTkmm1CLRq\nDVrvCARrFf1PzUNERDYaWR8YhLNoy0cdeiMJIu936MPtfnYsfKxhhG9Pgt7NY333XphkMya5oUH9\n+J0+OPzjKm/njWKmb38mOJaBbEauq8Mgm5oUl0VXMIXnj+fvyTwFi6IIptCsKIIpNCuKYArNiiKY\nQrOiCKbQrCiCKTQrimAKzYoimEKzogim0Kwogik0K4pgCs2KIphCs6IIptCsKIIpNCuKYArNiiKY\nQrOiCKbQrCiCKTQrimAKzYoimEKzogim0Kwogik0K/8D5Bwf68KAzYQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd07a8f0350>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h_W = sess.run(W, feed_dict={\n",
    "            _X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})\n",
    "h_bias = sess.run(bias, feed_dict={\n",
    "            _X:X3, y: y_batch, keep_prob: 1.0, batch_size: 1})\n",
    "h_bias.shape = [-1, 10]\n",
    "\n",
    "bar_index = range(class_num)\n",
    "for i in xrange(X3_outputs.shape[0]):\n",
    "    plt.subplot(7, 4, i+1)\n",
    "    plt.imshow(X3_outputs[i,:,:])\n",
    "    plt.axis('off')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.17"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}