Variational_autoenc_tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.examples.tutorials.mnist.input_data as input_data\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from scipy.misc import imsave as ims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge(images, size):\n",
    "    print images.shape##64x28x28\n",
    "    h, w = images.shape[1], images.shape[2]\n",
    "    img = np.zeros((h * size[0], w * size[1]))#28x28 matrix of zeros\n",
    "\n",
    "    for idx, image in enumerate(images):\n",
    "        i = idx % size[1]\n",
    "        j = idx / size[1]\n",
    "        img[j*h:j*h+h, i*w:i*w+w] = image\n",
    "\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LatentAttention():\n",
    "    def __init__(self):\n",
    "        self.mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "        self.n_samples = self.mnist.train.num_examples\n",
    "        print self.n_samples\n",
    "\n",
    "        self.n_hidden = 500\n",
    "        self.n_z = 20\n",
    "        self.batchsize = 100\n",
    "\n",
    "        self.images = tf.placeholder(tf.float32, [None, 784])\n",
    "        image_matrix = tf.reshape(self.images,[-1, 28, 28, 1])\n",
    "        z_mean, z_stddev = self.recognition(image_matrix)\n",
    "        print z_mean.shape,z_mean.shape\n",
    "        samples = tf.random_normal([self.batchsize,self.n_z],0,1,dtype=tf.float32)\n",
    "        guessed_z = z_mean + (z_stddev * samples)\n",
    "        print guessed_z.shape\n",
    "\n",
    "        self.generated_images = self.generation(guessed_z)\n",
    "        generated_flat = tf.reshape(self.generated_images, [self.batchsize, 28*28])\n",
    "\n",
    "        self.generation_loss = -tf.reduce_sum(self.images * tf.log(1e-8 + generated_flat) + (1-self.images) * tf.log(1e-8 + 1 - generated_flat),1)\n",
    "\n",
    "        self.latent_loss = 0.5 * tf.reduce_sum(tf.square(z_mean) + tf.square(z_stddev) - tf.log(tf.square(z_stddev)) - 1,1)\n",
    "        self.cost = tf.reduce_mean(self.generation_loss + self.latent_loss)\n",
    "        print self.cost.shape\n",
    "        self.optimizer = tf.train.AdamOptimizer(0.001).minimize(self.cost)\n",
    "        \n",
    "        \n",
    "    def recognition(self, input_images):\n",
    "        with tf.variable_scope(\"recognition\"):\n",
    "            filter_size = 5\n",
    "            n_filters = 16\n",
    "            filter_shape = [filter_size, filter_size, 1, n_filters]\n",
    "            W1 = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))\n",
    "            b1 = tf.Variable(tf.constant(0.1, shape=[n_filters]))\n",
    "            conv1 = tf.nn.conv2d(input=input_images,\n",
    "                                         filter=W1,\n",
    "                                          strides=[1, 2, 2, 1],\n",
    "                                          padding='SAME')\n",
    "            h1=tf.nn.bias_add(conv1, b1)\n",
    "            z1 = tf.nn.relu(h1)\n",
    "            \n",
    "            \n",
    "            filter_size = 5\n",
    "            n_filters = 32\n",
    "            filter_shape = [filter_size, filter_size, 16, n_filters]\n",
    "            W2 = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))\n",
    "            b2 = tf.Variable(tf.constant(0.1, shape=[n_filters]))\n",
    "            conv2 = tf.nn.conv2d(input=z1,\n",
    "                                         filter=W2,\n",
    "                                          strides=[1, 2, 2, 1],\n",
    "                                          padding='SAME')\n",
    "            h2=tf.nn.bias_add(conv2, b2)\n",
    "            z2 = tf.nn.relu(h2)\n",
    "            \n",
    "            \n",
    "            h2_flat = tf.reshape(h2,[self.batchsize, 7*7*32])\n",
    "            print h2_flat.shape\n",
    "\n",
    "            w_mean = tf.layers.dense(h2_flat,self.n_z,activation=None,use_bias=True,bias_initializer=tf.zeros_initializer())\n",
    "            print w_mean.shape\n",
    "    \n",
    "            w_stddev = tf.layers.dense(h2_flat,self.n_z,activation=None,use_bias=True,bias_initializer=tf.zeros_initializer())\n",
    "\n",
    "        return w_mean, w_stddev\n",
    "        \n",
    "        \n",
    "    def generation(self, z):\n",
    "        with tf.variable_scope(\"generation\"):\n",
    "            z_develop=tf.layers.dense(z,7*7*32,activation=None,use_bias=True,bias_initializer=tf.zeros_initializer())\n",
    "            z_matrix = tf.nn.relu(tf.reshape(z_develop, [self.batchsize, 7, 7, 32]))\n",
    "            print z_matrix.shape\n",
    "            \n",
    "            \n",
    "            filter_size = 5\n",
    "            filter_shape = [filter_size, filter_size, 16,32]\n",
    "            W1 = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))\n",
    "            \n",
    "            conv1 = tf.nn.conv2d_transpose(z_matrix, W1,[self.batchsize, 14, 14, 16], strides=[1,2,2,1])\n",
    "            \n",
    "            \n",
    "            #b1 = tf.Variable(tf.constant(0.1, shape=[16]))\n",
    "            \n",
    "            z1 = tf.nn.relu(conv1)\n",
    "            \n",
    "            print z1.shape\n",
    "            \n",
    "            filter_size = 5\n",
    "            filter_shape = [filter_size, filter_size, 1,16]\n",
    "            W2 = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))\n",
    "            \n",
    "            conv2 = tf.nn.conv2d_transpose(z1, W2,[self.batchsize, 28, 28, 1], strides=[1,2,2,1])\n",
    "            \n",
    "            z2 = tf.nn.sigmoid(conv2)\n",
    "            \n",
    "            \n",
    "            \n",
    "        return z2\n",
    "    \n",
    "    def train(self):\n",
    "        b=self.mnist.train.next_batch(self.batchsize)\n",
    "        visualization= self.mnist.train.next_batch(self.batchsize)[0]#100 images\n",
    "        print len(b),len(visualization)\n",
    "        reshaped_vis = visualization.reshape(self.batchsize,28,28)#100x28x28\n",
    "        ims(\"results/base.jpg\",merge(reshaped_vis[:64],[8,8]))#pass 64 images\n",
    "        # train\n",
    "        saver = tf.train.Saver(max_to_keep=2)\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.initialize_all_variables())\n",
    "            for epoch in range(10):\n",
    "                for idx in range(int(self.n_samples / self.batchsize)):\n",
    "                    batch = self.mnist.train.next_batch(self.batchsize)[0]\n",
    "                    _, gen_loss, lat_loss = sess.run((self.optimizer, self.generation_loss, self.latent_loss), feed_dict={self.images: batch})\n",
    "                    # dumb hack to print cost every epoch\n",
    "                    if idx % (self.n_samples - 3) == 0:\n",
    "                        print \"epoch %d: genloss %f latloss %f\" % (epoch, np.mean(gen_loss), np.mean(lat_loss))\n",
    "                        saver.save(sess, os.getcwd()+\"/training/train\",global_step=epoch)\n",
    "                        generated_test = sess.run(self.generated_images, feed_dict={self.images: visualization})\n",
    "                        generated_test = generated_test.reshape(self.batchsize,28,28)\n",
    "                        ims(\"results/\"+str(epoch)+\".jpg\",merge(generated_test[:64],[8,8]))\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "55000\n",
      "(100, 1568)\n",
      "(100, 20)\n",
      "(100, 20) (100, 20)\n",
      "(100, 20)\n",
      "(100, 7, 7, 32)\n",
      "(100, 14, 14, 16)\n",
      "()\n"
     ]
    }
   ],
   "source": [
    "model = LatentAttention()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 100\n",
      "(64, 28, 28)\n",
      "WARNING:tensorflow:From <ipython-input-5-14d0b8525d1f>:110: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "epoch 0: genloss 546.541443 latloss 26.686228\n",
      "(64, 28, 28)\n",
      "epoch 1: genloss 114.144508 latloss 23.428110\n",
      "(64, 28, 28)\n",
      "epoch 2: genloss 96.980049 latloss 24.964941\n",
      "(64, 28, 28)\n",
      "epoch 3: genloss 94.634842 latloss 24.394951\n",
      "(64, 28, 28)\n",
      "epoch 4: genloss 90.282257 latloss 24.470772\n",
      "(64, 28, 28)\n",
      "epoch 5: genloss 87.564743 latloss 23.690863\n",
      "(64, 28, 28)\n",
      "epoch 6: genloss 94.319870 latloss 24.259581\n",
      "(64, 28, 28)\n",
      "epoch 7: genloss 84.751640 latloss 24.703306\n",
      "(64, 28, 28)\n",
      "epoch 8: genloss 87.313240 latloss 24.946445\n",
      "(64, 28, 28)\n",
      "epoch 9: genloss 84.469246 latloss 24.784950\n",
      "(64, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}