Seminar4_TF_iris.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Structure your code with names\n",
    "\n",
    "https://docs.google.com/presentation/d/1SJMsa4BdOFVRCPD9uwaAqDBYYfgbQcbOm7HaxRVpaaY/edit\n",
    "\n",
    "see images\n",
    "\n",
    "1 упрощает чтение кода\n",
    "\n",
    "2 граф становится более читаемым\n",
    "\n",
    "3 не нужно писать длинные имена переменных"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shared Variables\n",
    "\n",
    "used for\n",
    "1. input, output embedding\n",
    "2. differeent model for train and test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for using only cpu (not gpu)\n",
    "\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load iris dataset (Ирисы Фишера)\n",
    "\n",
    "\n",
    "TRAIN_URL = \"http://download.tensorflow.org/data/iris_training.csv\"\n",
    "TEST_URL = \"http://download.tensorflow.org/data/iris_test.csv\"\n",
    "\n",
    "CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth',\n",
    "                    'PetalLength', 'PetalWidth', 'Species']\n",
    "SPECIES = ['Setosa', 'Versicolor', 'Virginica']\n",
    "\n",
    "def maybe_download():\n",
    "    train_path = tf.keras.utils.get_file(TRAIN_URL.split('/')[-1], TRAIN_URL)\n",
    "    test_path = tf.keras.utils.get_file(TEST_URL.split('/')[-1], TEST_URL)\n",
    "\n",
    "    return train_path, test_path\n",
    "\n",
    "def load_data(y_name='Species'):\n",
    "    \"\"\"Returns the iris dataset as (train_x, train_y), (test_x, test_y).\"\"\"\n",
    "    train_path, test_path = maybe_download()\n",
    "\n",
    "    train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)\n",
    "    train_x, train_y = train, train.pop(y_name)\n",
    "\n",
    "    test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)\n",
    "    test_x, test_y = test, test.pop(y_name)\n",
    "\n",
    "    return (train_x, train_y), (test_x, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(train_x, train_y), (test_x, test_y) = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create generator\n",
    "def next_batch(X_data, Y_data, batch_size=10):\n",
    "    length = X_data.shape[0]\n",
    "    reduced_length = (length // batch_size) * batch_size\n",
    "    \n",
    "    X_data = X_data[:reduced_length].reshape([batch_size, -1, 4])\n",
    "    Y_data = Y_data[:reduced_length].reshape([batch_size, -1]) \n",
    "    \n",
    "    X_data = np.transpose(X_data, axes=(1, 0, 2))\n",
    "    Y_data = np.transpose(Y_data, axes=(1, 0))\n",
    "    for x_i, y_i in zip(X_data, Y_data):\n",
    "        yield x_i, y_i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create placeholders\n",
    "batch_size = 10\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[batch_size, 4], name = 'input')\n",
    "Y = tf.placeholder(tf.int32, shape=[batch_size], name = 'output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create simple network\n",
    "\n",
    "layer_size = 50\n",
    "\n",
    "# initialization - http://cs231n.github.io/neural-networks-2/#init\n",
    "initializer = tf.random_normal_initializer(stddev=np.sqrt(2/layer_size), mean=0.0)\n",
    "\n",
    "# create weight and bias\n",
    "with tf.variable_scope(name_or_scope='layer_1'):\n",
    "    w1 = tf.get_variable('weights', shape=[4,layer_size], initializer=initializer)\n",
    "    b1 = tf.get_variable('bias', shape=[layer_size], initializer=initializer)\n",
    "    output_from_layer = tf.nn.relu(tf.matmul(X, w1) + b1)\n",
    "\n",
    "    \n",
    "with tf.variable_scope(name_or_scope='layer_2'):\n",
    "    w2 = tf.get_variable('weights', shape=[layer_size,3], initializer=initializer)\n",
    "    b2 = tf.get_variable('bias', shape=[3], initializer=initializer)\n",
    "    Y_predicted = tf.matmul(output_from_layer, w2) + b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# use tf.reduce_mean for batches\n",
    "loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = Y_predicted, labels = Y), name='loss')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "нарисуем граф"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# choose optimizer to minimize loss\n",
    "optimizer = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TAKE SUMMARY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# make summaries\n",
    "tf.summary.scalar('loss', loss)\n",
    "tf.summary.histogram('weights1', w1)\n",
    "tf.summary.histogram('bias1', b1)\n",
    "\n",
    "merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for name of summaries\n",
    "\n",
    "from datetime import datetime\n",
    "def time_str():\n",
    "    return datetime.now().replace(microsecond=0).isoformat().replace(':','-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "в граф добавились узлы оптимизации"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: 7.0857910215854645\n",
      "Epoch 19: 4.9306690990924835\n",
      "Epoch 29: 3.757598102092743\n",
      "Epoch 39: 2.9656500667333603\n",
      "Epoch 49: 2.40960706025362\n",
      "Epoch 59: 2.0175601691007614\n",
      "Epoch 69: 1.7376424074172974\n",
      "Epoch 79: 1.5337669402360916\n",
      "Epoch 89: 1.3816078826785088\n",
      "Epoch 99: 1.2654829286038876\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "with tf.Session() as sess:\n",
    "    # initialize the necessary variables\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    train_writer = tf.summary.FileWriter('./iris/summuries%s' %time_str(), sess.graph)\n",
    "    \n",
    "    # train the model for 100 epochs\n",
    "    for i in range(100):\n",
    "        total_loss = 0\n",
    "        for x_i, y_i in next_batch(train_x.values, train_y.values):\n",
    "            summary, _, l = sess.run(\n",
    "                [merged,\n",
    "                 optimizer, \n",
    "                 loss],feed_dict= \n",
    "                {X:x_i, \n",
    "                 Y:y_i}) \n",
    "            \n",
    "            total_loss += l  \n",
    "        train_writer.add_summary(summary, global_step=i)\n",
    "        if (i + 1) % 10 == 0:\n",
    "            print('Epoch {0}: {1}'.format(i, total_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "see summaries in TensorBoard\n",
    "\n",
    "for example: tensorboard --logdir=\"./iris/summuries2018-03-29T14-08-35/\" --port 6006\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test----------------------------------------\n",
      "predicted -- true\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "1 -- 2\n",
      "1 -- 2\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "0 -- 1\n",
      "1 -- 1\n",
      "1 -- 2\n",
      "1 -- 1\n",
      "1 -- 2\n",
      "1 -- 2\n",
      "1 -- 1\n",
      "0 -- 1\n",
      "1 -- 2\n",
      "1 -- 2\n",
      "1 -- 1\n",
      "1 -- 2\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "0 -- 1\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "0 -- 1\n"
     ]
    }
   ],
   "source": [
    "# test our model\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    print ('test' + '--'*20)\n",
    "    print ('predicted -- true')\n",
    "\n",
    "    for x_i, y_i in next_batch(test_x.values, test_y.values):\n",
    "        Y_pred = sess.run(\n",
    "            [Y_predicted],\n",
    "            feed_dict= \n",
    "            {X:x_i})\n",
    "        for index_i, Y_in_batch in enumerate(Y_pred[0]):\n",
    "#             print (Y_in_batch)\n",
    "            print ('{0} -- {1}'.format(np.argsort(Y_in_batch)[-1], y_i[index_i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# why so bad results?\n",
    "# we test a network with random value in layers! because we initialized it again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: 7.24475422501564\n",
      "Epoch 19: 5.384702891111374\n",
      "Epoch 29: 4.360525518655777\n",
      "Epoch 39: 3.5466431379318237\n",
      "Epoch 49: 2.910990431904793\n",
      "Epoch 59: 2.433864362537861\n",
      "Epoch 69: 2.080517955124378\n",
      "Epoch 79: 1.818356692790985\n",
      "Epoch 89: 1.621521070599556\n",
      "Epoch 99: 1.4709897227585316\n",
      "test----------------------------------------\n",
      "predicted -- true\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "2 -- 2\n",
      "2 -- 2\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "2 -- 2\n",
      "1 -- 1\n",
      "1 -- 2\n",
      "2 -- 2\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "2 -- 2\n",
      "2 -- 2\n",
      "1 -- 1\n",
      "2 -- 2\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "2 -- 1\n",
      "0 -- 0\n",
      "1 -- 1\n",
      "Took: 1.133533 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for i in range(100):\n",
    "        total_loss = 0\n",
    "        for x_i, y_i in next_batch(train_x.values, train_y.values):\n",
    "            _, l = sess.run(\n",
    "                [optimizer, \n",
    "                 loss],\n",
    "                feed_dict= \n",
    "                {X:x_i, \n",
    "                 Y:y_i}) \n",
    "            \n",
    "            total_loss += l  \n",
    "        if (i + 1) % 10 == 0:\n",
    "            print('Epoch {0}: {1}'.format(i, total_loss))\n",
    "            \n",
    "    print ('test' + '--'*20)\n",
    "    print ('predicted -- true')\n",
    "\n",
    "    for x_i, y_i in next_batch(test_x.values, test_y.values):\n",
    "        Y_pred = sess.run(\n",
    "            [Y_predicted],\n",
    "            feed_dict= \n",
    "            {X:x_i})\n",
    "        for index_i, Y_in_batch in enumerate(Y_pred[0]):\n",
    "#             print (Y_in_batch)\n",
    "            print ('{0} -- {1}'.format(np.argsort(Y_in_batch)[-1], y_i[index_i]))\n",
    "\n",
    "print('Took: %f seconds' %(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save  and restore model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "tf.train.Saver.save(sess, save_path, global_step=None...)\n",
    "\n",
    "\n",
    "tf.train.Saver.restore(sess, save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver(max_to_keep=50)\n",
    "# saver = tf.train.Saver({'variable': v1, 'v2': v2})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: 9.786037623882294\n",
      "Epoch 9: 7.0703694224357605\n",
      "Epoch 14: 5.780054569244385\n",
      "Epoch 19: 5.004405647516251\n",
      "Epoch 24: 4.439634948968887\n",
      "Epoch 29: 3.97695292532444\n",
      "Epoch 34: 3.581219270825386\n",
      "Epoch 39: 3.232933893799782\n",
      "Epoch 44: 2.9260697960853577\n",
      "Epoch 49: 2.6579074561595917\n",
      "Epoch 54: 2.4251612573862076\n",
      "Epoch 59: 2.224899247288704\n",
      "Epoch 64: 2.053098790347576\n",
      "Epoch 69: 1.9058754295110703\n",
      "Epoch 74: 1.7795924842357635\n",
      "Epoch 79: 1.670658115297556\n",
      "Epoch 84: 1.5765843093395233\n",
      "Epoch 89: 1.4948071464896202\n",
      "Epoch 94: 1.4234918057918549\n",
      "Epoch 99: 1.3609264269471169\n",
      "Took: 1.923270 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "with tf.Session() as sess:\n",
    "    # Step 7: initialize the necessary variables, in this case, w and b\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # Step 8: train the model for 100 epochs\n",
    "    for i in range(100):\n",
    "        total_loss = 0\n",
    "        for x_i, y_i in next_batch(train_x.values, train_y.values, batch_size=10):\n",
    "            _, l = sess.run(\n",
    "                [optimizer, \n",
    "                 loss], \n",
    "                {X:x_i, \n",
    "                 Y:y_i}) \n",
    "            \n",
    "            total_loss += l  \n",
    "        if (i + 1) % 5 == 0:\n",
    "            print('Epoch {0}: {1}'.format(i, total_loss))\n",
    "            saver.save(sess, \n",
    "            './iris/model_network/model',\n",
    "            global_step=i)\n",
    "\n",
    "\n",
    "print('Took: %f seconds' %(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./iris/model_network/model-4\n",
      "test----------------------------------------\n",
      "predicted -- true\n",
      "2 -- 1\n",
      "2 -- 1\n",
      "0 -- 0\n",
      "2 -- 2\n",
      "2 -- 2\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "2 -- 1\n",
      "2 -- 1\n",
      "2 -- 1\n",
      "2 -- 2\n",
      "2 -- 1\n",
      "2 -- 2\n",
      "2 -- 2\n",
      "2 -- 1\n",
      "2 -- 1\n",
      "2 -- 2\n",
      "2 -- 2\n",
      "2 -- 1\n",
      "2 -- 2\n",
      "0 -- 0\n",
      "2 -- 1\n",
      "2 -- 1\n",
      "0 -- 0\n",
      "2 -- 1\n",
      "0 -- 0\n",
      "0 -- 0\n",
      "2 -- 1\n",
      "0 -- 0\n",
      "2 -- 1\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver.restore(sess, './iris/model_network/model-4')\n",
    "    \n",
    "    print ('test' + '--'*20)\n",
    "    print ('predicted -- true')\n",
    "\n",
    "    for x_i, y_i in next_batch(test_x.values, test_y.values):\n",
    "        Y_pred = sess.run(\n",
    "            [Y_predicted],\n",
    "            feed_dict= \n",
    "            {X:x_i})\n",
    "        for index_i, Y_in_batch in enumerate(Y_pred[0]):\n",
    "#             print (Y_in_batch)\n",
    "            print ('{0} -- {1}'.format(np.argsort(Y_in_batch)[-1], y_i[index_i]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (whatever you want to call it)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}