ch09-tensorflow-setup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Intro\n",
    "* Graphs defined in Python, executed in C++\n",
    "* Open-sourced 2015. Windows, Linux, macOS, iOS, Android\n",
    "* Python API: **tensorflow.contrib.learn** (trains NNs)\n",
    "* Simpler API: **tensorflow.contrib.slim** (simplifies building NNs)\n",
    "* Other high-level APIs: **[Keras](http://keras.io)**, **[Pretty Tensor](https://github.com/google/prettytensor/)**.\n",
    "* Libraries: **Caffe**, **DeepLearning4J**, **H2O**, **MXNet**, **Theano**, **Torch**\n",
    "* **TensorBoard** visualization tool\n",
    "* [Cloud service](https://cloud.google.com/ml)\n",
    "* Resources: [home page](https://www.tensorflow.org/), [GitHub](https://github.com/jtoy/awesome-tensorflow), [StackOverflow](http://stackoverflow.com/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Installation & Test\n",
    "\n",
    "$ cd <your working directory>\n",
    "\n",
    "$ source env/bin/activate (if using virtualenv)\n",
    "\n",
    "$ pip3 install --upgrade tensorflow (or tensorflow-gpu for GPU support)\n",
    "\n",
    "$ python3 -c 'import tensorflow; print(tensorflow.__version__)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!python3 -c 'import tensorflow; print(tensorflow.__version__)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "# create your first graph\n",
    "import tensorflow as tf\n",
    "x = tf.Variable(3, name=\"x\")\n",
    "y = tf.Variable(4, name=\"y\")\n",
    "f = x*x*y + y + 2\n",
    "\n",
    "# run graph by opening a session\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(x.initializer)\n",
    "sess.run(y.initializer)\n",
    "result = sess.run(f)\n",
    "print(result)\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "# for repeated session \"runs\"\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    x.initializer.run()\n",
    "    y.initializer.run()\n",
    "    result = f.eval()\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "# use global_variables_initializer() to set up initialization\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run() # actually initialize all the variables\n",
    "    result = f.eval()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "# interactive sessions (from within Jupyter or Python shell)\n",
    "# interactive sesions are auto-set as default sessions\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "init.run()\n",
    "result = f.eval()\n",
    "print(result)\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Managing Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# any created node = added to default graph\n",
    "x1 = tf.Variable(1)\n",
    "x1.graph is tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, False)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# handling multiple graphs\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    x2 = tf.Variable(2)\n",
    "\n",
    "x2.graph is graph, x2.graph is tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Node Lifecycles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "15\n",
      "10\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# TF finds node's dependencies & evaluates them first\n",
    "\n",
    "w = tf.constant(3)\n",
    "x = w + 2\n",
    "y = x + 5\n",
    "z = x * 3\n",
    "\n",
    "# previous eval results = NOT reused. above code evals w & x twice.\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print(y.eval())\n",
    "    print(z.eval())\n",
    "    \n",
    "# a more efficient evaluation call:\n",
    "with tf.Session() as sess:\n",
    "    y_val, z_val = sess.run([y,z])\n",
    "    print(y_val)\n",
    "    print(z_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression with TF\n",
    "* TF ops take any number of inputs & produce any number of outputs\n",
    "* Constants & variables = source ops (no inputs)\n",
    "* Inputs & outputs = multidimensional \"tensors\" = **NumPy ndarrays** in Python API. Typically floats, can also be strings.\n",
    "\n",
    "#### Below: Linear Regression on 2D arrays (California Housing dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  1.,  1.],\n",
       "       [ 1.,  0.,  0.,  0.,  0.,  1.,  1.]])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (never used Numpy c_ operator before. Had to check it out.)\n",
    "a = np.ones((6,1))\n",
    "b = np.zeros((6,4))\n",
    "c = np.ones((6,2))\n",
    "np.c_[a,b,c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20640 8 (20640, 9)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "import numpy as np\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "m, n = housing.data.shape\n",
    "\n",
    "# add bias feature, x0 = 1\n",
    "housing_data_plus_bias = np.c_[np.ones((m, 1)), housing.data]\n",
    "\n",
    "print(m,n,housing_data_plus_bias.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shape:  (20640, 9)\n",
      "XT shape:  (9, 20640)\n",
      "theta: \n",
      " [[ -3.69419202e+01]\n",
      " [  4.36693293e-01]\n",
      " [  9.43577803e-03]\n",
      " [ -1.07322041e-01]\n",
      " [  6.45065694e-01]\n",
      " [ -3.97638942e-06]\n",
      " [ -3.78654265e-03]\n",
      " [ -4.21314378e-01]\n",
      " [ -4.34513755e-01]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.constant(\n",
    "    housing_data_plus_bias,        \n",
    "    dtype=tf.float64, name=\"X\")\n",
    "\n",
    "print(\"X shape: \",X.shape)\n",
    "\n",
    "# housing.target = 1D array. Reshape to col vector to compute theta.\n",
    "# reshape() accepts -1 = \"unspecified\" for a dimension.\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float64, name=\"y\")\n",
    "\n",
    "XT = tf.transpose(X)\n",
    "\n",
    "print(\"XT shape: \",XT.shape)\n",
    "\n",
    "# normal equation: theta = (XT * X)^-1 * XT * y\n",
    "\n",
    "theta = tf.matmul(\n",
    "            tf.matmul(\n",
    "                tf.matrix_inverse(\n",
    "                    tf.matmul(XT, X)), \n",
    "                    XT), \n",
    "                y)\n",
    "\n",
    "# TF doesn't immediately run the code. It creates nodes that will run with eval().\n",
    "# TF will auto-run on GPU if available.\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    result = theta.eval()\n",
    "\n",
    "print(\"theta: \\n\",result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "theta: \n",
      " [[ -3.69419202e+01]\n",
      " [  4.36693293e-01]\n",
      " [  9.43577803e-03]\n",
      " [ -1.07322041e-01]\n",
      " [  6.45065694e-01]\n",
      " [ -3.97638942e-06]\n",
      " [ -3.78654265e-03]\n",
      " [ -4.21314378e-01]\n",
      " [ -4.34513755e-01]]\n"
     ]
    }
   ],
   "source": [
    "# compare to pure NumPy\n",
    "X = housing_data_plus_bias\n",
    "y = housing.target.reshape(-1, 1)\n",
    "\n",
    "theta_numpy = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y)\n",
    "\n",
    "print(\"theta: \\n\",theta_numpy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "theta: \n",
      " [[ -3.69419202e+01]\n",
      " [  4.36693293e-01]\n",
      " [  9.43577803e-03]\n",
      " [ -1.07322041e-01]\n",
      " [  6.45065694e-01]\n",
      " [ -3.97638942e-06]\n",
      " [ -3.78654265e-03]\n",
      " [ -4.21314378e-01]\n",
      " [ -4.34513755e-01]]\n"
     ]
    }
   ],
   "source": [
    "# compare to Scikit\n",
    "from sklearn.linear_model import LinearRegression\n",
    "lin_reg = LinearRegression()\n",
    "\n",
    "lin_reg.fit(\n",
    "    housing.data, \n",
    "    housing.target.reshape(-1, 1))\n",
    "\n",
    "print(\"theta: \\n\",np.r_[\n",
    "    lin_reg.intercept_.reshape(-1, 1), \n",
    "    lin_reg.coef_.T])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch Gradient Descent (instead of Normal Equation):\n",
    "* Could use TF; let's use Scikit first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 20640 entries, 0 to 20639\n",
      "Data columns (total 9 columns):\n",
      "0    20640 non-null float64\n",
      "1    20640 non-null float64\n",
      "2    20640 non-null float64\n",
      "3    20640 non-null float64\n",
      "4    20640 non-null float64\n",
      "5    20640 non-null float64\n",
      "6    20640 non-null float64\n",
      "7    20640 non-null float64\n",
      "8    20640 non-null float64\n",
      "dtypes: float64(9)\n",
      "memory usage: 1.4 MB\n"
     ]
    }
   ],
   "source": [
    "# normalize input features first - otherwise training = much slower.\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "\n",
    "scaled_housing_data = scaler.fit_transform(\n",
    "    housing.data)\n",
    "\n",
    "scaled_housing_data_plus_bias = np.c_[\n",
    "    np.ones((m, 1)), \n",
    "    scaled_housing_data]\n",
    "\n",
    "import pandas as pd\n",
    "pd.DataFrame(scaled_housing_data_plus_bias).info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean (axis=0): \n",
      " [  1.00000000e+00   6.60969987e-17   5.50808322e-18   6.60969987e-17\n",
      "  -1.06030602e-16  -1.10161664e-17   3.44255201e-18  -1.07958431e-15\n",
      "  -8.52651283e-15]\n",
      "mean (axis=1): \n",
      " [ 0.38915536  0.36424355  0.5116157  ..., -0.06612179 -0.06360587\n",
      "  0.01359031]\n",
      "mean (w/bias): \n",
      " 0.111111111111\n",
      "data shape:    \n",
      " (20640, 9)\n"
     ]
    }
   ],
   "source": [
    "print(\"mean (axis=0): \\n\",scaled_housing_data_plus_bias.mean(axis=0))\n",
    "print(\"mean (axis=1): \\n\",scaled_housing_data_plus_bias.mean(axis=1))\n",
    "print(\"mean (w/bias): \\n\",scaled_housing_data_plus_bias.mean())\n",
    "print(\"data shape:    \\n\",scaled_housing_data_plus_bias.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual gradient computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 2.75443\n",
      "Epoch 100 MSE = 0.632222\n",
      "Epoch 200 MSE = 0.57278\n",
      "Epoch 300 MSE = 0.558501\n",
      "Epoch 400 MSE = 0.549069\n",
      "Epoch 500 MSE = 0.542288\n",
      "Epoch 600 MSE = 0.537379\n",
      "Epoch 700 MSE = 0.533822\n",
      "Epoch 800 MSE = 0.531243\n",
      "Epoch 900 MSE = 0.529371\n",
      "Best theta: \n",
      " [[  2.06855226e+00]\n",
      " [  7.74078071e-01]\n",
      " [  1.31192386e-01]\n",
      " [ -1.17845096e-01]\n",
      " [  1.64778158e-01]\n",
      " [  7.44080753e-04]\n",
      " [ -3.91945168e-02]\n",
      " [ -8.61356616e-01]\n",
      " [ -8.23479712e-01]]\n"
     ]
    }
   ],
   "source": [
    "# batch gradient step:\n",
    "# theta(next) = theta - learning_rate * MSE(theta)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(\n",
    "    scaled_housing_data_plus_bias, \n",
    "    dtype=tf.float32, name=\"X\")\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float32, name=\"y\")\n",
    "\n",
    "theta = tf.Variable( # tf.random_uniform = generates random tensor\n",
    "    tf.random_uniform([n+1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "\n",
    "error       = y_pred - y\n",
    "mse         = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "gradients   = 2/m * tf.matmul(tf.transpose(X), error)\n",
    "training_op = tf.assign(theta, theta - learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0: # do every 100th epoch:\n",
    "            print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"Best theta: \\n\",best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using autodiff\n",
    "* automatically finds gradients. Note the different gradients assignment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 2.75443\n",
      "Epoch 100 MSE = 0.632222\n",
      "Epoch 200 MSE = 0.57278\n",
      "Epoch 300 MSE = 0.558501\n",
      "Epoch 400 MSE = 0.549069\n",
      "Epoch 500 MSE = 0.542288\n",
      "Epoch 600 MSE = 0.537379\n",
      "Epoch 700 MSE = 0.533822\n",
      "Epoch 800 MSE = 0.531243\n",
      "Epoch 900 MSE = 0.529371\n",
      "Best theta: \n",
      " [[  2.06855249e+00]\n",
      " [  7.74078071e-01]\n",
      " [  1.31192386e-01]\n",
      " [ -1.17845066e-01]\n",
      " [  1.64778143e-01]\n",
      " [  7.44078017e-04]\n",
      " [ -3.91945094e-02]\n",
      " [ -8.61356676e-01]\n",
      " [ -8.23479772e-01]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(\n",
    "    scaled_housing_data_plus_bias, \n",
    "    dtype=tf.float32, name=\"X\")\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float32, name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n + 1, 1], -1.0, 1.0, \n",
    "                      seed=42), name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "\n",
    "error       = y_pred - y\n",
    "mse         = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "\n",
    "# AutoDiff to the rescue\n",
    "# creates list of ops, one/variable, to find gradients per variable\n",
    "gradients   = tf.gradients(mse, [theta])[0]\n",
    "#\n",
    "\n",
    "training_op = tf.assign(theta, theta - learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"Best theta: \\n\", best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Four ways to run autodiff - see Appendix D\n",
    "* reverse-mode (default): best for many inputs, few outputs\n",
    "* symbolic diff: high accuracy\n",
    "* forward mode: high accuracy\n",
    "* numerical diff: low accuracy, but trivial to implement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using a predefined optimizer (Gradient Descent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 2.75443\n",
      "Epoch 100 MSE = 0.632222\n",
      "Epoch 200 MSE = 0.57278\n",
      "Epoch 300 MSE = 0.558501\n",
      "Epoch 400 MSE = 0.549069\n",
      "Epoch 500 MSE = 0.542288\n",
      "Epoch 600 MSE = 0.537379\n",
      "Epoch 700 MSE = 0.533822\n",
      "Epoch 800 MSE = 0.531243\n",
      "Epoch 900 MSE = 0.529371\n",
      "Best theta:\n",
      " [[  2.06855249e+00]\n",
      " [  7.74078071e-01]\n",
      " [  1.31192386e-01]\n",
      " [ -1.17845066e-01]\n",
      " [  1.64778143e-01]\n",
      " [  7.44078017e-04]\n",
      " [ -3.91945094e-02]\n",
      " [ -8.61356676e-01]\n",
      " [ -8.23479772e-01]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(\n",
    "    scaled_housing_data_plus_bias, \n",
    "    dtype=tf.float32, name=\"X\")\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float32, name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "\n",
    "error       = y_pred - y\n",
    "mse         = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "\n",
    "#####\n",
    "optimizer   = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "#####\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"Best theta:\\n\", best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using a predefined optimizer (Momentum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best theta:\n",
      " [[  2.06855392e+00]\n",
      " [  7.94067979e-01]\n",
      " [  1.25333667e-01]\n",
      " [ -1.73580602e-01]\n",
      " [  2.18767926e-01]\n",
      " [ -1.64708309e-03]\n",
      " [ -3.91250364e-02]\n",
      " [ -8.85289013e-01]\n",
      " [ -8.50607991e-01]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(\n",
    "    scaled_housing_data_plus_bias, \n",
    "    dtype=tf.float32, name=\"X\")\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float32, name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred      = tf.matmul(X, theta, name=\"predictions\")\n",
    "error       = y_pred - y\n",
    "mse         = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "#####\n",
    "optimizer   = tf.train.MomentumOptimizer(\n",
    "    learning_rate=learning_rate, \n",
    "    momentum=0.25)\n",
    "#####\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"Best theta:\\n\", best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training & Data Feeds\n",
    "* Goal: modify previous code for Minibatch gradient descent\n",
    "* Best practice: *placeholder* nodes (no computation, just data output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 6.  7.  8.]] \n",
      " [[  9.  10.  11.]\n",
      " [ 12.  13.  14.]]\n"
     ]
    }
   ],
   "source": [
    "A = tf.placeholder(tf.float32, shape=(None, 3))\n",
    "B = A + 5\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    B_val_1 = B.eval(\n",
    "        feed_dict={A: [[1, 2, 3]]})\n",
    "    \n",
    "    B_val_2 = B.eval(\n",
    "        feed_dict={A: [[4, 5, 6], [7, 8, 9]]})\n",
    "    \n",
    "print(B_val_1, \"\\n\", B_val_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# definition phase: change X,y to placeholder nodes\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "##########\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n+1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1),   name=\"y\")\n",
    "\n",
    "##########\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n+1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "\n",
    "error  = y_pred - y\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate)\n",
    "\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m:  20640 \n",
      " n_batches:  207 \n",
      "\n",
      "Best theta: \n",
      " [[ 2.07001591]\n",
      " [ 0.82045609]\n",
      " [ 0.1173173 ]\n",
      " [-0.22739051]\n",
      " [ 0.31134021]\n",
      " [ 0.00353193]\n",
      " [-0.01126994]\n",
      " [-0.91643935]\n",
      " [-0.87950081]]\n"
     ]
    }
   ],
   "source": [
    "# execution phase: fetch minibatches one-by-one. \n",
    "# use feed_dict to provide values to dependent nodes\n",
    "\n",
    "import numpy.random as rnd\n",
    "\n",
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "print(\"m: \",m,\"\\n\",\"n_batches: \",n_batches,\"\\n\")\n",
    "\n",
    "def fetch_batch(epoch, batch_index, batch_size):\n",
    "    \n",
    "    rnd.seed(epoch * n_batches + batch_index)\n",
    "    indices = rnd.randint(m, size=batch_size)\n",
    "    \n",
    "    X_batch = scaled_housing_data_plus_bias[indices]\n",
    "    y_batch = housing.target.reshape(-1, 1)[indices]\n",
    "    \n",
    "    return X_batch, y_batch\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            \n",
    "            X_batch, y_batch = fetch_batch(\n",
    "                epoch, batch_index, batch_size)\n",
    "            \n",
    "            sess.run(\n",
    "                training_op, \n",
    "                feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()\n",
    "    \n",
    "print(\"Best theta: \\n\",best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Save/Restore\n",
    "* Use a *saver* node once construction is complete.\n",
    "* call **save()** method - pass it session and filepath info."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(\n",
    "    scaled_housing_data_plus_bias, \n",
    "    dtype=tf.float32, name=\"X\")\n",
    "\n",
    "y = tf.constant(\n",
    "    housing.target.reshape(-1, 1), \n",
    "    dtype=tf.float32, name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n+1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "\n",
    "error = y_pred - y\n",
    "\n",
    "mse = tf.reduce_mean(\n",
    "    tf.square(error), \n",
    "    name=\"mse\")\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate)\n",
    "\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "# can specify which vars to save:\n",
    "# saver = tf.train.Saver({\"weights\": theta})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 2.75443\n",
      "Epoch 100 MSE = 0.632222\n",
      "Epoch 200 MSE = 0.57278\n",
      "Epoch 300 MSE = 0.558501\n",
      "Epoch 400 MSE = 0.549069\n",
      "Epoch 500 MSE = 0.542288\n",
      "Epoch 600 MSE = 0.537379\n",
      "Epoch 700 MSE = 0.533822\n",
      "Epoch 800 MSE = 0.531243\n",
      "Epoch 900 MSE = 0.529371\n",
      "Best theta:\n",
      " [[  2.06855249e+00]\n",
      " [  7.74078071e-01]\n",
      " [  1.31192386e-01]\n",
      " [ -1.17845066e-01]\n",
      " [  1.64778143e-01]\n",
      " [  7.44078017e-04]\n",
      " [ -3.91945094e-02]\n",
      " [ -8.61356676e-01]\n",
      " [ -8.23479772e-01]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"Epoch\", epoch, \"MSE =\", mse.eval())\n",
    "            save_path = saver.save(sess, \"/tmp/my_model.ckpt\")\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")\n",
    "\n",
    "print(\"Best theta:\\n\",best_theta)\n",
    "\n",
    "# model restoration:\n",
    "# 1) create Saver at end of construction phase\n",
    "# 2) call saver.restore() at start of execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_checkpoint_path: \"my_model_final.ckpt\"\r\n",
      "all_model_checkpoint_paths: \"/tmp/my_model.ckpt\"\r\n",
      "all_model_checkpoint_paths: \"my_model_final.ckpt\"\r\n"
     ]
    }
   ],
   "source": [
    "!cat checkpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization - inside Jupyter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "        \n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    \n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    # original was width=1200px, height=620px\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.1784179106002547&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 20640\\n          }\\n          dim {\\n            size: 9\\n          }\\n        }\\n        tensor_content: &quot;<stripped 743040 bytes>&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 20640\\n          }\\n          dim {\\n            size: 1\\n          }\\n        }\\n        tensor_content: &quot;<stripped 82560 bytes>&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\t\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 87654321\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;random_uniform/max&quot;\\n  input: &quot;random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;random_uniform/RandomUniform&quot;\\n  input: &quot;random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;random_uniform/mul&quot;\\n  input: &quot;random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 9\\n        }\\n        dim {\\n          size: 1\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;theta&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;predictions&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;theta/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;predictions&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Square&quot;\\n  op: &quot;Square&quot;\\n  input: &quot;sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\000\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;mse&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;Square&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\001\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/mse_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Tile/multiples&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\240P\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/mse_grad/Reshape&quot;\\n  input: &quot;gradients/mse_grad/Tile/multiples&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\240P\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/mse_grad/Shape&quot;\\n  input: &quot;gradients/mse_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/mse_grad/Shape_1&quot;\\n  input: &quot;gradients/mse_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;gradients/mse_grad/Prod_1&quot;\\n  input: &quot;gradients/mse_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;gradients/mse_grad/Prod&quot;\\n  input: &quot;gradients/mse_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;gradients/mse_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;gradients/mse_grad/Tile&quot;\\n  input: &quot;gradients/mse_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/mul/x&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^gradients/mse_grad/truediv&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 2.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/Square_grad/mul/x&quot;\\n  input: &quot;sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/mse_grad/truediv&quot;\\n  input: &quot;gradients/Square_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\240P\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\240P\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;gradients/sub_grad/Shape&quot;\\n  input: &quot;gradients/sub_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/Square_grad/mul_1&quot;\\n  input: &quot;gradients/sub_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/sub_grad/Sum&quot;\\n  input: &quot;gradients/sub_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/Square_grad/mul_1&quot;\\n  input: &quot;gradients/sub_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Neg&quot;\\n  op: &quot;Neg&quot;\\n  input: &quot;gradients/sub_grad/Sum_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/sub_grad/Neg&quot;\\n  input: &quot;gradients/sub_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/sub_grad/Reshape&quot;\\n  input: &quot;^gradients/sub_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/sub_grad/Reshape&quot;\\n  input: &quot;^gradients/sub_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/sub_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/sub_grad/Reshape_1&quot;\\n  input: &quot;^gradients/sub_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/sub_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  input: &quot;theta/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/predictions_grad/MatMul&quot;\\n  input: &quot;^gradients/predictions_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/predictions_grad/MatMul&quot;\\n  input: &quot;^gradients/predictions_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/predictions_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/predictions_grad/MatMul_1&quot;\\n  input: &quot;^gradients/predictions_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/predictions_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_theta/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/predictions_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^GradientDescent/update_theta/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^theta/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;theta&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.1784179106002547&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization - using TensorBoard\n",
    "\n",
    "* Start **TensorBoard**: $ tensorboard --logdir tf_logs/ (starts on localhost:6006)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Need a logging directory for TB data\n",
    "# with timestamps to avoid mixing runs together\n",
    "\n",
    "from datetime import datetime\n",
    "now         = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir      = \"{}/run-{}/\".format(root_logdir, now)\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, n + 1), \n",
    "    name=\"X\")\n",
    "\n",
    "y = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, 1), \n",
    "    name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, name=\"predictions\")\n",
    "    \n",
    "error = y_pred - y\n",
    "\n",
    "mse = tf.reduce_mean(\n",
    "    tf.square(error), \n",
    "    name=\"mse\")\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate)\n",
    "\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "mse_summary = tf.summary.scalar('MSE', mse)\n",
    "\n",
    "# FileWriter - creates logdir if not already present,\n",
    "# then writes graph def to a binary logfile.\n",
    "\n",
    "summary_writer = tf.summary.FileWriter(\n",
    "    logdir, \n",
    "    tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best theta:\n",
      "[[ 2.07001591]\n",
      " [ 0.82045609]\n",
      " [ 0.1173173 ]\n",
      " [-0.22739051]\n",
      " [ 0.31134021]\n",
      " [ 0.00353193]\n",
      " [-0.01126994]\n",
      " [-0.91643935]\n",
      " [-0.87950081]]\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            \n",
    "            X_batch, y_batch = fetch_batch(\n",
    "                epoch, \n",
    "                batch_index, \n",
    "                batch_size)\n",
    "            \n",
    "            # evaluate mse_summary on periodic basis,\n",
    "            # eg every 10 minibatches.\n",
    "            # adds summary for addition to events file.\n",
    "            \n",
    "            if batch_index % 10 == 0:\n",
    "                summary_str = mse_summary.eval(\n",
    "                    feed_dict={X: X_batch, y: y_batch})\n",
    "                \n",
    "                step = epoch * n_batches + batch_index\n",
    "                \n",
    "                summary_writer.add_summary(\n",
    "                    summary_str, \n",
    "                    step)\n",
    "                \n",
    "            sess.run(\n",
    "                training_op, \n",
    "                feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()\n",
    "\n",
    "summary_writer.flush()\n",
    "summary_writer.close()\n",
    "print(\"Best theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#!ls tf_logs/run*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Name Scopes\n",
    "* Graphs can contain thousands of nodes. *name scopes* group related nodes to aid visualization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, n + 1), \n",
    "    name=\"X\")\n",
    "\n",
    "y = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, 1), \n",
    "    name=\"y\")\n",
    "\n",
    "theta = tf.Variable(\n",
    "    tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), \n",
    "    name=\"theta\")\n",
    "\n",
    "y_pred = tf.matmul(\n",
    "    X, theta, \n",
    "    name=\"predictions\")\n",
    "\n",
    "##### Name Scope\n",
    "with tf.name_scope('loss') as scope:\n",
    "    error = y_pred - y\n",
    "    mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "#####\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate)\n",
    "\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "mse_summary = tf.summary.scalar(\n",
    "    'MSE', mse)\n",
    "\n",
    "summary_writer = tf.summary.FileWriter(\n",
    "    logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best theta:\n",
      "[[ 2.07001591]\n",
      " [ 0.82045609]\n",
      " [ 0.1173173 ]\n",
      " [-0.22739051]\n",
      " [ 0.31134021]\n",
      " [ 0.00353193]\n",
      " [-0.01126994]\n",
      " [-0.91643935]\n",
      " [-0.87950081]]\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            \n",
    "            X_batch, y_batch = fetch_batch(\n",
    "                epoch, \n",
    "                batch_index, \n",
    "                batch_size)\n",
    "            \n",
    "            if batch_index % 10 == 0:\n",
    "                \n",
    "                summary_str = mse_summary.eval(\n",
    "                    feed_dict={X: X_batch, y: y_batch})\n",
    "                \n",
    "                step = epoch * n_batches + batch_index\n",
    "                \n",
    "                summary_writer.add_summary(\n",
    "                    summary_str, \n",
    "                    step)\n",
    "                \n",
    "            sess.run(\n",
    "                training_op, \n",
    "                feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()\n",
    "\n",
    "summary_writer.flush()\n",
    "summary_writer.close()\n",
    "print(\"Best theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In TensorBoard:\n",
    "\n",
    "![Collapsed Name Scope](pics/collapsed-namescope-tensorboard.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modularity\n",
    "* ex: create graph, adds two ReLU nodes\n",
    "* output: result if >0, 0 otherwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# UGLY\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_features = 3\n",
    "X = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, n_features), \n",
    "    name=\"X\")\n",
    "\n",
    "w1 = tf.Variable(\n",
    "    tf.random_normal(\n",
    "        (n_features, 1)), \n",
    "    name=\"weights1\")\n",
    "\n",
    "w2 = tf.Variable(\n",
    "    tf.random_normal(\n",
    "        (n_features, 1)), \n",
    "    name=\"weights2\")\n",
    "\n",
    "b1 = tf.Variable(\n",
    "    0.0, name=\"bias1\")\n",
    "b2 = tf.Variable(\n",
    "    0.0, name=\"bias2\")\n",
    "\n",
    "linear1 = tf.add(\n",
    "    tf.matmul(X, w1), b1, name=\"linear1\")\n",
    "\n",
    "linear2 = tf.add(\n",
    "    tf.matmul(X, w2), b2, name=\"linear2\")\n",
    "\n",
    "relu1 = tf.maximum(\n",
    "    linear1, 0, name=\"relu1\")\n",
    "\n",
    "relu2 = tf.maximum(\n",
    "    linear2, 0, name=\"relu2\")\n",
    "\n",
    "output = tf.add_n([relu1, relu2], name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# better -- you can create functions that build ReLUs!\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "def relu(X):\n",
    "    w_shape = int(\n",
    "        X.get_shape()[1]), 1\n",
    "    \n",
    "    w = tf.Variable(\n",
    "        tf.random_normal(w_shape), \n",
    "        name=\"weights\")\n",
    "    \n",
    "    b = tf.Variable(\n",
    "        0.0, \n",
    "        name=\"bias\")\n",
    "    \n",
    "    linear = tf.add(\n",
    "        tf.matmul(X, w), \n",
    "        b, \n",
    "        name=\"linear\")\n",
    "    \n",
    "    return tf.maximum(linear, 0, name=\"relu\")\n",
    "\n",
    "n_features = 3\n",
    "\n",
    "X = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, n_features), \n",
    "    name=\"X\")\n",
    "\n",
    "relus = [relu(X) for i in range(5)]\n",
    "\n",
    "output = tf.add_n(\n",
    "    relus, \n",
    "    name=\"output\")\n",
    "\n",
    "summary_writer = tf.summary.FileWriter(\n",
    "    \"logs/relu1\", \n",
    "    tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sharing Variables\n",
    "* Simplest option: define it first, then share it as parameter to all functions that need it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# better, with name scopes\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        \n",
    "        w_shape = int(\n",
    "            X.get_shape()[1]), 1\n",
    "        \n",
    "        w = tf.Variable(\n",
    "            tf.random_normal(w_shape), name=\"weights\")\n",
    "        \n",
    "        b = tf.Variable(\n",
    "            0.0, name=\"bias\")\n",
    "        \n",
    "        linear = tf.add(\n",
    "            tf.matmul(X, w), b, name=\"linear\")\n",
    "        \n",
    "        return tf.maximum(\n",
    "            linear, 0, name=\"max\")\n",
    "\n",
    "n_features = 3\n",
    "\n",
    "X = tf.placeholder(\n",
    "    tf.float32, \n",
    "    shape=(None, n_features), \n",
    "    name=\"X\")\n",
    "\n",
    "relus = [relu(X) for i in range(5)]\n",
    "\n",
    "output = tf.add_n(\n",
    "    relus, name=\"output\")\n",
    "\n",
    "summary_writer = tf.summary.FileWriter(\n",
    "    \"logs/relu2\", \n",
    "    tf.get_default_graph())\n",
    "\n",
    "summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relu1  relu2  relu6\r\n"
     ]
    }
   ],
   "source": [
    "!ls logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "def relu(X, threshold):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        w_shape = int(X.get_shape()[1]), 1\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n",
    "        b = tf.Variable(0.0, name=\"bias\")\n",
    "        linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n",
    "        return tf.maximum(linear, threshold, name=\"max\")\n",
    "\n",
    "threshold = tf.Variable(0.0, name=\"threshold\")\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X, threshold) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        if not hasattr(relu, \"threshold\"):\n",
    "            relu.threshold = tf.Variable(0.0, name=\"threshold\")\n",
    "        w_shape = int(X.get_shape()[1]), 1\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n",
    "        b = tf.Variable(0.0, name=\"bias\")\n",
    "        linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n",
    "        return tf.maximum(linear, relu.threshold, name=\"max\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.variable_scope(\"relu\", reuse=True):\n",
    "        threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n",
    "        w_shape = int(X.get_shape()[1]), 1\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n",
    "        b = tf.Variable(0.0, name=\"bias\")\n",
    "        linear = tf.add(tf.matmul(X, w), b, name=\"linear\")\n",
    "        return tf.maximum(linear, threshold, name=\"max\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "with tf.variable_scope(\"relu\"):\n",
    "    threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n",
    "relus = [relu(X) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")\n",
    "\n",
    "summary_writer = tf.summary.FileWriter(\"logs/relu6\", tf.get_default_graph())\n",
    "summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}