explore_micrograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Value:\n",
    "    \"\"\" stores a single scalar value and its gradient \"\"\"\n",
    "\n",
    "    def __init__(self, data, _children=(), _op='', label=''):\n",
    "        self.data = data\n",
    "        self.grad = 0\n",
    "        self.label = label\n",
    "        # internal variables used for autograd graph construction\n",
    "        self._backward = lambda: None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op # the op that produced this node, for graphviz / debugging / etc\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += out.grad\n",
    "            other.grad += out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad\n",
    "            other.grad += self.data * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "        out = Value(self.data**other, (self,), f'**{other}')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (other * self.data**(other-1)) * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def relu(self):\n",
    "        out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (out.data > 0) * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "    \n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out = Value(t, (self, ), 'tanh')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "\n",
    "        # topological order all of the children in the graph\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                topo.append(v)\n",
    "        build_topo(self)\n",
    "\n",
    "        # go one variable at a time and apply the chain rule to get its gradient\n",
    "        self.grad = 1\n",
    "        for v in reversed(topo):\n",
    "            v._backward()\n",
    "\n",
    "    def __neg__(self): # -self\n",
    "        return self * -1\n",
    "\n",
    "    def __radd__(self, other): # other + self\n",
    "        return self + other\n",
    "\n",
    "    def __sub__(self, other): # self - other\n",
    "        return self + (-other)\n",
    "\n",
    "    def __rsub__(self, other): # other - self\n",
    "        return other + (-self)\n",
    "\n",
    "    def __rmul__(self, other): # other * self\n",
    "        return self * other\n",
    "\n",
    "    def __truediv__(self, other): # self / other\n",
    "        return self * other**-1\n",
    "\n",
    "    def __rtruediv__(self, other): # other / self\n",
    "        return other * self**-1\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Value(data={self.data}, grad={self.grad})\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "class Module:\n",
    "\n",
    "    def zero_grad(self):\n",
    "        for p in self.parameters():\n",
    "            p.grad = 0\n",
    "\n",
    "    def parameters(self):\n",
    "        return []\n",
    "\n",
    "class Neuron(Module):\n",
    "\n",
    "    def __init__(self, nin, nonlin=True):\n",
    "        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "        self.b = Value(0)\n",
    "        self.nonlin = nonlin\n",
    "\n",
    "    def __call__(self, x):\n",
    "        act = sum((wi*xi for wi,xi in zip(self.w, x)), self.b)\n",
    "        out = act.tanh()\n",
    "        return out\n",
    "\n",
    "    def parameters(self):\n",
    "        return self.w + [self.b]\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"{'ReLU' if self.nonlin else 'Linear'}Neuron({len(self.w)})\"\n",
    "\n",
    "class Layer(Module):\n",
    "\n",
    "    def __init__(self, nin, nout, **kwargs):\n",
    "        self.neurons = [Neuron(nin, **kwargs) for _ in range(nout)]\n",
    "\n",
    "    def __call__(self, x):\n",
    "        out = [n(x) for n in self.neurons]\n",
    "        return out[0] if len(out) == 1 else out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for n in self.neurons for p in n.parameters()]\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Layer of [{', '.join(str(n) for n in self.neurons)}]\"\n",
    "\n",
    "class MLP(Module):\n",
    "\n",
    "    def __init__(self, nin, nouts):\n",
    "        sz = [nin] + nouts\n",
    "        self.layers = [Layer(sz[i], sz[i+1], nonlin=i!=len(nouts)-1) for i in range(len(nouts))]\n",
    "\n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"MLP of [{', '.join(str(layer) for layer in self.layers)}]\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for child in v._prev:\n",
    "                edges.add((child, v))\n",
    "                build(child)\n",
    "    build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "def draw_dot(root, format='svg', rankdir='LR'):\n",
    "    \"\"\"\n",
    "    format: png | svg | ...\n",
    "    rankdir: TB (top to bottom graph) | LR (left to right)\n",
    "    \"\"\"\n",
    "    assert rankdir in ['LR', 'TB']\n",
    "    nodes, edges = trace(root)\n",
    "    dot = Digraph(format=format, graph_attr={'rankdir': rankdir}) #, node_attr={'rankdir': 'TB'})\n",
    "    \n",
    "    for n in nodes:\n",
    "        dot.node(name=str(id(n)), label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "        if n._op:\n",
    "            dot.node(name=str(id(n)) + n._op, label=n._op)\n",
    "            dot.edge(str(id(n)) + n._op, str(id(n)))\n",
    "    \n",
    "    for n1, n2 in edges:\n",
    "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "    \n",
    "    return dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gout.svg'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(-4.0, label='a')\n",
    "b = Value(2.0, label='b')\n",
    "c = a + b\n",
    "c.label = 'c'\n",
    "d = a * b + b**3\n",
    "d.label = 'd'\n",
    "# c += c + 1\n",
    "# c += 1 + c + (-a)\n",
    "# d += d * 2 + (b + a).relu()\n",
    "# d += 3 * d + (b - a).relu()\n",
    "# e = c - d\n",
    "# f = e**2\n",
    "# g = f / 2.0\n",
    "# g += 10.0 / f\n",
    "\n",
    "# print(f'{g.data:.4f}') # prints 24.7041, the outcome of this forward pass\n",
    "# g.backward()\n",
    "# print(f'{a.grad:.4f}') # prints 138.8338, i.e. the numerical value of dg/da\n",
    "# print(f'{b.grad:.4f}') # prints 645.5773, i.e. the numerical value of dg/db\n",
    "\n",
    "dot = draw_dot(d)\n",
    "dot.render('gout')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.7807842703143062\n",
      "Predictions: 0.13426891720880468, -0.3468271805728145, -0.9056605341987619, 0.22814554755616712\n",
      "Loss: 0.6564923568163181\n",
      "Predictions: 0.7250530269957218, -0.3123401484710329, -0.8565688107590216, 0.7042840190314695\n",
      "Loss: 0.3277313404609997\n",
      "Predictions: 0.6935300661147346, -0.6643616500881849, -0.9066936972592055, 0.6646668044725098\n",
      "Loss: 0.24235782238475737\n",
      "Predictions: 0.7450658951219907, -0.7001013202505436, -0.9127571074683168, 0.7174829489357435\n",
      "Loss: 0.19116265787236472\n",
      "Predictions: 0.7759265567464545, -0.7324675281528693, -0.9188208495606531, 0.7494205276919617\n",
      "Loss: 0.1570227161317133\n",
      "Predictions: 0.7980923061892357, -0.757605902004099, -0.9238510033276911, 0.7726183145583484\n",
      "Loss: 0.1327650036588094\n",
      "Predictions: 0.815115606399299, -0.7773869343310219, -0.9280209527072091, 0.79060753498277\n",
      "Loss: 0.11471207415539927\n",
      "Predictions: 0.8287211297069762, -0.7933525608048204, -0.931534619664141, 0.8051027185513212\n",
      "Loss: 0.10079217974792187\n",
      "Predictions: 0.8399062875115028, -0.8065415778362889, -0.9345450076347178, 0.817102034352592\n",
      "Loss: 0.08975469104198455\n",
      "Predictions: 0.8493019878352686, -0.8176519268099961, -0.9371620358214126, 0.827241872453518\n",
      "Loss: 0.08080260307889653\n",
      "Predictions: 0.8573311524393498, -0.8271646709378359, -0.9394654136434482, 0.8359520926764773\n",
      "Loss: 0.07340526207803565\n",
      "Predictions: 0.864289611054048, -0.8354211196807346, -0.9415141047763498, 0.8435354844607915\n",
      "Loss: 0.06719636158618326\n",
      "Predictions: 0.8703913211775038, -0.8426700533489766, -0.9433526520403777, 0.8502124243603699\n",
      "Loss: 0.061915269934591524\n",
      "Predictions: 0.8757953019217767, -0.8490972132346566, -0.9450153711514251, 0.8561476366314176\n",
      "Loss: 0.0573717087748343\n",
      "Predictions: 0.8806224685851862, -0.854844229048373, -0.9465291704787773, 0.8614670208293687\n",
      "Loss: 0.05342365126584893\n",
      "Predictions: 0.8849665694997132, -0.8600211097847565, -0.9479154849717453, 0.8662686472005725\n",
      "Loss: 0.04996302536538897\n",
      "Predictions: 0.8889015230319773, -0.8647147046038519, -0.9491916303134547, 0.8706301727598471\n",
      "Loss: 0.046906195453229466\n",
      "Predictions: 0.8924864757223938, -0.8689945717480415, -0.9503717701435792, 0.8746139795651953\n",
      "Loss: 0.04418746313800053\n",
      "Predictions: 0.8957693725745458, -0.872917137795685, -0.9514676197794487, 0.8782708184181639\n",
      "Loss: 0.04175452944264602\n",
      "Predictions: 0.8987895297178017, -0.8765287027839883, -0.9524889669584047, 0.8816424457374266\n",
      "Loss: 0.03956526291390987\n",
      "Predictions: 0.901579522371669, -0.8798676495982106, -0.9534440631733468, 0.884763566393022\n",
      "Loss: 0.03758535653750135\n",
      "Predictions: 0.9041665930938305, -0.8829660940997276, -0.9543399219267542, 0.8876632882877143\n",
      "Loss: 0.035786601607153204\n",
      "Predictions: 0.9065737176859178, -0.8858511352712541, -0.955182548980177, 0.8903662271681447\n",
      "Loss: 0.03414559752940451\n",
      "Predictions: 0.9088204227175094, -0.8885458147092413, -0.9559771222032948, 0.8928933567529053\n",
      "Loss: 0.03264277466773451\n",
      "Predictions: 0.9109234201237763, -0.891069861819024, -0.9567281335741822, 0.895262670660414\n",
      "Loss: 0.03126164530513265\n",
      "Predictions: 0.9128971052374903, -0.8934402788928536, -0.9574395024097134, 0.8974897033876196\n",
      "Loss: 0.029988223092857792\n",
      "Predictions: 0.9147539515912047, -0.8956718050812243, -0.9581146664811834, 0.8995879444262266\n",
      "Loss: 0.02881056848873763\n",
      "Predictions: 0.9165048267930156, -0.8977772877259651, -0.9587566559540261, 0.9015691704438157\n",
      "Loss: 0.027718429486200156\n",
      "Predictions: 0.918159247420322, -0.8997679820902177, -0.9593681538590803, 0.9034437139886492\n",
      "Loss: 0.026702955178297898\n",
      "Predictions: 0.9197255863365731, -0.9016537952078815, -0.9599515459081959, 0.905220682545535\n",
      "Loss: 0.025756465539108164\n",
      "Predictions: 0.9212112425537664, -0.9034434857304801, -0.9605089618094503, 0.906908138411893\n",
      "Loss: 0.024872264992105774\n",
      "Predictions: 0.9226227813619257, -0.905144828834537, -0.9610423097487314, 0.9085132473994276\n",
      "Loss: 0.02404449037359493\n",
      "Predictions: 0.9239660506702395, -0.9067647531691433, -0.9615533053378276, 0.910042402539545\n",
      "Loss: 0.023267986128371013\n",
      "Predictions: 0.9252462781767624, -0.9083094552656918, -0.9620434960514118, 0.9115013276016433\n",
      "Loss: 0.022538201226813666\n",
      "Predictions: 0.9264681529816954, -0.9097844956559838, -0.9625142819630051, 0.9128951641979848\n",
      "Loss: 0.021851103528706273\n",
      "Predictions: 0.9276358944964724, -0.9111948800495379, -0.9629669334263694, 0.9142285454587735\n",
      "Loss: 0.02120310825226848\n",
      "Predictions: 0.9287533109152349, -0.9125451282333302, -0.9634026062216636, 0.9155056586531617\n",
      "Loss: 0.020591017917351462\n",
      "Predictions: 0.9298238490620732, -0.9138393328250175, -0.9638223545862333, 0.9167302986605186\n",
      "Loss: 0.020011971676945323\n",
      "Predictions: 0.9308506370740841, -0.9150812095957249, -0.964227142471499, 0.9179059138280615\n",
      "Loss: 0.019463402372662496\n",
      "Predictions: 0.9318365211028995, -0.9162741407526448, -0.964617853305231, 0.9190356454612961\n",
      "Loss: 0.018942999978060485\n",
      "Predictions: 0.9327840969980954, -0.9174212123141435, -0.9649952984888639, 0.9201223619643799\n",
      "Loss: 0.01844868035092595\n",
      "Predictions: 0.9336957377615392, -0.9185252465052196, -0.9653602248196418, 0.9211686884648087\n",
      "Loss: 0.01797855841857351\n",
      "Predictions: 0.934573617422237, -0.9195888299372439, -0.9657133209952056, 0.9221770326104022\n",
      "Loss: 0.01753092508125643\n",
      "Predictions: 0.9354197318690056, -0.9206143382040272, -0.9660552233321105, 0.9231496071085656\n",
      "Loss: 0.017104227247322627\n",
      "Predictions: 0.936235917087521, -0.9216039574195509, -0.9663865208084413, 0.924088449482202\n",
      "Loss: 0.01669705051689767\n",
      "Predictions: 0.9370238651744714, -0.9225597031359477, -0.9667077595232166, 0.9249954394388048\n",
      "Loss: 0.016308104114083177\n",
      "Predictions: 0.9377851384412509, -0.9234834370094172, -0.9670194466508845, 0.9258723141855678\n",
      "Loss: 0.015936207735111597\n",
      "Predictions: 0.9385211818701263, -0.9243768815235746, -0.9673220539572926, 0.9267206819709958\n",
      "Loss: 0.01558028003483938\n",
      "Predictions: 0.9392333341450101, -0.9252416330317367, -0.9676160209336347, 0.9275420340902736\n",
      "Loss: 0.015239328518912137\n",
      "Predictions: 0.9399228374451737, -0.9260791733399231, -0.9679017575966115, 0.9283377555558209\n",
      "Loss: 0.01491244064587362\n",
      "Predictions: 0.9405908461621646, -0.9268908800193087, -0.9681796469961401, 0.9291091346046286\n",
      "Loss: 0.01459877597397036\n",
      "Predictions: 0.9412384346767387, -0.927678035609292, -0.9684500474661392, 0.9298573711890498\n",
      "Loss: 0.014297559212653257\n",
      "Predictions: 0.9418666043129914, -0.9284418358492389, -0.9687132946490137, 0.9305835845768147\n",
      "Loss: 0.014008074059775439\n",
      "Predictions: 0.9424762895703604, -0.9291833970575476, -0.968969703320322, 0.9312888201684426\n",
      "Loss: 0.013729657723010285\n",
      "Predictions: 0.943068363720258, -0.9299037627602766, -0.9692195690365866, 0.9319740556253688\n",
      "Loss: 0.01346169603869202\n",
      "Predictions: 0.943643643842295, -0.930603909657728, -0.969463169626212, 0.9326402063895112\n",
      "Loss: 0.013203619113617813\n",
      "Predictions: 0.9442028953650649, -0.9312847530055781, -0.9697007665409123, 0.9332881306643023\n",
      "Loss: 0.012954897425750385\n",
      "Predictions: 0.9447468361679286, -0.931947151477131, -0.9699326060828546, 0.9339186339180906\n",
      "Loss: 0.012715038328557738\n",
      "Predictions: 0.9452761402929675, -0.9325919115646896, -0.9701589205208438, 0.9345324729630116\n",
      "Loss: 0.0124835829111893\n",
      "Predictions: 0.9457914413100339, -0.9332197915707005, -0.9703799291072437, 0.9351303596557434\n",
      "Loss: 0.012260103173037284\n",
      "Predictions: 0.9462933353724811, -0.9338315052330216, -0.9705958390059297, 0.935712964260812\n",
      "Loss: 0.012044199476650502\n",
      "Predictions: 0.9467823839965352, -0.9344277250232231, -0.9708068461403544, 0.9362809185121498\n",
      "Loss: 0.011835498247602012\n",
      "Predictions: 0.947259116593299, -0.9350090851521464, -0.9710131359697481, 0.9368348184043336\n",
      "Loss: 0.0116336498938882\n",
      "Predictions: 0.9477240327789235, -0.935576184312883, -0.9712148842005639, 0.9373752267412123\n",
      "Loss: 0.011438326920856862\n",
      "Predictions: 0.9481776044855038, -0.9361295881878051, -0.9714122574394705, 0.9379026754664139\n",
      "Loss: 0.011249222220608004\n",
      "Predictions: 0.9486202778926454, -0.9366698317432256, -0.9716054137935025, 0.9384176677974158\n",
      "Loss: 0.01106604751736002\n",
      "Predictions: 0.9490524751973902, -0.9371974213325781, -0.9717945034223614, 0.938920680182417\n",
      "Loss: 0.010888531952478284\n",
      "Predictions: 0.9494745962382086, -0.9377128366266902, -0.9719796690473219, 0.9394121640971123\n",
      "Loss: 0.010716420794779954\n",
      "Predictions: 0.9498870199870325, -0.9382165323876664, -0.9721610464207278, 0.9398925476965898\n",
      "Loss: 0.010549474263393938\n",
      "Predictions: 0.950290105921786, -0.9387089401011077, -0.9723387647596448, 0.9403622373359395\n",
      "Loss: 0.010387466451907185\n",
      "Predictions: 0.950684195290535, -0.9391904694798288, -0.9725129471468651, 0.9408216189717019\n",
      "Loss: 0.010230184343797915\n",
      "Predictions: 0.951069612277204, -0.9396615098508306, -0.9726837109021412, 0.941271059455022\n",
      "Loss: 0.0100774269102669\n",
      "Predictions: 0.9514466650777658, -0.940122431436075, -0.972851167926229, 0.9417109077262401\n",
      "Loss: 0.009929004282550589\n",
      "Predictions: 0.9518156468949023, -0.9405735865365206, -0.9730154250200719, 0.9421414959196656\n",
      "Loss: 0.009784736991657413\n",
      "Predictions: 0.9521768368583164, -0.9410153106279213, -0.9731765841812233, 0.9425631403863884\n",
      "Loss: 0.009644455269219717\n",
      "Predictions: 0.9525305008771602, -0.941447923376043, -0.9733347428794115, 0.9429761426422099\n",
      "Loss: 0.009507998403819832\n",
      "Predictions: 0.9528768924304029, -0.9418717295781917, -0.9734899943129561, 0.9433807902470722\n",
      "Loss: 0.00937521414773327\n",
      "Predictions: 0.9532162533003976, -0.9422870200372846, -0.9736424276476017, 0.9437773576217552\n",
      "Loss: 0.00924595816955276\n",
      "Predictions: 0.953548814254397, -0.9426940723740898, -0.9737921282391723, 0.944166106807046\n",
      "Loss: 0.009120093548615489\n",
      "Predictions: 0.9538747956783142, -0.9430931517827287, -0.9739391778413332, 0.9445472881701082\n",
      "Loss: 0.008997490307564644\n",
      "Predictions: 0.9541944081666286, -0.9434845117340581, -0.9740836547996259, 0.9449211410623236\n",
      "Loss: 0.008878024979738958\n",
      "Predictions: 0.9545078530719662, -0.9438683946311202, -0.9742256342328363, 0.9452878944324938\n",
      "Loss: 0.00876158020840692\n",
      "Predictions: 0.9548153230175668, -0.9442450324204673, -0.9743651882026663, 0.9456477673989317\n",
      "Loss: 0.008648044375150444\n",
      "Predictions: 0.9551170023755556, -0.9446146471628234, -0.9745023858725911, 0.9460009697836547\n",
      "Loss: 0.008537311254960132\n",
      "Predictions: 0.9554130677136786, -0.944977451566232, -0.9746372936567088, 0.9463477026116052\n",
      "Loss: 0.008429279695833259\n",
      "Predictions: 0.9557036882129224, -0.9453336494845697, -0.9747699753593233, 0.9466881585775709\n",
      "Loss: 0.008323853320872692\n",
      "Predictions: 0.9559890260582318, -0.9456834363840441, -0.9749004923059316, 0.9470225224832391\n",
      "Loss: 0.008220940251068424\n",
      "Predictions: 0.9562692368043426, -0.9460269997800768, -0.975028903466242, 0.9473509716466137\n",
      "Loss: 0.008120452847109712\n",
      "Predictions: 0.9565444697185784, -0.9463645196467626, -0.9751552655697864, 0.9476736762858329\n",
      "Loss: 0.008022307468724451\n",
      "Predictions: 0.956814868102301, -0.9466961688009112, -0.9752796332146495, 0.9479907998792562\n",
      "Loss: 0.00792642425017603\n",
      "Predictions: 0.957080569592565, -0.9470221132625155, -0.9754020589697987, 0.9483024995035313\n",
      "Loss: 0.007832726890669425\n",
      "Predictions: 0.9573417064453982, -0.9473425125933312, -0.9755225934714525, 0.948608926151212\n",
      "Loss: 0.007741142458526534\n",
      "Predictions: 0.9575984058020115, -0.9476575202151201, -0.9756412855138954, 0.948910225029371\n",
      "Loss: 0.007651601208089141\n",
      "Predictions: 0.9578507899391393, -0.9479672837089828, -0.9757581821351163, 0.9492065358405346\n",
      "Loss: 0.007564036408397293\n",
      "Predictions: 0.9580989765046135, -0.94827194509709, -0.9758733286976141, 0.9494979930471618\n",
      "Loss: 0.007478384182770342\n",
      "Predictions: 0.9583430787391878, -0.9485716411080253, -0.9759867689646912, 0.9497847261207911\n",
      "Loss: 0.0073945833584924915\n",
      "Predictions: 0.9585832056855459, -0.9488665034268448, -0.9760985451725309, 0.950066859776895\n",
      "Loss: 0.007312575325868848\n",
      "Predictions: 0.9588194623853619, -0.9491566589308884, -0.9762086980983308, 0.9503445141963968\n",
      "Loss: 0.007232303905979234\n",
      "Predictions: 0.9590519500652064, -0.9494422299122878, -0.976317267124746, 0.9506178052347379\n",
      "Loss: 0.007153715226511842\n",
      "Predictions: 0.9592807663120357, -0.9497233342880457, -0.9764242903008774, 0.9508868446193091\n"
     ]
    }
   ],
   "source": [
    "n = MLP(3, [4 ,4, 1])\n",
    "xs = [\n",
    "    [2.0, 3.0, -1.0],\n",
    "    [3.0, -1.0, 0.5],\n",
    "    [0.5, 1.0, 1.0],\n",
    "    [1.0, 1.0, -1.0],\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0] # desired targets\n",
    "iterations = 100\n",
    "learning_rate = 0.05\n",
    "\n",
    "for i in range(0, iterations):\n",
    "    ypred = [n(x) for x in xs]\n",
    "    loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "\n",
    "    #zero grad\n",
    "    for p in n.parameters():\n",
    "        p.grad = 0\n",
    "    \n",
    "    # backward pass\n",
    "    loss.backward()\n",
    "\n",
    "    # update\n",
    "    for p in n.parameters():\n",
    "        p.data -= learning_rate * p.grad\n",
    "    \n",
    "    print(f'Loss: {loss.data}')\n",
    "    print(f'Predictions: {ypred[0].data}, {ypred[1].data}, {ypred[2].data}, {ypred[3].data}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "saurav_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}