lasso-and-ridge-regression.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:06.150821Z",
     "iopub.status.busy": "2022-01-06T17:26:06.150338Z",
     "iopub.status.idle": "2022-01-06T17:26:07.330284Z",
     "shell.execute_reply": "2022-01-06T17:26:07.329295Z",
     "shell.execute_reply.started": "2022-01-06T17:26:06.150670Z"
    }
   },
   "source": [
    "# Lasso and Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import make_regression\n",
    "\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Common Regression class"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The most of things common in Lasson and Ridge Regression. <br/>\n",
    "The only different between two regression is, what regularization it is using. <br/>\n",
    "- L1 regularization ---> Lasso Regression\n",
    "- L2 regularization ---> Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.332791Z",
     "iopub.status.busy": "2022-01-06T17:26:07.332469Z",
     "iopub.status.idle": "2022-01-06T17:26:07.346895Z",
     "shell.execute_reply": "2022-01-06T17:26:07.346034Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.332748Z"
    }
   },
   "outputs": [],
   "source": [
    "class Regression:\n",
    "    def __init__(self, learning_rate, iteration, regularization):\n",
    "        \"\"\"\n",
    "        :param learning_rate: A samll value needed for gradient decent, default value id 0.1.\n",
    "        :param iteration: Number of training iteration, default value is 10,000.\n",
    "        \"\"\"\n",
    "        self.m = None\n",
    "        self.n = None\n",
    "        self.w = None\n",
    "        self.b = None\n",
    "        self.regularization = regularization # will be the L1/L2 regularization class according to the regression model.\n",
    "        self.lr = learning_rate\n",
    "        self.it = iteration\n",
    "\n",
    "    def cost_function(self, y, y_pred):\n",
    "        \"\"\"\n",
    "        :param y: Original target value.\n",
    "        :param y_pred: predicted target value.\n",
    "        \"\"\"\n",
    "        return (1 / (2*self.m)) * np.sum(np.square(y_pred - y)) + self.regularization(self.w)\n",
    "    \n",
    "    def hypothesis(self, weights, bias, X):\n",
    "        \"\"\"\n",
    "        :param weights: parameter value weight.\n",
    "        :param X: Training samples.\n",
    "        \"\"\"\n",
    "        return np.dot(X, weights) #+ bias\n",
    "\n",
    "    def train(self, X, y):\n",
    "        \"\"\"\n",
    "        :param X: training data feature values ---> N Dimentional vector.\n",
    "        :param y: training data target value -----> 1 Dimentional array.\n",
    "        \"\"\"\n",
    "        # Insert constant ones for bias weights.\n",
    "        X = np.insert(X, 0, 1, axis=1)\n",
    "\n",
    "        # Target value should be in the shape of (n, 1) not (n, ).\n",
    "        # So, this will check that and change the shape to (n, 1), if not.\n",
    "        try:\n",
    "            y.shape[1]\n",
    "        except IndexError as e:\n",
    "            # we need to change it to the 1 D array, not a list.\n",
    "            print(\"ERROR: Target array should be a one dimentional array not a list\"\n",
    "                  \"----> here the target value not in the shape of (n,1). \\nShape ({shape_y_0},1) and {shape_y} not match\"\n",
    "                  .format(shape_y_0 = y.shape[0] , shape_y = y.shape))\n",
    "            return \n",
    "        \n",
    "        # m is the number of training samples.\n",
    "        self.m = X.shape[0]\n",
    "        # n is the number of features.\n",
    "        self.n = X.shape[1]\n",
    "\n",
    "        # Set the initial weight.\n",
    "        self.w = np.zeros((self.n , 1))\n",
    "\n",
    "        # bias.\n",
    "        self.b = 0\n",
    "\n",
    "        for it in range(1, self.it+1):\n",
    "            # 1. Find the predicted value through the hypothesis.\n",
    "            # 2. Find the Cost function value.\n",
    "            # 3. Find the derivation of weights.\n",
    "            # 4. Apply Gradient Decent.\n",
    "            y_pred = self.hypothesis(self.w, self.b, X)\n",
    "            #print(\"iteration\",it)\n",
    "            #print(\"y predict value\",y_pred)\n",
    "            cost = self.cost_function(y, y_pred)\n",
    "            #print(\"Cost function\",cost)\n",
    "            # fin the derivative.\n",
    "            dw = (1/self.m) * np.dot(X.T, (y_pred - y)) + self.regularization.derivation(self.w)\n",
    "            #print(\"weights derivation\",dw)\n",
    "            #db = -(2 / self.m) * np.sum((y_pred - y))\n",
    "\n",
    "            # change the weight parameter.\n",
    "            self.w = self.w - self.lr * dw\n",
    "            #print(\"updated weights\",self.w)\n",
    "            #self.b = self.b - self.lr * db\n",
    "\n",
    "\n",
    "            if it % 10 == 0:\n",
    "                print(\"The Cost function for the iteration {}----->{} :)\".format(it, cost))\n",
    "    def predict(self, test_X):\n",
    "        \"\"\"\n",
    "        :param test_X: feature values to predict.\n",
    "        \"\"\"\n",
    "        # Insert constant ones for bias weights.\n",
    "        test_X = np.insert(test_X, 0, 1, axis=1)\n",
    "\n",
    "        y_pred = self.hypothesis(self.w, self.b, test_X)\n",
    "        return y_pred"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularization classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.348816Z",
     "iopub.status.busy": "2022-01-06T17:26:07.347965Z",
     "iopub.status.idle": "2022-01-06T17:26:07.365987Z",
     "shell.execute_reply": "2022-01-06T17:26:07.364904Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.348780Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create the regularization class we want.\n",
    "class L1Regularization:\n",
    "    \"\"\"Regularization used for Lasson Regression\"\"\"\n",
    "    def __init__(self, lamda):\n",
    "        self.lamda = lamda\n",
    "\n",
    "    def __call__(self, weights):\n",
    "        \"This will be retuned when we call this class.\"\n",
    "        return self.lamda * np.sum(np.abs(weights))\n",
    "    \n",
    "    def derivation(self, weights):\n",
    "        \"Derivation of the regulariozation function.\"\n",
    "        return self.lamda * np.sign(weights)\n",
    "\n",
    "\n",
    "class L2Regularization:\n",
    "    \"\"\"Regularization used for Ridge Regression\"\"\"\n",
    "    def __init__(self, lamda):\n",
    "        self.lamda = lamda\n",
    "\n",
    "    def __call__(self, weights):\n",
    "        \"This will be retuned when we call this class.\"\n",
    "        return self.lamda * np.sum(np.square(weights))\n",
    "    \n",
    "    def derivation(self, weights):\n",
    "        \"Derivation of the regulariozation function.\"\n",
    "        return self.lamda * 2 * (weights)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.368794Z",
     "iopub.status.busy": "2022-01-06T17:26:07.368210Z",
     "iopub.status.idle": "2022-01-06T17:26:07.437963Z",
     "shell.execute_reply": "2022-01-06T17:26:07.436880Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.368747Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "Number of training data samples-----> 50000\n",
      "Number of training features --------> 8\n",
      "Shape of the target value ----------> (50000, 1)\n"
     ]
    }
   ],
   "source": [
    "# Define the traning data.\n",
    "X, y = make_regression(n_samples=50000, n_features=8)\n",
    "\n",
    "# Chnage the shape of the target to 1 dimentional array.\n",
    "y = y[:, np.newaxis]\n",
    "\n",
    "print(\"=\"*100)\n",
    "print(\"Number of training data samples-----> {}\".format(X.shape[0]))\n",
    "print(\"Number of training features --------> {}\".format(X.shape[1]))\n",
    "print(\"Shape of the target value ----------> {}\".format(y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.440680Z",
     "iopub.status.busy": "2022-01-06T17:26:07.439884Z",
     "iopub.status.idle": "2022-01-06T17:26:07.475198Z",
     "shell.execute_reply": "2022-01-06T17:26:07.474173Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.440621Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.121669</td>\n",
       "      <td>-0.148495</td>\n",
       "      <td>0.617423</td>\n",
       "      <td>-1.662224</td>\n",
       "      <td>0.536879</td>\n",
       "      <td>-0.681744</td>\n",
       "      <td>-0.885964</td>\n",
       "      <td>0.869547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.078910</td>\n",
       "      <td>-0.534563</td>\n",
       "      <td>-0.258445</td>\n",
       "      <td>2.274082</td>\n",
       "      <td>2.007185</td>\n",
       "      <td>2.304888</td>\n",
       "      <td>-0.477328</td>\n",
       "      <td>0.173211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.492209</td>\n",
       "      <td>0.326460</td>\n",
       "      <td>0.724924</td>\n",
       "      <td>0.546021</td>\n",
       "      <td>-0.856196</td>\n",
       "      <td>0.292669</td>\n",
       "      <td>0.022331</td>\n",
       "      <td>-0.336342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.057504</td>\n",
       "      <td>-0.829756</td>\n",
       "      <td>1.446873</td>\n",
       "      <td>-0.206937</td>\n",
       "      <td>0.741066</td>\n",
       "      <td>0.265056</td>\n",
       "      <td>0.148822</td>\n",
       "      <td>-1.136302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.941642</td>\n",
       "      <td>-0.523965</td>\n",
       "      <td>-0.218161</td>\n",
       "      <td>-0.703982</td>\n",
       "      <td>-0.222337</td>\n",
       "      <td>0.464776</td>\n",
       "      <td>-1.604384</td>\n",
       "      <td>0.461822</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.121669 -0.148495  0.617423 -1.662224  0.536879 -0.681744 -0.885964   \n",
       "1  0.078910 -0.534563 -0.258445  2.274082  2.007185  2.304888 -0.477328   \n",
       "2  1.492209  0.326460  0.724924  0.546021 -0.856196  0.292669  0.022331   \n",
       "3 -0.057504 -0.829756  1.446873 -0.206937  0.741066  0.265056  0.148822   \n",
       "4  0.941642 -0.523965 -0.218161 -0.703982 -0.222337  0.464776 -1.604384   \n",
       "\n",
       "          7  \n",
       "0  0.869547  \n",
       "1  0.173211  \n",
       "2 -0.336342  \n",
       "3 -1.136302  \n",
       "4  0.461822  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display the data.\n",
    "data = pd.DataFrame(X)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.478199Z",
     "iopub.status.busy": "2022-01-06T17:26:07.477480Z",
     "iopub.status.idle": "2022-01-06T17:26:07.491656Z",
     "shell.execute_reply": "2022-01-06T17:26:07.490437Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.478149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-52.114639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>195.157729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>145.868942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-86.035711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32.734924</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0\n",
       "0  -52.114639\n",
       "1  195.157729\n",
       "2  145.868942\n",
       "3  -86.035711\n",
       "4   32.734924"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display the data.\n",
    "data_y = pd.DataFrame(y)\n",
    "data_y.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lasson Regression from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.631896Z",
     "iopub.status.busy": "2022-01-06T17:26:07.631589Z",
     "iopub.status.idle": "2022-01-06T17:26:07.638854Z",
     "shell.execute_reply": "2022-01-06T17:26:07.637945Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.631866Z"
    }
   },
   "outputs": [],
   "source": [
    "class LassoRegression(Regression):\n",
    "    \"\"\"\n",
    "    Lasso Regression is one of the variance of the Linear Regression. This model doing the parameter learning \n",
    "    and regularization at the same time. This model uses the L1Regularization. \n",
    "    * Regularization will be one of the soluions to the Overfitting.\n",
    "    * Overfitting happens when the model has \"High Variance and low bias\". So, regularization adds a little bias to the model.\n",
    "    * This model will try to keep the balance between learning the parameters and the complexity of the model( tries to keep the parameter having small value and small degree of palinamial).\n",
    "    * The Regularization parameter(lamda) controls how severe  the regularization is. \n",
    "    * large lamda adds more bias , hence the Variance will go very small --> this may cause underfitting(Low bias and High Varinace).\n",
    "    * Lamda can be found by tial and error methos. \n",
    "    \"\"\"\n",
    "    def __init__(self, lamda, learning_rate, iteration):\n",
    "        \"\"\"\n",
    "        Define the hyperparameters we are going to use in this model.\n",
    "        :param lamda: Regularization factor.\n",
    "        :param learning_rate: A samll value needed for gradient decent, default value id 0.1.\n",
    "        :param iteration: Number of training iteration, default value is 10,000.\n",
    "        \"\"\"\n",
    "        self.regularization = L1Regularization(lamda)\n",
    "        super(LassoRegression, self).__init__(learning_rate, iteration, self.regularization)\n",
    "\n",
    "    def train(self, X, y):\n",
    "        \"\"\"\n",
    "        :param X: training data feature values ---> N Dimentional vector.\n",
    "        :param y: training data target value -----> 1 Dimentional array.\n",
    "        \"\"\"\n",
    "        return super(LassoRegression, self).train(X, y)\n",
    "    def predict(self, test_X):\n",
    "        \"\"\"\n",
    "        parma test_X: Value need to be predicted.\n",
    "        \"\"\"\n",
    "        return super(LassoRegression, self).predict(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.720989Z",
     "iopub.status.busy": "2022-01-06T17:26:07.720476Z",
     "iopub.status.idle": "2022-01-06T17:26:07.815661Z",
     "shell.execute_reply": "2022-01-06T17:26:07.814715Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.720953Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "The Cost function for the iteration 10----->1790.7949491493525 :)\n",
      "The Cost function for the iteration 20----->249.1630251618197 :)\n",
      "The Cost function for the iteration 30----->62.47992614761185 :)\n",
      "The Cost function for the iteration 40----->39.84797838907741 :)\n",
      "The Cost function for the iteration 50----->37.10119387961131 :)\n",
      "The Cost function for the iteration 60----->36.76731980371954 :)\n",
      "The Cost function for the iteration 70----->36.72663483920857 :)\n",
      "The Cost function for the iteration 80----->36.72164454424515 :)\n",
      "The Cost function for the iteration 90----->36.72101958323096 :)\n",
      "The Cost function for the iteration 100----->36.72093598500494 :)\n",
      "The r2_score of the trained model 0.9999965334550955\n"
     ]
    }
   ],
   "source": [
    "# define the parameters\n",
    "param = {\n",
    "    \"lamda\" : 0.1,\n",
    "    \"learning_rate\" : 0.1,\n",
    "    \"iteration\" : 100\n",
    "}\n",
    "print(\"=\"*100)\n",
    "linear_reg = LassoRegression(**param)\n",
    "\n",
    "# Train the model.\n",
    "linear_reg.train(X, y) \n",
    "\n",
    "# Predict the values.\n",
    "y_pred = linear_reg.predict(X)\n",
    "\n",
    "# Root mean square error.\n",
    "score = r2_score(y, y_pred)\n",
    "print(\"The r2_score of the trained model\", score)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lasso Regression using skicit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:07.988625Z",
     "iopub.status.busy": "2022-01-06T17:26:07.988280Z",
     "iopub.status.idle": "2022-01-06T17:26:08.084069Z",
     "shell.execute_reply": "2022-01-06T17:26:08.082943Z",
     "shell.execute_reply.started": "2022-01-06T17:26:07.988591Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "Number of training data samples-----> 50000\n",
      "Number of training features --------> 8\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# data is already defined, going to use the same data for comparision.\n",
    "print(\"=\"*100)\n",
    "print(\"Number of training data samples-----> {}\".format(X.shape[0]))\n",
    "print(\"Number of training features --------> {}\".format(X.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:08.086479Z",
     "iopub.status.busy": "2022-01-06T17:26:08.086108Z",
     "iopub.status.idle": "2022-01-06T17:26:08.111239Z",
     "shell.execute_reply": "2022-01-06T17:26:08.110281Z",
     "shell.execute_reply.started": "2022-01-06T17:26:08.086410Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "R2 score of the model is 0.9996615456176722\n"
     ]
    }
   ],
   "source": [
    "lasso_sklearn = Lasso()\n",
    "lasso_sklearn.fit(X, y)\n",
    "\n",
    "# predict the value\n",
    "y_pred_sklearn = lasso_sklearn.predict(X)\n",
    "score = r2_score(y, y_pred_sklearn)\n",
    "print(\"=\"*100)\n",
    "print(\"R2 score of the model is {}\".format(score))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "Our model (from scratch) also works great as compared to skiti-learn model. Both the models are giving 0.99..% r2_socre which is good."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge Regression from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:08.327556Z",
     "iopub.status.busy": "2022-01-06T17:26:08.327229Z",
     "iopub.status.idle": "2022-01-06T17:26:08.334849Z",
     "shell.execute_reply": "2022-01-06T17:26:08.333930Z",
     "shell.execute_reply.started": "2022-01-06T17:26:08.327524Z"
    }
   },
   "outputs": [],
   "source": [
    "class RidgeRegression(Regression):\n",
    "    \"\"\"\n",
    "    Ridge Regression is one of the variance of the Linear Regression. This model doing the parameter learning \n",
    "    and regularization at the same time. This model uses the L2Regularization. \n",
    "    This is very similar to the Lasso regression.\n",
    "    * Regularization will be one of the soluions to the Overfitting.\n",
    "    * Overfitting happens when the model has \"High Variance and low bias\". So, regularization adds a little bias to the model.\n",
    "    * This model will try to keep the balance between learning the parameters and the complexity of the model( tries to keep the parameter having small value and small degree of palinamial).\n",
    "    * The Regularization parameter(lamda) controls how severe  the regularization is. \n",
    "    * large lamda adds more bias , hence the Variance will go very small --> this may cause underfitting(Low bias and High Varinace).\n",
    "    * Lamda can be found by tial and error methos. \n",
    "    \"\"\"\n",
    "    def __init__(self, lamda, learning_rate, iteration):\n",
    "        \"\"\"\n",
    "        Define the hyperparameters we are going to use in this model.\n",
    "        :param lamda: Regularization factor.\n",
    "        :param learning_rate: A samll value needed for gradient decent, default value id 0.1.\n",
    "        :param iteration: Number of training iteration, default value is 10,000.\n",
    "        \"\"\"\n",
    "        self.regularization = L2Regularization(lamda)\n",
    "        super(RidgeRegression, self).__init__(learning_rate, iteration, self.regularization)\n",
    "\n",
    "    def train(self, X, y):\n",
    "        \"\"\"\n",
    "        :param X: training data feature values ---> N Dimentional vector.\n",
    "        :param y: training data target value -----> 1 Dimentional array.\n",
    "        \"\"\"\n",
    "        return super(RidgeRegression, self).train(X, y)\n",
    "    def predict(self, test_X):\n",
    "        \"\"\"\n",
    "        parma test_X: Value need to be predicted.\n",
    "        \"\"\"\n",
    "        return super(RidgeRegression, self).predict(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:08.446536Z",
     "iopub.status.busy": "2022-01-06T17:26:08.446182Z",
     "iopub.status.idle": "2022-01-06T17:26:08.543871Z",
     "shell.execute_reply": "2022-01-06T17:26:08.542769Z",
     "shell.execute_reply.started": "2022-01-06T17:26:08.446497Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "The Cost function for the iteration 10----->2938.4966309980255 :)\n",
      "The Cost function for the iteration 20----->2035.1369099078838 :)\n",
      "The Cost function for the iteration 30----->1965.3537240779558 :)\n",
      "The Cost function for the iteration 40----->1959.9564565636458 :)\n",
      "The Cost function for the iteration 50----->1959.538491832234 :)\n",
      "The Cost function for the iteration 60----->1959.5060835072245 :)\n",
      "The Cost function for the iteration 70----->1959.5035673795078 :)\n",
      "The Cost function for the iteration 80----->1959.503371776776 :)\n",
      "The Cost function for the iteration 90----->1959.5033565506537 :)\n",
      "The Cost function for the iteration 100----->1959.503355363845 :)\n",
      "The r2_score of the trained model 0.9723201967872726\n"
     ]
    }
   ],
   "source": [
    "# define the parameters\n",
    "param = {\n",
    "    \"lamda\" : 0.1,\n",
    "    \"learning_rate\" : 0.1,\n",
    "    \"iteration\" : 100\n",
    "}\n",
    "print(\"=\"*100)\n",
    "linear_reg = RidgeRegression(**param)\n",
    "\n",
    "# Train the model.\n",
    "linear_reg.train(X, y) \n",
    "\n",
    "# Predict the values.\n",
    "y_pred = linear_reg.predict(X)\n",
    "\n",
    "# Root mean square error.\n",
    "score = r2_score(y, y_pred)\n",
    "print(\"The r2_score of the trained model\", score)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge Regression using scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:11.438105Z",
     "iopub.status.busy": "2022-01-06T17:26:11.437404Z",
     "iopub.status.idle": "2022-01-06T17:26:11.445713Z",
     "shell.execute_reply": "2022-01-06T17:26:11.444732Z",
     "shell.execute_reply.started": "2022-01-06T17:26:11.438051Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "Number of training data samples-----> 50000\n",
      "Number of training features --------> 8\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# data is already defined, going to use the same data for comparision.\n",
    "print(\"=\"*100)\n",
    "print(\"Number of training data samples-----> {}\".format(X.shape[0]))\n",
    "print(\"Number of training features --------> {}\".format(X.shape[1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-01-06T17:26:11.447879Z",
     "iopub.status.busy": "2022-01-06T17:26:11.447201Z",
     "iopub.status.idle": "2022-01-06T17:26:11.480795Z",
     "shell.execute_reply": "2022-01-06T17:26:11.479788Z",
     "shell.execute_reply.started": "2022-01-06T17:26:11.447844Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "R2 score of the model is 0.999999999601675\n"
     ]
    }
   ],
   "source": [
    "ridge_sklearn = Ridge()\n",
    "ridge_sklearn.fit(X, y)\n",
    "\n",
    "# predict the value\n",
    "y_pred_sklearn = ridge_sklearn.predict(X)\n",
    "score = r2_score(y, y_pred_sklearn)\n",
    "print(\"=\"*100)\n",
    "print(\"R2 score of the model is {}\".format(score))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Acknowledgments\n",
    "Thanks to Pavithra Devi M for creating the Notebook [Lasso and Ridge Regression from scratch](https://www.kaggle.com/code/ninjaac/lasso-and-ridge-regression-from-scratch), lisensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0). It inspires the majority of the content in this chapter."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}