IoT_Attack_Anomaly.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/AthanasiosDelis/faster-kan.git\n",
        "!cd faster-kan && pip install ."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e7T5l90saU1V",
        "outputId": "79a30228-9ea4-44e9-95ea-64e27e074b1b"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'faster-kan'...\n",
            "remote: Enumerating objects: 356, done.\u001b[K\n",
            "remote: Counting objects: 100% (146/146), done.\u001b[K\n",
            "remote: Compressing objects: 100% (86/86), done.\u001b[K\n",
            "remote: Total 356 (delta 86), reused 111 (delta 58), pack-reused 210\u001b[K\n",
            "Receiving objects: 100% (356/356), 955.13 KiB | 25.13 MiB/s, done.\n",
            "Resolving deltas: 100% (167/167), done.\n",
            "Processing /content/faster-kan\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torch>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from efficient-kan==0.1.0) (2.3.0+cu121)\n",
            "Collecting pytest>=8.2.0 (from efficient-kan==0.1.0)\n",
            "  Downloading pytest-8.2.2-py3-none-any.whl (339 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m339.9/339.9 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.66.2 in /usr/local/lib/python3.10/dist-packages (from efficient-kan==0.1.0) (4.66.4)\n",
            "Requirement already satisfied: torchvision>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from efficient-kan==0.1.0) (0.18.0+cu121)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.10/dist-packages (from pytest>=8.2.0->efficient-kan==0.1.0) (2.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from pytest>=8.2.0->efficient-kan==0.1.0) (24.1)\n",
            "Requirement already satisfied: pluggy<2.0,>=1.5 in /usr/local/lib/python3.10/dist-packages (from pytest>=8.2.0->efficient-kan==0.1.0) (1.5.0)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /usr/local/lib/python3.10/dist-packages (from pytest>=8.2.0->efficient-kan==0.1.0) (1.2.1)\n",
            "Requirement already satisfied: tomli>=1 in /usr/local/lib/python3.10/dist-packages (from pytest>=8.2.0->efficient-kan==0.1.0) (2.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (3.14.0)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2.3.0->efficient-kan==0.1.0) (2.3.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.3.0->efficient-kan==0.1.0)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision>=0.18.0->efficient-kan==0.1.0) (1.25.2)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision>=0.18.0->efficient-kan==0.1.0) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.3.0->efficient-kan==0.1.0) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=2.3.0->efficient-kan==0.1.0) (1.3.0)\n",
            "Building wheels for collected packages: efficient-kan\n",
            "  Building wheel for efficient-kan (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for efficient-kan: filename=efficient_kan-0.1.0-py3-none-any.whl size=31507 sha256=3e173918c66c0a6ef0c6446894786fe512672f9a7285882323e743806ce98075\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/30/06/b086b3bba225918eb4fa22ef36492bb5bdf4ef2678075ec97b\n",
            "Successfully built efficient-kan\n",
            "Installing collected packages: pytest, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, efficient-kan\n",
            "  Attempting uninstall: pytest\n",
            "    Found existing installation: pytest 7.4.4\n",
            "    Uninstalling pytest-7.4.4:\n",
            "      Successfully uninstalled pytest-7.4.4\n",
            "Successfully installed efficient-kan-0.1.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 pytest-8.2.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rJeNC631D-fX",
        "outputId": "449180c2-d8a0-4c15-d8f3-558a1a76009c"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade scikit-learn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TRo5KNS1eC6O",
        "outputId": "5b6cc5f8-708b-4101-b233-5ca9fbb487d8"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Collecting scikit-learn\n",
            "  Downloading scikit_learn-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.3/13.3 MB\u001b[0m \u001b[31m72.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.25.2)\n",
            "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.5.0)\n",
            "Installing collected packages: scikit-learn\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 1.2.2\n",
            "    Uninstalling scikit-learn-1.2.2:\n",
            "      Successfully uninstalled scikit-learn-1.2.2\n",
            "Successfully installed scikit-learn-1.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ***Libraries :-***"
      ],
      "metadata": {
        "id": "sjh_RdwmD8T2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm\n",
        "\n",
        "import math\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from tabulate import tabulate\n",
        "import seaborn as sns\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler, TargetEncoder, LabelEncoder\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "\n",
        "from fasterkan import fasterkan as fkan\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "metadata": {
        "id": "Gq806lAMtn7J"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ***Preprocessing :-***"
      ],
      "metadata": {
        "id": "4au4OfxNR1Uh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "filePath = \"/content/drive/MyDrive/mainSimulationAccessTraces.csv\"\n",
        "df = pd.read_csv(filePath)\n",
        "\n",
        "# For Missing Data\n",
        "df['accessedNodeType'] = df['accessedNodeType'].fillna(value='/Malicious')\n",
        "\n",
        "# For Unexpected Data\n",
        "df.loc[df.value=='twenty',\"value\"] = '20.0'\n",
        "df.loc[df.value=='false',\"value\"] = '0'\n",
        "df.loc[df.value=='true',\"value\"] = '1'\n",
        "df.loc[df.value=='none',\"value\"] = '0'\n",
        "df.loc[df.value=='0',\"value\"] = '0.0'\n",
        "df['value'] = df['value'].fillna(value='60.0')\n",
        "df = df.drop(df.index[df.value.str.contains(\"org.*\")])\n",
        "df.value = df.value.astype(float)\n",
        "\n",
        "# Dropping TimeStamp Column\n",
        "df = df.drop('timestamp', axis=1)\n",
        "\n",
        "# Converting Label class from Categorical to Numerical class\n",
        "def cat2num(x) :\n",
        "  if x == 'normal' :\n",
        "    return 0\n",
        "  else :\n",
        "    return 1\n",
        "\n",
        "df['normality'] = df['normality'].apply(cat2num)\n",
        "\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        },
        "id": "uh2h7HbtH7zn",
        "outputId": "fb806832-ff43-4bfe-9867-75e82ce4967b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        sourceID          sourceAddress       sourceType   sourceLocation  \\\n",
              "0  lightcontrol2  /agent2/lightcontrol2  /lightControler   BedroomParents   \n",
              "1  lightcontrol3  /agent3/lightcontrol3  /lightControler      Dinningroom   \n",
              "2  lightcontrol1  /agent1/lightcontrol1  /lightControler  BedroomChildren   \n",
              "3  lightcontrol4  /agent4/lightcontrol4  /lightControler          Kitchen   \n",
              "4      movement4      /agent4/movement4  /movementSensor          Kitchen   \n",
              "\n",
              "  destinationServiceAddress destinationServiceType destinationLocation  \\\n",
              "0     /agent2/lightcontrol2        /lightControler      BedroomParents   \n",
              "1     /agent3/lightcontrol3        /lightControler         Dinningroom   \n",
              "2     /agent1/lightcontrol1        /lightControler     BedroomChildren   \n",
              "3     /agent4/lightcontrol4        /lightControler             Kitchen   \n",
              "4         /agent4/movement4        /movementSensor             Kitchen   \n",
              "\n",
              "     accessedNodeAddress accessedNodeType        operation  value  normality  \n",
              "0  /agent2/lightcontrol2  /lightControler  registerService    0.0          0  \n",
              "1  /agent3/lightcontrol3  /lightControler  registerService    0.0          0  \n",
              "2  /agent1/lightcontrol1  /lightControler  registerService    0.0          0  \n",
              "3  /agent4/lightcontrol4  /lightControler  registerService    0.0          0  \n",
              "4      /agent4/movement4  /movementSensor  registerService    0.0          0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-885c80ba-049f-4696-86dd-16ad8fd69533\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sourceID</th>\n",
              "      <th>sourceAddress</th>\n",
              "      <th>sourceType</th>\n",
              "      <th>sourceLocation</th>\n",
              "      <th>destinationServiceAddress</th>\n",
              "      <th>destinationServiceType</th>\n",
              "      <th>destinationLocation</th>\n",
              "      <th>accessedNodeAddress</th>\n",
              "      <th>accessedNodeType</th>\n",
              "      <th>operation</th>\n",
              "      <th>value</th>\n",
              "      <th>normality</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>lightcontrol2</td>\n",
              "      <td>/agent2/lightcontrol2</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>BedroomParents</td>\n",
              "      <td>/agent2/lightcontrol2</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>BedroomParents</td>\n",
              "      <td>/agent2/lightcontrol2</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>registerService</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>lightcontrol3</td>\n",
              "      <td>/agent3/lightcontrol3</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>Dinningroom</td>\n",
              "      <td>/agent3/lightcontrol3</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>Dinningroom</td>\n",
              "      <td>/agent3/lightcontrol3</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>registerService</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>lightcontrol1</td>\n",
              "      <td>/agent1/lightcontrol1</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>BedroomChildren</td>\n",
              "      <td>/agent1/lightcontrol1</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>BedroomChildren</td>\n",
              "      <td>/agent1/lightcontrol1</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>registerService</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>lightcontrol4</td>\n",
              "      <td>/agent4/lightcontrol4</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>Kitchen</td>\n",
              "      <td>/agent4/lightcontrol4</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>Kitchen</td>\n",
              "      <td>/agent4/lightcontrol4</td>\n",
              "      <td>/lightControler</td>\n",
              "      <td>registerService</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>movement4</td>\n",
              "      <td>/agent4/movement4</td>\n",
              "      <td>/movementSensor</td>\n",
              "      <td>Kitchen</td>\n",
              "      <td>/agent4/movement4</td>\n",
              "      <td>/movementSensor</td>\n",
              "      <td>Kitchen</td>\n",
              "      <td>/agent4/movement4</td>\n",
              "      <td>/movementSensor</td>\n",
              "      <td>registerService</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-885c80ba-049f-4696-86dd-16ad8fd69533')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-885c80ba-049f-4696-86dd-16ad8fd69533 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-885c80ba-049f-4696-86dd-16ad8fd69533');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-44b50c7c-70ec-4961-b089-6bc7531e474e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-44b50c7c-70ec-4961-b089-6bc7531e474e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-44b50c7c-70ec-4961-b089-6bc7531e474e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df"
            }
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ***Data Exploration :-***"
      ],
      "metadata": {
        "id": "568UnujseLrn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sourceLocationData = []\n",
        "\n",
        "for i, j in zip(df.sourceLocation.value_counts().index, df.sourceLocation.value_counts().values):\n",
        "  sourceLocationData.append([i, j])\n",
        "\n",
        "sourceLocationData = sorted(sourceLocationData, key=lambda x: x[1], reverse=True)\n",
        "print(tabulate(sourceLocationData, headers=[\"Source Location\", \"Count\"], tablefmt=\"grid\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-RPPzD9dUeit",
        "outputId": "379c7f2d-add1-4dff-ebb4-19e4d4f7232f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------+---------+\n",
            "| Source Location   |   Count |\n",
            "+===================+=========+\n",
            "| Garage            |   39499 |\n",
            "+-------------------+---------+\n",
            "| Watterroom        |   38368 |\n",
            "+-------------------+---------+\n",
            "| Entrance          |   38217 |\n",
            "+-------------------+---------+\n",
            "| Bathroom          |   28461 |\n",
            "+-------------------+---------+\n",
            "| Showerroom        |   28315 |\n",
            "+-------------------+---------+\n",
            "| Kitchen           |   18962 |\n",
            "+-------------------+---------+\n",
            "| Dinningroom       |   13429 |\n",
            "+-------------------+---------+\n",
            "| BedroomChildren   |   12836 |\n",
            "+-------------------+---------+\n",
            "| room_6            |   11642 |\n",
            "+-------------------+---------+\n",
            "| Bedroom           |   11009 |\n",
            "+-------------------+---------+\n",
            "| BedroomParents    |   10946 |\n",
            "+-------------------+---------+\n",
            "| room_8            |   10798 |\n",
            "+-------------------+---------+\n",
            "| room_9            |   10752 |\n",
            "+-------------------+---------+\n",
            "| Livingroom        |   10620 |\n",
            "+-------------------+---------+\n",
            "| room_2            |   10606 |\n",
            "+-------------------+---------+\n",
            "| room_1            |   10601 |\n",
            "+-------------------+---------+\n",
            "| room_3            |   10597 |\n",
            "+-------------------+---------+\n",
            "| room_5            |   10596 |\n",
            "+-------------------+---------+\n",
            "| room_4            |   10572 |\n",
            "+-------------------+---------+\n",
            "| room_7            |   10558 |\n",
            "+-------------------+---------+\n",
            "| room_10           |   10557 |\n",
            "+-------------------+---------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "destinationLocationData = []\n",
        "\n",
        "for i, j in zip(df.destinationLocation.value_counts().index, df.destinationLocation.value_counts().values):\n",
        "  destinationLocationData.append([i, j])\n",
        "\n",
        "destinationLocationData = sorted(destinationLocationData, key=lambda x: x[1], reverse=True)\n",
        "print(tabulate(destinationLocationData, headers=[\"Destination Location\", \"Count\"], tablefmt=\"grid\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ce5uPZzzcv7W",
        "outputId": "3b144e45-522d-4c58-8739-4ecb12c7817a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------------------+---------+\n",
            "| Destination Location   |   Count |\n",
            "+========================+=========+\n",
            "| Garage                 |   57100 |\n",
            "+------------------------+---------+\n",
            "| Entrance               |   54448 |\n",
            "+------------------------+---------+\n",
            "| Watterroom             |   38524 |\n",
            "+------------------------+---------+\n",
            "| Kitchen                |   19437 |\n",
            "+------------------------+---------+\n",
            "| BedroomChildren        |   12768 |\n",
            "+------------------------+---------+\n",
            "| Bathroom               |   12684 |\n",
            "+------------------------+---------+\n",
            "| Showerroom             |   12583 |\n",
            "+------------------------+---------+\n",
            "| BedroomParents         |   10938 |\n",
            "+------------------------+---------+\n",
            "| room_9                 |   10843 |\n",
            "+------------------------+---------+\n",
            "| room_2                 |   10828 |\n",
            "+------------------------+---------+\n",
            "| room_1                 |   10805 |\n",
            "+------------------------+---------+\n",
            "| Livingroom             |   10781 |\n",
            "+------------------------+---------+\n",
            "| Dinningroom            |   10759 |\n",
            "+------------------------+---------+\n",
            "| room_6                 |   10741 |\n",
            "+------------------------+---------+\n",
            "| room_3                 |   10732 |\n",
            "+------------------------+---------+\n",
            "| room_4                 |   10708 |\n",
            "+------------------------+---------+\n",
            "| room_5                 |   10687 |\n",
            "+------------------------+---------+\n",
            "| room_8                 |   10651 |\n",
            "+------------------------+---------+\n",
            "| room_7                 |   10649 |\n",
            "+------------------------+---------+\n",
            "| room_10                |   10648 |\n",
            "+------------------------+---------+\n",
            "| Bedroom                |   10627 |\n",
            "+------------------------+---------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accessedNodeTypeData = []\n",
        "\n",
        "for i, j in zip(df.accessedNodeType.value_counts().index, df.accessedNodeType.value_counts().values):\n",
        "  accessedNodeTypeData.append([i, j])\n",
        "\n",
        "accessedNodeTypeData = sorted(accessedNodeTypeData, key=lambda x: x[1], reverse=True)\n",
        "print(tabulate(accessedNodeTypeData, headers=[\"Accessed Node Type\", \"Count\"], tablefmt=\"grid\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8yq_pNw2eG1S",
        "outputId": "da1060c0-0a30-42c0-b57a-0dcea63bb1c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------+---------+\n",
            "| Accessed Node Type   |   Count |\n",
            "+======================+=========+\n",
            "| /sensorService       |  130155 |\n",
            "+----------------------+---------+\n",
            "| /derived/boolean     |   94725 |\n",
            "+----------------------+---------+\n",
            "| /basic/number        |   90010 |\n",
            "+----------------------+---------+\n",
            "| /basic/text          |   42428 |\n",
            "+----------------------+---------+\n",
            "| /thermostat          |     313 |\n",
            "+----------------------+---------+\n",
            "| /Malicious           |     148 |\n",
            "+----------------------+---------+\n",
            "| /basic/composed      |     100 |\n",
            "+----------------------+---------+\n",
            "| /lightControler      |      23 |\n",
            "+----------------------+---------+\n",
            "| /movementSensor      |      22 |\n",
            "+----------------------+---------+\n",
            "| /batteryService      |       6 |\n",
            "+----------------------+---------+\n",
            "| /doorLockService     |       5 |\n",
            "+----------------------+---------+\n",
            "| /washingService      |       3 |\n",
            "+----------------------+---------+\n",
            "| /smartPhone          |       3 |\n",
            "+----------------------+---------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "operationData = []\n",
        "\n",
        "for i, j in zip(df.operation.value_counts().index, df.operation.value_counts().values):\n",
        "  operationData.append([i, j])\n",
        "\n",
        "operationData = sorted(operationData, key=lambda x: x[1], reverse=True)\n",
        "print(tabulate(operationData, headers=[\"Accessed Node Type\", \"Count\"], tablefmt=\"grid\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MyPb46UNowgC",
        "outputId": "944e8736-529b-4199-855d-f0d2bc87c94a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------+---------+\n",
            "| Accessed Node Type   |   Count |\n",
            "+======================+=========+\n",
            "| read                 |  248061 |\n",
            "+----------------------+---------+\n",
            "| write                |  109648 |\n",
            "+----------------------+---------+\n",
            "| lockSubtree          |     148 |\n",
            "+----------------------+---------+\n",
            "| registerService      |      84 |\n",
            "+----------------------+---------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "normalityData = []\n",
        "\n",
        "for i, j in zip(df.normality.value_counts().index, df.normality.value_counts().values):\n",
        "  normalityData.append([i, j])\n",
        "\n",
        "normalityData = sorted(normalityData, key=lambda x: x[1], reverse=True)\n",
        "print(tabulate(normalityData, headers=[\"Normality\", \"Count\"], tablefmt=\"grid\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IGYanVtUb5GT",
        "outputId": "c5052695-54d6-4c1b-f7fd-13d7296e3431"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------------------------+---------+\n",
            "| Normality                     |   Count |\n",
            "+===============================+=========+\n",
            "| normal                        |  347924 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(DoSattack)          |    5780 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(scan)               |    1547 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(malitiousControl)   |     889 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(malitiousOperation) |     805 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(spying)             |     532 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(dataProbing)        |     342 |\n",
            "+-------------------------------+---------+\n",
            "| anomalous(wrongSetUp)         |     122 |\n",
            "+-------------------------------+---------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ***Kolmogorov-Arnold-Network Implementation :-***"
      ],
      "metadata": {
        "id": "X5t-N66FmqDD"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Kolmogorov-Arnold Representation Theorem :-\n",
        "  * $f(x) = f(x_{1},...,x_{n}) = ∑_{q=1}^{2n + 1}𝚽_{q}(∑_{p=1}^{n}Φ_{q, p}(x_{p}))$\n",
        "* Gausian Radial Basis Function (RBF) :-\n",
        "  * Computationally efficient and easy to calculate, with significant speed-ups in forward and backward passes.\n",
        "  * $b_{i}(u) = e^{(-(u - u_{i})^{2}/h)}$\n",
        "* Reflextional Switch Activation Function (RSWAF) :-\n",
        "  * Uses function which have reflextionary symmetry, allows us to retain performance while reducing computation time.\n",
        "  * $b_{i}(u) = 1 - (tanh((u - u_{i})/h))^{2}$"
      ],
      "metadata": {
        "id": "ikV33U4luJge"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class KANLayer(nn.Module):\n",
        "  def __init__(self,\n",
        "               in_features,\n",
        "               out_features,\n",
        "               grid_size=5,\n",
        "               spline_order=3,\n",
        "               scale_noise=0.1,\n",
        "               scale_base=1.0,\n",
        "               scale_spline=1.0,\n",
        "               enable_standalone_scale_spline=True,\n",
        "               base_activation=torch.nn.SiLU,\n",
        "               grid_eps=0.2,\n",
        "               grid_range=[-1, 1],) :\n",
        "\n",
        "    super(KANLayer, self).__init__()\n",
        "    self.in_features = in_features\n",
        "    self.out_features = out_features\n",
        "    self.grid_size = grid_size\n",
        "    self.spline_order = spline_order\n",
        "\n",
        "    h = (grid_range[1] - grid_range[0])/grid_size\n",
        "\n",
        "    grid = ((torch.arange(-spline_order, grid_size + spline_order + 1) * h + grid_range[0]).expand(in_features, -1).contiguous())\n",
        "    self.register_buffer(\"grid\", grid)\n",
        "\n",
        "    self.base_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n",
        "    self.spline_weight = torch.nn.Parameter(torch.Tensor(out_features, in_features, grid_size + spline_order))\n",
        "\n",
        "    if enable_standalone_scale_spline:\n",
        "      self.spline_scaler = torch.nn.Parameter(torch.Tensor(out_features, in_features))\n",
        "\n",
        "    self.scale_noise = scale_noise\n",
        "    self.scale_base = scale_base\n",
        "    self.scale_spline = scale_spline\n",
        "    self.enable_standalone_scale_spline = enable_standalone_scale_spline\n",
        "    self.base_activation = base_activation()\n",
        "    self.grid_eps = grid_eps\n",
        "\n",
        "    self.reset_parameters()\n",
        "\n",
        "  def reset_parameters(self) :\n",
        "    torch.nn.init.kaiming_uniform_(self.base_weight, a=math.sqrt(5) * self.scale_base)\n",
        "\n",
        "    with torch.no_grad() :\n",
        "      noise = ((torch.rand(self.grid_size + 1, self.in_features, self.out_features)- 1 / 2)* self.scale_noise/ self.grid_size)\n",
        "      self.spline_weight.data.copy_((self.scale_spline if not self.enable_standalone_scale_spline else 1.0)* self.curve2coeff(self.grid.T[self.spline_order : -self.spline_order],noise,))\n",
        "\n",
        "      if self.enable_standalone_scale_spline :\n",
        "        torch.nn.init.kaiming_uniform_(self.spline_scaler, a=math.sqrt(5) * self.scale_spline)\n",
        "\n",
        "  def b_splines(self, x : torch.Tensor) :\n",
        "    assert x.dim() == 2 and x.size(1) == self.in_features\n",
        "\n",
        "    grid : torch.Tensor = (self.grid)\n",
        "    x = x.unsqueeze(-1)\n",
        "    bases = ((x >= grid[:, :-1]) & (x < grid[:, 1:])).to(x.dtype)\n",
        "\n",
        "    for k in range(1, self.spline_order + 1) :\n",
        "      bases = ((x - grid[:, : -(k + 1)])/ (grid[:, k:-1] - grid[:, : -(k + 1)])* bases[:, :, :-1]) + ((grid[:, k + 1 :] - x)/ (grid[:, k + 1 :] - grid[:, 1:(-k)])* bases[:, :, 1:])\n",
        "\n",
        "    assert bases.size() == (x.size(0), self.in_features, self.grid_size + self.spline_order,)\n",
        "    return bases.contiguous()\n",
        "\n",
        "  def curve2coeff(self, x : torch.Tensor, y : torch.Tensor) :\n",
        "    assert x.dim() == 2 and x.size(1) == self.in_features\n",
        "    assert y.size() == (x.size(0), self.in_features, self.out_features)\n",
        "\n",
        "    A = self.b_splines(x).transpose(0, 1)\n",
        "    B = y.transpose(0, 1)\n",
        "    solution = torch.linalg.lstsq(A, B).solution\n",
        "    result = solution.permute(2, 0, 1)\n",
        "\n",
        "    assert result.size() == (self.out_features, self.in_features, self.grid_size + self.spline_order,)\n",
        "    return result.contiguous()\n",
        "\n",
        "    @property\n",
        "    def scaled_spline_weight(self):\n",
        "      return self.spline_weight * (self.spline_scaler.unsqueeze(-1) if self.enable_standalone_scale_spline else 1.0)\n",
        "\n",
        "  def forward(self, x : torch.Tensor) :\n",
        "    assert x.size(-1) == self.in_features\n",
        "    original_shape = x.shape\n",
        "    x = x.view(-1, self.in_features)\n",
        "\n",
        "    base_output = F.linear(self.base_activation(x), self.base_weight)\n",
        "    spline_output = F.linear(self.b_splines(x).view(x.size(0), -1), self.scaled_spline_weight.view(self.out_features, -1),)\n",
        "    output = base_output + spline_output\n",
        "\n",
        "    output = output.view(*original_shape[:-1], self.out_features)\n",
        "    return output\n",
        "\n",
        "  def update_grid(self, x : torch.Tensor, margin=0.01) :\n",
        "    assert x.dim() == 2 and x.size(1) == self.in_features\n",
        "    batch = x.size(0)\n",
        "\n",
        "    splines = self.b_splines(x)\n",
        "    splines = splines.permute(1, 0, 2)\n",
        "\n",
        "    orig_coeff = self.scaled_spline_weight\n",
        "    orig_coeff = orig_coeff.permute(1, 2, 0)\n",
        "\n",
        "    unreduced_spline_output = torch.bmm(splines, orig_coeff)\n",
        "    unreduced_spline_output = unreduced_spline_output.permute(1, 0, 2)\n",
        "\n",
        "    x_sorted = torch.sort(x, dim=0)[0]\n",
        "    grid_adaptive = x_sorted[torch.linspace(0, batch - 1, self.grid_size + 1, dtype=torch.int64, device=x.device)]\n",
        "\n",
        "    uniform_step = (x_sorted[-1] - x_sorted[0] + 2 * margin) / self.grid_size\n",
        "    grid_uniform = (torch.arange(self.grid_size + 1, dtype=torch.float32, device=x.device).unsqueeze(1) * uniform_step + x_sorted[0] - margin)\n",
        "\n",
        "    grid = self.grid_eps * grid_uniform + (1 - self.grid_eps) * grid_adaptive\n",
        "    grid = torch.concatenate([grid[:1] - uniform_step * torch.arange(self.spline_order, 0, -1, device=x.device).unsqueeze(1), grid, grid[-1:] + uniform_step * torch.arange(1, self.spline_order + 1, device=x.device).unsqueeze(1),], dim=0,)\n",
        "\n",
        "    self.grid.copy_(grid.T)\n",
        "    self.spline_weight.data.copy_(self.curve2coeff(x, unreduced_spline_output))\n",
        "\n",
        "  def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0) :\n",
        "    l1_fake = self.spline_weight.abs().mean(-1)\n",
        "    regularization_loss_activation = l1_fake.sum()\n",
        "    p = l1_fake / regularization_loss_activation\n",
        "    regularization_loss_entropy = -torch.sum(p * p.log())\n",
        "\n",
        "    return (regularize_activation * regularization_loss_activation + regularize_entropy * regularization_loss_entropy)\n",
        "\n",
        "class KAN(torch.nn.Module):\n",
        "  def __init__(\n",
        "      self,\n",
        "      layers_hidden,\n",
        "      grid_size=5,\n",
        "      spline_order=3,\n",
        "      scale_noise=0.1,\n",
        "      scale_base=1.0,\n",
        "      scale_spline=1.0,\n",
        "      base_activation=torch.nn.SiLU,\n",
        "      grid_eps=0.02,\n",
        "      grid_range=[-1, 1],\n",
        "  ) :\n",
        "    super(KAN, self).__init__()\n",
        "    self.grid_size = grid_size\n",
        "    self.spline_order = spline_order\n",
        "\n",
        "    self.layers = torch.nn.ModuleList()\n",
        "    for in_features, out_features in zip(layers_hidden, layers_hidden[1:]) :\n",
        "      self.layers.append(KANLayer(\n",
        "          in_features,\n",
        "          out_features,\n",
        "          grid_size=grid_size,\n",
        "          spline_order=spline_order,\n",
        "          scale_noise=scale_noise,\n",
        "          scale_base=scale_base,\n",
        "          scale_spline=scale_spline,\n",
        "          base_activation=base_activation,\n",
        "          grid_eps=grid_eps,\n",
        "          grid_range=grid_range, ))\n",
        "\n",
        "  def forward(self, x: torch.Tensor, update_grid=False) :\n",
        "    for layer in self.layers :\n",
        "      if update_grid :\n",
        "        layer.update_grid(x)\n",
        "        x = layer(x)\n",
        "    return x\n",
        "\n",
        "  def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0):\n",
        "    return sum(layer.regularization_loss(regularize_activation, regularize_entropy) for layer in self.layers)"
      ],
      "metadata": {
        "id": "0nE8iEfAmpfX"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X0 = df.iloc[:, 0].values.reshape(-1, 1)\n",
        "X1 = df.iloc[:, 1].values.reshape(-1, 1)\n",
        "X2 = df.iloc[:, 2].values.reshape(-1, 1)\n",
        "X3 = df.iloc[:, 3].values.reshape(-1, 1)\n",
        "X4 = df.iloc[:, 4].values.reshape(-1, 1)\n",
        "X5 = df.iloc[:, 5].values.reshape(-1, 1)\n",
        "X6 = df.iloc[:, 6].values.reshape(-1, 1)\n",
        "X7 = df.iloc[:, 7].values.reshape(-1, 1)\n",
        "X8 = df.iloc[:, 8].values.reshape(-1, 1)\n",
        "X9 = df.iloc[:, 9].values.reshape(-1, 1)\n",
        "X10 = df.iloc[:, 10].values.reshape(-1, 1)\n",
        "\n",
        "y = df['normality'].values\n",
        "\n",
        "X0 = TargetEncoder(target_type=\"binary\").fit_transform(X0, y)\n",
        "X1 = TargetEncoder(target_type=\"binary\").fit_transform(X1, y)\n",
        "X2 = TargetEncoder(target_type=\"binary\").fit_transform(X2, y)\n",
        "X3 = TargetEncoder(target_type=\"binary\").fit_transform(X3, y)\n",
        "X4 = TargetEncoder(target_type=\"binary\").fit_transform(X4, y)\n",
        "X5 = TargetEncoder(target_type=\"binary\").fit_transform(X5, y)\n",
        "X6 = TargetEncoder(target_type=\"binary\").fit_transform(X6, y)\n",
        "X7 = TargetEncoder(target_type=\"binary\").fit_transform(X7, y)\n",
        "X8 = TargetEncoder(target_type=\"binary\").fit_transform(X8, y)\n",
        "X9 = TargetEncoder(target_type=\"binary\").fit_transform(X9, y)\n",
        "\n",
        "X10 = StandardScaler().fit_transform(X10.reshape(-1, 1))\n",
        "\n",
        "X = np.concatenate((X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10), axis=1)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=41)\n",
        "\n",
        "X_train = torch.tensor(X_train, dtype=torch.float32)\n",
        "X_test = torch.tensor(X_test, dtype=torch.float32)\n",
        "\n",
        "y_train = torch.tensor(y_train, dtype=torch.long)\n",
        "y_test = torch.tensor(y_test, dtype=torch.long)"
      ],
      "metadata": {
        "id": "ZdeVvGntsHJR"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Training Data Shape : \", X_train.shape)\n",
        "print(\"Testing Data Shape : \", X_test.shape)\n",
        "print(\"Training Label Shape : \", y_train.shape)\n",
        "print(\"Testing Label Shape : \", y_test.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g6GKzBz-exbJ",
        "outputId": "6f4bc684-01ac-49a2-d9b8-569c4ddc0dae"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training Data Shape :  torch.Size([286352, 11])\n",
            "Testing Data Shape :  torch.Size([71589, 11])\n",
            "Training Label Shape :  torch.Size([286352])\n",
            "Testing Label Shape :  torch.Size([71589])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainLoader = DataLoader(torch.utils.data.TensorDataset(X_train, y_train), batch_size=64, shuffle=True)\n",
        "valLoader = DataLoader(torch.utils.data.TensorDataset(X_test, y_test), batch_size=64, shuffle=False)"
      ],
      "metadata": {
        "id": "YXMzF0kjfEr0"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def count_parameters(model):\n",
        "    total_params = sum(p.numel() for p in model.parameters())\n",
        "    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "    return total_params, trainable_params"
      ],
      "metadata": {
        "id": "ArmsaNeNMsZU"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class EarlyStopping:\n",
        "    def __init__(self, patience=5, min_delta=0):\n",
        "        self.patience = patience\n",
        "        self.min_delta = min_delta\n",
        "        self.counter = 0\n",
        "        self.best_loss = None\n",
        "        self.early_stop = False\n",
        "\n",
        "    def __call__(self, val_loss):\n",
        "        if self.best_loss is None:\n",
        "            self.best_loss = val_loss\n",
        "        elif val_loss > self.best_loss - self.min_delta:\n",
        "            self.counter += 1\n",
        "            if self.counter >= self.patience:\n",
        "                self.early_stop = True\n",
        "        else:\n",
        "            self.best_loss = val_loss\n",
        "            self.counter = 0"
      ],
      "metadata": {
        "id": "82x8ZgVKWHX7"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "***Efficient -KAN***"
      ],
      "metadata": {
        "id": "rcMEwZxvfPVV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = KAN([X_train.shape[1], 100, len(set(y_train))])\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.to(device)\n",
        "\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-5)\n",
        "scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n",
        "criterion = torch.nn.CrossEntropyLoss()"
      ],
      "metadata": {
        "id": "277YnuMqmfps"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for name, param in model.named_parameters():\n",
        "    if not param.requires_grad:\n",
        "        print(f\"Parameter {name} does not require grad!\")\n",
        "    else: print(f\"Parameter {name} requires grad!\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7T5VaT3hv5y2",
        "outputId": "4157c92a-bead-470b-c22b-dc36ef5960e7"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Parameter layers.0.base_weight requires grad!\n",
            "Parameter layers.0.spline_weight requires grad!\n",
            "Parameter layers.0.spline_scaler requires grad!\n",
            "Parameter layers.1.base_weight requires grad!\n",
            "Parameter layers.1.spline_weight requires grad!\n",
            "Parameter layers.1.spline_scaler requires grad!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train\n",
        "epochs = 15\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  model.train()\n",
        "  epoch_loss = 0\n",
        "  epoch_accuracy = 0\n",
        "  with tqdm(trainLoader, unit=\"batch\") as pbar:\n",
        "        for i, (feats, labels) in enumerate(pbar):\n",
        "            feats = feats.to(device).requires_grad_(True)\n",
        "            labels = labels.to(device)\n",
        "            #if not feats.requires_grad : print(\"Inputs do not require gradients!\")\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(feats)\n",
        "            #if output.grad_fn is None : print(\"Output does not have a grad_fn!\")\n",
        "\n",
        "            loss = criterion(output, labels)\n",
        "            #if loss.grad_fn is None : print(\"Loss does not have a grad_fn!\")\n",
        "\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_accuracy += accuracy.item()\n",
        "            pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer.param_groups[0]['lr'])\n",
        "\n",
        "  scheduler.step()\n",
        "  print(f\"Epoch {epoch+1}/{epochs} - Loss: {epoch_loss/len(trainLoader):.4f} - Accuracy: {epoch_accuracy/len(trainLoader):.4f}\")\n",
        "\n",
        "model.eval()\n",
        "\n",
        "val_loss = 0\n",
        "val_accuracy = 0\n",
        "with torch.no_grad():\n",
        "    for feats, labels in valLoader:\n",
        "        feats, labels = feats.to(device), labels.to(device)\n",
        "        output = model(feats)\n",
        "        loss = criterion(output, labels)\n",
        "        accuracy = (output.argmax(dim=1) == labels).float().mean()\n",
        "        val_loss += loss.item()\n",
        "        val_accuracy += accuracy.item()\n",
        "\n",
        "print(f\"Validation - Loss: {val_loss/len(valLoader):.4f} - Accuracy: {val_accuracy/len(valLoader):.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 460
        },
        "id": "JaHp0gDTrV9x",
        "outputId": "53b2906b-bbf8-445a-ddfd-9cec1852aedd"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [00:18<00:00, 243.30batch/s, accuracy=0, loss=2.38, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/15 - Loss: 2.4296 - Accuracy: 0.0253\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [00:19<00:00, 232.04batch/s, accuracy=0, loss=2.41, lr=0.0008]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2/15 - Loss: 2.4296 - Accuracy: 0.0253\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 93%|█████████▎| 4157/4475 [00:18<00:01, 225.65batch/s, accuracy=0.0156, loss=2.39, lr=0.00064]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-8daeb84238f3>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mepoch_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0mepoch_accuracy\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m             \u001b[0mpbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_postfix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_groups\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m   \u001b[0mscheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36mset_postfix\u001b[0;34m(self, ordered_dict, refresh, **kwargs)\u001b[0m\n\u001b[1;32m   1429\u001b[0m                                  for key in postfix.keys())\n\u001b[1;32m   1430\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrefresh\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1431\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1432\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1433\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_postfix_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrefresh\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36mrefresh\u001b[0;34m(self, nolock, lock_args)\u001b[0m\n\u001b[1;32m   1345\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1346\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1347\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1348\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mnolock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36mdisplay\u001b[0;34m(self, msg, pos)\u001b[0m\n\u001b[1;32m   1493\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1494\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoveto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1495\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__str__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1496\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1497\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoveto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36mprint_status\u001b[0;34m(s)\u001b[0m\n\u001b[1;32m    457\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mprint_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    458\u001b[0m             \u001b[0mlen_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdisp_len\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 459\u001b[0;31m             \u001b[0mfp_write\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\r'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ms\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_len\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    460\u001b[0m             \u001b[0mlast_len\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen_s\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/std.py\u001b[0m in \u001b[0;36mfp_write\u001b[0;34m(s)\u001b[0m\n\u001b[1;32m    450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mfp_write\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 452\u001b[0;31m             \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    453\u001b[0m             \u001b[0mfp_flush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tqdm/utils.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    194\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, string)\u001b[0m\n\u001b[1;32m    400\u001b[0m             \u001b[0mis_child\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_master_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m             \u001b[0;31m# only touch the buffer in the IO thread to avoid races\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 402\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    403\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_child\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                 \u001b[0;31m# mp.Pool cannot be trusted to flush promptly (or ever),\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/ipykernel/iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[0;34m(self, f)\u001b[0m\n\u001b[1;32m    201\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;31m# wake event thread (message content is ignored)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mb''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    204\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, data, flags, copy, track, routing_id, group)\u001b[0m\n\u001b[1;32m    618\u001b[0m                 )\n\u001b[1;32m    619\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 620\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m     def send_multipart(\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._send_copy\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/zmq/backend/cython/checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "parameter_kan = count_parameters(model)\n",
        "print(f\"Total Parameters : {parameter_kan[0]}\")\n",
        "print(f\"Trainable Parameters : {parameter_kan[1]}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DcBTmYTCQZT9",
        "outputId": "26a1c33f-83a9-4b02-b38c-8f4fdde8e781"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Parameters : 286363000\n",
            "Trainable Parameters : 286363000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "***Faster-KAN***"
      ],
      "metadata": {
        "id": "Bkl2TiH2OqV2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_flag = True\n",
        "\n",
        "model_ = fkan.FasterKAN([X_train.shape[1], 100, len(set(y_train))], grid_min=-1.2, grid_max=0.2, num_grids=5, exponent=2, inv_denominator=0.5, train_grid=train_flag, train_inv_denominator=train_flag).to(device)\n",
        "model_.to(device)\n",
        "\n",
        "optimizer_ = torch.optim.AdamW(model_.parameters(), lr=1e-3, weight_decay=1e-5)\n",
        "scheduler_ = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.6, patience=1, verbose=True)\n",
        "criterion_ = torch.nn.CrossEntropyLoss()\n",
        "earlyStopping = EarlyStopping(patience=7, min_delta=0.001)"
      ],
      "metadata": {
        "id": "gFQOQNem4F3P"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs_ = 100\n",
        "val_loss = 0.0\n",
        "\n",
        "for epoch in range(epochs_):\n",
        "  model_.train()\n",
        "  with tqdm(trainLoader) as pbar:\n",
        "    for i, (feats, labels) in enumerate(pbar):\n",
        "      feats = feats.to(device).requires_grad_(True)\n",
        "      labels = labels.to(device)\n",
        "\n",
        "      optimizer_.zero_grad()\n",
        "\n",
        "      output = model_(feats)\n",
        "\n",
        "      loss = criterion_(output, labels)\n",
        "      loss.backward()\n",
        "\n",
        "      torch.nn.utils.clip_grad_norm_(model_.parameters(), max_norm=1.0)\n",
        "\n",
        "      optimizer_.step()\n",
        "\n",
        "      accuracy = (output.argmax(dim=1) == labels.to(device)).float().mean()\n",
        "      pbar.set_postfix(loss=loss.item(), accuracy=accuracy.item(), lr=optimizer_.param_groups[0]['lr'])\n",
        "\n",
        "  model_.eval()\n",
        "  with torch.no_grad():\n",
        "    val_correct = 0\n",
        "    for feats, labels in valLoader:\n",
        "        feats = feats.to(device)\n",
        "        output = model_(feats)\n",
        "        val_loss += criterion_(output, labels.to(device)).item()\n",
        "        preds = output.argmax(dim=1)\n",
        "        val_correct += (preds == labels.to(device)).sum().item()\n",
        "\n",
        "\n",
        "  val_loss /= len(valLoader.dataset)\n",
        "  val_accuracy = val_correct / len(valLoader.dataset)\n",
        "\n",
        "  scheduler_.step(val_loss)\n",
        "\n",
        "  print(f\"Epoch {epoch + 1}, Val Loss: {val_loss}, Val Accuracy: {val_accuracy}\")\n",
        "  print(f\"Current Learning Rate: {optimizer.param_groups[0]['lr']}\")\n",
        "\n",
        "  earlyStopping(val_loss)\n",
        "  if earlyStopping.early_stop:\n",
        "    print(\"Triggering Early Stop!\")\n",
        "    break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7mzB4jqPP0X1",
        "outputId": "be725c21-3642-41ca-ec5d-1701974779c3"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:46<00:00, 12.92it/s, accuracy=1, loss=0.00233, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Val Loss: 0.0003442798067289987, Val Accuracy: 0.9914372319769797\n",
            "Current Learning Rate: 0.001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:43<00:00, 13.01it/s, accuracy=1, loss=2.5e-5, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2, Val Loss: 0.00025768938037364655, Val Accuracy: 0.9931553730321697\n",
            "Current Learning Rate: 0.001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:44<00:00, 12.97it/s, accuracy=1, loss=1.6e-5, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3, Val Loss: 0.00025329322583038, Val Accuracy: 0.9926105966000364\n",
            "Current Learning Rate: 0.001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:42<00:00, 13.07it/s, accuracy=1, loss=0.00098, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4, Val Loss: 0.000269417303026877, Val Accuracy: 0.9932950592968194\n",
            "Current Learning Rate: 0.001\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:42<00:00, 13.07it/s, accuracy=1, loss=0.0191, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5, Val Loss: 0.00036488787682472306, Val Accuracy: 0.9901660869686684\n",
            "Current Learning Rate: 0.0006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:42<00:00, 13.08it/s, accuracy=1, loss=0.000997, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6, Val Loss: 0.0003810813055150816, Val Accuracy: 0.983726550168322\n",
            "Current Learning Rate: 0.0006\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:43<00:00, 13.03it/s, accuracy=0.938, loss=0.372, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7, Val Loss: 0.00037272968605219436, Val Accuracy: 0.9899844948246239\n",
            "Current Learning Rate: 0.00035999999999999997\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4475/4475 [05:43<00:00, 13.02it/s, accuracy=0.938, loss=0.0666, lr=0.001]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8, Val Loss: 0.000431507375765532, Val Accuracy: 0.9896911536688597\n",
            "Current Learning Rate: 0.00035999999999999997\n",
            "Triggering Early Stop!\n"
          ]
        }
      ]
    }
  ]
}