Data Processing/data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ac914b24-8ea0-45e1-a630-d9c94c2456e9",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a08548ef-979c-4b84-a42c-aaef3e1e7ae3",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cbbb510-abd5-4656-8b67-2ed0b6b1d210",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "\n",
    "from boruta import BorutaPy\n",
    "from sklearn.feature_selection import SelectFdr, SequentialFeatureSelector, chi2\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "from sklearn.decomposition import PCA, FastICA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbadd5b7-8315-4880-b14f-e8eb173e377f",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0c763e0-0588-4204-8436-49451fceea25",
   "metadata": {},
   "outputs": [],
   "source": [
    "MCAD_DATASET_PATH = \"\"\n",
    "COLLECTED_DATASET_PATH = \"\"\n",
    "\n",
    "MCAD_CLASS_FILE_MAP = { \n",
    "    \"ddos\": [\n",
    "        \"attack_ddos_tcp.csv\",\n",
    "        \"attack_ddos_udp.csv\",\n",
    "        \"ddos_attack_scapy_new.csv\"\n",
    "    ],\n",
    "    \"probe\": [\n",
    "        \"attack_os_port_scan.csv\"\n",
    "    ],\n",
    "    \"web\": [\n",
    "        \"attack_sql_injection.csv\"\n",
    "    ],\n",
    "    \"r2l\": [\n",
    "        \"attack_bruteforce.csv\",\n",
    "        \"attack_cmd.csv\"\n",
    "    ],\n",
    "    \"u2r\": [\n",
    "        \"attack_vnc.csv\",\n",
    "        \"attack_samba.csv\"\n",
    "    ],\n",
    "    \"normal\": [\n",
    "        \"normal_ditg.csv\",\n",
    "        \"normal_internet1.csv\",\n",
    "        \"normal_internet2.csv\",\n",
    "        \"normal_internet3.csv\",\n",
    "        \"normal_iperf.csv\",\n",
    "    ]\n",
    "}\n",
    "COLLECTED_CLASS_FILE_MAP = {\n",
    "    \"ddos\": [\n",
    "        \"attack_ddos_icmp.csv\",\n",
    "        \"attack_ddos_tcp.csv\",\n",
    "        \"attack_ddos_udp.csv\",\n",
    "    ],\n",
    "    \"probe\": [\n",
    "        \"attack_nmap_probe.csv\"\n",
    "    ],\n",
    "    \"web\": [\n",
    "        \"attack_selenium_sqli.csv\",\n",
    "        \"attack_selenium_xss.csv\",\n",
    "    ],\n",
    "    \"r2l\": [\n",
    "        \"attack_selenium_bruteforce.csv\",\n",
    "        \"attack_selenium_cmd.csv\",\n",
    "        \"attack_selenium_file_upload.csv\",\n",
    "    ],\n",
    "    \"u2r\": [\n",
    "        \"attack_msf_samba.csv\",\n",
    "        \"attack_msf_vnc.csv\",\n",
    "    ],\n",
    "    \"normal\": [\n",
    "        \"normal_dns.csv\",\n",
    "        \"normal_iperf.csv\",\n",
    "        \"normal_ping.csv\",\n",
    "        \"normal_telnet.csv\",\n",
    "        \"normal_w3m.csv\",\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4008927-6394-4bda-906c-90920d8fa0ad",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Data Loading"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d6b622-da3b-45b6-ae43-5d082266f5ca",
   "metadata": {
    "tags": []
   },
   "source": [
    "### MCAD Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfdb0922-4d0a-4684-9392-5b349452ee0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "mcad_dfs = []\n",
    "\n",
    "for attack_class, files in MCAD_CLASS_FILE_MAP.items():\n",
    "    for file in files:\n",
    "        try:\n",
    "            data = pd.read_csv(MCAD_DATASET_PATH + file)\n",
    "            data[\"type\"] = attack_class\n",
    "            mcad_dfs.append(data)\n",
    "        except FileNotFoundError:\n",
    "            print(f\"Error: File '{file}' not found. Skipping...\")\n",
    "\n",
    "# Concatenate all DataFrames\n",
    "mcad_combined_data = pd.concat(mcad_dfs, ignore_index=True)\n",
    "print(\"Successfully combined MCAD data into a single DataFrame!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d2ec1eb-e767-45fe-93c9-19d8acafcd75",
   "metadata": {},
   "outputs": [],
   "source": [
    "mcad_combined_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f325ff2c-3145-4522-8e25-09097b5854ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot of counts per class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bef2e34-93d3-4eb5-8003-abcd3789f3b7",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Collected Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a996951d-3578-4e34-9223-7095d9e33767",
   "metadata": {},
   "outputs": [],
   "source": [
    "collected_dfs = []\n",
    "\n",
    "for attack_class, files in COLLECTED_CLASS_FILE_MAP.items():\n",
    "    for file in files:\n",
    "        try:\n",
    "            data = pd.read_csv(COLLECTED_DATASET_PATH + file)\n",
    "            data[\"type\"] = attack_class\n",
    "            collected_dfs.append(data)\n",
    "        except FileNotFoundError:\n",
    "            print(f\"Error: File '{file}' not found. Skipping...\")\n",
    "\n",
    "# Concatenate all DataFrames\n",
    "collected_combined_data = pd.concat(collected_dfs, ignore_index=True)\n",
    "print(\"Successfully combined collected data into a single DataFrame!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "253bb1fb-bca6-4457-a798-07e9832f2477",
   "metadata": {},
   "outputs": [],
   "source": [
    "collected_combined_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd0cf846-af4b-463e-aae0-69ed2c634241",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot of counts per attack class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95c1746d-e302-4f6f-8c9d-7ba551a61093",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot of counts per normal class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "096ccbc5-32a0-42c8-a02d-3bf7c5a0cfeb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Dataset Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "486b912e-2cf8-481f-a79f-34a35acffec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot of MCAD, collected dataset counts comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "942dece7-6583-4eba-aa3b-0ee6d4243f66",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34d9293d-1b60-493c-a2fb-99075a51177c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Cleansing / Shuffling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4059f237-cd4c-438a-b40f-761f747b8ec1",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = mcad_combined_data.sample(frac=1).reset_index().drop('index', axis = 1)\n",
    "combined.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "276c5ef8-b7f6-4287-ab57-9eb8700d3c31",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Division Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed46ee4e-6639-4821-b96c-850a280264f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Laplacian Correction\n",
    "relevant_attributes = [\n",
    "    'ip_bytes',\n",
    "    'ip_packet',\n",
    "    'ip_duration',\n",
    "    'port_bytes',\n",
    "    'port_packet',\n",
    "    'port_flow_count',\n",
    "    'table_matched_count',\n",
    "    'table_active_count',\n",
    "    'table_lookup_count',\n",
    "    'port_rx_packets',\n",
    "    'port_tx_packets',\n",
    "    'port_rx_bytes',\n",
    "    'port_tx_bytes',\n",
    "    'port_duration_sec',\n",
    "]\n",
    "for attribute in relevant_attributes:\n",
    "    combined[attribute] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0f38b5-bf3d-4217-b4f1-674a2ce2a738",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined['ip_bytes_sec'] = combined['ip_bytes'] / combined['ip_duration']\n",
    "combined['ip_packets_sec'] = combined['ip_packet'] / combined['ip_duration']\n",
    "combined['ip_bytes_packet'] = combined['ip_bytes'] / combined['ip_packet']\n",
    "combined['port_bytes_sec'] = combined['port_bytes'] / combined['ip_duration']\n",
    "combined['port_packet_sec'] = combined['port_packet'] / combined['ip_duration']\n",
    "combined['port_byte_packet'] = combined['port_bytes'] / combined['port_packet']\n",
    "combined['port_flow_count_sec'] = combined['port_flow_count'] / combined['ip_duration']\n",
    "combined['table_matched_lookup'] = combined['table_matched_count'] / combined['table_lookup_count']\n",
    "combined['table_active_lookup'] = combined['table_active_count'] / combined['table_lookup_count']\n",
    "combined['port_rx_packets_sec'] = combined['port_rx_packets'] / combined['port_duration_sec']\n",
    "combined['port_tx_packets_sec'] = combined['port_tx_packets'] / combined['port_duration_sec']\n",
    "combined['port_rx_bytes_sec'] = combined['port_rx_bytes'] / combined['port_duration_sec']\n",
    "combined['port_tx_bytes_sec'] = combined['port_tx_bytes'] / combined['port_duration_sec']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6e12dc5-b289-4692-a8d4-238e97b97b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = combined[\n",
    "    [\n",
    "        'ip_bytes_sec',\n",
    "        'ip_packets_sec',\n",
    "        'ip_bytes_packet',\n",
    "        'port_bytes_sec',\n",
    "        'port_packet_sec',\n",
    "        'port_byte_packet',\n",
    "        'port_flow_count_sec',\n",
    "        'table_matched_lookup',\n",
    "        'table_active_lookup',\n",
    "        'port_rx_packets_sec',\n",
    "        'port_tx_packets_sec',\n",
    "        'port_rx_bytes_sec',\n",
    "        'port_tx_bytes_sec',\n",
    "        'type'\n",
    "    ]\n",
    "].copy()\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4392342-079a-4fad-aac4-3f03b2666d26",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "430d5a8c-cd30-44e0-8477-c49b2724dd21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting into dependent and independent variables\n",
    "X = data.loc[ : , data.columns != 'type']\n",
    "y = data['type']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8089d66a-afcf-40b7-a7fc-838ede3f00ff",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Benjamini–Hochberg False Discovery Rate (FDR) method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4910e32-8682-42d2-b5fd-b2f4d72de3df",
   "metadata": {},
   "outputs": [],
   "source": [
    "selector = SelectFdr(score_func=chi2, alpha=1e-8).fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7640321-317b-4cc1-83e8-95f43cc5f6ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdr_selected = set(selector.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa7ce581-8efd-4c0e-9c3c-94e6ff65a9ec",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Sequential Feature Selection  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6da2b8ae-453d-4f39-8257-ce5d90549e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9491cabc-f9dd-4c48-8046-6c051fb7d600",
   "metadata": {},
   "outputs": [],
   "source": [
    "sfs = SequentialFeatureSelector(knn, n_features_to_select=7)\n",
    "sfs.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbe801a0-bfe9-427d-adfb-aeec01bcbfcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sfs_selected = set(sfs.get_feature_names_out())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79afd078-290f-4b19-88ca-e541aeb4f6a8",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Boruta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d8430db-913c-47ef-aeb9-a684d058b884",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(n_jobs=-1, max_depth=5, class_weight='balanced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85835b63-6136-49e4-8d67-a9f496dac503",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_selector = BorutaPy(rf, n_estimators='auto', verbose=1, max_iter=10)\n",
    "feat_selector.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1e00ab9-7b6b-4959-b0a1-c9bc3635cf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\n------Support and Ranking for each feature------\")\n",
    "for i in range(len(feat_selector.support_)):\n",
    "    if feat_selector.support_[i]:\n",
    "        print(\"Passes the test: \", X.columns[i],\n",
    "              \" - Ranking: \", feat_selector.ranking_[i])\n",
    "    else:\n",
    "        print(\"Doesn't pass the test: \",\n",
    "              X.columns[i], \" - Ranking: \", feat_selector.ranking_[i], \" X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab61f1b5-81b7-41a2-8a67-d035968eb519",
   "metadata": {},
   "outputs": [],
   "source": [
    "boruta_selected = set()\n",
    "for i in range(len(feat_selector.support_)):\n",
    "    if feat_selector.support_[i]:\n",
    "        boruta_selected.add(X.columns[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "971aae2f-07cb-485b-9613-3f0337d7eccd",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Intersection of Selected Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d5af38b-879e-484f-b5ea-4b3cc0480383",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected = list(fdr_selected & sfs_selected & boruta_selected)\n",
    "selected"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4644b87b-c026-44a8-aca2-f42c5b64c60a",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de73e47c-28e4-4f61-a34a-701e41089462",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X[selected]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b6d9f02-9a1b-44e4-a451-43aaa4303623",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baaf6e92-0c9c-4106-9580-9c5680fb5fc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = RobustScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "819c2f20-3f92-4c38-a0ab-936026acc2b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.DataFrame(scaler.fit_transform(X_train), columns=selected)\n",
    "X_test = pd.DataFrame(scaler.transform(X_test), columns=selected)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f93d23d4-ae1b-4d1c-9d80-c7445362c593",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4997d0e3-2889-4897-8c93-643aaeeb28d9",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Without dimensionality reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "527fef2e-bc03-4861-97f3-c423b0cae6ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97b2faf6-6e5e-47ec-b17d-2387d74151fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "precision, recall, fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7cf4179-b599-4f30-a728-bd0279d74c25",
   "metadata": {
    "tags": []
   },
   "source": [
    "### With PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f06c033-01d0-4c24-8c23-a76f057fa9cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=4)\n",
    "X_train_pca = pd.DataFrame(pca.fit_transform(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e74c3a5-0109-4752-8637-22c233667c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train_pca, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87625bef-3009-4159-9a2e-81d3585307c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_pca = pd.DataFrame(pca.transform(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1ebdc1-e9bb-4bc5-9634-83622975beff",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test_pca)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "pca_precision, pca_recall, pca_fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d569d327-464e-457c-82bc-3180bdd294a2",
   "metadata": {
    "tags": []
   },
   "source": [
    "### With ICA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9110ed6d-1c97-41d8-8ba0-c6d4a07ba6ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "ica = FastICA(n_components=4)\n",
    "X_train_ica = pd.DataFrame(ica.fit_transform(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "659c514b-cc2c-40dd-bbea-6104bda7df48",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train_ica, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bb0ebdf-ac4b-43b3-a077-b6ca47f6a7f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_ica = pd.DataFrame(ica.transform(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0f73b39-e6de-40e7-899e-fc5325b29162",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test_ica)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "ica_precision, ica_recall, ica_fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d43977d-d1c4-4f82-b6cd-08d29b2244aa",
   "metadata": {
    "tags": []
   },
   "source": [
    "### With LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2909e8b5-757a-4b9e-bab3-90b7021ef3bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = LinearDiscriminantAnalysis(n_components=4)\n",
    "lda.fit(X_train, y_train)\n",
    "X_train_lda = pd.DataFrame(lda.transform(X_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4fe4755-6968-4f9c-8c8f-51d68df98a0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train_lda, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c113ba2-5d8c-42da-aadd-55e6d9897880",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_lda = pd.DataFrame(lda.transform(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59be8b9c-cf8f-4b64-935e-3155d273c806",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test_lda)\n",
    "accuracy = accuracy_score(y_test, y_pred)\n",
    "lda_precision, lda_recall, lda_fscore, _ = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "958d2b10-34f2-4e39-8736-f069f62ff084",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Dimensionality Reduction Methods Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "993d74fb-4647-4fd7-ac89-d805545f30c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot comparing all four, with Accuracy, Precision, Recall and F1 score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}