Assignment1_Ell409.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ji2rU4xyYigY"
   },
   "source": [
    "# This Notebook contains complete code for different statistical approach for classifying the data with its test accuracy.\n",
    "\n",
    "---\n",
    "The code is written in python and divided in many section\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FhwSiQLUYz9W"
   },
   "source": [
    "# Section 1: \n",
    "Reading data from the mnist binary data file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9MXJxW6JXFk9"
   },
   "outputs": [],
   "source": [
    "def load_mnist(path, kind='train'):\n",
    "    import os\n",
    "    import gzip\n",
    "    import numpy as np\n",
    "\n",
    "    \"\"\"Load MNIST data from `path`\"\"\"\n",
    "    labels_path = os.path.join(path,\n",
    "                               '%s-labels-idx1-ubyte.gz'\n",
    "                               % kind)\n",
    "    images_path = os.path.join(path,\n",
    "                               '%s-images-idx3-ubyte.gz'\n",
    "                               % kind)\n",
    "\n",
    "    with gzip.open(labels_path, 'rb') as lbpath:\n",
    "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,\n",
    "                               offset=8)\n",
    "\n",
    "    with gzip.open(images_path, 'rb') as imgpath:\n",
    "        images = np.frombuffer(imgpath.read(), dtype=np.uint8,\n",
    "                               offset=16).reshape(len(labels), 784)\n",
    "\n",
    "    return images, labels\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lfPfb5EYY77t"
   },
   "source": [
    "Importing required python Module and load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oHJIO1UeZNXX"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numpy\n",
    "import math,copy\n",
    "import csv\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1bqDdQmyZX-w"
   },
   "source": [
    "# Section 2: \n",
    "Bayesian Classifier with Maximum Likelihood Estimation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OqdDKYPaZbAR"
   },
   "source": [
    "# PART A\n",
    "Fashion Minst dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R5ybK7AUZSic"
   },
   "outputs": [],
   "source": [
    "mus = []\n",
    "training_freq = {}\n",
    "training_data = []\n",
    "test_data = []\n",
    "sigs = []\n",
    "dets = 1\n",
    "priori = []\n",
    "m0 = 1\n",
    "s0 = 1\n",
    "n = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Sxc3VgiaZe4v"
   },
   "outputs": [],
   "source": [
    "#\n",
    "def cal_priori():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    dic = training_freq.copy()\n",
    "    for i in dic.keys():\n",
    "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
    "    priori = dic.copy()\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nLf_vF6vZg2N"
   },
   "outputs": [],
   "source": [
    "#\n",
    "def mean_mat():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    global n\n",
    "    dic = {}\n",
    "    for y,x in training_data:\n",
    "        if y not in dic:\n",
    "            dic[y] = []\n",
    "        dic[y].append(x)\n",
    "    for j in dic.keys():\n",
    "        dic[j] = np.mean(dic.get(j),axis=0)\n",
    "    ns0 = np.array(n*s0)\n",
    "    sigs = sigs.astype(float)\n",
    "    ns0 = ns0.astype(float)\n",
    "    for va in dic.keys():\n",
    "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
    "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
    "        mn = np.array(mn0+mn1)\n",
    "        dic[va] = mn\n",
    "    mus = dic\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ju2sDxyrZifo"
   },
   "outputs": [],
   "source": [
    "def mulvar_nor(X,y):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    #print dets\n",
    "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
    "    #math.log(2*math.pi) = 1.8378770664093453\n",
    "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
    "    xut = np.array([X-mus.get(y)])\n",
    "    sig_i = np.linalg.inv(sigs)\n",
    "    pd = np.matmul(xut , sig_i)\n",
    "    xu = np.array([X-mus.get(y)]).T\n",
    "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
    "    return c+e\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ur4dqqt4ZkiH"
   },
   "outputs": [],
   "source": [
    "def mul_exp(X,y):\n",
    "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "60wVAZGSZmdQ"
   },
   "outputs": [],
   "source": [
    "def train_rn():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    X_train, y_train = load_mnist('drive/ml1/data/fashion', kind='train')\n",
    "    #X_train = np.array(pca(X_train))\n",
    "    #X_train = np.array(X_train).real.astype(float)\n",
    "    sigs = np.cov(X_train,rowvar=0)\n",
    "    dets = np.linalg.slogdet(sigs)[-1]\n",
    "    X_test, y_test = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
    "    #X_test = pca(X_test)\n",
    "    #X_test = np.array(X_test).real.astype(float)\n",
    "    training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
    "    val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
    "    m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
    "    s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
    "    n=len(X_train)-len(X_train)/10\n",
    "    sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
    "    sigs = np.array(sn+sigs)\n",
    "    test_data = [x for x in zip(y_test,X_test)]\n",
    "    unique, counts = np.unique(y_train, return_counts=True)\n",
    "    training_freq = dict(zip(unique,counts))\n",
    "    priori = cal_priori()\n",
    "    mus = mean_mat()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dU4Y6zEWZoJX"
   },
   "outputs": [],
   "source": [
    "def bayes_c(X):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    clas = \"\"\n",
    "    max = 0\n",
    "    dik={}\n",
    "    for i in training_freq.keys():\n",
    "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
    "    pm = -999999\n",
    "    for v in dik.keys():\n",
    "        if dik.get(v)>pm:\n",
    "            pm = dik.get(v)\n",
    "            clas = v\n",
    "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
    "    return clas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3qoE7spFZp7B"
   },
   "outputs": [],
   "source": [
    "def test_rn():\n",
    "    fp = 0\n",
    "    cp = 0\n",
    "    fn = 0\n",
    "    cn = 0\n",
    "    p = 0\n",
    "    f = 0\n",
    "    t = 0\n",
    "    for v,u in test_data:\n",
    "        t+=1\n",
    "        k = str(bayes_c(u))\n",
    "        if k==str(v):\n",
    "            p+=1\n",
    "        else:\n",
    "            f+=1\n",
    "    print \"accuracy = \" + str(float(p)/float(t))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b6ioASADZrxs"
   },
   "outputs": [],
   "source": [
    "def pca(X):\n",
    "    a = []\n",
    "    for i in X:\n",
    "        M = np.mean(i)\n",
    "        i = i-M\n",
    "        b = i.reshape(28,28)\n",
    "        c = np.cov(b)\n",
    "        values,vectors = np.linalg.eig(c)\n",
    "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "        eig_pair.sort(key=lambda x:x[0])\n",
    "        eig_pair.reverse()\n",
    "        ab = []\n",
    "        for j in range():\n",
    "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
    "        a.append(ab)\n",
    "    return a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8gG_mK7mZtmM"
   },
   "outputs": [],
   "source": [
    "train_rn()\n",
    "test_rn()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lWkQ_8rRkdX-"
   },
   "source": [
    "![alt text](https://2.bp.blogspot.com/-dvmGCHt1IsE/W57oy9mGLOI/AAAAAAAAKPE/FfqahezKlzgq0DvW_5n1JcMKfEWd4tJWQCLcBGAs/s320/Capture.PNG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZA_wTQMxaX9E"
   },
   "source": [
    "# Part B\n",
    "Medical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pMh7YgL8bZ4W"
   },
   "outputs": [],
   "source": [
    "mus = []\n",
    "training_freq = {}\n",
    "training_data = []\n",
    "test_data = []\n",
    "sigs = []\n",
    "dets = 1\n",
    "priori = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "up0lLROeaoHA"
   },
   "outputs": [],
   "source": [
    "def cal_priori():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    dic = training_freq.copy()\n",
    "    for i in dic.keys():\n",
    "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
    "    priori = dic.copy()\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uq26GgIbasOP"
   },
   "outputs": [],
   "source": [
    "def mean_mat():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    dic = {}\n",
    "    for y,x in training_data:\n",
    "        if y not in dic:\n",
    "            dic[y] = []\n",
    "        dic[y].append(x)\n",
    "    for j in dic.keys():\n",
    "        dic[j] = np.mean(dic.get(j),axis=0)\n",
    "    mus = dic\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "roFsl5isaxO9"
   },
   "outputs": [],
   "source": [
    "def mulvar_nor(X,y):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
    "    xut = np.array([X-mus.get(y)])\n",
    "    sig_i = np.linalg.inv(sigs)\n",
    "    pd = np.matmul(xut , sig_i)\n",
    "    xu = np.array([X-mus.get(y)]).T\n",
    "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
    "    return c+e\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ULeYJnWza1K-"
   },
   "outputs": [],
   "source": [
    "def train_rn():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    priori = cal_priori()\n",
    "    mus = mean_mat()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GFprrV88a3I4"
   },
   "outputs": [],
   "source": [
    "def bayes_c(X):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    clas = \"\"\n",
    "    max = 0\n",
    "    dik={}\n",
    "    for i in training_freq.keys():\n",
    "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
    "    pm = -999999\n",
    "    for v in dik.keys():\n",
    "        if dik.get(v)>pm:\n",
    "            pm = dik.get(v)\n",
    "            clas = v\n",
    "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
    "    return clas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ixaBjUGwa5IU"
   },
   "outputs": [],
   "source": [
    "def test_rn():\n",
    "    fp = 0\n",
    "    cp = 0\n",
    "    fn = 0\n",
    "    cn = 0\n",
    "    p = 0\n",
    "    f = 0\n",
    "    t = 0\n",
    "    for v,u in test_data:\n",
    "        t+=1\n",
    "        k = str(bayes_c(u))\n",
    "        if k==str(v):\n",
    "            p+=1\n",
    "        else:\n",
    "            f+=1\n",
    "    print \"accuracy = \" + str(float(p)/float(t))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "M5n8BtZza7e2",
    "outputId": "71189eef-d644-4c38-8e11-59d87fa7b779"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy = 0.816666666667\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"drive/ml1/Medical_data.csv\")\n",
    "y_train = np.array(df['Health'].values.tolist())\n",
    "X_train = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "df = pd.read_csv(\"drive/ml1/test_medical.csv\")\n",
    "y_test = np.array(df['Health'].values.tolist())\n",
    "X_test = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "sigs = np.cov(X_train,rowvar=0)\n",
    "dets = np.linalg.det(sigs)\n",
    "training_data = [x for x in zip(y_train,X_train)]\n",
    "test_data = [x for x in zip(y_test,X_test)]\n",
    "unique, counts = np.unique(y_train, return_counts=True)\n",
    "training_freq = dict(zip(unique,counts))\n",
    "train_rn()\n",
    "test_rn()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Lmk2oj-fbH43"
   },
   "source": [
    "# Part C\n",
    "Railway Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nHUsQ7s6bIrr"
   },
   "outputs": [],
   "source": [
    "mus = []\n",
    "training_freq = {}\n",
    "training_data = []\n",
    "test_data = []\n",
    "sigs = []\n",
    "dets = 1\n",
    "priori = []\n",
    "priori_d = {}\n",
    "recal = 0.5\n",
    "precisio = 0.5\n",
    "fscore = 0.5\n",
    "k = 1\n",
    "fp = 0\n",
    "cp = 0\n",
    "fn = 0\n",
    "cn = 0\n",
    "accu = 0.92"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fEf2mu5ObxGP"
   },
   "outputs": [],
   "source": [
    "def cal_priori_d():\n",
    "    global dets\n",
    "    global k\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global priori\n",
    "    global sigs\n",
    "    global mus\n",
    "    global priori_d\n",
    "    dicd = {}\n",
    "    for y,x,x1 in training_data:\n",
    "        if str(x1) not in dicd.keys():\n",
    "            #print x1\n",
    "            dicd[str(x1)] = 0\n",
    "        if y==1:\n",
    "            dicd[str(x1)]+=1\n",
    "    for a in dicd.keys():\n",
    "        dicd[a] = dicd.get(a)/float(len(training_data))\n",
    "    priori_d = dicd.copy()\n",
    "    return dicd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R-O0JXQXb17a"
   },
   "outputs": [],
   "source": [
    "def cal_priori():\n",
    "    global dets\n",
    "    global k\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global priori\n",
    "    global sigs\n",
    "    global mus\n",
    "    global priori_d\n",
    "    dic = training_freq.copy()\n",
    "    for i in dic.keys():\n",
    "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
    "    priori = dic.copy()\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S1TBMnqbb4-U"
   },
   "outputs": [],
   "source": [
    "def mean_mat():\n",
    "    global dets\n",
    "    global k\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global priori\n",
    "    global sigs\n",
    "    global mus\n",
    "    global priori_d\n",
    "    dic = {}\n",
    "    for y,x,x1 in training_data:\n",
    "        if y not in dic:\n",
    "            dic[y] = []\n",
    "        dic[y].append(x)\n",
    "    for j in dic.keys():\n",
    "        dic[j] = np.mean(dic.get(j),axis=0)\n",
    "    mus = dic.copy()\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OafUmW6hb9o7"
   },
   "outputs": [],
   "source": [
    "def mulvar_nor(X,y):\n",
    "    global training_freq\n",
    "    global k\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
    "    xut = np.array([X-mus.get(y)])\n",
    "    sig_i = np.linalg.inv(sigs)\n",
    "    pd = np.matmul(xut , sig_i)\n",
    "    xu = np.array([X-mus.get(y)]).T\n",
    "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
    "    return c+e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gTaPWzRmcBib"
   },
   "outputs": [],
   "source": [
    "def train_rn():\n",
    "    global dets\n",
    "    global k\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global priori\n",
    "    global sigs\n",
    "    global mus\n",
    "    global priori_d\n",
    "    df = pd.read_csv(\"drive/ml1/railwayBookingList.csv\")\n",
    "    y_train = np.array(df['boarded'].values.tolist())\n",
    "    X_train = df[['budget', 'memberCount', 'age']].values.tolist()\n",
    "    sigs = np.cov(X_train,rowvar=0)\n",
    "    dets = np.linalg.det(sigs)\n",
    "    X_train1 = df[['preferredClass', 'sex']].values.tolist()\n",
    "    training_data = [x for x in zip(y_train[:1100],X_train[:1100],X_train1[:1100])]\n",
    "    #print \"length of training_data: \"+str(len(training_data))\n",
    "    test_data = [x for x in zip(y_train[1100:],X_train[1100:],X_train1[1100:])]\n",
    "    #print \"length of test_data: \"+str(len(test_data))\n",
    "    unique, counts = np.unique(y_train, return_counts=True)\n",
    "    training_freq = dict(zip(unique,counts))\n",
    "    cal_priori()\n",
    "    cal_priori_d()\n",
    "    mean_mat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N2aJmd8tcuss"
   },
   "outputs": [],
   "source": [
    "def bayes_c(X,X1):\n",
    "    global dets\n",
    "    global k\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global priori\n",
    "    global sigs\n",
    "    global mus\n",
    "    global priori_d\n",
    "    clas = 1\n",
    "    max = 0\n",
    "    dik ={}\n",
    "    if X1 not in priori_d:\n",
    "        return 1\n",
    "    for i in training_freq.keys():\n",
    "        if i==1:\n",
    "            post_d = priori_d.get(X1)\n",
    "        else:\n",
    "            post_d = 1-priori_d.get(X1)\n",
    "        dik[i] = mulvar_nor(X,i)*priori.get(i)*post_d\n",
    "    #print str(dik.get(0))+\" \"+str(k*dik.get(1))\n",
    "    if dik.get(0)>k*dik.get(1):\n",
    "        return 0\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7OvF8MIScwMs"
   },
   "outputs": [],
   "source": [
    "def test_rn():\n",
    "    global fscore\n",
    "    global precisio\n",
    "    global recal\n",
    "    global k\n",
    "    global fp\n",
    "    global cp\n",
    "    global fn\n",
    "    global cn\n",
    "    global accu\n",
    "    accu = 0.85\n",
    "    p = 0\n",
    "    f = 0\n",
    "    t = 0\n",
    "    for v,u,c in test_data:\n",
    "        t+=1\n",
    "        st = str(v)\n",
    "        te = str(bayes_c(u,str(c)))\n",
    "        if (st==\"0\" and te==\"0\"):\n",
    "            cn+=1\n",
    "        if (st==\"0\" and te==\"1\"):\n",
    "            fp+=1\n",
    "        if (st==\"1\" and te==\"0\"):\n",
    "            fn+=1\n",
    "        if (st==\"1\" and te==\"1\"):\n",
    "            cp+=1\n",
    "        if te==st:\n",
    "            p+=1\n",
    "        else:\n",
    "            f+=1\n",
    "    recal = cp/(float(cp)+ float(fn))\n",
    "    precisio = cp/(float(cp)+ float(fp))\n",
    "    fscore = 2*precisio*recal/(precisio + recal)\n",
    "    accu = str(float(p)/float(t))\n",
    "    print \"Accuracy: \" + str(accu)\n",
    "    print \"Precision: \" + str(precisio)\n",
    "    print \"recall: \" + str(recal)\n",
    "    print \"F1 score: \" + str(fscore)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ulbhA6BTc0bw"
   },
   "outputs": [],
   "source": [
    "def roc():\n",
    "    global k\n",
    "    global fp\n",
    "    global cp\n",
    "    global fn\n",
    "    global cn\n",
    "    ro = [(0,0)]\n",
    "    for i in range(100):\n",
    "        k = -2.5 + i/float(10)\n",
    "        #train_rn()\n",
    "        test_rn()\n",
    "        le = len(test_data)\n",
    "        ro.append((fp/float(le),cp/float(le)))\n",
    "        fp = 0\n",
    "        cp = 0\n",
    "        fn = 0\n",
    "        cn = 0\n",
    "    #print ro\n",
    "    plt.scatter(*zip(*ro))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "G_5qRfOr455c"
   },
   "source": [
    "![alt text](https://2.bp.blogspot.com/-XzCZ_uO9UHY/W57x3Qpe9tI/AAAAAAAAKPY/HT0ULZWjyd8oSfpocvIJz6H_AufDolXfgCLcBGAs/s320/Froc.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "id": "lrFU58ikc6lu",
    "outputId": "ab4df8e1-4b67-422a-87d0-b3a6518448f3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.828571428571\n",
      "Precision: 0.828571428571\n",
      "recall: 1.0\n",
      "F1 score: 0.90625\n"
     ]
    }
   ],
   "source": [
    "train_rn()\n",
    "test_rn()\n",
    "#roc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GOhTMGsphEQX"
   },
   "source": [
    "#Section 3\n",
    "Bayesian Classifier with Bayesian Estimation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "doe70KWfhiXr"
   },
   "source": [
    "#Part A\n",
    "Fashion Mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3sP4-dBRhDKE"
   },
   "outputs": [],
   "source": [
    "mus = []\n",
    "training_freq = {}\n",
    "training_data = []\n",
    "test_data = []\n",
    "sigs = []\n",
    "dets = 1\n",
    "priori = []\n",
    "m0 = 1\n",
    "s0 = 1\n",
    "n = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "svjQqMNqiLxS"
   },
   "outputs": [],
   "source": [
    "def cal_priori():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    dic = training_freq.copy()\n",
    "    for i in dic.keys():\n",
    "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
    "    priori = dic.copy()\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "evXd8ZjriO_f"
   },
   "outputs": [],
   "source": [
    "def mean_mat():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    global n\n",
    "    dic = {}\n",
    "    for y,x in training_data:\n",
    "        if y not in dic:\n",
    "            dic[y] = []\n",
    "        dic[y].append(x)\n",
    "    for j in dic.keys():\n",
    "        dic[j] = np.mean(dic.get(j),axis=0)\n",
    "    ns0 = np.array(n*s0)\n",
    "    sigs = sigs.astype(float)\n",
    "    ns0 = ns0.astype(float)\n",
    "    for va in dic.keys():\n",
    "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
    "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
    "        mn = np.array(mn0+mn1)\n",
    "        dic[va] = mn\n",
    "    mus = dic\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SrnVsv1fiR2p"
   },
   "outputs": [],
   "source": [
    "def mulvar_nor(X,y):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    #print dets\n",
    "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
    "    #math.log(2*math.pi) = 1.8378770664093453\n",
    "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
    "    xut = np.array([X-mus.get(y)])\n",
    "    sig_i = np.linalg.inv(sigs)\n",
    "    pd = np.matmul(xut , sig_i)\n",
    "    xu = np.array([X-mus.get(y)]).T\n",
    "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
    "    return c+e\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jSsIgIEfiWdG"
   },
   "outputs": [],
   "source": [
    "def mul_exp(X,y):\n",
    "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EHdMitwxiXbc"
   },
   "outputs": [],
   "source": [
    "def train_rn():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    X_train, y_train = load_mnist('drive/ml1/data/fashion', kind='train')\n",
    "    #X_train = np.array(pca(X_train))\n",
    "    #X_train = np.array(X_train).real.astype(float)\n",
    "    sigs = np.cov(X_train,rowvar=0)\n",
    "    dets = np.linalg.slogdet(sigs)[-1]\n",
    "    X_test, y_test = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
    "    #X_test = pca(X_test)\n",
    "    #X_test = np.array(X_test).real.astype(float)\n",
    "    training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
    "    val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
    "    m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
    "    s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
    "    n=len(X_train)-len(X_train)/10\n",
    "    sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
    "    sigs = np.array(sn+sigs)\n",
    "    test_data = [x for x in zip(y_test,X_test)]\n",
    "    unique, counts = np.unique(y_train, return_counts=True)\n",
    "    training_freq = dict(zip(unique,counts))\n",
    "    priori = cal_priori()\n",
    "    mus = mean_mat()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bsHLovtNiaaE"
   },
   "outputs": [],
   "source": [
    "def bayes_c(X):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    clas = \"\"\n",
    "    max = 0\n",
    "    dik={}\n",
    "    for i in training_freq.keys():\n",
    "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
    "    pm = -999999\n",
    "    for v in dik.keys():\n",
    "        if dik.get(v)>pm:\n",
    "            pm = dik.get(v)\n",
    "            clas = v\n",
    "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
    "    return clas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "37TrX60oidy-"
   },
   "outputs": [],
   "source": [
    "def test_rn():\n",
    "    fp = 0\n",
    "    cp = 0\n",
    "    fn = 0\n",
    "    cn = 0\n",
    "    p = 0\n",
    "    f = 0\n",
    "    t = 0\n",
    "    for v,u in test_data:\n",
    "        t+=1\n",
    "        k = str(bayes_c(u))\n",
    "        if k==str(v):\n",
    "            p+=1\n",
    "        else:\n",
    "            f+=1\n",
    "    print \"accuracy = \" + str(float(p)/float(t))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gIzfr_DPijKh"
   },
   "outputs": [],
   "source": [
    "def pca(X):\n",
    "    a = []\n",
    "    for i in X:\n",
    "        M = np.mean(i)\n",
    "        i = i-M\n",
    "        b = i.reshape(28,28)\n",
    "        c = np.cov(b)\n",
    "        values,vectors = np.linalg.eig(c)\n",
    "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "        eig_pair.sort(key=lambda x:x[0])\n",
    "        eig_pair.reverse()\n",
    "        ab = []\n",
    "        for j in range():\n",
    "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
    "        a.append(ab)\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Lipzvntiilxl"
   },
   "outputs": [],
   "source": [
    "train_rn()\n",
    "test_rn()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6xsMpIqDnQvq"
   },
   "source": [
    "![alt text](https://2.bp.blogspot.com/-aDBDQgbZ5k8/W57o1JoQ9yI/AAAAAAAAKPI/8rW8eReCTDca9AnH--smS7d8QCm1fu_DQCLcBGAs/s320/Capture2.PNG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "I5OFxYKxiq13"
   },
   "source": [
    "#Part B\n",
    "Medical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ut2-o4oxiuV6"
   },
   "outputs": [],
   "source": [
    "mus = []\n",
    "training_freq = {}\n",
    "training_data = []\n",
    "test_data = []\n",
    "sigs = []\n",
    "dets = 1\n",
    "priori = []\n",
    "m0 = 1\n",
    "s0 = 1\n",
    "n = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "buVtfgZWi0EQ"
   },
   "outputs": [],
   "source": [
    "def cal_priori():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    dic = training_freq.copy()\n",
    "    for i in dic.keys():\n",
    "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
    "    priori = dic.copy()\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eWAvVYCWi4Wf"
   },
   "outputs": [],
   "source": [
    "def mean_mat():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    global n\n",
    "    dic = {}\n",
    "    for y,x in training_data:\n",
    "        if y not in dic:\n",
    "            dic[y] = []\n",
    "        dic[y].append(x)\n",
    "    for j in dic.keys():\n",
    "        dic[j] = np.mean(dic.get(j),axis=0)\n",
    "    ns0 = np.array(n*s0)\n",
    "    sigs = sigs.astype(float)\n",
    "    ns0 = ns0.astype(float)\n",
    "    for va in dic.keys():\n",
    "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
    "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
    "        mn = np.array(mn0+mn1)\n",
    "        dic[va] = mn\n",
    "    mus = dic\n",
    "    return dic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ooxWfeAhi5h7"
   },
   "outputs": [],
   "source": [
    "def mulvar_nor(X,y):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    #print dets\n",
    "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
    "    #math.log(2*math.pi) = 1.8378770664093453\n",
    "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
    "    xut = np.array([X-mus.get(y)])\n",
    "    sig_i = np.linalg.inv(sigs)\n",
    "    pd = np.matmul(xut , sig_i)\n",
    "    xu = np.array([X-mus.get(y)]).T\n",
    "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
    "    return c+e\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CskyMfEpi8eN"
   },
   "outputs": [],
   "source": [
    "def mul_exp(X,y):\n",
    "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dFoI7U9LjCjR"
   },
   "outputs": [],
   "source": [
    "def train_rn():\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    global mus\n",
    "    priori = cal_priori()\n",
    "    mus = mean_mat()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZTyUA7LmjEeb"
   },
   "outputs": [],
   "source": [
    "def bayes_c(X):\n",
    "    global training_freq\n",
    "    global training_data\n",
    "    global test_data\n",
    "    global m0,s0\n",
    "    global sigs\n",
    "    global dets\n",
    "    global priori\n",
    "    clas = \"\"\n",
    "    max = 0\n",
    "    dik={}\n",
    "    for i in training_freq.keys():\n",
    "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
    "    pm = -999999\n",
    "    for v in dik.keys():\n",
    "        if dik.get(v)>pm:\n",
    "            pm = dik.get(v)\n",
    "            clas = v\n",
    "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
    "    return clas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BPYAWDLZjIOg"
   },
   "outputs": [],
   "source": [
    "def test_rn():\n",
    "    fp = 0\n",
    "    cp = 0\n",
    "    fn = 0\n",
    "    cn = 0\n",
    "    p = 0\n",
    "    f = 0\n",
    "    t = 0\n",
    "    for v,u in test_data:\n",
    "        t+=1\n",
    "        k = str(bayes_c(u))\n",
    "        if k==str(v):\n",
    "            p+=1\n",
    "        else:\n",
    "            f+=1\n",
    "    print \"accuracy = \" + str(float(p)/float(t))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z9TxdkU0jK8W"
   },
   "outputs": [],
   "source": [
    "def pca(X):\n",
    "    a = []\n",
    "    for i in X:\n",
    "        M = np.mean(i)\n",
    "        i = i-M\n",
    "        b = i.reshape(28,28)\n",
    "        c = np.cov(b)\n",
    "        values,vectors = np.linalg.eig(c)\n",
    "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "        eig_pair.sort(key=lambda x:x[0])\n",
    "        eig_pair.reverse()\n",
    "        ab = []\n",
    "        for j in range():\n",
    "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
    "        a.append(ab)\n",
    "    return a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "pr2RagHBjNnK",
    "outputId": "6ced8432-4c4c-4c62-a13a-77bb4975c4e3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy = 0.817\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"drive/ml1/Medical_data.csv\")\n",
    "y_train = np.array(df['Health'].values.tolist())\n",
    "X_train = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "df = pd.read_csv(\"drive/ml1/test_medical.csv\")\n",
    "y_test = np.array(df['Health'].values.tolist())\n",
    "X_test = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "sigs = np.cov(X_train,rowvar=0)\n",
    "dets = np.linalg.slogdet(sigs)[-1]\n",
    "test_data = [x for x in zip(y_test,X_test)]\n",
    "training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
    "val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
    "m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
    "s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
    "n=len(X_train)-len(X_train)/10\n",
    "sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
    "sigs = np.array(sn+sigs)\n",
    "unique, counts = np.unique(y_train, return_counts=True)\n",
    "training_freq = dict(zip(unique,counts))\n",
    "train_rn()\n",
    "test_rn()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QJvWGsF1n_l-"
   },
   "source": [
    "# Section 3\n",
    "Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GfX7Eln8u9Bm"
   },
   "source": [
    "# Part A\n",
    "Medical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fRAdUbY6oXfn"
   },
   "outputs": [],
   "source": [
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = dataset[1:leng]\n",
    "\treturn dataset\n",
    "def mean(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\treturn np.sum(arr)/arr.size\n",
    "\n",
    "def standev(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\tmea = mean(arr);\n",
    "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
    "\treturn math.sqrt(squaresum) \n",
    "def trainprob(arr):\n",
    "\tarr = np.array(arr)\n",
    "\tunique, counts = np.unique(arr, return_counts=True)\n",
    "\tcounts = np.float_(counts)/float(arr.size)\n",
    "\treturn (unique,counts)\n",
    "def probdata(x,mean,stdev):\n",
    "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
    "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
    "def dsepbyclass(dataset):\n",
    "\tdicte = {}\n",
    "\tfor i in dataset:\n",
    "\t\tif(i[0] not in dicte):\n",
    "\t\t\tdicte[i[0]] = []\n",
    "\t\tdicte[i[0]].append(i[1:4])\n",
    "\treturn dicte\n",
    "def meanvar(dataset):\n",
    "\tdicte = {}\n",
    "\tfor i in dataset:\n",
    "\t\ta = zip(*dataset[i])\n",
    "\t\tdicte[i] = []\n",
    "\t\tfor j in a:\n",
    "\t\t\tp = np.array(j)\n",
    "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
    "\treturn dicte"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "rnaHnNSAtKpW",
    "outputId": "629acab4-b38f-43eb-e939-60285210fdc2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89.6666666667\n"
     ]
    }
   ],
   "source": [
    "def predictclass(mdict,data,probmatrix):\n",
    "\tdicter = predictprob(mdict,data)\n",
    "\tbestprob,bestclass = -1,None\n",
    "\tt =0\n",
    "\tfor clas,prob in dicter.iteritems():\n",
    "\t\tif bestprob<prob*probmatrix[t] :\n",
    "\t\t\tbestprob = prob*probmatrix[t]\n",
    "\t\t\tbestclass = clas\n",
    "\t\tt+=1\n",
    "\n",
    "\treturn bestclass\n",
    "\n",
    "\n",
    "   \n",
    "def predictprob(mdict,data):\n",
    "\tdicter = {}\n",
    "\t\n",
    "\tfor classvalue,classdata in mdict.iteritems():\n",
    "\t\tprob = 1\n",
    "\t\tt=1\n",
    "\t\tfor j in classdata:\n",
    "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
    "\t\t\tt +=1\n",
    "\t\tdicter[classvalue] = prob\n",
    "\treturn dicter\n",
    "\n",
    "\n",
    "            \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\t#return [(mean(x),standev(x)) for x in a]\n",
    "def answerdata(file1,file2):\n",
    "\tdataset = loadCsv(file1)\n",
    "\tdic = dsepbyclass(dataset)\n",
    "\tmdict = meanvar(dic)\n",
    "\tdataset1 = loadCsv(file2)\n",
    "\tdataset1 = dataset1[400:700]\n",
    "\tunique ,counts = trainprob(dataset[1:500])\n",
    "\taccuracy =0\n",
    "\tfor j in dataset1:\n",
    "\t\ta =  predictclass(mdict,j,counts)\n",
    "\t\tif(a==j[0]):\n",
    "\t\t\taccuracy += 1\n",
    "\tprint (float(accuracy)/float(len(dataset1)))*100\n",
    "\n",
    "\n",
    "\n",
    "answerdata(\"drive/ml1/Medical_data.csv\",\"drive/ml1/test_medical.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v-pNstvmu4iP"
   },
   "source": [
    "# Part B\n",
    "Fashion Mnist Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o0jhg6iDtXub"
   },
   "outputs": [],
   "source": [
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = dataset[1:leng]\n",
    "\treturn dataset\n",
    "def mean(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\n",
    "\treturn np.sum(arr)/arr.size\n",
    "\n",
    "def standev(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\tmea = mean(arr)\n",
    "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
    "\treturn math.sqrt(squaresum)\n",
    "def trainprob(dicte,X_train):\n",
    "\t#unique, counts = np.unique(arr, return_counts=True)\n",
    "\t#counts = np.float_(counts)/float(arr.size)\n",
    "\t#return (unique,counts)\n",
    "\ta ={}\n",
    "\tlent = len(X_train)\n",
    "\tfor j in dicte:\n",
    "\t\ta[j] =   float(len(dicte[j]))/float(lent)\n",
    "\treturn a\n",
    "def probdata(x,mean,stdev):\n",
    "\t#print mean\n",
    "\t#print stdev\n",
    "\tif stdev==0:\n",
    "\t\treturn 1\n",
    "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
    "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
    "def dsepbyclass(X_train,y_train):\n",
    "\tdicte = {}\n",
    "\tfor i in range(len(X_train)):\n",
    "\t\tif(y_train[i] not in dicte):\n",
    "\t\t\tdicte[y_train[i]] = []\n",
    "\t\tdicte[y_train[i]].append(X_train[i])\n",
    "\treturn dicte\n",
    "def meanvar(dataset):\n",
    "\tdicte = {}\n",
    "\tfor i in dataset:\n",
    "\t\ta = zip(*dataset[i])\n",
    "\t\t#print len(a)\n",
    "\t\tdicte[i] = []\n",
    "\t\tfor j in a:\n",
    "\t\t\tp = np.array(j)\n",
    "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
    "\treturn dicte\n",
    "def predictclass(mdict,data,probmatrix):\n",
    "\tdicter = predictprob(mdict,data)\n",
    "\tbestprob,bestclass = -1,None\n",
    "\tt =0\n",
    "\tfor clas,prob in dicter.iteritems():\n",
    "\t\tif bestprob<prob*probmatrix[clas] :\n",
    "\t\t\tbestprob = prob*probmatrix[clas]\n",
    "\t\t\tbestclass = clas\n",
    "\t\tt+=1\n",
    "\n",
    "\treturn bestclass\n",
    "\n",
    "\n",
    "\n",
    "def predictprob(mdict,data):\n",
    "\tdicter = {}\n",
    "\n",
    "\tfor classvalue,classdata in mdict.iteritems():\n",
    "\t\tprob = 1\n",
    "\t\tt=0\n",
    "\t\tfor j in classdata:\n",
    "\t\t\t#print j\n",
    "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
    "\t\t\tt +=1\n",
    "\t\tdicter[classvalue] = prob\n",
    "\treturn dicter\n",
    "\n",
    "\t#return [(mean(x),standev(x)) for x in a]\n",
    "def answerdata():\n",
    "\tX_train, y_train = load_mnist(\"drive/ml1/data/fashion\", kind='train')\n",
    "\tX_test, y_test = load_mnist(\"drive/ml1/data/fashion\", kind='t10k')\n",
    "\t#pcareduction([X_train[0]])\n",
    "\t#return\n",
    "\tX_test = pcareduction(X_test)\n",
    "\tX_train = pcareduction(X_train)\n",
    "\t#dataset = loadCsv(file1)\n",
    "\tdic = dsepbyclass(X_train,y_train)\n",
    "\tmdict = meanvar(dic)\n",
    "\tcounts = trainprob(dic,X_train)\n",
    "\taccuracy =0\n",
    "\tfor j in range(len(X_test)):\n",
    "\t\ta =  predictclass(mdict,X_test[j],counts)\n",
    "\t\tif(a==y_test[j]):\n",
    "\t\t\taccuracy += 1\n",
    "\t\t\t#print \"correct\"\n",
    "\t\t#else:\n",
    "\t\t\t#print \"incorrect\"\n",
    "\tprint \"Accuracy: \" + str((float(accuracy)/float(len(X_test)))*100)\n",
    "def pcareduction(X_train):\n",
    "\ta = []\n",
    "\tfor i in X_train:\n",
    "\t    M = np.mean(i)\n",
    "\t    i = i-M\n",
    "\t    b = i.reshape(28,28)\n",
    "\t    #m = np.mean(b.T,axis =1)\n",
    "\t    c = np.cov(b)\n",
    "\t    values,vectors = np.linalg.eig(c)\n",
    "\t    eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "\t    eig_pair.sort(key=lambda x:x[0])\n",
    "\t    eig_pair.reverse()\n",
    "\t    #print eig_pair\n",
    "\t    ab = []\n",
    "\t    #print eig_pair[0][0]\n",
    "\t    for j in range(6):\n",
    "\t    \tab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
    "\t    #print ab\n",
    "\t    a.append(ab)\n",
    "\treturn a\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def pca1(X_train):\n",
    "\ta = []\n",
    "\tb = X_train.T\n",
    "\tM = np.mean(b,axis=1)\n",
    "\tprint M.shape\n",
    "\tac = X_train- M\n",
    "\tc = np.cov(ac.T)\n",
    "\tprint c.shape\n",
    "\tvalues,vectors = np.linalg.eig(c)\n",
    "\teig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "\teig_pair.sort(key=lambda x:x[0])\n",
    "\teig_pair.reverse()\n",
    "\tfor i,j in eig_pair:\n",
    "\t\tprint i\n",
    "\tab =[]\n",
    "\tfor j in range(6):\n",
    "\t\tab.append(eig_pair[j][1])\n",
    "\tab = np.array(ab)\n",
    "\tprint ab.shape\n",
    "\treturn\n",
    "\tp = ab.dot(ac.T)\n",
    "\tprint p.shape\n",
    "\treturn p.T\n",
    "def a(x):\n",
    "\tx.sort(key=lambda x:x[0])\n",
    "\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 125
    },
    "colab_type": "code",
    "id": "uGzqvXRIvQXA",
    "outputId": "2d1a471f-d844-4caa-bcba-38c899966c33"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: ComplexWarning: Casting complex values to real discards the imaginary part\n",
      "  \n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:13: ComplexWarning: Casting complex values to real discards the imaginary part\n",
      "  del sys.path[0]\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:72: ComplexWarning: Casting complex values to real discards the imaginary part\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 52.48\n"
     ]
    }
   ],
   "source": [
    "answerdata()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GWVlBSMxwvJt"
   },
   "source": [
    "#Part C\n",
    "Railway Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "id": "n5mKGc7awyy2",
    "outputId": "5efa0a0f-2b94-4f51-fbdb-c84e41de8db0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuaracy = 76.0305343511\n",
      "precision = 0.745059288538\n",
      "recall = 0.930864197531\n",
      "F1 score = 0.827661909989\n"
     ]
    }
   ],
   "source": [
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = np.array(dataset[1:leng])\n",
    "\tdataset1 = dataset[:,[1,2,3,6]]\n",
    "\tdataset2 = dataset[:,[1,4,5]]\n",
    "\treturn dataset1,dataset2\n",
    "def mean(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\treturn np.sum(arr)/arr.size\n",
    "\n",
    "def standev(arr):\n",
    "\tarr = arr.astype(np.float)\n",
    "\tmea = mean(arr);\n",
    "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
    "\treturn math.sqrt(squaresum) \n",
    "def trainprob(arr):\n",
    "\tarr = np.array(arr)\n",
    "\tunique, counts = np.unique(arr, return_counts=True)\n",
    "\tcounts = np.float_(counts)/float(arr.size)\n",
    "\treturn (unique,counts)\n",
    "def probdata(x,mean,stdev):\n",
    "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
    "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
    "def dsepbyclass(dataset):\n",
    "\tdicte = {}\n",
    "\tfor i in dataset:\n",
    "\t\tif(i[0] not in dicte):\n",
    "\t\t\tdicte[i[0]] = []\n",
    "\t\tdicte[i[0]].append(i[1:4])\n",
    "\treturn dicte\n",
    "def meanvar(dataset):\n",
    "\tdicte = {}\n",
    "\tfor i in dataset:\n",
    "\t\ta = zip(*dataset[i])\n",
    "\t\tdicte[i] = []\n",
    "\t\tfor j in a:\n",
    "\t\t\tp = np.array(j)\n",
    "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
    "\treturn dicte\n",
    "def newprob(dicter):\n",
    "\tdic ={}\n",
    "\tfor j in dicter:\n",
    "\t\ta = zip(*dicter[j])\n",
    "\t\tunique1,counts1 = np.unique(a[0], return_counts=True)\n",
    "\t\tunique2,counts2 = np.unique(a[1], return_counts=True) \n",
    "\t\tb = {}\n",
    "\t\tc =0\n",
    "\t\td= 0 \n",
    "\t\tfor k in counts1:\n",
    "\t\t\tc +=k\n",
    "\t\tfor l in counts2:\n",
    "\t\t\td +=l\n",
    "\t\tfor qw in range(len(unique1)):\n",
    "\t\t\tb[unique1[qw]] = float(counts1[qw])/float(c)\n",
    "\t\tfor qa in range(len(unique2)):\n",
    "\t\t\tb[unique2[qa]] = float(counts2[qa])/float(c)\n",
    "\t\tdic[j] = b\n",
    "\tif len(dic['0']) != len(dic['1']):\n",
    "\t\tfor j in dic['0']:\n",
    "\t\t\tif not j in dic['1']:\n",
    "\t\t\t\tdic['1'][j] =0\n",
    "\t\tfor j in dic['1']:\n",
    "\t\t\tif not j in dic['0']:\n",
    "\t\t\t\tdic['0'][j] =0\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\treturn dic\n",
    "def predictclass(mdict,data,data1,probmatrix,ndict):\n",
    "\tdicter = predictprob(mdict,data,probmatrix)\n",
    "\tbestprob,bestclass = -1,None\n",
    "\t#print dicter\n",
    "\tfor clas,prob in dicter.iteritems():\n",
    "\t\t\n",
    "\t\tab = ndict[clas]\n",
    "\t\tif data1[1] not in ab:\n",
    "\t\t\tp=0\n",
    "\t\telif data1[2] not in ab:\n",
    "\t\t\tp = 0\n",
    "\t\telse:\n",
    "\t\t\tp = prob*ab[data1[1]]*ab[data1[2]]\n",
    "\t\t\n",
    "\t\tif bestprob< p:\n",
    "\t\t\tbestprob = p\n",
    "\t\t\tbestclass = clas\n",
    "\n",
    "\treturn bestclass\n",
    "\n",
    "\n",
    "   \n",
    "def predictprob(mdict,data,cd):\n",
    "\tdicter = {}\n",
    "\tre = 0\n",
    "\tfor classvalue,classdata in mdict.iteritems():\n",
    "\t\tprob = 1\n",
    "\t\tt=1\n",
    "\t\tfor j in classdata:\n",
    "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
    "\t\t\tt +=1\n",
    "\t\tdicter[classvalue] = prob*cd[re]\n",
    "\t\tre += 1\n",
    "\treturn dicter\n",
    "\n",
    "\t#return [(mean(x),standev(x)) for x in a]\n",
    "def answerdata(file1,file2):\n",
    "\tdataset1,dataset2 = loadCsv(file1)\n",
    "\tdic = dsepbyclass(dataset1)\n",
    "\tdic1 = dsepbyclass(dataset2)\n",
    "\tmdict = meanvar(dic)\n",
    "\tndict = newprob(dic1)\n",
    "\tdataset3 ,dataset4 = loadCsv(file2)\n",
    "\t#print trainprob\n",
    "\tunique ,counts = trainprob(dataset1[:,1])\n",
    "\taccuracy =0\n",
    "\ttp = 0\n",
    "\tfp =0\n",
    "\tfn =0\n",
    "\tfor s in range(len(dataset3)):\n",
    "\t\tj = dataset3[s]\n",
    "\t\tk = dataset4[s]\n",
    "\t\ta =  predictclass(mdict,j,k,counts,ndict)\n",
    "\t\tif(a==j[0]):\n",
    "\t\t\taccuracy += 1\n",
    "\t\t\tif(a ==\"1\"):\n",
    "\t\t\t\ttp +=1\n",
    "\t\telse:\n",
    "\t\t\tif(a==\"1\"):\n",
    "\t\t\t\tfp +=1\n",
    "\t\t\tif(a==\"0\"):\n",
    "\t\t\t\tfn += 1\n",
    "\tprecision = float(tp)/float(tp+fp)\n",
    "\trecall = float(tp)/float(tp+fn)\n",
    "\tprint \"accuaracy = \" + str((float(accuracy)/float(len(dataset1)))*100)\n",
    "\tprint \"precision = \" + str(precision)\n",
    "\tprint \"recall = \" + str(recall)\n",
    "\tprint \"F1 score = \" +str(float(2*precision*recall)/float(precision+recall))\n",
    "\n",
    "\n",
    "\n",
    "answerdata(\"drive/ml1/railwayBookingList.csv\",\"drive/ml1/railwayBookingList.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OW_64bLDxLWf"
   },
   "source": [
    "#Section 4\n",
    "K Nearest Neighbor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Nj1CUy19xsSO"
   },
   "source": [
    "#Part A\n",
    "Medical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "me7jAEFDxByr",
    "outputId": "7eabedc4-00bd-4690-9e41-e3c19db3f14b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79.6\n"
     ]
    }
   ],
   "source": [
    "def knnclassifier(file1,file2,k):\n",
    "\tdataset1 = np.array(loadCsv(file1))\n",
    "\tdataset1 = dataset1[1:1500]\n",
    "\tdataset2 = np.array(loadCsv(file2))\n",
    "\tdataset2 = dataset2[1500:3000]\n",
    "\n",
    "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
    "\taccuracy =0\n",
    "\tfor i in dataset2:\n",
    "\t\t#print i\n",
    "\t\tclas = find(i,dataset1,k)\n",
    "\t\t#print \"---\"\n",
    "\t\t#print clas\n",
    "\t\t#print\"-----\"\n",
    "\t\tif clas == i[0]:\n",
    "\t\t\taccuracy +=1\n",
    "\tprint (float(accuracy)/float(len(dataset2)))*100\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def find(data,file,k):\n",
    "\ta = []\n",
    "\tfor j in file:\n",
    "\t\tdis = finddis(data,j)\n",
    "\t\ta.append([j[0],dis])\n",
    "\t#print len(a)\n",
    "\tb = findclass(a,k)\n",
    "\treturn b\n",
    "\n",
    "def findclass(a,k):\n",
    "\tm = {}\n",
    "\tb =[]\n",
    "\tn = len(a)\n",
    "\tfor i in range(k):\n",
    "\t\tp =float(\"inf\")\n",
    "\t\tc = ''\n",
    "\t\tfor j in range(i,n-1):\n",
    "\t\t\tif a[j][1] <p:\n",
    "\t\t\t\tp = a[j][1]\n",
    "\t\t\t\tc = a[j][0] \n",
    "\t\tb.append([c,p])\n",
    "\t#print b\n",
    "\tfor d in range(k-1):\n",
    "\t\tif b[d][0] not in m:\n",
    "\t\t\tm[b[d][0]] =1\n",
    "\t\telse:\n",
    "\t\t\tm[b[d][0]] +=1\n",
    "\tmaxclass = None\n",
    "\tmaxvalue = -1\n",
    "\tfor clas,value in m.iteritems():\n",
    "\t\tif maxvalue<value:\n",
    "\t\t\tmaxvalue = value\n",
    "\t\t\tmaxclass = clas\n",
    "\treturn maxclass\n",
    "\n",
    "def finddis(data1,data2):\n",
    "\ta = np.array(data1[1:3])\n",
    "\ta = a.astype(np.float)\n",
    "\tb = np.array(data2[1:3])\n",
    "\tb = b.astype(np.float)\n",
    "\treturn math.sqrt(np.sum((a-b)**2))\n",
    "\n",
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = dataset[1:leng]\n",
    "\treturn dataset\n",
    "knnclassifier(\"drive/ml1/Medical_data.csv\",\"drive/ml1/test_medical.csv\",50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MSdeYLL4yPfx"
   },
   "source": [
    "#Part B\n",
    "Railway Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "id": "P4MxAXNvymuJ",
    "outputId": "b86a08bf-e352-40e8-d7d2-907e4764b5e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuaracy = 79.7709923664\n",
      "precision = 0.827130852341\n",
      "recall = 0.850617283951\n",
      "F1 score = 0.838709677419\n"
     ]
    }
   ],
   "source": [
    "def knnclassifier(file1,file2,k):\n",
    "\tdataset1 = np.array(loadCsv(file1))\n",
    "\tdataset2 = np.array(loadCsv(file2))\n",
    "\tdataset11 = dataset1[:,[1,2,3,6]]\n",
    "\tdataset12 = dataset1[:,[1,4,5]]\n",
    "\tdataset21 = dataset2[:,[1,2,3,6]]\n",
    "\tdataset22 = dataset2[:,[1,4,5]]\n",
    "\n",
    "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
    "\taccuracy =0\n",
    "\tfp =0\n",
    "\tfn=0\n",
    "\ttp = 0\n",
    "\tfor i in range(len(dataset2)):\n",
    "\t\t#print i\n",
    "\t\tclas = find(dataset21[i],dataset22[i],dataset11,dataset22,k)\n",
    "\t\t#print \"---\"\n",
    "\t\t#print clas\n",
    "\t\t#print\"-----\"\n",
    "\t\tif clas == dataset21[i][0]:\n",
    "\t\t\taccuracy +=1\n",
    "\t\t\tif(clas ==\"1\"):\n",
    "\t\t\t\ttp +=1\n",
    "\t\telse:\n",
    "\t\t\tif(clas==\"1\"):\n",
    "\t\t\t\tfp +=1\n",
    "\t\t\tif(clas==\"0\"):\n",
    "\t\t\t\tfn += 1\n",
    "\n",
    "\taccuracy =  (float(accuracy)/float(len(dataset2)))*100\n",
    "\tprecision = float(tp)/float(tp+fp)\n",
    "\trecall = float(tp)/float(tp+fn)\n",
    "\tprint \"accuaracy = \" + str(accuracy)\n",
    "\tprint \"precision = \" + str(precision)\n",
    "\tprint \"recall = \" + str(recall)\n",
    "\tprint \"F1 score = \" +str(float(2*precision*recall)/float(precision+recall))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def find(data,data1,file,file1,k):\n",
    "\ta = []\n",
    "\tfor j in range(len(file)):\n",
    "\t\tdis = finddis(data,data1,file[j],file1[j])\n",
    "\t\ta.append([file[j][0],dis])\n",
    "\t#print len(a)\n",
    "\tb = findclass(a,k)\n",
    "\treturn b\n",
    "\n",
    "def findclass(a,k):\n",
    "\tm = {}\n",
    "\tb =[]\n",
    "\tn = len(a)\n",
    "\tfor i in range(k):\n",
    "\t\tp =float(\"inf\")\n",
    "\t\tc = ''\n",
    "\t\tfor j in range(i,n-1):\n",
    "\t\t\tif a[j][1] <p:\n",
    "\t\t\t\tp = a[j][1]\n",
    "\t\t\t\tc = a[j][0] \n",
    "\t\tb.append([c,p])\n",
    "\t#print b\n",
    "\tfor d in range(k-1):\n",
    "\t\tif b[d][0] not in m:\n",
    "\t\t\tm[b[d][0]] =1\n",
    "\t\telse:\n",
    "\t\t\tm[b[d][0]] +=1\n",
    "\tmaxclass = None\n",
    "\tmaxvalue = -1\n",
    "\tfor clas,value in m.iteritems():\n",
    "\t\tif maxvalue<value:\n",
    "\t\t\tmaxvalue = value\n",
    "\t\t\tmaxclass = clas\n",
    "\treturn maxclass\n",
    "\n",
    "def finddis(data1,data3,data2,data4):\n",
    "\ta = np.array(data1[1:3])\n",
    "\ta = a.astype(np.float)\n",
    "\tb = np.array(data2[1:3])\n",
    "\tb = b.astype(np.float)\n",
    "\tc = math.sqrt(np.sum((a-b)**2))\n",
    "\tif data3[1] != data4[1]:\n",
    "\t\tc+=2\n",
    "\tif data3[2] != data4[2]:\n",
    "\t\tc+=2\n",
    "\treturn c\n",
    "\n",
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = dataset[1:leng]\n",
    "\treturn dataset\n",
    "knnclassifier(\"drive/ml1/railwayBookingList.csv\",\"drive/ml1/railwayBookingList.csv\",25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EPamaqley6Rx"
   },
   "source": [
    "#Part C\n",
    "Fashion Mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xQAWGZVAy0Vf"
   },
   "outputs": [],
   "source": [
    "def pcareduction(X_train):\n",
    "\ta = []\n",
    "\tfor i in X_train:\n",
    "\t    M = np.mean(i)\n",
    "\t    i = i-M\n",
    "\t    b = i.reshape(28,28)\n",
    "\t    c = np.cov(b)\n",
    "\t    values,vectors = np.linalg.eig(c)\n",
    "\t    eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
    "\t    eig_pair.sort(key=lambda x:x[0])\n",
    "\t    eig_pair.reverse()\n",
    "\t    #print eig_pair\n",
    "\t    ab = []\n",
    "\t    #print eig_pair[0][0]\n",
    "\t    for j in range(6):\n",
    "\t    \tab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
    "\t    #print ab\n",
    "\t    a.append(ab)\n",
    "\t#print a\n",
    "\n",
    "\n",
    "\n",
    "\treturn a\n",
    "\n",
    "def knnclassifier(k):\n",
    "\tX_train, y_train = mnist_reader.load_mnist(\"drive/ml1/data/fashion\", kind='train')\n",
    "\tX_test, y_test = mnist_reader.load_mnist(\"drive/ml1/data/fashion\", kind='t10k')\n",
    "\tX_train  = pcareduction(X_train)\n",
    "\tX_test = pcareduction(X_test)\n",
    "\ty_train = np.reshape(y_train, (-1, 1))\n",
    "\ty_test = np.reshape(y_test, (-1, 1))\n",
    "\tdataset1 = np.hstack((y_train,X_train))\n",
    "\tdataset2 = np.hstack((y_test,X_test))\n",
    "\tdataset1 = dataset1\n",
    "\tdataset2 = dataset2\n",
    "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
    "\taccuracy =0\n",
    "\tfor i in dataset2:\n",
    "\t\t#print i\n",
    "\t\tclas = find(i,dataset1,k)\n",
    "\t\t#print \"---\"\n",
    "\t\t#print clas\n",
    "\t\t#print\"-----\"\n",
    "\t\tif clas == i[0]:\n",
    "\t\t\taccuracy +=1\n",
    "\t\t\t#print \"correct\"\n",
    "\t\t#else :\n",
    "\t\t\t#print \"incorrect\"\n",
    "\tprint \"accurracy=\" + str((float(accuracy)/float(len(dataset2)))*100)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def find(data,file,k):\n",
    "\ta = []\n",
    "\tfor j in file:\n",
    "\t\tdis = finddis(data,j)\n",
    "\t\ta.append([j[0],dis])\n",
    "\t#print len(a)\n",
    "\tb = findclass(a,k)\n",
    "\treturn b\n",
    "\n",
    "def findclass(a,k):\n",
    "\tm = {}\n",
    "\tb =[]\n",
    "\tn = len(a)\n",
    "\tfor i in range(k):\n",
    "\t\tp =float(\"inf\")\n",
    "\t\tc = ''\n",
    "\t\tfor j in range(i,n-1):\n",
    "\t\t\tif a[j][1] <p:\n",
    "\t\t\t\tp = a[j][1]\n",
    "\t\t\t\tc = a[j][0]\n",
    "\t\tb.append([c,p])\n",
    "\t#print b\n",
    "\tfor d in range(k-1):\n",
    "\t\tif b[d][0] not in m:\n",
    "\t\t\tm[b[d][0]] =1\n",
    "\t\telse:\n",
    "\t\t\tm[b[d][0]] +=1\n",
    "\tmaxclass = None\n",
    "\tmaxvalue = -1\n",
    "\tfor clas,value in m.iteritems():\n",
    "\t\tif maxvalue<value:\n",
    "\t\t\tmaxvalue = value\n",
    "\t\t\tmaxclass = clas\n",
    "\treturn maxclass\n",
    "\n",
    "def finddis(data1,data2):\n",
    "\ta = np.array(data1[1:len(data1)])\n",
    "\ta = a.astype(np.float)\n",
    "\tb = np.array(data2[1:len(data2)])\n",
    "\tb = b.astype(np.float)\n",
    "\treturn math.sqrt(np.sum((a-b)**2))\n",
    "\n",
    "def loadCsv(filename):\n",
    "\tlines = csv.reader(open(filename, \"rb\"))\n",
    "\tdataset = list(lines)\n",
    "\tleng = len(dataset)\n",
    "\tdataset = dataset[1:leng]\n",
    "\treturn dataset\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YYL9gAIA3MUX"
   },
   "outputs": [],
   "source": [
    "knnclassifier(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dr2KnpvO3OWX"
   },
   "source": [
    "![alt text](https://3.bp.blogspot.com/-VaR_5oYII5U/W57oNE53MPI/AAAAAAAAKO0/s1FnDpGl07g2yDGNl0ve6lWdXQCliP_3gCLcBGAs/s320/Screenshot_5.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Z_UwDAgfzoZo"
   },
   "source": [
    "#Section 5\n",
    "K Means Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zsOhyZrLzm_y"
   },
   "outputs": [],
   "source": [
    "\n",
    "def precision(predicted_test,test):\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    for i in range(len(test)):\n",
    "        if predicted_test[i][1] == test[i][1]:\n",
    "            if predicted_test[i][1]==1:\n",
    "                TP = TP + 1\n",
    "        else:\n",
    "            if predicted_test[i][1] == 1:\n",
    "                FP = FP + 1\n",
    "    #print (str(TP)+\" \"+str(FP))\n",
    "    return TP*(1.0/(TP+FP))\n",
    "def accuracy(predicted_test,test):\n",
    "    T = 0\n",
    "    for i in range(len(test)):\n",
    "        if predicted_test[i][1] == test[i][1]:\n",
    "            T = T + 1\n",
    "    #print (str(T)+\" \")\n",
    "    return T*(1.0/len(test))\n",
    "def recall(predicted_test,test):\n",
    "    TP = 0\n",
    "    FN = 0\n",
    "    for i in range(len(test)):\n",
    "        if predicted_test[i][1] == test[i][1]:\n",
    "            if predicted_test[i][1]==1:\n",
    "                TP = TP + 1\n",
    "        else:\n",
    "            if predicted_test[i][1] == 0:\n",
    "                FN = FN + 1\n",
    "\n",
    "    return TP*(1.0/(TP+FN))\n",
    "def F1(precision,recall):\n",
    "    return 2.0*(precision*recall)*(1.0/(precision+recall))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class kmean:\n",
    "    name = \"\"\n",
    "    k=2\n",
    "    def __init__(self,name=\"K-Means Clustering\",k=2):\n",
    "        self.name = name\n",
    "        self.k = k\n",
    "\n",
    "    def initialize(self,train):\n",
    "\n",
    "        l=[]\n",
    "        i=0\n",
    "        for t in train:\n",
    "            if t[1]==i:\n",
    "                l.append(t[0])\n",
    "                i+=1\n",
    "            if i==self.k:\n",
    "                break\n",
    "        print(\"number of classes: \"+str(len(l)))\n",
    "        return l\n",
    "\n",
    "    def make_cluster(self,train,centroids):\n",
    "        clusters = {c: [] for c in range(self.k)}\n",
    "        #initialze cluster with key as label\n",
    "\n",
    "        for X,Y in train:\n",
    "            min = float('inf')\n",
    "            its_cluster = 0\n",
    "\n",
    "            for i in range(self.k):\n",
    "                centroid_i = centroids[i]\n",
    "                euclidean_dis = np.linalg.norm(X-centroid_i)\n",
    "                if euclidean_dis<min:\n",
    "                    its_cluster = i\n",
    "                    min = euclidean_dis\n",
    "            clusters[its_cluster].append((Y,X))\n",
    "        return clusters.values()\n",
    "\n",
    "    def avg(self,cluster):\n",
    "        #print(len(cluster))\n",
    "        l = list(map(lambda x: x[1], cluster))\n",
    "        return np.array(l).mean(axis=0)\n",
    "    def change_centroids(self,clusters):\n",
    "        centroids = []\n",
    "        #print(len(clusters))\n",
    "        for cluster in clusters:\n",
    "            #print(len(cluster))\n",
    "            centroids.append(self.avg(cluster))\n",
    "\n",
    "        return centroids\n",
    "\n",
    "    def train_the_model(self,train):\n",
    "        centroids = self.initialize(train)\n",
    "        clusters = self.make_cluster(train,centroids)\n",
    "        diff = 0\n",
    "        i=0\n",
    "        while True:\n",
    "            old_centroids = centroids\n",
    "            centroids = self.change_centroids(clusters)\n",
    "            clusters = self.make_cluster(train,centroids)\n",
    "            #print(\"diff\",end=\"\\r\")\n",
    "            #print(list(map(lambda a,b : np.linalg.norm(a-b),old_centroids,centroids)))\n",
    "            new_diff = max(list(map(lambda a,b : np.linalg.norm(a-b),old_centroids,centroids)))\n",
    "            #print (\"new_diff-: \"+ str(new_diff),end=\"\\r\")\n",
    "            try:\n",
    "                change_in_diff = abs((new_diff-diff)/np.mean([diff,new_diff])) * 100\n",
    "                #print (\"iteration-:\"+str(i) + \" \" +str(change_in_diff) ,end=\"\\r\")\n",
    "            except:\n",
    "                break\n",
    "            if np.isnan(change_in_diff):\n",
    "                break\n",
    "\n",
    "            diff = new_diff\n",
    "            i += 1\n",
    "        return clusters,centroids\n",
    "\n",
    "    def labelling(self,clusters,centroids):\n",
    "        new_centroids = []\n",
    "        i=0\n",
    "        for cluster in clusters:\n",
    "            labels = [y for y,x in cluster]\n",
    "            high_freq = max(set(labels),key=labels.count)\n",
    "            centroid = (high_freq,centroids[i])\n",
    "            new_centroids.append(centroid)\n",
    "            i = i + 1\n",
    "        return new_centroids\n",
    "\n",
    "    def test(self,testimage,centroids):\n",
    "        min = float(\"inf\")\n",
    "        y = 0\n",
    "        for (label, centroid) in centroids:\n",
    "            distance = np.linalg.norm(centroid - testimage)\n",
    "            if distance < min:\n",
    "                min = distance\n",
    "                y = label\n",
    "        #print(y)\n",
    "        return y\n",
    "def main(train,test,k):\n",
    "    #print(\"Creating classifier Object...\")\n",
    "    \n",
    "    classifier = kmean(k=k)\n",
    "    #print(\"classifier name: \" + classifier.name)\n",
    "    #print(\"Training Start Here...\")\n",
    "    \n",
    "    clusters,centroids=classifier.train_the_model(train)\n",
    "    labelled_centroids = classifier.labelling(clusters,centroids)\n",
    "    predicted_test = []\n",
    "    #print(\"Prediction Start Here...\")\n",
    "    \n",
    "    for x,y in test:\n",
    "        yy = classifier.test(x,labelled_centroids)\n",
    "        predicted_test.append((x,yy))\n",
    "\n",
    "    predicted_train = []\n",
    "    for x,y in train:\n",
    "        yy = classifier.test(x,labelled_centroids)\n",
    "        predicted_train.append((x,yy))\n",
    "\n",
    "    print(\"Test Accuracy: \" + str(accuracy(predicted_test,test)))\n",
    "    #print(\"Training Accuracy: \" + str(accuracy(predicted_train,test)))\n",
    "\n",
    "    if k==2:\n",
    "        p=precision(predicted_test,test)\n",
    "        r=recall(predicted_test,test)\n",
    "        print(\"Precision \"+str(p))\n",
    "        print(\"Recall \"+str(r))\n",
    "        print(\"F1 Score \"+ str(F1(p,r)))\n",
    "\n",
    "\n",
    "\n",
    "def mnist():\n",
    "    trainX , trainY = load_mnist('drive/ml1/data/fashion', kind='train')\n",
    "    testX , testY = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
    "    train = [list(a) for a in zip(trainX, trainY)]\n",
    "    test = [list(a) for a in zip(testX, testY)]\n",
    "\n",
    "    main(train,test,10)\n",
    "\n",
    "def medical():\n",
    "    df = pd.read_csv('drive/ml1/Medical_data.csv')\n",
    "    df2 = pd.read_csv('drive/ml1/test_medical.csv')\n",
    "    df['Health'] = df['Health'].map({'HEALTHY':0,'MEDICATION':1,'SURGERY':2})\n",
    "    df2['Health'] = df2['Health'].map({'HEALTHY':0,'MEDICATION':1,'SURGERY':2})\n",
    "    trainY = df['Health'].values.tolist()\n",
    "    trainXX = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "    trainX = list(map(lambda x: np.array(x),trainXX))\n",
    "    train = [list(a) for a in zip(trainX, trainY)]\n",
    "    testY = df2['Health'].values.tolist()\n",
    "    testXX = df2[['TEST1','TEST2','TEST3']].values.tolist()\n",
    "    testX = list(map(lambda x: np.array(x),testXX))\n",
    "    test = [list(a) for a in zip(testX, testY)]\n",
    "    #print(type(train))\n",
    "    main(train,test,3)\n",
    "\n",
    "def rail():\n",
    "    df = pd.read_csv('drive/ml1/railwayBookingList.csv')\n",
    "    dataY = df['boarded'].values.tolist()\n",
    "    df.memberCount = df.memberCount*df['budget'].mean()\n",
    "    df.age = df.age*df['budget'].mean()\n",
    "    df['preferredClass'] = df['preferredClass'].map({'FIRST_AC':100000,'SECOND_AC':500000,'THIRD_AC':100000,'NO_PREF':0})\n",
    "    df['sex'] = df['sex'].map({'female':300000,'male':300000})\n",
    "    dataXX = df[['budget','memberCount','preferredClass','sex','age']].values.tolist()\n",
    "    dataX = list(map(lambda x: np.array(x),dataXX))\n",
    "    data = [list(a) for a in zip(dataX, dataY)]\n",
    "    train = data[:int(len(data)*0.75)]\n",
    "    test = data[int(len(data)*0.75):]\n",
    "    #print(df[['budget','memberCount','preferredClass','sex','age']])\n",
    "\n",
    "    main(train,test,2)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "ZXGFuKlP0GoI",
    "outputId": "981c6835-a5da-4717-e658-86ea0c344ed9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of classes: 3\n",
      "Test Accuracy: 0.534666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "medical()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 125
    },
    "colab_type": "code",
    "id": "rY_1bnSA0-Eh",
    "outputId": "80ad67e9-7e73-4043-d6ee-4a0cde8d26db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of classes: 2\n",
      "Test Accuracy: 0.743902439024\n",
      "Precision 0.746177370031\n",
      "Recall 0.995918367347\n",
      "F1 Score 0.853146853147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "rail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "EqrltOIj0_ky",
    "outputId": "670a2282-35c3-404e-9841-5c29969fbdb5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of classes: 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.5275\n"
     ]
    }
   ],
   "source": [
    "mnist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ipKxKL7x1EJv"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Assignment1-Ell409.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}