tutorial_day5_self_organizing_maps.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Implementation of Self-Organizing Maps with Python for processing COVID 19 databases.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fGCr1YH77HXt"
      },
      "source": [
        "# Implementation of Self-Organizing Maps with Python for processing COVID 19 databases"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ovME1Oki7v5u"
      },
      "source": [
        "Programmer Information"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QQt9WLhX26dA"
      },
      "source": [
        "# Author = Darío Sebastián Cabezas Erazo\n",
        "# Email = dario.cabezas@yachaytech.edu.ec\n",
        "# LinkedIn =https://www.linkedin.com/in/dario-cabezas/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ujyb7bU37Qd-"
      },
      "source": [
        "# LIBRARIES"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XuPnI6xVb4KF"
      },
      "source": [
        "import gdown\n",
        "import math \n",
        "import numpy as np\n",
        "import csv\n",
        "import webbrowser\n",
        "import timeit\n",
        "import os"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TxhqdKgg7fcb"
      },
      "source": [
        "# UTILITY"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jaUYWJm3Huc"
      },
      "source": [
        "def dimension (matriz):\n",
        "    filas = len(matriz) \n",
        "    if filas > 0: \n",
        "        columnas = len(matriz[0]) \n",
        "    else: \n",
        "        columnas = 0 \n",
        "    return filas, columnas"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FhqY4o-x7n3p"
      },
      "source": [
        "# DATABASE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oxGl1I6K_az7",
        "outputId": "1424eb9a-b710-420c-ae3c-0cde50b1206d"
      },
      "source": [
        "url = 'https://drive.google.com/file/d/1VXq08OkQQBrS_iYe5UmJ38_A5dHjMQ8-'\n",
        "output = 'Epidemic-Data-for-Novel-Coronavirus-COVID-19.csv'\n",
        "gdown.download(url, output, quiet=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gdown/parse_url.py:31: UserWarning: You specified Google Drive Link but it is not the correct link to download the file. Maybe you should try: https://drive.google.com/uc?id=None\n",
            "  .format(url='https://drive.google.com/uc?id={}'.format(file_id))\n",
            "Downloading...\n",
            "From: https://drive.google.com/file/d/1VXq08OkQQBrS_iYe5UmJ38_A5dHjMQ8-\n",
            "To: /content/Epidemic-Data-for-Novel-Coronavirus-COVID-19.csv\n",
            "66.1kB [00:00, 14.6MB/s]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Epidemic-Data-for-Novel-Coronavirus-COVID-19.csv'"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZS32pGM3rwa"
      },
      "source": [
        "with open('Epidemic-Data-for-Novel-Coronavirus-COVID-19.csv', newline='') as csvfile:\n",
        "    reader = csv.DictReader(csvfile)\n",
        "    data = list(reader)\n",
        "    countries=[]\n",
        "    data_to_work=[]\n",
        "    for v in data:\n",
        "        for i in v.keys():\n",
        "            k=[]\n",
        "            if i==\"Country\":\n",
        "                k.append(v[i])\n",
        "                k.append(v['GeoPosition'])\n",
        "            if k!=[]:\n",
        "                countries.append(k)                \n",
        "            else:\n",
        "                continue\n",
        "    # Saving Countries\n",
        "    for v in data:\n",
        "        for i in v.keys():\n",
        "            k=[]\n",
        "            if i==\"ConfirmedCases\":\n",
        "                k.append(v[\"ConfirmedCases\"])\n",
        "                k.append(v[\"RecoveredCases\"])\n",
        "                k.append(v[\"Deaths\"])\n",
        "            if k!=[]:\n",
        "                data_to_work.append(k)                \n",
        "            else:\n",
        "                continue\n",
        "    # Saving Raw Data for working with\n",
        "# Partitioning Raw Data \n",
        "def clean_countries():\n",
        "    k=0\n",
        "    X=[]\n",
        "    for i in countries:\n",
        "        x=i[0].split(\",\")\n",
        "        x.append(k)\n",
        "        if isinstance(x[1], str):\n",
        "            temp=''\n",
        "            for j in x[1]:\n",
        "                if j==\"]\" or j=='\\\"' or j==' ':\n",
        "                    continue\n",
        "                temp= temp + j\n",
        "            x[1]=temp\n",
        "        k = k+1\n",
        "        x.pop(0)\n",
        "        X.append(x)\n",
        "    return X\n",
        "    # Cleaning countries for being used as strings\n",
        "def search_country(string):\n",
        "    countries_cleaned=clean_countries()\n",
        "    countries_list=[]\n",
        "    for i in countries_cleaned:\n",
        "        if i[0]==string:\n",
        "            countries_list.append(i[1])\n",
        "    return countries_list\n",
        "    # Function to look for specific countries and return a list of databases found\n",
        "def raw_data_manager(n):\n",
        "    raw_data=[]\n",
        "    d,e=dimension(data_to_work)\n",
        "    for i in range(e):\n",
        "        raw_data.append((data_to_work[n][i].split(\"{{{\")[1].split(\"}}\")[0]).split(\",\"))\n",
        "    r,c=dimension(raw_data)\n",
        "    data=np.zeros((c, r))\n",
        "    for i in range(r):\n",
        "        for j in range(c):\n",
        "            if  raw_data[i][j]==' Missing[\"NotAvailable\"]' or raw_data[i][j]=='Missing[\"NotAvailable\"]':\n",
        "                raw_data[i][j]=0\n",
        "                data[j][i]=raw_data[i][j]\n",
        "            else:\n",
        "                raw_data[i][j]=int(raw_data[i][j])\n",
        "                data[j][i]=raw_data[i][j]\n",
        "        np.append(data, raw_data[i])\n",
        "    return data\n",
        "    # Function to replace not consistent data in raw data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MvEe8Vcf71B7"
      },
      "source": [
        "# MOST IMPORTANT FUNCTIONS"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0M-wwFVl8zP4"
      },
      "source": [
        "EUCLIDEAN DISTANCE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3bx7yWGv82b1"
      },
      "source": [
        "def eu_distance(A,B):\n",
        "    return np.sqrt(np.sum((A-B)**2))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cawDsoUg8_Dk"
      },
      "source": [
        "WEIGHT UPDATE FORMULA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLcY841T9CyP"
      },
      "source": [
        "def learning(init_learning_rate,i,n_iter):\n",
        "    return init_learning_rate* np.exp(-i/n_iter)\n",
        "def topological_neighborhood(distance,radius):\n",
        "    return np.exp(-distance/(2*(radius**2)))\n",
        "def neighborhood_size(init_radius,i,time_constant):\n",
        "    return init_radius*np.exp(-i/time_constant)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N-xKtbIV9Dsc"
      },
      "source": [
        "BEST MATCHING UNIT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T49KLsvb4lNd"
      },
      "source": [
        "def find_bmu(t,net):\n",
        "    min_dist=1000000\n",
        "    for x in range(net.shape[0]):\n",
        "        for y in range(net.shape[1]):\n",
        "            unit=net[x,y].reshape(1,-1)\n",
        "            t=t.reshape(1,-1)\n",
        "            euc_dist=eu_distance(unit,t)\n",
        "            if euc_dist < min_dist:\n",
        "                min_dist=euc_dist\n",
        "                bmu=net[x,y]\n",
        "                bmu_idx=np.array([x,y])\n",
        "    return bmu, bmu_idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6i9SGIVd76ym"
      },
      "source": [
        "# SOM MAIN FUNCTION"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6N-Eaog84_fW"
      },
      "source": [
        "def SOM_COVID():\n",
        "    '''SETUP PARAMETERS'''\n",
        "    \n",
        "    #User input data\n",
        "    \n",
        "    country=input(\"What country do you want to analyze?\\n\")\n",
        "    print(search_country(country))\n",
        "    n=int(input(\"Which country database do you want to analyze?\\n\"))\n",
        "    \n",
        "    # Initialization Parameters & Benchmark\n",
        "    \n",
        "    start = timeit.default_timer()                                                      # Benchmark Constant \n",
        "    #\n",
        "    data=raw_data_manager(n)\n",
        "    iterations=1000         \n",
        "    learning_rate=0.1       \n",
        "    row=data.shape[0]       \n",
        "    columns=data.shape[1]   \n",
        "    # K is optimal number of neurons\n",
        "    K=int(5*math.sqrt(row)) \n",
        "    N=2                     \n",
        "    network_dim=np.array([K,N])\n",
        "    # Creating Neural Network 'net'\n",
        "    net=np.random.random((network_dim[0],network_dim[1],columns))\n",
        "    \n",
        "    init_radius=max(network_dim[0],network_dim[1])/2 \n",
        "    time_constant=iterations/np.log(init_radius)     \n",
        "\n",
        "    # TRAINING NEURAL NETWORK by Iterations\n",
        "\n",
        "    print(\"\\nProcessing COVID 19 databases...\")\n",
        "    for i in range(iterations):\n",
        "        t=data[np.random.randint(0,row),:]\n",
        "        r=neighborhood_size(init_radius,i,time_constant)\n",
        "        l=learning(learning_rate,i,iterations)\n",
        "        bmu,bmu_idx=find_bmu(t,net)\n",
        "        # Calculate Best Matching Unit\n",
        "        for x in range(net.shape[0]):\n",
        "            for y in range(net.shape[1]):\n",
        "                w=net[x,y].reshape(1,columns)\n",
        "                w_dist=eu_distance(np.array([[x,y]]),bmu_idx.reshape(1,2))\n",
        "                # Calculing weight distance for being used in neighborhood\n",
        "                if w_dist<=r:\n",
        "                    influence=topological_neighborhood(w_dist,r)\n",
        "                    new_w=w+(l*influence*(t.reshape(1,-1)-w))\n",
        "                    net[x,y]=new_w.reshape(1,3)\n",
        "    print(\"\\nCOVID 19 databases processed. Saving...\")\n",
        "    #OPERATIVE SISTEM\n",
        "    os.mkdir(str(country) + \" Processed Data\",0o777)\n",
        "    os.chdir(str(country) + \" Processed Data\")\n",
        "    for i in range(len(net)):\n",
        "        np.savetxt(str(country) + str(i) + str(\".txt\"),net[i])\n",
        "    os.chdir(\"../\")\n",
        "    #It Saves trained neurons\n",
        "    \n",
        "    #USEFULL DATA\n",
        "    end = timeit.default_timer()\n",
        "    print(\"\\nSOM\",end - start,\"seconds\")\n",
        "SOM_COVID()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}