Feature_extraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMuGbUoCK1bHAkvv0RnHSkw",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/t-willi/Simula/blob/main/Feature_extraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SZhL9CEfBDHw",
        "collapsed": true
      },
      "outputs": [],
      "source": [
        "!pip install wandb\n",
        "!pip install ecg_plot\n",
        "!pip install wfdb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rd4f-cYuBfkl"
      },
      "outputs": [],
      "source": [
        "import pathlib\n",
        "from pathlib import Path\n",
        "# current_position=pathlib.PurePath(__file__)\n",
        "# dir=current_position.parent\n",
        "# main_folder=dir.joinpath(\"main_folder\")\n",
        "current_position=pathlib.PurePath(\"/content/\")\n",
        "main_folder=current_position.joinpath(\"main_folder\")\n",
        "Path(main_folder).mkdir(parents=False,exist_ok = True)\n",
        "#create folder and directory for artifacts\n",
        "artifact_dir=main_folder.joinpath(\"artifacts\")\n",
        "Path(artifact_dir).mkdir(parents=False,exist_ok = True)\n",
        "#create folder and directory for train_data\n",
        "train_dir=main_folder.joinpath(\"train_dir\")\n",
        "Path(train_dir).mkdir(parents=False,exist_ok = True)\n",
        "#create folder and directory for ecg_files\n",
        "ecg_dir=main_folder.joinpath(\"ecg\")\n",
        "Path(ecg_dir).mkdir(parents=False,exist_ok = True)\n",
        "#create folder and directory for saved model sate dicts\n",
        "model_dir=main_folder.joinpath(\"model\")\n",
        "Path(model_dir).mkdir(parents=False,exist_ok = True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "19G_1YgVBm62"
      },
      "outputs": [],
      "source": [
        "import wandb\n",
        "import pandas as pd\n",
        "import torch\n",
        "from torch import nn\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import glob\n",
        "import torch.optim as optim\n",
        "from random import shuffle\n",
        "from tqdm.auto import tqdm\n",
        "import requests\n",
        "import zipfile\n",
        "from pathlib import Path\n",
        "\n",
        "if torch.cuda.is_available()==True:\n",
        "  device=\"cuda:0\"\n",
        "else:\n",
        "  device =\"cpu\"\n",
        "\n",
        "apikey = \"7a8ee9d41cc2d51eb77fd795e14f74a215e63c2d\"\n",
        "api = wandb.Api()\n",
        "artifact_syn = api.artifact('ecg_simula/setup_weights and biases/ecg_25000.zip:v0')\n",
        "artifact_normal = api.artifact('ecg_simula/upload_PTB_XL_normal.zip/PTB_normal:v0', type='dataset')\n",
        "artifact_syn.download()\n",
        "artifact_normal.download()\n",
        "pTOP_7_7_syn = api.artifact('ecg_simula/Feature_extraction_7to7_syn/Model:v4', type='Model')\n",
        "pTOP_7_7_norm = api.artifact('ecg_simula/Feature_extraction_7to7_PTB/Model:v6', type='Model')\n",
        "pTOP_1_7_syn = api.artifact('ecg_simula/AE_pTOP_serverrun_synthetic_data/Model:v6', type='Model')\n",
        "## pTOP_1_7_norm comes from training of pTOP on PTB_normal data with pretraining from ecg_simula/AE_pTOP_serverrun_synthetic_data/Model:v3\n",
        "pTOP_1_7_norm = api.artifact('ecg_simula/pTOP_PTB_normal/Model:v54', type='Model')\n",
        "## download model states \n",
        "pTOP_7_7_syn.download(\"/content/artifacts/pTOP_7_7_syn\")\n",
        "pTOP_7_7_norm.download(\"/content/artifacts/pTOP_7_7_norm\")\n",
        "pTOP_1_7_syn.download(\"/content/artifacts/pTOP_1_7_syn\")\n",
        "pTOP_1_7_norm.download(\"/content/artifacts/pTOP_1_7_norm\")\n",
        "\n",
        "torch.manual_seed(42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6_tEBvjPCWuA"
      },
      "outputs": [],
      "source": [
        "# previous_model = api.artifact('ecg_simula/AE_pTOP_LS_optim/Model:v8', type='Model')\n",
        "# previous_model.download()\n",
        "\n",
        "def request(path=None,name=None):\n",
        "  import requests\n",
        "  from pathlib import Path\n",
        "  request = requests.get(path)\n",
        "  name=name+\".py\"\n",
        "  with open(name,\"wb\") as f:\n",
        "    f.write(request.content)\n",
        "\n",
        "unzip_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/unzip.py\"\n",
        "get_pred_no_reshape_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/get_predictions_no_reshape.py\"\n",
        "get_pred_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/get_predictions.py\"\n",
        "plt_ECG_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/plot_ECG.py\"\n",
        "Dataloader_syn_1to7_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/dataloader_syn_1to7.py\"\n",
        "Dataloader_syn_7to7_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/dataloader_syn_7to7.py\"\n",
        "Dataloader_normal_1to7_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/dataloader_PTB_normal_1to7.py\"\n",
        "Dataloader_normal_7to7_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/dataloader_PTB_normal_7to7.py\"\n",
        "pTOP_git_dir=\"https://raw.githubusercontent.com/t-willi/NeuralNetworks/main/pTOP_model.py\"\n",
        "\n",
        "request(unzip_git_dir,\"Unzip\")\n",
        "from Unzip import unzip\n",
        "\n",
        "#request(pTOP_git_dir,\"pTOP\")\n",
        "#from pTOP import *\n",
        "request(Dataloader_syn_1to7_git_dir,\"dataset_and_loader\")\n",
        "#from dataset_and_loader import Custom_dataset as CD\n",
        "from dataset_and_loader import make_loader as ml\n",
        "request(Dataloader_syn_1to7_git_dir,\"syn1_7\")\n",
        "from syn1_7 import Custom_dataset as CD_syn_1_7\n",
        "request(Dataloader_syn_7to7_git_dir,\"syn7_7\")\n",
        "from syn7_7 import Custom_dataset as CD_syn_7_7\n",
        "request(Dataloader_normal_1to7_git_dir,\"normal1_7\")\n",
        "from normal1_7 import Custom_dataset_PTB as CD_normal_1_7\n",
        "request(Dataloader_normal_7to7_git_dir,\"normal7_7\")\n",
        "from normal7_7 import Custom_dataset_PTB as CD_normal_7_7\n",
        "\n",
        "\n",
        "#download prediction generator\n",
        "request(get_pred_no_reshape_git_dir,\"get_predictions\")\n",
        "from get_predictions import get_pred\n",
        "#download ECG plotter\n",
        "request(plt_ECG_git_dir,\"plot_ECG\")\n",
        "from plot_ECG import plotECG\n",
        "\n",
        "unzip(save_path=\"custom_data/syn_train\",zip_path=\"/content/artifacts/ecg_25000.zip:v0/ecg_25000.zip\",reload=True)\n",
        "unzip(save_path=\"custom_data/norm_train\",zip_path=\"/content/artifacts/PTB_normal:v0/PTB_normal.zip\",reload=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b1lg4MJPC6ig"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.utils.data\n",
        "\n",
        "class Transpose1dLayer(nn.Module):\n",
        "    def __init__(self, in_channels, out_channels, kernel_size, stride, padding=11, upsample=None, output_padding=1):\n",
        "        super(Transpose1dLayer, self).__init__()\n",
        "        self.upsample = upsample\n",
        "        self.upsample_layer = torch.nn.Upsample(scale_factor=upsample)\n",
        "        reflection_pad = kernel_size // 2\n",
        "        self.reflection_pad = nn.ConstantPad1d(reflection_pad, value=0)\n",
        "        self.conv1d = torch.nn.Conv1d(in_channels, out_channels, kernel_size, stride)\n",
        "        self.Conv1dTrans = nn.ConvTranspose1d(in_channels, out_channels, kernel_size, stride, padding, output_padding)\n",
        "\n",
        "    def forward(self, x):\n",
        "        if self.upsample:\n",
        "            #x = torch.cat((x, in_feature), 1)\n",
        "            return self.conv1d(self.reflection_pad(self.upsample_layer(x)))\n",
        "        else:\n",
        "            return self.Conv1dTrans(x)\n",
        "\n",
        "\n",
        "class pTOP_7to7(nn.Module):\n",
        "    def __init__(self,latent_dim=100, post_proc_filt_len=512,upsample=True):\n",
        "        super(pTOP_7to7, self).__init__()\n",
        "        # \"Dense\" is the same meaning as fully connection.\n",
        "        stride = 4\n",
        "        if upsample:\n",
        "            stride = 1\n",
        "            upsample = 5\n",
        "        # if upsample is anything but none Transpose1dLayer will do\n",
        "        # self.conv1d(self.reflection_pad(self.upsample_layer(x)))\n",
        "        # which is a 1d convolution on padded and upsampled data x\n",
        "        self.deconv_1 = Transpose1dLayer(250 , 250, 25, stride, upsample=upsample)\n",
        "        self.deconv_2 = Transpose1dLayer(250, 150, 25, stride, upsample=upsample)\n",
        "        self.deconv_3 = Transpose1dLayer(150, 50, 25, stride, upsample=upsample)\n",
        "        self.deconv_4 = Transpose1dLayer( 50, 25, 25, stride, upsample=2)\n",
        "        self.deconv_5 = Transpose1dLayer( 25, 10, 25, stride, upsample=upsample)\n",
        "        self.deconv_6 = Transpose1dLayer(  10, 7, 25, stride, upsample=2)\n",
        "\n",
        "\n",
        "        #new convolutional layers\n",
        "        self.conv_1 = nn.Conv1d(7, 10, 25, stride=2, padding=25 // 2)\n",
        "        self.conv_2 = nn.Conv1d(10, 25, 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_3 = nn.Conv1d(25, 50 , 25, stride=2, padding= 25 // 2)\n",
        "        self.conv_4 = nn.Conv1d(50, 150 , 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_5 = nn.Conv1d(150, 250 , 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_6 = nn.Conv1d(250, 250 , 25, stride=5, padding= 25 // 2)\n",
        "        self.flatt = nn.Flatten()\n",
        "        self.linear1 = nn.Linear(500,100)\n",
        "        self.linear2 = nn.Linear(100,500)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.ConvTranspose1d) or isinstance(m, nn.Linear):\n",
        "                nn.init.kaiming_normal_(m.weight.data)\n",
        "\n",
        "    def forward(self, x, LS=False):\n",
        "        self.LS=LS\n",
        "        if x.ndim==2:\n",
        "          x=x.unsqueeze(0)\n",
        "        x = F.leaky_relu(self.conv_1(x)) #(1,7,5000 --> 1, 10, 2500)\n",
        "        x = F.leaky_relu(self.conv_2(x)) #( --> 1, 25, 500)\n",
        "        x = F.leaky_relu(self.conv_3(x)) #(--> 1, 50, 250)\n",
        "        x = F.leaky_relu(self.conv_4(x)) # --> 1, 150, 50)\n",
        "        x = F.leaky_relu(self.conv_5(x)) #(--> 1, 250, 10)\n",
        "        x = F.leaky_relu(self.conv_6(x)) #(--> 1, 250, 2)-->flatten into (1,500)), then to linear ((1,100)), and then back\n",
        "        x = self.flatt(x) # (1,500)\n",
        "        LS = self.linear1(x) #(1,100)\n",
        "        if self.LS is True:\n",
        "          return LS\n",
        "        x = self.linear2(LS) #(1,500)\n",
        "        zero_dim=x.shape[0]\n",
        "        x=torch.reshape(x,(zero_dim,250,2)) #1(1,250,2)\n",
        "        x = F.relu(self.deconv_1(x)) #(--> 1, 250, 10)\n",
        "        x = F.relu(self.deconv_2(x)) #(--> 1, 150, 50)\n",
        "        x = F.relu(self.deconv_3(x)) #( --> 1, 50, 250)\n",
        "        x = F.relu(self.deconv_4(x)) #(--> 1, 25, 500)\n",
        "        x = F.relu(self.deconv_5(x)) #(--> 1, 10, 2500)\n",
        "        x = torch.tanh(self.deconv_6(x)) #(1, 7, 5000)\n",
        "        x=x.squeeze()\n",
        "        return x\n",
        "\n",
        "model_7to7_syn=pTOP_7to7().to(device)\n",
        "model_7to7_norm=pTOP_7to7().to(device)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.utils.data\n",
        "\n",
        "class Transpose1dLayer(nn.Module):\n",
        "    def __init__(self, in_channels, out_channels, kernel_size, stride, padding=11, upsample=None, output_padding=1):\n",
        "        super(Transpose1dLayer, self).__init__()\n",
        "        self.upsample = upsample\n",
        "        self.upsample_layer = torch.nn.Upsample(scale_factor=upsample)\n",
        "        reflection_pad = kernel_size // 2\n",
        "        self.reflection_pad = nn.ConstantPad1d(reflection_pad, value=0)\n",
        "        self.conv1d = torch.nn.Conv1d(in_channels, out_channels, kernel_size, stride)\n",
        "        self.Conv1dTrans = nn.ConvTranspose1d(in_channels, out_channels, kernel_size, stride, padding, output_padding)\n",
        "\n",
        "    def forward(self, x):\n",
        "        if self.upsample:\n",
        "            #x = torch.cat((x, in_feature), 1)\n",
        "            return self.conv1d(self.reflection_pad(self.upsample_layer(x)))\n",
        "        else:\n",
        "            return self.Conv1dTrans(x)\n",
        "\n",
        "\n",
        "class pTOP_1to7(nn.Module):\n",
        "    def __init__(self,latent_dim=100, post_proc_filt_len=512,upsample=True):\n",
        "        super(pTOP_1to7, self).__init__()\n",
        "        # \"Dense\" is the same meaning as fully connection.\n",
        "        stride = 4\n",
        "        if upsample:\n",
        "            stride = 1\n",
        "            upsample = 5\n",
        "        # if upsample is anything but none Transpose1dLayer will do\n",
        "        # self.conv1d(self.reflection_pad(self.upsample_layer(x)))\n",
        "        # which is a 1d convolution on padded and upsampled data x\n",
        "        self.deconv_1 = Transpose1dLayer(250 , 250, 25, stride, upsample=upsample)\n",
        "        self.deconv_2 = Transpose1dLayer(250, 150, 25, stride, upsample=upsample)\n",
        "        self.deconv_3 = Transpose1dLayer(150, 50, 25, stride, upsample=upsample)\n",
        "        self.deconv_4 = Transpose1dLayer( 50, 25, 25, stride, upsample=2)\n",
        "        self.deconv_5 = Transpose1dLayer( 25, 10, 25, stride, upsample=upsample)\n",
        "        self.deconv_6 = Transpose1dLayer(  10, 7, 25, stride, upsample=2)\n",
        "\n",
        "\n",
        "        #new convolutional layers\n",
        "        self.conv_1 = nn.Conv1d(1, 10, 25, stride=2, padding=25 // 2)\n",
        "        self.conv_2 = nn.Conv1d(10, 25, 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_3 = nn.Conv1d(25, 50 , 25, stride=2, padding= 25 // 2)\n",
        "        self.conv_4 = nn.Conv1d(50, 150 , 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_5 = nn.Conv1d(150, 250 , 25, stride=5, padding= 25 // 2)\n",
        "        self.conv_6 = nn.Conv1d(250, 250 , 25, stride=5, padding= 25 // 2)\n",
        "        self.flatt = nn.Flatten()\n",
        "        self.linear1 = nn.Linear(500,100)\n",
        "        self.linear2 = nn.Linear(100,500)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.ConvTranspose1d) or isinstance(m, nn.Linear):\n",
        "                nn.init.kaiming_normal_(m.weight.data)\n",
        "\n",
        "    def forward(self, x, LS=False):\n",
        "        self.LS=LS\n",
        "        if x.ndim==2:\n",
        "          x=x.unsqueeze(0)\n",
        "        x = F.leaky_relu(self.conv_1(x)) #(1,1,5000 --> 1, 10, 2500)\n",
        "        x = F.leaky_relu(self.conv_2(x)) #( --> 1, 25, 500)\n",
        "        x = F.leaky_relu(self.conv_3(x)) #(--> 1, 50, 250)\n",
        "        x = F.leaky_relu(self.conv_4(x)) # --> 1, 150, 50)\n",
        "        x = F.leaky_relu(self.conv_5(x)) #(--> 1, 250, 10)\n",
        "        x = F.leaky_relu(self.conv_6(x)) #(--> 1, 250, 2)-->flatten into (1,500)), then to linear ((1,100)), and then back\n",
        "        x = self.flatt(x) # (1,500)\n",
        "        LS = self.linear1(x) #(1,100)\n",
        "        if self.LS is True:\n",
        "          return LS\n",
        "        x = self.linear2(LS) #(1,500)\n",
        "        zero_dim=x.shape[0]\n",
        "        x=torch.reshape(x,(zero_dim,250,2)) #1(1,250,2)\n",
        "        x = F.relu(self.deconv_1(x)) #(--> 1, 250, 10)\n",
        "        x = F.relu(self.deconv_2(x)) #(--> 1, 150, 50)\n",
        "        x = F.relu(self.deconv_3(x)) #( --> 1, 50, 250)\n",
        "        x = F.relu(self.deconv_4(x)) #(--> 1, 25, 500)\n",
        "        x = F.relu(self.deconv_5(x)) #(--> 1, 10, 2500)\n",
        "        x = torch.tanh(self.deconv_6(x)) #(1, 7, 5000)\n",
        "        x=x.squeeze()\n",
        "        return x\n",
        "\n",
        "model_1to7_syn=pTOP_1to7().to(device)\n",
        "model_1to7_norm=pTOP_1to7().to(device)\n",
        "# PATH=\"/content/artifacts/Model:v3/model\"\n",
        "# model.load_state_dict(torch.load(PATH))"
      ],
      "metadata": {
        "id": "NMLiWd3pWgh9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create data to run trough the pTOP7to7 for feature extraction. A sample from syn data will be run trough trained pTOP1to7 models, the 7 lead output will be run trough a pTOP7to7 model and its latent space compared to the latent space of the real 7 leads.\n"
      ],
      "metadata": {
        "id": "wyOtGwE0Uf5Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## get datasets\n",
        "Syn_data_dir=\"/content/custom_data/syn_train\"\n",
        "Norm_data_dir=\"/content/custom_data/norm_train\"\n",
        "test_data_1to7_syn=CD_syn_1_7(Syn_data_dir,target=\"test\")\n",
        "test_data_1to7_norm=CD_normal_1_7(Norm_data_dir,target=\"test\")\n",
        "test_data_7to7_syn=CD_syn_7_7(Syn_data_dir,target=\"test\")\n",
        "test_data_7to7_norm=CD_normal_7_7(Norm_data_dir,target=\"test\")"
      ],
      "metadata": {
        "id": "KpHKwdX-UfN-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(test_data_7to7_norm)"
      ],
      "metadata": {
        "id": "n4N0rsvRX6HX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#set up 1to7 models for prediction\n",
        "syn1_7_path=\"/content/artifacts/pTOP_1_7_syn/model\"\n",
        "model_1to7_syn.load_state_dict(torch.load(syn1_7_path, map_location=device))\n",
        "norm1_7_path=\"/content/artifacts/pTOP_1_7_norm/model\"\n",
        "model_1to7_norm.load_state_dict(torch.load(norm1_7_path, map_location=device))\n",
        "\n",
        "#set up 7to7 models\n",
        "syn7_7_path=\"/content/artifacts/pTOP_7_7_syn/model\"\n",
        "model_7to7_syn.load_state_dict(torch.load(syn7_7_path, map_location=device))\n",
        "norm7_7_path=\"/content/artifacts/pTOP_7_7_norm/model\"\n",
        "model_7to7_norm.load_state_dict(torch.load(norm7_7_path, map_location=device))"
      ],
      "metadata": {
        "id": "AfFFQxlGWEix"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "\n",
        "def get_pred_12lead(dataset=None,Set=None,model=None,random=True,upscale=None):\n",
        "  \"\"\"\n",
        "  Function takes a Tensor Dataset as input,first a random file from the dataset is selected,\n",
        "  then the Tensor pair is recombined and shaped into a df-->df_Input. \n",
        "  X is used afterwards as input into the model. The predictions are safed as --> df_output.\n",
        "  Both dataframes are now unscaled by 5011, the max value of the whole dataset.\n",
        "  Then a tuple pair of input and output is returned.\n",
        "  \"\"\"\n",
        "  if random:\n",
        "    import random\n",
        "    limit=len(dataset)\n",
        "    rand_idx=random.randint(0,limit)\n",
        "    X,y=dataset[rand_idx]\n",
        "  if random is False:\n",
        "      X,y=Set\n",
        "  #need to combine tensors to make dataframe for plotting input and output side by side\n",
        "  full_tensor=torch.cat((X,y.squeeze()))\n",
        "  full_tensor=full_tensor.numpy()\n",
        "  df_input=pd.DataFrame(full_tensor).T\n",
        "  df_input.columns = [\"real_I\",\"real_II\",\"real_v1\",\"real_v2\",\"real_v3\",\"real_v4\",\"real_v5\",\"real_v6\"]\n",
        "  model.to(\"cpu\")\n",
        "  model.eval()\n",
        "  with torch.inference_mode():\n",
        "    output=model(X)\n",
        "  output=output.detach().numpy()\n",
        "  output=output.squeeze().T\n",
        "  #unscale data\n",
        "  df_output = pd.DataFrame(output,columns=[\"pred_II\",\"pred_v1\",\"pred_v2\",\"pred_v3\",\"pred_v4\",\"pred_v5\",\"pred_v6\"])\n",
        "  #calculate the 4 missing leads\n",
        "  df_input.insert(2, \"real_III\", df_input[\"real_II\"] - df_input[\"real_I\"])\n",
        "  df_input.insert(3,\"real_aVR\",0.5*(df_input[\"real_I\"] + df_input[\"real_II\"]))\n",
        "  df_input.insert(4,\"real_aVL\",(df_input[\"real_I\"] -0.5) * df_input[\"real_II\"])\n",
        "  df_input.insert(5,\"real_aVF\",(df_input[\"real_II\"] -0.5) * df_input[\"real_I\"])\n",
        "  df_output.insert(0,\"real_I\",df_input[\"real_I\"])\n",
        "  df_output.insert(2,\"real_III\",df_output[\"pred_II\"] - df_output[\"real_I\"])\n",
        "  df_output.insert(3,\"real_aVR\",0.5*(df_output[\"real_I\"] + df_output[\"pred_II\"]))\n",
        "  df_output.insert(4,\"real_aVL\",(df_output[\"real_I\"] -0.5) * df_output[\"pred_II\"])\n",
        "  df_output.insert(5,\"real_aVF\",(df_output[\"pred_II\"] -0.5) * df_output[\"real_I\"])\n",
        "  if upscale:\n",
        "    df_input=df_input*upscale\n",
        "    df_output=df_output*upscale\n",
        "    \n",
        "  return df_input,df_output\n",
        "\n",
        "# lead III value = (lead II value) - (lead I value)\n",
        "# lead aVR value = -0.5*(lead I value + lead II value)\n",
        "# lead aVL value = lead I value - 0.5 * lead II value\n",
        "# lead aVF value = lead II value - 0.5 * lead I value\n"
      ],
      "metadata": {
        "id": "DUrAOCtvwC2W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "def plotECG_12Lead(df1=None,df2=None,title=None,pad_df2=True,path=None,createECG=True,scale=None):\n",
        "  \"\"\"\n",
        "  takes two dataframes with identical columns, concats them and plots them as ecg using ecg_plot\n",
        "  it also takes the first column of df1 and ads it to df1 if pad_df2 is True\n",
        "  \"\"\"\n",
        "  import ecg_plot\n",
        "  index=[\"real_I\",\"real_II\",\"real_III\",\"real_aVR\",\"real_aVL\",\"real_aVF\",\"real_v1\",\"real_v2\",\"real_v3\",\"real_v4\",\"real_v5\",\"real_v6\",\"real_I\",\n",
        "         \"pred_II\",\"pred_III\",\"pred_aVR\",\"pred_aVL\",\"pred_aVF\",\"pred_v1\",\"pred_v2\",\"pred_v3\",\"pred_v4\",\"pred_v5\",\"pred_v6\"]\n",
        "  if createECG==True:\n",
        "    ecg_path=path\n",
        "    if Path(ecg_path).is_dir():\n",
        "        print(f\"{ecg_path} directory exists.\")\n",
        "    else:\n",
        "        print(f\"Did not find {ecg_path} directory, creating one...\")\n",
        "        Path(ecg_path).mkdir(parents=True, exist_ok=False)\n",
        "  if scale:\n",
        "    frames=[df1/1000,df2/1000]\n",
        "  if scale is None:\n",
        "    frames=[df1,df2]\n",
        "  combined_df=pd.concat(frames,axis=1,join=\"outer\",)\n",
        "  if createECG is True:\n",
        "    ecg_plot.plot(combined_df.values.T, sample_rate = 500,title = title,\n",
        "                      lead_index = index )\n",
        "    ecg_plot.save_as_png('ecg',ecg_path+\"/\")\n",
        "  return combined_df\n",
        "\n",
        "  "
      ],
      "metadata": {
        "id": "Chb-P9hL2sJK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#get prediction from dataset for synthetic data, this is for testing purposes to see that the ecg looks right.\n",
        "syn_df_in_1to7,syn_df_out_1to7=get_pred(dataset=test_data_1to7_syn,model=model_1to7_syn,upscale=5011)\n",
        "combined_df=plotECG(syn_df_in_1to7,syn_df_out_1to7,path=str(ecg_dir),scale=True)\n"
      ],
      "metadata": {
        "id": "QUT4Kd-hV-ER"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# syn_df_in_1to7=syn_df_in_1to7/1000\n",
        "# import ecg_plot\n",
        "# ecg_plot.plot(syn_df_in_1to7.values.T, sample_rate = 500)\n",
        "# import cv2 as cv\n",
        "# from PIL import Image\n",
        "# def png_to_jpg(path=None):\n",
        "#   im = cv.imread(\"/content/main_folder/ecg/ecg.png\")\n",
        "#   im = Image.fromarray(im)\n",
        "#   im.save(\"/content/main_folder/ecg/ecg.pdf\")\n",
        "# png_to_jpg()"
      ],
      "metadata": {
        "id": "vEt2GDH3yORy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#get prediction from dataset for PTB normal data, this is for testing purposes to see that the ecg looks right\n",
        "norm_df_in_1to7,norm_df_out_1to7=get_pred(dataset=test_data_1to7_norm,model=model_1to7_norm,upscale=33)\n",
        "combined_df=plotECG(norm_df_in_1to7,norm_df_out_1to7,path=str(ecg_dir),scale=None)\n"
      ],
      "metadata": {
        "id": "Y7Pxx7KLZiBo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#get ecgs from 20 random sets\n",
        "lenght=len(test_data_1to7_norm)\n",
        "range=list(np.arange(lenght))\n",
        "import random\n",
        "indexes = random.sample(range,21)\n",
        "for k,idx in enumerate(indexes):\n",
        "  x,y=test_data_1to7_norm[idx]\n",
        "  Set=(x,y)\n",
        "  df_in,df_out=get_pred(model=model_1to7_norm,Set=Set,random=False,upscale=33)\n",
        "  combined_df=plotECG(df_in,df_out,title=f\"ecg{k+1}\",scale=None,path=\"/content/ecg_files\")"
      ],
      "metadata": {
        "id": "1pctefXtZwTi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#get prediction from dataset for synthetic data, this is for testing purposes to see that the ecg looks right.\n",
        "syn_df_in_7to7,syn_df_out_7to7=get_pred_FE(dataset=test_data_7to7_syn,model=model_7to7_syn,upscale=5011)\n",
        "combined_df=plotECG_FE(syn_df_in_7to7,syn_df_out_7to7,path=str(ecg_dir),scale=True)"
      ],
      "metadata": {
        "id": "8Bx2gR4sdv77"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#get prediction from dataset for synthetic data, this is for testing purposes to see that the ecg looks right.\n",
        "norm_df_in_7to7,norm_df_out_7to7=get_pred_FE(dataset=test_data_7to7_norm,model=model_7to7_norm,upscale=33)\n",
        "combined_df=plotECG_FE(norm_df_in_7to7,norm_df_out_7to7,path=str(ecg_dir))"
      ],
      "metadata": {
        "id": "EpL6yfkofK82"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## getting the latent spaces for real and predicted data\n",
        "# first for synthetic data\n",
        "#first run real data trough the 7t7syn model adn extract latent spaces, \n",
        "#then run predicted data trough 7to7syn model and extract latent spaces,\n",
        "#then compare\n",
        "## look at latent space\n",
        "### get latent spaces for real data\n",
        "test_loader_7to7_syn=ml(test_data_7to7_syn,32)\n",
        "Latent_spaces_real = torch.empty((1,100), dtype=torch.float32)\n",
        "for x in tqdm(test_loader_7to7_syn):\n",
        "  model_7to7_syn.to(device)\n",
        "  model_7to7_syn.eval()\n",
        "  with torch.inference_mode():\n",
        "    x=x.to(device)\n",
        "    output=model_7to7_syn(x,LS=True)\n",
        "    Latent_spaces_real = torch.cat((Latent_spaces_real, output.detach().cpu()), 0)\n",
        "# Latent_spaces=Latent_spaces.numpy()\n",
        "Latent_spaces_real.shape"
      ],
      "metadata": {
        "id": "ASZPjel3bFXF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# get latent spaces for predicted data\n",
        "test_loader_1to7_syn=ml(test_data_1to7_syn,32)\n",
        "Latent_spaces_predicted = torch.empty((1,100), dtype=torch.float32)\n",
        "for (x,y) in tqdm(test_loader_1to7_syn):\n",
        "  model_1to7_syn.to(device)\n",
        "  model_7to7_syn.to(device)\n",
        "  model_1to7_syn.eval()\n",
        "  model_7to7_syn.eval()\n",
        "  with torch.inference_mode():\n",
        "    x=x.to(device)\n",
        "    output=model_1to7_syn(x)\n",
        "    output=model_7to7_syn(output,LS=True)\n",
        "    Latent_spaces_predicted = torch.cat((Latent_spaces_predicted, output.detach().cpu()), 0)\n",
        "# Latent_spaces=Latent_spaces.numpy()\n",
        "Latent_spaces_predicted.shape"
      ],
      "metadata": {
        "id": "b26yKd_vhXRy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install umap-learn\n",
        "!pip install umap-learn[plot]"
      ],
      "metadata": {
        "id": "oqmvbCRgkj4R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Latent_spaces_predicted.shape"
      ],
      "metadata": {
        "id": "8ThXcAgCoUie"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "##use UMAP to compare latent spaces\n",
        "## using UMAP\n",
        "import umap.umap_ as umap\n",
        "from sklearn.datasets import load_digits\n",
        "embedding_real = umap.UMAP(n_neighbors=7,\n",
        "                      min_dist=0.3,\n",
        "                      n_components=2,\n",
        "                      random_state=42,\n",
        "                      metric='correlation').fit(Latent_spaces_real.data)\n",
        "\n",
        "embedding_predicted = umap.UMAP(n_neighbors=7,\n",
        "                      min_dist=0.3,\n",
        "                      n_components=2,\n",
        "                      random_state=42,\n",
        "                      metric='correlation').fit(Latent_spaces_predicted.data)\n"
      ],
      "metadata": {
        "id": "Kg_iH8qskVwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.DataFrame(embedding_real)"
      ],
      "metadata": {
        "id": "55MmsfU7tSKP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import umap.plot\n",
        "umap.plot.points(embedding_real)"
      ],
      "metadata": {
        "id": "xklLq3IGkhcF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import umap.plot\n",
        "umap.plot.points(embedding_predicted)"
      ],
      "metadata": {
        "id": "sWWURJjtqd_f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "\n",
        "def get_pred_FE(dataset=None,Set=None,model=None,random=True,upscale=None):\n",
        "  \"\"\"\n",
        "  Function takes a Tensor Dataset as input,first a random file from the dataset is selected,\n",
        "  then the Tensor pair is recombined and shaped into a df-->df_Input. \n",
        "  X is used afterwards as input into the model. The predictions are safed as --> df_output.\n",
        "  Both dataframes are now unscaled by 5011, the max value of the whole dataset.\n",
        "  Then a tuple pair of input and output is returned.\n",
        "  \"\"\"\n",
        "  if random:\n",
        "    import random\n",
        "    limit=len(dataset)\n",
        "    rand_idx=random.randint(0,limit)\n",
        "    X=dataset[rand_idx]\n",
        "  if random is False:\n",
        "      X=Set\n",
        "  #need to combine tensors to make dataframe for plotting input and output side by side\n",
        "  # full_tensor=torch.cat((X,y.squeeze()))\n",
        "  # full_tensor=full_tensor.numpy()\n",
        "  input=X.detach().numpy()\n",
        "  input=input.squeeze().T\n",
        "  df_input=pd.DataFrame(input,columns = [\"real_II\",\"real_v1\",\"real_v2\",\"real_v3\",\"real_v4\",\"real_v5\",\"real_v6\"])\n",
        "  model.to(\"cpu\")\n",
        "  model.eval()\n",
        "  with torch.inference_mode():\n",
        "    output=model(X)\n",
        "  output=output.detach().numpy()\n",
        "  output=output.squeeze().T\n",
        "  #unscale data\n",
        "  df_output = pd.DataFrame(output,columns=[\"pred_II\",\"pred_v1\",\"pred_v2\",\"pred_v3\",\"pred_v4\",\"pred_v5\",\"pred_v6\"])\n",
        "  if upscale:\n",
        "    df_input=df_input*upscale\n",
        "    df_output=df_output*upscale\n",
        "  return df_input,df_output"
      ],
      "metadata": {
        "id": "Rm0NQo98yNBE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "def plotECG_FE(df1=None,df2=None,title=None,pad_df2=True,path=None,createECG=True,scale=None):\n",
        "  \"\"\"\n",
        "  takes two dataframes with identical columns, concats them and plots them as ecg using ecg_plot\n",
        "  it also takes the first column of df1 and ads it to df1 if pad_df2 is True\n",
        "  \"\"\"\n",
        "  index=[\"real_II\",\"real_v1\",\"real_v2\",\"real_v3\",\"real_v4\",\"real_v5\",\"real_v6\",\n",
        "         \"pred_II\",\"pred_v1\",\"pred_v2\",\"pred_v3\",\"pred_v4\",\"pred_v5\",\"pred_v6\"]\n",
        "  if createECG==True:\n",
        "    ecg_path=path\n",
        "    if Path(ecg_path).is_dir():\n",
        "        print(f\"{ecg_path} directory exists.\")\n",
        "    else:\n",
        "        print(f\"Did not find {ecg_path} directory, creating one...\")\n",
        "        Path(ecg_path).mkdir(parents=True, exist_ok=True)\n",
        "    import ecg_plot\n",
        "  if scale:\n",
        "    frames=[df1/1000,df2/1000]\n",
        "  if scale is None:\n",
        "    frames=[df1,df2]\n",
        "  combined_df=pd.concat(frames,axis=1,join=\"outer\",)\n",
        "  if createECG is True:\n",
        "    ecg_plot.plot(combined_df.values.T, sample_rate = 500,title = title,\n",
        "                      lead_index = index )\n",
        "    ecg_plot.save_as_png('ecg',ecg_path+\"/\")\n",
        "  return combined_df\n",
        "\n",
        "  "
      ],
      "metadata": {
        "id": "eJIltqUYzJ-L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# dataset=Custom_dataset(\"/content/main_folder/train_dir\")\n",
        "# df_in,df_out=get_pred_FE(model=model,dataset=dataset,upscale=5011)\n"
      ],
      "metadata": {
        "id": "PFNGc7UK05hi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# combined_df=plotECG_FE(df_in,df_out,scale=True,path=str(ecg_dir))"
      ],
      "metadata": {
        "id": "rsr_30sO1tou"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o0lGHAC7Qpk-"
      },
      "outputs": [],
      "source": []
    }
  ]
}