RELATHEDataExperiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.io import loadmat\n",
    "from scipy.sparse import issparse\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from Data import VFLDataset\n",
    "from torch.utils.data import DataLoader\n",
    "import VFL\n",
    "import torch\n",
    "import os\n",
    "DIR = \"Data\"\n",
    "file_name = 'BASEHOCK.mat'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BASEHOCK.mat (1993, 4862) (1993,)\n",
      "Client 0: Feature Index 0-1620\n",
      "Client 1: Feature Index 1621-3241\n",
      "Server : Feature Index 3242-4861\n"
     ]
    }
   ],
   "source": [
    "mat = loadmat(os.path.join(DIR, file_name))\n",
    "X = mat[\"X\"]\n",
    "y = mat[\"Y\"]\n",
    "if issparse(X):\n",
    "    X = X.todense()\n",
    "y = y.flatten()\n",
    "print(file_name, X.shape, y.shape)\n",
    "y[np.where(y == 1)] = 0\n",
    "y[np.where(y == 2)] = 1\n",
    "scaler = MinMaxScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "dataset = VFLDataset(data_source=(X, y), \n",
    "                    num_clients=2,\n",
    "                    gini_portion=None,\n",
    "                    insert_noise=False,\n",
    "                    test_size=0.2)\n",
    "train_loader = DataLoader(dataset.train(), batch_size=256, shuffle=False)\n",
    "val_loader = DataLoader(dataset.valid(), batch_size=1000, shuffle=False)\n",
    "test_loader = DataLoader(dataset.test(), batch_size=1000, shuffle=False)\n",
    "input_dim_list = dataset.get_input_dim_list()\n",
    "output_dim = np.unique(y).size\n",
    "criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "models, top_model = VFL.make_binary_models(\n",
    "                            input_dim_list=input_dim_list,\n",
    "                            type='FNN',\n",
    "                            emb_dim=4,\n",
    "                            output_dim=output_dim, hidden_dims=[8],\n",
    "                            activation='relu')\n",
    "fnn_history = VFL.train(models, top_model, train_loader, val_loader, test_loader,\n",
    "                epochs=30,  optimizer='Adam',verbose=True, save_mask_at=10000, \n",
    "                criterion=criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.932573</td>\n",
       "      <td>0.936148</td>\n",
       "      <td>0.942356</td>\n",
       "      <td>0.942356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.893441</td>\n",
       "      <td>0.939644</td>\n",
       "      <td>0.957393</td>\n",
       "      <td>0.957393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.842558</td>\n",
       "      <td>0.952906</td>\n",
       "      <td>0.957393</td>\n",
       "      <td>0.957393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.790051</td>\n",
       "      <td>0.962576</td>\n",
       "      <td>0.967419</td>\n",
       "      <td>0.967419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.736935</td>\n",
       "      <td>0.968436</td>\n",
       "      <td>0.969925</td>\n",
       "      <td>0.969925</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc\n",
       "25    0.932573   0.936148  0.942356  0.942356\n",
       "26    0.893441   0.939644  0.957393  0.957393\n",
       "27    0.842558   0.952906  0.957393  0.957393\n",
       "28    0.790051   0.962576  0.967419  0.967419\n",
       "29    0.736935   0.968436  0.969925  0.969925"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnn_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "stg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "models, top_model = VFL.make_binary_models(\n",
    "                            input_dim_list=input_dim_list,\n",
    "                            type='STG',\n",
    "                            emb_dim=4,\n",
    "                            output_dim=output_dim, hidden_dims=[8],\n",
    "                            activation='relu', lam=0.1)\n",
    "stg_history = VFL.train(models, top_model, train_loader, val_loader, test_loader,\n",
    "                epochs=30,  optimizer='Adam',verbose=True, save_mask_at=10000, \n",
    "                criterion=criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>num_feats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.120966</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.119726</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.120107</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>3612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.123952</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.123778</td>\n",
       "      <td>0.998047</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>3433</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc  num_feats\n",
       "25    0.120966   1.000000  1.000000  1.000000       3852\n",
       "26    0.119726   1.000000  1.000000  1.000000       3714\n",
       "27    0.120107   1.000000  0.997494  0.997494       3612\n",
       "28    0.123952   1.000000  1.000000  1.000000       3517\n",
       "29    0.123778   0.998047  1.000000  0.997494       3433"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stg_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STG with GINI Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "gini_labels = dataset.gini_filter(0.5)\n",
    "feat_idx_list = dataset.get_feature_index_list()\n",
    "mus = VFL.initialize_mu(gini_labels, feat_idx_list)\n",
    "models, top_model = VFL.make_binary_models(\n",
    "                            input_dim_list=input_dim_list,\n",
    "                            type='STG',\n",
    "                            emb_dim=4,\n",
    "                            output_dim=output_dim, hidden_dims=[8],\n",
    "                            activation='relu', lam=0.1, mus=mus)\n",
    "stg_gini_history = VFL.train(models, top_model, train_loader, val_loader, test_loader,\n",
    "                epochs=30,  optimizer='Adam',verbose=True, save_mask_at=10000, \n",
    "                criterion=criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>num_feats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.138072</td>\n",
       "      <td>0.996503</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.121633</td>\n",
       "      <td>0.998047</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.118509</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>2995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.118073</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>2970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.127384</td>\n",
       "      <td>0.996094</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc  num_feats\n",
       "25    0.138072   0.996503  0.997494  1.000000       3042\n",
       "26    0.121633   0.998047  1.000000  1.000000       3020\n",
       "27    0.118509   1.000000  0.997494  0.997494       2995\n",
       "28    0.118073   1.000000  1.000000  0.997494       2970\n",
       "29    0.127384   0.996094  1.000000  1.000000       2952"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stg_gini_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dual STG with GINI Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "mus = VFL.initialize_mu(gini_labels, feat_idx_list)\n",
    "models, top_model = VFL.make_binary_models(\n",
    "    input_dim_list=input_dim_list,\n",
    "    type=\"DualSTG\",\n",
    "    emb_dim=4,\n",
    "    output_dim=output_dim,\n",
    "    hidden_dims=[8],\n",
    "    activation=\"relu\",\n",
    "    mus=mus, top_lam=0.1, lam=0.1)\n",
    "dual_stg_gini_history = VFL.train(\n",
    "    models,\n",
    "    top_model,\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    test_loader,\n",
    "    epochs=30,\n",
    "    optimizer='Adam',\n",
    "    criterion=criterion,\n",
    "    verbose=True,\n",
    "    save_mask_at=100000, freeze_top_till=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>num_feats</th>\n",
       "      <th>num_emb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.293749</td>\n",
       "      <td>0.990234</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>0.989975</td>\n",
       "      <td>3035</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.301293</td>\n",
       "      <td>0.992597</td>\n",
       "      <td>0.994987</td>\n",
       "      <td>0.994987</td>\n",
       "      <td>3030</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.260955</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>0.984962</td>\n",
       "      <td>3018</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.305416</td>\n",
       "      <td>0.993007</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3013</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.252432</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.989975</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc  num_feats  num_emb\n",
       "25    0.293749   0.990234  0.997494  0.989975       3035       11\n",
       "26    0.301293   0.992597  0.994987  0.994987       3030       11\n",
       "27    0.260955   1.000000  0.997494  0.984962       3018       10\n",
       "28    0.305416   0.993007  0.997494  1.000000       3013       10\n",
       "29    0.252432   1.000000  0.989975  1.000000       3000       10"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dual_stg_gini_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dual stg with longer training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "mus = VFL.initialize_mu(gini_labels, feat_idx_list)\n",
    "models, top_model = VFL.make_binary_models(\n",
    "    input_dim_list=input_dim_list,\n",
    "    type=\"DualSTG\",\n",
    "    emb_dim=4,\n",
    "    output_dim=output_dim,\n",
    "    hidden_dims=[8],\n",
    "    activation=\"relu\",\n",
    "    mus=mus, top_lam=0.1, lam=0.1)\n",
    "longer_dual_stg_gini_history = VFL.train(\n",
    "    models,\n",
    "    top_model,\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    test_loader,\n",
    "    epochs=60,\n",
    "    optimizer='Adam',\n",
    "    criterion=criterion,\n",
    "    verbose=True,\n",
    "    save_mask_at=100000, freeze_top_till=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "      <th>num_feats</th>\n",
       "      <th>num_emb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>0.239650</td>\n",
       "      <td>0.992188</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2076</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>0.216584</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2061</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>0.228709</td>\n",
       "      <td>0.998047</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>0.997494</td>\n",
       "      <td>2040</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.216636</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2018</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.213594</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1998</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc  num_feats  num_emb\n",
       "55    0.239650   0.992188  0.997494  1.000000       2076        9\n",
       "56    0.216584   1.000000  1.000000  1.000000       2061        9\n",
       "57    0.228709   0.998047  0.997494  0.997494       2040        9\n",
       "58    0.216636   1.000000  1.000000  1.000000       2018        9\n",
       "59    0.213594   1.000000  1.000000  1.000000       1998        9"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "longer_dual_stg_gini_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SFFS Filtered (0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total computation time for pinv is: 30.316142797470093\n"
     ]
    }
   ],
   "source": [
    "from SFFS import get_f_stat_index\n",
    "index = get_f_stat_index(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1993, 2431)\n"
     ]
    }
   ],
   "source": [
    "X_filtered = X[:, index[:int(0.5*len(index))]]\n",
    "print(X_filtered.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Client 0: Feature Index 0-810\n",
      "Client 1: Feature Index 811-1620\n",
      "Server : Feature Index 1621-2430\n"
     ]
    }
   ],
   "source": [
    "dataset = VFLDataset(data_source=(X_filtered, y), \n",
    "                    num_clients=2,\n",
    "                    gini_portion=None,\n",
    "                    insert_noise=False,\n",
    "                    test_size=0.2)\n",
    "train_loader = DataLoader(dataset.train(), batch_size=256, shuffle=True)\n",
    "val_loader = DataLoader(dataset.valid(), batch_size=1000, shuffle=True)\n",
    "test_loader = DataLoader(dataset.test(), batch_size=1000, shuffle=True)\n",
    "input_dim_list = dataset.get_input_dim_list()\n",
    "output_dim = np.unique(y).size\n",
    "criterion = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "models, top_model = VFL.make_binary_models(\n",
    "                            input_dim_list=input_dim_list,\n",
    "                            type='FNN',\n",
    "                            emb_dim=4,\n",
    "                            output_dim=output_dim, hidden_dims=[8],\n",
    "                            activation='relu')\n",
    "sffs_fnn_history = VFL.train(models, top_model, train_loader, val_loader, test_loader,\n",
    "                epochs=30,  optimizer='Adam',verbose=True, save_mask_at=10000, \n",
    "                criterion=criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_loss</th>\n",
       "      <th>train_acc</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>test_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.129539</td>\n",
       "      <td>0.807119</td>\n",
       "      <td>0.827068</td>\n",
       "      <td>0.827068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.101950</td>\n",
       "      <td>0.838683</td>\n",
       "      <td>0.852130</td>\n",
       "      <td>0.852130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.069046</td>\n",
       "      <td>0.862434</td>\n",
       "      <td>0.879699</td>\n",
       "      <td>0.879699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1.039511</td>\n",
       "      <td>0.877336</td>\n",
       "      <td>0.907268</td>\n",
       "      <td>0.907268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.998699</td>\n",
       "      <td>0.921247</td>\n",
       "      <td>0.927318</td>\n",
       "      <td>0.927318</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    train_loss  train_acc   val_acc  test_acc\n",
       "25    1.129539   0.807119  0.827068  0.827068\n",
       "26    1.101950   0.838683  0.852130  0.852130\n",
       "27    1.069046   0.862434  0.879699  0.879699\n",
       "28    1.039511   0.877336  0.907268  0.907268\n",
       "29    0.998699   0.921247  0.927318  0.927318"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sffs_fnn_history.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Model                 | # Features | Test Acc | Ratio Embedding |\n",
    "|-----------------------|------------|----------|-----------------|\n",
    "| FNN                   | 4862       | 0.9669   | 1               |\n",
    "| STG                   | 3433       | 0.9974   | 1               |\n",
    "| STG+GINI              | 2952       | 1        | 1               |\n",
    "| DualSTG+GINI          | 3000       | 1        | 0.833           |\n",
    "| DualSTG+GINI (double) | 1998       | 1        | 0.75            |\n",
    "| SFFS->FNN             | 2431       | 0.9273   | 1               |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill\n",
    "dill.dump_session('RELATHEDataExperiments.db')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "1579d4a348c2ef16482c05d3cfac916f73c8945ddf1938a1e045b3bdea82eece"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('VFL')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}