1_thesis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2a8bdd2c-0a1a-4b9d-8a59-b648d44d145a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>train_env_0</th>\n",
       "      <th>train_env_1</th>\n",
       "      <th>binary_label</th>\n",
       "      <th>seed</th>\n",
       "      <th>es_f1</th>\n",
       "      <th>ID</th>\n",
       "      <th>job_id</th>\n",
       "      <th>State</th>\n",
       "      <th>Job Type</th>\n",
       "      <th>...</th>\n",
       "      <th>test.PAD_acc</th>\n",
       "      <th>test.MIMIC_prec_bin</th>\n",
       "      <th>test.MIMIC_rec_bin</th>\n",
       "      <th>test.CXP_prec_bin</th>\n",
       "      <th>test.CXP_rec_bin</th>\n",
       "      <th>test.NIH_prec_bin</th>\n",
       "      <th>test.NIH_rec_bin</th>\n",
       "      <th>test.PAD_prec_bin</th>\n",
       "      <th>test.PAD_rec_bin</th>\n",
       "      <th>es_opt_thresh</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(CXP,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.714091</td>\n",
       "      <td>c2ub14qr</td>\n",
       "      <td>27039924</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.993154</td>\n",
       "      <td>0.364995</td>\n",
       "      <td>0.353306</td>\n",
       "      <td>0.518785</td>\n",
       "      <td>0.485522</td>\n",
       "      <td>0.329213</td>\n",
       "      <td>0.457098</td>\n",
       "      <td>0.306122</td>\n",
       "      <td>0.306122</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NIH</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.719801</td>\n",
       "      <td>8dr2o2tb</td>\n",
       "      <td>27039926</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.994765</td>\n",
       "      <td>0.379154</td>\n",
       "      <td>0.259298</td>\n",
       "      <td>0.500491</td>\n",
       "      <td>0.527404</td>\n",
       "      <td>0.457082</td>\n",
       "      <td>0.332293</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.183673</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>PAD</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.727782</td>\n",
       "      <td>qvmz92ez</td>\n",
       "      <td>27039921</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.994362</td>\n",
       "      <td>0.363122</td>\n",
       "      <td>0.331612</td>\n",
       "      <td>0.496194</td>\n",
       "      <td>0.539297</td>\n",
       "      <td>0.332740</td>\n",
       "      <td>0.291732</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.265306</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.699785</td>\n",
       "      <td>k2nenx8j</td>\n",
       "      <td>27039933</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.994060</td>\n",
       "      <td>0.335703</td>\n",
       "      <td>0.439050</td>\n",
       "      <td>0.332840</td>\n",
       "      <td>0.581179</td>\n",
       "      <td>0.346687</td>\n",
       "      <td>0.351014</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.102041</td>\n",
       "      <td>0.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>CXP</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.728350</td>\n",
       "      <td>xcos9u5l</td>\n",
       "      <td>27039925</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.992550</td>\n",
       "      <td>0.395455</td>\n",
       "      <td>0.449380</td>\n",
       "      <td>0.481245</td>\n",
       "      <td>0.583764</td>\n",
       "      <td>0.383769</td>\n",
       "      <td>0.435257</td>\n",
       "      <td>0.295082</td>\n",
       "      <td>0.367347</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>(MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NIH</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.782981</td>\n",
       "      <td>79zle1pd</td>\n",
       "      <td>27032544</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.834994</td>\n",
       "      <td>0.747722</td>\n",
       "      <td>0.860343</td>\n",
       "      <td>0.808696</td>\n",
       "      <td>0.918881</td>\n",
       "      <td>0.582698</td>\n",
       "      <td>0.660704</td>\n",
       "      <td>0.696129</td>\n",
       "      <td>0.519845</td>\n",
       "      <td>0.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>(MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.797440</td>\n",
       "      <td>6k6vvrol</td>\n",
       "      <td>27032541</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.849391</td>\n",
       "      <td>0.747550</td>\n",
       "      <td>0.866619</td>\n",
       "      <td>0.809577</td>\n",
       "      <td>0.907282</td>\n",
       "      <td>0.508246</td>\n",
       "      <td>0.753429</td>\n",
       "      <td>0.662451</td>\n",
       "      <td>0.723037</td>\n",
       "      <td>0.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>(NIH,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.732541</td>\n",
       "      <td>xfbh7izc</td>\n",
       "      <td>27032534</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.824021</td>\n",
       "      <td>0.731447</td>\n",
       "      <td>0.807277</td>\n",
       "      <td>0.811347</td>\n",
       "      <td>0.855601</td>\n",
       "      <td>0.538858</td>\n",
       "      <td>0.748360</td>\n",
       "      <td>0.649215</td>\n",
       "      <td>0.534944</td>\n",
       "      <td>0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>(NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.756103</td>\n",
       "      <td>qdvkvcia</td>\n",
       "      <td>27032543</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.842948</td>\n",
       "      <td>0.708218</td>\n",
       "      <td>0.835875</td>\n",
       "      <td>0.820862</td>\n",
       "      <td>0.827852</td>\n",
       "      <td>0.539066</td>\n",
       "      <td>0.757007</td>\n",
       "      <td>0.644436</td>\n",
       "      <td>0.729508</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>(PAD,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>PAD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.784636</td>\n",
       "      <td>6tcn4hwc</td>\n",
       "      <td>27032550</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.848082</td>\n",
       "      <td>0.728937</td>\n",
       "      <td>0.705195</td>\n",
       "      <td>0.857627</td>\n",
       "      <td>0.520482</td>\n",
       "      <td>0.512081</td>\n",
       "      <td>0.720334</td>\n",
       "      <td>0.667634</td>\n",
       "      <td>0.694996</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Name train_env_0 train_env_1  \\\n",
       "0       (CXP,)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NaN   \n",
       "1    (CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NIH   \n",
       "2    (CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)         CXP         PAD   \n",
       "3     (MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)       MIMIC         NaN   \n",
       "4   (MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         CXP   \n",
       "..                                                ...         ...         ...   \n",
       "85  (MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         NIH   \n",
       "86  (MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         PAD   \n",
       "87      (NIH,)-test(MIMIC)-shift(None)-bal(None,None)         NIH         NaN   \n",
       "88   (NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)         NIH         PAD   \n",
       "89      (PAD,)-test(MIMIC)-shift(None)-bal(None,None)         PAD         NaN   \n",
       "\n",
       "    binary_label  seed     es_f1        ID    job_id     State    Job Type  \\\n",
       "0   Pneumothorax     0  0.714091  c2ub14qr  27039924  finished  1_baseline   \n",
       "1   Pneumothorax     0  0.719801  8dr2o2tb  27039926  finished  1_baseline   \n",
       "2   Pneumothorax     0  0.727782  qvmz92ez  27039921  finished  1_baseline   \n",
       "3   Pneumothorax     0  0.699785  k2nenx8j  27039933  finished  1_baseline   \n",
       "4   Pneumothorax     0  0.728350  xcos9u5l  27039925  finished  1_baseline   \n",
       "..           ...   ...       ...       ...       ...       ...         ...   \n",
       "85           All     0  0.782981  79zle1pd  27032544  finished  1_baseline   \n",
       "86           All     0  0.797440  6k6vvrol  27032541  finished  1_baseline   \n",
       "87           All     0  0.732541  xfbh7izc  27032534  finished  1_baseline   \n",
       "88           All     0  0.756103  qdvkvcia  27032543  finished  1_baseline   \n",
       "89           All     0  0.784636  6tcn4hwc  27032550  finished  1_baseline   \n",
       "\n",
       "    ... test.PAD_acc  test.MIMIC_prec_bin  test.MIMIC_rec_bin  \\\n",
       "0   ...     0.993154             0.364995            0.353306   \n",
       "1   ...     0.994765             0.379154            0.259298   \n",
       "2   ...     0.994362             0.363122            0.331612   \n",
       "3   ...     0.994060             0.335703            0.439050   \n",
       "4   ...     0.992550             0.395455            0.449380   \n",
       "..  ...          ...                  ...                 ...   \n",
       "85  ...     0.834994             0.747722            0.860343   \n",
       "86  ...     0.849391             0.747550            0.866619   \n",
       "87  ...     0.824021             0.731447            0.807277   \n",
       "88  ...     0.842948             0.708218            0.835875   \n",
       "89  ...     0.848082             0.728937            0.705195   \n",
       "\n",
       "    test.CXP_prec_bin  test.CXP_rec_bin  test.NIH_prec_bin  test.NIH_rec_bin  \\\n",
       "0            0.518785          0.485522           0.329213          0.457098   \n",
       "1            0.500491          0.527404           0.457082          0.332293   \n",
       "2            0.496194          0.539297           0.332740          0.291732   \n",
       "3            0.332840          0.581179           0.346687          0.351014   \n",
       "4            0.481245          0.583764           0.383769          0.435257   \n",
       "..                ...               ...                ...               ...   \n",
       "85           0.808696          0.918881           0.582698          0.660704   \n",
       "86           0.809577          0.907282           0.508246          0.753429   \n",
       "87           0.811347          0.855601           0.538858          0.748360   \n",
       "88           0.820862          0.827852           0.539066          0.757007   \n",
       "89           0.857627          0.520482           0.512081          0.720334   \n",
       "\n",
       "    test.PAD_prec_bin  test.PAD_rec_bin  es_opt_thresh  \n",
       "0            0.306122          0.306122           0.29  \n",
       "1            0.428571          0.183673           0.33  \n",
       "2            0.393939          0.265306           0.17  \n",
       "3            0.250000          0.102041           0.15  \n",
       "4            0.295082          0.367347           0.23  \n",
       "..                ...               ...            ...  \n",
       "85           0.696129          0.519845           0.42  \n",
       "86           0.662451          0.723037           0.41  \n",
       "87           0.649215          0.534944           0.22  \n",
       "88           0.644436          0.729508           0.25  \n",
       "89           0.667634          0.694996           0.23  \n",
       "\n",
       "[90 rows x 58 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "import pandas as pd\n",
    "from ast import literal_eval\n",
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "label = \"Cardiomegaly\"\n",
    "in_domain = \"MIMIC\"\n",
    "\n",
    "df = pd.read_csv(\"1_baseline_EVAL-wandb_export_2022-11-22T17_52_23.068-05_00.csv\")\n",
    "# df = df.query(\"binary_label == @label\")\n",
    "# df = df.query(\"train_envs == 'MIMIC'\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0d6aeb3-838f-4d2f-a642-ce3fc82effac",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# 1 - Parse Embeddings from CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "969dcdef-41a2-455b-8de2-02f6febb09ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "row_idx = 2\n",
    "row = df.iloc[row_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "26a95e89-e751-44b3-8727-afd3637d2eaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on row: Name                    (CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)\n",
      "train_env_0                                                          CXP\n",
      "train_env_1                                                          PAD\n",
      "binary_label                                                Pneumothorax\n",
      "seed                                                                   0\n",
      "es_f1                                                           0.727782\n",
      "ID                                                              qvmz92ez\n",
      "job_id                                                          27039921\n",
      "State                                                           finished\n",
      "Job Type                                                      1_baseline\n",
      "Created                                         2022-11-16T05:17:08.000Z\n",
      "Runtime                                                            16362\n",
      "orig-MIMIC-trn_len                                                194213\n",
      "MIMIC-trn_len                                                     194213\n",
      "orig-MIMIC-trn_prop                                             0.041928\n",
      "MIMIC-trn_prop                                                  0.041928\n",
      "orig-CXP-trn_len                                                  153400\n",
      "CXP-trn_len                                                       153400\n",
      "orig-CXP-trn_prop                                               0.092549\n",
      "CXP-trn_prop                                                    0.092549\n",
      "orig-NIH-trn_len                                                   89568\n",
      "NIH-trn_len                                                        89568\n",
      "orig-NIH-trn_prop                                               0.046579\n",
      "NIH-trn_prop                                                    0.046579\n",
      "orig-PAD-trn_len                                                   80069\n",
      "PAD-trn_len                                                        80069\n",
      "orig-PAD-trn_prop                                               0.003097\n",
      "PAD-trn_prop                                                    0.003097\n",
      "balance_method                                                       NaN\n",
      "train_envs                                                       CXP,PAD\n",
      "resample_method                                                      NaN\n",
      "img_size                                                             224\n",
      "output_dir             /scratch/rc4499/thesis/output/(CXP,PAD)-test(M...\n",
      "test.MIMIC_f1                                                   0.660497\n",
      "test.MIMIC_f1_bin                                               0.346652\n",
      "test.CXP_f1                                                     0.730232\n",
      "test.CXP_f1_bin                                                 0.516848\n",
      "test.NIH_f1                                                     0.636774\n",
      "test.NIH_f1_bin                                                 0.310889\n",
      "test.PAD_f1                                                     0.657121\n",
      "test.PAD_f1_bin                                                 0.317073\n",
      "test.MIMIC_roc                                                  0.821056\n",
      "test.CXP_roc                                                    0.860112\n",
      "test.NIH_roc                                                    0.783389\n",
      "test.PAD_roc                                                    0.787094\n",
      "test.MIMIC_acc                                                  0.950622\n",
      "test.CXP_acc                                                    0.899016\n",
      "test.NIH_acc                                                    0.929157\n",
      "test.PAD_acc                                                    0.994362\n",
      "test.MIMIC_prec_bin                                             0.363122\n",
      "test.MIMIC_rec_bin                                              0.331612\n",
      "test.CXP_prec_bin                                               0.496194\n",
      "test.CXP_rec_bin                                                0.539297\n",
      "test.NIH_prec_bin                                                0.33274\n",
      "test.NIH_rec_bin                                                0.291732\n",
      "test.PAD_prec_bin                                               0.393939\n",
      "test.PAD_rec_bin                                                0.265306\n",
      "es_opt_thresh                                                       0.17\n",
      "Name: 2, dtype: object\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>subject_id</th>\n",
       "      <th>path</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>env</th>\n",
       "      <th>frontal</th>\n",
       "      <th>study_id</th>\n",
       "      <th>No Finding</th>\n",
       "      <th>Atelectasis</th>\n",
       "      <th>Cardiomegaly</th>\n",
       "      <th>Effusion</th>\n",
       "      <th>Pneumonia</th>\n",
       "      <th>Pneumothorax</th>\n",
       "      <th>Consolidation</th>\n",
       "      <th>Edema</th>\n",
       "      <th>img_exists</th>\n",
       "      <th>All</th>\n",
       "      <th>preds</th>\n",
       "      <th>targets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>10001217</td>\n",
       "      <td>/mimic-cxr/p10/p10001217/s52067803/a917c883-72...</td>\n",
       "      <td>F</td>\n",
       "      <td>40-60</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>True</td>\n",
       "      <td>52067803</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.00901925377547741, 0.01077312882989645...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10001217</td>\n",
       "      <td>/mimic-cxr/p10/p10001217/s58913004/5e54fc9c-37...</td>\n",
       "      <td>F</td>\n",
       "      <td>40-60</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>True</td>\n",
       "      <td>58913004</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.009735546074807644, 0.0132189877331256...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>10002131</td>\n",
       "      <td>/mimic-cxr/p10/p10002131/s52823782/051b7911-cb...</td>\n",
       "      <td>F</td>\n",
       "      <td>80-</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>True</td>\n",
       "      <td>52823782</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.0062325638718903065, 0.014277804642915...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>10002661</td>\n",
       "      <td>/mimic-cxr/p10/p10002661/s53368584/599b85f2-14...</td>\n",
       "      <td>F</td>\n",
       "      <td>0-20</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>True</td>\n",
       "      <td>53368584</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.007936063222587109, 0.0114700403064489...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>10003412</td>\n",
       "      <td>/mimic-cxr/p10/p10003412/s59172281/80c12354-23...</td>\n",
       "      <td>M</td>\n",
       "      <td>40-60</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>True</td>\n",
       "      <td>59172281</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.006472188513725996, 0.0116815753281116...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19305</th>\n",
       "      <td>361</td>\n",
       "      <td>64670</td>\n",
       "      <td>/CheXpert-v1.0/valid/patient64670/study1/view1...</td>\n",
       "      <td>M</td>\n",
       "      <td>60-80</td>\n",
       "      <td>CXP</td>\n",
       "      <td>True</td>\n",
       "      <td>patient64670/study1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.005553290713578463, 0.0075003942474722...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19306</th>\n",
       "      <td>362</td>\n",
       "      <td>64673</td>\n",
       "      <td>/CheXpert-v1.0/valid/patient64673/study1/view1...</td>\n",
       "      <td>M</td>\n",
       "      <td>40-60</td>\n",
       "      <td>CXP</td>\n",
       "      <td>True</td>\n",
       "      <td>patient64673/study1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.0061642746441066265, 0.005955003201961...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19307</th>\n",
       "      <td>363</td>\n",
       "      <td>64688</td>\n",
       "      <td>/CheXpert-v1.0/valid/patient64688/study1/view1...</td>\n",
       "      <td>F</td>\n",
       "      <td>20-40</td>\n",
       "      <td>CXP</td>\n",
       "      <td>True</td>\n",
       "      <td>patient64688/study1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.005653426516801119, 0.0080270217731595...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19308</th>\n",
       "      <td>364</td>\n",
       "      <td>64704</td>\n",
       "      <td>/CheXpert-v1.0/valid/patient64704/study1/view1...</td>\n",
       "      <td>F</td>\n",
       "      <td>60-80</td>\n",
       "      <td>CXP</td>\n",
       "      <td>True</td>\n",
       "      <td>patient64704/study1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.008805961348116398, 0.0050149606540799...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19309</th>\n",
       "      <td>365</td>\n",
       "      <td>64720</td>\n",
       "      <td>/CheXpert-v1.0/valid/patient64720/study1/view1...</td>\n",
       "      <td>F</td>\n",
       "      <td>60-80</td>\n",
       "      <td>CXP</td>\n",
       "      <td>True</td>\n",
       "      <td>patient64720/study1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.0, 0.0046728565357625484, 0.010310535319149...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65450 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0 subject_id  \\\n",
       "0               0   10001217   \n",
       "1               1   10001217   \n",
       "2               2   10002131   \n",
       "3               3   10002661   \n",
       "4               4   10003412   \n",
       "...           ...        ...   \n",
       "19305         361      64670   \n",
       "19306         362      64673   \n",
       "19307         363      64688   \n",
       "19308         364      64704   \n",
       "19309         365      64720   \n",
       "\n",
       "                                                    path Sex    Age    env  \\\n",
       "0      /mimic-cxr/p10/p10001217/s52067803/a917c883-72...   F  40-60  MIMIC   \n",
       "1      /mimic-cxr/p10/p10001217/s58913004/5e54fc9c-37...   F  40-60  MIMIC   \n",
       "2      /mimic-cxr/p10/p10002131/s52823782/051b7911-cb...   F    80-  MIMIC   \n",
       "3      /mimic-cxr/p10/p10002661/s53368584/599b85f2-14...   F   0-20  MIMIC   \n",
       "4      /mimic-cxr/p10/p10003412/s59172281/80c12354-23...   M  40-60  MIMIC   \n",
       "...                                                  ...  ..    ...    ...   \n",
       "19305  /CheXpert-v1.0/valid/patient64670/study1/view1...   M  60-80    CXP   \n",
       "19306  /CheXpert-v1.0/valid/patient64673/study1/view1...   M  40-60    CXP   \n",
       "19307  /CheXpert-v1.0/valid/patient64688/study1/view1...   F  20-40    CXP   \n",
       "19308  /CheXpert-v1.0/valid/patient64704/study1/view1...   F  60-80    CXP   \n",
       "19309  /CheXpert-v1.0/valid/patient64720/study1/view1...   F  60-80    CXP   \n",
       "\n",
       "       frontal             study_id No Finding Atelectasis Cardiomegaly  \\\n",
       "0         True             52067803        NaN         1.0          NaN   \n",
       "1         True             58913004        NaN         NaN          0.0   \n",
       "2         True             52823782        NaN         NaN          NaN   \n",
       "3         True             53368584        1.0         NaN          NaN   \n",
       "4         True             59172281        1.0         NaN          NaN   \n",
       "...        ...                  ...        ...         ...          ...   \n",
       "19305     True  patient64670/study1        0.0         1.0          0.0   \n",
       "19306     True  patient64673/study1        0.0         0.0          1.0   \n",
       "19307     True  patient64688/study1        0.0         0.0          0.0   \n",
       "19308     True  patient64704/study1        0.0         0.0          0.0   \n",
       "19309     True  patient64720/study1        0.0         0.0          0.0   \n",
       "\n",
       "      Effusion Pneumonia Pneumothorax Consolidation Edema  img_exists  All  \\\n",
       "0          NaN       NaN          NaN           NaN   NaN        True    1   \n",
       "1          NaN       NaN          0.0           NaN   NaN        True    0   \n",
       "2          1.0       NaN          NaN           0.0   NaN        True    1   \n",
       "3          NaN       NaN          NaN           NaN   NaN        True    0   \n",
       "4          NaN       NaN          NaN           NaN   NaN        True    0   \n",
       "...        ...       ...          ...           ...   ...         ...  ...   \n",
       "19305      1.0       0.0          0.0           0.0   0.0        True    1   \n",
       "19306      1.0       0.0          0.0           1.0   0.0        True    1   \n",
       "19307      1.0       0.0          0.0           1.0   1.0        True    1   \n",
       "19308      1.0       0.0          0.0           1.0   0.0        True    1   \n",
       "19309      1.0       0.0          0.0           0.0   0.0        True    1   \n",
       "\n",
       "                                                   preds  targets  \n",
       "0      [0.0, 0.00901925377547741, 0.01077312882989645...        0  \n",
       "1      [0.0, 0.009735546074807644, 0.0132189877331256...        0  \n",
       "2      [0.0, 0.0062325638718903065, 0.014277804642915...        0  \n",
       "3      [0.0, 0.007936063222587109, 0.0114700403064489...        0  \n",
       "4      [0.0, 0.006472188513725996, 0.0116815753281116...        0  \n",
       "...                                                  ...      ...  \n",
       "19305  [0.0, 0.005553290713578463, 0.0075003942474722...        0  \n",
       "19306  [0.0, 0.0061642746441066265, 0.005955003201961...        0  \n",
       "19307  [0.0, 0.005653426516801119, 0.0080270217731595...        0  \n",
       "19308  [0.0, 0.008805961348116398, 0.0050149606540799...        0  \n",
       "19309  [0.0, 0.0046728565357625484, 0.010310535319149...        0  \n",
       "\n",
       "[65450 rows x 20 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def load_all_dfs(row):\n",
    "    dfs = []\n",
    "    for csv_file in glob(f\"{row.output_dir}/emb-test-*.csv\"):\n",
    "        tmp_df = pd.read_csv(csv_file)\n",
    "        dfs.append(tmp_df)\n",
    "        \n",
    "    return pd.concat(dfs)\n",
    "\n",
    "print(f\"Running on row: {row}\")\n",
    "\n",
    "embed_df = load_all_dfs(row)\n",
    "embed_df\n",
    "# load_all_dfs(df.iloc[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "42e0f3a5-986e-4d42-94de-ca670822d9ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "285cd7d4dd2a485d98d6d313c540cab6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/65450 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/state/partition1/job-27698519/ipykernel_1769287/1546253545.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tmp_df[\"preds\"] = tmp_df[\"preds\"].progress_apply(literal_eval)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>env</th>\n",
       "      <th>targets</th>\n",
       "      <th>preds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.00901925377547741, 0.01077312882989645...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.009735546074807644, 0.0132189877331256...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0062325638718903065, 0.014277804642915...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.007936063222587109, 0.0114700403064489...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.006472188513725996, 0.0116815753281116...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19305</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.005553290713578463, 0.0075003942474722...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19306</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0061642746441066265, 0.005955003201961...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19307</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.005653426516801119, 0.0080270217731595...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19308</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.008805961348116398, 0.0050149606540799...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19309</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>[0.0, 0.0046728565357625484, 0.010310535319149...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65450 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         env  targets                                              preds\n",
       "0      MIMIC        0  [0.0, 0.00901925377547741, 0.01077312882989645...\n",
       "1      MIMIC        0  [0.0, 0.009735546074807644, 0.0132189877331256...\n",
       "2      MIMIC        0  [0.0, 0.0062325638718903065, 0.014277804642915...\n",
       "3      MIMIC        0  [0.0, 0.007936063222587109, 0.0114700403064489...\n",
       "4      MIMIC        0  [0.0, 0.006472188513725996, 0.0116815753281116...\n",
       "...      ...      ...                                                ...\n",
       "19305    CXP        0  [0.0, 0.005553290713578463, 0.0075003942474722...\n",
       "19306    CXP        0  [0.0, 0.0061642746441066265, 0.005955003201961...\n",
       "19307    CXP        0  [0.0, 0.005653426516801119, 0.0080270217731595...\n",
       "19308    CXP        0  [0.0, 0.008805961348116398, 0.0050149606540799...\n",
       "19309    CXP        0  [0.0, 0.0046728565357625484, 0.010310535319149...\n",
       "\n",
       "[65450 rows x 3 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert the string representation of the array to an actual array\n",
    "\n",
    "tmp_df = embed_df[[\"env\", \"targets\", \"preds\"]]\n",
    "tmp_df[\"preds\"] = tmp_df[\"preds\"].progress_apply(literal_eval)\n",
    "tmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6c2d0866-a5d2-4669-a8b9-1ad0df6d1581",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the array out into individual embedding columns\n",
    "emb_cols = pd.DataFrame(tmp_df[\"preds\"].to_list(), columns=[f\"emb{x}\" for x in range(1024)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9073949d-4270-4e4f-a3cc-a3074b17ea4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>env</th>\n",
       "      <th>targets</th>\n",
       "      <th>emb0</th>\n",
       "      <th>emb1</th>\n",
       "      <th>emb2</th>\n",
       "      <th>emb3</th>\n",
       "      <th>emb4</th>\n",
       "      <th>emb5</th>\n",
       "      <th>emb6</th>\n",
       "      <th>emb7</th>\n",
       "      <th>...</th>\n",
       "      <th>emb1014</th>\n",
       "      <th>emb1015</th>\n",
       "      <th>emb1016</th>\n",
       "      <th>emb1017</th>\n",
       "      <th>emb1018</th>\n",
       "      <th>emb1019</th>\n",
       "      <th>emb1020</th>\n",
       "      <th>emb1021</th>\n",
       "      <th>emb1022</th>\n",
       "      <th>emb1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009019</td>\n",
       "      <td>0.010773</td>\n",
       "      <td>0.014118</td>\n",
       "      <td>0.082006</td>\n",
       "      <td>1.000930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003739</td>\n",
       "      <td>...</td>\n",
       "      <td>1.716854</td>\n",
       "      <td>1.217442</td>\n",
       "      <td>0.596091</td>\n",
       "      <td>0.204451</td>\n",
       "      <td>1.505431</td>\n",
       "      <td>0.858514</td>\n",
       "      <td>1.026665</td>\n",
       "      <td>0.957266</td>\n",
       "      <td>0.212563</td>\n",
       "      <td>1.233507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009736</td>\n",
       "      <td>0.013219</td>\n",
       "      <td>0.009420</td>\n",
       "      <td>0.053224</td>\n",
       "      <td>0.545036</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004477</td>\n",
       "      <td>...</td>\n",
       "      <td>1.781648</td>\n",
       "      <td>1.129378</td>\n",
       "      <td>1.378103</td>\n",
       "      <td>0.932024</td>\n",
       "      <td>1.258978</td>\n",
       "      <td>1.031995</td>\n",
       "      <td>0.645231</td>\n",
       "      <td>1.608750</td>\n",
       "      <td>0.219032</td>\n",
       "      <td>0.972057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006233</td>\n",
       "      <td>0.014278</td>\n",
       "      <td>0.009311</td>\n",
       "      <td>0.070191</td>\n",
       "      <td>0.962579</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005800</td>\n",
       "      <td>...</td>\n",
       "      <td>0.735749</td>\n",
       "      <td>0.414505</td>\n",
       "      <td>2.033327</td>\n",
       "      <td>1.666464</td>\n",
       "      <td>0.369142</td>\n",
       "      <td>0.335040</td>\n",
       "      <td>0.734385</td>\n",
       "      <td>1.085055</td>\n",
       "      <td>0.173630</td>\n",
       "      <td>1.405971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007936</td>\n",
       "      <td>0.011470</td>\n",
       "      <td>0.009832</td>\n",
       "      <td>0.139990</td>\n",
       "      <td>0.659141</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004092</td>\n",
       "      <td>...</td>\n",
       "      <td>1.269481</td>\n",
       "      <td>2.204899</td>\n",
       "      <td>0.709529</td>\n",
       "      <td>1.043449</td>\n",
       "      <td>1.115657</td>\n",
       "      <td>1.829118</td>\n",
       "      <td>0.469219</td>\n",
       "      <td>0.656466</td>\n",
       "      <td>0.775826</td>\n",
       "      <td>0.916508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MIMIC</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006472</td>\n",
       "      <td>0.011682</td>\n",
       "      <td>0.011859</td>\n",
       "      <td>0.109972</td>\n",
       "      <td>0.603312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006205</td>\n",
       "      <td>...</td>\n",
       "      <td>0.992668</td>\n",
       "      <td>1.700704</td>\n",
       "      <td>0.496869</td>\n",
       "      <td>0.318498</td>\n",
       "      <td>1.064452</td>\n",
       "      <td>2.143820</td>\n",
       "      <td>0.266140</td>\n",
       "      <td>1.388599</td>\n",
       "      <td>1.040022</td>\n",
       "      <td>1.492086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65445</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005553</td>\n",
       "      <td>0.007500</td>\n",
       "      <td>0.008953</td>\n",
       "      <td>0.188211</td>\n",
       "      <td>0.562518</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010716</td>\n",
       "      <td>...</td>\n",
       "      <td>0.206591</td>\n",
       "      <td>0.268932</td>\n",
       "      <td>1.365327</td>\n",
       "      <td>1.063059</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.691384</td>\n",
       "      <td>1.118519</td>\n",
       "      <td>0.057419</td>\n",
       "      <td>0.162391</td>\n",
       "      <td>0.223064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65446</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006164</td>\n",
       "      <td>0.005955</td>\n",
       "      <td>0.007303</td>\n",
       "      <td>0.179337</td>\n",
       "      <td>0.602694</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008311</td>\n",
       "      <td>...</td>\n",
       "      <td>1.167723</td>\n",
       "      <td>0.803143</td>\n",
       "      <td>0.878279</td>\n",
       "      <td>0.396848</td>\n",
       "      <td>0.324516</td>\n",
       "      <td>2.550550</td>\n",
       "      <td>2.510952</td>\n",
       "      <td>0.363813</td>\n",
       "      <td>1.066585</td>\n",
       "      <td>1.349051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65447</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005653</td>\n",
       "      <td>0.008027</td>\n",
       "      <td>0.006180</td>\n",
       "      <td>0.198867</td>\n",
       "      <td>0.520103</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009355</td>\n",
       "      <td>...</td>\n",
       "      <td>0.516233</td>\n",
       "      <td>0.689941</td>\n",
       "      <td>1.120229</td>\n",
       "      <td>1.693037</td>\n",
       "      <td>0.135447</td>\n",
       "      <td>0.362442</td>\n",
       "      <td>1.767007</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.442268</td>\n",
       "      <td>2.402235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65448</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008806</td>\n",
       "      <td>0.005015</td>\n",
       "      <td>0.009229</td>\n",
       "      <td>0.199519</td>\n",
       "      <td>0.552450</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009826</td>\n",
       "      <td>...</td>\n",
       "      <td>0.559241</td>\n",
       "      <td>0.397526</td>\n",
       "      <td>1.635013</td>\n",
       "      <td>1.093209</td>\n",
       "      <td>0.025182</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.753329</td>\n",
       "      <td>0.339818</td>\n",
       "      <td>0.022096</td>\n",
       "      <td>0.131124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65449</th>\n",
       "      <td>CXP</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004673</td>\n",
       "      <td>0.010311</td>\n",
       "      <td>0.006958</td>\n",
       "      <td>0.223502</td>\n",
       "      <td>0.571176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012401</td>\n",
       "      <td>...</td>\n",
       "      <td>0.175857</td>\n",
       "      <td>0.187619</td>\n",
       "      <td>1.029836</td>\n",
       "      <td>0.827870</td>\n",
       "      <td>0.155176</td>\n",
       "      <td>2.141144</td>\n",
       "      <td>3.095485</td>\n",
       "      <td>0.482666</td>\n",
       "      <td>0.128750</td>\n",
       "      <td>1.659392</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65450 rows × 1026 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         env  targets  emb0      emb1      emb2      emb3      emb4      emb5  \\\n",
       "0      MIMIC        0   0.0  0.009019  0.010773  0.014118  0.082006  1.000930   \n",
       "1      MIMIC        0   0.0  0.009736  0.013219  0.009420  0.053224  0.545036   \n",
       "2      MIMIC        0   0.0  0.006233  0.014278  0.009311  0.070191  0.962579   \n",
       "3      MIMIC        0   0.0  0.007936  0.011470  0.009832  0.139990  0.659141   \n",
       "4      MIMIC        0   0.0  0.006472  0.011682  0.011859  0.109972  0.603312   \n",
       "...      ...      ...   ...       ...       ...       ...       ...       ...   \n",
       "65445    CXP        0   0.0  0.005553  0.007500  0.008953  0.188211  0.562518   \n",
       "65446    CXP        0   0.0  0.006164  0.005955  0.007303  0.179337  0.602694   \n",
       "65447    CXP        0   0.0  0.005653  0.008027  0.006180  0.198867  0.520103   \n",
       "65448    CXP        0   0.0  0.008806  0.005015  0.009229  0.199519  0.552450   \n",
       "65449    CXP        0   0.0  0.004673  0.010311  0.006958  0.223502  0.571176   \n",
       "\n",
       "       emb6      emb7  ...   emb1014   emb1015   emb1016   emb1017   emb1018  \\\n",
       "0       0.0  0.003739  ...  1.716854  1.217442  0.596091  0.204451  1.505431   \n",
       "1       0.0  0.004477  ...  1.781648  1.129378  1.378103  0.932024  1.258978   \n",
       "2       0.0  0.005800  ...  0.735749  0.414505  2.033327  1.666464  0.369142   \n",
       "3       0.0  0.004092  ...  1.269481  2.204899  0.709529  1.043449  1.115657   \n",
       "4       0.0  0.006205  ...  0.992668  1.700704  0.496869  0.318498  1.064452   \n",
       "...     ...       ...  ...       ...       ...       ...       ...       ...   \n",
       "65445   0.0  0.010716  ...  0.206591  0.268932  1.365327  1.063059  0.000000   \n",
       "65446   0.0  0.008311  ...  1.167723  0.803143  0.878279  0.396848  0.324516   \n",
       "65447   0.0  0.009355  ...  0.516233  0.689941  1.120229  1.693037  0.135447   \n",
       "65448   0.0  0.009826  ...  0.559241  0.397526  1.635013  1.093209  0.025182   \n",
       "65449   0.0  0.012401  ...  0.175857  0.187619  1.029836  0.827870  0.155176   \n",
       "\n",
       "        emb1019   emb1020   emb1021   emb1022   emb1023  \n",
       "0      0.858514  1.026665  0.957266  0.212563  1.233507  \n",
       "1      1.031995  0.645231  1.608750  0.219032  0.972057  \n",
       "2      0.335040  0.734385  1.085055  0.173630  1.405971  \n",
       "3      1.829118  0.469219  0.656466  0.775826  0.916508  \n",
       "4      2.143820  0.266140  1.388599  1.040022  1.492086  \n",
       "...         ...       ...       ...       ...       ...  \n",
       "65445  0.691384  1.118519  0.057419  0.162391  0.223064  \n",
       "65446  2.550550  2.510952  0.363813  1.066585  1.349051  \n",
       "65447  0.362442  1.767007  0.000000  0.442268  2.402235  \n",
       "65448  0.000000  1.753329  0.339818  0.022096  0.131124  \n",
       "65449  2.141144  3.095485  0.482666  0.128750  1.659392  \n",
       "\n",
       "[65450 rows x 1026 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate with the metadata columns (env and label)\n",
    "ml_df = pd.concat([\n",
    "    tmp_df.drop(columns=\"preds\").reset_index(drop=True),\n",
    "    emb_cols.reset_index(drop=True)\n",
    "], axis=1)\n",
    "\n",
    "ml_df\n",
    "# pd.concat(ml_df.drop(columns=\"preds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "9f1c5321-d6a4-4d53-ba00-78d98d3114f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "wandb version 0.13.6 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/scratch/rc4499/thesis/ood-generalization/wandb/run-20221206_130743-3iwyh1b8</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/basedrhys/ood-generalization/runs/3iwyh1b8\" target=\"_blank\">row 2</a></strong> to <a href=\"https://wandb.ai/basedrhys/ood-generalization\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e370f77a2e954ebf8a7d392cc142d24d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e4fff4474774ac5943a7d2ce8fb32cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: SINGLE PRED VAL: ('CXP',) for col: env... Subsetting through the middle\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/rc4499/envs_dirs/thesis/lib/python3.8/site-packages/sklearn/svm/_base.py:1225: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: SINGLE PRED VAL: ('MIMIC',) for col: env... Subsetting through the middle\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [39]\u001b[0m, in \u001b[0;36m<cell line: 117>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    113\u001b[0m         json\u001b[38;5;241m.\u001b[39mdump(results_dict, f, indent\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m    115\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m results_dict\n\u001b[0;32m--> 117\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [39]\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     80\u001b[0m env_pred_col \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     81\u001b[0m env_pred_vals \u001b[38;5;241m=\u001b[39m [(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCXP\u001b[39m\u001b[38;5;124m\"\u001b[39m, ), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMIMIC\u001b[39m\u001b[38;5;124m\"\u001b[39m, ), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNIH\u001b[39m\u001b[38;5;124m\"\u001b[39m, ), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPAD\u001b[39m\u001b[38;5;124m\"\u001b[39m, ), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCXP\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNIH\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCXP\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPAD\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMIMIC\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCXP\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMIMIC\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNIH\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMIMIC\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPAD\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNIH\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPAD\u001b[39m\u001b[38;5;124m\"\u001b[39m), (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCXP\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMIMIC\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNIH\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPAD\u001b[39m\u001b[38;5;124m\"\u001b[39m)]\n\u001b[0;32m---> 83\u001b[0m \u001b[43mtrain_models\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mml_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m     84\u001b[0m \u001b[43m             \u001b[49m\u001b[43mpred_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_pred_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     85\u001b[0m \u001b[43m             \u001b[49m\u001b[43mpred_vals\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_pred_vals\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     86\u001b[0m \u001b[43m             \u001b[49m\u001b[43mfix_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_fix_col\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     87\u001b[0m \u001b[43m             \u001b[49m\u001b[43mfix_vals\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv_fix_vals\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     88\u001b[0m \u001b[43m             \u001b[49m\u001b[43mresults_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresults_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     89\u001b[0m \u001b[43m             \u001b[49m\u001b[43mtask_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43menv_pred\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;66;03m# Label prediction task\u001b[39;00m\n\u001b[1;32m     92\u001b[0m label_fix_col \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "Input \u001b[0;32mIn [39]\u001b[0m, in \u001b[0;36mtrain_models\u001b[0;34m(df, pred_col, pred_vals, fix_col, fix_vals, results_dict, task_type)\u001b[0m\n\u001b[1;32m     38\u001b[0m     df_bal\u001b[38;5;241m.\u001b[39mloc[mid_val:, pred_col] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     40\u001b[0m \u001b[38;5;66;03m# Now lets pass this dataframe into our train method\u001b[39;00m\n\u001b[0;32m---> 41\u001b[0m acc \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf_bal\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpred_col\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     43\u001b[0m \u001b[38;5;66;03m# Store the results in our global dictionary\u001b[39;00m\n\u001b[1;32m     44\u001b[0m results_dict[task_type]\u001b[38;5;241m.\u001b[39mappend({\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfix_val\u001b[39m\u001b[38;5;124m\"\u001b[39m: fix_val,\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpred_val\u001b[39m\u001b[38;5;124m\"\u001b[39m: pred_val,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124macc\u001b[39m\u001b[38;5;124m\"\u001b[39m: acc,\n\u001b[1;32m     50\u001b[0m })\n",
      "Input \u001b[0;32mIn [39]\u001b[0m, in \u001b[0;36mtrain_model\u001b[0;34m(df, pred_col, max_iter)\u001b[0m\n\u001b[1;32m     55\u001b[0m model \u001b[38;5;241m=\u001b[39m make_pipeline(StandardScaler(), LinearSVC(random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, tol\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1e-5\u001b[39m, max_iter\u001b[38;5;241m=\u001b[39mmax_iter))\n\u001b[1;32m     57\u001b[0m X_train, X_test, y_train, y_test \u001b[38;5;241m=\u001b[39m train_test_split(X, y, test_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.2\u001b[39m, random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m42\u001b[39m)\n\u001b[0;32m---> 59\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     61\u001b[0m preds \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict(X_test)\n\u001b[1;32m     63\u001b[0m acc \u001b[38;5;241m=\u001b[39m accuracy_score(y_test, preds)\n",
      "File \u001b[0;32m/scratch/rc4499/envs_dirs/thesis/lib/python3.8/site-packages/sklearn/pipeline.py:382\u001b[0m, in \u001b[0;36mPipeline.fit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_final_estimator \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpassthrough\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    381\u001b[0m         fit_params_last_step \u001b[38;5;241m=\u001b[39m fit_params_steps[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msteps[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m][\u001b[38;5;241m0\u001b[39m]]\n\u001b[0;32m--> 382\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_final_estimator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfit_params_last_step\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[0;32m/scratch/rc4499/envs_dirs/thesis/lib/python3.8/site-packages/sklearn/svm/_classes.py:257\u001b[0m, in \u001b[0;36mLinearSVC.fit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    254\u001b[0m check_classification_targets(y)\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclasses_ \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39munique(y)\n\u001b[0;32m--> 257\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcoef_, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mintercept_, n_iter_ \u001b[38;5;241m=\u001b[39m \u001b[43m_fit_liblinear\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    258\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    259\u001b[0m \u001b[43m    \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    260\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mC\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    261\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_intercept\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    262\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mintercept_scaling\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    263\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclass_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    264\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpenalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    265\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdual\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    266\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    267\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_iter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    268\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    269\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    270\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmulti_class\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    271\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    272\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    273\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;66;03m# Backward compatibility: _fit_liblinear is used both by LinearSVC/R\u001b[39;00m\n\u001b[1;32m    275\u001b[0m \u001b[38;5;66;03m# and LogisticRegression but LogisticRegression sets a structured\u001b[39;00m\n\u001b[1;32m    276\u001b[0m \u001b[38;5;66;03m# `n_iter_` attribute with information about the underlying OvR fits\u001b[39;00m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;66;03m# while LinearSVC/R only reports the maximum value.\u001b[39;00m\n\u001b[1;32m    278\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_iter_ \u001b[38;5;241m=\u001b[39m n_iter_\u001b[38;5;241m.\u001b[39mmax()\u001b[38;5;241m.\u001b[39mitem()\n",
      "File \u001b[0;32m/scratch/rc4499/envs_dirs/thesis/lib/python3.8/site-packages/sklearn/svm/_base.py:1205\u001b[0m, in \u001b[0;36m_fit_liblinear\u001b[0;34m(X, y, C, fit_intercept, intercept_scaling, class_weight, penalty, dual, verbose, max_iter, tol, random_state, multi_class, loss, epsilon, sample_weight)\u001b[0m\n\u001b[1;32m   1202\u001b[0m sample_weight \u001b[38;5;241m=\u001b[39m _check_sample_weight(sample_weight, X, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mfloat64)\n\u001b[1;32m   1204\u001b[0m solver_type \u001b[38;5;241m=\u001b[39m _get_liblinear_solver_type(multi_class, penalty, loss, dual)\n\u001b[0;32m-> 1205\u001b[0m raw_coef_, n_iter_ \u001b[38;5;241m=\u001b[39m \u001b[43mliblinear\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_wrap\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1206\u001b[0m \u001b[43m    \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1207\u001b[0m \u001b[43m    \u001b[49m\u001b[43my_ind\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1208\u001b[0m \u001b[43m    \u001b[49m\u001b[43msp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43misspmatrix\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1209\u001b[0m \u001b[43m    \u001b[49m\u001b[43msolver_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1210\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1211\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1212\u001b[0m \u001b[43m    \u001b[49m\u001b[43mC\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1213\u001b[0m \u001b[43m    \u001b[49m\u001b[43mclass_weight_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1214\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_iter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1215\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrnd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrandint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miinfo\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mi\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1216\u001b[0m \u001b[43m    \u001b[49m\u001b[43mepsilon\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1217\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1218\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1219\u001b[0m \u001b[38;5;66;03m# Regarding rnd.randint(..) in the above signature:\u001b[39;00m\n\u001b[1;32m   1220\u001b[0m \u001b[38;5;66;03m# seed for srand in range [0..INT_MAX); due to limitations in Numpy\u001b[39;00m\n\u001b[1;32m   1221\u001b[0m \u001b[38;5;66;03m# on 32-bit platforms, we can't get to the UINT_MAX limit that\u001b[39;00m\n\u001b[1;32m   1222\u001b[0m \u001b[38;5;66;03m# srand supports\u001b[39;00m\n\u001b[1;32m   1223\u001b[0m n_iter_max \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(n_iter_)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import wandb\n",
    "import json\n",
    "\n",
    "def train_models(df, pred_col, pred_vals, fix_col, fix_vals, results_dict, task_type):   \n",
    "    for fix_val in tqdm(fix_vals):\n",
    "        # Subset the dataframe to just have this val in the column\n",
    "        df_fix = df[df[fix_col] == fix_val]\n",
    "        # Drop this column to avoid any oddities during training\n",
    "        df_fix = df_fix.drop(columns=fix_col)\n",
    "        \n",
    "        # Get just the 2/4 classes that we're trying to \n",
    "        for pred_val in tqdm(pred_vals):\n",
    "            df_pred = df_fix[df_fix[pred_col].isin(pred_val)]\n",
    "            \n",
    "            # We have the final dataframe, but we need to create a perfectly balanced \n",
    "            # version of it\n",
    "            grouped = df_pred.groupby(pred_col)\n",
    "            # print(\"Count per class:\", grouped[\"emb0\"].count())\n",
    "            min_group_size = grouped.count()[\"emb0\"].min()\n",
    "            df_bal = grouped.sample(n=min_group_size, random_state=0)\n",
    "            # print(\"Count per class after balancing:\", df_bal.groupby(pred_col)[\"emb0\"].count())\n",
    "            \n",
    "            # Note that we may have a single class remaining in our dataset (if we're doing the baseline \n",
    "            # CXP vs CXP prediction, for example). We need to check that and manually change our dataset\n",
    "            # If that is the case\n",
    "            df_bal = df_bal.sample(frac=1, random_state=0).reset_index(drop=True)\n",
    "            \n",
    "            if len(pred_val) == 1:\n",
    "                print(f\"INFO: SINGLE PRED VAL: {pred_val} for col: {pred_col}... Subsetting through the middle\")\n",
    "                mid_val = len(df_bal) // 2\n",
    "                \n",
    "                df_bal.loc[:mid_val, pred_col] = \"0\"\n",
    "                df_bal.loc[mid_val:, pred_col] = \"1\"\n",
    "            \n",
    "            # Now lets pass this dataframe into our train method\n",
    "            acc = train_model(df_bal, pred_col)\n",
    "            \n",
    "            # Store the results in our global dictionary\n",
    "            results_dict[task_type].append({\n",
    "                \"fix_val\": fix_val,\n",
    "                \"pred_val\": pred_val,\n",
    "                \"min_group_size\": min_group_size,\n",
    "                \"df_size\": len(df_bal),\n",
    "                \"acc\": acc,\n",
    "            })\n",
    "            \n",
    "def train_model(df, pred_col, max_iter=5000):\n",
    "    X, y = df.drop(columns=pred_col), df[pred_col]\n",
    "    \n",
    "    model = make_pipeline(StandardScaler(), LinearSVC(random_state=0, tol=1e-5, max_iter=max_iter))\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    preds = model.predict(X_test)\n",
    "\n",
    "    acc = accuracy_score(y_test, preds)\n",
    "    return acc            \n",
    "            \n",
    "def main():\n",
    "    wandb.init(project=\"ood-generalization\",\n",
    "            job_type=\"emb_train\", \n",
    "            entity=\"basedrhys\",\n",
    "              name=f\"row {row_idx}\")\n",
    "    \n",
    "    results_dict = {}\n",
    "    results_dict[\"env_pred\"] = []\n",
    "    results_dict[\"label_pred\"] = []\n",
    "\n",
    "    # Environment Prediction Task\n",
    "    env_fix_col = \"targets\"\n",
    "    env_fix_vals = [0, 1]\n",
    "\n",
    "    env_pred_col = \"env\"\n",
    "    env_pred_vals = [(\"CXP\", ), (\"MIMIC\", ), (\"NIH\", ), (\"PAD\", ), (\"CXP\",\"NIH\"), (\"CXP\",\"PAD\"), (\"MIMIC\",\"CXP\"), (\"MIMIC\",\"NIH\"), (\"MIMIC\",\"PAD\"), (\"NIH\",\"PAD\"), (\"CXP\", \"MIMIC\", \"NIH\", \"PAD\")]\n",
    "\n",
    "    train_models(df=ml_df, \n",
    "                 pred_col=env_pred_col,\n",
    "                 pred_vals=env_pred_vals,\n",
    "                 fix_col=env_fix_col,\n",
    "                 fix_vals=env_fix_vals,\n",
    "                 results_dict=results_dict,\n",
    "                 task_type=\"env_pred\")\n",
    "    \n",
    "    # Label prediction task\n",
    "    label_fix_col = \"env\"\n",
    "    label_fix_vals = [\"CXP\", \"MIMIC\", \"NIH\", \"PAD\"]\n",
    "\n",
    "    label_pred_col = \"targets\"\n",
    "    label_pred_vals = [(0, 1), (0, ), (1, )]\n",
    "    \n",
    "    train_models(df=ml_df,\n",
    "                 pred_col=label_pred_col,\n",
    "                 pred_vals=label_pred_vals,\n",
    "                 fix_col=label_fix_col,\n",
    "                 fix_vals=label_fix_vals,\n",
    "                 results_dict=results_dict,\n",
    "                 task_type=\"label_pred\")\n",
    "    \n",
    "    wandb.log(results_dict)\n",
    "    \n",
    "    output_dir = row[\"output_dir\"]\n",
    "    \n",
    "    print(\"Outputting JSON to\", output_dir)\n",
    "    \n",
    "    with open(f\"{output_dir}/emb_test_results.json\", mode=\"w\") as f:\n",
    "        json.dump(results_dict, f, indent=True)\n",
    "    \n",
    "    return results_dict\n",
    "        \n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2ff87878-06d6-4ffd-a5a0-1942ae198d92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">row 2</strong>: <a href=\"https://wandb.ai/basedrhys/ood-generalization/runs/3bqvivzy\" target=\"_blank\">https://wandb.ai/basedrhys/ood-generalization/runs/3bqvivzy</a><br/>Synced 7 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20221206_125621-3bqvivzy/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f9579851-f2cf-4ce0-ae70-b2983876ebc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total DF Length: 61858\n",
      "Count per class: env\n",
      "CXP      17376\n",
      "MIMIC    23537\n",
      "NIH      11061\n",
      "PAD       9884\n",
      "Name: emb0, dtype: int64\n",
      "Count per class: env\n",
      "CXP      9884\n",
      "MIMIC    9884\n",
      "NIH      9884\n",
      "PAD      9884\n",
      "Name: emb0, dtype: int64\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 's'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36m<cell line: 30>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     clf\u001b[38;5;241m.\u001b[39mfit(X, y)\n\u001b[1;32m     27\u001b[0m     \u001b[38;5;28mprint\u001b[39m(clf\u001b[38;5;241m.\u001b[39mscore(X, y))\n\u001b[0;32m---> 30\u001b[0m \u001b[43mrun_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mml_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mCXP\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msubsample\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_iter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36mrun_model\u001b[0;34m(df, target_label, ood_env, max_iter, subsample)\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;66;03m# assert set(df.env.unique()) == set([in_domain, ood_env])\u001b[39;00m\n\u001b[1;32m     21\u001b[0m X, y \u001b[38;5;241m=\u001b[39m df\u001b[38;5;241m.\u001b[39mdrop(columns\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv\u001b[39m\u001b[38;5;124m\"\u001b[39m), df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menv\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m---> 23\u001b[0m clf \u001b[38;5;241m=\u001b[39m make_pipeline(StandardScaler(), \u001b[43mLinearSVC\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1e-5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_iter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_iter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43ms\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[1;32m     25\u001b[0m clf\u001b[38;5;241m.\u001b[39mfit(X, y)\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28mprint\u001b[39m(clf\u001b[38;5;241m.\u001b[39mscore(X, y))\n",
      "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 's'"
     ]
    }
   ],
   "source": [
    "\n",
    "# Split df into in-domain and OOD\n",
    "\n",
    "# Train model to classify between the two\n",
    "\n",
    "# Print classification results\n",
    "\n",
    "def run_model(df, target_label, ood_env, max_iter=50000, subsample=None):\n",
    "    df = df.query(\"targets == @target_label\").drop(columns=\"targets\")\n",
    "\n",
    "    # df = df[df[\"env\"].isin([in_domain, ood_env])]\n",
    "    \n",
    "    print(\"Total DF Length:\", len(df))\n",
    "    print(\"Count per class:\", df.groupby(\"env\")[\"emb0\"].count())\n",
    "    if subsample:\n",
    "        min_group_size = df.groupby(\"env\").count()[\"emb0\"].min()\n",
    "        df = df.groupby(\"env\").sample(n=min_group_size, random_state=0)\n",
    "    \n",
    "    print(\"Count per class:\", df.groupby(\"env\")[\"emb0\"].count())\n",
    "    # assert set(df.env.unique()) == set([in_domain, ood_env])\n",
    "    \n",
    "    X, y = df.drop(columns=\"env\"), df[\"env\"]\n",
    "    \n",
    "    clf = make_pipeline(StandardScaler(), LinearSVC(random_state=0, tol=1e-5, max_iter=max_iter, verbose=1))\n",
    "    \n",
    "    clf.fit(X, y)\n",
    "    \n",
    "    print(clf.score(X, y))\n",
    "    \n",
    "    \n",
    "run_model(ml_df, 0, \"CXP\", subsample=1000, max_iter=100)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8e723dd6-56c9-4f35-ac89-af2c677257e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_test(df, target_label=0, subsample=None):\n",
    "    df = df.query(\"targets == @target_label\").drop(columns=\"targets\")\n",
    "\n",
    "    df = df.query(\"env == 'MIMIC'\")\n",
    "    \n",
    "    mid_val = len(df) // 2\n",
    "    \n",
    "    df.loc[:mid_val, \"env\"] = \"0\"\n",
    "    df.loc[mid_val:, \"env\"] = \"1\"\n",
    "    \n",
    "    print(\"Total DF Length:\", len(df))\n",
    "    \n",
    "    print(df.head())\n",
    "    print(df.tail())\n",
    "    \n",
    "    print(\"Count per class:\", df.groupby(\"env\")[\"emb0\"].count())\n",
    "    \n",
    "    if subsample:\n",
    "        df = df.groupby(\"env\").sample(n=subsample, random_state=0)\n",
    "    \n",
    "    X, y = df.drop(columns=\"env\"), df[\"env\"]\n",
    "    \n",
    "    clf = make_pipeline(StandardScaler(), LinearSVC(random_state=0, tol=1e-5, max_iter=100000, verbose=1))\n",
    "    \n",
    "    clf.fit(X, y)\n",
    "    \n",
    "    print(clf.score(X, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1a80cf56-8fb8-4f38-a709-92882e5f1f8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total DF Length: 20201\n",
      "  env  emb0      emb1      emb2  emb3      emb4      emb5      emb6      emb7  \\\n",
      "0   0   0.0  0.007503  0.000000   0.0  0.087218  0.703008  0.004383  0.006953   \n",
      "1   0   0.0  0.007088  0.000040   0.0  0.102328  0.798866  0.003676  0.007626   \n",
      "2   0   0.0  0.008093  0.000011   0.0  0.091230  0.679302  0.004168  0.005339   \n",
      "3   0   0.0  0.010434  0.000000   0.0  0.149953  0.868009  0.004373  0.006010   \n",
      "4   0   0.0  0.008966  0.000016   0.0  0.148700  0.811602  0.003370  0.004825   \n",
      "\n",
      "       emb8  ...   emb1014   emb1015   emb1016   emb1017   emb1018   emb1019  \\\n",
      "0  0.312130  ...  0.879122  3.747355  1.287988  1.130645  1.926350  0.810758   \n",
      "1  0.320931  ...  1.415729  3.744271  1.754291  1.041173  1.925371  0.798239   \n",
      "2  0.309972  ...  0.858417  0.344237  0.230802  0.359181  0.607738  1.049954   \n",
      "3  0.432212  ...  1.861471  4.430128  0.345798  2.046093  2.349850  0.695919   \n",
      "4  0.418812  ...  1.635271  2.765703  2.714306  3.090220  2.080219  0.870097   \n",
      "\n",
      "    emb1020   emb1021   emb1022   emb1023  \n",
      "0  3.074003  1.973089  2.014313  1.579566  \n",
      "1  2.538288  2.277111  2.350302  1.778218  \n",
      "2  0.704531  1.235538  0.325283  0.141733  \n",
      "3  2.925310  2.744299  2.347250  1.622971  \n",
      "4  3.413409  1.452121  2.993020  2.212436  \n",
      "\n",
      "[5 rows x 1025 columns]\n",
      "      env  emb0      emb1      emb2  emb3      emb4      emb5      emb6  \\\n",
      "24494   1   0.0  0.009842  0.000022   0.0  0.167326  0.708037  0.003472   \n",
      "24495   1   0.0  0.012393  0.000000   0.0  0.197373  0.770302  0.003214   \n",
      "24496   1   0.0  0.006506  0.000004   0.0  0.105408  1.018255  0.003196   \n",
      "24498   1   0.0  0.010435  0.000024   0.0  0.091225  0.652333  0.003451   \n",
      "24503   1   0.0  0.005702  0.000069   0.0  0.119710  0.880928  0.004015   \n",
      "\n",
      "           emb7      emb8  ...   emb1014   emb1015   emb1016   emb1017  \\\n",
      "24494  0.006315  0.308179  ...  1.449434  2.669827  1.207075  2.277984   \n",
      "24495  0.006862  0.280143  ...  0.835820  0.646861  0.938638  0.447083   \n",
      "24496  0.006998  0.333114  ...  0.701544  1.004197  0.445281  0.919188   \n",
      "24498  0.004043  0.237064  ...  0.149594  0.000000  0.191375  0.012661   \n",
      "24503  0.007774  0.183278  ...  0.976559  0.223400  0.102840  0.013986   \n",
      "\n",
      "        emb1018   emb1019   emb1020   emb1021   emb1022   emb1023  \n",
      "24494  1.677248  0.932988  3.260410  1.427409  1.740650  2.115317  \n",
      "24495  0.794900  0.870651  1.211172  2.446810  0.294744  0.782535  \n",
      "24496  0.919774  0.965946  0.709070  1.989115  0.748326  1.035873  \n",
      "24498  0.096507  1.214237  0.287017  1.440121  0.004318  0.087370  \n",
      "24503  0.305100  1.392447  0.413866  2.526262  0.341999  0.251946  \n",
      "\n",
      "[5 rows x 1025 columns]\n",
      "Count per class: env\n",
      "0     8256\n",
      "1    11945\n",
      "Name: emb0, dtype: int64\n",
      "[LibLinear].......................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*...........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................*\n",
      "optimization finished, #iter = 36708\n",
      "Objective value = -221.103145\n",
      "nSV = 877\n",
      "0.9975\n"
     ]
    }
   ],
   "source": [
    "run_test(ml_df, subsample=800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e4a210b7-35ef-4de9-b40a-45341d9b908a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total DF Length: 20201\n",
      "  env  emb0      emb1      emb2  emb3      emb4      emb5      emb6      emb7  \\\n",
      "0   0   0.0  0.007503  0.000000   0.0  0.087218  0.703008  0.004383  0.006953   \n",
      "1   0   0.0  0.007088  0.000040   0.0  0.102328  0.798866  0.003676  0.007626   \n",
      "2   0   0.0  0.008093  0.000011   0.0  0.091230  0.679302  0.004168  0.005339   \n",
      "3   0   0.0  0.010434  0.000000   0.0  0.149953  0.868009  0.004373  0.006010   \n",
      "4   0   0.0  0.008966  0.000016   0.0  0.148700  0.811602  0.003370  0.004825   \n",
      "\n",
      "       emb8  ...   emb1014   emb1015   emb1016   emb1017   emb1018   emb1019  \\\n",
      "0  0.312130  ...  0.879122  3.747355  1.287988  1.130645  1.926350  0.810758   \n",
      "1  0.320931  ...  1.415729  3.744271  1.754291  1.041173  1.925371  0.798239   \n",
      "2  0.309972  ...  0.858417  0.344237  0.230802  0.359181  0.607738  1.049954   \n",
      "3  0.432212  ...  1.861471  4.430128  0.345798  2.046093  2.349850  0.695919   \n",
      "4  0.418812  ...  1.635271  2.765703  2.714306  3.090220  2.080219  0.870097   \n",
      "\n",
      "    emb1020   emb1021   emb1022   emb1023  \n",
      "0  3.074003  1.973089  2.014313  1.579566  \n",
      "1  2.538288  2.277111  2.350302  1.778218  \n",
      "2  0.704531  1.235538  0.325283  0.141733  \n",
      "3  2.925310  2.744299  2.347250  1.622971  \n",
      "4  3.413409  1.452121  2.993020  2.212436  \n",
      "\n",
      "[5 rows x 1025 columns]\n",
      "      env  emb0      emb1      emb2  emb3      emb4      emb5      emb6  \\\n",
      "24494   1   0.0  0.009842  0.000022   0.0  0.167326  0.708037  0.003472   \n",
      "24495   1   0.0  0.012393  0.000000   0.0  0.197373  0.770302  0.003214   \n",
      "24496   1   0.0  0.006506  0.000004   0.0  0.105408  1.018255  0.003196   \n",
      "24498   1   0.0  0.010435  0.000024   0.0  0.091225  0.652333  0.003451   \n",
      "24503   1   0.0  0.005702  0.000069   0.0  0.119710  0.880928  0.004015   \n",
      "\n",
      "           emb7      emb8  ...   emb1014   emb1015   emb1016   emb1017  \\\n",
      "24494  0.006315  0.308179  ...  1.449434  2.669827  1.207075  2.277984   \n",
      "24495  0.006862  0.280143  ...  0.835820  0.646861  0.938638  0.447083   \n",
      "24496  0.006998  0.333114  ...  0.701544  1.004197  0.445281  0.919188   \n",
      "24498  0.004043  0.237064  ...  0.149594  0.000000  0.191375  0.012661   \n",
      "24503  0.007774  0.183278  ...  0.976559  0.223400  0.102840  0.013986   \n",
      "\n",
      "        emb1018   emb1019   emb1020   emb1021   emb1022   emb1023  \n",
      "24494  1.677248  0.932988  3.260410  1.427409  1.740650  2.115317  \n",
      "24495  0.794900  0.870651  1.211172  2.446810  0.294744  0.782535  \n",
      "24496  0.919774  0.965946  0.709070  1.989115  0.748326  1.035873  \n",
      "24498  0.096507  1.214237  0.287017  1.440121  0.004318  0.087370  \n",
      "24503  0.305100  1.392447  0.413866  2.526262  0.341999  0.251946  \n",
      "\n",
      "[5 rows x 1025 columns]\n",
      "Count per class: env\n",
      "0     8256\n",
      "1    11945\n",
      "Name: emb0, dtype: int64\n",
      "[LibLinear]................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
      "optimization finished, #iter = 100000\n",
      "\n",
      "WARNING: reaching max number of iterations\n",
      "Using -s 2 may be faster (also see FAQ)\n",
      "\n",
      "Objective value = -17925.267959\n",
      "nSV = 20133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/scratch/rc4499/envs_dirs/thesis/lib/python3.8/site-packages/sklearn/svm/_base.py:1225: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6379387159051532\n"
     ]
    }
   ],
   "source": [
    "def run_test(df, target_label=0):\n",
    "    df = df.query(\"targets == @target_label\").drop(columns=\"targets\")\n",
    "\n",
    "    df = df.query(\"env == 'MIMIC'\")\n",
    "    \n",
    "    mid_val = len(df) // 2\n",
    "    \n",
    "    df.loc[:mid_val, \"env\"] = \"0\"\n",
    "    df.loc[mid_val:, \"env\"] = \"1\"\n",
    "    \n",
    "    print(\"Total DF Length:\", len(df))\n",
    "    \n",
    "    print(df.head())\n",
    "    print(df.tail())\n",
    "    \n",
    "    print(\"Count per class:\", df.groupby(\"env\")[\"emb0\"].count())\n",
    "    \n",
    "    X, y = df.drop(columns=\"env\"), df[\"env\"]\n",
    "    \n",
    "    clf = make_pipeline(StandardScaler(), LinearSVC(random_state=0, tol=1e-5, max_iter=100000, verbose=1))\n",
    "    \n",
    "    clf.fit(X, y)\n",
    "    \n",
    "    print(clf.score(X, y))\n",
    "    \n",
    "run_test(ml_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b570db2b-c13b-42cc-9bcd-a46bb49aa0e8",
   "metadata": {},
   "source": [
    "# 2 - Parse Embedding Training Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5734c2c5-2581-47e8-82be-aa202703c838",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/scratch/rc4499/thesis/output/(CXP,)-test(MIMIC)-shift(None)-bal(None,None)-27040988\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Name                       (CXP,)-test(MIMIC)-shift(None)-bal(None,None)\n",
       "train_env_0                                                          CXP\n",
       "train_env_1                                                          NaN\n",
       "binary_label                                                  No Finding\n",
       "seed                                                                   0\n",
       "es_f1                                                           0.719991\n",
       "ID                                                              n7nowxuc\n",
       "job_id                                                          27040988\n",
       "State                                                           finished\n",
       "Job Type                                                      1_baseline\n",
       "Created                                         2022-11-16T08:02:12.000Z\n",
       "Runtime                                                             6528\n",
       "orig-MIMIC-trn_len                                                194213\n",
       "MIMIC-trn_len                                                     194213\n",
       "orig-MIMIC-trn_prop                                             0.347567\n",
       "MIMIC-trn_prop                                                  0.347567\n",
       "orig-CXP-trn_len                                                  153400\n",
       "CXP-trn_len                                                       153400\n",
       "orig-CXP-trn_prop                                               0.089804\n",
       "CXP-trn_prop                                                    0.089804\n",
       "orig-NIH-trn_len                                                   89568\n",
       "NIH-trn_len                                                        89568\n",
       "orig-NIH-trn_prop                                               0.536509\n",
       "NIH-trn_prop                                                    0.536509\n",
       "orig-PAD-trn_len                                                   80069\n",
       "PAD-trn_len                                                        80069\n",
       "orig-PAD-trn_prop                                               0.361176\n",
       "PAD-trn_prop                                                    0.361176\n",
       "balance_method                                                       NaN\n",
       "train_envs                                                           CXP\n",
       "resample_method                                                      NaN\n",
       "img_size                                                             224\n",
       "output_dir             /scratch/rc4499/thesis/output/(CXP,)-test(MIMI...\n",
       "test.MIMIC_f1                                                   0.703672\n",
       "test.MIMIC_f1_bin                                               0.565599\n",
       "test.CXP_f1                                                     0.705676\n",
       "test.CXP_f1_bin                                                 0.462321\n",
       "test.NIH_f1                                                     0.683572\n",
       "test.NIH_f1_bin                                                 0.674811\n",
       "test.PAD_f1                                                     0.727028\n",
       "test.PAD_f1_bin                                                 0.683662\n",
       "test.MIMIC_roc                                                   0.84031\n",
       "test.CXP_roc                                                    0.868514\n",
       "test.NIH_roc                                                     0.74448\n",
       "test.PAD_roc                                                    0.825034\n",
       "test.MIMIC_acc                                                  0.768007\n",
       "test.CXP_acc                                                    0.906888\n",
       "test.NIH_acc                                                    0.683815\n",
       "test.PAD_acc                                                    0.733917\n",
       "test.MIMIC_prec_bin                                             0.807021\n",
       "test.MIMIC_rec_bin                                              0.435361\n",
       "test.CXP_prec_bin                                               0.457667\n",
       "test.CXP_rec_bin                                                0.467069\n",
       "test.NIH_prec_bin                                               0.760801\n",
       "test.NIH_rec_bin                                                0.606286\n",
       "test.PAD_prec_bin                                               0.596118\n",
       "test.PAD_rec_bin                                                0.801347\n",
       "es_opt_thresh                                                        0.4\n",
       "Name: 20, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row_idx = 20\n",
    "\n",
    "row = df.iloc[row_idx]\n",
    "\n",
    "print(row.output_dir)\n",
    "row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "03a93580-6479-4d69-9756-7d6abd22c9f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>train_env_0</th>\n",
       "      <th>train_env_1</th>\n",
       "      <th>binary_label</th>\n",
       "      <th>seed</th>\n",
       "      <th>es_f1</th>\n",
       "      <th>ID</th>\n",
       "      <th>job_id</th>\n",
       "      <th>State</th>\n",
       "      <th>Job Type</th>\n",
       "      <th>...</th>\n",
       "      <th>test.MIMIC_prec_bin</th>\n",
       "      <th>test.MIMIC_rec_bin</th>\n",
       "      <th>test.CXP_prec_bin</th>\n",
       "      <th>test.CXP_rec_bin</th>\n",
       "      <th>test.NIH_prec_bin</th>\n",
       "      <th>test.NIH_rec_bin</th>\n",
       "      <th>test.PAD_prec_bin</th>\n",
       "      <th>test.PAD_rec_bin</th>\n",
       "      <th>es_opt_thresh</th>\n",
       "      <th>emb_results</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(CXP,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.714091</td>\n",
       "      <td>c2ub14qr</td>\n",
       "      <td>27039924</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.364995</td>\n",
       "      <td>0.353306</td>\n",
       "      <td>0.518785</td>\n",
       "      <td>0.485522</td>\n",
       "      <td>0.329213</td>\n",
       "      <td>0.457098</td>\n",
       "      <td>0.306122</td>\n",
       "      <td>0.306122</td>\n",
       "      <td>0.29</td>\n",
       "      <td>{'single_label_acc': 0.4597479952886929, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NIH</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.719801</td>\n",
       "      <td>8dr2o2tb</td>\n",
       "      <td>27039926</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.379154</td>\n",
       "      <td>0.259298</td>\n",
       "      <td>0.500491</td>\n",
       "      <td>0.527404</td>\n",
       "      <td>0.457082</td>\n",
       "      <td>0.332293</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.183673</td>\n",
       "      <td>0.33</td>\n",
       "      <td>{'single_label_acc': 0.514287492657534, 'label...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>PAD</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.727782</td>\n",
       "      <td>qvmz92ez</td>\n",
       "      <td>27039921</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.363122</td>\n",
       "      <td>0.331612</td>\n",
       "      <td>0.496194</td>\n",
       "      <td>0.539297</td>\n",
       "      <td>0.332740</td>\n",
       "      <td>0.291732</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.265306</td>\n",
       "      <td>0.17</td>\n",
       "      <td>{'single_label_acc': 0.5130444629227122, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.699785</td>\n",
       "      <td>k2nenx8j</td>\n",
       "      <td>27039933</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.335703</td>\n",
       "      <td>0.439050</td>\n",
       "      <td>0.332840</td>\n",
       "      <td>0.581179</td>\n",
       "      <td>0.346687</td>\n",
       "      <td>0.351014</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.102041</td>\n",
       "      <td>0.15</td>\n",
       "      <td>{'single_label_acc': 0.5187337072422323, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>CXP</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.728350</td>\n",
       "      <td>xcos9u5l</td>\n",
       "      <td>27039925</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.395455</td>\n",
       "      <td>0.449380</td>\n",
       "      <td>0.481245</td>\n",
       "      <td>0.583764</td>\n",
       "      <td>0.383769</td>\n",
       "      <td>0.435257</td>\n",
       "      <td>0.295082</td>\n",
       "      <td>0.367347</td>\n",
       "      <td>0.23</td>\n",
       "      <td>{'single_label_acc': 0.5023510759290241, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>(MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NIH</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.782981</td>\n",
       "      <td>79zle1pd</td>\n",
       "      <td>27032544</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.747722</td>\n",
       "      <td>0.860343</td>\n",
       "      <td>0.808696</td>\n",
       "      <td>0.918881</td>\n",
       "      <td>0.582698</td>\n",
       "      <td>0.660704</td>\n",
       "      <td>0.696129</td>\n",
       "      <td>0.519845</td>\n",
       "      <td>0.42</td>\n",
       "      <td>{'single_label_acc': 0.49601488288609774, 'lab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>(MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.797440</td>\n",
       "      <td>6k6vvrol</td>\n",
       "      <td>27032541</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.747550</td>\n",
       "      <td>0.866619</td>\n",
       "      <td>0.809577</td>\n",
       "      <td>0.907282</td>\n",
       "      <td>0.508246</td>\n",
       "      <td>0.753429</td>\n",
       "      <td>0.662451</td>\n",
       "      <td>0.723037</td>\n",
       "      <td>0.41</td>\n",
       "      <td>{'single_label_acc': 0.4948974594021861, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>(NIH,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.732541</td>\n",
       "      <td>xfbh7izc</td>\n",
       "      <td>27032534</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.731447</td>\n",
       "      <td>0.807277</td>\n",
       "      <td>0.811347</td>\n",
       "      <td>0.855601</td>\n",
       "      <td>0.538858</td>\n",
       "      <td>0.748360</td>\n",
       "      <td>0.649215</td>\n",
       "      <td>0.534944</td>\n",
       "      <td>0.22</td>\n",
       "      <td>{'single_label_acc': 0.4952684628999458, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>(NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.756103</td>\n",
       "      <td>qdvkvcia</td>\n",
       "      <td>27032543</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.708218</td>\n",
       "      <td>0.835875</td>\n",
       "      <td>0.820862</td>\n",
       "      <td>0.827852</td>\n",
       "      <td>0.539066</td>\n",
       "      <td>0.757007</td>\n",
       "      <td>0.644436</td>\n",
       "      <td>0.729508</td>\n",
       "      <td>0.25</td>\n",
       "      <td>{'single_label_acc': 0.4929569577615811, 'labe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>(PAD,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>PAD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.784636</td>\n",
       "      <td>6tcn4hwc</td>\n",
       "      <td>27032550</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>0.728937</td>\n",
       "      <td>0.705195</td>\n",
       "      <td>0.857627</td>\n",
       "      <td>0.520482</td>\n",
       "      <td>0.512081</td>\n",
       "      <td>0.720334</td>\n",
       "      <td>0.667634</td>\n",
       "      <td>0.694996</td>\n",
       "      <td>0.23</td>\n",
       "      <td>{'single_label_acc': 0.5004807208995008, 'labe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Name train_env_0 train_env_1  \\\n",
       "0       (CXP,)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NaN   \n",
       "1    (CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NIH   \n",
       "2    (CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)         CXP         PAD   \n",
       "3     (MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)       MIMIC         NaN   \n",
       "4   (MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         CXP   \n",
       "..                                                ...         ...         ...   \n",
       "85  (MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         NIH   \n",
       "86  (MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         PAD   \n",
       "87      (NIH,)-test(MIMIC)-shift(None)-bal(None,None)         NIH         NaN   \n",
       "88   (NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)         NIH         PAD   \n",
       "89      (PAD,)-test(MIMIC)-shift(None)-bal(None,None)         PAD         NaN   \n",
       "\n",
       "    binary_label  seed     es_f1        ID    job_id     State    Job Type  \\\n",
       "0   Pneumothorax     0  0.714091  c2ub14qr  27039924  finished  1_baseline   \n",
       "1   Pneumothorax     0  0.719801  8dr2o2tb  27039926  finished  1_baseline   \n",
       "2   Pneumothorax     0  0.727782  qvmz92ez  27039921  finished  1_baseline   \n",
       "3   Pneumothorax     0  0.699785  k2nenx8j  27039933  finished  1_baseline   \n",
       "4   Pneumothorax     0  0.728350  xcos9u5l  27039925  finished  1_baseline   \n",
       "..           ...   ...       ...       ...       ...       ...         ...   \n",
       "85           All     0  0.782981  79zle1pd  27032544  finished  1_baseline   \n",
       "86           All     0  0.797440  6k6vvrol  27032541  finished  1_baseline   \n",
       "87           All     0  0.732541  xfbh7izc  27032534  finished  1_baseline   \n",
       "88           All     0  0.756103  qdvkvcia  27032543  finished  1_baseline   \n",
       "89           All     0  0.784636  6tcn4hwc  27032550  finished  1_baseline   \n",
       "\n",
       "    ... test.MIMIC_prec_bin  test.MIMIC_rec_bin  test.CXP_prec_bin  \\\n",
       "0   ...            0.364995            0.353306           0.518785   \n",
       "1   ...            0.379154            0.259298           0.500491   \n",
       "2   ...            0.363122            0.331612           0.496194   \n",
       "3   ...            0.335703            0.439050           0.332840   \n",
       "4   ...            0.395455            0.449380           0.481245   \n",
       "..  ...                 ...                 ...                ...   \n",
       "85  ...            0.747722            0.860343           0.808696   \n",
       "86  ...            0.747550            0.866619           0.809577   \n",
       "87  ...            0.731447            0.807277           0.811347   \n",
       "88  ...            0.708218            0.835875           0.820862   \n",
       "89  ...            0.728937            0.705195           0.857627   \n",
       "\n",
       "    test.CXP_rec_bin  test.NIH_prec_bin  test.NIH_rec_bin  test.PAD_prec_bin  \\\n",
       "0           0.485522           0.329213          0.457098           0.306122   \n",
       "1           0.527404           0.457082          0.332293           0.428571   \n",
       "2           0.539297           0.332740          0.291732           0.393939   \n",
       "3           0.581179           0.346687          0.351014           0.250000   \n",
       "4           0.583764           0.383769          0.435257           0.295082   \n",
       "..               ...                ...               ...                ...   \n",
       "85          0.918881           0.582698          0.660704           0.696129   \n",
       "86          0.907282           0.508246          0.753429           0.662451   \n",
       "87          0.855601           0.538858          0.748360           0.649215   \n",
       "88          0.827852           0.539066          0.757007           0.644436   \n",
       "89          0.520482           0.512081          0.720334           0.667634   \n",
       "\n",
       "    test.PAD_rec_bin  es_opt_thresh  \\\n",
       "0           0.306122           0.29   \n",
       "1           0.183673           0.33   \n",
       "2           0.265306           0.17   \n",
       "3           0.102041           0.15   \n",
       "4           0.367347           0.23   \n",
       "..               ...            ...   \n",
       "85          0.519845           0.42   \n",
       "86          0.723037           0.41   \n",
       "87          0.534944           0.22   \n",
       "88          0.729508           0.25   \n",
       "89          0.694996           0.23   \n",
       "\n",
       "                                          emb_results  \n",
       "0   {'single_label_acc': 0.4597479952886929, 'labe...  \n",
       "1   {'single_label_acc': 0.514287492657534, 'label...  \n",
       "2   {'single_label_acc': 0.5130444629227122, 'labe...  \n",
       "3   {'single_label_acc': 0.5187337072422323, 'labe...  \n",
       "4   {'single_label_acc': 0.5023510759290241, 'labe...  \n",
       "..                                                ...  \n",
       "85  {'single_label_acc': 0.49601488288609774, 'lab...  \n",
       "86  {'single_label_acc': 0.4948974594021861, 'labe...  \n",
       "87  {'single_label_acc': 0.4952684628999458, 'labe...  \n",
       "88  {'single_label_acc': 0.4929569577615811, 'labe...  \n",
       "89  {'single_label_acc': 0.5004807208995008, 'labe...  \n",
       "\n",
       "[90 rows x 59 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def convert(results, key):\n",
    "    df = pd.DataFrame(results)\n",
    "    single_val_rows = df[df[\"pred_val\"].str.len() == 1]\n",
    "    \n",
    "    single_val_acc = single_val_rows.acc.mean()\n",
    "    \n",
    "    multi_val_rows = df[df[\"pred_val\"].str.len() >= 2]\n",
    "    \n",
    "    row_dict = {f\"single_{key}_acc\": single_val_acc}\n",
    "    for _, row in multi_val_rows.iterrows():\n",
    "        pred_val = \",\".join([str(x) for x in row.pred_val])\n",
    "        dict_key = f\"{key}_{pred_val}_{row.fix_val}\"\n",
    "        row_dict[f\"{dict_key}_acc\"] = row.acc\n",
    "        row_dict[f\"{dict_key}_len\"] = row.df_size\n",
    "        # Get coefficients out\n",
    "        row_dict[f\"{dict_key}_coef\"] = row.coef\n",
    "    \n",
    "    return row_dict\n",
    "\n",
    "import json\n",
    "\n",
    "def extract_main(row):\n",
    "    results_dict_path = f\"{row.output_dir}/emb_test_results_coef.json\"\n",
    "    \n",
    "    try:\n",
    "        with open(results_dict_path) as f:\n",
    "            results_dict = json.load(f)\n",
    "    except Exception as e:\n",
    "        return {}\n",
    "        \n",
    "    # env_pred_df = pd.DataFrame(results_dict[\"env_pred\"])\n",
    "    # label_pred_df = pd.DataFrame(results_dict[\"label_pred\"])\n",
    "    row_dict = {}\n",
    "    row_dict.update(convert(results_dict[\"label_pred\"], \"label\"))\n",
    "    row_dict.update(convert(results_dict[\"env_pred\"], \"env\"))\n",
    "    \n",
    "    return row_dict\n",
    "\n",
    "df[\"emb_results\"] = df.apply(extract_main, axis=1)\n",
    "df\n",
    "# extract_main(row)\n",
    "# label_pred_df[\"pred_val\"].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ac9d03b4-f29f-401c-9db6-09f77f631ea6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>train_env_0</th>\n",
       "      <th>train_env_1</th>\n",
       "      <th>binary_label</th>\n",
       "      <th>seed</th>\n",
       "      <th>es_f1</th>\n",
       "      <th>ID</th>\n",
       "      <th>job_id</th>\n",
       "      <th>State</th>\n",
       "      <th>Job Type</th>\n",
       "      <th>...</th>\n",
       "      <th>env_MIMIC,NIH_1_coef</th>\n",
       "      <th>env_MIMIC,PAD_1_acc</th>\n",
       "      <th>env_MIMIC,PAD_1_len</th>\n",
       "      <th>env_MIMIC,PAD_1_coef</th>\n",
       "      <th>env_NIH,PAD_1_acc</th>\n",
       "      <th>env_NIH,PAD_1_len</th>\n",
       "      <th>env_NIH,PAD_1_coef</th>\n",
       "      <th>env_CXP,MIMIC,NIH,PAD_1_acc</th>\n",
       "      <th>env_CXP,MIMIC,NIH,PAD_1_len</th>\n",
       "      <th>env_CXP,MIMIC,NIH,PAD_1_coef</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(CXP,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.714091</td>\n",
       "      <td>c2ub14qr</td>\n",
       "      <td>27039924</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.005256527418282763, -0.00924869023743721, ...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[-0.006265830233764563, 0.005642347998679152,...</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.004647425836362778, 0.0052440106903279705,...</td>\n",
       "      <td>0.925000</td>\n",
       "      <td>196</td>\n",
       "      <td>[[0.006324464965272984, 0.029136769808494393, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>NIH</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.719801</td>\n",
       "      <td>8dr2o2tb</td>\n",
       "      <td>27039926</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[-0.009902525034324207, 0.012665668545215707,...</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[-0.005814294272023986, 0.018762227719207974,...</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.004211760677588215, 0.009943577366169078, ...</td>\n",
       "      <td>0.925000</td>\n",
       "      <td>196</td>\n",
       "      <td>[[0.0031619048729606375, -0.013679478472567301...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>CXP</td>\n",
       "      <td>PAD</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.727782</td>\n",
       "      <td>qvmz92ez</td>\n",
       "      <td>27039921</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, 0.00016563287665611812, 0.0241397397607...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, -0.0034404295594303988, 0.0136016880056...</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, -0.005128896448979347, 0.00497298184186...</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>196</td>\n",
       "      <td>[[0.0, -0.004616143149733463, -0.0082155582712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.699785</td>\n",
       "      <td>k2nenx8j</td>\n",
       "      <td>27039933</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, -0.001959716435394835, 0.01294514366212...</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, -0.007236304973804693, 0.01298263381709...</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, -0.013780975326034546, -0.0075937957266...</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>196</td>\n",
       "      <td>[[0.0, 0.03653691437288172, 0.0074711491011369...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>CXP</td>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>0</td>\n",
       "      <td>0.728350</td>\n",
       "      <td>xcos9u5l</td>\n",
       "      <td>27039925</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, 0.0127624992119481, 0.00745361325309276...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, 0.00583283447724515, -0.003130491669885...</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>98</td>\n",
       "      <td>[[0.0, -0.007406990964977571, -0.0076130520773...</td>\n",
       "      <td>0.925000</td>\n",
       "      <td>196</td>\n",
       "      <td>[[0.0, 0.014224403066599759, -0.01132285809548...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>(MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>NIH</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.782981</td>\n",
       "      <td>79zle1pd</td>\n",
       "      <td>27032544</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[-0.0011189870328174136, 0.005705315384278041...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.005911982525605574, 0.00983284460614124, -...</td>\n",
       "      <td>0.966595</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[-0.013771272302760611, -0.006457639385987544...</td>\n",
       "      <td>0.958491</td>\n",
       "      <td>9272</td>\n",
       "      <td>[[-0.06428134053742168, -0.026448361711687865,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>(MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...</td>\n",
       "      <td>MIMIC</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.797440</td>\n",
       "      <td>6k6vvrol</td>\n",
       "      <td>27032541</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, 0.011653308352520746, -0.00111354381296...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0, 0.030753264275056483, -0.00198842710246...</td>\n",
       "      <td>0.948276</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0, -0.02222587818400612, 0.016381002586556...</td>\n",
       "      <td>0.947170</td>\n",
       "      <td>9272</td>\n",
       "      <td>[[0.0, 0.1612512559240793, -0.1838569081075190...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>(NIH,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.732541</td>\n",
       "      <td>xfbh7izc</td>\n",
       "      <td>27032534</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, 0.01650976145270204, 0.0147109709710174...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0, 0.011982438415704379, 0.002291842186967...</td>\n",
       "      <td>0.970905</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0, -0.043517766084606134, 0.03530515872668...</td>\n",
       "      <td>0.965499</td>\n",
       "      <td>9272</td>\n",
       "      <td>[[0.0, 0.04294427487360091, 0.0989272880416249...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>(NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>NIH</td>\n",
       "      <td>PAD</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.756103</td>\n",
       "      <td>qdvkvcia</td>\n",
       "      <td>27032543</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.01511092895376117, 0.04668931806488876, -0...</td>\n",
       "      <td>0.997845</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.018895594936006576, 0.04569684800878787, -...</td>\n",
       "      <td>0.975216</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.06966767082958969, 0.04459568478383912, -0...</td>\n",
       "      <td>0.967116</td>\n",
       "      <td>9272</td>\n",
       "      <td>[[-0.09289703225092409, -0.01871920551253282, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>(PAD,)-test(MIMIC)-shift(None)-bal(None,None)</td>\n",
       "      <td>PAD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0.784636</td>\n",
       "      <td>6tcn4hwc</td>\n",
       "      <td>27032550</td>\n",
       "      <td>finished</td>\n",
       "      <td>1_baseline</td>\n",
       "      <td>...</td>\n",
       "      <td>[[0.0, 0.025913550511208305, -0.00763752921855...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0, 0.04105170739515691, -0.016961163385524...</td>\n",
       "      <td>0.948276</td>\n",
       "      <td>4636</td>\n",
       "      <td>[[0.0030406910109971124, 0.07245017060052873, ...</td>\n",
       "      <td>0.953639</td>\n",
       "      <td>9272</td>\n",
       "      <td>[[0.02728790714647353, 0.04130202500870749, -0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90 rows × 114 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Name train_env_0 train_env_1  \\\n",
       "0       (CXP,)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NaN   \n",
       "1    (CXP,NIH)-test(MIMIC)-shift(None)-bal(None,None)         CXP         NIH   \n",
       "2    (CXP,PAD)-test(MIMIC)-shift(None)-bal(None,None)         CXP         PAD   \n",
       "3     (MIMIC,)-test(MIMIC)-shift(None)-bal(None,None)       MIMIC         NaN   \n",
       "4   (MIMIC,CXP)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         CXP   \n",
       "..                                                ...         ...         ...   \n",
       "85  (MIMIC,NIH)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         NIH   \n",
       "86  (MIMIC,PAD)-test(MIMIC)-shift(None)-bal(None,N...       MIMIC         PAD   \n",
       "87      (NIH,)-test(MIMIC)-shift(None)-bal(None,None)         NIH         NaN   \n",
       "88   (NIH,PAD)-test(MIMIC)-shift(None)-bal(None,None)         NIH         PAD   \n",
       "89      (PAD,)-test(MIMIC)-shift(None)-bal(None,None)         PAD         NaN   \n",
       "\n",
       "    binary_label  seed     es_f1        ID    job_id     State    Job Type  \\\n",
       "0   Pneumothorax     0  0.714091  c2ub14qr  27039924  finished  1_baseline   \n",
       "1   Pneumothorax     0  0.719801  8dr2o2tb  27039926  finished  1_baseline   \n",
       "2   Pneumothorax     0  0.727782  qvmz92ez  27039921  finished  1_baseline   \n",
       "3   Pneumothorax     0  0.699785  k2nenx8j  27039933  finished  1_baseline   \n",
       "4   Pneumothorax     0  0.728350  xcos9u5l  27039925  finished  1_baseline   \n",
       "..           ...   ...       ...       ...       ...       ...         ...   \n",
       "85           All     0  0.782981  79zle1pd  27032544  finished  1_baseline   \n",
       "86           All     0  0.797440  6k6vvrol  27032541  finished  1_baseline   \n",
       "87           All     0  0.732541  xfbh7izc  27032534  finished  1_baseline   \n",
       "88           All     0  0.756103  qdvkvcia  27032543  finished  1_baseline   \n",
       "89           All     0  0.784636  6tcn4hwc  27032550  finished  1_baseline   \n",
       "\n",
       "    ...                               env_MIMIC,NIH_1_coef  \\\n",
       "0   ...  [[0.005256527418282763, -0.00924869023743721, ...   \n",
       "1   ...  [[-0.009902525034324207, 0.012665668545215707,...   \n",
       "2   ...  [[0.0, 0.00016563287665611812, 0.0241397397607...   \n",
       "3   ...  [[0.0, -0.001959716435394835, 0.01294514366212...   \n",
       "4   ...  [[0.0, 0.0127624992119481, 0.00745361325309276...   \n",
       "..  ...                                                ...   \n",
       "85  ...  [[-0.0011189870328174136, 0.005705315384278041...   \n",
       "86  ...  [[0.0, 0.011653308352520746, -0.00111354381296...   \n",
       "87  ...  [[0.0, 0.01650976145270204, 0.0147109709710174...   \n",
       "88  ...  [[0.01511092895376117, 0.04668931806488876, -0...   \n",
       "89  ...  [[0.0, 0.025913550511208305, -0.00763752921855...   \n",
       "\n",
       "    env_MIMIC,PAD_1_acc  env_MIMIC,PAD_1_len  \\\n",
       "0              1.000000                   98   \n",
       "1              0.900000                   98   \n",
       "2              1.000000                   98   \n",
       "3              0.950000                   98   \n",
       "4              1.000000                   98   \n",
       "..                  ...                  ...   \n",
       "85             1.000000                 4636   \n",
       "86             1.000000                 4636   \n",
       "87             1.000000                 4636   \n",
       "88             0.997845                 4636   \n",
       "89             1.000000                 4636   \n",
       "\n",
       "                                 env_MIMIC,PAD_1_coef  env_NIH,PAD_1_acc  \\\n",
       "0   [[-0.006265830233764563, 0.005642347998679152,...           0.900000   \n",
       "1   [[-0.005814294272023986, 0.018762227719207974,...           0.900000   \n",
       "2   [[0.0, -0.0034404295594303988, 0.0136016880056...           0.800000   \n",
       "3   [[0.0, -0.007236304973804693, 0.01298263381709...           0.800000   \n",
       "4   [[0.0, 0.00583283447724515, -0.003130491669885...           0.850000   \n",
       "..                                                ...                ...   \n",
       "85  [[0.005911982525605574, 0.00983284460614124, -...           0.966595   \n",
       "86  [[0.0, 0.030753264275056483, -0.00198842710246...           0.948276   \n",
       "87  [[0.0, 0.011982438415704379, 0.002291842186967...           0.970905   \n",
       "88  [[0.018895594936006576, 0.04569684800878787, -...           0.975216   \n",
       "89  [[0.0, 0.04105170739515691, -0.016961163385524...           0.948276   \n",
       "\n",
       "    env_NIH,PAD_1_len                                 env_NIH,PAD_1_coef  \\\n",
       "0                  98  [[0.004647425836362778, 0.0052440106903279705,...   \n",
       "1                  98  [[0.004211760677588215, 0.009943577366169078, ...   \n",
       "2                  98  [[0.0, -0.005128896448979347, 0.00497298184186...   \n",
       "3                  98  [[0.0, -0.013780975326034546, -0.0075937957266...   \n",
       "4                  98  [[0.0, -0.007406990964977571, -0.0076130520773...   \n",
       "..                ...                                                ...   \n",
       "85               4636  [[-0.013771272302760611, -0.006457639385987544...   \n",
       "86               4636  [[0.0, -0.02222587818400612, 0.016381002586556...   \n",
       "87               4636  [[0.0, -0.043517766084606134, 0.03530515872668...   \n",
       "88               4636  [[0.06966767082958969, 0.04459568478383912, -0...   \n",
       "89               4636  [[0.0030406910109971124, 0.07245017060052873, ...   \n",
       "\n",
       "    env_CXP,MIMIC,NIH,PAD_1_acc  env_CXP,MIMIC,NIH,PAD_1_len  \\\n",
       "0                      0.925000                          196   \n",
       "1                      0.925000                          196   \n",
       "2                      0.875000                          196   \n",
       "3                      0.875000                          196   \n",
       "4                      0.925000                          196   \n",
       "..                          ...                          ...   \n",
       "85                     0.958491                         9272   \n",
       "86                     0.947170                         9272   \n",
       "87                     0.965499                         9272   \n",
       "88                     0.967116                         9272   \n",
       "89                     0.953639                         9272   \n",
       "\n",
       "                         env_CXP,MIMIC,NIH,PAD_1_coef  \n",
       "0   [[0.006324464965272984, 0.029136769808494393, ...  \n",
       "1   [[0.0031619048729606375, -0.013679478472567301...  \n",
       "2   [[0.0, -0.004616143149733463, -0.0082155582712...  \n",
       "3   [[0.0, 0.03653691437288172, 0.0074711491011369...  \n",
       "4   [[0.0, 0.014224403066599759, -0.01132285809548...  \n",
       "..                                                ...  \n",
       "85  [[-0.06428134053742168, -0.026448361711687865,...  \n",
       "86  [[0.0, 0.1612512559240793, -0.1838569081075190...  \n",
       "87  [[0.0, 0.04294427487360091, 0.0989272880416249...  \n",
       "88  [[-0.09289703225092409, -0.01871920551253282, ...  \n",
       "89  [[0.02728790714647353, 0.04130202500870749, -0...  \n",
       "\n",
       "[90 rows x 114 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_comb = pd.concat([df.drop(columns=\"emb_results\"), df[\"emb_results\"].apply(pd.Series)], axis=1)\n",
    "df_comb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "30021b5c-96d8-4913-a5bf-447cb20da075",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAit0lEQVR4nO3de3BU9f3G8WeTLBuCiYiYZIP5xYAIWCxS0IAgF5UgdSiIHbVRBhyvNTLSDMPEImVBiYoW6QzCjJShXhqhLaJOoZB4CbdIFUpmBBEBg6CQUhASILpsyPf3h5MMMeGyyTnJl933a2Yns2fPfvfzkGX3mbNJjscYYwQAAGCxmLYeAAAA4HwoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA68W19QA/VVtbqwMHDigxMVEej6etxwEAABfAGKPjx48rLS1NMTHOHw+xrrAcOHBA6enpbT0GAABohv379+vKK690fF3rCktiYqKkHwMnJSU5vn4oFFJRUZGys7Pl9XodX98mZI1M0ZI1WnJKZI1U0ZK1LufAgQOVmZlZ/z7uNOsKS93HQElJSa4VloSEBCUlJUX0E0gia6SKlqzRklMia6SKlqx1OeuKils/zsEP3QIAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYL66tBwDQfFflr2zrEcK29/k72noEABchjrAAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD1OfgigVV3oCRt9sUZzbpR6B9YoeNrj8lTnx0kbgbbFERYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHphFZbnnntON9xwgxITE5WcnKyxY8dq586dDfaZOHGiPB5Pg8uAAQMcHRoAAESXsArL2rVrlZubq02bNqm4uFg1NTXKzs7WyZMnG+x3++236+DBg/WXVatWOTo0AACILmH9af7Vq1c3uL5kyRIlJydry5YtGjJkSP12n8+n1NRUZyYEAABRr0XnEqqsrJQkderUqcH2kpISJScnq2PHjho6dKhmz56t5OTkJtcIBoMKBoP116uqqiRJoVBIoVCoJeM1qW5NN9a2DVkj05lZfbGmjadxjy/GNPja1tx8bkXr8zfSRUvW1srpMcY069XAGKMxY8bo6NGjWr9+ff32ZcuW6ZJLLlFGRobKy8s1ffp01dTUaMuWLfL5fI3WCQQCmjlzZqPthYWFSkhIaM5oAACglVVXVysnJ0eVlZVKSkpyfP1mF5bc3FytXLlSGzZs0JVXXnnW/Q4ePKiMjAwtXbpU48aNa3R7U0dY0tPTdfjwYVcCh0IhFRcXa8SIEfJ6vY6vbxOyRqYzs/ad/WFbj+MaX4zRM/1rNX1zjIK1bX+25m2Bka6tHa3PX7JGhrqcWVlZ8vv9rhWWZn0kNGnSJL333ntat27dOcuKJPn9fmVkZGjXrl1N3u7z+Zo88uL1el39Bru9vk3IGpm8Xq+Cp9v+jdxtwVqPFTlb43kVbc9fskYWtzOGVViMMZo0aZJWrFihkpISZWZmnvc+R44c0f79++X3+5s9JAAAiG5h/Vpzbm6u3nzzTRUWFioxMVEVFRWqqKjQ999/L0k6ceKEpkyZoo8//lh79+5VSUmJRo8erc6dO+vOO+90JQAAAIh8YR1hWbhwoSRp2LBhDbYvWbJEEydOVGxsrD777DO9/vrrOnbsmPx+v4YPH65ly5YpMTHRsaEBAEB0CfsjoXNp37691qxZ06KBAAAAfopzCQEAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWC+swvLcc8/phhtuUGJiopKTkzV27Fjt3LmzwT7GGAUCAaWlpal9+/YaNmyYtm/f7ujQAAAguoRVWNauXavc3Fxt2rRJxcXFqqmpUXZ2tk6ePFm/z5w5czR37lzNnz9fn376qVJTUzVixAgdP37c8eEBAEB0iAtn59WrVze4vmTJEiUnJ2vLli0aMmSIjDGaN2+epk2bpnHjxkmSXnvtNaWkpKiwsFCPPvqoc5MDAICo0aKfYamsrJQkderUSZJUXl6uiooKZWdn1+/j8/k0dOhQlZaWtuShAABAFAvrCMuZjDHKy8vT4MGD1bt3b0lSRUWFJCklJaXBvikpKfr666+bXCcYDCoYDNZfr6qqkiSFQiGFQqHmjndWdWu6sbZtyBqZzszqizVtPI17fDGmwde25uZzK1qfv5EuWrK2Vk6PMaZZrwa5ublauXKlNmzYoCuvvFKSVFpaqkGDBunAgQPy+/31+z788MPav39/o4+UJCkQCGjmzJmNthcWFiohIaE5owEAgFZWXV2tnJwcVVZWKikpyfH1m3WEZdKkSXrvvfe0bt26+rIiSampqZJ+PNJyZmE5dOhQo6MudZ566inl5eXVX6+qqlJ6erqys7NdCRwKhVRcXKwRI0bI6/U6vr5NyBqZzszad/aHbT2Oa3wxRs/0r9X0zTEK1nraehxtC4x0be1off6SNTLU5Rw+fLirjxNWYTHGaNKkSVqxYoVKSkqUmZnZ4PbMzEylpqaquLhYffv2lSSdOnVKa9eu1QsvvNDkmj6fTz6fr9F2r9fr6jfY7fVtQtbI5PV6FTzd9m/kbgvWeqzI2RrPq2h7/pI1sridMazCkpubq8LCQr377rtKTEys/5mVSy+9VO3bt5fH49HkyZNVUFCg7t27q3v37iooKFBCQoJycnJcCQAAACJfWIVl4cKFkqRhw4Y12L5kyRJNnDhRkjR16lR9//33evzxx3X06FFlZWWpqKhIiYmJjgwMAACiT9gfCZ2Px+NRIBBQIBBo7kwAAAANcC4hAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1wi4s69at0+jRo5WWliaPx6N33nmnwe0TJ06Ux+NpcBkwYIBT8wIAgCgUdmE5efKk+vTpo/nz5591n9tvv10HDx6sv6xatapFQwIAgOgWF+4dRo0apVGjRp1zH5/Pp9TU1GYPBQAAcKawC8uFKCkpUXJysjp27KihQ4dq9uzZSk5ObnLfYDCoYDBYf72qqkqSFAqFFAqFHJ+tbk031rYNWSPTmVl9saaNp3GPL8Y0+NrW3HxuRevzN9JFS9bWyukxxjT71cDj8WjFihUaO3Zs/bZly5bpkksuUUZGhsrLyzV9+nTV1NRoy5Yt8vl8jdYIBAKaOXNmo+2FhYVKSEho7mgAAKAVVVdXKycnR5WVlUpKSnJ8fccLy08dPHhQGRkZWrp0qcaNG9fo9qaOsKSnp+vw4cOuBA6FQiouLtaIESPk9XodX98mZI1MZ2btO/vDth7HNb4Yo2f612r65hgFaz1tPY62BUa6tna0Pn/JGhnqcmZlZcnv97tWWFz5SOhMfr9fGRkZ2rVrV5O3+3y+Jo+8eL1eV7/Bbq9vE7JGJq/Xq+Dptn8jd1uw1mNFztZ4XkXb85eskcXtjK7/HZYjR45o//798vv9bj8UAACIUGEfYTlx4oR2795df728vFxlZWXq1KmTOnXqpEAgoLvuukt+v1979+7V73//e3Xu3Fl33nmno4MDAIDoEXZh2bx5s4YPH15/PS8vT5I0YcIELVy4UJ999plef/11HTt2TH6/X8OHD9eyZcuUmJjo3NQAACCqhF1Yhg0bpnP9nO6aNWtaNBAAAMBPcS4hAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWC+urQcAgIvBVfkrXVvbF2s050apd2CNgqc9jq279/k7HFsLaGscYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANaLa+sBAFtclb+yrUe4IL5Yozk3Sr0DayR52nocAGgVHGEBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKwXdmFZt26dRo8erbS0NHk8Hr3zzjsNbjfGKBAIKC0tTe3bt9ewYcO0fft2p+YFAABRKOzCcvLkSfXp00fz589v8vY5c+Zo7ty5mj9/vj799FOlpqZqxIgROn78eIuHBQAA0SnsP80/atQojRo1qsnbjDGaN2+epk2bpnHjxkmSXnvtNaWkpKiwsFCPPvpoy6YFAABRydFzCZWXl6uiokLZ2dn123w+n4YOHarS0tImC0swGFQwGKy/XlVVJUkKhUIKhUJOjle/7plfIxlZw+OLNU6N4ypfjGnwNVJFS07Jvaw2/t/ndSnytFZOjzGm2f9DPB6PVqxYobFjx0qSSktLNWjQIH377bdKS0ur3++RRx7R119/rTVr1jRaIxAIaObMmY22FxYWKiEhobmjAQCAVlRdXa2cnBxVVlYqKSnJ8fVdOVuzx9PwDLLGmEbb6jz11FPKy8urv15VVaX09HRlZ2e7EjgUCqm4uFgjRoyQ1+t1fH2bkDU8P5792H6+GKNn+tdq+uYYBWsj92zN0ZJTci/rtsBIx9ZyCq9Lkacu5/Dhw119HEcLS2pqqiSpoqJCfr+/fvuhQ4eUkpLS5H18Pp98Pl+j7V6v19VvsNvr24SsFyZ4+uJ6UwzWei66mZsjWnJKzme1+f89r0uRx+2Mjv4dlszMTKWmpqq4uLh+26lTp7R27VrddNNNTj4UAACIImEfYTlx4oR2795df728vFxlZWXq1KmT/u///k+TJ09WQUGBunfvru7du6ugoEAJCQnKyclxdHAAABA9wi4smzdvbvA5Vd3Pn0yYMEF/+ctfNHXqVH3//fd6/PHHdfToUWVlZamoqEiJiYnOTQ0AAKJK2IVl2LBhOtcvFnk8HgUCAQUCgZbMBQAAUI9zCQEAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALBeXFsPAABwx1X5K9t6hEZ8sUZzbpR6B9YoeNrT6Pa9z9/RBlPhYsARFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAeo4XlkAgII/H0+CSmprq9MMAAIAo4spfuv3Zz36m999/v/56bGysGw8DAACihCuFJS4ujqMqAADAMa4Ull27diktLU0+n09ZWVkqKChQ165dm9w3GAwqGAzWX6+qqpIkhUIhhUIhx2erW9ONtW1D1vD4Yo1T47jKF2MafI1U0ZJTIuuZIun1Klpeg1srp8cY4+j/kH/961+qrq7WNddco//+97969tln9cUXX2j79u26/PLLG+0fCAQ0c+bMRtsLCwuVkJDg5GgAAMAl1dXVysnJUWVlpZKSkhxf3/HC8lMnT55Ut27dNHXqVOXl5TW6vakjLOnp6Tp8+LArgUOhkIqLizVixAh5vV7H17dJW2btHVjTqo/nizF6pn+tpm+OUbC28RlgI0m0ZI2WnBJZz7QtMLINpnJHtLzf1OXMysqS3+93rbC48pHQmTp06KDrrrtOu3btavJ2n88nn8/XaLvX63X1G+z2+jZpi6xNnTa+VR631tNmj93aoiVrtOSUyCopIl+Xo+X9xu2Mrv8dlmAwqB07dsjv97v9UAAAIEI5XlimTJmitWvXqry8XP/+97/161//WlVVVZowYYLTDwUAAKKE4x8JffPNN/rNb36jw4cP64orrtCAAQO0adMmZWRkOP1QAAAgSjheWJYuXer0kgAAIMpxLiEAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWo7AAAADrUVgAAID1KCwAAMB6FBYAAGA9CgsAALAehQUAAFiPwgIAAKxHYQEAANajsAAAAOtRWAAAgPUoLAAAwHoUFgAAYD0KCwAAsB6FBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADWi2vrAVpb78Aazbnxx6/B0562HueC7H3+jrYeAQBwFlflr2xyuy/WWPt+czG+r3CEBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwnmuFZcGCBcrMzFR8fLz69eun9evXu/VQAAAgwrlSWJYtW6bJkydr2rRp2rp1q26++WaNGjVK+/btc+PhAABAhHOlsMydO1cPPvigHnroIfXq1Uvz5s1Tenq6Fi5c6MbDAQCACOf4uYROnTqlLVu2KD8/v8H27OxslZaWNto/GAwqGAzWX6+srJQkfffddwqFQk6Pp7jQSVVX1youFKPTtXad2+Fsjhw50qz7hUIhVVdX68iRI/J6vQ5PdW5xNSdb9/FqzUX3fW2uaMkaLTklsp6pua93belsr3c2f1+d/Heue6/57rvvJEnGGMfWbsA47NtvvzWSzMaNGxtsnz17trnmmmsa7T9jxgwjiQsXLly4cOESAZf9+/c7XS2MMca4drZmj6dhmzTGNNomSU899ZTy8vLqr9fW1uq7777T5Zdf3uT+LVVVVaX09HTt379fSUlJjq9vE7JGpmjJGi05JbJGqmjJWpdz37598ng8SktLc+VxHC8snTt3VmxsrCoqKhpsP3TokFJSUhrt7/P55PP5Gmzr2LGj02M1kpSUFNFPoDORNTJFS9ZoySmRNVJFS9ZLL73U1ZyO/9Btu3bt1K9fPxUXFzfYXlxcrJtuusnphwMAAFHAlY+E8vLyNH78ePXv318DBw7Uq6++qn379umxxx5z4+EAAECEc6Ww3HPPPTpy5IhmzZqlgwcPqnfv3lq1apUyMjLceLiw+Hw+zZgxo9HHUJGIrJEpWrJGS06JrJEqWrK2Vk6PMW79/hEAAIAzOJcQAACwHoUFAABYj8ICAACsR2EBAADWi4jCsmDBAmVmZio+Pl79+vXT+vXrL+h+GzduVFxcnK6//vpGtx07dky5ubny+/2Kj49Xr169tGrVKocnD5/TWYcNGyaPx9Pocscdd7gw/YVz43s6b9489ejRQ+3bt1d6erp+97vf6YcffnB48vA5nTUUCmnWrFnq1q2b4uPj1adPH61evdqFycMXTtaSkpImn5tffPFFg/2WL1+ua6+9Vj6fT9dee61WrFjhdozzcjrn9u3bddddd+mqq66Sx+PRvHnzWiHFhXE666JFi3TzzTfrsssu02WXXabbbrtNn3zySWtEOS+ns7799tvq37+/OnbsqA4dOuj666/XG2+80RpRzsuN/6t1li5dKo/Ho7Fjx4Y3lCt/8L8VLV261Hi9XrNo0SLz+eefmyeffNJ06NDBfP311+e837Fjx0zXrl1Ndna26dOnT4PbgsGg6d+/v/nlL39pNmzYYPbu3WvWr19vysrKXExyfm5kPXLkiDl48GD9Zdu2bSY2NtYsWbLEvSDn4UbON9980/h8PvPXv/7VlJeXmzVr1hi/328mT57sYpLzcyPr1KlTTVpamlm5cqXZs2ePWbBggYmPjzf/+c9/XExyfuFm/eijj4wks3PnzgbP0Zqamvp9SktLTWxsrCkoKDA7duwwBQUFJi4uzmzatKm1YjXiRs5PPvnETJkyxbz11lsmNTXVvPzyy62U5tzcyJqTk2NeeeUVs3XrVrNjxw7zwAMPmEsvvdR88803rRWrSW5k/eijj8zbb79tPv/8c7N7924zb948Exsba1avXt1asZrkRtY6e/fuNV26dDE333yzGTNmTFhzXfSF5cYbbzSPPfZYg209e/Y0+fn557zfPffcY55++mkzY8aMRi/4CxcuNF27djWnTp1yetwWcSPrT7388ssmMTHRnDhxoqXjNpsbOXNzc80tt9zSYFteXp4ZPHiwIzM3lxtZ/X6/mT9/foNtY8aMMffdd58jMzdXuFnrXgSPHj161jXvvvtuc/vttzfYNnLkSHPvvfe2eN7mciPnmTIyMqwpLG5nNcaYmpoak5iYaF577bWWjNpirZHVGGP69u1rnn766eaO6Qi3stbU1JhBgwaZP//5z2bChAlhF5aL+iOhU6dOacuWLcrOzm6wPTs7W6WlpWe935IlS7Rnzx7NmDGjydvfe+89DRw4ULm5uUpJSVHv3r1VUFCg06dPOzp/ONzK+lOLFy/Wvffeqw4dOrRo3uZyK+fgwYO1ZcuW+kPLX331lVatWtWmH325lTUYDCo+Pr7Btvbt22vDhg0tH7qZmptVkvr27Su/369bb71VH330UYPbPv7440Zrjhw58rxrusWtnDZqrazV1dUKhULq1KlTi2durtbIaozRBx98oJ07d2rIkCGOzN0cbmadNWuWrrjiCj344IPNms21szW3hsOHD+v06dONTqqYkpLS6OSLdXbt2qX8/HytX79ecXFNx//qq6/04Ycf6r777tOqVau0a9cu5ebmqqamRn/4wx8cz3Eh3Mp6pk8++UTbtm3T4sWLHZm5OdzKee+99+p///ufBg8eLGOMampq9Nvf/lb5+fmOZ7hQbmUdOXKk5s6dqyFDhqhbt2764IMP9O6777Zp4W5OVr/fr1dffVX9+vVTMBjUG2+8oVtvvVUlJSX1L+gVFRVhrek2t3LaqLWy5ufnq0uXLrrtttscz3Ch3MxaWVmpLl26KBgMKjY2VgsWLNCIESNczXMubmXduHGjFi9erLKysmbPdlEXljoej6fBdWNMo22SdPr0aeXk5GjmzJm65pprzrpebW2tkpOT9eqrryo2Nlb9+vXTgQMH9OKLL7ZZYanjdNYzLV68WL1799aNN97oyKwt4XTOkpISzZ49WwsWLFBWVpZ2796tJ598Un6/X9OnT3d8/nA4nfVPf/qTHn74YfXs2VMej0fdunXTAw88oCVLljg+e7guNKsk9ejRQz169Ki/PnDgQO3fv18vvfRSgxf8cNZsLW7ktJWbWefMmaO33npLJSUljY4atgU3siYmJqqsrEwnTpzQBx98oLy8PHXt2lXDhg1zJcOFcjLr8ePHdf/992vRokXq3Llzs2e6qAtL586dFRsb26j1HTp0qFE7lKTjx49r8+bN2rp1q5544glJP5YTY4zi4uJUVFSkW265RX6/X16vV7GxsfX37dWrlyoqKnTq1Cm1a9fO3WBNcCtrnerqai1dulSzZs1yN8h5uJVz+vTpGj9+vB566CFJ0nXXXaeTJ0/qkUce0bRp0xQT0/qfjrqV9YorrtA777yjH374QUeOHFFaWpry8/OVmZnZKrmaEm7WsxkwYIDefPPN+uupqaktXtNJbuW0kdtZX3rpJRUUFOj999/Xz3/+8xbP2xJuZo2JidHVV18tSbr++uu1Y8cOPffcc21WWNzIumfPHu3du1ejR4+uv722tlaSFBcXp507d6pbt27nXfOi/hmWdu3aqV+/fiouLm6wvbi4WDfddFOj/ZOSkvTZZ5+prKys/vLYY4+pR48eKisrU1ZWliRp0KBB2r17d/0/qCR9+eWX8vv9bVJWJPey1vnb3/6mYDCo+++/39Uc5+NWzurq6kalJDY2VubHHzx3L9A5uP09jY+PV5cuXVRTU6Ply5drzJgxruY5l3Czns3WrVvl9/vrrw8cOLDRmkVFRWGt6SS3ctrIzawvvviinnnmGa1evVr9+/d3ZN6WaM3vqzFGwWCwWXM6wY2sPXv2bPTa9atf/UrDhw9XWVmZ0tPTL2zRsH5E10J1v361ePFi8/nnn5vJkyebDh06mL179xpjjMnPzzfjx48/6/2b+i2Lffv2mUsuucQ88cQTZufOneaf//ynSU5ONs8++6ybUc7Ljax1Bg8ebO655x43xg6bGzlnzJhhEhMTzVtvvWW++uorU1RUZLp162buvvtuN6OclxtZN23aZJYvX2727Nlj1q1bZ2655RaTmZkZ9m8rOC3crC+//LJZsWKF+fLLL822bdtMfn6+kWSWL19ev8/GjRtNbGysef75582OHTvM888/b82vNTuZMxgMmq1bt5qtW7cav99vpkyZYrZu3Wp27drV6vnO5EbWF154wbRr18784x//aPArssePH2/1fGdyI2tBQYEpKioye/bsMTt27DB//OMfTVxcnFm0aFGr5zuTG1l/qjm/JXTRFxZjjHnllVdMRkaGadeunfnFL35h1q5dW3/bhAkTzNChQ89637O9iZeWlpqsrCzj8/lM165dzezZs5v8nfLW5kbWnTt3GkmmqKjIhYmbx+mcoVDIBAIB061bNxMfH2/S09PN448/3uZv4sY4n7WkpMT06tXL+Hw+c/nll5vx48ebb7/91qXpwxNO1hdeeKH++3XZZZeZwYMHm5UrVzZa8+9//7vp0aOH8Xq9pmfPnud8kWwtTucsLy83khpdzvXcaC1OZ83IyGgy64wZM1op0dk5nXXatGnm6quvrt9n4MCBZunSpa0V55zc+L96puYUFo8xbXQ8HAAA4AJd1D/DAgAAogOFBQAAWI/CAgAArEdhAQAA1qOwAAAA61FYAACA9SgsAADAehQWAABgPQoLAACwHoUFAABYj8ICAACsR2EBAADW+3/VKRwp6lIVCwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_comb[\"single_label_acc\"].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c8b66bf2-fabe-472e-89cd-87208bc60def",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'single_label_acc': 0.5130444629227122,\n",
       " 'label_0,1_CXP_acc': 0.7131782945736435,\n",
       " 'label_0,1_CXP_len': 3868,\n",
       " 'label_0,1_MIMIC_acc': 0.7036082474226805,\n",
       " 'label_0,1_MIMIC_len': 1936,\n",
       " 'label_0,1_NIH_acc': 0.7003891050583657,\n",
       " 'label_0,1_NIH_len': 1282,\n",
       " 'label_0,1_PAD_acc': 0.8,\n",
       " 'label_0,1_PAD_len': 98}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "convert(results_dict[\"label_pred\"], \"label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c16b8910-865f-4f5d-a4d7-bcdc8d744288",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_comb.to_csv(\"1a_baseline_EVAL.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "301996eb-b2dd-429a-92e9-dc1659fdfa95",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_comb.to_csv(\"1a_baseline_EVAL_coef.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "thesis",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 (default, Mar 28 2022, 11:38:47) \n[GCC 7.5.0]"
  },
  "vscode": {
   "interpreter": {
    "hash": "770583d413ddd366163ce940894a7aadba90c4b700c3ba64411cd727eada5441"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}