federated_forest.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Federated Forest\n",
    "\n",
    "This file imports the fully preprocessed data (preprocessing IV) and applies sklearn's random forest classifiers on each of the ethnicities (representing local models). These models are stored and aggregated in the federated model (global model) such that each ethnicity is equally weighted. Consequently, the performance of the local models and the global model is evaluated using accuracy, recall, precision and AUC (ROC)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ethnic_group_names = ['unknown', 'white', 'other', 'asian', 'hispanic_latino', 'black_african_american', 'unable_to_obtain', 'american_indian_alaska_native']\n",
    "path = './data/preprocessing_IV/'\n",
    "\n",
    "def read_train_val_test(name, path):\n",
    "    train = pd.read_csv(path + name + '_train.csv')\n",
    "    # NOTE: drope datetime columns for now, because sklearn does not support it\n",
    "    train = train.drop(columns=['admittime', 'edregtime', 'emar_charttime', 'lab_charttime'])\n",
    "    \n",
    "    validate = pd.read_csv(path + name + '_validate.csv')\n",
    "    # NOTE: drope datetime columns for now, because sklearn does not support it\n",
    "    validate = validate.drop(columns=['admittime', 'edregtime', 'emar_charttime', 'lab_charttime'])\n",
    "    \n",
    "    test = pd.read_csv(path + name + '_test.csv')\n",
    "    # NOTE: drope datetime columns for now, because sklearn does not support it\n",
    "    test = test.drop(columns=['admittime', 'edregtime', 'emar_charttime', 'lab_charttime'])\n",
    "\n",
    "    return train, validate, test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Train each local model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# split features and labels\n",
    "def splitFeaturesLabels(df):\n",
    "    X = df.copy()\n",
    "    X = X.drop('has_kidney_issue', axis=1)\n",
    "    \n",
    "    return X, df.has_kidney_issue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def trainModel(name, path, all_params):\n",
    "    # get data and optimal parameters\n",
    "    train, val, test = read_train_val_test(name, path)\n",
    "    hyperparams = all_params.loc[all_params['ethnicity']==name]\n",
    "    X_train, y_train = splitFeaturesLabels(train)\n",
    "    \n",
    "    # train a model with the specified params\n",
    "    clf = RandomForestClassifier(n_estimators=int(hyperparams['n_estimators']),\n",
    "                                 max_depth=int(hyperparams['max_depth']),\n",
    "                                 random_state=42)\n",
    "    clf.fit(X_train, y_train)\n",
    "    \n",
    "    return clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "models = []\n",
    "params = pd.read_csv('hyperparameters.csv')\n",
    "for eth in ethnic_group_names:\n",
    "    models.append(trainModel(eth, path, params))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Aggregate models into federated forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def aggregateForests(agg, model, n):\n",
    "    # Aggregate the estimators\n",
    "    size = model.n_estimators\n",
    "    dupl = math.floor(n/size)\n",
    "    for i in range(dupl):\n",
    "        agg.estimators_ += model.estimators_\n",
    "        agg.n_estimators = len(agg.estimators_)\n",
    "        \n",
    "    return agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def getMaxForest(model_list):\n",
    "    # normalize over the maximum number of trees so each model is weighted the same in the aggregated model\n",
    "    sizes = [len(model.estimators_) for model in model_list]\n",
    "    \n",
    "    return max(sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def federatedForest(model_list):\n",
    "    # Aggregate all models and normalize\n",
    "    ff = model_list[0]\n",
    "    max_size = getMaxForest(model_list)\n",
    "    for i, model in enumerate(model_list):\n",
    "        if i > 0:\n",
    "            ff = aggregateForests(ff, model_list[i], max_size)\n",
    "            \n",
    "    return ff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "fed = federatedForest(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Model evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def evaluateMetrics(model, X, y):\n",
    "    y_pr = model.predict(X)\n",
    "    accuracy = accuracy_score(y, y_pr)\n",
    "    precision = precision_score(y, y_pr)\n",
    "    recall = recall_score(y, y_pr)\n",
    "    auc = roc_auc_score(y, y_pr)\n",
    "    \n",
    "    return accuracy, precision, recall, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def printMetrics(model, ethnicity, path):\n",
    "    train, val, test = read_train_val_test(ethnicity, path)\n",
    "    X, y = splitFeaturesLabels(test)\n",
    "    acc, prec, rec, auc = evaluateMetrics(model, X, y)\n",
    "    \n",
    "    print('Local model:\\t\\t' + ethnicity)\n",
    "    print('Accuracy: \\t\\t\\t' + str(acc))\n",
    "    print('Precision: \\t\\t\\t' + str(prec))\n",
    "    print('Recall: \\t\\t\\t' + str(rec))\n",
    "    print('AUC: \\t\\t\\t\\t' + str(auc) + '\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printFedMetrics(model, ethnicity, path):\n",
    "    train, val, test = read_train_val_test(ethnicity, path)\n",
    "    X, y = splitFeaturesLabels(test)\n",
    "    acc, prec, rec, auc = evaluateMetrics(model, X, y)\n",
    "    \n",
    "    print('Federated model: \\t' + ethnicity)\n",
    "    print('Accuracy: \\t\\t\\t' + str(acc))\n",
    "    print('Precision: \\t\\t\\t' + str(prec))\n",
    "    print('Recall: \\t\\t\\t' + str(rec))\n",
    "    print('AUC: \\t\\t\\t\\t' + str(auc) + '\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metricsDF(model_type, model, ethnicity, path):\n",
    "    train, val, test = read_train_val_test(ethnicity, path)\n",
    "    X, y = splitFeaturesLabels(test)\n",
    "    acc, prec, rec, auc = evaluateMetrics(model, X, y)\n",
    "    \n",
    "    return [model_type, ethnicity, round(acc, 4), round(prec,4), round(rec, 4), round(auc, 4)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Local model:\t\tunknown\n",
      "Accuracy: \t\t\t0.8851699279093718\n",
      "Precision: \t\t\t0.6855895196506551\n",
      "Recall: \t\t\t0.5097402597402597\n",
      "AUC: \t\t\t\t0.7328383061247199\n",
      "\n",
      "\n",
      "Federated model: \tunknown\n",
      "Accuracy: \t\t\t0.8748712667353244\n",
      "Precision: \t\t\t0.6006191950464397\n",
      "Recall: \t\t\t0.6298701298701299\n",
      "AUC: \t\t\t\t0.7754613807245386\n",
      "\n",
      "\n",
      "Local model:\t\twhite\n",
      "Accuracy: \t\t\t0.8144894095373328\n",
      "Precision: \t\t\t0.7039790118058592\n",
      "Recall: \t\t\t0.3957232395231658\n",
      "AUC: \t\t\t\t0.6714853389340611\n",
      "\n",
      "\n",
      "Federated model: \twhite\n",
      "Accuracy: \t\t\t0.8155543722636375\n",
      "Precision: \t\t\t0.6778900112233446\n",
      "Recall: \t\t\t0.445372987587563\n",
      "AUC: \t\t\t\t0.689141474897923\n",
      "\n",
      "\n",
      "Local model:\t\tother\n",
      "Accuracy: \t\t\t0.8319672131147541\n",
      "Precision: \t\t\t0.5949367088607594\n",
      "Recall: \t\t\t0.2842741935483871\n",
      "AUC: \t\t\t\t0.6201992539954001\n",
      "\n",
      "\n",
      "Federated model: \tother\n",
      "Accuracy: \t\t\t0.8438897168405365\n",
      "Precision: \t\t\t0.6222222222222222\n",
      "Recall: \t\t\t0.3951612903225806\n",
      "AUC: \t\t\t\t0.6703868608834109\n",
      "\n",
      "\n",
      "Local model:\t\tasian\n",
      "Accuracy: \t\t\t0.8715334420880914\n",
      "Precision: \t\t\t0.6263345195729537\n",
      "Recall: \t\t\t0.45595854922279794\n",
      "AUC: \t\t\t\t0.7025678515717088\n",
      "\n",
      "\n",
      "Federated model: \tasian\n",
      "Accuracy: \t\t\t0.8752039151712887\n",
      "Precision: \t\t\t0.6694915254237288\n",
      "Recall: \t\t\t0.40932642487046633\n",
      "AUC: \t\t\t\t0.6857861553200348\n",
      "\n",
      "\n",
      "Local model:\t\thispanic_latino\n",
      "Accuracy: \t\t\t0.8340582134493142\n",
      "Precision: \t\t\t0.6884615384615385\n",
      "Recall: \t\t\t0.30134680134680136\n",
      "AUC: \t\t\t\t0.6337631710283068\n",
      "\n",
      "\n",
      "Federated model: \thispanic_latino\n",
      "Accuracy: \t\t\t0.8390766142522583\n",
      "Precision: \t\t\t0.7116104868913857\n",
      "Recall: \t\t\t0.31986531986531985\n",
      "AUC: \t\t\t\t0.6438575033564595\n",
      "\n",
      "\n",
      "Local model:\t\tblack_african_american\n",
      "Accuracy: \t\t\t0.8230473115609089\n",
      "Precision: \t\t\t0.7173373075012419\n",
      "Recall: \t\t\t0.6278260869565218\n",
      "AUC: \t\t\t\t0.7644605838919581\n",
      "\n",
      "\n",
      "Federated model: \tblack_african_american\n",
      "Accuracy: \t\t\t0.7936172854836707\n",
      "Precision: \t\t\t0.8097087378640777\n",
      "Recall: \t\t\t0.3626086956521739\n",
      "AUC: \t\t\t\t0.6642697571777296\n",
      "\n",
      "\n",
      "Local model:\t\tunable_to_obtain\n",
      "Accuracy: \t\t\t0.9144385026737968\n",
      "Precision: \t\t\t0.4666666666666667\n",
      "Recall: \t\t\t0.22580645161290322\n",
      "AUC: \t\t\t\t0.6012414182262766\n",
      "\n",
      "\n",
      "Federated model: \tunable_to_obtain\n",
      "Accuracy: \t\t\t0.9144385026737968\n",
      "Precision: \t\t\t0.4782608695652174\n",
      "Recall: \t\t\t0.3548387096774194\n",
      "AUC: \t\t\t\t0.6599266434684473\n",
      "\n",
      "\n",
      "Local model:\t\tamerican_indian_alaska_native\n",
      "Accuracy: \t\t\t0.9155844155844156\n",
      "Precision: \t\t\t0.75\n",
      "Recall: \t\t\t0.7777777777777778\n",
      "AUC: \t\t\t\t0.8613298337707787\n",
      "\n",
      "\n",
      "Federated model: \tamerican_indian_alaska_native\n",
      "Accuracy: \t\t\t0.8961038961038961\n",
      "Precision: \t\t\t0.7894736842105263\n",
      "Recall: \t\t\t0.5555555555555556\n",
      "AUC: \t\t\t\t0.7620297462817148\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dfs = []\n",
    "columns = ['Model', 'Ethnicity', 'Accuracy', 'Precision', 'Recall', 'AUC']\n",
    "\n",
    "for i in range(len(models)):\n",
    "    # Since 'unknown' is overwritten as the aggregated model, redefine it\n",
    "    if i == 0:\n",
    "        unknown = trainModel('unknown', path, params)\n",
    "        printMetrics(unknown, ethnic_group_names[i], path)\n",
    "        dfs.append(metricsDF('Local', unknown, ethnic_group_names[i], path))\n",
    "    else:\n",
    "        printMetrics(models[i], ethnic_group_names[i], path)\n",
    "        dfs.append(metricsDF('Local', models[i], ethnic_group_names[i], path))\n",
    "        \n",
    "    printFedMetrics(fed, ethnic_group_names[i], path)\n",
    "\n",
    "    dfs.append(metricsDF('Federated', fed, ethnic_group_names[i], path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\annik\\AppData\\Local\\Temp\\ipykernel_9272\\1199555007.py:3: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.loc['Local mean'] = df[df['Model']=='Local'].mean()\n",
      "C:\\Users\\annik\\AppData\\Local\\Temp\\ipykernel_9272\\1199555007.py:4: FutureWarning: Dropping of nuisance columns in DataFrame reductions (with 'numeric_only=None') is deprecated; in a future version this will raise TypeError.  Select only valid columns before calling the reduction.\n",
      "  df.loc['Federated mean'] = df[df['Model']=='Federated'].mean()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Local</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.885200</td>\n",
       "      <td>0.685600</td>\n",
       "      <td>0.509700</td>\n",
       "      <td>0.732800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Federated</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.874900</td>\n",
       "      <td>0.600600</td>\n",
       "      <td>0.629900</td>\n",
       "      <td>0.775500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Local</td>\n",
       "      <td>white</td>\n",
       "      <td>0.814500</td>\n",
       "      <td>0.704000</td>\n",
       "      <td>0.395700</td>\n",
       "      <td>0.671500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Federated</td>\n",
       "      <td>white</td>\n",
       "      <td>0.815600</td>\n",
       "      <td>0.677900</td>\n",
       "      <td>0.445400</td>\n",
       "      <td>0.689100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Local</td>\n",
       "      <td>other</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.594900</td>\n",
       "      <td>0.284300</td>\n",
       "      <td>0.620200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Federated</td>\n",
       "      <td>other</td>\n",
       "      <td>0.843900</td>\n",
       "      <td>0.622200</td>\n",
       "      <td>0.395200</td>\n",
       "      <td>0.670400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Local</td>\n",
       "      <td>asian</td>\n",
       "      <td>0.871500</td>\n",
       "      <td>0.626300</td>\n",
       "      <td>0.456000</td>\n",
       "      <td>0.702600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Federated</td>\n",
       "      <td>asian</td>\n",
       "      <td>0.875200</td>\n",
       "      <td>0.669500</td>\n",
       "      <td>0.409300</td>\n",
       "      <td>0.685800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Local</td>\n",
       "      <td>hispanic_latino</td>\n",
       "      <td>0.834100</td>\n",
       "      <td>0.688500</td>\n",
       "      <td>0.301300</td>\n",
       "      <td>0.633800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Federated</td>\n",
       "      <td>hispanic_latino</td>\n",
       "      <td>0.839100</td>\n",
       "      <td>0.711600</td>\n",
       "      <td>0.319900</td>\n",
       "      <td>0.643900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Local</td>\n",
       "      <td>black_african_american</td>\n",
       "      <td>0.823000</td>\n",
       "      <td>0.717300</td>\n",
       "      <td>0.627800</td>\n",
       "      <td>0.764500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Federated</td>\n",
       "      <td>black_african_american</td>\n",
       "      <td>0.793600</td>\n",
       "      <td>0.809700</td>\n",
       "      <td>0.362600</td>\n",
       "      <td>0.664300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Local</td>\n",
       "      <td>unable_to_obtain</td>\n",
       "      <td>0.914400</td>\n",
       "      <td>0.466700</td>\n",
       "      <td>0.225800</td>\n",
       "      <td>0.601200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Federated</td>\n",
       "      <td>unable_to_obtain</td>\n",
       "      <td>0.914400</td>\n",
       "      <td>0.478300</td>\n",
       "      <td>0.354800</td>\n",
       "      <td>0.659900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Local</td>\n",
       "      <td>american_indian_alaska_native</td>\n",
       "      <td>0.915600</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.777800</td>\n",
       "      <td>0.861300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Federated</td>\n",
       "      <td>american_indian_alaska_native</td>\n",
       "      <td>0.896100</td>\n",
       "      <td>0.789500</td>\n",
       "      <td>0.555600</td>\n",
       "      <td>0.762000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Local mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.861287</td>\n",
       "      <td>0.654162</td>\n",
       "      <td>0.447300</td>\n",
       "      <td>0.698487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Federated mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.856600</td>\n",
       "      <td>0.669913</td>\n",
       "      <td>0.434088</td>\n",
       "      <td>0.693863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Model                      Ethnicity  Accuracy  Precision  \\\n",
       "0                   Local                        unknown  0.885200   0.685600   \n",
       "1               Federated                        unknown  0.874900   0.600600   \n",
       "2                   Local                          white  0.814500   0.704000   \n",
       "3               Federated                          white  0.815600   0.677900   \n",
       "4                   Local                          other  0.832000   0.594900   \n",
       "5               Federated                          other  0.843900   0.622200   \n",
       "6                   Local                          asian  0.871500   0.626300   \n",
       "7               Federated                          asian  0.875200   0.669500   \n",
       "8                   Local                hispanic_latino  0.834100   0.688500   \n",
       "9               Federated                hispanic_latino  0.839100   0.711600   \n",
       "10                  Local         black_african_american  0.823000   0.717300   \n",
       "11              Federated         black_african_american  0.793600   0.809700   \n",
       "12                  Local               unable_to_obtain  0.914400   0.466700   \n",
       "13              Federated               unable_to_obtain  0.914400   0.478300   \n",
       "14                  Local  american_indian_alaska_native  0.915600   0.750000   \n",
       "15              Federated  american_indian_alaska_native  0.896100   0.789500   \n",
       "Local mean            NaN                            NaN  0.861287   0.654162   \n",
       "Federated mean        NaN                            NaN  0.856600   0.669913   \n",
       "\n",
       "                  Recall       AUC  \n",
       "0               0.509700  0.732800  \n",
       "1               0.629900  0.775500  \n",
       "2               0.395700  0.671500  \n",
       "3               0.445400  0.689100  \n",
       "4               0.284300  0.620200  \n",
       "5               0.395200  0.670400  \n",
       "6               0.456000  0.702600  \n",
       "7               0.409300  0.685800  \n",
       "8               0.301300  0.633800  \n",
       "9               0.319900  0.643900  \n",
       "10              0.627800  0.764500  \n",
       "11              0.362600  0.664300  \n",
       "12              0.225800  0.601200  \n",
       "13              0.354800  0.659900  \n",
       "14              0.777800  0.861300  \n",
       "15              0.555600  0.762000  \n",
       "Local mean      0.447300  0.698487  \n",
       "Federated mean  0.434088  0.693863  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add results to dataframe\n",
    "df = pd.DataFrame(dfs, columns=columns)\n",
    "df.loc['Local mean'] = df[df['Model']=='Local'].mean()\n",
    "df.loc['Federated mean'] = df[df['Model']=='Federated'].mean()\n",
    "\n",
    "# store as csv\n",
    "df.to_csv('./metrics.csv', index=False)\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('sklearn-env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "34e7f5b862476ca5b132d6fa3a551e06a58745cd7626854c77bc546bf08a4906"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}