book_disbalance_public_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "93d3633a-cdd1-423d-8643-3aadb8a47325",
   "metadata": {},
   "source": [
    "# Эксперименты с дисбалансом классов\n",
    "\n",
    "2020, Дьяконов Александр (https://dyakonov.org/ag/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e005d44f-1098-401a-98c0-0ba76b310db1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# подгружаем все нужные пакеты\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score, cohen_kappa_score, f1_score, log_loss, matthews_corrcoef, roc_auc_score, average_precision_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.datasets import make_moons\n",
    "from imblearn.datasets import make_imbalance\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_circles\n",
    "\n",
    "# за какими показателями качества следить\n",
    "scores = [accuracy_score, balanced_accuracy_score, cohen_kappa_score, f1_score, matthews_corrcoef]\n",
    "scores2 = [log_loss, roc_auc_score, average_precision_score]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cac57796-5444-43b7-b598-8a99a68e6dbe",
   "metadata": {},
   "source": [
    "### Полезные функции"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b414360e-a244-4c94-92a3-b252e854d622",
   "metadata": {},
   "outputs": [],
   "source": [
    "def imbalancing(X, y, p=0.1):\n",
    "    \"\"\"\n",
    "    Внести дисбаланс в данные\n",
    "    \"\"\"\n",
    "    I = (y <= 0) | (np.random.rand(X.shape[0]) < p) \n",
    "    X = X[I, :] # прореживание\n",
    "    y = y[I]\n",
    "    return (X, y)\n",
    "\n",
    "from imblearn import FunctionSampler\n",
    "def func(X, y):\n",
    "    \"\"\"\n",
    "    Тождественный сэмлер\n",
    "    \"\"\"\n",
    "    return X, y\n",
    "\n",
    "def investigate_model(model, X, y, cv, X_test, y_test):\n",
    "    \"\"\"\n",
    "    оценить стратегию подбора порога\n",
    "    \"\"\"\n",
    "    df = pd.DataFrame(index=['train', 'test0', 'cv', 'test', 'theta'])\n",
    "    # ф-ии качества бинаризации\n",
    "    scores = [accuracy_score, balanced_accuracy_score, cohen_kappa_score, f1_score, matthews_corrcoef]\n",
    "    # ф-ии качества оценок\n",
    "    scores2 = [log_loss, roc_auc_score, average_precision_score]\n",
    "    # на cv\n",
    "    a_cv = cross_val_predict(model, X, y, cv=cv, method='predict_proba')[:,1]\n",
    "    model.fit(X, y)\n",
    "    # на обучении\n",
    "    a_train = model.predict_proba(X)[:,1]\n",
    "    b_train = model.predict(X)\n",
    "    # на тесте\n",
    "    a_test = model.predict_proba(X_test)[:,1]\n",
    "    b_test = model.predict(X_test)\n",
    "    \n",
    "    # оптимальный порог по cv\n",
    "    thetas = np.linspace(0, 1, 101)\n",
    "    for s in scores:\n",
    "        q = np.array([s(y, a_cv > t) for t in thetas])\n",
    "        i = np.argmax(q)\n",
    "        t = thetas[i]\n",
    "        s_cv = q[i]\n",
    "        s_test = s(y_test, a_test > t)\n",
    "        sb_train = s(y, b_train)\n",
    "        sb_test = s(y_test, b_test)\n",
    "        print (s.__name__, sb_train, sb_test, s_cv, s_test, t)\n",
    "        df[s.__name__] = [sb_train, sb_test, s_cv, s_test, t]\n",
    "    for s in scores2:\n",
    "        sb_train = s(y, a_train)\n",
    "        sb_test = s(y_test, a_test)\n",
    "        s_cv = s(y, a_cv)\n",
    "        \n",
    "        print (s.__name__, sb_train, sb_test, s_cv)\n",
    "        df[s.__name__] = [sb_train, sb_test, s_cv, sb_test, np.NaN]\n",
    "    return (df)\n",
    "\n",
    "def make_statistics(model, X, y, X_test, y_test):\n",
    "    \"\"\"\n",
    "    Посчитать статистику\n",
    "    \"\"\" \n",
    "    df = pd.DataFrame(index=['train', 'test'])\n",
    "    a = model.predict_proba(X)[:, 1]\n",
    "    b = model.predict(X)\n",
    "    a_test = model.predict_proba(X_test)[:, 1]\n",
    "    b_test = model.predict(X_test)\n",
    "\n",
    "    for s in scores:\n",
    "        sb_test = s(y_test, b_test)\n",
    "        sb_train = s(y, b)\n",
    "        print (s.__name__, sb_train, sb_test)\n",
    "        df[s.__name__] = [sb_train, sb_test]\n",
    "        \n",
    "    for s in scores2:\n",
    "        sb_train = s(y, a)\n",
    "        sb_test = s(y_test, a_test)\n",
    "        #s_cv = s(y, a_cv)\n",
    "        df[s.__name__] = [sb_train, sb_test]\n",
    "        print (s.__name__, sb_train, sb_test)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d2f9d0a-de6e-416f-83fc-392e85df2e09",
   "metadata": {},
   "source": [
    "# LogReg + TwoMoons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5ed34e94-0662-4049-87ec-493384442a76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------ DataSet --------\n",
      "X.shape=(526, 2), y.mean=0.049429657794676805\n",
      "X_test.shape=(529806, 2), y_test.mean=0.05625832852025081\n",
      "------ None --------\n",
      "accuracy_score 0.9771863117870723 0.9687678131240491\n",
      "balanced_accuracy_score 0.7692307692307692 0.7227056126954304\n",
      "cohen_kappa_score 0.6892477353288696 0.6022842497269536\n",
      "f1_score 0.7000000000000001 0.616087793786687\n",
      "matthews_corrcoef 0.7251491888941517 0.6561664944867404\n",
      "log_loss 0.08936455351083333 0.09757047610645972\n",
      "roc_auc_score 0.9610769230769232 0.9626310765617659\n",
      "average_precision_score 0.7757950101877162 0.7890625601162011\n",
      "------ weight_balanced --------\n",
      "accuracy_score 0.8821292775665399 0.8813358097114793\n",
      "balanced_accuracy_score 0.8468461538461538 0.8817444625914246\n",
      "cohen_kappa_score 0.3561048807455378 0.40589639923394794\n",
      "f1_score 0.40384615384615385 0.45548636312457236\n",
      "matthews_corrcoef 0.42310898355047466 0.4778614572799835\n",
      "log_loss 0.2554812209422345 0.2552905595748801\n",
      "roc_auc_score 0.9613076923076923 0.9618829709454473\n",
      "average_precision_score 0.7860861866549161 0.7859703615202439\n",
      "------ Selection of a threshold --------\n",
      "accuracy_score 0.9771863117870723 0.9687678131240491 0.9771863117870723 0.9746190114872236 0.35000000000000003\n",
      "balanced_accuracy_score 0.7692307692307692 0.7227056126954304 0.873 0.8618223339596054 0.03\n",
      "cohen_kappa_score 0.6892477353288696 0.6022842497269536 0.7060514130608335 0.6992707488203829 0.26\n",
      "f1_score 0.7000000000000001 0.616087793786687 0.7199999999999999 0.7148419813456488 0.26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matthews_corrcoef 0.7251491888941517 0.6561664944867404 0.7251491888941517 0.7312322788798225 0.35000000000000003\n",
      "log_loss 0.08936455351083333 0.09757047610645972 0.09203396371918841\n",
      "roc_auc_score 0.9610769230769232 0.9626310765617659 0.9576923076923077\n",
      "average_precision_score 0.7757950101877162 0.7890625601162011 0.758585625992195\n",
      "accuracy_score 0.8821292775665399 0.8813358097114793 0.9790874524714829 0.9742962518355776 0.91\n",
      "balanced_accuracy_score 0.8468461538461538 0.8817444625914246 0.879 0.8646628845869959 0.29\n",
      "cohen_kappa_score 0.3561048807455378 0.40589639923394794 0.7337811723566762 0.7018911591351833 0.91\n",
      "f1_score 0.40384615384615385 0.45548636312457236 0.744186046511628 0.7144714219818006 0.91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n",
      "/home/dash/anaconda3/lib/python3.7/site-packages/sklearn/metrics/_classification.py:873: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matthews_corrcoef 0.42310898355047466 0.4778614572799835 0.7518326869207617 0.7270715961034643 0.91\n",
      "log_loss 0.2554812209422345 0.2552905595748801 0.258477311412552\n",
      "roc_auc_score 0.9613076923076923 0.9618829709454473 0.9566153846153846\n",
      "average_precision_score 0.7860861866549161 0.7859703615202439 0.7766214965623943\n",
      "------ Oversampling methods ------\n",
      "Technique: RandomOverSampler\n",
      "X_resampled=(1000, 2), y_resampled=0.5\n",
      "accuracy_score 0.8821292775665399 0.878368685896347\n",
      "balanced_accuracy_score 0.8468461538461538 0.8769070073810642\n",
      "cohen_kappa_score 0.3561048807455378 0.3968983413323913\n",
      "f1_score 0.40384615384615385 0.4474133273879451\n",
      "matthews_corrcoef 0.42310898355047466 0.46926078902318197\n",
      "log_loss 0.2517942761438466 0.2523349171788454\n",
      "roc_auc_score 0.9603846153846154 0.9604503643561699\n",
      "average_precision_score 0.7877516877681922 0.7821651826096271\n",
      "Technique: SMOTE\n",
      "X_resampled=(1000, 2), y_resampled=0.5\n",
      "accuracy_score 0.8954372623574145 0.8896086492036708\n",
      "balanced_accuracy_score 0.8356153846153846 0.873286493793196\n",
      "cohen_kappa_score 0.3762666551679531 0.41797416548862554\n",
      "f1_score 0.4210526315789474 0.4656275126087274\n",
      "matthews_corrcoef 0.4309837415188403 0.48134078688270815\n",
      "log_loss 0.22720868333575703 0.22883087278770492\n",
      "roc_auc_score 0.9603846153846154 0.9594248375494865\n",
      "average_precision_score 0.7918771619616175 0.7796159968920074\n",
      "Technique: ADASYN\n",
      "X_resampled=(998, 2), y_resampled=0.49899799599198397\n",
      "accuracy_score 0.908745247148289 0.9009750738949729\n",
      "balanced_accuracy_score 0.8608461538461538 0.8821480200630745\n",
      "cohen_kappa_score 0.4263382713805326 0.4504201644029773\n",
      "f1_score 0.46666666666666673 0.49449829456766803\n",
      "matthews_corrcoef 0.47853397544063225 0.5080912899642254\n",
      "log_loss 0.2105933944183684 0.21122373475459613\n",
      "roc_auc_score 0.960923076923077 0.9626258523787157\n",
      "average_precision_score 0.778720238563835 0.7888841404527688\n",
      "------ Undersampling methods ------\n",
      "Technique: RandomUnderSampler\n",
      "X_resampled=(52, 2), y_resampled=0.5\n",
      "accuracy_score 0.8403041825095057 0.8512738625081634\n",
      "balanced_accuracy_score 0.8248461538461538 0.8701398525800175\n",
      "cohen_kappa_score 0.27657344947278795 0.3457812726818805\n",
      "f1_score 0.3333333333333333 0.40276198705413313\n",
      "matthews_corrcoef 0.3589009446159568 0.4324160698769609\n",
      "log_loss 0.34554581810920504 0.3412245615833309\n",
      "roc_auc_score 0.9547692307692308 0.9601938535194255\n",
      "average_precision_score 0.742989451683136 0.7881744970934119\n",
      "Technique: NearMiss\n",
      "X_resampled=(52, 2), y_resampled=0.5\n",
      "accuracy_score 0.908745247148289 0.9129190684892206\n",
      "balanced_accuracy_score 0.8061538461538462 0.7975796291350734\n",
      "cohen_kappa_score 0.38670812281383604 0.4205378857164487\n",
      "f1_score 0.4285714285714286 0.46312286173109596\n",
      "matthews_corrcoef 0.42374488438190394 0.44559330165661604\n",
      "log_loss 0.34167509719227396 0.3442399089842096\n",
      "roc_auc_score 0.9390769230769231 0.9153578121183654\n",
      "average_precision_score 0.703716792597569 0.635584420699456\n",
      "Technique: NearMiss\n",
      "X_resampled=(52, 2), y_resampled=0.5\n",
      "accuracy_score 0.8574144486692015 0.8549336927101618\n",
      "balanced_accuracy_score 0.7973846153846154 0.7695404039455143\n",
      "cohen_kappa_score 0.2816039625596386 0.28294860260003096\n",
      "f1_score 0.336283185840708 0.343074490362836\n",
      "matthews_corrcoef 0.34699904758235145 0.33500078740950245\n",
      "log_loss 0.37689616855729585 0.3799586033780342\n",
      "roc_auc_score 0.904 0.8708986425551903\n",
      "average_precision_score 0.5126277153333255 0.4386204208158697\n",
      "Technique: TomekLinks\n",
      "X_resampled=(526, 2), y_resampled=0.049429657794676805\n",
      "accuracy_score 0.9771863117870723 0.9687678131240491\n",
      "balanced_accuracy_score 0.7692307692307692 0.7227056126954304\n",
      "cohen_kappa_score 0.6892477353288696 0.6022842497269536\n",
      "f1_score 0.7000000000000001 0.616087793786687\n",
      "matthews_corrcoef 0.7251491888941517 0.6561664944867404\n",
      "log_loss 0.08936455351083333 0.09757047610645972\n",
      "roc_auc_score 0.9610769230769232 0.9626310765617659\n",
      "average_precision_score 0.7757950101877162 0.7890625601162011\n",
      "Technique: EditedNearestNeighbours\n",
      "X_resampled=(526, 2), y_resampled=0.049429657794676805\n",
      "accuracy_score 0.9771863117870723 0.9687678131240491\n",
      "balanced_accuracy_score 0.7692307692307692 0.7227056126954304\n",
      "cohen_kappa_score 0.6892477353288696 0.6022842497269536\n",
      "f1_score 0.7000000000000001 0.616087793786687\n",
      "matthews_corrcoef 0.7251491888941517 0.6561664944867404\n",
      "log_loss 0.08936455351083335 0.09757047610645972\n",
      "roc_auc_score 0.9610769230769232 0.9626310765617659\n",
      "average_precision_score 0.7757950101877162 0.7890625601162011\n"
     ]
    }
   ],
   "source": [
    "# модель\n",
    "\n",
    "model = LogisticRegression()\n",
    "model_w = LogisticRegression(class_weight='balanced')\n",
    "\n",
    "# данные\n",
    "\n",
    "X, y = make_moons(n_samples=1000, shuffle=True, noise=0.1, random_state=1)\n",
    "X, y = imbalancing(X, y, p=0.06)\n",
    "X_test, y_test = make_moons(n_samples=1000000, shuffle=True, noise=0.1, random_state=2)\n",
    "X_test, y_test = imbalancing(X_test, y_test, p=0.06)\n",
    "\n",
    "print('------ DataSet --------')\n",
    "print(f'X.shape={X.shape}, y.mean={y.mean()}')\n",
    "print(f'X_test.shape={X_test.shape}, y_test.mean={y_test.mean()}')\n",
    "\n",
    "dfs = [] # для создания отчётности\n",
    "\n",
    "print('------ None --------')\n",
    "\n",
    "model.fit(X, y)\n",
    "dfs.append(make_statistics(model, X, y, X_test, y_test))\n",
    "\n",
    "\n",
    "print('------ weight_balanced --------')\n",
    "\n",
    "model_w.fit(X, y)\n",
    "dfs.append(make_statistics(model_w, X, y, X_test, y_test))\n",
    "\n",
    "print('------ Selection of a threshold --------')\n",
    "\n",
    "dfs.append(investigate_model(model, X, y, 10, X_test, y_test))\n",
    "dfs.append(investigate_model(model_w, X, y, 10, X_test, y_test))\n",
    "\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler, SMOTE, ADASYN\n",
    "from imblearn.under_sampling import RandomUnderSampler, NearMiss, TomekLinks, EditedNearestNeighbours\n",
    "\n",
    "\n",
    "print('------ Oversampling methods ------')\n",
    "techniques = [RandomOverSampler(), # FunctionSampler(func=func),\n",
    "              SMOTE(),\n",
    "              ADASYN()]\n",
    "\n",
    "for sampler in techniques:\n",
    "    technique = sampler.__class__.__name__\n",
    "    print(f'Technique: {technique}')\n",
    "    X_resampled, y_resampled = sampler.fit_resample(X, y)\n",
    "    print(f'X_resampled={X_resampled.shape}, y_resampled={y_resampled.mean()}')\n",
    "    model.fit(X_resampled, y_resampled)\n",
    "    dfs.append(make_statistics(model, X, y, X_test, y_test))\n",
    "    \n",
    "print('------ Undersampling methods ------')\n",
    "techniques = [RandomUnderSampler(),\n",
    "              NearMiss(version=1),\n",
    "              NearMiss(version=2),\n",
    "              TomekLinks(),\n",
    "              EditedNearestNeighbours()]\n",
    "# dfs = []\n",
    "\n",
    "for sampler in techniques:\n",
    "    technique = sampler.__class__.__name__\n",
    "    print(f'Technique: {technique}')\n",
    "    X_resampled, y_resampled = sampler.fit_resample(X, y)\n",
    "    print(f'X_resampled={X_resampled.shape}, y_resampled={y_resampled.mean()}')\n",
    "    model.fit(X_resampled, y_resampled)\n",
    "    dfs.append(make_statistics(model, X, y, X_test, y_test)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfe4bc11-2116-4f17-8e10-4d2444264098",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_5c58a_row0_col2,#T_5c58a_row1_col1,#T_5c58a_row1_col6,#T_5c58a_row2_col3,#T_5c58a_row3_col2,#T_5c58a_row4_col2,#T_5c58a_row5_col0,#T_5c58a_row5_col2,#T_5c58a_row5_col10,#T_5c58a_row5_col11,#T_5c58a_row6_col0,#T_5c58a_row6_col2,#T_5c58a_row6_col6,#T_5c58a_row6_col10,#T_5c58a_row6_col11,#T_5c58a_row7_col0,#T_5c58a_row7_col2,#T_5c58a_row7_col6,#T_5c58a_row7_col10,#T_5c58a_row7_col11{\n",
       "            background-color:  #FFDDDD;\n",
       "        }</style><table id=\"T_5c58a_\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >None</th>        <th class=\"col_heading level0 col1\" >Weights</th>        <th class=\"col_heading level0 col2\" >Th-d</th>        <th class=\"col_heading level0 col3\" >Th-d + W</th>        <th class=\"col_heading level0 col4\" >RandOS</th>        <th class=\"col_heading level0 col5\" >SMOTE</th>        <th class=\"col_heading level0 col6\" >ADASYN</th>        <th class=\"col_heading level0 col7\" >RandUS</th>        <th class=\"col_heading level0 col8\" >NM1</th>        <th class=\"col_heading level0 col9\" >NM2</th>        <th class=\"col_heading level0 col10\" >TLinks</th>        <th class=\"col_heading level0 col11\" >ENNs</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_5c58a_level0_row0\" class=\"row_heading level0 row0\" >accuracy_score</th>\n",
       "                        <td id=\"T_5c58a_row0_col0\" class=\"data row0 col0\" >0.969</td>\n",
       "                        <td id=\"T_5c58a_row0_col1\" class=\"data row0 col1\" >0.881</td>\n",
       "                        <td id=\"T_5c58a_row0_col2\" class=\"data row0 col2\" >0.975</td>\n",
       "                        <td id=\"T_5c58a_row0_col3\" class=\"data row0 col3\" >0.974</td>\n",
       "                        <td id=\"T_5c58a_row0_col4\" class=\"data row0 col4\" >0.878</td>\n",
       "                        <td id=\"T_5c58a_row0_col5\" class=\"data row0 col5\" >0.890</td>\n",
       "                        <td id=\"T_5c58a_row0_col6\" class=\"data row0 col6\" >0.901</td>\n",
       "                        <td id=\"T_5c58a_row0_col7\" class=\"data row0 col7\" >0.851</td>\n",
       "                        <td id=\"T_5c58a_row0_col8\" class=\"data row0 col8\" >0.913</td>\n",
       "                        <td id=\"T_5c58a_row0_col9\" class=\"data row0 col9\" >0.855</td>\n",
       "                        <td id=\"T_5c58a_row0_col10\" class=\"data row0 col10\" >0.969</td>\n",
       "                        <td id=\"T_5c58a_row0_col11\" class=\"data row0 col11\" >0.969</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row1\" class=\"row_heading level0 row1\" >balanced_accuracy_score</th>\n",
       "                        <td id=\"T_5c58a_row1_col0\" class=\"data row1 col0\" >0.723</td>\n",
       "                        <td id=\"T_5c58a_row1_col1\" class=\"data row1 col1\" >0.882</td>\n",
       "                        <td id=\"T_5c58a_row1_col2\" class=\"data row1 col2\" >0.862</td>\n",
       "                        <td id=\"T_5c58a_row1_col3\" class=\"data row1 col3\" >0.865</td>\n",
       "                        <td id=\"T_5c58a_row1_col4\" class=\"data row1 col4\" >0.877</td>\n",
       "                        <td id=\"T_5c58a_row1_col5\" class=\"data row1 col5\" >0.873</td>\n",
       "                        <td id=\"T_5c58a_row1_col6\" class=\"data row1 col6\" >0.882</td>\n",
       "                        <td id=\"T_5c58a_row1_col7\" class=\"data row1 col7\" >0.870</td>\n",
       "                        <td id=\"T_5c58a_row1_col8\" class=\"data row1 col8\" >0.798</td>\n",
       "                        <td id=\"T_5c58a_row1_col9\" class=\"data row1 col9\" >0.770</td>\n",
       "                        <td id=\"T_5c58a_row1_col10\" class=\"data row1 col10\" >0.723</td>\n",
       "                        <td id=\"T_5c58a_row1_col11\" class=\"data row1 col11\" >0.723</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row2\" class=\"row_heading level0 row2\" >cohen_kappa_score</th>\n",
       "                        <td id=\"T_5c58a_row2_col0\" class=\"data row2 col0\" >0.602</td>\n",
       "                        <td id=\"T_5c58a_row2_col1\" class=\"data row2 col1\" >0.406</td>\n",
       "                        <td id=\"T_5c58a_row2_col2\" class=\"data row2 col2\" >0.699</td>\n",
       "                        <td id=\"T_5c58a_row2_col3\" class=\"data row2 col3\" >0.702</td>\n",
       "                        <td id=\"T_5c58a_row2_col4\" class=\"data row2 col4\" >0.397</td>\n",
       "                        <td id=\"T_5c58a_row2_col5\" class=\"data row2 col5\" >0.418</td>\n",
       "                        <td id=\"T_5c58a_row2_col6\" class=\"data row2 col6\" >0.450</td>\n",
       "                        <td id=\"T_5c58a_row2_col7\" class=\"data row2 col7\" >0.346</td>\n",
       "                        <td id=\"T_5c58a_row2_col8\" class=\"data row2 col8\" >0.421</td>\n",
       "                        <td id=\"T_5c58a_row2_col9\" class=\"data row2 col9\" >0.283</td>\n",
       "                        <td id=\"T_5c58a_row2_col10\" class=\"data row2 col10\" >0.602</td>\n",
       "                        <td id=\"T_5c58a_row2_col11\" class=\"data row2 col11\" >0.602</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row3\" class=\"row_heading level0 row3\" >f1_score</th>\n",
       "                        <td id=\"T_5c58a_row3_col0\" class=\"data row3 col0\" >0.616</td>\n",
       "                        <td id=\"T_5c58a_row3_col1\" class=\"data row3 col1\" >0.455</td>\n",
       "                        <td id=\"T_5c58a_row3_col2\" class=\"data row3 col2\" >0.715</td>\n",
       "                        <td id=\"T_5c58a_row3_col3\" class=\"data row3 col3\" >0.714</td>\n",
       "                        <td id=\"T_5c58a_row3_col4\" class=\"data row3 col4\" >0.447</td>\n",
       "                        <td id=\"T_5c58a_row3_col5\" class=\"data row3 col5\" >0.466</td>\n",
       "                        <td id=\"T_5c58a_row3_col6\" class=\"data row3 col6\" >0.494</td>\n",
       "                        <td id=\"T_5c58a_row3_col7\" class=\"data row3 col7\" >0.403</td>\n",
       "                        <td id=\"T_5c58a_row3_col8\" class=\"data row3 col8\" >0.463</td>\n",
       "                        <td id=\"T_5c58a_row3_col9\" class=\"data row3 col9\" >0.343</td>\n",
       "                        <td id=\"T_5c58a_row3_col10\" class=\"data row3 col10\" >0.616</td>\n",
       "                        <td id=\"T_5c58a_row3_col11\" class=\"data row3 col11\" >0.616</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row4\" class=\"row_heading level0 row4\" >matthews_corrcoef</th>\n",
       "                        <td id=\"T_5c58a_row4_col0\" class=\"data row4 col0\" >0.656</td>\n",
       "                        <td id=\"T_5c58a_row4_col1\" class=\"data row4 col1\" >0.478</td>\n",
       "                        <td id=\"T_5c58a_row4_col2\" class=\"data row4 col2\" >0.731</td>\n",
       "                        <td id=\"T_5c58a_row4_col3\" class=\"data row4 col3\" >0.727</td>\n",
       "                        <td id=\"T_5c58a_row4_col4\" class=\"data row4 col4\" >0.469</td>\n",
       "                        <td id=\"T_5c58a_row4_col5\" class=\"data row4 col5\" >0.481</td>\n",
       "                        <td id=\"T_5c58a_row4_col6\" class=\"data row4 col6\" >0.508</td>\n",
       "                        <td id=\"T_5c58a_row4_col7\" class=\"data row4 col7\" >0.432</td>\n",
       "                        <td id=\"T_5c58a_row4_col8\" class=\"data row4 col8\" >0.446</td>\n",
       "                        <td id=\"T_5c58a_row4_col9\" class=\"data row4 col9\" >0.335</td>\n",
       "                        <td id=\"T_5c58a_row4_col10\" class=\"data row4 col10\" >0.656</td>\n",
       "                        <td id=\"T_5c58a_row4_col11\" class=\"data row4 col11\" >0.656</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row5\" class=\"row_heading level0 row5\" >$-$log_loss</th>\n",
       "                        <td id=\"T_5c58a_row5_col0\" class=\"data row5 col0\" >-0.098</td>\n",
       "                        <td id=\"T_5c58a_row5_col1\" class=\"data row5 col1\" >-0.255</td>\n",
       "                        <td id=\"T_5c58a_row5_col2\" class=\"data row5 col2\" >-0.098</td>\n",
       "                        <td id=\"T_5c58a_row5_col3\" class=\"data row5 col3\" >-0.255</td>\n",
       "                        <td id=\"T_5c58a_row5_col4\" class=\"data row5 col4\" >-0.252</td>\n",
       "                        <td id=\"T_5c58a_row5_col5\" class=\"data row5 col5\" >-0.229</td>\n",
       "                        <td id=\"T_5c58a_row5_col6\" class=\"data row5 col6\" >-0.211</td>\n",
       "                        <td id=\"T_5c58a_row5_col7\" class=\"data row5 col7\" >-0.341</td>\n",
       "                        <td id=\"T_5c58a_row5_col8\" class=\"data row5 col8\" >-0.344</td>\n",
       "                        <td id=\"T_5c58a_row5_col9\" class=\"data row5 col9\" >-0.380</td>\n",
       "                        <td id=\"T_5c58a_row5_col10\" class=\"data row5 col10\" >-0.098</td>\n",
       "                        <td id=\"T_5c58a_row5_col11\" class=\"data row5 col11\" >-0.098</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row6\" class=\"row_heading level0 row6\" >roc_auc_score</th>\n",
       "                        <td id=\"T_5c58a_row6_col0\" class=\"data row6 col0\" >0.963</td>\n",
       "                        <td id=\"T_5c58a_row6_col1\" class=\"data row6 col1\" >0.962</td>\n",
       "                        <td id=\"T_5c58a_row6_col2\" class=\"data row6 col2\" >0.963</td>\n",
       "                        <td id=\"T_5c58a_row6_col3\" class=\"data row6 col3\" >0.962</td>\n",
       "                        <td id=\"T_5c58a_row6_col4\" class=\"data row6 col4\" >0.960</td>\n",
       "                        <td id=\"T_5c58a_row6_col5\" class=\"data row6 col5\" >0.959</td>\n",
       "                        <td id=\"T_5c58a_row6_col6\" class=\"data row6 col6\" >0.963</td>\n",
       "                        <td id=\"T_5c58a_row6_col7\" class=\"data row6 col7\" >0.960</td>\n",
       "                        <td id=\"T_5c58a_row6_col8\" class=\"data row6 col8\" >0.915</td>\n",
       "                        <td id=\"T_5c58a_row6_col9\" class=\"data row6 col9\" >0.871</td>\n",
       "                        <td id=\"T_5c58a_row6_col10\" class=\"data row6 col10\" >0.963</td>\n",
       "                        <td id=\"T_5c58a_row6_col11\" class=\"data row6 col11\" >0.963</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_5c58a_level0_row7\" class=\"row_heading level0 row7\" >average_precision_score</th>\n",
       "                        <td id=\"T_5c58a_row7_col0\" class=\"data row7 col0\" >0.789</td>\n",
       "                        <td id=\"T_5c58a_row7_col1\" class=\"data row7 col1\" >0.786</td>\n",
       "                        <td id=\"T_5c58a_row7_col2\" class=\"data row7 col2\" >0.789</td>\n",
       "                        <td id=\"T_5c58a_row7_col3\" class=\"data row7 col3\" >0.786</td>\n",
       "                        <td id=\"T_5c58a_row7_col4\" class=\"data row7 col4\" >0.782</td>\n",
       "                        <td id=\"T_5c58a_row7_col5\" class=\"data row7 col5\" >0.780</td>\n",
       "                        <td id=\"T_5c58a_row7_col6\" class=\"data row7 col6\" >0.789</td>\n",
       "                        <td id=\"T_5c58a_row7_col7\" class=\"data row7 col7\" >0.788</td>\n",
       "                        <td id=\"T_5c58a_row7_col8\" class=\"data row7 col8\" >0.636</td>\n",
       "                        <td id=\"T_5c58a_row7_col9\" class=\"data row7 col9\" >0.439</td>\n",
       "                        <td id=\"T_5c58a_row7_col10\" class=\"data row7 col10\" >0.789</td>\n",
       "                        <td id=\"T_5c58a_row7_col11\" class=\"data row7 col11\" >0.789</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fb1698ab650>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys = ['None', 'Weights', 'Th-d', 'Th-d + W', 'RandOS', 'SMOTE', 'ADASYN', 'RandUS', 'NM1', 'NM2', 'TLinks', 'ENNs']\n",
    "d = pd.concat([d.T['test'] for d in dfs], keys=keys, axis=1).round(3)\n",
    "d.loc['log_loss',:] = -d.loc['log_loss',:]\n",
    "d = d.rename(index={'log_loss':'$-$log_loss'})\n",
    "\n",
    "d.style.set_precision(3).highlight_max(axis=1, color='#FFDDDD')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}