notebooks/compute-calib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "faced-problem",
   "metadata": {
    "papermill": {
     "duration": 0.009988,
     "end_time": "2021-05-17T13:16:51.511069",
     "exception": false,
     "start_time": "2021-05-17T13:16:51.501081",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adequate-provider",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:51.536605Z",
     "iopub.status.busy": "2021-05-17T13:16:51.534589Z",
     "iopub.status.idle": "2021-05-17T13:16:52.433350Z",
     "shell.execute_reply": "2021-05-17T13:16:52.434579Z"
    },
    "papermill": {
     "duration": 0.914916,
     "end_time": "2021-05-17T13:16:52.435187",
     "exception": false,
     "start_time": "2021-05-17T13:16:51.520271",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cp -R ../input/calibration/calib calib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "statewide-shape",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:52.464984Z",
     "iopub.status.busy": "2021-05-17T13:16:52.464003Z",
     "iopub.status.idle": "2021-05-17T13:16:54.917361Z",
     "shell.execute_reply": "2021-05-17T13:16:54.916429Z"
    },
    "papermill": {
     "duration": 2.472416,
     "end_time": "2021-05-17T13:16:54.917535",
     "exception": false,
     "start_time": "2021-05-17T13:16:52.445119",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Main imports\n",
    "\n",
    "# !pip install --upgrade \"numpy==1.20.2\"\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "### Plots\n",
    "\n",
    "from matplotlib.cm import get_cmap\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import set_matplotlib_formats\n",
    "from cycler import cycler\n",
    "\n",
    "plt.rc('axes', axisbelow=True, grid=True)\n",
    "plt.rc('grid', c='grey', ls=':')\n",
    "plt.rc('font', family='serif')\n",
    "plt.rc('axes', prop_cycle=cycler(color='bmrcgyk'))\n",
    "plt.rc('image', cmap='gist_rainbow')\n",
    "plt.rc('savefig', bbox='tight', pad_inches=0.1, format='pdf')\n",
    "# set_matplotlib_formats('png')\n",
    "\n",
    "# Calibration utils\n",
    "from calib.eval import *\n",
    "from calib.calibrators import *\n",
    "from calib.utils import *\n",
    "\n",
    "# Measuer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import defaultdict\n",
    "from scipy.special import softmax\n",
    "from scipy.special import expit as sigmoid\n",
    "\n",
    "### Also\n",
    "\n",
    "rs = {'random_state': 0}\n",
    "rng = np.random.default_rng(seed=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "comfortable-criterion",
   "metadata": {
    "papermill": {
     "duration": 0.008857,
     "end_time": "2021-05-17T13:16:54.935926",
     "exception": false,
     "start_time": "2021-05-17T13:16:54.927069",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "following-cleaners",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:54.964811Z",
     "iopub.status.busy": "2021-05-17T13:16:54.962595Z",
     "iopub.status.idle": "2021-05-17T13:16:54.965692Z",
     "shell.execute_reply": "2021-05-17T13:16:54.966223Z"
    },
    "papermill": {
     "duration": 0.021601,
     "end_time": "2021-05-17T13:16:54.966390",
     "exception": false,
     "start_time": "2021-05-17T13:16:54.944789",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_logits(path):\n",
    "    logitss = {}\n",
    "    targets = None\n",
    "    for fname in os.listdir(path):\n",
    "        if fname == 'targets.txt':\n",
    "            with open(os.path.join(path, fname), 'r') as fin:\n",
    "                targets = np.array([int(target) for target in fin.read().split()])\n",
    "        elif fname[-3:] == '.pt':\n",
    "            logitss[fname[:-3]] = torch.load(os.path.join(path, fname)).numpy()\n",
    "        else:\n",
    "            raise ValueError('Invalid name in logits path!')\n",
    "    return logitss, targets    \n",
    "\n",
    "def upd_metrics(metrics_dict, new_metrics,\n",
    "                calib_name, model_dataset_tuple):\n",
    "    for metric_name, value in new_metrics.items():\n",
    "        metrics_dict[metric_name][calib_name][model_dataset_tuple] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "distinct-geometry",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:54.993792Z",
     "iopub.status.busy": "2021-05-17T13:16:54.991629Z",
     "iopub.status.idle": "2021-05-17T13:16:54.994966Z",
     "shell.execute_reply": "2021-05-17T13:16:54.995580Z"
    },
    "papermill": {
     "duration": 0.019603,
     "end_time": "2021-05-17T13:16:54.995758",
     "exception": false,
     "start_time": "2021-05-17T13:16:54.976155",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "logit_path = '../input/calibration/logits/'\n",
    "\n",
    "dataset_names = [\n",
    "    'cifar10_v1',\n",
    "    'cifar100',\n",
    "    'imagenet',\n",
    "    'focal_CIFAR10',\n",
    "    'focal_CIFAR100',\n",
    "    'focal_TinyImageNet',\n",
    "]\n",
    "\n",
    "calibrators_confs = {\n",
    "    'Hist-binning': HistogramBinningMulticlass(n_bins=20),\n",
    "    'Isotonic': IsotonicRegressionMulticlass(),\n",
    "}\n",
    "\n",
    "calibrators_logits = {\n",
    "    'T-scaling': LogitScaling(scale_type='temperature', bias_type='none'),\n",
    "    'V-scaling': LogitScaling(scale_type='vector', bias_type='none'),\n",
    "    'V-scaling-b': LogitScaling(scale_type='vector', bias_type='vector'),\n",
    "    'M-scaling-b': LogitScaling(scale_type='matrix', bias_type='vector'),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "greater-bailey",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:55.077385Z",
     "iopub.status.busy": "2021-05-17T13:16:55.076079Z",
     "iopub.status.idle": "2021-05-17T13:16:55.080879Z",
     "shell.execute_reply": "2021-05-17T13:16:55.081365Z"
    },
    "papermill": {
     "duration": 0.075522,
     "end_time": "2021-05-17T13:16:55.081548",
     "exception": false,
     "start_time": "2021-05-17T13:16:55.006026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "assigned-madagascar",
   "metadata": {
    "papermill": {
     "duration": 0.010362,
     "end_time": "2021-05-17T13:16:55.102496",
     "exception": false,
     "start_time": "2021-05-17T13:16:55.092134",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Compute all metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "entire-question",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:16:55.237094Z",
     "iopub.status.busy": "2021-05-17T13:16:55.236117Z",
     "iopub.status.idle": "2021-05-17T13:27:25.498891Z",
     "shell.execute_reply": "2021-05-17T13:27:25.498296Z"
    },
    "papermill": {
     "duration": 630.386257,
     "end_time": "2021-05-17T13:27:25.499041",
     "exception": false,
     "start_time": "2021-05-17T13:16:55.112784",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== dataset cifar10_v1 ==\n",
      "densenet121, googlenet, mobilenet_v2, densenet169, resnet34, vgg13_bn, vgg16_bn, resnet50, vgg19_bn, inception_v3, vgg11_bn, resnet18, densenet161, DONE\n",
      "== dataset cifar100 ==\n",
      "shufflenetv2_x1_0, shufflenetv2_x2_0, resnet56, resnet32, vgg13_bn, mobilenetv2_x1_0, mobilenetv2_x0_5, vgg16_bn, resnet44, vgg19_bn, vgg11_bn, shufflenetv2_x0_5, resnet20, shufflenetv2_x1_5, mobilenetv2_x1_4, DONE\n",
      "== dataset imagenet ==\n",
      "tf_efficientnet_b8, mobilenetv2_120d, repvgg_b3, vgg19_bn, DONE\n",
      "== dataset focal_CIFAR10 ==\n",
      "wide_resnet_focal_loss_gamma_3.0, resnet110_cross_entropy, densenet121_focal_loss_gamma_2.0, densenet121_cross_entropy_smoothed_smoothing_0.05, densenet121_cross_entropy, resnet50_focal_loss_gamma_1.0, resnet110_focal_loss_gamma_1.0, resnet50_cross_entropy, resnet50_focal_loss_gamma_3.0, densenet121_focal_loss_gamma_3.0, wide_resnet_focal_loss_gamma_1.0, wide_resnet_cross_entropy_smoothed_smoothing_0.05, resnet110_cross_entropy_smoothed_smoothing_0.05, densenet121_focal_loss_gamma_1.0, resnet50_cross_entropy_smoothed_smoothing_0.05, wide_resnet_cross_entropy, wide_resnet_focal_loss_gamma_2.0, resnet110_focal_loss_gamma_2.0, resnet50_focal_loss_gamma_2.0, resnet110_focal_loss_gamma_3.0, DONE\n",
      "== dataset focal_CIFAR100 ==\n",
      "wide_resnet_focal_loss_gamma_3.0, resnet110_cross_entropy, densenet121_focal_loss_gamma_2.0, densenet121_cross_entropy_smoothed_smoothing_0.05, densenet121_cross_entropy, resnet50_focal_loss_gamma_1.0, resnet110_focal_loss_gamma_1.0, resnet50_cross_entropy, resnet50_focal_loss_gamma_3.0, densenet121_focal_loss_gamma_3.0, wide_resnet_focal_loss_gamma_1.0, wide_resnet_cross_entropy_smoothed_smoothing_0.05, resnet110_cross_entropy_smoothed_smoothing_0.05, densenet121_focal_loss_gamma_1.0, resnet50_cross_entropy_smoothed_smoothing_0.05, wide_resnet_cross_entropy, wide_resnet_focal_loss_gamma_2.0, resnet110_focal_loss_gamma_2.0, resnet50_focal_loss_gamma_2.0, resnet110_focal_loss_gamma_3.0, DONE\n",
      "== dataset focal_TinyImageNet ==\n",
      "resnet50_focal_loss_gamma_1.0, resnet50_cross_entropy, resnet50_focal_loss_gamma_3.0, resnet50_cross_entropy_smoothed_smoothing_0.05, resnet50_focal_loss_gamma_2.0, DONE\n",
      "CPU times: user 9min 57s, sys: 28.5 s, total: 10min 26s\n",
      "Wall time: 10min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "metrics_val = defaultdict(lambda: defaultdict(dict))\n",
    "metrics_test = defaultdict(lambda: defaultdict(dict))\n",
    "rels_val = defaultdict(dict)\n",
    "rels_test = defaultdict(dict)\n",
    "\n",
    "for dataset_name in dataset_names:\n",
    "    print(f'== dataset {dataset_name} ==')\n",
    "    fpath = os.path.join(logit_path, dataset_name)\n",
    "    logitss, targets = get_logits(fpath)\n",
    "    for model_name, logits in logitss.items():\n",
    "        print(f'{model_name}, ', end='')\n",
    "        model_dataset_tuple = (dataset_name, model_name)\n",
    "        logits_val, logits_test, targets_val, targets_test = train_test_split(\n",
    "            logits, targets, test_size=0.5, **rs)\n",
    "        confs_val = softmax(logits_val, axis=1)\n",
    "        confs_test = softmax(logits_test, axis=1)\n",
    "\n",
    "        # No calibration\n",
    "        upd_metrics(metrics_val, all_metrics(targets_val, confs_val),\n",
    "                    'До калибровки', model_dataset_tuple)\n",
    "        upd_metrics(metrics_test, all_metrics(targets_test, confs_test),\n",
    "                    'До калибровки', model_dataset_tuple)\n",
    "        rels_val[model_dataset_tuple]['До калибровки'] = bins_reliability_multiclass(targets_val, confs_val, 10)\n",
    "        rels_test[model_dataset_tuple]['До калибровки'] = bins_reliability_multiclass(targets_test, confs_test, 10)\n",
    "\n",
    "        # Transforming confs\n",
    "        for cal_name, calibrator in calibrators_confs.items():\n",
    "            calibrator.fit(confs_val, targets_val)\n",
    "            confs_val_cal = calibrator.transform(confs_val)\n",
    "            confs_test_cal = calibrator.transform(confs_test)\n",
    "            upd_metrics(metrics_val, all_metrics(targets_val, confs_val_cal),\n",
    "                        cal_name, model_dataset_tuple)\n",
    "            upd_metrics(metrics_test, all_metrics(targets_test, confs_test_cal),\n",
    "                        cal_name, model_dataset_tuple)\n",
    "            rels_val[model_dataset_tuple][cal_name] = bins_reliability_multiclass(targets_val, confs_val_cal, 10)\n",
    "            rels_test[model_dataset_tuple][cal_name] = bins_reliability_multiclass(targets_test, confs_test_cal, 10)\n",
    "\n",
    "        # Transforming logits\n",
    "        for cal_name, calibrator in calibrators_logits.items():\n",
    "            calibrator.fit(logits_val, targets_val, device=device)\n",
    "            confs_val_cal = calibrator.transform(logits_val)\n",
    "            confs_test_cal = calibrator.transform(logits_test)\n",
    "            upd_metrics(metrics_val, all_metrics(targets_val, confs_val_cal),\n",
    "                        cal_name, model_dataset_tuple)\n",
    "            upd_metrics(metrics_test, all_metrics(targets_test, confs_test_cal),\n",
    "                        cal_name, model_dataset_tuple)\n",
    "            rels_val[model_dataset_tuple][cal_name] = bins_reliability_multiclass(targets_val, confs_val_cal, 10)\n",
    "            rels_test[model_dataset_tuple][cal_name] = bins_reliability_multiclass(targets_test, confs_test_cal, 10)\n",
    "    print('DONE')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sporting-celebration",
   "metadata": {
    "papermill": {
     "duration": 0.037475,
     "end_time": "2021-05-17T13:27:25.574239",
     "exception": false,
     "start_time": "2021-05-17T13:27:25.536764",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Save checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "possible-animation",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:27:25.669657Z",
     "iopub.status.busy": "2021-05-17T13:27:25.668551Z",
     "iopub.status.idle": "2021-05-17T13:27:25.703011Z",
     "shell.execute_reply": "2021-05-17T13:27:25.702423Z"
    },
    "papermill": {
     "duration": 0.09138,
     "end_time": "2021-05-17T13:27:25.703152",
     "exception": false,
     "start_time": "2021-05-17T13:27:25.611772",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('metrics_test.pickle', 'wb') as fout:\n",
    "    pickle.dump(dict(metrics_test), fout)\n",
    "    \n",
    "with open('metrics_val.pickle', 'wb') as fout:\n",
    "    pickle.dump(dict(metrics_val), fout)\n",
    "\n",
    "with open('rels_test.pickle', 'wb') as fout:\n",
    "    pickle.dump(dict(rels_test), fout)\n",
    "    \n",
    "with open('rels_val.pickle', 'wb') as fout:\n",
    "    pickle.dump(dict(rels_val), fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "spectacular-carpet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-17T13:27:25.785576Z",
     "iopub.status.busy": "2021-05-17T13:27:25.784846Z",
     "iopub.status.idle": "2021-05-17T13:27:27.464119Z",
     "shell.execute_reply": "2021-05-17T13:27:27.464667Z"
    },
    "papermill": {
     "duration": 1.722596,
     "end_time": "2021-05-17T13:27:27.464872",
     "exception": false,
     "start_time": "2021-05-17T13:27:25.742276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: metrics_test.pickle (deflated 54%)\r\n",
      "  adding: metrics_val.pickle (deflated 54%)\r\n",
      "  adding: rels_test.pickle (deflated 51%)\r\n",
      "  adding: rels_val.pickle (deflated 52%)\r\n"
     ]
    }
   ],
   "source": [
    "!zip exp_results.zip *.pickle\n",
    "!rm *.pickle"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 645.976539,
   "end_time": "2021-05-17T13:27:29.716597",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-17T13:16:43.740058",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}