Document/Document.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p  style=\"z-index: 101;background: #fde073;text-align: center;line-height: 2.5;overflow: hidden;font-size:22px;\">Please <a href=\"#Cite\">cite us</a> if you use the software</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyCM Document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version : 4.2 <a class=\"tocSkip\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "html_hide"
    ]
   },
   "source": [
    "## Table of contents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "html_hide"
    ]
   },
   "source": [
    "<ul>\n",
    "    <li><a href=\"#Overview\">Overview</a></li>\n",
    "    <li><a href=\"#Installation\">Installation</a></li>\n",
    "        <ol>\n",
    "            <li><a href=\"#PyPI\">PyPI</a></li>\n",
    "            <li><a href=\"#Source-code\">Source Code</a></li>\n",
    "            <li><a href=\"#MATLAB\">MATLAB</a></li>\n",
    "        </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Usage\">Usage</a></li>\n",
    "        <ol>\n",
    "            <li><a href=\"#From-vector\">From Vector</a></li>\n",
    "            <li><a href=\"#Direct-CM\">Direct CM</a></li>\n",
    "            <li><a href=\"#Iterating-and-casting\">Iterating And Casting</a></li>\n",
    "            <li><a href=\"#Activation-threshold\">Activation Threshold</a></li>\n",
    "            <li><a href=\"#Load-from-file\">Load From File</a></li>\n",
    "            <li><a href=\"#Sample-weights\">Sample Weights</a></li>\n",
    "            <li><a href=\"#Transpose\">Transpose</a></li>\n",
    "            <li><a href=\"#Metrics-off\">Metrics Off</a></li>\n",
    "            <li><a href=\"#Relabel\">Relabel</a></li>\n",
    "            <li><a href=\"#Position\">Position</a></li>\n",
    "            <li><a href=\"#To-array\">To Array</a></li>\n",
    "            <li><a href=\"#Combine\">Combine</a></li>\n",
    "            <li><a href=\"#Plot\">Plot</a></li>\n",
    "            <li><a href=\"#Distance/Similarity\">Distance/Similarity</a></li>\n",
    "            <li><a href=\"#Parameter-recommender\">Parameter Recommender</a></li>\n",
    "            <li><a href=\"#Compare\">Compare</a></li>\n",
    "            <li><a href=\"#ROC-curve\">ROC Curve</a></li>\n",
    "            <li><a href=\"#Precision-Recall-curve\">Precision-Recall Curve</a></li>\n",
    "            <li><a href=\"#Multilabel-confusion-matrix\">Multilabel Confusion Matrix</a></li>\n",
    "            <li><a href=\"#Online-help\">Online Help</a></li>\n",
    "            <li><a href=\"#Acceptable-data-types\">Acceptable Data Types</a></li>\n",
    "        </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Basic-parameters\">Basic Parameters</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#TP-(True-positive)\">True Positive</a></li>\n",
    "        <li><a href=\"#TN-(True-negative)\">True Negative</a></li>\n",
    "        <li><a href=\"#FP-(False-positive)\">False Positive</a></li>\n",
    "        <li><a href=\"#FN-(False-negative)\">False Negative</a></li>\n",
    "        <li><a href=\"#P-(Condition-positive)\">Condition Positive</a></li>\n",
    "        <li><a href=\"#N-(Condition-negative)\">Condition Negative</a></li>\n",
    "        <li><a href=\"#TOP-(Test-outcome-positive)\">Test Outcome Positive</a></li>\n",
    "        <li><a href=\"#TON-(Test-outcome-negative)\">Test Outcome Negative</a></li>\n",
    "        <li><a href=\"#POP-(Population)\">Population</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Class-statistics\">Class Statistics</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#TPR-(True-positive-rate)\">True Positive Rate</a></li>\n",
    "        <li><a href=\"#TNR-(True-negative-rate)\">True Negative Rate</a></li>\n",
    "        <li><a href=\"#PPV-(Positive-predictive-value)\">Positive Predictive Value</a></li>\n",
    "        <li><a href=\"#NPV-(Negative-predictive-value)\">Negative Predictive Value</a></li>\n",
    "        <li><a href=\"#FNR-(False-negative-rate)\">False Negative Rate</a></li>\n",
    "        <li><a href=\"#FPR-(False-positive-rate)\">False Positive Rate</a></li>\n",
    "        <li><a href=\"#FDR-(False-discovery-rate)\">False Discovery Rate</a></li>\n",
    "        <li><a href=\"#FOR-(False-omission-rate)\">False Omission Rate</a></li>\n",
    "        <li><a href=\"#ACC-(Accuracy)\">Accuracy</a></li>\n",
    "        <li><a href=\"#ERR-(Error-rate)\">Error Rate</a></li>\n",
    "        <li><a href=\"#FBeta-Score\">FBeta Score</a></li>\n",
    "        <li><a href=\"#MCC-(Matthews-correlation-coefficient)\">Matthews Correlation Coefficient</a></li>\n",
    "        <li><a href=\"#BM-(Bookmaker-informedness)\">Informedness</a></li>\n",
    "        <li><a href=\"#MK-(Markedness)\">Markedness</a></li>\n",
    "        <li><a href=\"#PLR-(Positive-likelihood-ratio)\">Positive Likelihood Ratio</a></li>\n",
    "        <li><a href=\"#NLR-(Negative-likelihood-ratio)\">Negative Likelihood Ratio</a></li>\n",
    "        <li><a href=\"#DOR-(Diagnostic-odds-ratio)\">Diagnostic Odds Ratio</a></li>\n",
    "        <li><a href=\"#PRE-(Prevalence)\">Prevalence</a></li>\n",
    "        <li><a href=\"#G-(G-measure)\">G-Measure</a></li>\n",
    "        <li><a href=\"#RACC-(Random-accuracy)\">Random Accuracy</a></li>\n",
    "        <li><a href=\"#RACCU-(Random-accuracy-unbiased)\">Random Accuracy Unbiased</a></li>\n",
    "        <li><a href=\"#J-(Jaccard-index)\">Jaccard Index</a></li>\n",
    "        <li><a href=\"#IS-(Information-score)\">Information Score</a></li>\n",
    "        <li><a href=\"#CEN-(Confusion-entropy)\">Confusion Entropy</a></li>\n",
    "        <li><a href=\"#MCEN-(Modified-confusion-entropy)\">Modified Confusion Entropy</a></li>\n",
    "        <li><a href=\"#AUC-(Area-under-the-ROC-curve)\">Area Under The ROC Curve</a></li>\n",
    "        <li><a href=\"#dInd-(Distance-index)\">Distance Index</a></li>\n",
    "        <li><a href=\"#sInd-(Similarity-index)\">Similarity Index</a></li>\n",
    "        <li><a href=\"#DP-(Discriminant-power)\">Discriminant Power</a></li>\n",
    "        <li><a href=\"#Y-(Youden-index)\">Youden Index</a></li>\n",
    "        <li><a href=\"#PLRI-(Positive-likelihood-ratio-interpretation)\">Positive Likelihood Ratio Interpretation</a></li>\n",
    "        <li><a href=\"#NLRI-(Negative-likelihood-ratio-interpretation)\">Negative Likelihood Ratio Interpretation</a></li>\n",
    "        <li><a href=\"#DPI-(Discriminant-power-interpretation)\">Discriminant Power Interpretation</a></li>\n",
    "        <li><a href=\"#AUCI-(AUC-value-interpretation)\">AUC Value Interpretation</a></li>\n",
    "        <li><a href=\"#MCCI-(Matthews-correlation-coefficient-interpretation)\">Matthews Correlation Coefficient Interpretation</a></li>\n",
    "        <li><a href=\"#QI-(Yule's-Q-interpretation)\">Yule's Q Interpretation</a></li>\n",
    "        <li><a href=\"#GI-(Gini-index)\">Gini Index</a></li>\n",
    "        <li><a href=\"#LS-(Lift-score)\">Lift Score</a></li>\n",
    "        <li><a href=\"#AM-(Automatic/Manual)\">Automatic/Manual</a></li>\n",
    "        <li><a href=\"#BCD-(Bray-Curtis-dissimilarity)\">Bray-Curtis Dissimilarity</a></li>\n",
    "        <li><a href=\"#OP-(Optimized-precision)\">Optimized Precision</a></li>\n",
    "        <li><a href=\"#IBA-(Index-of-balanced-accuracy)\">Index of Balanced Accuracy</a></li>\n",
    "        <li><a href=\"#GM-(G-mean)\">G-Mean</a></li>\n",
    "        <li><a href=\"#Q-(Yule's-Q)\">Yule's Q</a></li>\n",
    "        <li><a href=\"#AGM-(Adjusted-G-mean)\">Adjusted G-Mean</a></li> \n",
    "        <li><a href=\"#AGF-(Adjusted-F-score)\">Adjusted F-Score</a></li>\n",
    "        <li><a href=\"#OC-(Overlap-coefficient)\">Overlap Coefficient</a></li>\n",
    "        <li><a href=\"#BB-(Braun-Blanquet-similarity)\">Braun-Blanquet Similarity</a></li>\n",
    "        <li><a href=\"#OOC-(Otsuka-Ochiai-coefficient)\">Otsuka Ochiai Coefficient</a></li>\n",
    "        <li><a href=\"#TI-(Tversky-index)\">Tversky Index</a></li> \n",
    "        <li><a href=\"#AUPR-(Area-under-the-PR-curve)\">Area Under The PR Curve</a></li> \n",
    "        <li><a href=\"#ICSI-(Individual-classification-success-index)\">Individual Classification Success Index</a></li> \n",
    "        <li><a href=\"#CI-(Confidence-interval)\">Confidence Interval</a></li> \n",
    "        <li><a href=\"#NB-(Net-benefit)\">Net Benefit</a></li>\n",
    "        <li><a href=\"#Average\">Average</a></li>\n",
    "        <li><a href=\"#Weighted-average\">Weighted Average</a></li>\n",
    "        <li><a href=\"#Sensitivity-index\">Sensitivity Index</a></li>\n",
    "        <li><a href=\"#HD-(Hamming-distance)\">Hamming Distance</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Overall-statistics\">Overall Statistics</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#Kappa\">Kappa</a></li>\n",
    "        <li><a href=\"#Kappa-unbiased\">Kappa Unbiased</a></li>\n",
    "        <li><a href=\"#Kappa-no-prevalence\">Kappa No Prevalence</a></li>\n",
    "        <li><a href=\"#Weighted-kappa\">Weighted Kappa</a></li>\n",
    "        <li><a href=\"#Kappa-standard-error\">Kappa Standard Error</a></li>\n",
    "        <li><a href=\"#Kappa-95%-CI\">Kappa 95% CI</a></li>\n",
    "        <li><a href=\"#Chi-squared\">Chi Squared</a></li>\n",
    "        <li><a href=\"#Chi-squared-DF\">Chi Squared DF</a></li>\n",
    "        <li><a href=\"#Phi-squared\">Phi Squared</a></li>\n",
    "        <li><a href=\"#Cramer's-V\">Cramer's V</a></li>\n",
    "        <li><a href=\"#Standard-error\">Standard Error</a></li>\n",
    "        <li><a href=\"#95%-CI\">95% CI</a></li>\n",
    "        <li><a href=\"#Bennett's-S\">Bennett's S</a></li>\n",
    "        <li><a href=\"#Scott's-Pi\">Scott's PI</a></li>\n",
    "        <li><a href=\"#Gwet's-AC1\">Gwet's AC1</a></li>\n",
    "        <li><a href=\"#Reference-entropy\">Reference Entropy</a></li>\n",
    "        <li><a href=\"#Response-entropy\">Response Entropy</a></li>\n",
    "        <li><a href=\"#Cross-entropy\">Cross Entropy</a></li>\n",
    "        <li><a href=\"#Joint-entropy\">Joint Entropy</a></li>\n",
    "        <li><a href=\"#Conditional-entropy\">Conditional Entropy</a></li>\n",
    "        <li><a href=\"#Kullback-Leibler-divergence\">Kullback-Leibler Divergence</a></li>\n",
    "        <li><a href=\"#Mutual-information\">Mutual Information</a></li>\n",
    "        <li><a href=\"#Goodman-&-Kruskal's-lambda-A\">Goodman-Kruskal's Lambda A</a></li>\n",
    "        <li><a href=\"#Goodman-&-Kruskal's-lambda-B\">Goodman-Kruskal's Lambda B</a></li>\n",
    "        <li><a href=\"#SOA1-(Landis-&-Koch's-benchmark)\">Landis-Koch's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA2-(Fleiss'-benchmark)\">Fleiss' Benchmark</a></li>\n",
    "        <li><a href=\"#SOA3-(Altman's-benchmark)\">Altman's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA4-(Cicchetti's-benchmark)\">Cicchetti's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA5-(Cramer's-benchmark)\">Cramer's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA6-(Matthews's-benchmark)\">Matthews's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA7-(Goodman-&-Kruskal's-lambda-A-benchmark)\">Goodman-Kruskal's Lambda A Benchmark</a></li>\n",
    "        <li><a href=\"#SOA8-(Goodman-&-Kruskal's-lambda-B-benchmark)\">Goodman-Kruskal's Lambda B Benchmark</a></li>\n",
    "        <li><a href=\"#SOA9-(Krippendorff's-alpha-benchmark)\">Krippendorff's Alpha Benchmark</a></li>\n",
    "        <li><a href=\"#SOA10-(Pearson's-C-benchmark)\">Pearson's C Benchmark</a></li>\n",
    "        <li><a href=\"#Overall_ACC\">Overall Accuracy</a></li>\n",
    "        <li><a href=\"#Overall_RACC\">Overall Random Accuracy</a></li>\n",
    "        <li><a href=\"#Overall_RACCU\">Overall Random Accuracy Unbiased</a></li>\n",
    "        <li><a href=\"#PPV_Micro\">Positive Predictive Value Micro</a></li>\n",
    "        <li><a href=\"#NPV_Micro\">Negative Predictive Value Micro</a></li>\n",
    "        <li><a href=\"#TPR_Micro\">True Positive Rate Micro</a></li>\n",
    "        <li><a href=\"#TNR_Micro\">True Negative Rate Micro</a></li>\n",
    "        <li><a href=\"#FPR_Micro\">False Positive Rate Micro</a></li>\n",
    "        <li><a href=\"#FNR_Micro\">False Negative Rate Micro</a></li>\n",
    "        <li><a href=\"#F1_Micro\">F1 Score Micro</a></li>\n",
    "        <li><a href=\"#PPV_Macro\">Positive Predictive Value Macro</a></li>\n",
    "        <li><a href=\"#NPV_Macro\">Negative Predictive Value Macro</a></li>\n",
    "        <li><a href=\"#TPR_Macro\">True Positive Rate Macro</a></li>\n",
    "        <li><a href=\"#TNR_Macro\">True Negative Rate Macro</a></li>\n",
    "        <li><a href=\"#FPR_Macro\">False Positive Rate Macro</a></li>\n",
    "        <li><a href=\"#FNR_Macro\">False Negative Rate Macro</a></li>\n",
    "        <li><a href=\"#F1_Macro\">F1 Score Macro</a></li>\n",
    "        <li><a href=\"#ACC_Macro\">Accuracy Macro</a></li>\n",
    "        <li><a href=\"#Overall_J\">Overall Jaccard Index</a></li>\n",
    "        <li><a href=\"#Hamming-loss\">Hamming Loss</a></li>\n",
    "        <li><a href=\"#Zero-one-loss\">Zero-one Loss</a></li>\n",
    "        <li><a href=\"#NIR-(No-information-rate)\">No Information Rate</a></li>\n",
    "        <li><a href=\"#P-Value\">P Value</a></li>\n",
    "        <li><a href=\"#Overall_CEN\">Overall Confusion Entropy</a></li>\n",
    "        <li><a href=\"#Overall_MCEN\">Overall Modified Confusion Entropy</a></li>\n",
    "        <li><a href=\"#Overall_MCC\">Overall Matthews Correlation Coefficient</a></li>\n",
    "        <li><a href=\"#RR-(Global-performance-index)\">Global Performance Index</a></li>\n",
    "        <li><a href=\"#CBA-(Class-balance-accuracy)\">Class Balance Accuracy</a></li>\n",
    "        <li><a href=\"#AUNU\">AUNU</a></li>\n",
    "        <li><a href=\"#AUNP\">AUNP</a></li>\n",
    "        <li><a href=\"#RCI-(Relative-classifier-information)\">Relative Classifier Information</a></li>\n",
    "        <li><a href=\"#Pearson's-C\">Pearson's C</a></li>\n",
    "        <li><a href=\"#CSI-(Classification-success-index)\">Classification Success Index</a></li>\n",
    "        <li><a href=\"#ARI-(Adjusted-Rand-index)\">Adjusted Rand Index</a></li>\n",
    "        <li><a href=\"#Bangdiwala's-B\">Bangdiwala's B</a></li>\n",
    "        <li><a href=\"#Krippendorff's-alpha\">Krippendorff's Alpha</a></li>\n",
    "        <li><a href=\"#Weighted-alpha\">Weighted Alpha</a></li>\n",
    "        <li><a href=\"#Aickin's-alpha\">Aickin's Alpha</a></li>\n",
    "        <li><a href=\"#Brier-score\">Brier Score</a></li>\n",
    "        <li><a href=\"#Log-loss\">Log Loss</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Print\">Print</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#Full\">Full</a></li>\n",
    "        <li><a href=\"#Matrix\">Matrix</a></li>\n",
    "        <li><a href=\"#Normalized-matrix\">Normalized Matrix</a></li>\n",
    "        <li><a href=\"#Stat\">Stat</a></li>\n",
    "        <li><a href=\"#Compare-report\">Compare Report</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Save\">Save</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#.pycm-file\">pycm</a></li>\n",
    "        <li><a href=\"#HTML\">HTML</a></li>\n",
    "        <li><a href=\"#CSV\">CSV</a></li>\n",
    "        <li><a href=\"#OBJ\">object</a></li>\n",
    "        <li><a href=\"#comp\">comp</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Input-errors\">Input Errors</a></li>\n",
    "    <li><a href=\"#Examples\">Examples</a></li>\n",
    "    <li><a href=\"#Cite\">Cite</a></li>\n",
    "    <li><a href=\"#References\">References</a></li>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"text-align:justify;\">\n",
    "PyCM is a multi-class confusion matrix library written in Python that supports both input data vectors and direct matrix, and a proper tool for post-classification model evaluation that supports most classes and overall statistics parameters.\t\n",
    "PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scientists that need a broad array of metrics for predictive models and accurate evaluation of a large variety of classifiers.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/block_diagram.jpg\" onerror='this.src=\"../images/block_diagram.jpg\"' width=\"700\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig1. ConfusionMatrix Block Diagram</p></center>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installation\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "⚠️  PyCM 3.9 is the last version to support **Python 3.5**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "⚠️  PyCM 2.4 is the last version to support **Python 2.7** & **Python 3.4**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "⚠️  Plotting capability requires **Matplotlib (>= 3.0.0)** or **Seaborn (>= 0.9.1)**   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyPI\n",
    "\n",
    "\n",
    "- Check [Python Packaging User Guide](https://packaging.python.org/installing/)     \n",
    "- Run `pip install pycm==4.2`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Source code\n",
    "- Download [Version 4.2](https://github.com/sepandhaghighi/pycm/archive/v4.2.zip) or [Latest Source](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n",
    "- Run `pip install .`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conda\n",
    "\n",
    "- Check [Conda Managing Package](https://conda.io/docs/user-guide/tasks/manage-pkgs.html#installing-packages-from-anaconda-org)\n",
    "- `conda install -c sepandhaghighi pycm`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MATLAB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Download and install [MATLAB](https://www.mathworks.com/products/matlab.html) (>=8.5, 64/32 bit)\n",
    "- Download and install [Python3.x](https://www.python.org/downloads/) (>=3.6, 64/32 bit) \n",
    "\t- Select `Add to PATH` option\n",
    "\t- Select `Install pip` option\n",
    "- Run `pip install pycm` or `pip3 install pycm`\n",
    "- Configure Python interpreter\n",
    "```\n",
    ">> pyversion PYTHON_EXECUTABLE_FULL_PATH\n",
    "```\n",
    "- Visit [MATLAB Examples](https://github.com/sepandhaghighi/pycm/tree/master/MATLAB)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Usage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Environment check"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking that the notebook is running on Google Colab or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "try:\n",
    "  import google.colab\n",
    "  !{sys.executable} -m pip -q -q install pycm\n",
    "except:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycm import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]\n",
    "y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred,digit=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li> <span style=\"color:red;\">Notice </span> :  <code>digit</code> (the number of digits to the right of the decimal point in a number) is new in <span style=\"color:red;\">version 0.6</span> (default value : 5)</li>\n",
    "    <li>Only for print and save</li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.actual_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.predict_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACC': {0: 0.8333333333333334, 1: 0.75, 2: 0.5833333333333334},\n",
       " 'AGF': {0: 0.9135962935560564, 1: 0.5399492471560389, 2: 0.5515973485146916},\n",
       " 'AGM': {0: 0.837285964012303, 1: 0.6919986974962765, 2: 0.6071224016819726},\n",
       " 'AM': {0: 2, 1: -1, 2: -1},\n",
       " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n",
       " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n",
       " 'AUPR': {0: 0.8, 1: 0.41666666666666663, 2: 0.55},\n",
       " 'BB': {0: 0.6, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'BCD': {0: 0.08333333333333333,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.041666666666666664},\n",
       " 'BM': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'CEN': {0: 0.25, 1: 0.49657842846620864, 2: 0.6044162769630221},\n",
       " 'DOR': {0: 'None', 1: 3.999999999999998, 2: 1.9999999999999998},\n",
       " 'DP': {0: 'None', 1: 0.331933069996499, 2: 0.16596653499824957},\n",
       " 'DPI': {0: 'None', 1: 'Poor', 2: 'Poor'},\n",
       " 'ERR': {0: 0.16666666666666663, 1: 0.25, 2: 0.41666666666666663},\n",
       " 'F0.5': {0: 0.6521739130434783,\n",
       "  1: 0.45454545454545453,\n",
       "  2: 0.5769230769230769},\n",
       " 'F1': {0: 0.75, 1: 0.4, 2: 0.5454545454545454},\n",
       " 'F2': {0: 0.8823529411764706, 1: 0.35714285714285715, 2: 0.5172413793103449},\n",
       " 'FDR': {0: 0.4, 1: 0.5, 2: 0.4},\n",
       " 'FN': {0: 0, 1: 2, 2: 3},\n",
       " 'FNR': {0: 0.0, 1: 0.6666666666666667, 2: 0.5},\n",
       " 'FOR': {0: 0.0, 1: 0.19999999999999996, 2: 0.4285714285714286},\n",
       " 'FP': {0: 2, 1: 1, 2: 2},\n",
       " 'FPR': {0: 0.2222222222222222,\n",
       "  1: 0.11111111111111116,\n",
       "  2: 0.33333333333333337},\n",
       " 'G': {0: 0.7745966692414834, 1: 0.408248290463863, 2: 0.5477225575051661},\n",
       " 'GI': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'GM': {0: 0.8819171036881969, 1: 0.5443310539518174, 2: 0.5773502691896257},\n",
       " 'HD': {0: 2, 1: 3, 2: 5},\n",
       " 'IBA': {0: 0.9506172839506174, 1: 0.1316872427983539, 2: 0.2777777777777778},\n",
       " 'ICSI': {0: 0.6000000000000001,\n",
       "  1: -0.16666666666666674,\n",
       "  2: 0.10000000000000009},\n",
       " 'IS': {0: 1.263034405833794, 1: 1.0, 2: 0.2630344058337938},\n",
       " 'J': {0: 0.6, 1: 0.25, 2: 0.375},\n",
       " 'LS': {0: 2.4, 1: 2.0, 2: 1.2},\n",
       " 'MCC': {0: 0.6831300510639732, 1: 0.25819888974716115, 2: 0.1690308509457033},\n",
       " 'MCCI': {0: 'Moderate', 1: 'Negligible', 2: 'Negligible'},\n",
       " 'MCEN': {0: 0.2643856189774724, 1: 0.5, 2: 0.6875},\n",
       " 'MK': {0: 0.6000000000000001, 1: 0.30000000000000004, 2: 0.17142857142857126},\n",
       " 'N': {0: 9, 1: 9, 2: 6},\n",
       " 'NLR': {0: 0.0, 1: 0.7500000000000001, 2: 0.75},\n",
       " 'NLRI': {0: 'Good', 1: 'Negligible', 2: 'Negligible'},\n",
       " 'NPV': {0: 1.0, 1: 0.8, 2: 0.5714285714285714},\n",
       " 'OC': {0: 1.0, 1: 0.5, 2: 0.6},\n",
       " 'OOC': {0: 0.7745966692414834, 1: 0.4082482904638631, 2: 0.5477225575051661},\n",
       " 'OP': {0: 0.7083333333333334, 1: 0.2954545454545454, 2: 0.4404761904761905},\n",
       " 'P': {0: 3, 1: 3, 2: 6},\n",
       " 'PLR': {0: 4.5, 1: 2.9999999999999987, 2: 1.4999999999999998},\n",
       " 'PLRI': {0: 'Poor', 1: 'Poor', 2: 'Poor'},\n",
       " 'POP': {0: 12, 1: 12, 2: 12},\n",
       " 'PPV': {0: 0.6, 1: 0.5, 2: 0.6},\n",
       " 'PRE': {0: 0.25, 1: 0.25, 2: 0.5},\n",
       " 'Q': {0: 'None', 1: 0.6, 2: 0.3333333333333333},\n",
       " 'QI': {0: 'None', 1: 'Moderate', 2: 'Weak'},\n",
       " 'RACC': {0: 0.10416666666666667,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.20833333333333334},\n",
       " 'RACCU': {0: 0.1111111111111111,\n",
       "  1: 0.04340277777777778,\n",
       "  2: 0.21006944444444442},\n",
       " 'TN': {0: 7, 1: 8, 2: 4},\n",
       " 'TNR': {0: 0.7777777777777778, 1: 0.8888888888888888, 2: 0.6666666666666666},\n",
       " 'TON': {0: 7, 1: 10, 2: 7},\n",
       " 'TOP': {0: 5, 1: 2, 2: 5},\n",
       " 'TP': {0: 3, 1: 1, 2: 3},\n",
       " 'TPR': {0: 1.0, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'Y': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'dInd': {0: 0.2222222222222222, 1: 0.6758625033664689, 2: 0.6009252125773316},\n",
       " 'sInd': {0: 0.8428651597363228, 1: 0.5220930407198541, 2: 0.5750817072006014}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.class_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>cm.statistic_result</code> prev versions (0.2 >)</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'95% CI': (0.30438856248221097, 0.8622781041844558),\n",
       " 'ACC Macro': 0.7222222222222223,\n",
       " 'ARI': 0.09206349206349207,\n",
       " 'AUNP': 0.6666666666666666,\n",
       " 'AUNU': 0.6944444444444443,\n",
       " 'Bangdiwala B': 0.37254901960784315,\n",
       " 'Bennett S': 0.37500000000000006,\n",
       " 'CBA': 0.4777777777777778,\n",
       " 'CSI': 0.1777777777777778,\n",
       " 'Chi-Squared': 6.6,\n",
       " 'Chi-Squared DF': 4,\n",
       " 'Conditional Entropy': 0.9591479170272448,\n",
       " 'Cramer V': 0.5244044240850757,\n",
       " 'Cross Entropy': 1.5935164295556343,\n",
       " 'F1 Macro': 0.5651515151515151,\n",
       " 'F1 Micro': 0.5833333333333334,\n",
       " 'FNR Macro': 0.38888888888888895,\n",
       " 'FNR Micro': 0.41666666666666663,\n",
       " 'FPR Macro': 0.22222222222222232,\n",
       " 'FPR Micro': 0.20833333333333337,\n",
       " 'Gwet AC1': 0.3893129770992367,\n",
       " 'Hamming Loss': 0.41666666666666663,\n",
       " 'Joint Entropy': 2.4591479170272446,\n",
       " 'KL Divergence': 0.09351642955563438,\n",
       " 'Kappa': 0.35483870967741943,\n",
       " 'Kappa 95% CI': (-0.07707577422109269, 0.7867531935759315),\n",
       " 'Kappa No Prevalence': 0.16666666666666674,\n",
       " 'Kappa Standard Error': 0.2203645326012817,\n",
       " 'Kappa Unbiased': 0.34426229508196726,\n",
       " 'Krippendorff Alpha': 0.3715846994535519,\n",
       " 'Lambda A': 0.16666666666666666,\n",
       " 'Lambda B': 0.42857142857142855,\n",
       " 'Mutual Information': 0.5242078379544426,\n",
       " 'NIR': 0.5,\n",
       " 'NPV Macro': 0.7904761904761904,\n",
       " 'NPV Micro': 0.7916666666666666,\n",
       " 'Overall ACC': 0.5833333333333334,\n",
       " 'Overall CEN': 0.4638112995385119,\n",
       " 'Overall J': (1.225, 0.4083333333333334),\n",
       " 'Overall MCC': 0.36666666666666664,\n",
       " 'Overall MCEN': 0.5189369467580801,\n",
       " 'Overall RACC': 0.3541666666666667,\n",
       " 'Overall RACCU': 0.3645833333333333,\n",
       " 'P-Value': 0.38720703125,\n",
       " 'PPV Macro': 0.5666666666666668,\n",
       " 'PPV Micro': 0.5833333333333334,\n",
       " 'Pearson C': 0.5956833971812705,\n",
       " 'Phi-Squared': 0.5499999999999999,\n",
       " 'RCI': 0.3494718919696284,\n",
       " 'RR': 4.0,\n",
       " 'Reference Entropy': 1.5,\n",
       " 'Response Entropy': 1.4833557549816874,\n",
       " 'SOA1(Landis & Koch)': 'Fair',\n",
       " 'SOA10(Pearson C)': 'Strong',\n",
       " 'SOA2(Fleiss)': 'Poor',\n",
       " 'SOA3(Altman)': 'Fair',\n",
       " 'SOA4(Cicchetti)': 'Poor',\n",
       " 'SOA5(Cramer)': 'Relatively Strong',\n",
       " 'SOA6(Matthews)': 'Weak',\n",
       " 'SOA7(Lambda A)': 'Very Weak',\n",
       " 'SOA8(Lambda B)': 'Moderate',\n",
       " 'SOA9(Krippendorff Alpha)': 'Low',\n",
       " 'Scott PI': 0.34426229508196726,\n",
       " 'Standard Error': 0.14231876063832777,\n",
       " 'TNR Macro': 0.7777777777777777,\n",
       " 'TNR Micro': 0.7916666666666666,\n",
       " 'TPR Macro': 0.611111111111111,\n",
       " 'TPR Micro': 0.5833333333333334,\n",
       " 'Zero-one Loss': 5}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.overall_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>_</code> removed from overall statistics names in <span style=\"color:red;\">version 1.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1.0, 1: 0.0, 2: 0.0},\n",
       " 1: {0: 0.0, 1: 0.33333, 2: 0.66667},\n",
       " 2: {0: 0.33333, 1: 0.16667, 2: 0.5}}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1.0, 1: 0.0, 2: 0.0},\n",
       " 1: {0: 0.0, 1: 0.33333, 2: 0.66667},\n",
       " 2: {0: 0.33333, 1: 0.16667, 2: 0.5}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>matrix</code>, <code>normalized_matrix</code> & <code>normalized_table</code> added in <span style=\"color:red;\">version 1.5</span> (changed from print style)</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_actu = numpy.array([2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2])\n",
    "y_pred = numpy.array([0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred,digit=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>numpy.array</code> support in <span style=\"color:red;\">versions > 0.7</span></li>\n",
    "</ul>  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred, classes=[1, 0, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 1       0       2       \n",
      "Actual\n",
      "1       1       0       2       \n",
      "\n",
      "0       0       3       0       \n",
      "\n",
      "2       1       2       3       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred, classes=[0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       \n",
      "Actual\n",
      "0       3       0       \n",
      "\n",
      "1       0       1       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-4.2-py3.5.egg\\pycm\\utils.py:399: RuntimeWarning: Specified classes are not a subset of the classes in the actual and predicted vectors.\n"
     ]
    }
   ],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred, classes=[1, 0, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 1       0       4       \n",
      "Actual\n",
      "1       1       0       0       \n",
      "\n",
      "0       0       3       0       \n",
      "\n",
      "4       0       0       0       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>classes</code> added in <span style=\"color:red;\">version 3.2</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 in cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "10 in cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm[1][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>__getitem__</code> and <code>__contains__</code> methods added in <span style=\"color:red;\">version 3.8</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Direct CM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2 = ConfusionMatrix(matrix={0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}, digit=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2.actual_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2.predict_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACC': {0: 0.8333333333333334, 1: 0.75, 2: 0.5833333333333334},\n",
       " 'AGF': {0: 0.9135962935560564, 1: 0.5399492471560389, 2: 0.5515973485146916},\n",
       " 'AGM': {0: 0.837285964012303, 1: 0.6919986974962765, 2: 0.6071224016819726},\n",
       " 'AM': {0: 2, 1: -1, 2: -1},\n",
       " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n",
       " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n",
       " 'AUPR': {0: 0.8, 1: 0.41666666666666663, 2: 0.55},\n",
       " 'BB': {0: 0.6, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'BCD': {0: 0.08333333333333333,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.041666666666666664},\n",
       " 'BM': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'CEN': {0: 0.25, 1: 0.49657842846620864, 2: 0.6044162769630221},\n",
       " 'DOR': {0: 'None', 1: 3.999999999999998, 2: 1.9999999999999998},\n",
       " 'DP': {0: 'None', 1: 0.331933069996499, 2: 0.16596653499824957},\n",
       " 'DPI': {0: 'None', 1: 'Poor', 2: 'Poor'},\n",
       " 'ERR': {0: 0.16666666666666663, 1: 0.25, 2: 0.41666666666666663},\n",
       " 'F0.5': {0: 0.6521739130434783,\n",
       "  1: 0.45454545454545453,\n",
       "  2: 0.5769230769230769},\n",
       " 'F1': {0: 0.75, 1: 0.4, 2: 0.5454545454545454},\n",
       " 'F2': {0: 0.8823529411764706, 1: 0.35714285714285715, 2: 0.5172413793103449},\n",
       " 'FDR': {0: 0.4, 1: 0.5, 2: 0.4},\n",
       " 'FN': {0: 0, 1: 2, 2: 3},\n",
       " 'FNR': {0: 0.0, 1: 0.6666666666666667, 2: 0.5},\n",
       " 'FOR': {0: 0.0, 1: 0.19999999999999996, 2: 0.4285714285714286},\n",
       " 'FP': {0: 2, 1: 1, 2: 2},\n",
       " 'FPR': {0: 0.2222222222222222,\n",
       "  1: 0.11111111111111116,\n",
       "  2: 0.33333333333333337},\n",
       " 'G': {0: 0.7745966692414834, 1: 0.408248290463863, 2: 0.5477225575051661},\n",
       " 'GI': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'GM': {0: 0.8819171036881969, 1: 0.5443310539518174, 2: 0.5773502691896257},\n",
       " 'HD': {0: 2, 1: 3, 2: 5},\n",
       " 'IBA': {0: 0.9506172839506174, 1: 0.1316872427983539, 2: 0.2777777777777778},\n",
       " 'ICSI': {0: 0.6000000000000001,\n",
       "  1: -0.16666666666666674,\n",
       "  2: 0.10000000000000009},\n",
       " 'IS': {0: 1.263034405833794, 1: 1.0, 2: 0.2630344058337938},\n",
       " 'J': {0: 0.6, 1: 0.25, 2: 0.375},\n",
       " 'LS': {0: 2.4, 1: 2.0, 2: 1.2},\n",
       " 'MCC': {0: 0.6831300510639732, 1: 0.25819888974716115, 2: 0.1690308509457033},\n",
       " 'MCCI': {0: 'Moderate', 1: 'Negligible', 2: 'Negligible'},\n",
       " 'MCEN': {0: 0.2643856189774724, 1: 0.5, 2: 0.6875},\n",
       " 'MK': {0: 0.6000000000000001, 1: 0.30000000000000004, 2: 0.17142857142857126},\n",
       " 'N': {0: 9, 1: 9, 2: 6},\n",
       " 'NLR': {0: 0.0, 1: 0.7500000000000001, 2: 0.75},\n",
       " 'NLRI': {0: 'Good', 1: 'Negligible', 2: 'Negligible'},\n",
       " 'NPV': {0: 1.0, 1: 0.8, 2: 0.5714285714285714},\n",
       " 'OC': {0: 1.0, 1: 0.5, 2: 0.6},\n",
       " 'OOC': {0: 0.7745966692414834, 1: 0.4082482904638631, 2: 0.5477225575051661},\n",
       " 'OP': {0: 0.7083333333333334, 1: 0.2954545454545454, 2: 0.4404761904761905},\n",
       " 'P': {0: 3, 1: 3, 2: 6},\n",
       " 'PLR': {0: 4.5, 1: 2.9999999999999987, 2: 1.4999999999999998},\n",
       " 'PLRI': {0: 'Poor', 1: 'Poor', 2: 'Poor'},\n",
       " 'POP': {0: 12, 1: 12, 2: 12},\n",
       " 'PPV': {0: 0.6, 1: 0.5, 2: 0.6},\n",
       " 'PRE': {0: 0.25, 1: 0.25, 2: 0.5},\n",
       " 'Q': {0: 'None', 1: 0.6, 2: 0.3333333333333333},\n",
       " 'QI': {0: 'None', 1: 'Moderate', 2: 'Weak'},\n",
       " 'RACC': {0: 0.10416666666666667,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.20833333333333334},\n",
       " 'RACCU': {0: 0.1111111111111111,\n",
       "  1: 0.04340277777777778,\n",
       "  2: 0.21006944444444442},\n",
       " 'TN': {0: 7, 1: 8, 2: 4},\n",
       " 'TNR': {0: 0.7777777777777778, 1: 0.8888888888888888, 2: 0.6666666666666666},\n",
       " 'TON': {0: 7, 1: 10, 2: 7},\n",
       " 'TOP': {0: 5, 1: 2, 2: 5},\n",
       " 'TP': {0: 3, 1: 1, 2: 3},\n",
       " 'TPR': {0: 1.0, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'Y': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'dInd': {0: 0.2222222222222222, 1: 0.6758625033664689, 2: 0.6009252125773316},\n",
       " 'sInd': {0: 0.8428651597363228, 1: 0.5220930407198541, 2: 0.5750817072006014}}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.class_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'95% CI': (0.30438856248221097, 0.8622781041844558),\n",
       " 'ACC Macro': 0.7222222222222223,\n",
       " 'ARI': 0.09206349206349207,\n",
       " 'AUNP': 0.6666666666666666,\n",
       " 'AUNU': 0.6944444444444443,\n",
       " 'Bangdiwala B': 0.37254901960784315,\n",
       " 'Bennett S': 0.37500000000000006,\n",
       " 'CBA': 0.4777777777777778,\n",
       " 'CSI': 0.1777777777777778,\n",
       " 'Chi-Squared': 6.6,\n",
       " 'Chi-Squared DF': 4,\n",
       " 'Conditional Entropy': 0.9591479170272448,\n",
       " 'Cramer V': 0.5244044240850757,\n",
       " 'Cross Entropy': 1.5935164295556343,\n",
       " 'F1 Macro': 0.5651515151515151,\n",
       " 'F1 Micro': 0.5833333333333334,\n",
       " 'FNR Macro': 0.38888888888888895,\n",
       " 'FNR Micro': 0.41666666666666663,\n",
       " 'FPR Macro': 0.22222222222222232,\n",
       " 'FPR Micro': 0.20833333333333337,\n",
       " 'Gwet AC1': 0.3893129770992367,\n",
       " 'Hamming Loss': 0.41666666666666663,\n",
       " 'Joint Entropy': 2.4591479170272446,\n",
       " 'KL Divergence': 0.09351642955563438,\n",
       " 'Kappa': 0.35483870967741943,\n",
       " 'Kappa 95% CI': (-0.07707577422109269, 0.7867531935759315),\n",
       " 'Kappa No Prevalence': 0.16666666666666674,\n",
       " 'Kappa Standard Error': 0.2203645326012817,\n",
       " 'Kappa Unbiased': 0.34426229508196726,\n",
       " 'Krippendorff Alpha': 0.3715846994535519,\n",
       " 'Lambda A': 0.16666666666666666,\n",
       " 'Lambda B': 0.42857142857142855,\n",
       " 'Mutual Information': 0.5242078379544426,\n",
       " 'NIR': 0.5,\n",
       " 'NPV Macro': 0.7904761904761904,\n",
       " 'NPV Micro': 0.7916666666666666,\n",
       " 'Overall ACC': 0.5833333333333334,\n",
       " 'Overall CEN': 0.4638112995385119,\n",
       " 'Overall J': (1.225, 0.4083333333333334),\n",
       " 'Overall MCC': 0.36666666666666664,\n",
       " 'Overall MCEN': 0.5189369467580801,\n",
       " 'Overall RACC': 0.3541666666666667,\n",
       " 'Overall RACCU': 0.3645833333333333,\n",
       " 'P-Value': 0.38720703125,\n",
       " 'PPV Macro': 0.5666666666666668,\n",
       " 'PPV Micro': 0.5833333333333334,\n",
       " 'Pearson C': 0.5956833971812705,\n",
       " 'Phi-Squared': 0.5499999999999999,\n",
       " 'RCI': 0.3494718919696284,\n",
       " 'RR': 4.0,\n",
       " 'Reference Entropy': 1.5,\n",
       " 'Response Entropy': 1.4833557549816874,\n",
       " 'SOA1(Landis & Koch)': 'Fair',\n",
       " 'SOA10(Pearson C)': 'Strong',\n",
       " 'SOA2(Fleiss)': 'Poor',\n",
       " 'SOA3(Altman)': 'Fair',\n",
       " 'SOA4(Cicchetti)': 'Poor',\n",
       " 'SOA5(Cramer)': 'Relatively Strong',\n",
       " 'SOA6(Matthews)': 'Weak',\n",
       " 'SOA7(Lambda A)': 'Very Weak',\n",
       " 'SOA8(Lambda B)': 'Moderate',\n",
       " 'SOA9(Krippendorff Alpha)': 'Low',\n",
       " 'Scott PI': 0.34426229508196726,\n",
       " 'Standard Error': 0.14231876063832777,\n",
       " 'TNR Macro': 0.7777777777777777,\n",
       " 'TNR Micro': 0.7916666666666666,\n",
       " 'TPR Macro': 0.611111111111111,\n",
       " 'TPR Micro': 0.5833333333333334,\n",
       " 'Zero-one Loss': 5}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.overall_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "    <li>In direct matrix mode <code>actual_vector</code> and <code>predict_vector</code> are empty</li>\n",
    "</ul> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = [[1, 2, 3], [4, 6, 1], [1, 2, 3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(matrix=array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  confusion matrices input in array format is new in <span style=\"color:red;\">version 3.6</span> </li>\n",
    "</ul> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1, 1: 2, 2: 3}, 1: {0: 4, 1: 6, 2: 1}, 2: {0: 1, 1: 2, 2: 3}}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1, 1: 2, 2: 3}, 1: {0: 4, 1: 6, 2: 1}, 2: {0: 1, 1: 2, 2: 3}}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 0.16667, 1: 0.33333, 2: 0.5},\n",
       " 1: {0: 0.36364, 1: 0.54545, 2: 0.09091},\n",
       " 2: {0: 0.16667, 1: 0.33333, 2: 0.5}}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 0.16667, 1: 0.33333, 2: 0.5},\n",
       " 1: {0: 0.36364, 1: 0.54545, 2: 0.09091},\n",
       " 2: {0: 0.16667, 1: 0.33333, 2: 0.5}}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       2       \n",
      "Actual\n",
      "0       1       2       3       \n",
      "\n",
      "1       4       6       1       \n",
      "\n",
      "2       1       2       3       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = numpy.array([[1, 2, 3], [4, 6, 1], [1, 2, 3]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(matrix=array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(matrix=array, classes=[\"L1\", \"L2\", \"L3\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3'])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Iterating and casting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From `version 3.5`, `ConfusionMatrix` is an **iterator** object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 {'L2': 6, 'L3': 1, 'L1': 4}\n",
      "L3 {'L2': 2, 'L3': 3, 'L1': 1}\n",
      "L1 {'L2': 2, 'L3': 3, 'L1': 1}\n"
     ]
    }
   ],
   "source": [
    "for row, col in cm:\n",
    "    print(row, col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('L2', {'L1': 4, 'L2': 6, 'L3': 1})"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_iter = iter(cm)\n",
    "next(cm_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': {'L1': 1, 'L2': 2, 'L3': 3},\n",
       " 'L2': {'L1': 4, 'L2': 6, 'L3': 1},\n",
       " 'L3': {'L1': 1, 'L2': 2, 'L3': 3}}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_dict = dict(cm)\n",
    "cm_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('L2', {'L1': 4, 'L2': 6, 'L3': 1}),\n",
       " ('L3', {'L1': 1, 'L2': 2, 'L3': 3}),\n",
       " ('L1', {'L1': 1, 'L2': 2, 'L3': 3})]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_list = list(cm)\n",
    "cm_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activation threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`threshold` is added in `version 0.9` for real value prediction.\t\t\t\n",
    "\t\t\t\t\t\t\n",
    "For more information visit <a href=\"#Example-3-(Activation-threshold)\">Example 3</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load from file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` is added in `version 0.9.5` in order to load saved confusion matrix with `.obj` format generated by `save_obj` method.\n",
    "\n",
    "For more information visit <a href=\"#Example-4-(File)\">Example 4</a>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`sample_weight` is added in `version 1.2`\n",
    "\n",
    "For more information visit <a href=\"#Example-5-(Sample-weights)\">Example 5</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transpose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`transpose` is added in `version 1.2` in order to transpose input matrix (only in `Direct CM` mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(matrix={0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}, digit=5, transpose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       2       \n",
      "Actual\n",
      "0       3       0       2       \n",
      "\n",
      "1       0       1       1       \n",
      "\n",
      "2       0       2       3       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics off"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`metrics_off` is added in `version 3.9` in order to bypass metrics calculation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm3 = ConfusionMatrix(y_actu, y_pred, metrics_off=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AGF': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AGM': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AM': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AUC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AUCI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'AUPR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'BB': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'BCD': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'BM': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'CEN': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'DOR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'DP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'DPI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'ERR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'F0.5': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'F1': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'F2': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FDR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FN': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FNR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FOR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'FPR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'G': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'GI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'GM': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'HD': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'IBA': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'ICSI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'IS': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'J': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'LS': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'MCC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'MCCI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'MCEN': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'MK': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'N': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'NLR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'NLRI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'NPV': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'OC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'OOC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'OP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'P': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'PLR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'PLRI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'POP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'PPV': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'PRE': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'Q': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'QI': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'RACC': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'RACCU': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TN': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TNR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TON': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TOP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TP': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'TPR': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'Y': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'dInd': {0: 'None', 1: 'None', 2: 'None'},\n",
       " 'sInd': {0: 'None', 1: 'None', 2: 'None'}}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm3.class_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'95% CI': 'None',\n",
       " 'ACC Macro': 'None',\n",
       " 'ARI': 'None',\n",
       " 'AUNP': 'None',\n",
       " 'AUNU': 'None',\n",
       " 'Bangdiwala B': 'None',\n",
       " 'Bennett S': 'None',\n",
       " 'CBA': 'None',\n",
       " 'CSI': 'None',\n",
       " 'Chi-Squared': 'None',\n",
       " 'Chi-Squared DF': 'None',\n",
       " 'Conditional Entropy': 'None',\n",
       " 'Cramer V': 'None',\n",
       " 'Cross Entropy': 'None',\n",
       " 'F1 Macro': 'None',\n",
       " 'F1 Micro': 'None',\n",
       " 'FNR Macro': 'None',\n",
       " 'FNR Micro': 'None',\n",
       " 'FPR Macro': 'None',\n",
       " 'FPR Micro': 'None',\n",
       " 'Gwet AC1': 'None',\n",
       " 'Hamming Loss': 'None',\n",
       " 'Joint Entropy': 'None',\n",
       " 'KL Divergence': 'None',\n",
       " 'Kappa': 'None',\n",
       " 'Kappa 95% CI': 'None',\n",
       " 'Kappa No Prevalence': 'None',\n",
       " 'Kappa Standard Error': 'None',\n",
       " 'Kappa Unbiased': 'None',\n",
       " 'Krippendorff Alpha': 'None',\n",
       " 'Lambda A': 'None',\n",
       " 'Lambda B': 'None',\n",
       " 'Mutual Information': 'None',\n",
       " 'NIR': 'None',\n",
       " 'NPV Macro': 'None',\n",
       " 'NPV Micro': 'None',\n",
       " 'Overall ACC': 'None',\n",
       " 'Overall CEN': 'None',\n",
       " 'Overall J': 'None',\n",
       " 'Overall MCC': 'None',\n",
       " 'Overall MCEN': 'None',\n",
       " 'Overall RACC': 'None',\n",
       " 'Overall RACCU': 'None',\n",
       " 'P-Value': 'None',\n",
       " 'PPV Macro': 'None',\n",
       " 'PPV Micro': 'None',\n",
       " 'Pearson C': 'None',\n",
       " 'Phi-Squared': 'None',\n",
       " 'RCI': 'None',\n",
       " 'RR': 'None',\n",
       " 'Reference Entropy': 'None',\n",
       " 'Response Entropy': 'None',\n",
       " 'SOA1(Landis & Koch)': 'None',\n",
       " 'SOA10(Pearson C)': 'None',\n",
       " 'SOA2(Fleiss)': 'None',\n",
       " 'SOA3(Altman)': 'None',\n",
       " 'SOA4(Cicchetti)': 'None',\n",
       " 'SOA5(Cramer)': 'None',\n",
       " 'SOA6(Matthews)': 'None',\n",
       " 'SOA7(Lambda A)': 'None',\n",
       " 'SOA8(Lambda B)': 'None',\n",
       " 'SOA9(Krippendorff Alpha)': 'None',\n",
       " 'Scott PI': 'None',\n",
       " 'Standard Error': 'None',\n",
       " 'TNR Macro': 'None',\n",
       " 'TNR Micro': 'None',\n",
       " 'TPR Macro': 'None',\n",
       " 'TPR Micro': 'None',\n",
       " 'Zero-one Loss': 'None'}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm3.overall_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Relabel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`relabel` method is added in `version 1.5` in order to change ConfusionMatrix class names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm.relabel(mapping={0:\"L1\",1:\"L2\",2:\"L3\"}, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3'])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `mapping` : mapping dictionary (type : `dict`)\n",
    "2. `sort` : flag for sorting new classes (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>sort</code> added in <span style=\"color:red;\">version 3.9</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Position"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`position` method is added in `version 2.8` in order to find the indexes of observations in `predict_vector` which made TP, TN, FP, FN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'FN': [], 'FP': [0, 7], 'TN': [2, 3, 5, 6, 8, 10, 11], 'TP': [1, 4, 9]},\n",
       " 1: {'FN': [5, 10], 'FP': [3], 'TN': [0, 1, 2, 4, 7, 8, 9, 11], 'TP': [6]},\n",
       " 2: {'FN': [0, 3, 7], 'FP': [5, 10], 'TN': [1, 4, 6, 9], 'TP': [2, 8, 11]}}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm3 = ConfusionMatrix(y_actu, y_pred, digit=5)\n",
    "cm3.position()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  only works in vector mode </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To array"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`to_array` method is added in `version 2.9` in order to returns the confusion matrix in the form of a NumPy array. This can be helpful to apply different operations over the confusion matrix for different purposes such as aggregation, normalization, and combination."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3, 0, 2],\n",
       "       [0, 1, 1],\n",
       "       [0, 2, 3]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.to_array()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.6, 0. , 0.4],\n",
       "       [0. , 0.5, 0.5],\n",
       "       [0. , 0.4, 0.6]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.to_array(normalized=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.6, 0.4],\n",
       "       [0. , 1. ]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.to_array(normalized=True, one_vs_all=True, class_name=\"L1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `normalized` : a flag for getting normalized confusion matrix (type : `bool`, default : `False`)\n",
    "2. `one_vs_all` : one-vs-all mode flag (type : `bool`, default : `False`)\n",
    "3. `class_name` : target class name for one-vs-all mode (type : `any valid type`, default : `None`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Confusion Matrix in NumPy array format`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`combine` method is added in `version 3.0` in order to merge two confusion matrices. This option will be useful in mini-batch learning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       2       \n",
      "Actual\n",
      "0       6       0       0       \n",
      "\n",
      "1       0       2       4       \n",
      "\n",
      "2       4       2       6       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm_combined = cm2.combine(cm3)\n",
    "cm_combined.print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `other` : the other matrix that is going to be combined (type : `ConfusionMatrix`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New `ConfusionMatrix`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`plot` method is added in `version 3.0` in order to plot a confusion matrix using Matplotlib or Seaborn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyring is skipped due to an exception: invalid syntax (core.py, line 48)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyring is skipped due to an exception: invalid syntax (core.py, line 48)\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable}  -m pip -q -q install matplotlib;\n",
    "!{sys.executable}  -m pip -q -q install seaborn;\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x41e71b0>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEWCAYAAAAZ7jAvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGztJREFUeJzt3Xu4HFWd7vHvm4DhfpGoaEBwuMigozjkIIoiKDIBEZwBFEaQKPPwqDDICF5QDyp6joIzcVQQDA/KVcSoOJGLqIijOILsxHAJoOSgSIYoCWAwckvwPX9U7dj09O7uvdO9+1Lv53nqSVfV2qt+u0l+rFVr1SrZJiJi2E3pdQAREZMhyS4iKiHJLiIqIckuIiohyS4iKiHJLiIqIcmuIiRtKOk7klZKmrcO9bxV0vc6GVsvSLpG0jG9jiMmT5Jdn5H0j5JGJK2StKz8R/mqDlR9GPAcYCvbh0+0EtuX2t6/A/E8jaR9JFnSt+qOv7Q8/qM26/mYpEtalbN9gO0LJxhuDKAkuz4i6b3AvwP/lyIxPR/4InBIB6rfDviV7TUdqKtblgOvlLRVzbFjgF916gIq5O99FdnO1gcbsDmwCji8SZlpFMnw/nL7d2BaeW4fYClwMvAAsAx4e3nu48CTwOryGscCHwMuqal7e8DAeuX+bOAe4I/Ar4G31hy/oebnXgncDKws/3xlzbkfAZ8AflrW8z1g+hi/22j85wLHl8emlsdOA35UU/ZzwH3AI8AC4NXl8Vl1v+ctNXH8nzKOx4Ady2P/VJ4/B/hGTf1nANcB6vXfi2yd2/J/uP7xCmAD4IomZT4M7AnsBrwU2AP4SM35rSmS5gyKhHa2pC1tf5SitXi57U1sn98sEEkbA58HDrC9KUVCW9Sg3DOBq8qyWwFzgKvqWmb/CLwdeDbwDOCUZtcGLgLeVn7+O2AxRWKvdTPFd/BM4KvAPEkb2P5u3e/50pqfORo4DtgUuLeuvpOBl0iaLenVFN/dMS4zXwyHJLv+sRWwws27mW8FTrf9gO3lFC22o2vOry7Pr7Z9NUXr5oUTjOfPwIslbWh7me3FDcq8Abjb9sW219i+DLgLeGNNma/Y/pXtx4CvUySpMdn+L+CZkl5IkfQualDmEtsPltf8N4oWb6vf8wLbi8ufWV1X36PAURTJ+hLgn20vbVFfDJgku/7xIDBd0npNyjyPp7dK7i2Pra2jLlk+Cmwy3kBs/wl4C/BOYJmkqyTt0kY8ozHNqNn/3QTiuRg4AdiXBi1dSSdLurMcWf4DRWt2eos672t20vbPKbrtokjKMWSS7PrHz4DHgTc1KXM/xUDDqOfzP7t47foTsFHN/ta1J21fa/v1wHMpWmvntRHPaEz/PcGYRl0MvBu4umx1rVV2Mz8AvBnY0vYWFPcLNRr6GHU27ZJKOp6ihXg/8P6Jhx79KsmuT9heSXEj/mxJb5K0kaT1JR0g6cyy2GXARyQ9S9L0snzLaRZjWATsLen5kjYHTh09Iek5kg4u7909QdEdfqpBHVcDO5fTZdaT9BZgV+DKCcYEgO1fA6+huEdZb1NgDcXI7XqSTgM2qzn/e2D78Yy4StoZ+CRFV/Zo4P2Smna3Y/Ak2fUR23OA91IMOiyn6HqdAHy7LPJJYAS4FbgNWFgem8i1vg9cXta1gKcnqCkUN+3vBx6iSDzvblDHg8BBZdkHKVpEB9leMZGY6uq+wXajVuu1wDUU01HupWgN13ZRRydMPyhpYavrlLcNLgHOsH2L7buBDwEXS5q2Lr9D9BdlwCkiqiAtu4iohCS7iOgrkjaQ9HNJt0haLOnjDcpMk3S5pCWSbpK0fat6k+wiot88Aby2nBS+GzBL0p51ZY4FHra9I/BZiqdemkqyi4i+4sKqcnf9cqsfXDgEGF3I4RvA6ySJJppNYO0LkjKC0sTuu+/e6xD63j333NPrEPreww8/vML2syb68+P8d7qYYhR91Fzbc+vqm0oxS2BH4GzbN9XVMYNyFN72GkkrKZ9CGuuifZ/sormRkZFeh9D33vzmN/c6hL43b968+idhuulx2zObFbD9FLCbpC2AKyS92PbtNUUateKaJtx0YyOiIyS1tY2H7T9QrFAzq+7UUmDb8rrrUTwy+FCzupLsIqIjpkyZ0tbWSvmE0Bbl5w2B/SgeWaw1n2KtQygWpv1hq1Vq0o2NiI4Yb6utiecCF5b37aYAX7d9paTTgRHb84HzKZ5yWULRojuiVaVJdhGxzibSRR2L7VuBlzU4flrN58eBcb1eIMkuIjqigy27rkiyi4iOSLKLiEpIsouIoSeprZHWXkqyi4iOSMsuIiohyS4iKiHJLiIqIckuIoZeBigiojLSsouISkiyi4hKSLKLiKHXyYUAuiXJLiI6IskuIioho7ERUQlp2UXE0Ms9u4iojCS7iKiEJLuIqIQMUETE0Ms9u4iojCS7iKiEJLuIqIQku4iohCS7iBh6WbwzIiqj31t2XUvFklY1OLa3pIWS1kg6rFvXjojJNzr9pNXWRj3bSrpe0p2SFkt6T4My+0haKWlRuZ3Wqt7Jbtn9FpgNnDLJ142ILutgy24NcLLthZI2BRZI+r7tO+rK/cT2Qe1WOqnJzvZvACT9eTKvGxHd1clJxbaXAcvKz3+UdCcwA6hPduPS33cUI2JgdKobW1fn9sDLgJsanH6FpFskXSPpRa3q6ssBCknHAcf1Oo6IaN84RmOnSxqp2Z9re259IUmbAN8ETrL9SN3phcB2tldJOhD4NrBTs4v2ZbIrf/G5AJLc43Aiog3jaLWtsD2zRV3rUyS6S21/q/58bfKzfbWkL0qabnvFWHX2ZbKLiMHSyXt2Kio6H7jT9pwxymwN/N62Je1BcUvuwWb1djPZbSRpac3+HOAnwBXAlsAbJX3cdsu+dkT0vw6Oxu4FHA3cJmlReexDwPMBbJ8LHAa8S9Ia4DHgCNtNe4FdS3a2x+rAb9Ota0ZE73RwNPYGoGllts8CzhpPvenGRkRH5HGxiBh6WbwzIiojyS4iKiHJLiIqIckuIiohyS4ihl4W74yIykjLLiIqIckuIiohyS4ihl4mFUdEZSTZRUQlZDQ2IiohLbuIGHq5ZxcRlZFkFxGVkGQXEZWQAYqIGHq5ZxcRlZFkFxGVkGQXEZWQZBcRlZBkFxFDL4t3RkRlpGUXEZWQZBcRlZBkFxFDbxAmFff3HcWIGBijCa/V1kY920q6XtKdkhZLek+DMpL0eUlLJN0q6W9b1ZuWXUR0RAdHY9cAJ9teKGlTYIGk79u+o6bMAcBO5fZy4Jzyz7Hj61R0EVFd7bbq2mnZ2V5me2H5+Y/AncCMumKHABe5cCOwhaTnNqu371t2u+++OyMjI70Oo2/Nmzev1yH0vcMPP7zXIfS9Tvw9Gsc9u+mSav9Rz7U9d4w6twdeBtxUd2oGcF/N/tLy2LKxLtr3yS4iBsM4kt0K2zPbqG8T4JvASbYfqT/d4EfcrL6W3VhJZ0raTNL6kq6TtELSUa1+LiKqpVPd2LKu9SkS3aW2v9WgyFJg25r9bYD7m9XZzj27/cuselB5gZ2B97UVcURUwujjYu1sbdQl4HzgTttzxig2H3hbOSq7J7DS9phdWGivG7t++eeBwGW2H+r3+TQRMfk6mBf2Ao4GbpO0qDz2IeD5ALbPBa6myElLgEeBt7eqtJ1k9x1JdwGPAe+W9Czg8XGHHxFDrVPJzvYNNL4nV1vGwPHjqbdlm9L2B4FXADNtr6bIooeM5yIRMfw6ec+uG9oZoNiIIoOeUx56HtByJCUiqmXgkx3wFeBJ4JXl/lLgk12LKCIGTicnFXdLO8luB9tnAqsBbD9Gi/50RFRPp0Zju6WdAYonJW1IOWFP0g7AE12NKiIGTr/P0mgn2X0U+C6wraRLKYaFZ3czqIgYPAOf7Gx/X9JCYE+K7ut7bK/oemQRMTB6fT+uHe2Mxu4FPG77KmAL4EOStut6ZBExUIZhgOIc4FFJL6V4TOxe4KKuRhURA6ffByjaufKacrbyIcDnbX8O2LS7YUXEoOn3ll07AxR/lHQqcBSwt6Sp/OV52YiInieydrTTsnsLxVSTY23/jmKBvM90NaqIGDhD0bIDPmf7KUk7A7sAl3U3rIgYNMPQsvsxME3SDOA6iqVULuhmUBExePq9ZddOspPtR4F/AL5g+++BF3U3rIgYJJ1cvLNb2unGStIrgLcCx5bHpnYvpIgYRP3ejW0n2b0HOBW4wvZiSX8FXN/dsCJi0Ax8srP9Y4r7dqP79wAndjOoiBg8A5/symXY309xn26D0eO2X9vFuCJiwPR7smvnbuGlwF3AC4CPA78Bbu5iTBExYIZl8c6tbJ8PrLb9n7bfQbECSkTEWsMwGru6/HOZpDdQvIh2m+6FFBGDqN+7se0ku09K2hw4GfgCsBnwL12NKiIGzsAnO9tXlh9XAvt2N5yIGES9vh/XjjGTnaQvUL53ohHbmX4SEWsNbLIDRiYtiogYeL0cfGhHs2R3ObCp7eW1ByU9G3ikq1FFxMDp95Zds1T8eeDVDY6/Hvhsq4olrWpw7L2S7pB0q6Tr8i6LiOHQyXl2kr4s6QFJt49xfh9JKyUtKrfT2omxWbJ7le1v1R+0fSmwdzuVN/ALYKbtlwDfAM6cYD0R0Wc6OKn4AmBWizI/sb1buZ3eTqXNkl2zqCbUObd9fblcFMCNZL5exNDoVLIrn8d/qNPxNUtaD0jao/6gpP8FLG9QfryOBa5pdELScZJGJI0sX96JS0VEt40j2U0f/fddbsdN4HKvkHSLpGsktbW+ZrMBivcBX5d0AbCgPDYTeBtwxASCW0vSUWVdr2l03vZcYC7AzJkzx5z+EhH9YXTxzjatsD1zHS63ENjO9ipJBwLfBnZq9UNjRmf758AeFN3Z2eUm4OW2b5polJL2Az4MHGz7iYnWExH9ZbIWArD9iO1V5eergfUlTW/1c02foLD9APDRdY6uJOllwJeAWWXdETEkJmvqiaStgd/bdnmrbQrwYKufa+fZ2InaSNLSmv05wIHAJsC88ov5re2DuxhDREySTiU7SZcB+1Dc21tK0eBaH8D2ucBhwLskrQEeA46w3fJ2V9eSne1GXeQ53bpeRPRWp5Kd7SNbnD8LOGu89XazZRcRFTHoCwF8h+YLAaT7GRFrDfKzsf86aVFExMAb2Jad7f+czEAiYrANbLIbJWkn4FPArjz97WJ/1cW4ImKADMI9u3Y62V8BzgHWUKxUfBFwcTeDiojBMwxvF9vQ9nWAbN9r+2NA3hkbEU8zDG8Xe1zSFOBuSScA/w08u7thRcSgGYZu7EnARsCJwO7A0cAx3QwqIgZLJxfv7JZ23i52c/lxFfD27oYTEYOq31t27YzGXk+DycW2c98uItYa+GQHnFLzeQPgUIqR2YiItQY+2dleUHfop5Iy4Tgi1hrn4p090U439pk1u1MoBim27lpEETGQBr5lR7EkuylWKV4D/Jri/REREWsNQ7L7a9uP1x6QNK1L8UTEgOr3ZNdOJ/u/Ghz7WacDiYjBNrDz7Mp13mcAG5bvjhiNcjOKScYREcBgLATQrBv7dxRvFNsG+Df+kuweAT7U3bAiYtAM7Gis7QuBCyUdavubkxhTRAygfm/ZtZOKd5e0xeiOpC0lfbKLMUXEAOr3e3btJLsDbP9hdMf2wxSvRIyIAIZkIQBgqqRptp8AkLQhkKknEfE0/d6NbSfZXQJcJ+krFJOL30GxWnFExFoDO0AxyvaZkm4F9qMYkf2E7Wu7HllEDIxed1Hb0dZLsm1/F/gugKS9JJ1t+/iuRhYRA2Uokp2k3YAjgbdQPBv7rW4GFRGDZ2CTnaSdgSMoktyDwOUUL93Zd5Jii4gB0qlkJ+nLwEHAA7Zf3OC8gM9RzAp5FJhte2GrepvdUbwLeB3wRtuvsv0F4KmJBB8Rw6+DU08uAGY1OX8AsFO5HUfxqteWmiW7Q4HfAddLOk/S6/jLI2MREWuNLt7ZiVcp2v4x8FCTIocAF7lwI7CFpOe2qrfZ42JXAFdI2hh4E/AvwHMknQNcYft7LaOOrps3b16vQ+h7+Y4mxzi6sdMljdTsz7U9dxyXmgHcV7O/tDy2rNkPtTP15E/ApcCl5arFhwMfBJLsImKtcSS7FbZnrsulGhz7Hy8FqzeuWYC2H7L9pbxZLCLqTeLjYkuBbWv2twHub/VD/T3lOSIGwiQ/GzsfeJsKewIrbTftwkKb8+wiIlrp1ONiki4D9qG4t7cU+CiwPoDtc4GrKaadLKGYevL2dupNsouIjujUPDvbR7Y4b2DcT3Al2UVERwzsExQREe0amoUAIiJaSbKLiEpIsouIShj4xTsjIlrJPbuIqIwku4iohCS7iKiEJLuIqIQku4gYeqOLd/azJLuI6Ii07CKiEpLsIqISkuwiYuhlUnFEVEYGKCKiEtKyi4hKSLKLiKGXe3YRURlJdhFRCUl2EVEJGY2NiKGXe3YRURlJdhFRCUl2EVEJSXYRUQn9nuy6NnwiaVWDY++UdJukRZJukLRrt64fEZNndPHOdrY265sl6ZeSlkj6YIPzsyUtL3PJIkn/1KrOyW7ZfdX2uQCSDgbmALMmOYaI6IJOtewkTQXOBl4PLAVuljTf9h11RS+3fUK79U7qxBjbj9Tsbgx4Mq8fEd0zOv2k1daGPYAltu+x/STwNeCQdY1v0mcBSjpe0v8DzgROnOzrR0R3dDDZzQDuq9lfWh6rd6ikWyV9Q9K2rSqd9GRn+2zbOwAfAD7SqIyk4ySNSBpZvnz55AYYEePWbqIrk9300X/f5XZcfXUNLlHfC/wOsL3tlwA/AC5sFWMvR2O/BpzT6ITtucBcgJkzZ6arGzEAxvG42ArbM5ucXwrUttS2Ae6vLWD7wZrd84AzWsbXbnSdIGmnmt03AHdP5vUjons62I29GdhJ0gskPQM4Aphfd63n1uweDNzZqtJutuw2krS0Zn8OsJ2k/YDVwMPAMV28fkRMok6NxtpeI+kE4FpgKvBl24slnQ6M2J4PnFjO6FgDPATMblVv15Kd7f5eAiEiOqbTCwHYvhq4uu7YaTWfTwVOHU+deYIiIjqi35+gSLKLiI5IsouISsjinREx9LJ4Z0RURpJdRFRCkl1EVEKSXURUQpJdRAy90cU7+1mSXUR0RFp2EVEJSXYRUQlJdhEx9DKpOCIqIwMUEVEJadlFRCUk2UXE0Ms9u4iojCS7iKiEJLuIGHp5XCwiKiMtu4iohCS7iKiEJLuIqIQku4gYeplnFxGVkdHYiKiEtOwiohL6Pdn1d7szIgbC6D27drY265sl6ZeSlkj6YIPz0yRdXp6/SdL2repMsouIjuhUspM0FTgbOADYFThS0q51xY4FHra9I/BZ4IxW9SbZRURHTJkypa2tDXsAS2zfY/tJ4GvAIXVlDgEuLD9/A3idWmTSvr9nt2DBghWS7u11HDWmAyt6HUSfy3fUXD9+P9utyw8vWLDgWknT2yy+gaSRmv25tufW7M8A7qvZXwq8vK6OtWVsr5G0EtiKJt9r3yc728/qdQy1JI3YntnrOPpZvqPmhvH7sT2rg9U1aqF5AmWeJt3YiOg3S4Fta/a3Ae4fq4yk9YDNgYeaVZpkFxH95mZgJ0kvkPQM4Ahgfl2Z+cAx5efDgB/abtqy6/tubB+a27pI5eU7ai7fTxPlPbgTgGuBqcCXbS+WdDowYns+cD5wsaQlFC26I1rVqxbJMCJiKKQbGxGVkGQXEZWQZNeEpFUNju0taaGkNZIO60Vc/WSM7+i9ku6QdKuk6ySt0xyuQTbG9/NOSbdJWiTphgZPB0QXJNmN32+B2cBXexxHP/sFMNP2Syhmt5/Z43j6zVdt/43t3Si+mzm9DqgKkuzGyfZvbN8K/LnXsfQr29fbfrTcvZFinlSUbD9Ss7sxLSbDRmdk6kl027HANb0Oot9IOh54L/AM4LU9DqcS0rKLrpF0FDAT+EyvY+k3ts+2vQPwAeAjvY6nCpLsoisk7Qd8GDjY9hO9jqePfQ14U6+DqIIku+g4SS8DvkSR6B7odTz9RtJONbtvAO7uVSxVkicompD0Z57+APIc4CfAFcCWwOPA72y/qAfh9YUxvqMDgb8BlpXHfmv74MmOrR+M8f1sB+wHrAYeBk6wvbgH4VVKkl1EVEK6sRFRCUl2EVEJSXYRUQlJdhFRCUl2EVEJSXYDQtJT5SoZt0uaJ2mjdahrH0lXlp8PbvQS4pqyW0h69wSu8TFJp4xx7m3l77G4XB3llPL4BVlJJrolyW5wPGZ7N9svBp4E3ll7UoVx//e0Pd/2p5sU2QIYd7Ibi6QDgJOA/cv5iX8LrOxU/RFjSbIbTD8BdpS0vaQ7JX0RWAhsK2l/ST8r19ybJ2kTAEmzJN0l6QbgH0YrkjRb0lnl5+dIukLSLeX2SuDTwA5lq/IzZbn3Sbq5XK/u4zV1fVjSLyX9AHjhGLGfCpxi+34A24/bPq++kKTTymvcLmnu6AuQJZ1Ys1be18pjrynjWyTpF5I2HStOSRtLuqr8/W6X9JZ1+O8Qg8R2tgHYgFXln+sB/wG8C9ieYqmpPctz04EfAxuX+x8ATgM2oHih8E4U79v8OnBlWWY2cFb5+XLgpPLzVIrX020P3F4Tx/4UL4wRxf8srwT2BnYHbgM2AjYDllAktfrf4yFg8zF+xwuAw8rPz6w5fjHwxvLz/cC08vMW5Z/fAfYqP29SfkdjxXkocF5N3Q1jyTZ8W1p2g2NDSYuAEYoFRM8vj99r+8by857ArsBPy7LHUDyatAvwa9t3u/gXfskY13gtcA6A7adsN+pe7l9uv6BoTe5CkURfDVxh+1EX67XVv/puvPaVdJOk28q4Rh/JuxW4tFxRZU157KfAHEknUiTANU3ivA3YT9IZkl49xu8YQyjr2Q2Ox1ysbLtW2bP7U+0h4Pu2j6wrtxudWyBSwKdsf6nuGie1eY3FFK3AH455AWkD4IsUqx3fJ+ljFK1TKB6c3xs4GPjfkl5k+9OSrqJ4JvfGcsWVhnGW9e9elv2UpO/ZPr2NuGPApWU3XG4E9pK0I4CkjSTtDNwFvEDSDmW5I8f4+esousdImippM+CPwKY1Za4F3lFzL3CGpGdTdJ//XtKG5T2zN45xjU8BZ0rauvz5aWWLrNZoYltRXuewsuwUYFvb1wPvpxg82UTSDrZvs30GRct3l7HilPQ84FHblwD/SjFAEhWQlt0Qsb1c0mzgMknTysMfsf0rSccBV0laAdwAvLhBFe8B5ko6FngKeJftn0n6qaTbgWtsv0/SXwM/K1uWq4CjbC+UdDmwCLiXYhClUYxXS3oO8INy0MHAl+vK/EHSeRRdzt9QvCEeivuIl0janKLl9tmy7Cck7VvGfEcZ5xON4gR2BD6jYjWS1ZTJPYZfVj2JiEpINzYiKiHJLiIqIckuIiohyS4iKiHJLiIqIckuIiohyS4iKuH/AzGYSAq9I88AAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1227c1d0>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEWCAYAAAAZ7jAvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt8FNX9//HXOwmKICAQFAgKFPACiheolypCEf2CKN5A0WoFaW2rVq3Feq1VWm/YqnxbasHWKyiKP2kREBUUr8CXixYFRVGhhIuGiyCCQsLn98dM6GbZJEOyye5mP08f83Bn5uyZzw7Zz56ZM3NGZoZzztV1OakOwDnnaoMnO+dcVvBk55zLCp7snHNZwZOdcy4reLJzzmUFT3ZJIGkfSS9I2iRpYjXq+ZGkl5MZWypIelHSpVV8bwtJSyXVT3ZcNUHSEElvxcxvkfS9JG9jlqSfhK8HSJqQzPqzRVYlO0kXSZof/kGuCb+UJyWh6oHAAUBzMxtU1UrMbLyZnZaEeMqQ1EuSSXo+bvmR4fJZEeu5XdK4ysqZWT8ze7yK4d4IPGpm34bbnCXpW0kHxsTRR9LyKtZfo8xsXzP7rAbrnwwcLqlrTW2jrsqaZCfpOuBB4C6CxHQQ8FfgrCRU3xb42MyKk1BXTSkCfiCpecyyS4GPk7UBBar8NyVp7zCm+IT6DfDb6sQWs43cZNSTYk8Dl6c6iIxjZnV+ApoAW4BBFZTZmyAZrg6nB4G9w3W9gELg18CXwBpgaLjuDmA7sCPcxjDgdmBcTN3tAAPywvkhwGfA18DnwI9ilr8V874fAPOATeH/fxCzbhbwe+DtsJ6XgfxyPltp/H8DrgyX5YbLbgNmxZQdBawENgMLgB7h8r5xn/PfMXHcGcaxDegYLvtJuP4h4LmY+u8FZgJKEOfJwLK4ZbOA34WfsWO4rA+wPKbMYWG5r4DFwICYdY+FMUwjSJp9wmV/BV4MP8vbQMvw33wj8BFwdEwdNwKfhjEsAc6JWRf/b2bhPmgd1l06bQUsptxlwIfh9l4C2sasOzWMYRPwF+D10v0Zrj8R+DzV36tMm1IeQK18yOCLWkyYbMopMwKYA+wPtADeAX4frusVvn8EUA84PfzjbRquv52yyS1+vl34JcgDGhIkkkPCda2ALuHrXV8coFn4RbgkfN+F4XzzcP2s8At4MLBPOH9POZ+tF0Fi+wEwN1x2evgl+wllk93FQPNwm78G1gL1E32umDj+A3QJ31OPssmuAUHrcQjQA1gHtCknziuBqQnq/wlwf+m2iUl24faWATcDewG9CZJS6f59jCBpnEhwJFM/XLYO6BbOv0rwo/Njgh+BPwCvxcQwiCB55QAXECTNVvH/ZuG8ESbluM8xHng6fH12GPNh4T67FXgnXJdP8PcxMPxsvyL424tNds3C7TRO9Xcrk6ZsOYxtDqyzig8zfwSMMLMvzayIoMV2Scz6HeH6HWY2jeDX+pAqxrOT4LzLPma2xswWJyjTH/jEzJ40s2Ize5rg1/7MmDKPmtnHZrYNeBY4qqKNmtk7QDNJhxB8sZ9IUGacma0Pt/knghZvZZ/zMTNbHL5nR1x9WwkS6P0Eh6e/NLPCcurZjyBRJXI3cKakLnHLjwf2JUj0283sVWAKwY9DqX+Z2dtmttPCc4HAJDNbEM5PAr41syfMrAR4Bjg65jNMNLPV4fufAT4Bjq1wj8SQdANwKEFrDuBnwN1m9mH4N3kXcJSktgQ/QkvM7LlwXz5I8IMTq3Qf7Rc1Bpc95+zWA/mS8ioo0xpYETO/Ily2q464ZLmV4Eu2R8zsG4LWwc+BNZKmSjo0QjylMRXEzMd+CaLG8yRwFfBDgi95GZJ+LenDsGf5K4JTAPmV1LmyopVm9n8Eh+0iSMrl2Qg0KqeOIoJDuhFxq1oDK81sZ8yy+P2UKL4vYl5vSzC/a19K+rGk9yR9Fe6Tw6l8n5S+tx9wDXB2+KMEwTneUTH1bSDYNwWln6f0/RY05eLjL91HX0WJwQWyJdnNBr4lOHwoz2qCP8JSB4XLquIbgsO3Ui1jV5rZS2Z2KsEh7EfAwxHiKY1pVRVjKvUkcAUwLWx17SKpB3ADcD7BIfp+BIeAKg29nDorHDpH0pUELcTVwG8qKLqI4LC8PPcRJOluMctWAwfGdYzE76cqD+0TtrYeJviBaB7ukw/47z6p6L2HAI8D55tZbMJaCfzMzPaLmfYJW95rgNieZ8XOhw4jOIzfXNXPlY2yItmZ2SaCE/GjJZ0tqYGkepL6SRoZFnsauDW8zis/LF/pZRbleA84WdJBkpoAN5WukHRAeK1UQ+A7gsPhkgR1TAMODi+XyZN0AdCZ4BCtyszsc6AncEuC1Y0Izg8VAXmSbgMax6z/Ami3Jz2ukg4mOAd2McFpgd9IKu9w+/+A/SQVJFppZl8Bf6JswpxL8OPym/DftBfBoX6yrkVrSJAsiwAkDSVo2VVIUmPgX8CtZvZW3Oq/ATeVHpJLaiKp9JKlqUAXSeeGRyJXE/djSfDv92IVP0/WyopkB2Bm9wPXEZwMLiL4db0K+GdY5A/AfILWxfvAwnBZVbb1CsF5n0UEPZqxCSqH4MT/aoLDl54ELa34OtYDZ4Rl1xN8wc8ws3VViSmu7rfMLFGr9SWCL9HHBIeC31L2EKr0gun1khZWtp3wyzoOuNfM/m1mnxB0JDwZXmYSH9d2gs6DiyuodhQxPw7hewYA/Qg6Hf4K/NjMPqosvijMbAlBgp1NkOyPIOi9rcwxBOc67w+v69wiaUtY5ySCXukJkjYTtBT7hevWEXSI3EPw794pwfYuBMZU86NlHQWnBJxLD5JaAG8SXPqxrbLy2UbSmcAlZnZ+qmPJNJ7snHNZIWsOY51zmUNS3/Ae6WWSbiynzPmSlkhaLOmpSuv0lp1zLp2Et/R9THAnSSHB3UMXhudPS8t0IriMqbeZbZS0v5l9WVG93rJzzqWbYwluG/ws7ICawO73sP8UGG1mGwEqS3QQ3KqS1rRXjlE/7cNMmWMOrvQqiKy3dP3yVIeQ9r5ZvnGdmbWo6vuVX9/YvrPyggBf71hM0NNfaqyZjY2ZL6DsVQCFwHFxtRwMIOltglv8bjez6RVtNv2zSP08OG7/VEeRtt6eHn8Jl4vXa9yQVIeQ9uYOmRh/t86e2b4z+vd0xqpvzax7BSUSXbAdf74tj+CynF5AG+BNSYeH12Im5IexzrnkkKJNlSuk7F0jbdj9bqZCgnued4QXyi8lSH7l8mTnnKs+AbmKNlVuHtBJUntJewGDgclxZf5JcOsg4R1PBxPcf10uT3bOueRQxKkS4YAbVxHc0fMh8KyZLZY0QtKAsNhLBHfyLAFeA64P7zoqV/qfs3POZYDIh6iRhMOoTYtbdlvMayO4/fO6qHV6snPOVZ9I++NET3bOueRIYsuuJniyc84lR3rnOk92zrkkKO2NTWOe7JxzyeGHsc65rJDeuc6TnXMuCQTkpHe282TnnEuO9M51nuycc0kgQW56X2jnyc45lxzesnPOZQXvjXXOZYX0znWe7JxzSeC9sc65rJHeuc6TnXMuSfx2MedcnRd9yPWU8WTnnEuO9M51nuycc0niLTvnXFZI7xsoPNk555LALz1xzmUNT3bOuazg5+ycc3VexGfCppInO+dcEghFbNlZDUdSHk92zrmk8GTnnKvzBORG7KDYWbOhlCvNr4xJnVO79eDff5/OB4+8wvDzL09Y5rwe/Vg4ZhoLxkzlsRv+VMsRpt7L89+g67D/ocvQPtz3zJjd1n+3fTsX33UNXYb2occ1A1mxtjAFUabOsa2O4IkBdzP+rHu5qEv/csv1PKg7sy5+jEOatau94JJNQcsuypQqNZbsJG1JsOxkSQslFUsaWFPbrq6cnBwevPJ3nHXrTzn68tMZ1OsMDj2oQ5kyHVq3ZfgFP6P3rwfT7Wf9uf5vd6Yo2tQoKSnh2tF38K8/PMy7Y6cxcdYUPlyxrEyZx16aSNN9m7D40Rn88pwh3PLIfSmKtvblSFxz7CXc8Or9XPrCzfRudxxtm7Terdw+efU595BTWVL0aQqiTK5kJjtJfSUtlbRM0o0J1g+RVCTpvXD6SWV11nbL7j/AEOCpWt7uHvn+IV35dM0Klq9dyY7iHUx8fSpnnNCnTJnL+p3PmCnj+WrLZgCKNm1IRagpM2/pIjq0akv7VgexV729GNSzP1NmzyhTZsrsmfyozzkAnNujL7Pem41Zqs7Y1K5Dm3+PVV9/wZotRRTvLOHV5XM5sc3Ru5UbduS5TFgyje07d6QgymSKluiiJDtJucBooB/QGbhQUucERZ8xs6PC6e+V1Vuryc7MlpvZIlJ32B5J6+YHUFi0dtf8qnVrKWh+QJkynQra06mgHa/+6Wlef+BZTu3Wo7bDTKnV67+gTYuWu+YL8luyav0XCcq0AiAvN4/GDRuxfvPGWo0zVVo0aErR1v/+ABZt3UiLBk3LlOnY9CBaNGzG7FX/ru3wakTpwCeVTREcCywzs8/MbDswATiruvH5ObsEEv36xLdIcnNz6di6Haf95hJ+fM91PHTtnTRp2Ki2Qky5RC20+P0WpUzdlehvKHatuKr7RTy0YEItxlRzRFIPYwuAlTHzheGyeOdJWiTpOUkHVlZpWiY7SZdLmi9pPjtqvxG4at3a3Votqzd8uVuZF+bMoLikmBVfFPJx4ed0LGhXy5GmTkF+y91av62b7Z+gzBoAikuK2fzN1zRrtF+txpkqRVs30KJBs13zLRo0Zd22/7ZqG9SrT/smBTx46o1MOPuPdM7vwJ29rsncTgpBjnIiTUB+6fc7nOJ7ABNlxPhfzheAdmbWFZgBPF5ZiGmZ7MxsrJl1N7Pu1Kv9EOcvfZ+OrdvR9oA21Murx6Ce/Zk6Z2aZMi+8M4OeXY8HoHnjpnRq047P16xMVF2d1P2QI1i2ejnL165k+47tTHx9Kv2PP6VMmf7H92b8jEkAPP/mdHoeeULWtOyWrv+cNo0OoGXDfPJycund7jjeKXx31/pvdmzjrOd+yeB/DmfwP4ezZN2n3DJrFEs3LE9d0NW0By27daXf73AaG1dVIRDbUmsDrI4tYGbrzey7cPZhoFtl8fl1dgmU7CzhV38dwQt3/oPcnFwef/k5PlyxjN9ecjULP/mAqXNe5ZUFb9Kn20ksHDONkp0l3Pz3kWz4+qtUh15r8nLzeOCK2zjzlmGU7Czh0tMG0rldJ0Y8MYpjOh3OGSecwpC+g7hs5PV0GdqHpo2a8ORND6Q67FpTYjsZNW8c950ynBzl8OKnb7J802qGdj2HpRs+553C91IdYtIl8XdsHtBJUntgFTAYuKjsttTKzNaEswOADyuNr6Z6xyTtpGw2vh94E5gENAW+BdaaWZcK62m8l3Hc/hUVyWrbpn+c6hDSXq9xQ1IdQtqbO2TiAjPrXtX357Xa1xpddniksl/dNbfSbUk6HXgQyAUeMbM7JY0A5pvZZEl3EyS5YmAD8Asz+6jCGCNFVwVmVt7xZ5ua2qZzLnWSeYrCzKYB0+KW3Rbz+ibgpj2p0w9jnXPVJ8jx8eycc3Vd6aUn6cyTnXMuKTzZOeeyQGpv8o/Ck51zrvrkLTvnXJZI81znyc45V30iGBotnXmyc84lRU6aN+082Tnnqi/68E0p48nOOVdt8t5Y51y2UJo/ONaTnXMuKbxl55zLCn5vrHOuzpNfVOycyw7eQeGcyxKe7JxzWSHNc50nO+dc9Ul+u5hzLkv4YaxzLiukea7zZOecSwbvjXXOZQlPds65Os8vKnbOZQ2/Xcw5lx28Zeecq/u8g8I5lw18pGLnXDYQ6d9Bkd73dzjnMoakSFPEuvpKWippmaQbKyg3UJJJ6l5Znd6yc84lRbJ6YyXlAqOBU4FCYJ6kyWa2JK5cI+BqYG6k+JISnXMuu0Vs1UVs2R0LLDOzz8xsOzABOCtBud8DI4Fvo1Sa9i27Yw4+nLenv5XqMNJW/s0npzoE5/b0nF2+pPkx82PNbGzMfAGwMma+EDiuzPako4EDzWyKpOFRNpr2yc45lxn2INmtM7OKzrElqshitpMDPAAMiRwcEQ5jJY2U1FhSPUkzJa2TdPGebMQ5V/cl8TC2EDgwZr4NsDpmvhFwODBL0nLgeGByZZ0UUc7ZnWZmm4EzwiAOBq6PErFzLkso6KCIMkUwD+gkqb2kvYDBwOTSlWa2yczyzaydmbUD5gADzGx+4uoCUZJdvfD/pwNPm9mGKNE657KHSF4HhZkVA1cBLwEfAs+a2WJJIyQNqGqMUc7ZvSDpI2AbcIWkFkTs/XDOZY9kXlRsZtOAaXHLbiunbK8odVbasjOzG4ETgO5mtgPYSuJuYOdcFpOiTakSpYOiAXAl8FC4qDVQ6dXKzrksouTeQVETopyzexTYDvwgnC8E/lBjETnnMlOaN+2iJLsOZjYS2AFgZttIfB2Mcy5LCcjNUaQpVaJ0UGyXtA/hRX2SOgDf1WhUzrkMUzfGs/sdMB04UNJ44ET28Mpl51wdJ8jJ9GRnZq9IWkhwlbKAa8xsXY1H5pzLGHViPDtJJwLfmtlUYD/gZkltazwy51xGyYk4pUqUbT8EbJV0JMFtYiuAJ2o0KudcRgk6KHIiTakSZcvFZmYEFxL/r5mNIrgR1znnQiJH0aZUidJB8bWkm4CLgZPDUUTrVfIe51w2yYCHZEdp2V1AcKnJMDNbSzCw3n01GpVzLqOI9D9nF6llB4wysxJJBwOHAk/XbFjOuUyT7peeREm0bwB7SyoAZgJDgcdqMijnXOapC/fGysy2AucCfzazc4AuNRuWcy6TCMiVIk2pEuUwVpJOAH4EDAuX5dZcSM65zJPantYooiS7a4CbgEnhaKHfA16r2bCcc5lEdeR2sTcIztuVzn9G8GBa55zbJd0vPak02YXDsP+G4Dxd/dLlZta7BuNyzmWYdG/ZRemgGA98BLQH7gCWEzz9xznngHAggIhTqkRJds3N7B/ADjN73cwuIxgBxTnnQiIvJyfSlCpROih2hP9fI6k/wcNq29RcSM65TKMMuF0sSrL7g6QmwK+BPwONgV/VaFTOuYyT7ufsovTGTglfbgJ+WLPhOOcyVXqnugqSnaQ/Ez53IhEz88tPnHNAOBBABrfs5tdaFM65DKeUDswZRUXJ7hmgkZkVxS6UtD+wuUajSgMvz3+D4Q/dScnOEob0HcT1F/yszPrvtm9n2B+v591PFtOs8X6Mu+lB2rbMnn6b3p2O5a4zriYnJ4dx86byv2+ML7N+8DF9ub3fFazZFPz5/GPO84ybPzUVoaZMNu2j0iGe0llFye5/CZ4q9nzc8lOBk4BfVFSxpC1mtm/csuuAnwDFQBFwmZmt2NOga1pJSQnXjr6DqXc9SkF+S066+jzOOP4UDmvbcVeZx16aSNN9m7D40Rk8O2sKtzxyH+NuHpXCqGtPjnK4d8CvGPjIdazeXMQrV4xl+kdv8fGXZf8p/7noVW584cEURZlaWbePktwbK6kvMIrgPvy/m9k9cet/DlwJlABbgMvNbElFdVaUjE8ys/hEh5mNB07ew9hLvQt0N7OuwHPAyCrWU6PmLV1Eh1Ztad/qIPaqtxeDevZnyuwZZcpMmT2TH/U5B4Bze/Rl1nuzCUavr/uOaXMYn69fxYqNa9hRUsykRTPpd9hJqQ4rrWTjPkrWsOzhaOijgX5AZ+BCSZ3jij1lZkeY2VEEeeT+SuOraJtVfF+5zOy1cLgogDmk6fV6q9d/QZsWLXfNF+S3ZNX6LxKUaQVAXm4ejRs2Yv3mjbUaZ6q0apLP6k1f7ppfvamIVo1b7FbuzC49ef2Xj/LIRSNo3WT/2gwx5bJtH5V2UCTpGRTHAsvM7DMz2w5MIHgGzi5mFnsqrSEVdKaWqihpfSnp2N0+lPR9gkPQ6hoGvJhohaTLJc2XNL+oqPYfUZuohRbfRI9Spq5Sgt9Bi/tbe+nDdzj6vvPp+eehvLFsPqMH3lxb4aWFbNxHezB4Z37p9zucLo+rqgBYGTNfGC6L396Vkj4laNlVenVIRcnueuBZSbdLOjOc7gCeDddVmaSLge6U8ywLMxtrZt3NrHuLFvnV2VSVFOS3pLBo7a75VevW0rrZ/gnKrAGguKSYzd98TbNG+9VqnKmyelNRmVZI6yYtWLu57I/Sxm2b2V4S3HzzxLwpHFlwcK3GmGrZt49ErnIiTcC60u93OI3drbLd7da6MLPRZtYBuAG4tbIIy012ZvZ/BM1JAUPCScBxZja3sorLI6kPcAswwMy+q2o9Nan7IUewbPVylq9dyfYd25n4+lT6H39KmTL9j+/N+BmTAHj+zen0PPKErGnZvbvqI76X34aDmraiXm4e53Q9hekfvl2mzAGNmu963fewE3c7MV/XZds+Kh3PLkmHsYXAgTHzbQhuUy3PBODsyiqt8A4KM/sS+F2U6KKQdDQwBugb1p2W8nLzeOCK2zjzlmGU7Czh0tMG0rldJ0Y8MYpjOh3OGSecwpC+g7hs5PV0GdqHpo2a8ORND6Q67FpTsrOEGyc/yMShfyRHOTy1YBpLv1zOjX0u473CpUz/6G1+esJ59D3sRIp3lvDVts1c9f/uTnXYtSob91GiQ/cqmgd0ktQeWAUMBi4qsy2pk5l9Es72Bz6hEqqpHkRJOymbje8HTgeOANaEy/5jZgMqqqdb92Ps7blv1UiMdUH+zVXtGHfuv74ZuWCBmXWv6vtbHdbKLnt0WOUFgbtOuLPSbUk6HXiQ4NKTR8zsTkkjgPlmNlnSKKAPwUAlG4GrzGxxRXVGGQigSsws0SFypd3DzrnMoyQ/g8LMpgHT4pbdFvP6mj2ts8aSnXMuuyjN76GoaCCAF6h4IIAKDz+dc9klk++N/WOtReGcy2gK/0tn5SY7M3u9NgNxzmWwuvAoRUmdgLsJ7lGLfbrY92owLudchkn360yjHGQ/CjxEMFLJD4EngCdrMijnXGYJhniK9l+qRNnyPmY2k+CavBVmdjvgz4x1zsUQOTk5kaZUiXLpybeScoBPJF1FcEVz5g7P4JyrETlp3kERJc1eCzQgGFWgG3AJcGlNBuWcyyxij0Y9SYkoTxebF77cAgyt2XCccxmpjvTGvkbi4VX8vJ1zLpTB19nFGB7zuj5wHkHPrHPOAaUjFWfuHRQAmNmCuEVvS/ILjp1zZWR8spPULGY2h6CTomU5xZ1zWSm5o57UhCiHsQsIztmJ4PD1c4LnRzjnHBD2xtaBc3aHmdm3sQsk7V1D8TjnMlS6t+yiHGS/k2DZ7GQH4pzLYAIpJ9KUKhWNZ9eS4PFl+4TPjihN240JLjJ2zrlQZl968j8ETxRrA/yJ/ya7zUBmP+DSOZdUIoMH7zSzx4HHJZ1nZv+vFmNyzmWgunBvbDdJu57+LKmppD/UYEzOuQyTCffGRkl2/czsq9IZM9tI8EhE55wLKXM7KGLkStrbzL4DkLQP4JeeOOfKSPfD2CjJbhwwU9KjBBcXX0YwWrFzzgEg1YHbxcxspKRFBE/fFvB7M3upxiNzzmWQ1J6PiyLSQ7LNbDowHUDSiZJGm9mVNRqZcy6j1IXDWCQdBVwIXEBwb+zzNRmUcy6zBL2x6X0YW250kg6WdJukD4G/AIUED935oZn9udYidM5lAEX+L1JtUl9JSyUtk3RjgvXXSVoiaZGkmZLaVlZnRan4I+AU4EwzOylMcCWRInXOZZ1kXWcnKRcYDfQjeF71hZI6xxV7F+huZl2B54CRldVbUbI7D1gLvCbpYUmnQJoflDvnUiZHOZGmCI4FlpnZZ2a2HZgAnBVbwMxeM7Ot4ewcgttaK1TR7WKTgEmSGgJnA78CDpD0EDDJzF6OErWrWYd3/l6qQ0h7c8clGrjHJVPwkOzIbaF8SfNj5sea2diY+QJgZcx8IXBcBfUNA16sbKNRLj35BhgPjA9HLR4E3Ah4snPOBfbsVrB1Zta9otoSLNvtoV/BZnUx0B3oWdlGI/XG7tqa2QZgTDg559wuinT3aSSFwIEx822A1bttT+oD3AL0LL3DqyJ7lOycc648SbyoeB7QSVJ7YBUwGLgobltHEzS6+prZl1Eq9WTnnKs2IXKTdJ2dmRVLugp4CcgFHjGzxZJGAPPNbDJwH7AvMDFMsv8xswEV1evJzjmXFMkcqdjMpgHT4pbdFvO6z57W6cnOOZcUdeLeWOecq0jwKMX0vl3Mk51zLgnqyKgnzjlXmTox6olzzlWkTgze6ZxzUfhhrHMuC8g7KJxz2SHHW3bOubouuPTEk51zLgv4OTvnXBaQ98Y65+q+YPBOT3bOubpOfhjrnMsK0Z8cliqe7JxzSeEtO+dcnefn7Jxz2cNbds65us/P2TnnsoSfs3POZQVv2TnnsoInO+dcnSe/Xcw5ly28Zeecq/v8djHnXLbwlp1zrs4T6d+yS+8ziin08vw36Drsf+gytA/3PTNmt/Xfbd/OxXddQ5ehfehxzUBWrC1MQZSpdWyrI3hiwN2MP+teLurSv9xyPQ/qzqyLH+OQZu1qL7g0cGq3Hvz779P54JFXGH7+5QnLnNejHwvHTGPBmKk8dsOfajnCZFLk/1KlxpKdpC0Jlv1c0vuS3pP0lqTONbX96igpKeHa0Xfwrz88zLtjpzFx1hQ+XLGsTJnHXppI032bsPjRGfzynCHc8sh9KYo2NXIkrjn2Em549X4ufeFmerc7jrZNWu9Wbp+8+px7yKksKfo0BVGmTk5ODg9e+TvOuvWnHH356QzqdQaHHtShTJkOrdsy/IKf0fvXg+n2s/5c/7c7UxRtcuQoJ9IUhaS+kpZKWibpxgTrT5a0UFKxpIGR4tvDz1NdT5nZEWZ2FDASuL+Wtx/JvKWL6NCqLe1bHcRe9fZiUM/+TJk9o0yZKbNn8qM+5wBwbo++zHpvNmaWinBT4tDm32PV11+wZksRxTtLeHX5XE5sc/Ru5YYdeS4Tlkxj+84dKYgydb5/SFc+XbOC5WtXsqN4BxNfn8oZJ/QpU+ayfuczZsp4vtqyGYCiTRtSEWrSJKtlJykXGA30AzoDFyZoGP0HGAI8FTW+Wk12ZrY5ZrYhkJbZYfX6L2jTouWu+YL8lqxa/0WCMq3XLK7RAAAKuUlEQVQAyMvNo3HDRqzfvLFW40ylFg2aUrT1v1/Ooq0badGgaZkyHZseRIuGzZi96t+1HV7KtW5+AIVFa3fNr1q3loLmB5Qp06mgPZ0K2vHqn57m9Qee5dRuPWo7zKQpfeBOkg5jjwWWmdlnZrYdmACcFVvAzJab2SJgZ9QYa72DQtKVwHXAXkDv2t5+FIlaaPEnX6OUqdt2/6yxu0SIq7pfxD3v/L0WY0ofif4W4v9mcnNz6di6Haf95hIK8lsy849P0e3n/dn0zde1FWYSKZl//wXAypj5QuC46lZa6x0UZjbazDoANwC3Jioj6XJJ8yXNLypaV7sBErTk4n+VWzfbP0GZNQAUlxSz+ZuvadZov1qNM5WKtm6gRYNmu+ZbNGjKum3/bdk2qFef9k0KePDUG5lw9h/pnN+BO3tdkzWdFKvWrd3t6GD1hi93K/PCnBkUlxSz4otCPi78nI4F7Wo50mRSxIn80u93OMX33iTKmtU+Ckxlb+wE4OxEK8xsrJl1N7PuLVrk13JY0P2QI1i2ejnL165k+47tTHx9Kv2PP6VMmf7H92b8jEkAPP/mdHoeeUJWteyWrv+cNo0OoGXDfPJycund7jjeKXx31/pvdmzjrOd+yeB/DmfwP4ezZN2n3DJrFEs3LE9d0LVo/tL36di6HW0PaEO9vHoM6tmfqXNmlinzwjsz6Nn1eACaN25Kpzbt+HzNykTVpT/tUQfFutLvdziNjautEDgwZr4NsLq6IdbqYaykTmb2STjbH/ikovKpkpebxwNX3MaZtwyjZGcJl542kM7tOjHiiVEc0+lwzjjhFIb0HcRlI6+ny9A+NG3UhCdveiDVYdeqEtvJqHnjuO+U4eQohxc/fZPlm1YztOs5LN3wOe8UvpfqEFOqZGcJv/rrCF648x/k5uTy+MvP8eGKZfz2kqtZ+MkHTJ3zKq8seJM+3U5i4ZhplOws4ea/j2TD11+lOvQqS+JlJfOATpLaA6uAwcBF1a1UNdWDKGknZbPx/UBboA+wA9gIXGVmiyuqp1v3Y+ztuW/VSIx1Qa9xQ1IdQtqbO+6dVIeQ/masWmBm3av69q7HHGGT33g+Utn2jQ6udFuSTgceBHKBR8zsTkkjgPlmNlnS94FJQFPgW2CtmXWpqM4aa9mZmV+w7FwWSeYFw2Y2DZgWt+y2mNfzCA5vI/PbxZxzSeH3xjrnskK6d9B5snPOVZsP3umcyxp+GOucyxKe7JxzWSC9U50nO+dckngHhXMuS3iyc87VeakdhTgKT3bOuWpTBjxdLL0vjHHOuSTxlp1zLin8MNY5lxU82TnnsoKfs3POuTTgLTvnXBL4pSfOuazhyc45V8ftem5YGvNk55xLinTvoPBk55xLCj9n55zLEp7snHN1ntL+MNavs3POZQVv2Tnnqi3ojU3vlp0nO+dckniyc85lgZw0P2fnyc45lwTpf1mxJzvnXFKkd6rz3ljnXNIo4hShJqmvpKWSlkm6McH6vSU9E66fK6ldZXV6snPOVV/4DIooU6VVSbnAaKAf0Bm4UFLnuGLDgI1m1hF4ALi3sno92Tnnqq300pMo/0VwLLDMzD4zs+3ABOCsuDJnAY+Hr58DTlElmTTtz9ktXPDuun3yGq5IdRwx8oF1qQ4izfk+qlg67p+21XnzwgXvvrRPXsP8iMXrS5ofMz/WzMbGzBcAK2PmC4Hj4urYVcbMiiVtAppTwX5N+2RnZi1SHUMsSfPNrHuq40hnvo8qVhf3j5n1TWJ1iVpoVoUyZfhhrHMu3RQCB8bMtwFWl1dGUh7QBNhQUaWe7Jxz6WYe0ElSe0l7AYOByXFlJgOXhq8HAq+aWYUtu7Q/jE1DYysvkvV8H1XM908FwnNwVwEvAbnAI2a2WNIIYL6ZTQb+ATwpaRlBi25wZfWqkmTonHN1gh/GOueygic751xW8GRXAUlbEiw7WdJCScWSBqYirnRSzj66TtISSYskzZRUrWu4Mlk5++fnkt6X9J6ktxLcHeBqgCe7PfcfYAjwVIrjSGfvAt3NrCvB1e0jUxxPunnKzI4ws6MI9s39qQ4oG3iy20NmttzMFgE7Ux1LujKz18xsazg7h+A6KRcys80xsw2p5GJYlxx+6YmracOAF1MdRLqRdCVwHbAX0DvF4WQFb9m5GiPpYqA7cF+qY0k3ZjbazDoANwC3pjqebODJztUISX2AW4ABZvZdquNJYxOAs1MdRDbwZOeSTtLRwBiCRPdlquNJN5I6xcz2Bz5JVSzZxO+gqICknZS9Afl+4E1gEtAU+BZYa2ZdUhBeWihnH50OHAGsCZf9x8wG1HZs6aCc/dMW6APsADYCV5nZ4hSEl1U82TnnsoIfxjrnsoInO+dcVvBk55zLCp7snHNZwZOdcy4reLLLEJJKwlEyPpA0UVKDatTVS9KU8PWARA8hjim7n6QrqrCN2yUNL2fdj8PPsTgcHWV4uPwxH0nG1RRPdpljm5kdZWaHA9uBn8euVGCP/z3NbLKZ3VNBkf2APU525ZHUD7gWOC28PvEYYFOy6neuPJ7sMtObQEdJ7SR9KOmvwELgQEmnSZodjrk3UdK+AJL6SvpI0lvAuaUVSRoi6S/h6wMkTZL073D6AXAP0CFsVd4Xlrte0rxwvLo7Yuq6RdJSSTOAQ8qJ/SZguJmtBjCzb83s4fhCkm4Lt/GBpLGlD0CWdHXMWHkTwmU9w/jek/SupEblxSmpoaSp4ef7QNIF1fh3cJnEzHzKgAnYEv4/D/gX8AugHcFQU8eH6/KBN4CG4fwNwG1AfYIHCncieN7ms8CUsMwQ4C/h62eAa8PXuQSPp2sHfBATx2kED4wRwY/lFOBkoBvwPtAAaAwsI0hq8Z9jA9CknM/4GDAwfN0sZvmTwJnh69XA3uHr/cL/vwCcGL7eN9xH5cV5HvBwTN0JY/Gp7k3esssc+0h6D5hPMIDoP8LlK8xsTvj6eKAz8HZY9lKCW5MOBT43s08s+IaPK2cbvYGHAMysxMwSHV6eFk7vErQmDyVIoj2ASWa21YLx2uIffbenfihprqT3w7hKb8lbBIwPR1QpDpe9Ddwv6WqCBFhcQZzvA30k3SupRzmf0dVBPp5d5thmwci2u4RHdt/ELgJeMbML48odRfIGiBRwt5mNidvGtRG3sZigFfhquRuQ6gN/JRjteKWk2wlapxDcOH8yMAD4raQuZnaPpKkE9+TOCUdcSRhnWH+3sOzdkl42sxER4nYZzlt2dcsc4ERJHQEkNZB0MPAR0F5Sh7DcheW8fybB4TGSciU1Br4GGsWUeQm4LOZcYIGk/QkOn8+RtE94zuzMcrZxNzBSUsvw/XuHLbJYpYltXbidgWHZHOBAM3sN+A1B58m+kjqY2ftmdi9By/fQ8uKU1BrYambjgD8SdJC4LOAtuzrEzIokDQGelrR3uPhWM/tY0uXAVEnrgLeAwxNUcQ0wVtIwoAT4hZnNlvS2pA+AF83sekmHAbPDluUW4GIzWyjpGeA9YAVBJ0qiGKdJOgCYEXY6GPBIXJmvJD1McMi5nOAJ8RCcRxwnqQlBy+2BsOzvJf0wjHlJGOd3ieIEOgL3KRiNZAdhcnd1n4964pzLCn4Y65zLCp7snHNZwZOdcy4reLJzzmUFT3bOuazgyc45lxU82TnnssL/B2F3o0ZvzOUMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm.plot(cmap=plt.cm.Greens, number_label=True, normalized=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x122b45d0>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEWCAYAAACg+rZnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+YlHW9//Hna3dxsSRJsVSW/IVa2Ek9Cf1OM06iFfYTsZ+mXhwLT3ikOuTpa2adk9LJrFBrOxaKBoJpagiFlgmWIoKioEe4IoLVIjVQUVDW9/eP+14bt92Z2WVm7rlnX4/rmouZ+77nc793gNd89nN/7vtWRGBmZvWtKesCzMysNIe1mVkOOKzNzHLAYW1mlgMOazOzHHBYm5nlgMPadpqkXSXdJGmLpHk70c7HJf2qkrVlQdICSZ/Oug5rLA7rAUTSxyQtk/S0pEfTUHl7BZr+CPBqYM+I+Gh/G4mIqyPiPRWo5yUkHSMpJF3Xbfnh6fLbymznPElXldouIo6PiCv6Wa5ZjxzWA4Sks4GLgf8mCdbXAJcCJ1ag+f2AhyNiRwXaqpa/Am+VtGfBsk8DD1dqB0r4/5RVhf9hDQCSdgfOByZHxHURsTUino+ImyLii+k2rZIulvRI+rhYUmu67hhJGyVNlbQp7ZV/Jl33NeBc4KS0x35a9x6opP3THmxL+voUSX+Q9JSkdZI+XrB8ScH73irp7nR45W5Jby1Yd5ukr0u6I23nV5KGFfkYngN+DkxM398MTACu7vZZfVfSBklPSrpH0jvS5eOAcwp+zvsK6vgvSXcAzwAHpstOT9dfJunagvYvlHSrJJX9F2iGw3qgeAswGLi+yDb/CbwZOAI4HBgDfKVg/d7A7sBw4DTgEkmvjIivkvTWr4mI3SLi8mKFSHo58D3g+IgYArwVuLeH7fYA5qfb7glcBMzv1jP+GPAZ4FXALsAXiu0buBL4VPr8OGAV8Ei3be4m+Qz2AH4KzJM0OCIWdvs5Dy94zyeBScAQYH239qYCb0i/iN5B8tl9OnydB+sjh/XAsCfwWIlhio8D50fEpoj4K/A1khDq8ny6/vmIuBl4Gji0n/W8ALxe0q4R8WhErOphm/cCayJiVkTsiIjZwEPA+wu2+UlEPBwRzwJzSUK2VxHxO2APSYeShPaVPWxzVUQ8nu7z20ArpX/OmRGxKn3P893aewb4BMmXzVXAv0XExhLtmf0Dh/XA8DgwrGsYohf78tJe4fp02YttdAv7Z4Dd+lpIRGwFTgLOAB6VNF/Sa8uop6um4QWv/9yPemYBZwLvooffNNKhngfToZfNJL9NFBteAdhQbGVELAX+AIjkS8WszxzWA8PvgW3AB4ps8wjJgcIur+EfhwjKtRV4WcHrvQtXRsQvI+JfgH1Iess/KqOerpo6+llTl1nA54Cb017vi9Jhiv8gGct+ZUQMBbaQhCxAb0MXRYc0JE0m6aE/Anyp/6XbQOawHgAiYgvJQcBLJH1A0sskDZJ0vKTp6Wazga9I2is9UHcuya/t/XEv8E5Jr0kPbn65a4WkV0san45dbycZTunsoY2bgUPS6YYtkk4CRgG/6GdNAETEOuBokjH67oYAO0hmjrRIOhd4RcH6vwD792XGh6RDgG+QDIV8EviSpKLDNWY9cVgPEBFxEXA2yUHDv5L86n4myQwJSAJlGbASuB9Yni7rz74WAdekbd3DSwO2ieSg2yPAEyTB+bke2ngceF+67eMkPdL3RcRj/ampW9tLIqKn3xp+CSwgmc63nuS3kcIhjq4Tfh6XtLzUftJhp6uACyPivohYQzKjZFbXTBuzcskHpc3M6p971mZmOeCwNjOrIEmDJS2VdJ+kVemJY923aZV0jaS1ku6StH+pdh3WZmaVtR04Nj1x6ghgnKQ3d9vmNOBvETES+A5wYalGHdZmZhUUiafTl4PSR/eDgycCXRf7uhZ4d6lLEBQ7SSJTknzks8p8cLn6JkyYkHUJA8LcuXN3+lorfcycfyW5xECX9ohoL2irmWQm1Ejgkoi4q9v7h5PONIqIHZK2kJ5p3NsO6zaszczqVRrM7UXWdwJHSBoKXC/p9RHxQMEmPX25FP2y8DCImRkgqexHuSJiM3AbMK7bqo3AiHS/LSSXNXiiWFsOazMzoKmpqexHMelZwEPT57sCY0kuq1DoRpLrqUNy845fl7oSo4dBzMygTz3mEvYBrkjHrZuAuRHxC0nnA8si4kbgcpIzWdeS9KgnlmrUYW1mRuXCOiJWAkf2sPzcgufbgD7dAs9hbWZGRXvWVeGwNjPDYW1mlgsOazOzHCg1yyNrDmszM9yzNjPLBYe1mVkOOKzNzHLAYW1mlgM+wGhmlgPuWZuZ5YDD2swsBxzWZmY54LA2M8sBh7WZWQ54NoiZWQ64Z21mlgMOazOzHHBYm5nlgMPazCwHfIDRzCwH3LM2M8sBh7WZWQ44rM3McsBhbWaWAw5rM7Mc8GyQBtfa2srtt99Oa2srLS0tXHvttZx33nlZl9WQFi5cyJQpU+js7OT0009n2rRpWZfUUPbcc08mT57M0KFDiQhuueUWFixYkHVZNeOedYPbvn07xx57LFu3bqWlpYUlS5awYMEC7rrrrqxLayidnZ1MnjyZRYsW0dbWxujRoxk/fjyjRo3KurSG0dnZyaxZs1i3bh2DBw/mggsuYOXKlXR0dGRdWk1UKqwljQCuBPYGXgDaI+K73bY5BrgBWJcuui4izi/WrsO6ArZu3QrAoEGDGDRoEBGRcUWNZ+nSpYwcOZIDDzwQgIkTJ3LDDTc4rCto8+bNbN68GYBt27bR0dHBHnvs4bDuux3A1IhYLmkIcI+kRRGxutt2iyPifeU2Wt+DNDnR1NTEihUr2LRpE4sWLWLp0qVZl9RwOjo6GDFixIuv29raBkyIZGGvvfbigAMOYO3atVmXUjOSyn4UExGPRsTy9PlTwIPA8J2tr+ZhLen+Wu+z2l544QWOPPJI2traGDNmDIcddljWJTWcnn5bqfcxxrxqbW1l6tSpzJw5k2effTbrcmqmUmHdrc39gSOBnsZF3yLpPkkLJJUMjaoMg0j6UG+rSMZxenvfJGBSNWqqhS1btnDbbbcxbtw4Vq1alXU5DaWtrY0NGza8+Hrjxo3su+++GVbUmJqbm5k6dSqLFy8ecL8h9mU2SA9Z1R4R7d222Q34GXBWRDzZrYnlwH4R8bSkE4CfAwcX22e1xqyvAa4Gehq8Hdzbm9Ifth1AUi4GfocNG8bzzz/Pli1bGDx4MGPHjuXCCy/MuqyGM3r0aNasWcO6desYPnw4c+bM4ac//WnWZTWcM844g46ODubPn591KTXXlx5zYVb10tYgkqC+OiKu6+H9TxY8v1nSpZKGRcRjvbVZrbBeCfxPRDzQfYWksVXaZyb22WcfrrjiCpqbm2lqamLu3LkD8h96tbW0tDBjxgyOO+44Ojs7OfXUUz3cVGGHHnooRx99NOvXr2f69OkAzJ49mxUrVmRcWW1UcDaIgMuBByPiol622Rv4S0SEpDEkQ9KPF223GjMXJL0DWB8Rf+ph3VERsayMNnLRs84zz1qpvgkTJmRdwoAwd+7cnU7aww8/vOz/EPfdd1+v+5P0dmAxcD/J1D2Ac4DXAETEDySdCXyWZObIs8DZEfG7YvusSs86IhYXWf12oGRYm5nVUqV61hGxhOT4XLFtZgAz+tJuFlP3zs5gn2ZmRTU1NZX9yEIWJ8V4vpWZ1Z16nwqaRVh7oNTM6s6ADGtJT9FzKAvYtRr7NDPbGQMyrCNiSDXaNTOrlgEZ1mZmeeOwNjPLAd98wMwsB9yzNjPLAYe1mVkOOKzNzHLAYW1mlgMOazOzHPBsEDOzHHDP2swsBxzWZmY54LA2M8sBh7WZWQ74AKOZWQ64Z21mlgMOazOzHHBYm5nlgMPazCwHHNZmZjng2SBmZjngnrWZWQ44rM3McsBhbWaWA/Ue1vU9om5mViOSyn6UaGeEpN9IelDSKklTethGkr4naa2klZL+uVR97lmbmVHR2SA7gKkRsVzSEOAeSYsiYnXBNscDB6ePNwGXpX/2qm7DOiKyLqHhzZs3L+sSGt5HP/rRrEuwMlVqGCQiHgUeTZ8/JelBYDhQGNYnAldGEnR3ShoqaZ/0vT3yMIiZGX0bBpE0SdKygsekXtrcHzgSuKvbquHAhoLXG9NlvarbnrWZWS31pWcdEe1Ae4n2dgN+BpwVEU92X91Ts8XaK9mzljRd0iskDZJ0q6THJH2i1PvMzPKkUgcY07YGkQT11RFxXQ+bbARGFLxuAx4p1mY5wyDvSb8V3pfu4BDgi2W8z8wsN5qamsp+FKMkzS8HHoyIi3rZ7EbgU+mskDcDW4qNV0N5wyCD0j9PAGZHxBP1Ph/RzKyvKphrbwM+Cdwv6d502TnAawAi4gfAzSSZuhZ4BvhMqUbLCeubJD0EPAt8TtJewLY+l29mVscqOBtkCT2PSRduE8DkvrRbMqwjYpqkC4EnI6JT0jMk007MzBpGvY8YlHOA8WUk3wCXpYv2BY6qZlFmZrVWyQOM1VDOAcafAM8Bb01fbwS+UbWKzMwyUO9hXc6Y9UERcZKkkwEi4lnV++8LZmZ91Ag3H3hO0q6kE7YlHQRsr2pVZmY1Vu990HLC+qvAQmCEpKtJpqWcUs2izMxqLfdhHRGLJC0H3kwyHWVKRDxW9crMzGqo3sO6nNkgbwO2RcR8YChwjqT9ql6ZmVkN1fsBxnJG1C8DnpF0OMlp5uuBK6talZlZjVXqdPOq1VfGNjvSs21OBL4XEd8FhlS3LDOz2qr3nnU5BxifkvRl4BPAOyU18/frhZiZNYTcj1kDJ5FM1TstIv5McoHsb1W1KjOzGmuInjXw3fS6IIcArwVmV7csM7PaaoSe9e1Aq6ThwK0kl/KbWc2izMxqrd571uWEtSLiGeBDwPcj4oPAYdUty8ystup9Nkg5wyCS9Bbg48Bp6bLm6pVkZlZ79T4MUk5YTwG+DFwfEaskHQj8prplmZnVVu7DOiJuJxm37nr9B+Dz1SzKzKzWch/W6W28vkQyTj24a3lEHFvFuszMaqrew7qckfKrgYeAA4CvAX8E7q5iTWZmNdcIs0H2jIjLgecj4rcRcSrJFfjMzBpGI8wGeT7981FJ7wUeAdqqV5KZWe3V+zBIOWH9DUm7A1OB7wOvAP69qlWZmdVY7sM6In6RPt0CvKu65ZiZZSO3YS3p+6T3XexJRHj6npk1jNyGNbCsZlWYmWUsz3c3vwYYEhF/LVwo6VXAk1WtKmcWLlzIlClT6Ozs5PTTT2fatGlZl9RwLr30UpYvX87uu+/Ot7/97azLaUgD/TOu9551sa+S7wHv6GH5vwDfqU45+dPZ2cnkyZNZsGABq1evZvbs2axevTrrshrOMcccwznnnJN1GQ1toH/GlZxnLenHkjZJeqCX9cdI2iLp3vRxbqk2i4X12yPiuu4LI+Jq4J0lqx0gli5dysiRIznwwAPZZZddmDhxIjfccEPWZTWcUaNGsdtuu2VdRkMb6J9xhU+KmQmMK7HN4og4In2cX6rBYmFdrKL6HtypoY6ODkaMGPHi67a2Njo6OjKsyMz6o5JhnV5T6YlK1lcsdDdJGtN9oaTRwF972L5wmxGS5khaLOkcSYMK1v28yPsmSVomaVl7e3s59WcuuZfwS9X72JeZ/aO+hHVhVqWPSf3Y5Vsk3SdpgaSS9wgodoDxi8BcSTOBe9JlRwGfAiaWaPfHwM+AO0mugf1bSe+PiMeB/Xp7U0S0A10p3eu0wXrS1tbGhg0bXny9ceNG9t133wwrMrP+6MtskG5Z1R/Lgf0i4mlJJwA/Bw4uWl+RYpYCY0iGQ05JHwLeFBF3lShkr4j4QUTcGxH/BlwK3C7pIHISwuUaPXo0a9asYd26dTz33HPMmTOH8ePHZ12WmfVRLS/kFBFPRsTT6fObgUGShhV7T9EzGCNiE/DVftQySNLgiNiWtnOVpD8DvwRe3o/26lZLSwszZszguOOOo7Ozk1NPPZXDDvNdzyrt4osvZvXq1Tz11FOcccYZTJgwgWOP9VV6K2mgf8a1HL6UtDfwl4iIdLi5CXi82HvKuTZIf/wv8Cbgt10LIuIWSR8Fpldpn5k54YQTOOGEE7Iuo6GdddZZWZfQ8Ab6Z1zJsJY0GzgGGCZpI0mndxBARPwA+AjwWUk7gGeBidHTAbACVQnriOhxHnZErJA0vxr7NDPbGZUM64g4ucT6GcCMvrSZxRS8szPYp5lZUfV+84FiF3K6ieIXcurvUTTPazOzupPna4P8T5X22VCzQcysMdT7+RG9hnVE/La3daVIeoqeQ1nArv1t18ysWnIb1l0kHQx8ExjFS+9ufmBv74mIIRWpzsysRuo9rMsZpPkJcBmwg+ROMVcCs6pZlJlZrdX7AcZywnrXiLgVUESsj4jzgIEzU97MBoRGuLv5NklNwBpJZwIdwKuqW5aZWW01wjDIWcDLgM8DbwQ+CXy6mkWZmdVavQ+DlHN387vTp08Dn6luOWZm2aj3nnU5s0F+Qw/T8CLC49Zm1jByH9bAFwqeDwY+TDIzxMysYeQ+rCPinm6L7pDU7xNmzMzqUZ5PNwdA0h4FL5tIDjLuXbWKzMwykPueNcktvYLkVPEdwDqSW3WZmTWMRgjr13Xd8aWLpNYq1WNmlol6D+tyBml+18Oy31e6EDOzLOV2nnV6j7DhwK6SjuTv16F+BclJMmZmDaPee9bFhkGOI7mjeRvwbf4e1k8C51S3LDOz2srtbJCIuAK4QtKHI+JnNazJzKzm6r1nXc5XyRslDe16IemVkr5RxZrMzGqu3sesywnr4yNic9eLiPgbcEL1SjIzq716D+typu41S2qNiO0AknYFPHXPzBpKvQ+DlBPWVwG3SvoJyckxp5LcLcbMrGHk9gBjl4iYLmklMJZkRsjXI+KXVa/MzKyGGqFnTUQsBBYCSHqbpEsiYnJVKzMzq6GGCGtJRwAnAyeRXBvkumoWZWZWa7kNa0mHABNJQvpx4BqSm+a+q0a1mZnVTCXDWtKPgfcBmyLi9T2sF/Bdkpl1zwCnRMTyYm0WG1F/CHg38P6IeHtEfB/o7G/xZmb1rMJT92YC44qsPx44OH1MAi4r1WCxsP4w8GfgN5J+JOnd/P2UczOzhtLU1FT2o5SIuB14osgmJwJXRuJOYKikfYq1Wex08+uB6yW9HPgA8O/AqyVdBlwfEb8qWbHVtXnz5mVdQsPzZ1wbEf9wm9g+68swiKRJJD3iLu0R0d6H3Q0HNhS83pgue7S3N5QzdW8rcDVwdXrXmI8C0wCHtZk1jL6EdRrMfQnnf9hdT80We0OfZoFHxBMR8UPf2dzMGk2NTzffCIwoeN0GPFLsDfV9yo6ZWY3UOKxvBD6lxJuBLRHR6xAIlDnP2sys0VXydHNJs4FjgGGSNgJfBQYBRMQPgJtJpu2tJZm695lSbTqszcyo7DzriDi5xPoA+nQWuMPazIwcn8FoZjaQOKzNzHLAYW1mlgMOazOzHMj9zQfMzAYC96zNzHLAYW1mlgMOazOzHHBYm5nlgMPazCwHPBvEzCwH3LM2M8sBh7WZWQ44rM3McsBhbWaWAz7AaGaWA+5Zm5nlgMPazCwHHNZmZjngsDYzywGHtZlZDng2iJlZDrhnbWaWAw5rM7MccFibmeWAw3oAWLhwIVOmTKGzs5PTTz+dadOmZV1Sw9lzzz2ZPHkyQ4cOJSK45ZZbWLBgQdZlNZTW1lZuv/12WltbaWlp4dprr+W8887LuqyacVg3uM7OTiZPnsyiRYtoa2tj9OjRjB8/nlGjRmVdWkPp7Oxk1qxZrFu3jsGDB3PBBRewcuVKOjo6si6tYWzfvp1jjz2WrVu30tLSwpIlS1iwYAF33XVX1qXVRCVng0gaB3wXaAb+NyIu6Lb+FOBbQNc/4BkR8b9F66tYdQPU0qVLGTlyJAceeCC77LILEydO5IYbbsi6rIazefNm1q1bB8C2bdvo6Ohgjz32yLiqxrN161YABg0axKBBg4iIjCuqHUllP0q00wxcAhwPjAJOltRT7+2aiDgifRQNanBY77SOjg5GjBjx4uu2tjb39qpsr7324oADDmDt2rVZl9JwmpqaWLFiBZs2bWLRokUsXbo065JqplJhDYwB1kbEHyLiOWAOcOLO1leVsJb0WkkLJM2XdJCkmZI2S1oq6XXV2GdWeup51PvYV561trYydepUZs6cybPPPpt1OQ3nhRde4Mgjj6StrY0xY8Zw2GGHZV1SzVQwrIcDGwpeb0yXdfdhSSslXStpRA/rX6JaPet24FLgKuDXwELglcDXgRm9vUnSJEnLJC1rb2+vUmmV1dbWxoYNf/972bhxI/vuu2+GFTWu5uZmpk6dyuLFiwdUjy8LW7Zs4bbbbmPcuHFZl1IzfQnrwqxKH5MKm+qh+e69upuA/SPiDcAtwBWl6qtWWA+JiJsiYjbwfETMicRNJKHdo4hoj4ijIuKoSZMm9bZZXRk9ejRr1qxh3bp1PPfcc8yZM4fx48dnXVZDOuOMM+jo6GD+/PlZl9KQhg0bxu677w7A4MGDGTt2LA899FDGVdVOU1NT2Y/CrEofhb3LjUBhT7kNeKRwXxHxeERsT1/+CHhjqfqqNRukueD5Rd3W7VKlfWaipaWFGTNmcNxxx9HZ2cmpp546oH51rJVDDz2Uo48+mvXr1zN9+nQAZs+ezYoVKzKurHHss88+XHHFFTQ3N9PU1MTcuXMH1BdjBYcv7wYOlnQAyWyPicDHuu1rn4h4NH05HniwZH3VONor6V+BqyPi6W7LRwJnRsRZZTQzcA5DZ2TChAlZl9Dw5s2bl3UJA0JE7HTSPvzww2VnziGHHFJ0f5JOAC4m6bj+OCL+S9L5wLKIuFHSN0lCegfwBPDZiCj6a0xVetYR8cNelq+V9Mdq7NPMbGdUcmJARNwM3Nxt2bkFz78MfLkvbWYxde/sDPZpZlZUBWeDVEUWZzB6XpuZ1Z16n3KbRVh7LNrM6s6AvPmApKfoOZQF7FqNfZqZ7YwB2bOOiCHVaNfMrFoGZFibmeWNw9rMLAcc1mZmOeCwNjPLgQE5G8TMLG/cszYzywGHtZlZDjiszcxywGFtZpYDPsBoZpYD7lmbmeWAw9rMLAcc1mZmOeCwNjPLAYe1mVkOeDaImVkOuGdtZpYDDmszsxxwWJuZ5YDD2swsBxzWZmY54NkgZmY54J61mVkO1HtY13e/38ysRiSV/SijrXGS/k/SWknTeljfKumadP1dkvYv1abD2syMyoW1pGbgEuB4YBRwsqRR3TY7DfhbRIwEvgNcWKo+h7WZGckBxnIfJYwB1kbEHyLiOWAOcGK3bU4ErkifXwu8WyW+Bep5zLq+B5B6IGlSRLRnXUe55s6dm3UJfZa3zziPBvBnXHbmSJoETCpY1F7wmQ0HNhSs2wi8qVsTL24TETskbQH2BB7rbZ/uWVfWpNKb2E7yZ1x9/oxLiIj2iDiq4FH45dZT6Ee31+Vs8xIOazOzytoIjCh43QY80ts2klqA3YEnijXqsDYzq6y7gYMlHSBpF2AicGO3bW4EPp0+/wjw64go2rOu5zHrPBqI43y15s+4+vwZ74R0DPpM4JdAM/DjiFgl6XxgWUTcCFwOzJK0lqRHPbFUuyoR5mZmVgc8DGJmlgMOazOzHHBY95Okp3tY9k5JyyXtkPSRLOpqJL18xmdLWi1ppaRbJe2XRW2NpJfP+QxJ90u6V9KSHs7AsxpzWFfWn4BTgJ9mXEcjWwEcFRFvIDnza3rG9TSqn0bEP0XEESSf8UVZFzTQOawrKCL+GBErgReyrqVRRcRvIuKZ9OWdJHNYrcIi4smCly+nxAkbVn2eumd5dhqwIOsiGpWkycDZwC7AsRmXM+C5Z225JOkTwFHAt7KupVFFxCURcRDwH8BXsq5noHNYW+5IGgv8JzA+IrZnXc8AMAf4QNZFDHQOa8sVSUcCPyQJ6k1Z19OoJB1c8PK9wJqsarGEz2DsJ0kv8NKLs1wELAauB14JbAP+HBGHZVBeQ+jlMz4B+Cfg0XTZnyJifK1rayS9fM77AWOB54G/AWdGxKoMyrOUw9rMLAc8DGJmlgMOazOzHHBYm5nlgMPazCwHHNZmZjngsDYkdaZXV3tA0jxJL9uJto6R9Iv0+XhJ04psO1TS5/qxj/MkfaGXdZ9Kf45V6dX5vpAun+krIVqeOawN4NmIOCIiXg88B5xRuFKJPv9biYgbI+KCIpsMBfoc1r2RdDxwFvCedH77PwNbKtW+WZYc1tbdYmCkpP0lPSjpUmA5MELSeyT9Pr1m9zxJuwFIGifpIUlLgA91NSTpFEkz0uevlnS9pPvSx1uBC4CD0l79t9Ltvijp7vR61V8raOs/Jf2fpFuAQ3up/cvAFyLiEYCI2BYRP+q+kaRz0308IKldktLlny+4VvacdNnRaX33SlohaUhvdUp6uaT56c/3gKSTduLvwewlfNU9e5GkFuB4YGG66FDgMxHxOUnDSC7mMzYitkr6D+BsSdOBH5FclW0tcE0vzX8P+G1EfFBSM7AbMA14fXrNZCS9BzgYGAMIuFHSO4GtJDcUPZLk3+xy4J4e9vH6XpZ3NyMizk/3OQt4H3BTWs8BEbFd0tB02y8AkyPijvTLaVuROvcCHomI96Zt715GLWZlcc/aAHaVdC+wjOQGCpeny9dHxJ3p8zcDo4A70m0/TXJK8muBdRGxJpLTYa/qZR/HApcBRERnRPQ0PPGe9LGCJJBfSxKK7wCuj4hn0uss37hTPy28S9Jdku5P6+q6JMBK4Or0in470mV3ABdJ+jwwNCJ2FKnzfmCspAslvaOXn9GsX9yzNkjHrAsXpCMDWwsXAYsi4uRu2x1B5S5ML+CbEfHDbvs4q8x9rALeCPy61x1Ig4FLSe42s0HSecDgdPV7gXcC44H/J+mwiLhA0nySa5LcmV7xr8c60/bfmG77TUm/6urBm+0s96ytXHcCb5M0EkDSyyQdAjwEHCDpoHS7k3t5/63AZ9P3Nkt6BfAUMKRgm18CpxaMhQ+X9CrgduCDknZNx4zf38tCTDQ/AAABAUlEQVQ+vglMl7R3+v7WtEdcqCuYH0v385F02yZgRET8BvgSycHP3SQdFBH3R8SFJL95vLa3OiXtCzwTEVcB/0NygNOsItyztrJExF8lnQLMltSaLv5KRDwsaRIwX9JjwBKSsePupgDtkk4DOoHPRsTvJd0h6QFgQUR8UdLrgN+nPfungU9ExHJJ1wD3AutJDoL2VOPNkl4N3JIeNAzgx9222SzpRyRDFn8E7k5XNQNXpePMAr6Tbvt1Se9Ka16d1rm9pzqBkcC3lFzF7nnSLyezSvBV98zMcsDDIGZmOeCwNjPLAYe1mVkOOKzNzHLAYW1mlgMOazOzHHBYm5nlwP8HLRHi5FdX3UcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm.plot(plot_lib=\"seaborn\", number_label=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1b8fb4d0>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAEWCAYAAAAOzKDmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG1BJREFUeJzt3Xu8XeOdx/HP95xELoSE3EjiFiSNdiipkgxFK5W6tFOmGFTQl1eLupRqtZ2WqalLp0pVtTFFKxiqYkrUpYa6FJVEXEKKUak0QeIeEpKT3/yx1ontzNn7rJOz9tlrnXzfXut19l77Oc/6beGX53nWs55HEYGZWVk1NToAM7OucBIzs1JzEjOzUnMSM7NScxIzs1JzEjOzUnMSW0tI6ifpJklvSPpNF+o5VNLtecbWCJJ+L+mIRsdhXeckVjCS/kXSTElLJS1K/2f7xxyqPhAYBmwUEf+8ppVExFURMSmHeD5A0u6SQtINbc5vl56/O2M9Z0ia1lG5iJgcEb9aw3CtQJzECkTS14ALgB+QJJxNgZ8Bn82h+s2ApyNiZQ511ctiYIKkjSrOHQE8ndcFlPB/9z1JRPgowAFsACwF/rlGmT4kSW5helwA9Ek/2x1YAJwCvAwsAo5MPzsTeA9YkV7jaOAMYFpF3ZsDAfRK308BngPeAv4KHFpx/r6K35sAPAy8kf6cUPHZ3cD3gfvTem4HBlf5bq3x/xw4Lj3XnJ77LnB3RdkLgReAN4FZwK7p+b3bfM9HK+L49zSOZcBW6bkvpZ9fAlxfUf+5wJ2AGv3fhY+OD/+NVBy7AH2B6TXKfBvYGdge2A7YCfhOxefDSZLhCJJEdbGkQRHxPZLW3bURsV5E/LJWIJLWBX4CTI6IASSJak475TYEZqRlNwLOB2a0aUn9C3AkMBRYBzi11rWBXwNfTF9/GphLkrArPUzy72BD4GrgN5L6RsStbb7ndhW/czhwDDAAmN+mvlOAf5A0RdKuJP/ujog0o1mxOYkVx0bAkqjd3TsU+LeIeDkiFpO0sA6v+HxF+vmKiLiFpDUyZg3jWQV8WFK/iFgUEXPbKbMP8ExEXBkRKyPiGmAesF9Fmcsj4umIWAZcR5J8qoqIPwEbShpDksx+3U6ZaRHxSnrNH5G0UDv6nldExNz0d1a0qe8d4DCSJDwN+GpELOigPisIJ7HieAUYLKlXjTKb8MFWxPz03Oo62iTBd4D1OhtIRLwNHAR8GVgkaYaksRniaY1pRMX7F9cgniuB44E9aKdlKukUSU+ld1pfJ2l9Du6gzhdqfRgRfybpPosk2VpJOIkVxwPAcuBzNcosJBmgb7Up/7+rldXbQP+K98MrP4yI2yJiL2BjktbVpRniaY3p72sYU6srgWOBW9JW0mppd+8bwBeAQRExkGQ8Tq2hV6mzZtdQ0nEkLbqFwGlrHrp1NyexgoiIN0gGsC+W9DlJ/SX1ljRZ0nlpsWuA70gaImlwWr7D6QRVzAF2k7SppA2A01s/kDRM0v7p2Ni7JN3SlnbquAXYJp0W0kvSQcA44OY1jAmAiPgr8AmSMcC2BgArSe5k9pL0XWD9is9fAjbvzB1ISdsAZ5F0KQ8HTpNUs9trxeEkViARcT7wNZLB+sUkXaDjgRvTImcBM4HHgMeB2em5NbnWHcC1aV2z+GDiaSIZ7F4IvEqSUI5tp45XgH3Tsq+QtGD2jYglaxJTm7rvi4j2Wpm3Ab8nmXYxn6T1WtlVbJ3I+4qk2R1dJ+2+TwPOjYhHI+IZ4FvAlZL6dOU7WPeQb8CYWZm5JWZmpeYkZmaFImmMpDkVx5uSTqpa3t1JMysqSc0kd7s/HhFtp/MAbomZWbF9EvjfagkMoNbEykJYd4MNY9DwER0XtMLo37u50SFYJz0z99ElETFkTX+/ef3NIlYuy1Q2li2eS3JXudXUiJhapfjBJFOLqip8Ehs0fARf/fmNHRe0wthh+PodF7JCmTRuaNWWThaxchl9xnwhU9nlcy5eHhHjOyonaR1gfyrmMLan8EnMzMpAkP8KR5OB2RHxUq1CTmJm1nUCmnIfRjiEDrqS4IF9M8uLlO3IVJX6A3sBN3RU1i0xM8tBvt3J9MH/jTosiJOYmeUlYysrb05iZtZ1oh4D+5k4iZlZDrKPd+XNSczM8pH/3clMnMTMLAd1mSeWiZOYmXWdcHfSzErOLTEzKy93J82szAQ0e2DfzMrMY2JmVl7uTppZ2bklZmal5paYmZVWJ5bZyZuTmJnlw48dmVl5eWDfzMrO3UkzKy2vJ2Zm5ebupJmVnQf2zazUPCZmZqWlxnUnve+kmeUj330nB0q6XtI8SU9J2qVaWbfEzCwXyrc7eSFwa0QcKGkdoH+1gk5iZtZlyerU+SQxSesDuwFTACLiPeC9auXdnTSzrpNQU7YDGCxpZsVxTJvatgQWA5dLekTSf0pat9ql3RIzs1x0oiW2JCLG1/i8F7AD8NWIeEjShcA3gX9tr7BbYmaWC0mZjgwWAAsi4qH0/fUkSa1dTmJmlou8klhEvAi8IGlMeuqTwJPVyrs7aWZdp/TIz1eBq9I7k88BR1Yr6CRmZl0mMncVM4mIOUCtcbPVnMTMLBdNTX4A3MxKLOfJrpk5iZlZ1+U/JpaZk5iZ5cItMTMrrbwH9jvDSczMcpE+UtTtnMTMrOvk7qSZlZyTmJmVmpOYmZWWB/bNrPw8T8zMSkt+7MjMSs7dSTMrN3cn1w4rVqzk8p9eR8vKFlatWsW47bZmj70nNDosq2HxK29wwSU38trrS5HEp/fcgf0n79zosAqnx7XEJC2NiPXanNsNuAD4B+DgiLi+Xtcvql69mjni2APp02cdWlpauOyi69hq7BaM2nzjRodmVTQ3NXHUoZMYvcXGvLPsXb727als/5HRbDpySKNDK4xOLD2du+4eifsbyTZMV3fzdQtDEn36rANAS8sqWlpWNWr3d8tow0EDGL1F8pdM/359GDliCK+89maDoyqeHNfY75Ru7U5GxPMAklZ153WLZtWqVfzi/Kt5dcnr7DRxO0Zu5lZYWby0+HWee34RY0aPbHQoheNnJyuk+9AdAzBw2CYNjiZ/TU1NfOXUw1i2bDnXXnYTLy1awrCNBzc6LOvAsuXvcc6Pr+NLh+9N//59Gh1O4awt3clMImJqRIyPiPHrbrBho8Opm379+rL5ViN5dt7zjQ7FOrByZQvn/Pg6PjHxI0zY6UONDqd41LjuZCGTWE/29tJ3WLZsOQAr3lvJc0//jcFDe26i7gkigoum/o6RIwbzuX12aXQ4hSRAynbkrZDdyZ7srTff5sZrbmPVqiAi2Ha7bRiz7ZaNDstqeOovL3DXfY+x2aihnHj6zwE4/AufZPxHt25wZEXSM5+d7C9pQcX784F7genAIGA/SWdGxLZ1jKFwhm8yhC+fclijw7BOGDd2U3539fcaHUbhNeU4sC/peeAtoAVYGRFVt2+rWxKLiGpdVd/WMetp6tNV3CMilnRUyN1JM+sykW9LrDM8sG9muejEwP5gSTMrjmPaqS6A2yXNqvL5am6JmVkuOjGwv6TWGFdqYkQslDQUuEPSvIi4p72CbomZWddlbIVlzXMRsTD9+TLJzcCdqpV1EjOzLhOiqakp09FhXdK6kga0vgYmAU9UK+/upJnlIse7k8OA6Wn3tBdwdUTcWq2wk5iZ5SKvya4R8RywXdbyTmJm1nV1eqQoCycxM+uy5NnJnvfYkZmtRdwSM7NSa9SMfScxM+s6uTtpZiXWup5YIziJmVkOCrzbkaTzJK0vqbekOyUtkeQFsczsAxq1smuWx44mRcSbwL7AAmAb4Ov5h2JmpaVkYD/Lkbcs3cne6c/PANdExKuNajaaWTEVfZ7YTZLmAcuAYyUNAZbXNywzK5vCjolFxDeBXYDxEbECeAf4bL0DM7NyKeyYmKT+wHHAJempTYCOFjQzs7VMkfedvBx4D5iQvl8AnJV7JGZWXjkvitgZWZLY6Ig4D1gBEBHLkpDNzBLJoojFvTv5nqR+JAv3I2k08G7ukZhZqTUV+O7k94BbgVGSrgImAlPqGZSZlU9hHzuKiDskzQZ2JulGnphlQ0szW3uogQ+AZ7k7ORFYHhEzgIHAtyRtVvfIzKxUmpTtyP26GcpcArwjaTuSx43mA7/OPxQzK7NGDexnSWIrIyJIJrj+JCIuBAbkHomZlZZI7lBm+SdvWQb235J0OnAYsJukZt5/ntLMDKhPVzHTdTOUOYhkSsXREfEiMAL4YV2jMrNyyThbvzOD/5KaJT0i6eZa5TK1xIALI6JF0jbAWOCazJGY2VqhDjcnTwSeAtavVShLS+weoI+kEcCdwJHAFV2Nzsx6DpFMds1yZKpPGgnsA/xnR2WzJDFFxDvA54GLIuKfgG0zRWJma42c705eAJwGrOrwuhkqk6RdgEOBGem55qyRmFnPl/Xh77QhNljSzIrjmA/WpX2BlyNiVpZrZxkTOxE4HZgeEXMlbQnc1bmvaGY9XSeenVwSEbWW85oI7C/pM0BfYH1J0yKi3b09sjx2dA/JuFjr++eAE7JGa2Zrh7zG9SPidJKGE5J2B06tlsAgQxJLl6M+jWQcrG/FhfbsarBm1nMU9tlJ4CpgHrAFcCbwPPBwHWMys5JJ7k7m/+xkRNwdEfvWKpMliW0UEb8EVkTEHyPiKJIVLczMEir2oogr0p+LJO0DLARG5h6JmZVakbdsO0vSBsApwEUks2dPrmtUZlYqrd3JRshyd7L1uaU3gD3qG46ZlVXhWmKSLiJdV789EeFpFma2WqN2D6rVEpvZbVGYWalJ0Nyg/mStJHYtMCAiFleelDQUeLOuUZlZ6RRxnthPgF3bOb8X8OP6hGNmZVXEzXP/MSJuaHsyIq4Cdss/FDMrK5FtGZ567E1ZqztZ62pZJsma2dqiTq2sLGolsZcl7RQRf648KeljwOIqv5O7Yev14cRdR3fX5SwHgz52fKNDsAYo3BQLku3ZrpN0BdC6rs944IvAwXWOy8xKREBz0ZJYRPxZ0k7AccCU9PRc4OMR8XI3xGZmJVLIGftpsvpeN8ViZiVWyCRmZpZFMn2iYN1JM7POcEvMzEqtcFMsJN1E7QfA969LRGZWOgJ6FbA7+R/dFoWZlV7hWmIR8cfuDMTMykt1eqQoiyy7HW0NnA2M44O7HW1Zx7jMrGQa1RLL8gzk5cAlwEqSlV1/DVxZz6DMrHzqsdtRputmKNMvIu4EFBHzI+IMwHtOmtlqIlkUMcvRYV1SX0l/lvSopLmSzqxVPssUi+WSmoBnJB0P/B0YmuWLmdlaIt9W1rvAnhGxVFJv4D5Jv4+IB9srnKUldhLQHzgB2BE4HDgir2jNrGdQxn86Eoml6dve6VF1uleW3Y5ad/teChzZ8Vcxs7VN3lu2SWomWT1nK+DiiHioWtksdyfvop0sGBEeFzOz1TqRxAZLqtyIaGpETK0sEBEtwPaSBgLTJX04Ip5or7IsY2KnVrzuCxxAcqfSzGy1TjwAviQixmcpGBGvS7ob2BtYsyQWEbPanLpfkifCmtlqyZZtedWlIcCKNIH1Az4FnFutfJbu5IYVb5tIBveHdzVQM+tZcpyxvzHwq3RcrAm4LiJurlY4S3dyFsmYmEi6kX8Fjs4hUDPrIfIc2I+Ix4CPZi2fJYl9KCKWV56Q1KezgZlZz1bkx47+1M65B/IOxMzKTDRlPPJWaz2x4cAIoJ+kj/L+PpTrk0x+NTMDkuRQuKV4gE+T7HI0EvgR7yexN4Fv1TcsMysVQa8GrU9daz2xX5HcITggIn7bjTGZWck0siWWZUxsx3TWLACSBkk6q44xmVkJNaULI3Z05H7dDGUmR8TrrW8i4jXgM7lHYmallmzb1vGRtyxTLJol9YmId5NA1Q/wFAszW01kaxHVQ5YkNg24U9LlJJNejyJZ3dXMLKFcZ+x3SpZnJ8+T9BjJ80sCvh8Rt9U9MjMrjWTGfkGTGEBE3ArcCiBpoqSLI+K4ukZmZqXSoJuT2ZKYpO2BQ4CDSJ6dvKGeQZlZ+RRusqukbYCDSZLXK8C1JJuF7NFNsZlZaagz64nlqlZLbB5wL7BfRDwLIOnkbonKzEqlkXcna133AOBF4C5Jl0r6JI3r9ppZwRVusmtETI+Ig4CxwN3AycAwSZdImpR7JGZWXkqWp85y5K3DFmBEvB0RV0XEviQPg88Bvpl7JGZWWq3dySxH3jpVZ0S8GhG/8E5HZtZWo1pimaZYmJl1pNDzxMzMahHQXMApFmZmmRVusquZWXZCDepQOomZWS6KvLKrmVlNyRSLfHY7kjRK0l2SnpI0V9KJtcq7JWZmXZfvqq0rgVMiYrakAcAsSXdExJPtFXYSM7Nc5PVIUUQsAhalr9+S9BTJ9pFOYmZWH8miiJmLD5Y0s+L91IiY2m690ubAR4GHqlXmJGZmuejE3cklETG+w/qk9YDfAidFxJvVyjmJmVku8rw7Kak3SQK7KiJqLsLqJNYAf/jTk5z+o+tpWbWKwz87gZOneFGQIttqs6Fc9oOjVr/fbJONOHvqDH5+zd2NC6qA8ponpuQBy18CT0XE+R2VdxLrZi0tq/j6edcx/afHs8mwgex5xA+ZvNtHGLvlxo0Ozap4dv7L7HboOQA0NYknb/l3Ztz1aIOjKpZOjol1ZCJwOPC4pDnpuW9FxC3tFXYS62az5j7PlqMGs/nIwQB8fq8duOWPjzmJlcQnPjaG5xcs5oUXX2t0KMWS44KHEXEfnXie3JNdu9mixW8wYtig1e83GTaIRYvfaGBE1hmfn7Qjv71tVqPDKCRlPPLWbUlM0nGSHpJ0r6SjJG0t6XRJE9ope4ykmZJmLl6yuLtC7BYR8f/ONepxDeuc3r2ambzbR7jxzkcaHUrhtO47WajlqetgFElf90vAHsBNQD/gwbYFI2JqRIyPiPFDBg/pxhDrb5OhA/n7S+93RRa+9BrDB2/QwIgsq09NGMej815g8atvNTqUQmpUS6zbxsQionVJ67+QDNqtlXYYtxn/+7fFzP/7EjYeOpAb7pjNpd+f0uiwLIMDPz2e397urmRVXopn7dCrVzPnnfYFDjjhYlpagkP335kPjfagftH169Ob3Xcay8k/uKbRoRRWPbqKWTiJNcCkidsyaeK2jQ7DOmHZuysYvdc3Gh1GoXl5ajMrN3cnzayskkF7dyfNrKzyXU+sU5zEzCwXHhMzsxKrz8a4WTiJmVku3J00s9Kq12z8LJzEzCwfbomZWZl5ioWZlZrHxMysvDxPzMzKzt1JMyst4ZaYmZWcp1iYWbm5JWZmZdaoRRG925GZ5SKvNfYlXSbpZUlPZLmuk5iZ5SO/nUKuAPbOell3J82sy/JcFDEi7pG0edbyTmJm1nWdm+w6WNLMivdTI2Lqml7aSczMctGJdtiSiBif13WdxMwsB14U0cxKrlEz9n130sy6LOuNyYxTLK4BHgDGSFog6eha5d0SM7N85NQSi4hDOlPeSczMcuFVLMys1LyKhZmVl6DJSczMys3dSTMrKS+KaGal50URzazU3BIzs1LzY0dmVmruTppZacn7TppZ2XnGvpmVm1tiZlZmHhMzsxJTw7ZscxIzsy5r5Ix9L4poZqXmlpiZ5cJTLMys1DzFwszKy5NdzazMvBSPmZWeu5NmVmqeYmFmpZbXvpMAkvaW9BdJz0r6Zq2yTmJmlo+cspikZuBiYDIwDjhE0rhq5Z3EzKzLBDRJmY4MdgKejYjnIuI94L+Az1YrXPgxsdmzZy3p11vzGx1HHQwGljQ6COuUnvxntllXfnn27Fm39eutwRmL95U0s+L91IiYWvF+BPBCxfsFwMerVVb4JBYRQxodQz1ImhkR4xsdh2XnP7PqImLvHKtrr7kW1Qq7O2lmRbMAGFXxfiSwsFphJzEzK5qHga0lbSFpHeBg4HfVChe+O9mDTe24iBWM/8y6QUSslHQ8cBvQDFwWEXOrlVdE1a6mmVnhuTtpZqXmJGZmpeYk1g0kLW3n3G6SZktaKenARsRl1hM4iTXO34ApwNUNjsOs1Hx3skEi4nkASasaHIpZqbklZlaDpOMkPSTpXklHSdpa0umSJjQ6Nks4iZnVNgqYCHwJ2AO4CegHPNjIoOx97k6a1RARrWtZ/QU4vJGxWPvcEjOzUvOM/W6QDt5XPsB6PnAvMB0YBCwHXoyIbRsQnlmpOYmZWam5O2lmpeYkZmal5iRmZqXmJGZmpeYkZmal5iRWEpJaJM2R9ISk30jq34W6dpd0c/p6/1qbk0oaKOnYNbjGGZJOrfLZF9PvMVfSk63lJF3hFT2ss5zEymNZRGwfER8G3gO+XPmhEp3+84yI30XEOTWKDAQ6ncSqkTQZOAmYlM6L2wF4I6/6be3jJFZO9wJbSdpc0lOSfgbMBkZJmiTpgXStst9IWg9Wbws/T9J9wOdbK5I0RdJP09fDJE2X9Gh6TADOAUanrcAfpuW+LulhSY9JOrOirm+nW8//ARhTJfbTgVMjYiFARCyPiEvbFpL03fQaT0iaKiW7rko6IW29PSbpv9Jzn0jjmyPpEUkDqsUpaV1JM9Lv94Skg7rw52BFEBE+SnAAS9OfvYD/Br4CbA6sAnZOPxsM3AOsm77/BvBdoC/JZqRbk+zpdx1wc1pmCvDT9PW1wEnp62Zgg/QaT1TEMYlkwwyR/CV4M7AbsCPwONAfWB94liRZtf0erwIbVPmOVwAHpq83rDh/JbBf+noh0Cd9PTD9eRMwMX29XvrvqFqcBwCXVtTdbiw+ynO4JVYe/STNAWaSLKj4y/T8/IhoXVFhZ2AccH9a9giSnZ3HAn+NiGci+T93WpVr7AlcAhARLRHRXjdvUno8QtL6G0uSHHcFpkfEOxHxJjW22Mpoj3QJnMfTuFofyXoMuErSYcDK9Nz9wPmSTiBJbCtrxPk48ClJ50ratcp3tBLxKhblsSwitq88kfaw3q48BdwREYe0Kbc9NXZQ7iQBZ0fEL9pc46SM15hL0mr7n6oXkPoCPwPGR8QLks4gaU0C7EPSotof+FdJ20bEOZJmAJ8BHpT0qWpxpvXvmJY9W9LtEfFvGeK2gnJLrGd5EJgoaSsASf0lbQPMA7aQNDotd0iV37+TpJuKpGZJ6wNvAQMqytwGHFUx1jZC0lCSbuw/SeqXjkntV+UaZwPnSRqe/n6ftAVVqTVhLUmvc2BatgkYFRF3AaeR3HRYT9LoiHg8Is4laamOrRanpE2AdyJiGvAfJDcWrMTcEutBImKxpCnANZL6pKe/ExFPSzoGmCFpCXAf8OF2qjgRmCrpaKAF+EpEPCDpfklPAL+PiK9L+hDwQNoSXAocFhGzJV0LzAHmk9x8aC/GWyQNA/6QDtYHcFmbMq9LupSk6/c8yY7QkIzTTZO0AUlL68dp2e9L2iON+ck0znfbixPYCvihkpVFVpAmbSsvr2JhZqXm7qSZlZqTmJmVmpOYmZWak5iZlZqTmJmVmpOYmZWak5iZldr/AYilfmW3TWXgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm.plot(cmap=plt.cm.Blues, number_label=True, one_vs_all=True, class_name=\"L1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1b8fb730>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAEWCAYAAAD1m1U5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcFMX9//HXexdFVBB0QU4BFTWeqEg8olEjCKJo4n2j8efXKyTxiJJ4BY1n4pWgEQ3etwkRBYN4YNR4sCiigAriwbKgnApy735+f1TP0js7Mzuws7vM7Of5ePSD6erq6upZ9rPVXdVdMjOcc85VV9TYFXDOuQ2RB0fnnEvBg6NzzqXgwdE551Lw4Oiccyl4cHTOuRQ8OOaApBaSnpf0naRn6lDOqZJeymXdGoOkFyWduZ77tpX0qaRNcl2v+iBpkKQ3Y+tLJW2b42OMl3RO9HmgpCdzWb5LrUkFR0mnSCqN/gPPiX6Jf5KDoo8Dtga2MrPj17cQM3vMzPrmoD7VSDpYkkn6V1L6HlH6+CzLuVbSo7XlM7P+ZvbQelb3CuABM1sRHXO8pBWSusTqcZikL9ez/HplZpub2cx6LH8UsKuk3evrGC5oMsFR0sXAHcANhEC2DXA3cHQOiu8KfGZma3JQVn2ZB+wvaatY2pnAZ7k6gIL1/j8lqXlUp+QA/ANwVV3qFjtGcS7KaWRPAOc2diUKnpkV/AJsASwFjs+QpzkheJZHyx1A82jbwUAZcAnwLTAHOCva9kdgFbA6OsYvgWuBR2NldwMMaBatDwJmAkuAL4BTY+lvxvbbH5gAfBf9u39s23jgOuCtqJyXgJI055ao/9+BC6O04ijtamB8LO+dwCzge2AicGCU3i/pPD+M1eNPUT2WA9tHaedE2+8Bno2VfzPwCqAU9TwImJGUNh64JjrH7aO0w4AvY3l+FOVbDEwBBsa2PRjVYQwhyB4Wpd0NvBidy1tA++hnvgj4BNgzVsYVwOdRHaYCP49tS/6ZWfQddIzKTizLAIvlOxuYFh1vLNA1tq1PVIfvgL8Brye+z2j7AcAXjf17VehLU2k57gdsAozMkOcPwL5AT2APoDdwZWx7e0KQ7UQIgMMktTGzawit0acsXFL9I1NFJG0G3AX0N7OWhAA4KUW+LYHRUd6tgNuA0Uktv1OAs4B2wMbApZmODTwMnBF9PpwQSMqT8kwgfAdbAo8Dz0jaxMz+k3See8T2OZ3QkmkJfJVU3iXA7tG9uQMJ392ZFv2WJ9kN+DRF+mzgPsIfnWokbQQ8T/jj0A74FfCYpB1j2U4hBPCWQOL+4AmEn28JsBJ4G3g/Wn+W8H0nfA4cSPj5/xF4VFKHFPWsYmbl0fe0uZltTvi/92RU52OA3wO/ANoCbxBag0gqAf4Zq9vnhGAYNw3oJqlVpjq4umkqwXErYL5lvuw9FRhqZt+a2TzCL8Hpse2ro+2rzWwMoTWwY4pyslFJuG/UwszmmNmUFHkGANPN7BEzW2NmTxBaE0fF8jxgZp+Z2XLgaUJQS8vM/gdsGQWOMwjBMjnPo2a2IDrmXwgt6trO80EzmxLtszqpvGXAaYRg8yjwKzMrS1NOa0LrLJUbgaMk7ZKUvi+wOXCTma0ys1eBF4CTY3meM7O3zKzSonuZwEgzmxitjwRWmNnDZlYBPAXsGTuHZ6JgV2lmTwHTCX88syLpcmAnQmsR4P+AG81sWvR/8gagp6SuwBHAVDN7Nvou7wDmJhWZ+I5aZ1sHt+6aSnBcAJRIapYhT0eqt3q+itKqykgKrssIv5TrxMx+AE4EzgPmSBotaacs6pOoU6fYevyXJtv6PAJcBBxCipa0pEskTYt63hcTWksltZQ5K9NGM3uPcBtBhCCeziJC6y5VGfMIl5hDkzZ1BGaZWWUsLfl7SlW/b2Kfl6dYr/ouJZ0haZKkxdF3siu1fyeJffsDvwaOif6IQbhHfWesvIWE76ZT4nwS+0ct7OT6J76jxdnUwa2fphIc3wZWAMdkyFNO+E+bsA01Lzmz9QOwaWy9fXyjmY01sz5AB0Jr8L4s6pOo0+z1rFPCI8AFwJioVVcluuy9nHDJ2cbMWhPueylR9TRlZny1k6QLCS3QcuB3GbJOBnbIsP1WQlDfO5ZWDnRJ6ghK/p7W+9VTUWvuPsIflK2i7+Rj1n4nmfbdEXgIOMHM4gFuFvB/ZtY6trSIWvZzgHjPvOLrkR8R7rl+v77n5WrXJIKjmX1H6HgYJukYSZtK2khSf0m3RNmeAK6MxtmVRPlrHbaSxiTgIEnbSNoCGJLYIGnraKzaZoR7XUuBihRljAF2iIYfNZN0IrAz4ZJxvZnZF8BPCfdYk7UE1hB6tptJuhqI39f6hnCvK+v/N5J2AK4nXFqfDvxOUrrL//eA1pI6pdpoZouBv1A9wL5L+GP0u+hnejDh1kOuxgJuRgiu8wAknUVoOWYU3Q98DrjSzN5M2vx3YEjiFoGkLSQlhoCNBnaR9IvoSmcwSX9cCT+/F9fzfFyWmkRwBDCz24CLCTe65xH+el8E/DvKcj1QSmi9fES4OX/9eh5rHOG+1WRCj288oBUROinKCZdTPyW05JLLWAAcGeVdQAgIR5rZ/PWpU1LZb5pZqlbxWMIv3WeES9MVVL+kSwxwXyDp/dqOE/1yPwrcbGYfmtl0QkfEI9GwneR6rSL0JJ+Wodg7if0xifYZCPQH5hN6oc8ws09qq182zGwqISC/TfjjsBuhd7s2exHu1d4WjatdKmlpVOZIQq/9k5K+J7RE+0fb5gPHAzcRfu49UhzvZODeOp6aq4VSdxo61zgkJXpv94zdo3MRSUcBp5vZCY1dl0LnwdE551JoMpfVzrn8Ialf9Iz9DElXpMlzgqSpkqZIejyWfqak6dFyZix9b0kfRWXeFXV2pa+DtxydcxuS6BHPzwhPCpURHkw4Obr/m8jTgzAs7FAzWySpnZl9Gz08UQr0InSkTQT2jvK8RxhW9Q6hw/MuM0vbseUtR+fchqY34THSmVGH25PUfAfC/wOGmdkiADP7Nko/HBhnZgujbeOAftETTa3M7O1o7OjDZB7aR6ZB0RuETSRr6TE8r3Td018Yk0++/Ppr5s9fUOu4zUy6qJmtyHI46XwqpxBGQiQMN7PhsfVOVB8lUQb8OKmYHQAkvUV4T8C10SOuqfbtFC1lKdLT2uCDY0uKOLbaeGq3ofv7m+MbuwpuHfT6ycF1LmMFxrFsllXee1mywsx6ZciSKlAnR95mhGFOBwOdgTck7Zph32zKrMabZM65OhMhmGSzZKGM6k8Fdabm02plhGfmV0cPNnxKCJbp9i2LPmcqsxoPjs65OhPQTMpqycIEoIek7pI2Bk4CRiXl+TfhUdLEm4x2IDy/PxboK6mNpDZAX2Csmc0BlkjaN+qlPoPwBFNaG/xltXMuPxRle9eylluTZrZG0kWEQFcMjDCzKZKGAqUW3oaeCIJTCU9MXRY9VYak6wgBFsKbtBZGn88nPIHVgvAkWMZHMD04OudyIpeXodFrAcckpV0d+2yEx4EvTrHvCGBEivRSsnguPsGDo3OuzoQoyu6SOW94cHTO5UShdWB4cHTO1ZlYh3uOecKDo3Ou7gTFflntnHPVJcY5FhIPjs65nPDLauecS8Fbjs45lyR0yBRW09GDo3OuzsLjg41di9zy4Oicywm/rHbOuRSKap/KO694cHTO1ZkPAnfOuTT8sto555JI3nJ0zrmUsnyRbd7w4OicqzN/fNA559Lwy2rnnEsiVHBDeQqtJeycayRFym7JhqR+kj6VNEPSFSm2D5I0T9KkaDknSj8kljZJ0gpJx0TbHpT0RWxbz0x18Jajc67OBBTnqOEoqRgYBvQhTKk6QdIoM5ualPUpM7sonmBmrwE9o3K2BGYAL8WyXGZmz2ZTD285Oudyoii6tK5tyUJvYIaZzTSzVcCTwNHrUaXjgBfNbNl67OvB0TlXd8rykjrLy+pOwKzYelmUluxYSZMlPSupS4rtJwFPJKX9KdrndknNM1XCg6NzLieKslyAEkmlseXcpKJShdDk2a6fB7qZ2e7Ay8BD1QqQOgC7Eea3ThgC7ATsA2wJXJ7pfPyeo3MuJ9bhluN8M+uVYXsZEG8JdgbK4xnMbEFs9T7g5qQyTgBGmtnq2D5zoo8rJT0AXJqpkt5ydM7VWeJlt9ksWZgA9JDUXdLGhMvjUdWOF1qGCQOBaUllnEzSJXViH0kCjgE+zlQJbzk653IiVy0tM1sj6SLCJXExMMLMpkgaCpSa2ShgsKSBwBpgITAosb+kboSW5+tJRT8mqS0hlk8CzstUDw+OzrmcyOUQcDMbA4xJSrs69nkI4R5iqn2/JEUHjpkdui518ODonMsJ+YsnnHOuOpHbluOGwIOjcy4nCq1314Ojcy4nCuyq2oOjc67uwvscCys6enB0zuVEYYVGD47OuRzxl90651wNQgXWdvTg6JyrMx/K45xzqfjUrM45l5r3VjvnXBK/rHbOuTR8ELhzzqVQYLHRg6NzLjd8KI9zziXJ5dSsGwoPjs65nCiw2OjB0TmXG4V2WV1or2Db4Ozc52Cu/eB1hk5+k8MvuTBlnr1/cSTXlL7K1RNe4ewH/tbANXTJprz0Gtf0/ClX7fYT/vPnYWnzTRw5mvM268JX73/YgLXbcEnZLfmi3lqOkpaa2eZJaecBFwIVwFLgXDObWl91aGwqKuLk267nzqNOYdHsOQx5YzSTR7/EnE+mV+Vpt113Dr/0Im497OcsW/wdLdtu1Yg1dpUVFTxx8ZX8+vnHadOpAzceeCS7D+hDxx/tUC3fiiVLee2eEXTfZ89GqumGJbyyLIflSf2AOwkTbN1vZjclbR8E3ArMjpL+Zmb3R9sqgI+i9K/NbGCU3h14kjBn9fvA6Wa2Kl0dGrrl+LiZ7WZmPYFbgNsa+PgNqluvnnw780vmf/k1FatXM+HZ59j9yL7V8vzkrFN4/d6HWLb4OwCWzFuQqijXQL4snUS7bbvRtntXmm28MfscN5DJL7xUI9+ooX+m72/Pp9kmzRuhlhsmZbnUWo5UDAwD+gM7AydL2jlF1qfMrGe03B9LXx5LHxhLvxm43cx6AIuAX2aqR4MGRzP7Pra6GWANefyG1qZjBxaVzalaXzx7Lm06dKiWp9323dm6x7Zc9vJIfvfaKHbuc3AD19LFLSqfS5vOHavWW3fqwKI5c6vl+XrSxyyaXc7u/Q9r6Opt0HI4b3VvYIaZzYxadk8CR9elbtFc1YcCz0ZJDxHmrk6rwe85SrpQ0ueEluPgNHnOlVQqqXRFPsfPFP8PzKqfT1GzZrTbrjt/6Xc8/xh0IacPu5UWW7RqoAq6Gqzm/7f4rHqVlZU8c8UfOfbGqxqyVhu8bFuN0TdZkvj9jpZzk4rrBMyKrZeRYqpV4FhJkyU9K6lLLH2TqNx3JCUC4FbAYjNbU0uZVRq8t9rMhgHDJJ0CXAmcmSLPcGA4QFsV5210XDR7Dm06r20ptu7UnsVzq7dCFs+ewxcT3qdyzRoWfDWLb6Z/TrvtuvtN/kbSplMHFpWVV60vnj2H1u23rlpfuWQp5VM/5bZ+JwDw/TfzuPv4s7ngmRF03WuPBq/vBkNal6lZ55tZr0ylpUhLjgPPA0+Y2cqoL+MhQssQYBszK5e0LfCqpI+A76kpY2xpzN7qJ6mlWZvvvpr4Ie22685WXbtQvNFG7HPc0UwePa5ankkvjGWHg/YHYLOt2tBu+22Z/+VXjVFdB3Tdew++/TzcJ16zahUTnh3F7gP6VG1vsUUr/vL1ZG6Y9jY3THub7r339MAYKVJ2SxbKgHhLsDNQHs9gZgvMbGW0eh+wd2xbefTvTGA8sCcwH2gtKdEgrFFmsgZtOUrqYWaJrtoBwPRM+fNdZUUFT11yFYOfe4yi4iL+9/BTzJn2GUddeSlfvf8hk8eMY+q48ez8s4O4pvRVKisr+dcfrueHhYsbu+pNVnGzZpz4l+u46+jTqKyoYP8zTqTjzjsy6ro/03Wv3dljQN/aC2milLsXOk4AekS9y7OBk4BTqh1L6mBmiRv6A4FpUXobYFnUoiwBDgBuMTOT9BpwHKFhdibwXMbzSb4HliuSKqkemW8DugKHAasJvUUXmdmUTOW0VbEdy6b1UkdXP/7+Q1ljV8Gtg14/OZjS9z+oU2TbpXlze6J9h9ozAnt8/dXEWi6rkXQEcAdhKM8IM/uTpKFAqZmNknQjISiuARYC55vZJ5L2B+4FKglXxneY2T+iMrdl7VCeD4DTYq3PGuqt5WhmPsDcuSZkHe451srMxgBjktKujn0eAgxJsd//gN3SlDmT0BOeFX980DmXE/n09Es2PDg653Iily3HDYEHR+dcnQlvOTrnXE0i26df8oYHR+dcDoiiApub1YOjc67OBKjAxqd4cHTO1Z28Q8Y551IqsNjowdE5lxvecnTOuRQKLDZ6cHTO1Z0Exd5b7ZxzNflltXPOpVBgsdGDo3Ou7grx8cFah21KukVSK0kbSXpF0nxJpzVE5ZxzeUJCRdkt+SKbMe19o1kDjyS8vnwH4LJ6rZVzLu8UFymrJV9kc1m9UfTvEYQJbRYW2o1X51zdFOJldTbB8XlJnwDLgQsktQVW1G+1nHP5ptAaTbVeVpvZFcB+QC8zWw0so44TbDvnCoxCyzGbJavipH6SPpU0Q9IVKbYPkjRP0qRoOSdK7ynpbUlTojmtT4zt86CkL2L79MxUh1pbjpI2BS4EtgHOBToCOwIvZHeazrmmIFctR0nFwDCgD6GfY4KkUWY2NSnrU2Z2UVLaMuAMM5suqSMwUdJYM0tM6XmZmT2bTT2y6ZB5AFgF7B+tlwHXZ1O4c67pyGHLsTcww8xmmtkqwoyBWV2tmtlniemfo/mrvwXars/5ZBMctzOzWwjTqWJmywn3X51zDghBr6hYWS1AiaTS2HJuUnGdgFmx9bIoLdmx0aXzs5K61KyTegMbA5/Hkv8U7XO7pOaZzimb4LhKUgvAogNuB6Sd69U51xQJKbsFmG9mvWLL8BqF1WRJ688D3cxsd+Bl4KFqBUgdgEeAs8ysMkoeAuwE7EOYu/ryTGeUTXC8BvgP0EXSY8ArwO+y2M8515QUKbuldmVAvCXYGSiPZzCzBWaWaKTdB+yd2CapFTAauNLM3ontM8eClYTbhRnnsK61Q8bMxkl6H9iXENF/bWbza9vPOdfE5G4ozwSgh6TuwGzgJOCU6odSBzObE60OBKZF6RsDI4GHzeyZVPsoNF+PAT7OVIlsHh88AFhhZqOB1sDvJXXN4gSdc01FNE1ClpfVGZnZGuAiYCwh6D1tZlMkDZU0MMo2OBqu8yEwGBgUpZ8AHAQMSjFk5zFJHwEfASXU0rGczSDwe4A9JO1BeGxwBPAw8NMs9nXONQmC4tzNsGVmY4AxSWlXxz4PIdxDTN7vUeDRNGUeui51yOZs1piZEbrS7zKzO4GW63IQ51xhkyi4F09k03JcImkIcBpwUDRAc6Na9nHONTVN7fFB4ETC0J1fmtlcwnijW+u1Vs65vNMkW47AnWZWIWkHwjihJ+q3Ws65vNMEW47/BZpL6kQY43gW8GB9Vso5l2eU5RjHPGo5ZhMcZWbLgF8AfzWznwO71G+1nHP5RsVFWS35IqvgKGk/4FTCqHOA4vqrknMu7yTedpurd5ZtALK55/hrwniikdFAzG2B1+q3Ws65fKP8aRRmJZvHB/9LuO+YWJ9JGJHunHNr5VGrMBvZvOy2LeFFE7sAmyTS13W0uXOugCm/hulkI5uG8GPAJ0B34I/Al4QHw51zbq0Cu+eYTXDcysz+Aaw2s9fN7GzCG3qccw6I4l6B9VZn0yGzOvp3jqQBhPeqda6/Kjnn8lKBXVZnExyvl7QFcAnwV6AV8Nt6rZVzLs/k1yVzNrLprU7MMvgdcEj9Vsc5l68Kbd7qtMFR0l+pOW9DFTPz4TzOuUA0qcvq0garhXMu7+VTZ0s2MgXHp4CWZjYvniipHfB9vdbKOZdf8myYTjYyhfq7gANTpPcBbq+f6jjn8lUu3+coqZ+kTyXNkHRFiu2DJM2LzRNzTmzbmZKmR8uZsfS9JX0UlXmXarlJmik4/sTM/pWcaGaPESawcc65tXI0CDyabWAY0B/YGThZ0s4psj5lZj2j5f5o3y0J00n/mDD16jWS2kT57wHOBXpES79M9cgUHDOdRWHdXHDO1U2iQyY373PsDcwws5lmtgp4kjCHVTYOB8aZ2UIzWwSMA/pJ6gC0MrO3ozmxHiZMz5pWpnuO30rqbWbvxRMl7QPMS7NPznXddUfuHvlwQx3O5cDTnXZo7Cq4dbBoycKclLMOQ3lKJMU7fIeb2fDYeidgVmy9jNASTHaspIOAz4DfmtmsNPt2ipayFOlpZQqOlwFPS3oQmBil9QLOIEyy7ZxzkXWamnW+mfXKXFgNycMKnweeMLOVks4DHgIOzbBvNmVWk/ZsohZj76jQQdEi4Mdm9m6mQp1zTUxuX3ZbBnSJrXcmPLZcxcwWmNnKaPU+YO9a9i2j+mPPNcpMlvEJGTP7lnBz0znnMsvdUJ4JQA9J3YHZhCvVU6ofSh3MbE60OhCYFn0eC9wQ64TpCwwxs4WSlkjaF3iXcAX810yVyObZauecq4WgKDf9tGa2RtJFhEBXDIyIZiEYCpSa2ShgsKSBwBpgIeHKligIXsfa1yoONbPETdXzCZMDtgBejJa0PDg653Ijh4PAzWwMMCYp7erY5yGE6VtS7TsCGJEivRTYNds6eHB0ztVd4p5jAcn04onnyfziiYH1UiPnXB4SFBfWpKSZWo5/brBaOOfyX1NpOZrZ6w1ZEedcHmtKl9UJknoANxKecYzPPrhtPdbLOZdvCiw4ZtP3/gDhge01hDeBPww8Up+Vcs7lm2goTzZLnsimpi3M7BVAZvaVmV1LeEzHOecCUXDBMZuhPCskFQHTo4GZs4F29Vst51zeaYKX1b8BNgUGE55fPB04M+MezrkmRQgVFWW15ItsZh9MPIazFDirfqvjnMtbBdZyzKa3+jVSDAY3M7/v6JwLmuJQHuDS2OdNgGMJPdfOObdWUwuOZjYxKektST5A3DkX07QeHwSqJqxJKCJ0yrSvtxo55/JPE72snsja14yvAb4AflmflXLO5aEmGBx/ZGYr4gmSmtdTfZxzeSl3L7vdUGRzNv9LkfZ2rivinMtzuZtDZoOQ6X2O7QlTF7aQtCdrZ+9qRRgU7pxzQRO753g4YV6GzsBfWBscvwd+X7/Vcs7ll9z2VkvqB9xJmEPmfjO7KU2+44BngH3MrFTSqYRppRN2B/Yys0mSxgMdgOXRtr7RJIIpZXqf40PAQ5KONbN/rsN5Oeeaohy1HCUVA8OAPoQpVSdIGmVmU5PytSQ81lw1VbSZPQY8Fm3fDXjOzCbFdjs1mkumVtncc9xbUutYhdpIuj6bwp1zTURu563uDcwws5lmtgp4Ejg6Rb7rgFuAFSm2AZwMPLEeZwNkFxz7m9nixIqZLQKOWN8DOucK0Tq9z7FEUmlsOTepsE7ArNh6WZS29mihH6SLmb2QoVInUjM4PiBpkqSrpMyROpuhPMWSmpvZyqhSLQAfyuOcqy77y+r5ZtYrU0kp0qre7xC9QvF2ormqU1dFPwaWmdnHseRTzWx2dDn+T8Ibxh5OV0Y2wfFR4BVJD0QVPDtTgc65JkhAUc46ZMqALrH1zkB5bL0lYf7p8VHjrz0wStLA2P3Ek0hqNZrZ7OjfJZIeJ1y+r39wNLNbJE0GDiN8BdeZ2dja9nPONSWCopwN5ZkA9JDUnfBy7ZOAUxIbzew7oKTqyKEX+tJEYIxalscDB8XyNANam9l8SRsBRwIvZ6pENi1HzOw/wH+igxwgaZiZXZjNvs65JkK5eULGzNZEsw6MJQzlGWFmUyQNBUrNbFQtRRwElJnZzFhac2BsFBiLCYHxvkyFZBUcJfUk9PycSHi2+l/Z7Oeca0JyOAjczMYAY5LSrk6T9+Ck9fHAvklpPxBempO1TE/I7EBozp4MLACeIkyydci6HMA51wSo8J6tztRy/AR4AzjKzGYASPptg9TKOZd/CuzxwUyh/lhgLvCapPsk/YzUXezOORd6q7NZ8kTa4GhmI83sRGAnYDzwW2BrSfdI6ttA9XPO5QOt0yDwvFBrTc3sBzN7zMyOJIw3mgRcUe81c87llwJ7Zdk6hXEzW2hm9/rMg865GlSU3ZInshrK45xzGSmng8A3CB4cnXO5kUedLdnw4OicywHl1SVzNjw4OufqTvhltXPOpZRHPdHZ8ODonMsNv6x2zrkk3lvtnHNpeG+1c84l895q55yryXurnXMuDW85Nl1T/vsez1z/N6yikv1POILD/++UlPnef/F17h/8Ry7/1z103W1Hpr1Zyr//fB8Vq9dQvFEzfnH5/7HjfnsBMOH5Vxj798dBonW7rRj059+z+ZZb8MJdD/LW06Np2SZMGT7wkl+y68H78t5zL/Py/U9VHWv2pzO54t/30mXn7ev/C8hD7Q8+kJ5D/4CKivniiWf4ZNjwatu7nfBzdr/ycpbP/QaAGQ88yhdPPEPb/X9Mz2t/X5Wv1Xbb8vYFv6V87MtsP+g0epxzJi27d+Xfu/6YVYsWVSuzzR678bPnn+ad839D2eixbNqpI/vf/zdUXExRs2bMeOARPn/kyfo/+YbmQ3mapsqKCp669k4GP3grrdu35eZjz2f3Q/enQ49u1fKtWLqM8Q//i257/KgqbfM2W3D+vX+i9dYllH/2BX89+3fc+OYzVKyp4Jnrh3H1iw+w+ZZb8K+b72X8oyM5cvAgAA4ddBx9zjmxWvm9jz6M3kcfBoTA+PfzrvLAmIaKitjrT9fw+slnsXzOXA4b80/KX3qF76d/Xi3frFFj+ODKodXS5v3vXcb1DfPIb9x6C/q/OY5vXn8TgPkTJlL+8msc8uwjKY+5+x8u5Zvxb1alrfh2Hq8efSKVq1bTbNNNOfzVF5j90qus+ObbXJ9yI8rtm8Al9QPuJMz3cr+Z3ZQm33HAM8A+ZlYqqRswDfg0yvKOmZ0X5d0beBBoQZiC4ddmZsllJhRWO7gefTn5E9p27UTnSfXDAAAPrUlEQVTJNh1ptvFG7D3gUD585X818j1/xwj6/L+T2Kj5xlVpXXbpQeutw2RpHXp0Y83K1axeuQrMwIyVy5djZqxYuozW7UpqlJlO6Quv0usof0FSOlvuuTtLv/yKH76eReXq1Xz93Gg6Hn7YOpfTeUA/5r72XypWrABg8ZRpLCubnTLv9mefzuzRL7FiwYKqtMrVq6lctRqAouYb59U7DbOWmJo1By+7lVQMDAP6AzsDJ0vaOUW+lsBg4N2kTZ+bWc9oOS+Wfg9wLtAjWvplqkcB/pTqx+K582nToV3Vepv2JXz3zbxqeWZNmc6iOfPY7dD90pbzwX/+S+edt2ej5htTvFEzTvrjb/jTgHMYcsDxzJ3xJfsf378q7+uP/pvrjzyHR664hWXfLalR1sTRr7HPkR4c02nRfmuWlc+tWl8+Zy4t2m9dI1/nI/rSd9wo9ht+Fy06tq+xvcvRR/D1cy9kdbxO/frw+SNP1NzWsT19x43iyAmv8+mw+wqs1Qihtzpn73PsDcwws5lmtgp4Ejg6Rb7rgFuAFbXWTuoAtDKzt6PW4sPAMZn2abDgKOlCSe9KekPS2ZJ6SBoiaf8Uec+VVCqpdN7CRamKawQpWt+xH3RlZSXP3nA3xw45P20J5dO/4N+3DueUoWEqnorVa3jjiVEMee5ebnzrGTrttG24/wgcdMpAhr7yKL8fNZxW7bbinzfeU62sLyZNY+MWm9Bxh+45OLcCleoXMekqqnzca4ze9xBe6jOQb9/4H73vuLna9k3atWWLnXZkbuwyOZ2ef/w9k2+4FausrLFteflcXuozkDEH9KHr8T+neclW63Yu+SD7N4GXJH6/o+XcpJI6AbNi62VRWhVJewJdzCzVX63ukj6Q9LqkA2NllmUqs8bp1H7GOdMFOAA4BzgEeJ5w7f9OckYzG25mvcysV9st2zRgFdNr3b4ti+as/Wu/aO58tohdAq/8YRnl07/g9tN+y5UHn8wXk6by9/Ou5KuPwq2PRXPmMfyCazjz1iG07Rp+JrOmzQCgbddOSGKv/gcz84MpALQq2ZKi4mKKior4yQkD+HLyJ9XqM3H0q/TyVmNGy+fMZdNYS7BFh/YsT2qxrVq0uOqSd+ZjT9Nmt12rbe9yVH9mvzgOW7Om1uO12X1X9rv7dga88yqdBxzOXjdcW+MyfsU33/L9Z9Np++Ne63taGyaxLi3H+Ynf72gZnqK0ZFV/1SQVAbcDl6TINwfYxsz2BC4GHpfUqrYyU2mwDhkzS0yt8ClwekMdN1e67rYT3345m/mz5tB66xImjn6Vs277Q9X2Fi0359b3/l21fvupv+UXV5xH1912ZNn3S7n73CEcfck5bLf32l++1luXMGfGVyxZsJiWW7Xmk7cm0n67rgB89+0CtmgXWheTxr1RrYVYWVnJ+y++zsWP31Hfp53XFk76iM27d2OzLp1ZPvcbtjl6AO9ceHG1PJu0a8uKb8PtkY59f8aSGdU7a7Y55kgm3/iXrI43Zr+fVX3e5/abmPPya5SPfZkWHbZm1aLFVKxYyUZbtKJkn734bPgDdTu5DU5OB4GXERpTCZ2B8th6S2BXYLxCsG0PjJI00MxKgZUAZjZR0ufADlGZnTOUWYP3VmepuFkxJ17zK/529uVUVlSw33H96dijO8/f8QBdd9uB3X92QNp9X39kJPO+KufFYY/w4rDQw/mrB2+h9dYlHHHRGdx2ym8o3qgZW3Zsxxk3Xw7AyFvupWza5yCxVaetOeW6tb/UMyZMpnX7tpRs07F+TzrPWUUF7185lIMe/0cYyvPUs3z/2Qx2uXQwiz78mPJxr9Lj7DPo2PdQrKKCVYsX895v1k6PtGnnTrTo0IF5b79XrdweZ5/Ojhf8PzZpW8LhL49izqv/pfSyPyQfvkqr7bdjj6vXlvvp30fw3Sef5f6EG1vuHh+cAPSQ1B2YDZwEVI2bM7PvgKrLNknjgUuj3uq2wEIzq5C0LaHjZaaZLZS0RNK+hA6cM4C/ZqqEMvRkbxB67bazvTfy4cauhlsHz+4zoLGr4NbBkCUL+XzN6joNUuy103b27vAbssrb7KcnTTSzjPcVJB0B3EEYyjPCzP4kaShQamajkvKOZ21wPBYYCqwBKoBrzOz5KF8v1g7leRH4VaahPN5ydM7lRg6fkDGzMYSxiPG0q9PkPTj2+Z/AP9PkKyVcjmfFg6NzLjf8CRnnnEvmb+VxzrmU5C1H55xLIkFRYYWTwjob51zj8fc5OudcCn7P0TnnkiQeHywgHhydczngvdXOOZeatxydcy6JBMU+NatzztXkl9XOOZeCX1Y751wy75BxzrnUvOXonHNJJCgurHBSWGfjnGs0/uIJ55xLxe85OudckgJ8fLCwQr1zrpFEvdXZLNmUJvWT9KmkGZKuyJDvOEkWzQ+DpD6SJkr6KPr30Fje8VGZk6KlXaY6eMvROZcbOWo5SioGhgF9CFOqTpA0ysymJuVrCQwmzCaYMB84yszKJe0KjAU6xbafGs0lUytvOTrn6i7x+GA2S+16AzPMbKaZrQKeBI5Oke864BZgRSLBzD4ws8R81FOATSQ1X59T8uDonMuN3F1WdwJmxdbLqN76Q9KeQBczeyFDOccCH5jZyljaA9El9VWqpXvdL6udc7mR/WV1iaT4pe1wMxseLynFPlXzS0sqAm4HBqWvinYBbgb6xpJPNbPZ0eX4P4HTgYfTleHB0TmXI1kHx/lm1ivD9jKgS2y9M1AeW29JmH96fNT4aw+MkjTQzEoldQZGAmeY2eeJncxsdvTvEkmPEy7f0wZHv6x2zuWAQssxm6V2E4AekrpL2hg4CRiV2Ghm35lZiZl1M7NuwDtAIjC2BkYDQ8zsraraSc0klUSfNwKOBD7OVAkPjs653MhRcDSzNcBFhJ7macDTZjZF0lBJA2vZ/SJge+CqpCE7zYGxkiYDk4DZwH2ZCvLLaudc3YmcPiFjZmOAMUlpV6fJe3Ds8/XA9WmK3Xtd6uDB0TmXG4X1gIwHR+dcrhRWdPTg6JzLgaw7W/KGB0fnXG54cHTOuRT8lWXOOZeKtxydc6667Ad45w0Pjs653PDg6JxzqXhwdM65GnyCLeecq0HeW+2ccyl5y9E555IU4OyDHhydczniwdE552rylqNzzqVQWLHRg6NzLhe8t9o552ryDhnnnEunsIJjYbWDnXONJ3ezDyKpn6RPJc2QdEWGfMdJMkm9YmlDov0+lXT4upaZ4C1H51wO5O6tPJKKgWFAH8Ic1hMkjTKzqUn5WgKDgXdjaTsTpnLdBegIvCxph2hzrWXGecvROZcbKspuqV1vYIaZzTSzVcCTwNEp8l0H3AKsiKUdDTxpZivN7AtgRlRetmVW2eBbjhM/nja/uMc+XzV2PepBCTC/sSvh1kmh/sy61rWAiR9MGqvNWpdkmX0TSaWx9eFmNjy23gmYFVsvA34cL0DSnkAXM3tB0qVJ+76TtG+n6HPGMpNt8MHRzNo2dh3qg6RSM+tVe063ofCfWXpm1i+HxaW6PreqjVIRcDswaB32TdVktRRpVTb44Oica3LKgC6x9c5AeWy9JbArMD56TVp7YJSkgbXsm6nMGvyeo3NuQzMB6CGpu6SNCR0soxIbzew7Mysxs25m1o1wGT3QzEqjfCdJai6pO9ADeK+2MlPxlmPjGV57FreB8Z9ZAzCzNZIuAsYCxcAIM5siaShQamZpg1qU72lgKrAGuNDMKgBSlZmpHjLLeNntnHNNkl9WO+dcCh4cnXMuBQ+ODUDS0hRp50n6SNIkSW9GI/udcxsIv+fYACQtNbPNk9Jamdn30eeBwAU5HivmnKsDbzk2kkRgjGxGLQNSnXMNy4fyNCJJFwIXAxsDhzZydVwK0c/oDGAV8ADwBnAc8LqZ/a8x6+bql7ccG5GZDTOz7YDLgSsbuz4upS7AAcA5wCHA80ALqj+/6wqQ33NsAKnuOSZtLwIWmdkWDVgt51wG3nJsJJJ6xFYHANMbqy7OuZr8nmPD2FRSWWz9NqCrpMOA1cAi4MxGqZlzLiW/rHbOuRT8sto551Lw4Oiccyl4cHTOuRQ8ODrnXAoeHJ1zLgUPjnlCUkX0Bp+PJT0jadM6lHWwpBeizwNrmTS9taQL1uMY1ybNChffdkZ0HlMkTU3kk/SgpOPW9VjO1QcPjvljuZn1NLNdCc/5nhffqGCdf55mNsrMbsqQpTWwzsExHUn9gd8Afc1sF2Av4Ltcle9crnhwzE9vANtL6iZpmqS7gfeBLpL6Snpb0vtRC3NzAEn9JH0i6U3gF4mCJA2S9Lfo89aSRkr6MFr2B24CtotarbdG+S6TNEHSZEl/jJX1B0mfSnoZ2DFN3YcAl5pZOYCZrTCz+5IzSbo6OsbHkoYrmmZO0uCotTlZ0pNR2k+j+k2S9IGklunqKWkzSaOj8/tY0ol1+Dm4AuZPyOQZSc2A/sB/oqQdgbPM7AJJJYQXWBxmZj9Iuhy4WNItwH2EN//MAJ5KU/xdhLfN/FxSMbA5cAWwq5n1jI7flzCjW2/CHMGjJB0E/ECY0W1Pwv+r94GJKY6xa5r0ZH8zs6HRMR8BjiS89OEKoLuZrZTUOsp7KWEipbeiPwYrMtSzLVBuZgOisv15dpeStxzzRwtJk4BS4GvgH1H6V2aWeEPMvsDOwFtR3jOBrsBOwBdmNt3CI1GPpjnGocA9AGZWYWapLnf7RssHhAC4EyEIHQiMNLNl0bsqM057mYVDJL0r6aOoXrtE6ZOBxySdRphdDuAt4DZJg4HWZrYmQz0/Ag6TdLOkA9Oco3PecswjyxOtt4ToSvOHeBIwzsxOTsrXk9y9TFfAjWZ2b9IxfpPlMaYAewOvpj2AtAlwN9DLzGZJuhbYJNo8ADgIGAhcJWkXM7tJ0mjgCOCd6Jn1lPWMyt87ynujpJcSLVTn4rzlWFjeAQ6QtD2ApE0l7QB8AnSXtF2U7+Q0+78CnB/tWyypFbAEaBnLMxY4O3Yvs5OkdsB/gZ9LahHd8zsqzTFuBG6R1D7av3nU4otLBML50XGOi/IWAV3M7DXgd4TOos0lbWdmH5nZzYSW9U7p6impI7DMzB4F/kzoEHKuBm85FhAzmydpEPCEpOZR8pVm9pmkc4HRkuYDbxLu/SX7NTBc0i+BCuB8M3tb0luSPgZeNLPLJP0IeDtquS4FTjOz9yU9BUwCviJ0GqWq4xhJWwMvR50sBoxIyrNY0n2ES+AvgQnRpmLg0eg+oYDbo7zXSTokqvPUqJ4rU9UT2B64VVIl4Y1I52f37bqmxt/K45xzKfhltXPOpeDB0TnnUvDg6JxzKXhwdM65FDw4OudcCh4cnXMuBQ+OzjmXwv8HLnvIe1MLOQEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm.plot(cmap=plt.cm.Reds, number_label=True, normalized=True, one_vs_all=True, class_name=\"L3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `normalized` :normalized flag for matrix (type : `bool`, default : `False`)\n",
    "2. `one_vs_all` : one-vs-all mode flag (type : `bool`, default : `False`)\n",
    "3. `class_name` : target class name for one-vs-all mode (type : `any valid type`, default : `None`)\n",
    "4. `title` : plot title (type : `str`, default : `Confusion Matrix`)\n",
    "5. `number_label` : number label flag (type : `bool`, default : `False`)\n",
    "6. `cmap` : color map (type : `matplotlib.colors.ListedColormap`, default : `None`)\n",
    "7. `plot_lib` : plotting library (type : `str`, default : `matplotlib`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot axes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distance/Similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Distance.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Distance.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter recommender"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This option has been added in `version 1.9` to recommend the most related parameters considering the characteristics of the input dataset. The suggested parameters are selected according to some characteristics of the input such as being balance/imbalance and binary/multi-class. All suggestions can be categorized into three main groups: imbalanced dataset, binary classification for a balanced dataset, and multi-class classification for a balanced dataset. The recommendation lists have been gathered according to the respective paper of each parameter and the capabilities which had been claimed by the paper."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/recommendation_block_diagram.jpg\" onerror='this.src=\"../images/recommendation_block_diagram.jpg\"' width=\"400\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig2. Parameter Recommender Block Diagram</p></center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For determining if the dataset is imbalanced, we use the following strategy:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$R=\\frac{Max(P)}{Min(P)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$State=\\begin{cases}Balance & R\\leq 3\\\\Imbalance & R > 3\\end{cases}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ERR',\n",
       " 'TPR Micro',\n",
       " 'TPR Macro',\n",
       " 'F1 Macro',\n",
       " 'PPV Macro',\n",
       " 'NPV Macro',\n",
       " 'ACC',\n",
       " 'Overall ACC',\n",
       " 'MCC',\n",
       " 'MCCI',\n",
       " 'Overall MCC',\n",
       " 'SOA6(Matthews)',\n",
       " 'BCD',\n",
       " 'Hamming Loss',\n",
       " 'Zero-one Loss']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.recommended_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`is_imbalanced` parameter has been added in `version 3.3`, so the user can indicate whether the concerned dataset is imbalanced or not. As long as the user does not provide any information in this regard, the automatic detection algorithm will be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm4 = ConfusionMatrix(y_actu, y_pred, is_imbalanced=True)\n",
    "cm4.imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm4 = ConfusionMatrix(y_actu, y_pred, is_imbalanced=False)\n",
    "cm4.imbalance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  also available in HTML report </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : The recommender system assumes that the input is the result of classification over the whole data rather than just a part of it. If the confusion matrix is the result of test data classification, the recommendation is not valid. </li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>is_imbalanced</code> , new in <span style=\"color:red;\">version 3.3 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In `version 2.0`, a method for comparing several confusion matrices is introduced. This option is a combination of several overall and class-based benchmarks. Each of the benchmarks evaluates the performance of the classification algorithm from good to poor and give them a numeric score. The score of good and poor performances are 1 and 0, respectively.\n",
    "\n",
    "After that, two scores are calculated for each confusion matrices, overall and class-based. The overall score is the average of the score of seven overall benchmarks which are Landis & Koch, Cramer, Matthews, Goodman-Kruskal's Lambda A, Goodman-Kruskal's Lambda B, Krippendorff's Alpha, and Pearson's C. In the same manner, the class-based score is the average of the score of six class-based benchmarks which are Positive Likelihood Ratio Interpretation, Negative Likelihood Ratio Interpretation, Discriminant Power Interpretation, AUC value Interpretation, Matthews Correlation Coefficient Interpretation and Yule's Q Interpretation. It should be noticed that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminated in total averaging. If the user sets weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average.\n",
    "\n",
    "If the user sets the value of `by_class` boolean input `True`, the best confusion matrix is the one with the maximum class-based score. Otherwise, if a confusion matrix obtains the maximum of both overall and class-based scores, that will be reported as the best confusion matrix, but in any other case, the compared object doesn’t select the best confusion matrix."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/compare_block_diagram.jpg\" onerror='this.src=\"../images/compare_block_diagram.jpg\"' width=\"500\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig3. Compare Block Diagram</p></center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is how the overall and class-based scores are determined for each confusion matrix. Note that here $|Set|$ shows the cardinality of the set and the cardinality of each benchmark is equal to the maximum possible score for that benchmark."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$S_{Overall}=\\sum_{i=1}^{|B_O|}\\frac{W_{OB}(i)}{\\sum W_{OB}}\\times\\frac{R_O(i)}{|B_O(i)|}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$S_{Class}=\\sum_{i=1}^{|B_C|}\\sum_{j=1}^{|C|}\\frac{W_{CB}(i)}{\\sum W_{CB}}\\times\\frac{W_C(j)}{\\sum W_{C}}\\times\\frac{R_C(i,j)}{|B_C(i)|}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\leq S_{Overall},S_{Class}\\leq1$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- $B_C$ : Class benchmarks\n",
    "- $B_O$ : Overall benchmarks\n",
    "- $C$ : Classes\n",
    "- $W_{CB}$ : Class benchmark weights\n",
    "- $W_{OB}$ : Overall benchmark weights\n",
    "- $W_{C}$ : Class weights\n",
    "- $R_C$ : Class benchmark result\n",
    "- $R_O$ : Overall benchmark result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2 = ConfusionMatrix(matrix={0: {0:2, 1:50, 2:6}, 1:{0:5, 1:50, 2:3}, 2:{0:1, 1:7, 2:50}})\n",
    "cm3 = ConfusionMatrix(matrix={0: {0:50, 1:2, 2:6}, 1:{0:50, 1:5, 2:3}, 2:{0:1, 1:55, 2:2}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp = Compare({\"cm2\":cm2, \"cm3\":cm3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm2    0.50278           0.58095\n",
      "2     cm3    0.33611           0.52857\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cm2': {'class': 0.50278, 'overall': 0.58095},\n",
       " 'cm3': {'class': 0.33611, 'overall': 0.52857}}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cm2', 'cm3']"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cm2'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.best_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp2 = Compare({\"cm2\":cm2, \"cm3\":cm3}, by_class=True, class_weight={0:5, 1:1, 2:1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm3\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm3    0.45357           0.52857\n",
      "2     cm2    0.34881           0.58095\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp3 = Compare({\"cm2\":cm2, \"cm3\":cm3}, class_benchmark_weight={\"PLRI\":0, \"NLRI\":0, \"DPI\":0, \"AUCI\":1, \"MCCI\":0, \"QI\":0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm2    0.46667           0.58095\n",
      "2     cm3    0.33333           0.52857\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp4 = Compare(\n",
    "    {\"cm2\":cm2, \"cm3\":cm3},\n",
    "    overall_benchmark_weight={\"SOA1\":1, \"SOA2\":0, \"SOA3\":0, \"SOA4\":0, \"SOA5\":0, \"SOA6\":1, \"SOA7\":0, \"SOA8\":0, \"SOA9\":0, \"SOA10\":0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm2    0.50278           0.45\n",
      "2     cm3    0.33611           0.18333\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overall and class benchmark lists are available in `CLASS_BENCHMARK_LIST` and `OVERALL_BENCHMARK_LIST`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AUCI', 'DPI', 'MCCI', 'NLRI', 'PLRI', 'QI']\n",
      "['SOA1', 'SOA10', 'SOA2', 'SOA3', 'SOA4', 'SOA5', 'SOA6', 'SOA7', 'SOA8', 'SOA9']\n"
     ]
    }
   ],
   "source": [
    "from pycm import CLASS_BENCHMARK_LIST, OVERALL_BENCHMARK_LIST\n",
    "print(CLASS_BENCHMARK_LIST)\n",
    "print(OVERALL_BENCHMARK_LIST)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>overall_benchmark_weight</code> and <code>class_benchmark_weight</code>, new in <span style=\"color:red;\">version 3.3 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  From <span style=\"color:red;\">version 3.8</span>, Goodman-Kruskal's Lambda A, Goodman-Kruskal's Lambda B, Krippendorff's Alpha, and Pearson's C benchmarks are considered in the overall score and default weights of the overall benchmarks are modified accordingly. </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`ROCCurve`, added in `version 3.7`, is devised to compute the Receiver Operating Characteristic (ROC) or simply ROC curve. In ROC curves, the Y axis represents the True Positive Rate, and the X axis represents the False Positive Rate. Thus, the ideal point is located at the top left of the curve, and a larger area under the curve represents better performance. ROC curve is a graphical representation of binary classifiers' performance. In PyCM, `ROCCurve` binarizes the output based on the \"One vs. Rest\" strategy to provide an extension of ROC for multi-class classifiers. By getting the actual labels vector and the target probability estimates of the positive classes, this method is able to compute and plot TPR-FPR pairs for different discrimination thresholds and compute the area under the ROC curve. \n",
    "The thresholds for which the TPR-FPR pairs are calculated can be either specified by users (by setting `thresholds` input) or calculated automatically. Furthermore, sample weights can be adjusted via `sample_weight` as an input; otherwise, they are assumed to be 1. `ROCCurve` has two methods named `area()` and `plot()`. `area()` provides the user with the value of area under curve, which can be calculated using either `trapezoidal` (default method) or `midpoint` numerical integral technique. `plot()` is also provided to plot the given curve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pycm import ROCCurve\n",
    "crv = ROCCurve(\n",
    "    actual_vector=numpy.array([1, 1, 2, 2]),\n",
    "    probs=numpy.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]),\n",
    "    classes=[2, 1])\n",
    "crv.thresholds\n",
    "auc_trp = crv.area()\n",
    "auc_trp[1]\n",
    "auc_trp[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1b96d6b0>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEjCAYAAADdZh27AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt0nXWd7/H3t21ubdKmbZI2zaUpbaE3FKSFERgsS2UKjKDgKKgLmAV2nfGgeDk6lTnHC3POwoPjKB45MozjAjkKo7jQynRERUCmA9KyQKWFYoFkZ6dp7k3a5rZ38j1/7J3tbpqkaZtnXz+vtbLWfp7nl/18n9w+eW7fx9wdERERgFnpLkBERDKHQkFERBIUCiIikqBQEBGRBIWCiIgkKBRERCRBoSAiIgkKBclKZtZoZgNmdsTMDprZ/WZWOm7MhWb2azM7bGa9ZvYzM1s3bsx8M/uGmYXi77U/Pl0xyXrNzD5hZi+b2VEzC5vZj8zs7CC3VyRVFAqSzd7j7qXAOcC5wOfHFpjZ24FfAD8FlgErgN8BO83sjPiYQuAJYD2wBZgPXAh0AedPss67gduATwCLgDOBnwBXnmzxZjbnZD9HJGimO5olG5lZI3CLu/8qPn0XsN7dr4xPPwP8wd0/Nu7z/h3ocPcbzOwW4H8BK939yDTWuRp4FXi7uz8/yZingP/n7t+JT98Ur/Pi+LQDtwKfBOYAjwNH3P2/Jb3HT4Gn3f0fzWwZ8H+AS4AjwNfd/ZvT+BKJnBLtKUjWM7Na4HJgf3x6LrH/+H80wfAfAu+Ov34X8PPpBELcO4HwZIFwEt4LXACsA34AfNDMDMDMFgKXAQ+b2SzgZ8T2cGri6/+kmf3Faa5fZFIKBclmPzGzw0Az0A58MT5/EbGf7dYJPqcVGDtfsHiSMZM52fGTudPdu919AHgGcODP48veDzzr7geATUClu9/h7sPu/gbwz8B1M1CDyIQUCpLN3uvuZcBmYA1/+mPfA4wC1RN8TjXQGX/dNcmYyZzs+Mk0j73w2PHbh4Hr47M+BHw//no5sMzMDo19ALcDS2agBpEJKRQk67n708D9wD/Ep48CzwJ/NcHwDxA7uQzwK+AvzGzeNFf1BFBrZhunGHMUmJs0vXSiksdNPwS838yWEzus9OP4/GbgTXcvT/ooc/crplmvyElTKEiu+AbwbjM7Jz69DbgxfvlomZktNLP/Cbwd+HJ8zIPE/vD+2MzWmNksM1tsZreb2XF/eN39j8D/BR4ys81mVmhmxWZ2nZltiw97CbjGzOaa2Srg5hMV7u4vAh3Ad4DH3f1QfNHzQJ+Z/a2ZlZjZbDPbYGabTuULJDIdCgXJCe7eAXwP+B/x6f8A/gK4hth5gCZil61eHP/jjrsPETvZ/CrwS6CP2B/iCuC3k6zqE8C3gHuAQ8DrwPuInRAG+DowDLQBD/CnQ0En8lC8lh8kbdMI8B5il9y+Seyw13eABdN8T5GTpktSRUQkQXsKIiKSoFAQEZEEhYKIiCQoFEREJEGhICIiCQoFERFJUCiIiEiCQkFERBIUCiIikqBQEBGRBIWCiIgkKBRERCRBoSAiIgkKBRERSVAoiIhIgkJBREQSFAoiIpIwJ90FnKyKigpvaGhIdxkiIlnlhRde6HT3yhONy7pQaGhoYPfu3ekuQ0Qkq5hZ03TG6fCRiIgkKBRERCRBoSAiIglZd05hIpFIhHA4zODgYLpLkWkqLi6mtraWgoKCdJciIklyIhTC4TBlZWU0NDRgZukuR07A3enq6iIcDrNixYp0lyMiSQI7fGRm3zWzdjN7eZLlZmbfNLP9ZvZ7M3vbqa5rcHCQxYsXKxCyhJmxePFi7dmJZKAgzyncD2yZYvnlwOr4x1bg26ezMgVCdtH3SyQzBRYK7v4boHuKIVcD3/OY54ByM6sOqh4RkWw1OjrK8PBwStaVznMKNUBz0nQ4Pq91/EAz20psb4L6+vqUFHeympubueGGGzh48CCzZs1i69at3HbbbQB88pOf5JprruGSSy5Ja40vvPACN910EwMDA1xxxRXcfffdx/3H/tWvfpXvf//7AESjUV555RU6OjpYtGgRDQ0NlJWVMXv2bObMmZO4ibC7u5sPfvCDNDY20tDQwA9/+EMWLlzIY489xq5du/jyl7+c8m1Nh7d++Rf0DkTSXYbkmNGhfqJ97ZjNpqCijsavXBnsCt09sA+gAXh5kmX/BlycNP0EcN6J3vO8887z8fbu3XvcvFQ7cOCAv/DCC+7u3tfX56tXr/Y9e/Z4V1eXX3DBBSf1XpFIJIgSfdOmTf6f//mfPjo66lu2bPEdO3ZMOX779u1+6aWXJqaXL1/uHR0dx4377Gc/63feeae7u995553+uc99zt3dR0dH/ZxzzvGjR49O+P6Z8H2bScv/9rF0lyA5JBqNemNjo+/evdv/8Ic/eF9f32m9H7Dbp/F3O533KYSBuqTpWuBAmmo5bdXV1bztbbFz5WVlZaxdu5aWlhYeeeQRtmz506mVO+64g02bNrFhwwa2bt06Fohs3ryZ22+/nXe84x3cfffddHR0cO2117Jp0yY2bdrEzp07AXj++ee58MILOffcc7nwwgvZt2/ftOprbW2lr6+Pt7/97ZgZN9xwAz/5yU+m/JyHHnqI66+//oTv/dOf/pQbb7wRgBtvvDHxvmbG5s2beeyxx6ZVo4jEuDuvvvoqnZ2dLF26lHXr1lFWVpaSdaczFLYDN8SvQvozoNfdjzt0lI0aGxt58cUXueCCC9i5cyfnnXdeYtmtt97Krl27ePnllxkYGDjmD+ahQ4d4+umn+cxnPsNtt93Gpz71KXbt2sWPf/xjbrnlFgDWrFnDb37zG1588UXuuOMObr/9dgD27dvHOeecM+HHoUOHaGlpoba2NrGu2tpaWlpaJt2G/v5+fv7zn3Pttdcm5pkZl112Geeddx733XdfYn5bWxvV1bHTQdXV1bS3tyeWbdy4kWeeeeZUv5QieSUajQKx37WamhrWrl1LTU0Ns2al7k91YOcUzOwhYDNQYWZh4ItAAYC73wvsAK4A9gP9wF/P1Lobtv3bTL1VwnSP4x05coRrr72Wb3zjG8yfP5/W1lYqK//UmPDJJ5/krrvuor+/n+7ubtavX8973vMeAD74wQ8mxv3qV79i7969iem+vj4OHz5Mb28vN954I3/84x8xMyKR2DHss846i5deemnSusb2SJJNdQXQz372My666CIWLVqUmLdz506WLVtGe3s77373u1mzZs0Jz5NUVVVx4EDW7gCKpEx3dzfNzc3U1NRQUVFBeXl5WuoILBTcfcrjDvFjXP81iHUHfiJmEpFIhGuvvZYPf/jDXHPNNQCUlJQkrscfHBzkYx/7GLt376auro4vfelLx1yrP2/evMTr0dFRnn32WUpKSo5Zx8c//nEuvfRSHn30URobG9m8eTMQ21NIDpVkTz31FLW1tYTD4cS8cDjMsmXLJt2Whx9++LhDR2Pjq6qqeN/73sfzzz/PJZdcwpIlS2htbaW6uprW1laqqqoSnzM4OHjcNojInwwPDxMKhejt7WXevHmUlpamtR71Ppoh7s7NN9/M2rVr+fSnP52Yv3btWvbv3w+QCICKigqOHDnCI488Mun7XXbZZXzrW99KTI/tBfT29lJTUwPA/fffn1g+tqcw0Ud5eTnV1dWUlZXx3HPP4e5873vf4+qrr55w3b29vTz99NPHLD969CiHDx9OvP7FL37Bhg0bALjqqqt44IEHAHjggQeO+bzXXnstMU5EjtXd3c3evXs5fPgwdXV1nHXWWRQXF6e1JoXCDNm5cycPPvggv/71rxPH8nfs2MGVV17JU089BUB5eTkf/ehHOfvss3nve9/Lpk2bJn2/b37zm+zevZu3vOUtrFu3jnvvvReAz33uc3z+85/noosuYmRk5KRq/Pa3v80tt9zCqlWrWLlyJZdffjkA9957b+L9AR599FEuu+yyY/Zc2trauPjii3nrW9/K+eefz5VXXpk4gb5t2zZ++ctfsnr1an75y1+ybdu2xOc9+eSTXHllevbcRDLdnDlzmDdvHuvWraOqqiojbuq0iY41Z7KNGzf6+IfsvPLKK6xduzZNFZ3YxRdfzGOPPZa2Y4Tp0tbWxoc+9CGeeOKJCZdn+vftZDVs+7e0HbqU7ODutLe3Mzo6mrg4I1XM7AV333iicTnREC/Tfe1rXyMUCuVdKIRCIb72ta+luwyRjDAwMEBjYyP9/f0sXLgw3eVMSqGQAhdccEG6S0iLqQ6PieQLd6e1tZWDBw8ye/ZszjjjDIVCKrh7RhyPk+nJtsOWIqdqcHCQgwcPsmjRImpra5kzJ7P/7ObEiebi4mK6urr0hyZLePx5Cum+ykIkKKOjo3R3x/qBlpSUsH79ehoaGjI+ECBH9hTGrsHv6OhIdykyTWNPXhPJNX19fTQ1NTE8PMzcuXMpLi6mqKgo3WVNW06EQkFBgZ7gJSJpNTIyQjgcprOzk+Li4oy45+BU5EQoiIik01gDu6GhIZYuXUp1dXVK+xXNJIWCiMgpikajzJkzJ9HArrCwkLlz56a7rNOSnVEmIpJmXV1dvPzyy3R2dgKxjgXZHgigPQURkZMyPDxMU1MTfX19lJaWpr2B3UxTKIiITFN3dzdNTU0A1NXVHdMROFcoFEREpmnOnDmUlpayfPlyCgsL011OIBQKIiKTcHfa2tpwd6qrq5k/fz7z589Pd1mBUiiIiEygv7+fpqYm+vv7j3kCYa5TKIiIJBkdHaW1tZW2tjbmzJnDypUr86rDsUJBRCTJ0NAQbW1tLFq0iLq6OmbPnp3uklJKoSAieW90dJSenh4WL16caGCXTf2KZpJCQUTyWnIDu3nz5mVdA7uZplAQkbwUjUYJh8OJNu7Z2sBupikURCTvuDv79u1jaGiI6upqli5dmrUN7GaaQkFE8kYuNrCbaYpGEckLYw3sxh7GlSsN7Gaa9hREJKcNDQ0RCoUSDezKysrSXVJGUyiISM7q6uoiFAphZtTX11NZWZnukjKeQkFEclZBQQFlZWXU19fnbAO7maZQEJGc4e4cPHgQIG8a2M00hYKI5IT+/n4aGxsZGBjIqwZ2My3Qq4/MbIuZ7TOz/Wa2bYLl9Wb2pJm9aGa/N7MrgqxHRHLP6OgoLS0tvPrqq0SjUVauXMmKFSvSXVbWCmxPwcxmA/cA7wbCwC4z2+7ue5OG/Xfgh+7+bTNbB+wAGoKqSURyz1gDu8WLF1NbW5t3DexmWpB7CucD+939DXcfBh4Grh43xoGxA34LgAMB1iMiOWJkZISuri4ASkpK2LBhA8uXL1cgzIAgzynUAM1J02HggnFjvgT8wsw+DswD3hVgPSKSA3p7ewmFQsc0sNOVRTMnyD0Fm2Cej5u+Hrjf3WuBK4AHzey4msxsq5ntNrPdY3cjikh+iUajvPnmm+zfv59Zs2axZs0aNbALQJB7CmGgLmm6luMPD90MbAFw92fNrBioANqTB7n7fcB9ABs3bhwfLCKS48Y3sKuursZsov875XQFGQq7gNVmtgJoAa4DPjRuTAh4J3C/ma0FigHtCogIAJFIhIKCAsyM2tpaCgsLKSkpSXdZOS2ww0fuHgVuBR4HXiF2ldEeM7vDzK6KD/sM8FEz+x3wEHCTu2tPQETo7Oxkz549iQZ2CxYsUCCkQKA3r7n7DmKXmSbP+0LS673ARUHWICLZZWhoiKamJg4fPkxZWZnuSE4x3dEsIhlDDezST6EgIhlDDezST6EgImkz1sDO3Vm2bJka2GUAhYKIpMXRo0dpampiYGCAxYsXp7sciVMoiEhKjY6OcuDAAdra2igoKGDVqlUsWLAg3WVJnEJBRFJqaGiI9vZ2KisrqampUb+iDKNQEJHAjYyMcOjQIRYvXpxoYKcTyZlJoSAigert7aWpqYlIJKIGdllAoSAigYhGozQ3N9Pd3U1JSQkrV65UA7ssoFAQkRnn7rz66qsMDw+zbNkyli5dqgZ2WUKhICIzJrmBXV1dnRrYZaFAn9EsIvlDDexyg/YUROS0qIFdblEoiMgpS25gt3z5cioqKtJdkpwmhYKInLLCwkLmz59PfX09BQUF6S5HZoBCQUSmzd1pbW0FYNmyZZSVlVFWVpbmqmQmKRREZFrUwC4/KBREZErJDewKCwvVwC7HKRREZErDw8NqYJdHFAoicpyRkRF6enqoqKiguLhYDezyiEJBRI5x6NAhQqEQ0WiU0tJSNbDLMwoFEQFiDexCoRA9PT2UlJSwatUqNbDLQwoFEVEDO0lQKIjksfEN7IqKirR3kOfUEE8kT3V0dBzXwE6BINpTEMkzg4ODNDU1ceTIEebPn68GdnIMhYJIHuns7KS5uRkzo6GhQXcmy3EUCiJ5pKioSA3sZEoKBZEcpgZ2crIUCiI56siRIzQ1NTE4OKjnHMi0KRREcszo6CgtLS20t7dTWFjI6tWrdTJZpi3QS1LNbIuZ7TOz/Wa2bZIxHzCzvWa2x8x+EGQ9IvlgeHiYjo4OqqqqWLdunQJBTkpgewpmNhu4B3g3EAZ2mdl2d9+bNGY18HngInfvMbOqoOoRyWXjG9idffbZOpEspyTIw0fnA/vd/Q0AM3sYuBrYmzTmo8A97t4D4O7tAdYjkpMmamCnQJBTFeThoxqgOWk6HJ+X7EzgTDPbaWbPmdmWid7IzLaa2W4z2z1296VIvotEIrzxxhu8/vrrFBQUsGbNGt2RLKctyD2Fibpp+QTrXw1sBmqBZ8xsg7sfOuaT3O8D7gPYuHHj+PcQyTvuzr59+xgeHqampoYlS5aogZ3MiCBDIQzUJU3XAgcmGPOcu0eAN81sH7GQ2BVgXSJZa3h4mMLCQjWwk8AEefhoF7DazFaYWSFwHbB93JifAJcCmFkFscNJbwRYk0jWam9vVwM7CVxgewruHjWzW4HHgdnAd919j5ndAex29+3xZZeZ2V5gBPisu3cFVZNINhrfwG7BggXpLklyWKA3r7n7DmDHuHlfSHrtwKfjHyIyTmdnJ6FQiFmzZqmBnaSE7mgWyWBFRUWUl5dTV1eny0wlJRQKIhlkdHQ00cCupqZGDewk5RQKIhlCDewkEygURNJsZGSEAwcOqIGdZASFgkiaRSIROjs7qaqqoqamhlmz9Oh0SR+FgkgaRKNRenp6qKyspLi4mA0bNuhEsmSEk/6XxMxmm9mHgyhGJB/09PSwZ88empubGRwcBFAgSMaYNBTMbL6Zfd7MvmVml1nMx4ndcfyB1JUokhsikQivv/46b7zxBoWFhaxdu1Z3JEvGmerw0YNAD/AscAvwWaAQuNrdX0pBbSI5Y6yBXSQSUQM7yWhThcIZ7n42gJl9B+gE6t39cEoqE8kByQ3s6uvrKSws1N6BZLSpzilExl64+wjwpgJBZHrc/bgGdvPnz1cgSMabak/hrWbWx5+ei1CSNO3urgupRSYwODhIY2MjR48eZcGCBWpgJ1ll0lBw99mpLEQkF3R0dNDc3Mzs2bNZsWIFixYtSndJIidl0lAws2LgvwCrgN8Ta30dTVVhItmouLiY8vJy6uvrmTNHtwFJ9pnqp/YBYucVngGuANYDt6WiqKC89cu/oHcgcuKBItPkPsrIkW4AFldVq4GdZL2pQmFd0tVH/wI8n5qSgtM7EKHxK1emuwzJEUeOHKGxsZGhoWVUVlZSX1+f7pJETttUoZB89VFU11SLxIyMjNDS0kJHRwdFRUWceeaZ2juQnDFVKJwTv9oIYlcc6eojEWJ3Jnd1dbFkyRKWLVumBnaSU6YKhd+5+7kpq0Qkg6mBneSLqULBU1aFSAbr6ekhFAoxMjJCWVkZxcXFCgTJWVOFQpWZfXqyhe7+jwHUI5IxIpEIoVCIQ4cOMXfuXBoaGnRHsuS8qUJhNlDKn+5oFskbyQ3samtrqaqqUgM7yQtThUKru9+RskpEMsDw8DAFBQWJBnZFRUUUFRWluyyRlJnqsgn9WyR5w91pa2s7roGdAkHyzVR7Cu9MWRUiaTQwMEBTU1OigV15eXm6SxJJm6ka4nWnshCRdFADO5FjqWOX5LXi4mIWLlxIXV2dGtiJoFCQPDM6OsqBAwcwM2pqatTATmQchYLkjcOHD9PU1MTQ0BCVlZXpLkckIykUJOeNjIwQDofp7OxUAzuREwi0k5eZbTGzfWa238y2TTHu/WbmZrYxyHokP0UiEbq7u1myZAnr1q1TIIhMIbBQMLPZwD3A5cA64HozWzfBuDLgE8Bvg6pF8k80GqW9vR2InUw+++yzqa2tVUdTkRMI8jfkfGC/u7/h7sPAw8DVE4z7e+AuYDDAWiSPdHd3s2fPHsLhMIODsR8rXVkkMj1BhkIN0Jw0HY7PSzCzc4E6d38swDokTwwPD7N//37efPNNioqKWLt2rRrYiZykIP99mqhNRqIdt5nNAr4O3HTCNzLbCmwF9MhDmZC789prrxGJRKirq6OyslIN7EROQZChEAbqkqZrgQNJ02XABuCp+C/vUmC7mV3l7ruT38jd7wPuA9i4caOe8yAJamAnMrOCPHy0C1htZivMrBC4Dtg+ttDde929wt0b3L0BeA44LhBEJqIGdiLBCGxPwd2jZnYr8DixZzN81933mNkdwG533z71O4hMbGBggMbGRvr7+ykvL2fhwoXpLkkkZwR6SYa77wB2jJv3hUnGbg6yFskNyQ3szjjjDAWCyAzTdXqSVUpKStTATiRA+q2SjDY6OkpLSwtmRm1tLaWlpZSWlqa7LJGcpVCQjJXcwK6qqird5YjkBYWCZJzxDezOOuss7R2IpIhCQTLOWAO7pUuXUl1drX5FIimkUJCMEIlE6OnpoaqqKtHATieSRVJPv3WSdt3d3TQ3NzMyMsKCBQsoKipSIIikiX7zJG2Gh4cJhUL09vYyb948GhoadEeySJopFCQt1MBOJDMpFCSlhoaGKCwsxMxYvnw5RUVFFBYWprssEYnTZR2SEu7OwYMHj2lgV1ZWpkAQyTDaU5DA9ff309TUpAZ2IllAoSCBam9vJxwOM2fOHDWwE8kCCgUJ1Ny5c1m0aBG1tbW6zFQkC+i3VGaUGtiJZDeFgsyYvr4+mpqaGB4eVgM7kSylUJDTNjIyQnNzM11dXRQXF6uBnUgWUyjIaRvrW6QGdiLZT6Egp2Ssk+mSJUvUwE4kh+i3WE5aV1cXzc3NjI6OUl5ergZ2IjlEv8kybcPDwzQ1NdHX10dpaWmiTYWI5A6FgkyLu7Nv3z6i0Sh1dXW6ukgkRykUZErJDezGWlurX5FI7tJlIjIhNbATyU/aU5DjJDewW7hwofoVieQRhYIcI7mB3cqVKykvL093SSKSQgoFOcZYA7u6ujpmz56d7nJEJMUUCnluZGSElpYWZs2apQZ2IqJQyGfJDeyWLFmS7nJEJAMoFPJQNBolHA4nGtitWbOGefPmpbssEckACoU8FI1GOXToENXV1VRXV2Nm6S5JRDJEoPcpmNkWM9tnZvvNbNsEyz9tZnvN7Pdm9oSZLQ+ynnwWiURoa2sDSDSwW7ZsmQJBRI4RWCiY2WzgHuByYB1wvZmtGzfsRWCju78FeAS4K6h68llnZyd79uzhwIEDDA0NAejKIhGZUJCHj84H9rv7GwBm9jBwNbB3bIC7P5k0/jngIwHWk3eGhoYIhUJqYCci0xZkKNQAzUnTYeCCKcbfDPz7RAvMbCuwFaC+vn6m6stp7s5rr73GyMgI9fX1VFZWprskEckCQYbCRAerfcKBZh8BNgLvmGi5u98H3AewcePGCd9DYtTATkROR5ChEAbqkqZrgQPjB5nZu4C/A97h7kMB1pPTxhrYtba2UltbS1VVFWVlZekuS0SyTJChsAtYbWYrgBbgOuBDyQPM7Fzgn4At7t4eYC05rb+/n8bGRgYGBli4cCGLFi1Kd0kikqUCCwV3j5rZrcDjwGzgu+6+x8zuAHa7+3bgq0Ap8KP4pZEhd78qqJpyUXt7O83NzRQUFKiBnYictkBvXnP3HcCOcfO+kPT6XUGuPx/MnTuXiooKamtrdZmpiJw23dGcZcYa2JkZdXV1amAnIjNKoZBFent7CYVCamAnIoFRKGSBaDRKc3Mz3d3damAnIoFSKGSBkZERent71cBORAKnUMhQkUiErq4uli5dSlFREWeffbZOJItI4BQKGaizs5NwOIy7s3DhQoqKihQIIpISCoUMMjQ0RFNTE4cPH6asrEwN7EQk5RQKGSK5gd3y5cupqKhId0kikocUCmk2ODhIUVGRGtiJSEYI9MlrMjl3p7W1lb1799LR0QFAWVmZAkFE0kp7Cmlw9OhRmpqaGBgYYNGiRWpgJyIZQ6GQYskN7FatWsWCBQvSXZKISIJCIcXmzp1LZWUlNTU1usxURDKOQiFgIyMjhMNhZs2apQZ2IpLxFAoB6u3tpampiUgkwtKlS9NdjojICSkUApDcwK6kpISVK1eqgZ2IZAWFQgDGGtgtW7aMpUuXqoGdiGQNhcIMGR4epru7Ww3sRCSrKRRmQEdHBy0tLWpgJyJZT6FwGtTATkRyjULhFKmBnYjkIoXCSUpuYLdixQqKioooKChId1kiIjNCDfGmyd05cODAMQ3sSktLFQgiklO0pzANR48epbGxkcHBQRYvXqwGdiKSsxQKJ9DW1kY4HKawsFAN7EQk5ykUTmDevHlqYCcieUOhMI4a2IlIPlMoJDl06BChUIhoNMqSJUvSXY6ISMopFIg1sAuFQvT09DB37lxWrVrF3Llz012WiEjKKRSIHTLq6+ujpqaGJUuWqIGdiOStvA2F4eFhurq6qK6uVgM7EZG4QG9eM7MtZrbPzPab2bYJlheZ2b/Gl//WzBqCrGdMR0cHe/bs4eDBgwwNDQEoEERECDAUzGw2cA9wObAOuN7M1o0bdjPQ4+6rgK8D/zuoegA8Osy+ffsIhUKUlpayfv16NbATEUkS5J7C+cB+d3/D3YeBh4Grx425Gngg/voR4J0W0AF9dyfSc4CBgQEaGhpYvXo1hYWFQaxKRCRrBRkKNUBz0nQ4Pm/CMe4eBXqBxePfyMy2mtluM9s91nfoZJkZcxYsYf369Sxhp7xdAAAFQUlEQVRefNwqRESEYENhov/4/RTG4O73uftGd99YWVl5ygWF/vH9amAnIjKFIEMhDNQlTdcCByYbY2ZzgAVAd4A1iYjIFIIMhV3AajNbYWaFwHXA9nFjtgM3xl+/H/i1ux+3pyAiIqkR2H0K7h41s1uBx4HZwHfdfY+Z3QHsdvftwL8AD5rZfmJ7CNcFVY+IiJxYoDevufsOYMe4eV9Iej0I/FWQNYiIyPTpyWsiIpKgUBARkQSFgoiIJCgUREQkwbLtClAz6wCaTvHTK4DOGSwnG2ib84O2OT+czjYvd/cT3v2bdaFwOsxst7tvTHcdqaRtzg/a5vyQim3W4SMREUlQKIiISEK+hcJ96S4gDbTN+UHbnB8C3+a8OqcgIiJTy7c9BRERmUJOhkKmPhs6SNPY5k+b2V4z+72ZPWFmy9NR50w60TYnjXu/mbmZZf2VKtPZZjP7QPx7vcfMfpDqGmfaNH62683sSTN7Mf7zfUU66pwpZvZdM2s3s5cnWW5m9s341+P3Zva2GS3A3XPqg1hH1teBM4BC4HfAunFjPgbcG399HfCv6a47Bdt8KTA3/vpv8mGb4+PKgN8AzwEb0113Cr7Pq4EXgYXx6ap0152Cbb4P+Jv463VAY7rrPs1tvgR4G/DyJMuvAP6d2EPK/gz47UyuPxf3FDLq2dApcsJtdvcn3b0/PvkcsYceZbPpfJ8B/h64CxhMZXEBmc42fxS4x917ANy9PcU1zrTpbLMD8+OvF3D8w7yyirv/hqkfNnY18D2PeQ4oN7PqmVp/LobCjD0bOotMZ5uT3UzsP41sdsJtNrNzgTp3fyyVhQVoOt/nM4EzzWynmT1nZltSVl0wprPNXwI+YmZhYq36P56a0tLmZH/fT0qgz1NIkxl7NnQWmfb2mNlHgI3AOwKtKHhTbrOZzQK+DtyUqoJSYDrf5znEDiFtJrY3+IyZbXD3QwHXFpTpbPP1wP3u/jUzezuxB3dtcPfR4MtLi0D/fuXinkI+Pht6OtuMmb0L+DvgKncfSlFtQTnRNpcBG4CnzKyR2LHX7Vl+snm6P9s/dfeIu78J7CMWEtlqOtt8M/BDAHd/Figm1iMoV03r9/1U5WIo5OOzoU+4zfFDKf9ELBCy/TgznGCb3b3X3SvcvcHdG4idR7nK3Xenp9wZMZ2f7Z8Qu6gAM6sgdjjpjZRWObOms80h4J0AZraWWCh0pLTK1NoO3BC/CunPgF53b52pN8+5w0eeh8+GnuY2fxUoBX4UP6cecver0lb0aZrmNueUaW7z48BlZrYXGAE+6+5d6av69Exzmz8D/LOZfYrYYZSbsvmfPDN7iNjhv4r4eZIvAgUA7n4vsfMmVwD7gX7gr2d0/Vn8tRMRkRmWi4ePRETkFCkUREQkQaEgIiIJCgUREUlQKIiISIJCQWSazGzEzF5K+mgws81m1hvv0PmKmX0xPjZ5/qtm9g/prl9kOnLuPgWRAA24+znJM+Jt159x9780s3nAS2Y21mtpbH4J8KKZPeruO1NbssjJ0Z6CyAxx96PAC8DKcfMHgJeYwaZlIkFRKIhMX0nSoaNHxy80s8XEeiztGTd/IbH+Q79JTZkip06Hj0Sm77jDR3F/bmYvAqPAV+JtGDbH5/8eOCs+/2AKaxU5JQoFkdP3jLv/5WTzzexM4D/i5xReSnVxIidDh49EAuburwF3An+b7lpETkShIJIa9wKXmNmKdBciMhV1SRURkQTtKYiISIJCQUREEhQKIiKSoFAQEZEEhYKIiCQoFEREJEGhICIiCQoFERFJ+P8LjK4I8/KaBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "crv.plot(area=True, classes=[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision-Recall curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`PRCurve`, added in `version 3.7`, is devised to compute the Precision-Recall curve in which the Y axis represents the Precision, and the X axis represents the Recall of a classifier. Thus, the ideal point is located at the top right of the curve, and a larger area under the curve represents better performance. Precision-Recall curve is a graphical representation of binary classifiers' performance. In PyCM, `PRCurve` binarizes the output based on the \"One vs. Rest\" strategy to provide an extension of this curve for multi-class classifiers. By getting the actual labels vector and the target probability estimates of the positive classes, this method is able to compute and plot Precision-Recall pairs for different discrimination thresholds and compute the area under the curve. \n",
    "The thresholds for which the Precision-Recall pairs are calculated can be either specified by users (by setting `thresholds` input) or calculated automatically. Furthermore, sample weights can be adjusted via `sample_weight` as an input; otherwise, they are assumed to be 1. `PRCurve` has two methods named `area()` and `plot()`. `area()` provides the user with the value of area under curve, which can be calculated using either `trapezoidal` (default method) or `midpoint` numerical integral technique. `plot()` is also provided to plot the given curve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-4.2-py3.5.egg\\pycm\\curve.py:382: RuntimeWarning: The curve contains non-numerical value(s).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.29166666666666663"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pycm import PRCurve\n",
    "crv = PRCurve(\n",
    "    actual_vector=numpy.array([1, 1, 2, 2]),\n",
    "    probs=numpy.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]),\n",
    "    classes=[2, 1])\n",
    "crv.thresholds\n",
    "auc_trp = crv.area()\n",
    "auc_trp[1]\n",
    "auc_trp[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1b9a5e50>"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEjCAYAAADdZh27AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl0nPV97/H319ZuyZJtSbZWy3jBO3YiAiFpgCamQMoSSBtIchN6ADehhDRJs5Em4bo3t0la2sAJJ9S5UAgngbY0SX24DulNwhJoXDCxcWyDF0AzGnnRLi/ape/9Y6TJWJYt2daj0cx8XufonJl5npn5PpJGH/2e5fszd0dERARgWqILEBGRqUOhICIiMQoFERGJUSiIiEiMQkFERGIUCiIiEqNQEBGRGIWCpBQzqzOzLjM7ZmaHzeyfzSx/aNmzZtY9tKzZzH5sZmVjvN47zGyzmbWbWauZvWRmfzY5WyMy+RQKkoqucfd84G3AhcBfxy27c2jZIiAf+PtTvYiZvRP4FfDc0PpzgE8CV51NUWY2/WyeJzKZFAqSsty9AfgZsHKUZe3AT4E1p3mJvwMedfdvuXuzR73i7n8KYGa3mNkL8U8wMzezRUO3HzGz7w2NNI4DXzazQ/HhYGYfMLMdQ7enmdmXzOwNM2sxs381s9nn+G0QOSMKBUlZZlYFXA1sG2XZHOAGYP8pnpsHvBN48hzL+DDwDaCA6KjkOPCHI5b/aOj2XcD1wKVAOdAGPHCO7y9yRhQKkop+ambtwAtEd/3877hl95tZB9AMFAOfOsVrzCL6+Th4jrX8h7u/6O6D7t4NPA7cDGBmBURD6/Ghdf8c+Iq7R9y9B7gH+KCZZZxjDSLjplCQVHS9uxe5+3x3v8Pdu+KW3eXuhcBqon/4K0/xGm3AIHDaA9HjUD/i/o+AG8wsm+hI5bfuHhpaNh/4ydBB7XbgNWAAmHuONYiMm0JB0pK7/w74X8ADZmajLO8EfgPceJqXOQ7kDd8xs3mjvdWI190NhIgerI7fdQTRALlqKNCGv3KGjo2ITAqFgqSzR4FS4NpTLP8CcIuZfX7oGARmdoGZPTG0/FVghZmtMbMcort7xuNHRI8fvAf4t7jHHwS+YWbzh96rxMyuO5MNEjlXCgVJW+7eC9wPfPUUy/+L6EHhPwTeNLNWYCOweWj5XmAD8AtgH9FjGOPxOHAZ8Ct3b457/D5gE/CfZnYU2AJcdGZbJXJuTJPsiIjIMI0UREQkRqEgIiIxCgUREYlRKIiISIxCQUREYhQKIiISo1AQEZEYhYKIiMQoFEREJEahICIiMQoFERGJUSiIiEiMQkFERGIUCiIiEqNQEBGRGIWCiIjEKBRERCQmI9EFnKni4mKvqalJdBkiIknllVdeaXb3krHWS7pQqKmpYevWrYkuQ0QkqZhZaDzrafeRiIjEKBRERCRGoSAiIjFJd0xhNH19fUQiEbq7uxNdioxTTk4OlZWVZGZmJroUEYmTEqEQiUQoKCigpqYGM0t0OTIGd6elpYVIJMKCBQsSXY6IxAls95GZPWxmjWa28xTLzczuN7P9ZrbDzN52tu/V3d3NnDlzFAhJwsyYM2eORnYiU1CQxxQeAa48zfKrgMVDX+uB753LmykQkot+XiJTU2Ch4O7PA62nWeU64AcetQUoMrOyoOoRCdLgoPNfbzQnugxJUYODg/T29k7KeyXy7KMKoD7ufmTosZOY2Xoz22pmW5uamialuDNVX1/P5ZdfzrJly1ixYgX33XdfbNlf/uVf8vzzzyewuqhXXnmFVatWsWjRIu666y7c/aR1fvjDH7J69WpWr17NJZdcwquvvhpbdt9997Fy5UpWrFjBd77zndjjn//851m6dCmrV6/mAx/4AO3t7QD87ne/45Zbbgl8u6aC/kHn4w+/lOgyJAUdPXqU3bt388Ybb0zK+yUyFEbbf3DyXynA3Te6e62715aUjHmVdkJkZGRw77338tprr7FlyxYeeOABdu/eTWtrK1u2bOE973nPuF+rv78/kBo/+clPsnHjRvbt28e+fft4+umnT1pnwYIFPPfcc+zYsYOvfvWrrF+/HoCdO3fy/e9/n5deeolXX32Vp556in379gGwbt06du7cyY4dO1iyZAl/+7d/C8CqVauIRCKEw+FAtkcklQ0MDBAKhdi7dy8AlZWVk/K+iQyFCFAVd78SOJCgWs5ZWVkZb3tb9Fh5QUEBy5Yto6GhgSeffJIrr/z9oZUNGzZw4YUXsnLlStavXx/7b/2yyy7j7rvv5tJLL+W+++6jqamJG2+8kQsvvJALL7yQF198EYCXXnqJSy65hLVr13LJJZewZ8+ecdV38OBBjhw5wjvf+U7MjI997GP89Kc/PWm9Sy65hFmzZgFw8cUXE4lEAHjttde4+OKLycvLIyMjg0svvZSf/OQnAFxxxRVkZGSc9ByAa665hieeeOKMvpci6c7def3112lubmbevHksX76cgoKCSXnvRIbCJuBjQ2chXQx0uPvBBNYzYerq6ti2bRsXXXQRL774Im9/+9tjy+68805efvlldu7cSVdXF0899VRsWXt7O8899xyf+9zn+PSnP81nPvMZXn75Zf793/+d2267DYClS5fy/PPPs23bNjZs2MDdd98NwJ49e1izZs2oX+3t7TQ0NJzwn0ZlZSUNDQ2n3Y6HHnqIq666CoCVK1fy/PPP09LSQmdnJ5s3b6a+vv6k5zz88MOx5wDU1tby61//+iy+iyLpZ3gvgZlRUVHBsmXLqKioYNq0yftTHdh1Cmb2OHAZUGxmEeDrQCaAuz8IbAauBvYDncCfTdR713zp/07US8XUffP941rv2LFj3HjjjXznO99h5syZHDx4kPhdXs888wzf/va36ezspLW1lRUrVnDNNdcA8KEPfSi23i9+8Qt2794du3/kyBGOHj1KR0cHH//4x9m3bx9mRl9fHwDnn38+27dvP2Vdox0/ON0ZQM888wwPPfQQL7zwAgDLli3ji1/8IuvWrSM/P58LLrggNjoY9o1vfIOMjAw+8pGPxB4rLS3lwIGkHQCKTJrW1lbq6+upqKiguLiYoqKihNQRWCi4+81jLHfgL4J47/H+AZ9ofX193HjjjXzkIx/hhhtuACA3Nzd2Pn53dzd33HEHW7dupaqqinvuueeEc/VnzJgRuz04OMhvfvMbcnNzT3iPT33qU1x++eX85Cc/oa6ujssuuwyIjhTiQyXes88+S2Vl5Qm7dSKRCOXl5aOuv2PHDm677TZ+9rOfMWfOnNjjt956K7feeisAd9999wkjj0cffZSnnnqKX/7ylyeETXd390nbICK/19vbSzgcpqOjgxkzZpCfn5/QetT7aIK4O7feeivLli3js5/9bOzxZcuWsX//foBYABQXF3Ps2DGefPLJU77eFVdcwXe/+93Y/eFRQEdHBxUV0ZO0Hnnkkdjy4ZHCaF9FRUWUlZVRUFDAli1bcHd+8IMfcN111530vuFwmBtuuIHHHnuMJUuWnLCssbExts6Pf/xjbr45mvtPP/003/rWt9i0aRN5eXknPGfv3r2sXLny9N88kTTV2trK7t27OXr0KFVVVZx//vnk5OQktCaFwgR58cUXeeyxx/jVr34V25e/efNm3v/+9/Pss88CUFRUxO23386qVau4/vrrufDCC0/5evfffz9bt25l9erVLF++nAcffBCAL3zhC3z5y1/mXe96FwMDA2dU4/e+9z1uu+02Fi1axMKFC2P7/h988MHY62/YsIGWlhbuuOMO1qxZQ21tbez5N954I8uXL+eaa67hgQceiB2QvvPOOzl69Cjr1q1jzZo1fOITn4g955lnnuH970/MyE1kqsvIyGDGjBksX76c0tLSKXFRp422r3kqq62t9ZGT7Lz22mssW7YsQRWN7d3vfjdPPfVUwvYRJkpPTw+XXnopL7zwwknHH2Dq/9zORG//ICu+/jT7vnF1okuRKczdaWxsZHBwkLKyyb1W18xecffasdZLiYZ4U929995LOBxOu1AIh8N885vfHDUQRNJNV1cXdXV1dHZ2xkbZU5E+rZPgoosuSnQJCbF48WIWL16c6DJEEsrdOXjwIIcOHWL69Omcd955CoXJ4O5TYn+cjE+y7bYUOVvd3d0cOnSI2bNnU1lZOeVHzilxoDknJ4eWlhb9oUkSw/MpJPosC5GgDA4O0toa7Qeam5vLihUrqKmpmfKBACkyUhg+B3+qNsuTkw3PvCaSao4cOUIoFKK3t5e8vDxycnLIzs5OdFnjlhKhkJmZqRm8RCShBgYGiEQiNDc3k5OTMyWuOTgbKREKIiKJNNzArqenh3nz5lFWVjap/YomkkJBROQs9ff3k5GREWtgl5WVddJV/ckmOaNMRCTBWlpa2LlzJ83N0Rn3ioqKkj4QQCMFEZEz0tvbSygU4siRI+Tn5ye8gd1EUyiIiIxTa2sroVAIgKqqKkpLSxNc0cRTKIiIjFNGRgb5+fnMnz+frKysRJcTCIWCiMgpuDuHDx/G3SkrK2PmzJnMnDkz0WUFSqEgIjKKzs5OQqEQnZ2dzJ49O9HlTBqFgohInMHBQQ4ePMjhw4fJyMhg4cKFadXhWKEgIhKnp6eHw4cPM3v2bKqqqpg+fXqiS5pUCgURSXuDg4O0tbUxZ86cWAO7ZOpXNJEUCiKS1uIb2M2YMSPpGthNNIWCiKSl/v5+IpFIrI17sjawm2gKBRFJO+7Onj176OnpoaysjHnz5iVtA7uJplAQkbSRig3sJpqiUUTSwnADu+HJuFKlgd1E00hBRFJaT08P4XA41sCuoKAg0SVNaQoFEUlZLS0thMNhzIzq6mpKSkoSXdKUp1AQkZSVmZlJQUEB1dXVKdvAbqIpFEQkZbg7hw4dAkibBnYTTaEgIimhs7OTuro6urq60qqB3UQL9OwjM7vSzPaY2X4z+9Ioy6vN7Bkz22ZmO8zs6iDrEZHUMzg4SENDA6+//jr9/f0sXLiQBQsWJLqspBXYSMHMpgMPAOuACPCymW1y991xq/018K/u/j0zWw5sBmqCqklEUs9wA7s5c+ZQWVmZdg3sJlqQI4V3APvd/U137wWeAK4bsY4Dwzv8CoEDAdYjIiliYGCAlpYWAHJzc1m5ciXz589XIEyAII8pVAD1cfcjwEUj1rkH+E8z+xQwA3hfgPWISAro6OggHA6f0MBOZxZNnCBHCjbKYz7i/s3AI+5eCVwNPGZmJ9VkZuvNbKuZbR2+GlFE0kt/fz9vvfUW+/fvZ9q0aSxdulQN7AIQ5EghAlTF3a/k5N1DtwJXArj7b8wsBygGGuNXcveNwEaA2trakcEiIiluZAO7srIyzEb7v1POVZCh8DKw2MwWAA3ATcCHR6wTBt4LPGJmy4AcQEMBEQGgr6+PzMxMzIzKykqysrLIzc1NdFkpLbDdR+7eD9wJ/Bx4jehZRrvMbIOZXTu02ueA283sVeBx4BZ310hARGhubmbXrl2xBnaFhYUKhEkQ6MVr7r6Z6Gmm8Y99Le72buBdQdYgIsmlp6eHUCjE0aNHKSgo0BXJk0xXNIvIlKEGdomnUBCRKUMN7BJPoSAiCTPcwM7dKS8vVwO7KUChICIJcfz4cUKhEF1dXcyZMyfR5cgQhYKITKrBwUEOHDjA4cOHyczMZNGiRRQWFia6LBmiUBCRSdXT00NjYyMlJSVUVFSoX9EUo1AQkcANDAzQ3t7OnDlzYg3sdCB5alIoiEigOjo6CIVC9PX1qYFdElAoiEgg+vv7qa+vp7W1ldzcXBYuXKgGdklAoSAiE87def311+nt7aW8vJx58+apgV2SUCiIyISJb2BXVVWlBnZJKNA5mkUkfaiBXWrQSEFEzoka2KUWhYKInLX4Bnbz58+nuLg40SXJOVIoiMhZy8rKYubMmVRXV5OZmZnocmQCKBREZNzcnYMHDwJQXl5OQUEBBQUFCa5KJpJCQUTGRQ3s0oNCQUROK76BXVZWlhrYpTiFgoicVm9vrxrYpRGFgoicZGBggLa2NoqLi8nJyVEDuzSiUBCRE7S3txMOh+nv7yc/P18N7BLM3alv7WLv4aNcvrSU6dOCbReiUBARINrALhwO09bWRm5uLosWLVIDuwQ40t3HjvoOtte3sS3czvb6dqZPMzq6+tj86T9gYUl+oO+vUBARNbBLkP6BQfYePsb2+na2hdvYXt9OQ3sXK8pnsrZ6Fje+vZK/uX4lZYU5vPcfnsM9+JoUCiJpbGQDu+zsbI0OAnT4SDfbwu1sq29je7idnQ0dzC3MYW3VLNZUF/HxS2o4f14BmdMT15ZOoSCSppqammhoaKCiooKSkhKdZjrBunoH2Hmgg+1xIdDZN8CaqiLWVs3ijssXsaayiMK8qXUluEJBJM10d3cTCoU4duwYM2fOVAO7CTA46LzVcvz3AVDfzv7GYyyZW8DaqiLWLZ/LF/5oKfPn5E353XIKBZE00tzcTH19PWZGTU2Nrkw+S23He9keaY8dCH61vp387AzWVhexpqqID6ytYEV5ITmZyXdNh0JBJI1kZ2ergd0Z6u0f5PVDR4YOBkdDoOloD6sqCllbXcRHL6rm7/9kNaUFqXEsRqEgksLUwO7MuDsN7V0nBMDuA0eonp3H2uoiLlowm09cupBFpfmBXy+QKAoFkRR17NgxQqEQ3d3dmufgFI719LMjbjfQ9vp23IntBvrcuiWsqiykICd9RlUKBZEUMzg4SENDA42NjWRlZbF48WIdTAYGBp39jcdi1wNsr28n1NLJsrIC1lbP4toLyvn6NcupKMqd8geDgxRoKJjZlcB9wHTg/7j7N0dZ50+BewAHXnX3DwdZk0iq6+3tpampidLSUsrLy9O2gV3T0Z4TLgrbEemgOD+LtdWzWFNVxIcvqmbpvJlkZWiq+niBhYKZTQceANYBEeBlM9vk7rvj1lkMfBl4l7u3mVlpUPWIpLKRDexWrVqVVgeSu/sG2HXgSCwAtoXbOdrdx5qhALj9PeexprKIWTPUw2ksQY4U3gHsd/c3AczsCeA6YHfcOrcDD7h7G4C7NwZYj0hKGq2BXSoHgrsTaumMXRC2vb6dPYePsqg0nzVVRVx2fimfWbeEBXNmMC1FDwYHKchQqADq4+5HgItGrLMEwMxeJLqL6R53f3rkC5nZemA9QHV1dSDFiiSbvr4+6uvraWtrIy8vL2Ub2HV09fFq7Gyg6EggJ3N67GDwH19QzsryQnKz0nM32UQLMhRGi+iR7ZwygMXAZUAl8GszW+nu7Sc8yX0jsBGgtrZ2ElpCiUxt7s6ePXvo7e2loqKCuXPnpsTB0f6BQV4/dJRt9e2xq4MPd3SzsqKQNdVFfOjCar5542rmzky98JsqggyFCFAVd78SODDKOlvcvQ94y8z2EA2JlwOsSyRp9fb2kpWVlTIN7A52dP3+dNBwOzsPdFBRlMuaqiLeNr+IW9+9gCVz88lIYIO4dBNkKLwMLDazBUADcBMw8syinwI3A4+YWTHR3UlvBliTSNJqbGykoaGBysrKpGxg19nbz45IRywAttW30TfgrK2K7ga6672LWV1VyMw0uiZgKgosFNy938zuBH5O9HjBw+6+y8w2AFvdfdPQsivMbDcwAHze3VuCqkkkGY1sYJcMYTA46LzZfIzfDo0CtoXbqWs+zvnzClhTVcRVq+Zx99XLqJqd3tcETEWBXqfg7puBzSMe+1rcbQc+O/QlIiM0NzcTDoeZNm3alG5g13KsJ3ZB2PDXrLysaJvo6iL+5O2VLC+fSXaGDgZPdbqiWWQKy87OpqioiKqqqilzmmlP/wCvHTx6wjUBbZ29XFAZ3Q10yyU1XFBVRHF+dqJLlbOgUBCZQgYHB2MN7CoqKhLewM7dibR18du4ANhz6Cg1xTNYU1XEuxYVc+fli1hYkq9rAlKEQkFkipgKDexONWn82uoi1lbP4ktXLWVVRSEzsvWnI1XpJyuSYAMDAxw4cGDSG9jFTxo/HALDk8avqSo6YdJ4HQxOHwoFkQTr6+ujubmZ0tJSKioqmDYtmHPyhyeNH24SNzxpfPRg8Cw+9s7ETxoviadQEEmA/v5+2traKCkpIScnh5UrV07ogeTuvgF+15B8k8ZL4ikURCZZW1sb4XCYgYEBCgoKzrmBXfyk8dvroyHwRuNxFs/Nj00a//k/WkpNEkwaL4l32lAws+8CP3L3/5qkekRSVl9fH+FwmPb2dvLy8qipqTmrFhXDk8ZHRwG/nzR+TXURa6uKuH5tedJOGi+JN9ZIYR9wr5mVAf8CPO7u24MvSyS1DDew6+vrO6MGduk2abwk3mlDwd3vA+4zs/lEexf9s5nlAI8DT7j73kmoUSRpxTewq66uJisr65Sjg9EmjX/t4BGqZuWxpio6afyfX3oei0sLUnbSeEm8cR1TcPcQ8C3gW2a2FngY+DrRnkYiMoK709TUdEIDu5GnmQ5PGh8fAu7EWkOk46TxknjjCgUzywSuJDpaeC/wHPA/A6xLJGl1d3dTV1fH8ePHKSwspLCwMDZpfPxFYZo0XqaisQ40ryPa2vqPgf8GngDWu/vxSahNJOk0NTVRX19PR/cArRTwxv5+tj/zW34X6WBO3KTxN7+jmmVlmjRepp6xRgp3Az8C/srdWyehHpGk1T/ofGXT6/x2T4ienNmsmT/I2upZ3P4H53FBVRGzNWm8JIGxQuFq4BPABjPbQXROhP7gyxJJDsMN7Nydz7xvCeVFuXzx2rWaNF4m3PVrKpiZG/ylZWO9wyNAL/AC0YBYAXw64JpEksKxY8eoq6ujp6eHkpIS7nrv4kSXJClssn6/xgqF5e6+CsDMHgJeCr4kkaltYGCAhoYGmpqayM7OZsmSJQltby0ykcYKhb7hG0PTawZcjsjU19fXR0tLC3PnzqW8vDywBnYiiTBWKFxgZkeA4TTIjbvv7h58f1+RKSDoBnYiU8VYVzTr4jRJexPdwE5kKhvrOoUcomcfLQJ09pGklYlqYCeSTMbaffQo0eMKv0ZnH0kaiW9gV1lZSWlpqa40lrSgs49E4vT29pKZmRlrYJednU12dnaiyxKZNGOdNnHC2UcB1yKSMO7O4cOH2bVrF01NTQDMnDlTgSBpZ7xnH0H0jCOdfSQpp6uri1AoFGtgV1RUlOiSRBJGZx9JWhtuYDd9+nQWLFjA7NmzE12SSEJpjmZJazk5OcyaNYuqqioyMvRxENGnQNLK4OAgBw4cwMyoqKigoKBALSpE4igUJG0cPXqUUCgUa2AnIidTKEjKGxgYIBKJ0NzcrAZ2ImMItJOXmV1pZnvMbL+Zfek0633QzNzMaoOsR9JTX18fra2tzJ07l+XLlysQRE4jsFAws+nAA8BVwHLgZjNbPsp6BcBdRKf7FJkQ/f39NDY2AtGDyatWraKyslIdTUXGEOQn5B3Afnd/0917ic7vfN0o6/0N8G2gO8BaJI20traya9cuIpEI3d3RXyudWSQyPkGGQgVQH3c/MvRYjJmtBarc/akA65A00dvby/79+3nrrbfIzs5m2bJlamAncoaC/PdptO5hHltoNg34R+CWMV/IbD2wHqC6unqCypNU4u7s3buXvr4+qqqqKCkpUQM7kbMQZChEgKq4+5XAgbj7BcBK4NmhD+88YJOZXevuW+NfyN03AhsBamtrHZEhamAnMrGC3H30MrDYzBaYWRZwE7BpeKG7d7h7sbvXuHsNsAU4KRBERqMGdiLBCGykMDSn853Az4HpRCfo2WVmG4Ct7r7p9K8gMrquri7q6uro7OykqKiIWbNmJbokkZQR6CkZ7r4Z2Dzisa+dYt3LgqxFUkN8A7vzzjtPgSAywXSeniSV3NxcNbATCZA+VTKlDQ4O0tDQgJlRWVlJfn4++fn5iS5LJGUpFGTKim9gV1pamuhyRNKCQkGmnJEN7M4//3yNDkQmiUJBppzhBnbz5s2jrKxM/YpEJpFCQaaEvr4+2traKC0tjTWw04FkkcmnT50kXGtrK/X19QwMDFBYWEh2drYCQSRB9MmThOnt7SUcDtPR0cGMGTOoqanRFckiCaZQkIRQAzuRqUmhIJOqp6eHrKwszIz58+eTnZ1NVlZWossSkSE6rUMmhbtz6NChExrYFRQUKBBEphiNFCRwnZ2dhEIhNbATSQIKBQlUY2MjkUiEjIwMNbATSQIKBQlUXl4es2fPprKyUqeZiiQBfUplQqmBnUhyUyjIhDly5AihUIje3l41sBNJUgoFOWcDAwPU19fT0tJCTk6OGtiJJDGFgpyz4b5FamAnkvwUCnJWhjuZzp07Vw3sRFKIPsVyxlpaWqivr2dwcJCioiI1sBNJIfoky7j19vYSCoU4cuQI+fn5sTYVIpI6FAoyLu7Onj176O/vp6qqSmcXiaQohYKcVnwDu+HW1upXJJK6dJqIjEoN7ETSk0YKcpL4BnazZs1SvyKRNKJQkBPEN7BbuHAhRUVFiS5JRCaRQkFOMNzArqqqiunTpye6HBGZZAqFNDcwMEBDQwPTpk1TAzsRUSiks/gGdnPnzk10OSIyBSgU0lB/fz+RSCTWwG7p0qXMmDEj0WWJyBSgUEhD/f39tLe3U1ZWRllZGWaW6JJEZIoI9DoFM7vSzPaY2X4z+9Ioyz9rZrvNbIeZ/dLM5gdZTzrr6+vj8OHDALEGduXl5QoEETlBYKFgZtOBB4CrgOXAzWa2fMRq24Bad18NPAl8O6h60llzczO7du3iwIED9PT0AOjMIhEZVZC7j94B7Hf3NwHM7AngOmD38Aru/kzc+luAjwZYT9rp6ekhHA6rgZ2IjFuQoVAB1MfdjwAXnWb9W4GfjbbAzNYD6wGqq6snqr6U5u7s3buXgYEBqqurKSkpSXRJIpIEggyF0XZW+6grmn0UqAUuHW25u28ENgLU1taO+hoSpQZ2InIuggyFCFAVd78SODByJTN7H/AV4FJ37wmwnpQ23MDu4MGDVFZWUlpaSkFBQaLLEpEkE2QovAwsNrMFQANwE/Dh+BXMbC3wT8CV7t4YYC0prbOzk7q6Orq6upg1axazZ89OdEkikqQCCwV37zezO4GfA9OBh919l5ltALa6+ybg74B84N+GTo2+QAUcAAAIfklEQVQMu/u1QdWUihobG6mvryczM1MN7ETknAV68Zq7bwY2j3jsa3G33xfk+6eDvLw8iouLqays1GmmInLOdEVzkhluYGdmVFVVqYGdiEwohUIS6ejoIBwOq4GdiARGoZAE+vv7qa+vp7W1VQ3sRCRQCoUkMDAwQEdHhxrYiUjgFApTVF9fHy0tLcybN4/s7GxWrVqlA8kiEjiFwhTU3NxMJBLB3Zk1axbZ2dkKBBGZFAqFKaSnp4dQKMTRo0cpKChQAzsRmXQKhSkivoHd/PnzKS4uTnRJIpKGFAoJ1t3dTXZ2thrYiciUEOjMa3Jq7s7BgwfZvXs3TU1NABQUFCgQRCShNFJIgOPHjxMKhejq6mL27NlqYCciU4ZCYZLFN7BbtGgRhYWFiS5JRCRGoTDJ8vLyKCkpoaKiQqeZisiUo1AI2MDAAJFIhGnTpqmBnYhMeQqFAHV0dBAKhejr62PevHmJLkdEZEwKhQDEN7DLzc1l4cKFamAnIklBoRCA4QZ25eXlzJs3Tw3sRCRpKBQmSG9vL62trWpgJyJJTaEwAZqammhoaFADOxFJegqFc6AGdiKSahQKZ0kN7EQkFSkUzlB8A7sFCxaQnZ1NZmZmossSEZkQaog3Tu7OgQMHTmhgl5+fr0AQkZSikcI4HD9+nLq6Orq7u5kzZ44a2IlIylIojOHw4cNEIhGysrLUwE5EUp5CYQwzZsxQAzsRSRsKhRHUwE5E0plCIU57ezvhcJj+/n7mzp2b6HJERCadQoFoA7twOExbWxt5eXksWrSIvLy8RJclIjLpFApEdxkdOXKEiooK5s6dqwZ2IpK20jYUent7aWlpoaysTA3sRESGBHrxmpldaWZ7zGy/mX1plOXZZvYvQ8v/28xqgqxnWFNTE7t27eLQoUP09PQAKBBERAhwpGBm04EHgHVABHjZzDa5++641W4F2tx9kZndBHwL+FBQNXV3dxMKhTh27BgzZ85k/vz5ZGVlBfV2IiJJJ8iRwjuA/e7+prv3Ak8A141Y5zrg0aHbTwLvtYB26Ls7+/bto6uri5qaGhYvXqxAEBEZIchjChVAfdz9CHDRqdZx934z6wDmAM3xK5nZemA9QHV19VkVowZ2IiJjC3KkMNp//H4W6+DuG9291t1rS0pKzrogNbATETm9IEMhAlTF3a8EDpxqHTPLAAqB1gBrEhGR0wgyFF4GFpvZAjPLAm4CNo1YZxPw8aHbHwR+5e4njRRERGRyBHZMYegYwZ3Az4HpwMPuvsvMNgBb3X0T8BDwmJntJzpCuCmoekREZGyBXrzm7puBzSMe+1rc7W7gT4KsQURExk8zr4mISIxCQUREYhQKIiISo1AQEZEYS7YzQM2sCQid5dOLGXG1dBrQNqcHbXN6OJdtnu/uY179m3ShcC7MbKu71ya6jsmkbU4P2ub0MBnbrN1HIiISo1AQEZGYdAuFjYkuIAG0zelB25weAt/mtDqmICIip5duIwURETmNlAyFqTo3dJDGsc2fNbPdZrbDzH5pZvMTUedEGmub49b7oJm5mSX9mSrj2WYz+9Ohn/UuM/vRZNc40cbxu11tZs+Y2bah3++rE1HnRDGzh82s0cx2nmK5mdn9Q9+PHWb2tgktwN1T6otoR9Y3gPOALOBVYPmIde4AHhy6fRPwL4muexK2+XIgb+j2J9Nhm4fWKwCeB7YAtYmuexJ+zouBbcCsofulia57ErZ5I/DJodvLgbpE132O2/we4G3AzlMsvxr4GdFJyi4G/nsi3z8VRwpTam7oSTLmNrv7M+7eOXR3C9FJj5LZeH7OAH8DfBvonsziAjKebb4deMDd2wDcvXGSa5xo49lmB2YO3S7k5Mm8koq7P8/pJxu7DviBR20BisysbKLePxVDYbS5oStOtY679wPDc0Mnq/Fsc7xbif6nkczG3GYzWwtUuftTk1lYgMbzc14CLDGzF81si5ldOWnVBWM823wP8FEzixBt1f+pySktYc70835GAp1PIUEmbG7oJDLu7TGzjwK1wKWBVhS8026zmU0D/hG4ZbIKmgTj+TlnEN2FdBnR0eCvzWylu7cHXFtQxrPNNwOPuPu9ZvZOohN3rXT3weDLS4hA/36l4kghHeeGHs82Y2bvA74CXOvuPZNUW1DG2uYCYCXwrJnVEd33uinJDzaP93f7P9y9z93fAvYQDYlkNZ5tvhX4VwB3/w2QQ7RHUKoa1+f9bKViKKTj3NBjbvPQrpR/IhoIyb6fGcbYZnfvcPdid69x9xqix1GudfetiSl3Qoznd/unRE8qwMyKie5OenNSq5xY49nmMPBeADNbRjQUmia1ysm1CfjY0FlIFwMd7n5wol485XYfeRrODT3Obf47IB/4t6Fj6mF3vzZhRZ+jcW5zShnnNv8cuMLMdgMDwOfdvSVxVZ+bcW7z54Dvm9lniO5GuSWZ/8kzs8eJ7v4rHjpO8nUgE8DdHyR63ORqYD/QCfzZhL5/En/vRERkgqXi7iMRETlLCgUREYlRKIiISIxCQUREYhQKIiISk3KnpIoEwczmAL8cujuP6Omew+fCX0C0UVsG8BrwcXfvNLMB4HdDj78F/I8kvrJY0oROSRU5Q2Z2D3DM3f9+6P4xd88fuv1D4BV3/4cRjz8K7HX3bySqbpHx0O4jkYn1a2DRKI//hglsWiYSFIWCyAQZ6qN1FdFdRvGPTyfahiHlrrKW1KNQEDl3uWa2HdhKtA/PQyMebwFmA/8vQfWJjJsONIucuy53X3Oqx82sEHgK+Avg/sktTeTMaKQgEjB37wDuAv7KzDITXY/I6SgURCaBu28jetpqUnfkldSnU1JFRCRGIwUREYlRKIiISIxCQUREYhQKIiISo1AQEZEYhYKIiMQoFEREJEahICIiMf8f9tF+MrgrIL0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "crv.plot(area=True, classes=[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multilabel confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From `version 4.0`, `MultiLabelCM` has been added to calculate class-wise or sample-wise multilabel confusion matrices. In class-wise mode, confusion matrices are calculated for each class, and in sample-wise mode, they are generated per sample. All generated confusion matrices are binarized with a one-vs-rest transformation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 0, 1], [0, 1, 0]]\n",
      "[[1, 0, 0], [0, 1, 1]]\n",
      "Predict 0       1       \n",
      "Actual\n",
      "0       1       0       \n",
      "\n",
      "1       0       1       \n",
      "\n",
      "\n",
      "Predict 0       1       \n",
      "Actual\n",
      "0       1       0       \n",
      "\n",
      "1       1       1       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlcm = MultiLabelCM(actual_vector=[{\"cat\", \"bird\"}, {\"dog\"}], \n",
    "                    predict_vector=[{\"cat\"}, {\"dog\", \"bird\"}], \n",
    "                    classes=[\"cat\", \"dog\", \"bird\"])\n",
    "print(mlcm.actual_vector_multihot)\n",
    "print(mlcm.predict_vector_multihot)\n",
    "mlcm.get_cm_by_class(\"cat\").print_matrix()\n",
    "mlcm.get_cm_by_sample(0).print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 4.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Online help"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`online_help` function is added in `version 1.1` in order to open each statistics definition in web browser."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "\n",
    ">>> from pycm import online_help\n",
    ">>> online_help(\"J\")\n",
    ">>> online_help(\"J\", alt_link=True)\n",
    ">>> online_help(\"SOA1(Landis & Koch)\")\n",
    ">>> online_help(2)\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* List of items are available by calling `online_help()` (without argument)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* If PyCM website is not available, set `alt_link = True`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please choose one parameter : \n",
      "\n",
      "Example : online_help(\"J\") or online_help(2)\n",
      "\n",
      "1-95% CI\n",
      "2-ACC\n",
      "3-ACC Macro\n",
      "4-AGF\n",
      "5-AGM\n",
      "6-AM\n",
      "7-ARI\n",
      "8-AUC\n",
      "9-AUCI\n",
      "10-AUNP\n",
      "11-AUNU\n",
      "12-AUPR\n",
      "13-BB\n",
      "14-BCD\n",
      "15-BM\n",
      "16-Bangdiwala B\n",
      "17-Bennett S\n",
      "18-CBA\n",
      "19-CEN\n",
      "20-CSI\n",
      "21-Chi-Squared\n",
      "22-Chi-Squared DF\n",
      "23-Conditional Entropy\n",
      "24-Cramer V\n",
      "25-Cross Entropy\n",
      "26-DOR\n",
      "27-DP\n",
      "28-DPI\n",
      "29-ERR\n",
      "30-F0.5\n",
      "31-F1\n",
      "32-F1 Macro\n",
      "33-F1 Micro\n",
      "34-F2\n",
      "35-FDR\n",
      "36-FN\n",
      "37-FNR\n",
      "38-FNR Macro\n",
      "39-FNR Micro\n",
      "40-FOR\n",
      "41-FP\n",
      "42-FPR\n",
      "43-FPR Macro\n",
      "44-FPR Micro\n",
      "45-G\n",
      "46-GI\n",
      "47-GM\n",
      "48-Gwet AC1\n",
      "49-HD\n",
      "50-Hamming Loss\n",
      "51-IBA\n",
      "52-ICSI\n",
      "53-IS\n",
      "54-J\n",
      "55-Joint Entropy\n",
      "56-KL Divergence\n",
      "57-Kappa\n",
      "58-Kappa 95% CI\n",
      "59-Kappa No Prevalence\n",
      "60-Kappa Standard Error\n",
      "61-Kappa Unbiased\n",
      "62-Krippendorff Alpha\n",
      "63-LS\n",
      "64-Lambda A\n",
      "65-Lambda B\n",
      "66-MCC\n",
      "67-MCCI\n",
      "68-MCEN\n",
      "69-MK\n",
      "70-Mutual Information\n",
      "71-N\n",
      "72-NIR\n",
      "73-NLR\n",
      "74-NLRI\n",
      "75-NPV\n",
      "76-NPV Macro\n",
      "77-NPV Micro\n",
      "78-OC\n",
      "79-OOC\n",
      "80-OP\n",
      "81-Overall ACC\n",
      "82-Overall CEN\n",
      "83-Overall J\n",
      "84-Overall MCC\n",
      "85-Overall MCEN\n",
      "86-Overall RACC\n",
      "87-Overall RACCU\n",
      "88-P\n",
      "89-P-Value\n",
      "90-PLR\n",
      "91-PLRI\n",
      "92-POP\n",
      "93-PPV\n",
      "94-PPV Macro\n",
      "95-PPV Micro\n",
      "96-PRE\n",
      "97-Pearson C\n",
      "98-Phi-Squared\n",
      "99-Q\n",
      "100-QI\n",
      "101-RACC\n",
      "102-RACCU\n",
      "103-RCI\n",
      "104-RR\n",
      "105-Reference Entropy\n",
      "106-Response Entropy\n",
      "107-SOA1(Landis & Koch)\n",
      "108-SOA2(Fleiss)\n",
      "109-SOA3(Altman)\n",
      "110-SOA4(Cicchetti)\n",
      "111-SOA5(Cramer)\n",
      "112-SOA6(Matthews)\n",
      "113-SOA7(Lambda A)\n",
      "114-SOA8(Lambda B)\n",
      "115-SOA9(Krippendorff Alpha)\n",
      "116-SOA10(Pearson C)\n",
      "117-Scott PI\n",
      "118-Standard Error\n",
      "119-TN\n",
      "120-TNR\n",
      "121-TNR Macro\n",
      "122-TNR Micro\n",
      "123-TON\n",
      "124-TOP\n",
      "125-TP\n",
      "126-TPR\n",
      "127-TPR Macro\n",
      "128-TPR Micro\n",
      "129-Y\n",
      "130-Zero-one Loss\n",
      "131-dInd\n",
      "132-sInd\n"
     ]
    }
   ],
   "source": [
    "online_help()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `param` : input parameter (type : `int or str`, default : `None`)\n",
    "2. `alt_link` : alternative link for document flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>alt_link</code> , new in <span style=\"color:red;\">version 2.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acceptable data types\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<strong>ConfusionMatrix</strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `actual_vector` : python `list` or numpy `array` of any stringable objects\n",
    "2. `predict_vector` : python `list` or numpy `array` of any stringable objects\n",
    "3. `matrix` : `dict`\n",
    "4. `digit`: `int`\n",
    "5. `threshold` : `FunctionType (function or lambda)`\n",
    "6. `file` : `File object`\n",
    "7. `sample_weight` : python `list` or numpy `array` of numbers\n",
    "8. `transpose` : `bool`\n",
    "9. `classes` : python `list`\n",
    "10. `is_imbalanced` : `bool`\n",
    "11. `metrics_off` : `bool`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(ConfusionMatrix)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>metrics_off</code>, new in <span style=\"color:red;\">version 3.9 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<strong>Compare</strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `cm_dict` : python `dict` of `ConfusionMatrix` object (`str` : `ConfusionMatrix`)\n",
    "2. `by_class` : `bool`\n",
    "3. `class_weight` : python `dict` of class weights (`class_name` : `float`)\n",
    "4. `class_benchmark_weight`: python `dict` of class benchmark weights (`class_benchmark_name` : `float`) \n",
    "5. `overall_benchmark_weight`: python `dict` of overall benchmark weights (`overall_benchmark_name` : `float`) \n",
    "6. `digit`: `int`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(Compare)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>weight</code> renamed to <code>class_weight</code> in <span style=\"color:red;\">version 3.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>overall_benchmark_weight</code> and <code>class_benchmark_weight</code>, new in <span style=\"color:red;\">version 3.3 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<strong>ROCCurve</strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `actual_vector` : python `list` or numpy `array` of any stringable objects\n",
    "2. `probs` : python `list` or numpy `array`\n",
    "3. `classes` : python `list`\n",
    "4. `thresholds`: python `list` or numpy `array` \n",
    "5. `sample_weight`: python `list` or numpy `array`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(ROCCurve)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<strong>PRCurve</strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `actual_vector` : python `list` or numpy `array` of any stringable objects\n",
    "2. `probs` : python `list` or numpy `array`\n",
    "3. `classes` : python `list`\n",
    "4. `thresholds`: python `list` or numpy `array` \n",
    "5. `sample_weight`: python `list` or numpy `array`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(PRCurve)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<strong>MultiLabelCM</strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `actual_vector` : python `list` or numpy array of `sets`\n",
    "2. `predict_vector` : python `list` or numpy array of `sets`\n",
    "3. `sample_weight`: python `list` or numpy `array`\n",
    "4. `classes` : python `list`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(MultiLabelCM)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TP (True positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A true positive test result is one that detects the condition when the\n",
    "condition is present (correctly identified) [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 3, 'L2': 1, 'L3': 3}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TN (True negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A true negative test result is one that does not detect the condition when\n",
    "the condition is absent (correctly rejected) [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 7, 'L2': 8, 'L3': 4}"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FP (False positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A false positive test result is one that detects the condition when the\n",
    "condition is absent (incorrectly identified) [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0, 'L2': 2, 'L3': 3}"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FN (False negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A false negative test result is one that does not detect the condition when\n",
    "the condition is present (incorrectly rejected) [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 2, 'L2': 1, 'L3': 2}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### P (Condition positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of positive samples.\n",
    "Also known as support (the number of occurrences of each class in y_true) [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P=TP+FN$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 5, 'L2': 2, 'L3': 5}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.P"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### N (Condition negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of negative samples [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$N=TN+FP$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 7, 'L2': 10, 'L3': 7}"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.N"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TOP (Test outcome positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of positive outcomes [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TOP=TP+FP$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 3, 'L2': 3, 'L3': 6}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TOP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TON (Test outcome negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of negative outcomes [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TON=TN+FN$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 9, 'L2': 9, 'L3': 6}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POP (Population)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Total sample size [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$POP=TP+TN+FN+FP$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 12, 'L2': 12, 'L3': 12}"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.POP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <a href=\"https://en.wikipedia.org/wiki/Confusion_matrix\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR (True positive rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sensitivity (also called the true positive rate, the recall, or probability of detection in some fields) measures the proportion of positives that are correctly identified as such (e.g. the percentage of sick people who are correctly identified as having the condition) [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Sensitivity_and_specificity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR=\\frac{TP}{P}=\\frac{TP}{TP+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6, 'L2': 0.5, 'L3': 0.6}"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TNR (True negative rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specificity (also called the true negative rate) measures the proportion of negatives that are correctly identified as such (e.g. the percentage of healthy people who are correctly identified as not having the condition) [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Sensitivity_and_specificity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TNR=\\frac{TN}{N}=\\frac{TN}{TN+FP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.0, 'L2': 0.8, 'L3': 0.5714285714285714}"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TNR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV (Positive predictive value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Positive predictive value (PPV) is the proportion of positives that correspond to\n",
    "the presence of the condition [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV=\\frac{TP}{TP+FP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.0, 'L2': 0.3333333333333333, 'L3': 0.5}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NPV (Negative predictive value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Negative predictive value (NPV) is the proportion of negatives that correspond to\n",
    "the absence of the condition [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NPV=\\frac{TN}{TN+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7777777777777778, 'L2': 0.8888888888888888, 'L3': 0.6666666666666666}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NPV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FNR (False negative rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false negative rate is the proportion of positives which yield negative test outcomes with the test, i.e., the conditional probability of a negative test result given that the condition being looked for is present [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_positives_and_false_negatives#False_positive_and_false_negative_rates\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FNR=\\frac{FN}{P}=\\frac{FN}{FN+TP}=1-TPR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.5, 'L3': 0.4}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FNR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FPR (False positive rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false positive rate is the proportion of all negatives that still yield positive test outcomes, i.e., the conditional probability of a positive test result given an event that was not present [[3]](#ref3).\n",
    "\n",
    "The false positive rate is equal to the significance level. The specificity of the test is equal to $ 1 $ minus the false positive rate.\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_positives_and_false_negatives#False_positive_and_false_negative_rates\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FPR=\\frac{FP}{N}=\\frac{FP}{FP+TN}=1-TNR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.0, 'L2': 0.19999999999999996, 'L3': 0.4285714285714286}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FDR (False discovery rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false discovery rate (FDR) is a method of conceptualizing the rate of type I errors in null hypothesis testing when conducting multiple comparisons. FDR-controlling procedures are designed to control the expected proportion of \"discoveries\" (rejected null hypotheses) that are false (incorrect rejections) [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_discovery_rate\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FDR=\\frac{FP}{FP+TP}=1-PPV$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.0, 'L2': 0.6666666666666667, 'L3': 0.5}"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FDR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FOR (False omission rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "False omission rate (FOR) is a statistical method used in multiple hypothesis testing to correct for multiple comparisons and it is the complement of the negative predictive value. It measures the proportion of false negatives which are incorrectly rejected [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FOR=\\frac{FN}{FN+TN}=1-NPV$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.2222222222222222,\n",
       " 'L2': 0.11111111111111116,\n",
       " 'L3': 0.33333333333333337}"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FOR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ACC (Accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy is the number of correct predictions from all predictions made [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Accuracy_and_precision\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ACC=\\frac{TP+TN}{P+N}=\\frac{TP+TN}{TP+TN+FP+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8333333333333334, 'L2': 0.75, 'L3': 0.5833333333333334}"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ERR (Error rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The error rate is the number of incorrect predictions from all predictions made [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ERR=\\frac{FP+FN}{P+N}=\\frac{FP+FN}{TP+TN+FP+FN}=1-ACC$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.16666666666666663, 'L2': 0.25, 'L3': 0.41666666666666663}"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ERR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FBeta-Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistical analysis of classification, the F1 score (also F-score or F-measure) is a measure of a test's accuracy. It considers both the precision $ p $ and the recall $ r $ of the test to compute the score.\n",
    "The F1 score is the harmonic average of the precision and recall, where F1 score reaches its best value at $ 1 $ (perfect precision and recall) and worst at $ 0 $ [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/F1_score\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$F_{\\beta}=(1+\\beta^2)\\times \\frac{PPV\\times TPR}{(\\beta^2 \\times PPV)+TPR}=\\frac{(1+\\beta^2) \\times TP}{(1+\\beta^2)\\times TP+FP+\\beta^2 \\times FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.75, 'L2': 0.4, 'L3': 0.5454545454545454}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8823529411764706, 'L2': 0.35714285714285715, 'L3': 0.5172413793103449}"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6521739130434783, 'L2': 0.45454545454545453, 'L3': 0.5769230769230769}"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6144578313253012, 'L2': 0.4857142857142857, 'L3': 0.5930232558139535}"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F_beta(beta=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `beta` : beta parameter (type : `float`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`{class1: FBeta-Score1, class2: FBeta-Score2, ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCC (Matthews correlation coefficient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Matthews correlation coefficient is used in machine learning as a measure of the quality of binary (two-class) classifications, introduced by biochemist Brian W. Matthews in 1975. It takes into account true and false positives and negatives and is generally regarded as a balanced measure that can be used even if the classes are of very different sizes. The MCC is, in essence, a correlation coefficient between the observed and predicted binary classifications; it returns a value between  $ −1 $ and  $ +1 $. A coefficient of $ +1 $ represents a perfect prediction, $ 0 $ no better than random prediction and $ −1 $ indicates total disagreement between prediction and observation [[27]](#ref27).\n",
    "\n",
    "<a href=\"#MCCI-(Matthews-correlation-coefficient-interpretation)\">Interpretation</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Matthews_correlation_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCC=\\frac{TP \\times TN-FP \\times FN}{\\sqrt{(TP+FP)\\times (TP+FN)\\times (TN+FP)\\times (TN+FN)}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6831300510639732, 'L2': 0.25819888974716115, 'L3': 0.1690308509457033}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BM (Bookmaker informedness)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The informedness of a prediction method as captured by a contingency matrix is defined as the probability that the prediction method will make a correct decision as opposed to guessing and is calculated using the bookmaker algorithm [[2]](#ref2).\n",
    "\n",
    "Equals to <a href=\"#Y-(Youden-index)\">Youden Index</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BM=TPR+TNR-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.BM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MK (Markedness)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics and psychology, the social science concept of markedness is quantified as a measure of how much one variable is marked as a predictor or possible cause of another and is also known as $ \\triangle P $ in simple two-choice cases [[2]](#ref2)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MK=PPV+NPV-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7777777777777777, 'L2': 0.2222222222222221, 'L3': 0.16666666666666652}"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PLR (Positive likelihood ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Likelihood ratios are used for assessing the value of performing a diagnostic test. They use the sensitivity and specificity of the test to determine whether a test result usefully changes the probability that a condition (such as a disease state) exists. The first description of the use of likelihood ratios for decision rules was made at a symposium on information theory in 1954 [[28]](#ref28).\n",
    "\n",
    "<a href=\"#PLRI-(Positive-likelihood-ratio-interpretation)\">Interpretation</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Likelihood_ratios_in_diagnostic_testing\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LR_+=PLR=\\frac{TPR}{FPR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 2.5000000000000004, 'L3': 1.4}"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>LR+</code> renamed to <code>PLR</code> in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLR (Negative likelihood ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Likelihood ratios are used for assessing the value of performing a diagnostic test. They use the sensitivity and specificity of the test to determine whether a test result usefully changes the probability that a condition (such as a disease state) exists. The first description of the use of likelihood ratios for decision rules was made at a symposium on information theory in 1954 [[28]](#ref28).\n",
    "\n",
    "<a href=\"#NLRI-(Negative-likelihood-ratio-interpretation)\">Interpretation</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Likelihood_ratios_in_diagnostic_testing\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LR_-=NLR=\\frac{FNR}{TNR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.625, 'L3': 0.7000000000000001}"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>LR-</code> renamed to <code>NLR</code> in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DOR (Diagnostic odds ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The diagnostic odds ratio is a measure of the effectiveness of a diagnostic test. It is defined as the ratio of the odds of the test being positive if the subject has a disease relative to the odds of the test being positive if the subject does not have the disease [[28]](#ref28).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Diagnostic_odds_ratio\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DOR=\\frac{LR_+}{LR_-}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 4.000000000000001, 'L3': 1.9999999999999998}"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DOR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PRE (Prevalence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prevalence is a statistical concept referring to the number of cases of a disease that are present in a particular population at a given time (Reference Likelihood) [[14]](#ref14).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Prevalence\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Prevalence=\\frac{P}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4166666666666667, 'L2': 0.16666666666666666, 'L3': 0.4166666666666667}"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PRE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### G (G-measure)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The geometric mean of precision and sensitivity, also known as Fowlkes–Mallows index [[3]](#ref3).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Fowlkes%E2%80%93Mallows_index\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$G=\\sqrt{PPV\\times TPR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7745966692414834, 'L2': 0.408248290463863, 'L3': 0.5477225575051661}"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.G"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RACC (Random accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The expected accuracy from a strategy of randomly guessing categories according to reference and response distributions [[24]](#ref24)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACC=\\frac{TOP \\times P}{POP^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.10416666666666667,\n",
       " 'L2': 0.041666666666666664,\n",
       " 'L3': 0.20833333333333334}"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RACCU (Random accuracy unbiased)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The expected accuracy from a strategy of randomly guessing categories according to the average of the reference and response distributions [[25]](#ref25)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACCU=(\\frac{TOP+P}{2 \\times POP})^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.1111111111111111,\n",
       " 'L2': 0.04340277777777778,\n",
       " 'L3': 0.21006944444444442}"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### J (Jaccard index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Jaccard index, also known as Intersection over Union and the Jaccard similarity coefficient (originally coined coefficient de communauté by Paul Jaccard), is a statistic used for comparing the similarity and diversity of sample sets [[29]](#ref29).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Jaccard_index\">Wikipedia page</a>\n",
    "\n",
    "Some articles also named it as the F* (An Interpretable Transformation of the F-measure) [[77]](#ref77)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J=\\frac{TP}{TOP+P-TP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6, 'L2': 0.25, 'L3': 0.375}"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IS (Information score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The amount of information needed to correctly classify an example into\n",
    "class C, whose prior probability is $ p(C) $, is defined as $ -\\log_2(p(C)) $ [[18]](#ref18) [[39]](#ref39)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$IS=-log_2(\\frac{TP+FN}{POP})+log_2(\\frac{TP}{TP+FP})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.2630344058337937, 'L2': 0.9999999999999998, 'L3': 0.26303440583379367}"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CEN (Confusion entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CEN based upon the concept of entropy for evaluating classifier performances. By exploiting the misclassification information of confusion matrices, the measure evaluates the confusion level of the class distribution of\n",
    "misclassified samples. Both theoretical analysis and statistical results show that the proposed measure is more discriminating than accuracy and RCI while it remains relatively consistent with the two measures. Moreover, it is more capable of measuring how the samples of different classes have been separated from each\n",
    "other. Hence the proposed measure is more precise than the two measures and can substitute for them to evaluate classifiers in classification applications [[17]](#ref17)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{j}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{i}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(i,k)+Matrix(k,i)\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CEN_j=-\\sum_{k=1,k\\neq j}^{|C|}\\Bigg(P_{j,k}^jlog_{2(|C|-1)}\\Big(P_{j,k}^j\\Big)+P_{k,j}^jlog_{2(|C|-1)}\\Big(P_{k,j}^j\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.25, 'L2': 0.49657842846620864, 'L3': 0.6044162769630221}"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  $ |C| $ is the number of classes </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCEN (Modified confusion entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modified version of CEN [[19]](#ref19)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{j}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)-Matrix(j,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{i}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(i,k)+Matrix(k,i)\\Big)-Matrix(i,i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCEN_j=-\\sum_{k=1,k\\neq j}^{|C|}\\Bigg(P_{j,k}^jlog_{2(|C|-1)}\\Big(P_{j,k}^j\\Big)+P_{k,j}^jlog_{2(|C|-1)}\\Big(P_{k,j}^j\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.2643856189774724, 'L2': 0.5, 'L3': 0.6875}"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUC (Area under the ROC curve)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The area under the curve (often referred to as simply the AUC) is equal to the probability that a classifier will rank a randomly chosen positive instance higher than a randomly chosen negative one (assuming 'positive' ranks higher than 'negative').\n",
    "Thus, AUC corresponds to the arithmetic mean of sensitivity and specificity values of each class [[23]](#ref23).\n",
    "\n",
    "<a href=\"#AUCI-(AUC-value-interpretation)\">Interpretation</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUC=\\frac{TNR+TPR}{2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8, 'L2': 0.65, 'L3': 0.5857142857142856}"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  this is an approximate calculation of AUC </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dInd (Distance index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Euclidean distance of a ROC point from the top left corner of the ROC space, which can take values between 0 (perfect classification) and $ \\sqrt{2} $  [[23]](#ref23)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$dInd=\\sqrt{(1-TNR)^2+(1-TPR)^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.5385164807134504, 'L3': 0.5862367008195198}"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.dInd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sInd (Similarity index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sInd is comprised between $ 0 $ (no correct classifications) and $ 1 $ (perfect classification) [[23]](#ref23)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$sInd = 1 - \\sqrt{\\frac{(1-TNR)^2+(1-TPR)^2}{2}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.717157287525381, 'L2': 0.6192113447068046, 'L3': 0.5854680534700882}"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.sInd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DP (Discriminant power) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Discriminant power (DP) is a measure that summarizes sensitivity and specificity.\n",
    "The DP has been used mainly in feature selection over imbalanced data [[33]](#ref33).\n",
    "\n",
    "<a href=\"#DPI-(Discriminant-power-interpretation)\">Interpretation</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$X=\\frac{TPR}{1-TPR}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Y=\\frac{TNR}{1-TNR}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DP=\\frac{\\sqrt{3}}{\\pi}(log_{10}X+log_{10}Y)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 0.33193306999649924, 'L3': 0.1659665349982495}"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Y (Youden index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Youden’s index evaluates the algorithm’s ability to avoid failure; it’s derived from sensitivity and\n",
    "specificity and denotes a linear correspondence balanced accuracy.\n",
    "As Youden’s index is a linear transformation of the mean sensitivity and specificity, its values are difficult to\n",
    "interpret, we retain that a higher value of Y indicates better ability to avoid failure. \n",
    "Youden’s index has been conventionally used to evaluate tests diagnostic, improve the efficiency of\n",
    "Telemedical prevention [[33]](#ref33) [[34]](#ref34).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Youden%27s_J_statistic\">Wikipedia page</a>\n",
    "\n",
    "Equals to <a href=\"#BM-(Bookmaker-informedness)\">Bookmaker Informedness</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Y=BM=TPR+TNR-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PLRI (Positive likelihood ratio interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#ref33)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">PLR</td>\n",
    "           <td style=\"text-align:center\">Model contribution</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 1 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">1 - 5</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">5 - 10</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> > 10 </td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PLRI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLRI (Negative likelihood ratio interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[48]](#ref48)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">NLR</td>\n",
    "           <td style=\"text-align:center\">Model contribution</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.5 - 1</td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.2 - 0.5</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.1 - 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.1 >  </td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'Poor', 'L2': 'Negligible', 'L3': 'Negligible'}"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NLRI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DPI (Discriminant power interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#ref33)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">DP</td>\n",
    "           <td style=\"text-align:center\">Model contribution</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 1 > </td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">1 - 2</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Limited</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">2 - 3</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> > 3 </td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DPI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUCI (AUC value interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#ref33)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">AUC</td>\n",
    "           <td style=\"text-align:center\">Model performance</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.5 - 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.6 - 0.7</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.7 - 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:yellowgreen;\">Good</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.8 - 0.9</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Very Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.9 - 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'Very Good', 'L2': 'Fair', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCCI (Matthews correlation coefficient interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MCC is a confusion matrix method of calculating the Pearson product-moment correlation coefficient (not to be confused with Pearson's C). Therefore, it has the same interpretation [[2]](#ref2).\n",
    "\n",
    "For more information visit [[49]](#ref49)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">MCC</td>\n",
    "           <td style=\"text-align:center\">Interpretation</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.3 ></td>\n",
    "           <td style=\"text-align:center;background-color:Red;\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.3 - 0.5</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.5 - 0.7</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.7 - 0.9</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.9 - 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Very Strong</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'Moderate', 'L2': 'Negligible', 'L3': 'Negligible'}"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MCCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  only positive values are considered</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### QI (Yule's Q interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[67]](#ref67)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Q</td>\n",
    "           <td style=\"text-align:center\">Interpretation</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.25 > </td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.25 - 0.5</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.5 - 0.75</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> > 0.75 </td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Strong</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 'Moderate', 'L3': 'Weak'}"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.QI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GI (Gini index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A chance-standardized variant of the AUC is given by Gini coefficient, taking values between $ 0 $ (no difference\n",
    "between the score distributions of the two classes) and $ 1 $ (complete separation between the two distributions).\n",
    "Gini coefficient is widespread use metric in imbalanced data learning [[33]](#ref33). \n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Gini_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$GI=2\\times AUC-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.GI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LS (Lift score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the context of classification, lift compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [[35]](#ref35) [[36]](#ref36)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LS=\\frac{PPV}{PRE}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 2.4, 'L2': 2.0, 'L3': 1.2}"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AM (Automatic/Manual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Difference between automatic and manual classification i.e., the difference between positive outcomes and of positive samples."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AM=TOP-P=(TP+FP)-(TP+FN)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': -2, 'L2': 1, 'L3': 1}"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BCD (Bray-Curtis dissimilarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In ecology and biology, the Bray–Curtis dissimilarity, named after J. Roger Bray and John T. Curtis, is a statistic used to quantify the compositional dissimilarity between two different sites, based on counts at each site [[37]](#ref37).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Bray%E2%80%93Curtis_dissimilarity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BCD=\\frac{|AM|}{\\sum_{i=1}^{|C|}\\Big(TOP_i+P_i\\Big)}=\\frac{|AM|}{2\\times POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.08333333333333333,\n",
       " 'L2': 0.041666666666666664,\n",
       " 'L3': 0.041666666666666664}"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.BCD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OP (Optimized precision)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimized precision is a type of hybrid threshold metric and has been proposed as a\n",
    "discriminator for building an optimized heuristic classifier. This metric is a combination of\n",
    "accuracy, sensitivity and specificity metrics. The sensitivity and specificity metrics were used for \n",
    "stabilizing and optimizing the accuracy performance when dealing with an imbalanced class of two-class problems [[40]](#ref40) [[42]](#ref42)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OP = ACC - \\frac{|TNR-TPR|}{|TNR+TPR|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.5833333333333334, 'L2': 0.5192307692307692, 'L3': 0.5589430894308943}"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.OP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IBA (Index of balanced accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The method combines an unbiased index of its overall accuracy and a measure about\n",
    "how dominant is the class with the highest individual accuracy rate [[41]](#ref41) [[42]](#ref42)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$IBA_{\\alpha}=(1+\\alpha \\times(TPR-TNR))\\times TNR \\times TPR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.36, 'L2': 0.27999999999999997, 'L3': 0.35265306122448975}"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.48, 'L2': 0.34, 'L3': 0.3477551020408163}"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA_alpha(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.576, 'L2': 0.388, 'L3': 0.34383673469387754}"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA_alpha(0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `alpha` : alpha parameter (type : `float`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`{class1: IBA1, class2: IBA2, ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GM (G-mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Geometric mean of specificity and sensitivity [[3]](#ref3) [[41]](#ref41) [[42]](#ref42)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$GM=\\sqrt{TPR \\times TNR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7745966692414834, 'L2': 0.6324555320336759, 'L3': 0.5855400437691198}"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.GM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Q (Yule's Q)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, Yule's Q, also known as the coefficient of colligation, is a measure of association between two binary variables [[45]](#ref45).\n",
    "\n",
    "<a href=\"#QI-(Yule's-Q-interpretation)\">Interpretation</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Coefficient_of_colligation\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OR = \\frac{TP\\times TN}{FP\\times FN}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Q = \\frac{OR-1}{OR+1}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 0.6, 'L3': 0.3333333333333333}"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AGM (Adjusted G-mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An adjusted version of the geometric mean of specificity and sensitivity [[46]](#ref46)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$N_n=\\frac{N}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AGM=\\frac{GM+TNR\\times N_n}{1+N_n};TPR>0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AGM=0;TPR=0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8576400016262, 'L2': 0.708612108382005, 'L3': 0.5803410802752335}"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AGM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AGF (Adjusted F-score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The F-measures used only three of the four elements of the confusion matrix and hence two classifiers with different TNR values may have the same F-score. Therefore, the AGF metric is introduced to use all elements of the confusion matrix and provide more weights to samples which are correctly classified in the minority class [[50]](#ref50)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AGF=\\sqrt{F_2 \\times InvF_{0.5}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$F_{2}=5\\times \\frac{PPV\\times TPR}{(4 \\times PPV)+TPR}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$InvF_{0.5}=(1+0.5^2)\\times \\frac{NPV\\times TNR}{(0.5^2 \\times NPV)+TNR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7285871475307653, 'L2': 0.6286946134619315, 'L3': 0.610088876086563}"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AGF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OC (Overlap coefficient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The overlap coefficient, or Szymkiewicz–Simpson coefficient, is a similarity measure that measures the overlap between two finite sets. It is defined as the size of the intersection divided by the smaller of the size of the two sets [[52]](#ref52).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Overlap_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OC=\\frac{TP}{min(TOP,P)}=max(PPV,TPR)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.0, 'L2': 0.5, 'L3': 0.6}"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.OC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BB (Braun-Blanquet similarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Braun-Blanquet coefficient is a similarity measure that is mostly used in botany. It is defined as the size of the intersection divided by the larger of the size of the two sets [[82]](#ref82) [[83]](#ref83)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BB=\\frac{TP}{max(TOP,P)}=min(PPV,TPR)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6, 'L2': 0.3333333333333333, 'L3': 0.5}"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.BB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOC (Otsuka-Ochiai coefficient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In biology, there is a similarity index, known as the Otsuka-Ochiai coefficient named after Yanosuke Otsuka and Akira Ochiai, also known as the Ochiai-Barkman or Ochiai coefficient. If sets are represented as bit vectors, the Otsuka-Ochiai coefficient can be seen to be the same as the cosine similarity [[53]](#ref53).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Yanosuke_Otsuka\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OOC=\\frac{TP}{\\sqrt{TOP\\times P}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7745966692414834, 'L2': 0.4082482904638631, 'L3': 0.5477225575051661}"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.OOC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TI (Tversky index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Tversky index, named after Amos Tversky, is an asymmetric similarity measure on sets that compares a variant to a prototype. The Tversky index can be seen as a generalization of Dice's coefficient and Tanimoto coefficient [[54]](#ref54).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Tversky_index\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TI(\\alpha,\\beta)=\\frac{TP}{TP+\\alpha FN+\\beta FP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.42857142857142855, 'L2': 0.1111111111111111, 'L3': 0.1875}"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TI(2,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `alpha` : alpha coefficient (type : `float`)\n",
    "2. `beta` : beta coefficient (type : `float`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`{class1: TI1, class2: TI2, ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUPR (Area under the PR curve)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A PR curve is plotting precision against recall. The precision recall area under curve (AUPR) is just the area under the PR curve. The higher it is, the better the model is [[55]](#ref55) [[56]](#ref56).\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUPR=\\frac{TPR+PPV}{2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8, 'L2': 0.41666666666666663, 'L3': 0.55}"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.4</span> </li>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  this is an approximate calculation of AUPR </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICSI (Individual classification success index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Individual Classification Success Index (ICSI), is a\n",
    "class-specific symmetric measure defined for classification\n",
    "assessment purpose. ICSI is hence $ 1 $ minus the sum of type I and type II errors.\n",
    "It ranges from $ -1 $ (both errors are maximal, i.e. $ 1 $) to $ 1 $ (both\n",
    "errors are minimal, i.e. $ 0 $), but the value $ 0 $ does not have any\n",
    "clear meaning. The measure is symmetric, and linearly related\n",
    "to the arithmetic mean of TPR and PPV [[58]](#ref58)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ICSI=PPV+TPR-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': -0.16666666666666674,\n",
       " 'L3': 0.10000000000000009}"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ICSI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CI (Confidence interval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, a confidence interval (CI) is a type of interval estimate (of a population parameter) that is computed from the observed data. The confidence level is the frequency (i.e., the proportion) of possible confidence intervals that contain the true value of their corresponding parameter. In other words, if confidence intervals are constructed using a given confidence level in an infinite number of independent experiments, the proportion of those intervals that contain the true value of the parameter will match the confidence level [[31]](#ref31).\n",
    "\n",
    "Supported statistics : `ACC`,`AUC`,`PRE`,`Overall ACC`,`Kappa`,`TPR`,`TNR`,`PPV`,`NPV`,`PLR`,`NLR`\n",
    "\n",
    "Supported alpha values (two-sided) : 0.001, 0.002, 0.01, 0.02, 0.05, 0.1, 0.2\n",
    "\n",
    "Supported alpha values (one-sided) : 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confidence intervals for `TPR`,`TNR`,`PPV`,`NPV`,`ACC`,`PRE` and `Overall ACC` are calculated using the normal approximation to the binomial distribution [[59]](#ref59), Wilson score [[62]](#ref62) and Agresti-Coull method [[63]](#ref63): \n",
    "\n",
    "#### Normal approximation\n",
    "\n",
    "$$SE=\\sqrt{\\frac{\\hat{p}(1-\\hat{p})}{n}}$$\n",
    "\n",
    "$$CI=\\hat{p}\\pm z\\times SE$$\n",
    "\n",
    "$$n=\\begin{cases}P & \\hat{p} == TPR/FNR\\\\N & \\hat{p} == TNR/FPR\\\\TOP & \\hat{p} == PPV\\\\TON & \\hat{p} ==NPV \\\\POP& \\hat{p} == ACC/ACC_{Overall}\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Wilson score\n",
    "\n",
    "$$CI=\\frac{\\hat{p}+\\frac{z^2}{2n}}{1+\\frac{z^2}{n}}\\pm\\frac{z}{1+\\frac{z^2}{n}}\\sqrt{\\frac{\\hat{p}(1-\\hat{p})}{n}+\\frac{z^2}{4n^2}}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Agresti-Coull\n",
    "\n",
    "$$\\hat{p}=\\frac{x}{n}$$\n",
    "\n",
    "$$\\tilde{p}=\\frac{x+\\frac{z^2}{2}}{n+z^2}$$\n",
    "\n",
    "$$CI =\\tilde{p}\\pm\\sqrt{\\frac{\\tilde{p}(1-\\tilde{p})}{n+z^2}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confidence interval for `Kappa` are calculated using Fleiss formula [[24]](#ref24) [[38]](#ref38) :\n",
    "\n",
    "$$SE_{Kappa}=\\sqrt{\\frac{ACC_{Overall}\\times (1-RACC_{Overall})}{(1-RACC_{Overall})^2}}$$\n",
    "\n",
    "$$CI_{Kappa}=Kappa\\pm z\\times SE_{Kappa}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confidence intervals for `NLR` and `PLR` are calculated using the log method [[60]](#ref60) :\n",
    "\n",
    "$$SE_{LR}=\\sqrt{\\frac{1}{a}-\\frac{1}{b}+\\frac{1}{c}-\\frac{1}{d}}$$\n",
    "\n",
    "$$CI_{LR}=e^{ln(LR)\\pm z\\times SE_{LR}}$$\n",
    "\n",
    "$$PLR:\\begin{cases}a=TP\\\\b=P\\\\c=FP\\\\d=N\\end{cases}$$\n",
    "\n",
    "$$NLR:\\begin{cases}a=FN\\\\b=P\\\\c=TN\\\\d=N\\end{cases}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confidence interval for `AUC` is calculated using Hanley and McNeil formula [[61]](#ref61) :\n",
    "\n",
    "$$SE_{AUC}=\\sqrt{\\frac{q_0+(N-1)q_1+(P-1)q_2}{N\\times P}}$$\n",
    "\n",
    "$$q_0=AUC(1-AUC)$$\n",
    "\n",
    "$$q_1=\\frac{AUC}{2-AUC}-AUC^2$$\n",
    "\n",
    "$$q_2=\\frac{2AUC^2}{1+AUC}-AUC^2$$\n",
    "\n",
    "$$CI_{AUC}=AUC\\pm z\\times SE_{AUC}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': [0.21908902300206645, (0.17058551491594975, 1.0294144850840503)],\n",
       " 'L2': [0.3535533905932738, (-0.19296464556281656, 1.1929646455628165)],\n",
       " 'L3': [0.21908902300206645, (0.17058551491594975, 1.0294144850840503)]}"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI(\"TPR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': [0.21908902300206645, (-0.2769850810763853, 1.0769850810763852)],\n",
       " 'L2': [0.3535533905932738, (-0.5924799769332159, 1.5924799769332159)],\n",
       " 'L3': [0.21908902300206645, (-0.2769850810763853, 1.0769850810763852)]}"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI(\"FNR\", alpha=0.001, one_sided=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': [0.14231876063832774, (0.19325746190524654, 0.6804926643446272)],\n",
       " 'L2': [0.10758287072798381, (0.04696414761482223, 0.44803635738467273)],\n",
       " 'L3': [0.14231876063832774, (0.19325746190524654, 0.6804926643446272)]}"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI(\"PRE\", alpha=0.05, binom_method=\"wilson\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.14231876063832777, (0.2805568916340536, 0.8343177950165198)]"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI(\"Overall ACC\", alpha=0.02, binom_method=\"agresti-coull\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.14231876063832777, (0.30438856248221097, 0.8622781041844558)]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI(\"Overall ACC\", alpha=0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `param` : input parameter (type : `str`)\n",
    "2. `alpha` : type I error (type : `float`, default : `0.05`)\n",
    "3. `one_sided` : one-sided mode flag (type : `bool`, default : `False`)\n",
    "4. `binom_method` : binomial confidence intervals method (type : `str`, default : `normal-approx`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Two-sided : `{class1: [SE1, (Lower CI, Upper CI)], ...}`\n",
    "2. One-sided : `{class1: [SE1, (Lower one-sided CI, Upper one-sided CI)], ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  For more information visit <a href=\"#Example-8-(Confidence-interval)\">Example 8</a></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NB (Net benefit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NB is a weighted sum of true positive\n",
    "classifications with compensation for false positive classifications by giving\n",
    "these a weight $ w $ [[64]](#ref64) [[65]](#ref65)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NB=\\frac{TP-w\\times FP}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vickers and Elkin (2006) suggested considering a range of thresholds and\n",
    "calculating the NB across these thresholds. The results can be plotted in a\n",
    "decision curve [[66]](#ref66)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_t=threshold$$\n",
    "$$w=\\frac{p_t}{1-p_t}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.25, 'L2': 0.0735, 'L3': 0.23525}"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NB(w=0.059)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `w` : weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`{class1: NB1, class2: NB2, ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Average"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here \"average\" refers to the arithmetic mean, the sum of the numbers divided by how many numbers are being averaged.\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Average\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6111111111111112"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.average(\"PPV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5651515151515151"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.average(\"F1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0000000000000004"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.average(\"DOR\", none_omit=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `param` : input parameter (type : `str`)\n",
    "2. `none_omit` : none items omitting flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Average`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Weighted average"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The weighted average is similar to an ordinary average, except that instead of each of the data points contributing equally to the final average, some data points contribute more than others.\n",
    "\n",
    "Default weight is **condition positive** (number of positive samples).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Weighted_arithmetic_mean\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6805555555555555"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_average(\"PPV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.606439393939394"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_average(\"F1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.5714285714285716"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_average(\"DOR\", none_omit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7152097902097901"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_average(\"F1\", weight={\"L1\": 23, \"L2\": 2, \"L3\": 1})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `param` : input parameter (type : `str`)\n",
    "2. `weight` : explicitly passes weights (type : `dict`, default : `None`)\n",
    "3. `none_omit` : none items omitting flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Weighted average`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sensitivity index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sensitivity index or d′ is a statistic used in signal detection theory. It provides the separation between the means of the signal and the noise distributions, compared against the standard deviation of the signal or noise distribution.\n",
    "d′ can be estimated from the observed hit rate and false-alarm rate, as follows [[76]](#ref76):"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$d^{\\prime}=Z(TPR) - Z(FPR)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function Z(p), p ∈ [0,1], is the inverse of the cumulative distribution function of the Gaussian distribution.\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Sensitivity_index\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 0.8416212335729143, 'L3': 0.4333594729285047}"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.sensitivity_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`{class1: SI1, class2: SI2, ...}`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HD (Hamming distance)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In information theory, the Hamming distance between two strings of equal length is the number of positions at which the corresponding symbols are different. In other words, it measures the minimum number of substitutions required to change one string into the other, or the minimum number of errors that could have transformed one string into the other. In a more general context, the Hamming distance is one of several string metrics for measuring the edit distance between two sequences. It is named after the American mathematician Richard Hamming [[80]](#ref80) [[81]](#ref81).\n",
    "\n",
    "A major application is in coding theory, more specifically to block codes, in which the equal-length strings are vectors over a finite field."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://en.wikipedia.org/wiki/Hamming_distance\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$HD = FN + FP$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 2, 'L2': 3, 'L3': 5}"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.HD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kappa is a statistic that measures inter-rater agreement for qualitative (categorical) items. It is generally thought to be a more robust measure than simple percent agreement calculation, as kappa takes into account the possibility of the agreement occurring by chance [[24]](#ref24).\n",
    "\n",
    "<a href=\"#SOA1-(Landis-&-Koch's-benchmark)\">Benchmark1</a>\n",
    "<a href=\"#SOA2-(Fleiss'-benchmark)\">Benchmark2</a>\n",
    "<a href=\"#SOA3-(Altman's-benchmark)\">Benchmark3</a>\n",
    "<a href=\"#SOA4-(Cicchetti's-benchmark)\">Benchmark4</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cohen%27s_kappa\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa=\\frac{ACC_{Overall}-RACC_{Overall}}{1-RACC_{Overall}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35483870967741943"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa unbiased"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The unbiased kappa value is defined in terms of total accuracy and a slightly different computation of expected likelihood that averages the reference and response probabilities [[25]](#ref25).\n",
    "\n",
    "Equals to [Scott's Pi](#Scott's-Pi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa_{Unbiased}=\\frac{ACC_{Overall}-RACCU_{Overall}}{1-RACCU_{Overall}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34426229508196726"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KappaUnbiased"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa no prevalence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prevalence-adjusted and bias-adjusted kappa (PABAK) [[14]](#ref14)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa_{NoPrevalence}=2 \\times ACC_{Overall}-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16666666666666674"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KappaNoPrevalence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The weighted kappa allows disagreements to be weighted differently and is especially useful when codes are ordered. Three matrices are involved, the matrix of observed scores, the matrix of expected scores based on chance agreement, and the weight matrix [[70]](#ref70) [[71]](#ref71)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$v_{ij}=1-\\frac{w_{ij}}{max(w)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_e=\\sum_{i,j=1}^{|C|}\\frac{TOP_i \\times P_j}{POP^2}\\times v_{ij}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_a=\\sum_{i,j=1}^{|C|}\\frac{Matrix(i,j)}{POP}\\times v_{ij}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa_{Weighted}=\\frac{P_a-P_e}{1-P_e}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.39130434782608675"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_kappa(\n",
    "    weight={\n",
    "        \"L1\": {\"L1\": 0, \"L2\": 1, \"L3\": 2},\n",
    "        \"L2\": {\"L1\": 1, \"L2\": 0, \"L3\": 1},\n",
    "        \"L3\": {\"L1\": 2, \"L2\": 1, \"L3\": 0}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-4.2-py3.5.egg\\pycm\\cm.py:850: RuntimeWarning: Invalid weight format; the result is for unweighted kappa.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.35483870967741943"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_kappa()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `weight` : weight matrix (type : `dict`, default : `None`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Weighted kappa`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa standard error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard error(s) of the Kappa coefficient was obtained by Fleiss (1969) [[24]](#ref24) [[38]](#ref38)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$SE_{Kappa}=\\sqrt{\\frac{ACC_{Overall}\\times (1-RACC_{Overall})}{(1-RACC_{Overall})^2}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2203645326012817"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa_SE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa 95% CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kappa 95% Confidence Interval [[24]](#ref24) [[38]](#ref38)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CI_{Kappa}=Kappa \\pm 1.96\\times SE_{Kappa}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.07707577422109269, 0.7867531935759315)"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa_CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chi-squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pearson's chi-squared test is a statistical test applied to sets of categorical data to evaluate how likely it is that any observed difference between the sets arose by chance. It is suitable for unpaired data from large samples [[10]](#ref10).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Chi-squared_test\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\chi^2=\\sum_{i=1}^{|C|}\\sum_{j=1}^{|C|}\\frac{\\Big(Matrix(i,j)-E(i,j)\\Big)^2}{E(i,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$E(i,j)=\\frac{TOP_j\\times P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.6000000000000005"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Chi_Squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chi-squared DF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of degrees of freedom of this confusion matrix for the chi-squared statistic [[10]](#ref10)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DF=(|C|-1)^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phi-squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, the phi coefficient (or mean square contingency coefficient) is a measure of association for two binary variables. Introduced by Karl Pearson, this measure is similar to the Pearson correlation coefficient in its interpretation. In fact, a Pearson correlation coefficient estimated for two binary variables will return the phi coefficient [[10]](#ref10).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Phi_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\phi^2=\\frac{\\chi^2}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.55"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Phi_Squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cramer's V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, Cramér's V (sometimes referred to as Cramér's phi) is a measure of association between two nominal variables, giving a value between $ 0 $ and $ +1 $ (inclusive). It is based on Pearson's chi-squared statistic and was published by Harald Cramér in 1946 [[26]](#ref26).\n",
    "\n",
    "<a href=\"#SOA5-(Cramer's-benchmark)\">Benchmark</a>\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$V=\\sqrt{\\frac{\\phi^2}{|C|-1}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5244044240850758"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard error (SE) of a statistic (usually an estimate of a parameter) is the standard deviation of its sampling distribution or an estimate of that standard deviation [[31]](#ref31).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Standard_error\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$SE_{ACC}=\\sqrt{\\frac{ACC\\times (1-ACC)}{POP}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14231876063832777"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 95% CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, a confidence interval (CI) is a type of interval estimate (of a population parameter) that is computed from the observed data. The confidence level is the frequency (i.e., the proportion) of possible confidence intervals that contain the true value of their corresponding parameter. In other words, if confidence intervals are constructed using a given confidence level in an infinite number of independent experiments, the proportion of those intervals that contain the true value of the parameter will match the confidence level [[31]](#ref31).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Confidence_interval\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CI=ACC \\pm 1.96\\times SE_{ACC}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.30438856248221097, 0.8622781041844558)"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI95"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>CI</code> renamed to <code>CI95</code> in <span style=\"color:red;\">version 2.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bennett's S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bennett, Alpert & Goldstein’s S is a statistical measure of inter-rater agreement. It was created by Bennett et al. in 1954.\n",
    "Bennett et al. suggested adjusting inter-rater reliability to accommodate the percentage of rater agreement that might be expected by chance was a better measure than a simple agreement between raters [[8]](#ref8).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Bennett,_Alpert,_and_Goldstein%E2%80%99s_S\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\frac{1}{|C|}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$S=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37500000000000006"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scott's Pi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scott's pi (named after William A. Scott) is a statistic for measuring inter-rater reliability for nominal data in communication studies. Textual entities are annotated with categories by different annotators, and various measures are used to assess the extent of agreement between the annotators, one of which is Scott's pi. Since automatically annotating text is a popular problem in natural language processing, and the goal is to get the computer program that is being developed to agree with the humans in the annotations it creates, assessing the extent to which humans agree with each other is important for establishing a reasonable upper limit on computer performance [[7]](#ref7).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Scott%27s_Pi\">Wikipedia page</a>\n",
    "\n",
    "\n",
    "Equals to [Kappa Unbiased](#Kappa-unbiased)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\sum_{i=1}^{|C|}(\\frac{TOP_i + P_i}{2\\times POP})^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\pi=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34426229508196726"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gwet's AC1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AC1 was originally introduced by Gwet in 2001 (Gwet, 2001). The interpretation of AC1 is similar to generalized kappa (Fleiss, 1971), which is used to assess inter-rater reliability when there are multiple raters. Gwet (2002) demonstrated that AC1 can overcome the limitations that kappa is sensitive to trait prevalence and rater's classification probabilities (i.e., marginal probabilities), whereas AC1 provides more robust measure of inter-rater reliability [[6]](#ref6)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\pi_i=\\frac{TOP_i + P_i}{2\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\frac{1}{|C|-1}\\sum_{i=1}^{|C|}\\Big(\\pi_i\\times (1-\\pi_i)\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AC_1=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3893129770992367"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AC1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the decision problem itself as defined by the counts for the reference. The entropy of a distribution is the average negative log probability of outcomes [[30]](#ref30)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Reference}=-\\sum_{i=1}^{|C|}Likelihood_{Reference}(i)\\times\\log_{2}{Likelihood_{Reference}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4833557549816874"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ReferenceEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Response entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the response distribution. The entropy of a distribution is the average negative log probability of outcomes [[30]](#ref30)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Response}=-\\sum_{i=1}^{|C|}Likelihood_{Response}(i)\\times\\log_{2}{Likelihood_{Response}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ResponseEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cross-entropy of the response distribution against the reference distribution. The cross-entropy is defined by the negative log probabilities of the response distribution weighted by the reference distribution [[30]](#ref30).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cross_entropy\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Cross}=-\\sum_{i=1}^{|C|}Likelihood_{Reference}(i)\\times\\log_{2}{Likelihood_{Response}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5833333333333335"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CrossEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Joint entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the joint reference and response distribution as defined by the underlying matrix [[30]](#ref30)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(i,j)=\\frac{Matrix(i,j)}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Joint}=-\\sum_{i=1}^{|C|}\\sum_{j=1}^{|C|}P^{'}(i,j)\\times\\log_{2}{P^{'}(i,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.4591479170272446"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.JointEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conditional entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the distribution of categories in the response given that the reference category was as specified [[30]](#ref30).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Conditional_entropy\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(j|i)=\\frac{Matrix(j,i)}{P_i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Conditional}=\\sum_{i=1}^{|C|}\\Bigg(Likelihood_{Reference}(i)\\times\\Big(-\\sum_{j=1}^{|C|}P^{'}(j|i)\\times\\log_{2}{P^{'}(j|i)}\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9757921620455572"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ConditionalEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kullback-Leibler divergence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In mathematical statistics, the Kullback–Leibler divergence (also called relative entropy) is a measure of how one probability distribution diverges from a second, expected probability distribution  [[11]](#ref11) [[30]](#ref30).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Divergence=-\\sum_{i=1}^{|C|}Likelihood_{Reference}\\times\\log_{2}{\\frac{Likelihood_{Reference}}{Likelihood_{Response}}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09997757835164581"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mutual information is defined as Kullback-Leibler divergence, between the product of the individual distributions and the joint distribution.\n",
    "Mutual information is symmetric. We could also subtract the conditional entropy of the reference given the response from the reference entropy to get the same result [[11]](#ref11) [[30]](#ref30).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Mutual_information\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(i,j)=\\frac{Matrix(i,j)}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MI=-\\sum_{i=1}^{|C|}\\sum_{j=1}^{|C|}P^{'}(i,j)\\times\\log_{2}\\Big({\\frac{P^{'}(i,j)}{Likelihood_{Reference}(i)\\times Likelihood_{Response}(i) }\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MI=Entropy_{Response}-Entropy_{Conditional}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5242078379544428"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MutualInformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goodman & Kruskal's lambda A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In probability theory and statistics, Goodman & Kruskal's lambda  is a measure of proportional reduction in error in cross tabulation analysis [[12]](#ref12).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Goodman_and_Kruskal%27s_lambda\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\lambda_A=\\frac{\\sum_{j=1}^{|C|}Max\\Big(Matrix(-,j)\\Big)-Max(P)}{POP-Max(P)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42857142857142855"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LambdaA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goodman & Kruskal's lambda B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In probability theory and statistics, Goodman & Kruskal's lambda  is a measure of proportional reduction in error in cross tabulation analysis [[13]](#ref13).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Goodman_and_Kruskal%27s_lambda\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\lambda_B=\\frac{\\sum_{i=1}^{|C|}Max\\Big(Matrix(i,-)\\Big)-Max(TOP)}{POP-Max(TOP)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16666666666666666"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LambdaB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA1 (Landis & Koch's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[1]](#ref1)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0 - 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:orangered;\">Slight</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.2 – 0.4</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.4 – 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.6 – 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Substantial</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.8 – 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Almost perfect</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fair'"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA2 (Fleiss' benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[4]](#ref4)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.40 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.40 - 0.75</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Intermediate to Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">More than 0.75</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Poor'"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA3 (Altman's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[5]](#ref5)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.2 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.2 – 0.4</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.4 – 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.6 – 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.8 – 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Very Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fair'"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA4 (Cicchetti's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[9]](#ref9)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.40 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.40 – 0.59</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.59 – 0.74</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.74 – 1.00</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Poor'"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA5 (Cramer's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[47]](#ref47)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Cramer's V</td>\n",
    "           <td style=\"text-align:center\">Strength of Association</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.1 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Negligible</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.1 – 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.2 – 0.4</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.4 – 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:yellowgreen;\">Relatively Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.6 – 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.8 – 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Very Strong</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Relatively Strong'"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA6 (Matthews's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MCC is a confusion matrix method of calculating the Pearson product-moment correlation coefficient (not to be confused with Pearson's C). Therefore, it has the same interpretation [[2]](#ref2).\n",
    "\n",
    "For more information visit [[49]](#ref49)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Overall MCC</td>\n",
    "           <td style=\"text-align:center\">Strength of Association</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.3 ></td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.3 - 0.5</td>\n",
    "           <td style=\"text-align:center;background-color:orange;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.5 - 0.7</td>\n",
    "           <td style=\"text-align:center;background-color:yellow;\">Moderate</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.7 - 0.9</td>\n",
    "           <td style=\"text-align:center;background-color:lawngreen;\">Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.9 - 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:green;\">Very Strong</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Weak'"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  only positive values are considered</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA7 (Goodman & Kruskal's lambda A benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[84]](#ref84)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Lambda A</td>\n",
    "           <td style=\"text-align:center\">Strength of Association</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0 - 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Very Weak</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.2 - 0.4</td>\n",
    "           <td style=\"text-align:center;background-color:OrangeRed;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.4 - 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:Orange;\">Moderate</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.6 - 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:Yellow;\">Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.8 - 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:LawnGreen;\">Very Strong</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">1.0</td>\n",
    "           <td style=\"text-align:center;background-color:Green;\">Perfect</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Moderate'"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA8 (Goodman & Kruskal's lambda B benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[84]](#ref84)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Lambda B</td>\n",
    "           <td style=\"text-align:center\">Strength of Association</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0 - 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:red;\">Very Weak</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.2 - 0.4</td>\n",
    "           <td style=\"text-align:center;background-color:OrangeRed;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.4 - 0.6</td>\n",
    "           <td style=\"text-align:center;background-color:Orange;\">Moderate</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.6 - 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:Yellow;\">Strong</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.8 - 1.0</td>\n",
    "           <td style=\"text-align:center;background-color:LawnGreen;\">Very Strong</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">1.0</td>\n",
    "           <td style=\"text-align:center;background-color:Green;\">Perfect</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Very Weak'"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA9 (Krippendorff's alpha benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[85]](#ref85)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Alpha</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.667 ></td>\n",
    "           <td style=\"text-align:center;background-color:Red;\">Low</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.667 - 0.8</td>\n",
    "           <td style=\"text-align:center;background-color:LawnGreen;\">Tentative</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.8 < </td>\n",
    "           <td style=\"text-align:center;background-color:Green;\">High</td>\n",
    "       </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Low'"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA10 (Pearson's C benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[86]](#ref86)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">C</td>\n",
    "           <td style=\"text-align:center\">Strength of Association</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0 - 0.1</td>\n",
    "           <td style=\"text-align:center;background-color:Red;\">Not Appreciable</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.1 - 0.2</td>\n",
    "           <td style=\"text-align:center;background-color:Orange;\">Weak</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.2 - 0.3</td>\n",
    "           <td style=\"text-align:center;background-color:LawnGreen;\">Medium</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.3 < </td>\n",
    "           <td style=\"text-align:center;background-color:Green;\">Strong</td>\n",
    "       </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Strong'"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ACC_{Overall}=\\frac{\\sum_{i=1}^{|C|}TP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Equals to [TPR Micro](#TPR_Micro), [F1 Micro](#F1_Micro) and [PPV Micro](#PPV_Micro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[24]](#ref24)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACC_{Overall}=\\sum_{i=1}^{|C|}RACC_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3541666666666667"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[25]](#ref25)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACCU_{Overall}=\\sum_{i=1}^{|C|}RACCU_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3645833333333333"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV_{Micro}=\\frac{\\sum_{i=1}^{|C|}TP_i}{\\sum_{i=1}^{|C|}TP_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Equals to [TPR Micro](#TPR_Micro), [F1 Micro](#F1_Micro) and [Overall ACC](#Overall_ACC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NPV_{Micro}=\\frac{\\sum_{i=1}^{|C|}TN_i}{\\sum_{i=1}^{|C|}TN_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7916666666666666"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR_{Micro}=\\frac{\\sum_{i=1}^{|C|}TP_i}{\\sum_{i=1}^{|C|}TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Equals to [PPV Micro](#PPV_Micro), [F1 Micro](#F1_Micro) and [Overall ACC](#Overall_ACC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TNR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TNR_{Micro}=\\frac{\\sum_{i=1}^{|C|}TN_i}{\\sum_{i=1}^{|C|}TN_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7916666666666666"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TNR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FPR_{Micro}=\\frac{\\sum_{i=1}^{|C|}FP_i}{\\sum_{i=1}^{|C|}TN_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20833333333333337"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FNR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FNR_{Micro}=\\frac{\\sum_{i=1}^{|C|}FN_i}{\\sum_{i=1}^{|C|}TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41666666666666663"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FNR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F1_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$F_{1_{Micro}}=2\\frac{\\sum_{i=1}^{|C|}TPR_i\\times PPV_i}{\\sum_{i=1}^{|C|}TPR_i+PPV_i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Equals to [PPV Micro](#PPV_Micro), [TPR Micro](#TPR_Micro) and [Overall ACC](#Overall_ACC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F1_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TP_i}{TP_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.611111111111111"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NPV_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TN_i}{TN_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7777777777777777"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TP_i}{TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5666666666666668"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TNR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TNR_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TN_i}{TN_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7904761904761904"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TNR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FPR_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{FP_i}{TN_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20952380952380956"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FNR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FNR_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{FN_i}{TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.43333333333333324"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FNR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F1_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$F_{1_{Macro}}=\\frac{2}{|C|}\\sum_{i=1}^{|C|}\\frac{TPR_i\\times PPV_i}{TPR_i+PPV_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5651515151515151"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F1_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ACC_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#ref3)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ACC_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}{ACC_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7222222222222223"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ACC_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[29]](#ref29)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Mean}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}J_i$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Sum}=\\sum_{i=1}^{|C|}J_i$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Overall}=(J_{Sum},J_{Mean})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.225, 0.4083333333333334)"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hamming loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The average Hamming loss or Hamming distance between two sets of samples [[31]](#ref31)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$L_{Hamming}=\\frac{1}{POP}\\sum_{i=1}^{POP}1(y_i \\neq \\widehat{y}_i)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41666666666666663"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.HammingLoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zero-one loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zero-one loss is a common  loss function used with  classification learning. It assigns $ 0 $ to loss for a correct classification and $ 1 $ for an incorrect classification [[31]](#ref31)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$L_{0-1}=\\sum_{i=1}^{POP}1(y_i \\neq \\widehat{y}_i)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ZeroOneLoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NIR (No information rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Largest class percentage in the data [[57]](#ref57)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NIR=\\frac{1}{POP}Max(P)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4166666666666667"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### P-Value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistical hypothesis testing, the p-value or probability value is, for a given statistical model, the probability that, when the null hypothesis is true, the statistical summary (such as the absolute value of the sample mean difference between two compared groups) would be greater than or equal to the actual observed results [[31]](#ref31) .    \n",
    "Here a one-sided binomial test to see if the accuracy is better than the no information rate [[57]](#ref57).\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/P-value\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$x=\\sum_{i=1}^{|C|}TP_{i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p=NIR$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$n=POP$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P-Value_{(ACC > NIR)}=1-\\sum_{i=1}^{x}\\left(\\begin{array}{c}n\\\\ i\\end{array}\\right)p^{i}(1-p)^{n-i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18926430237560654"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PValue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[17]](#ref17)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_j=\\frac{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)}{2\\sum_{k,l=1}^{|C|}Matrix(k,l)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CEN_{Overall}=\\sum_{j=1}^{|C|}P_jCEN_j$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4638112995385119"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[19]](#ref19)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha=\\begin{cases}1 & |C| > 2\\\\0 & |C| = 2\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_j=\\frac{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)-Matrix(j,j)}{2\\sum_{k,l=1}^{|C|}Matrix(k,l)-\\alpha \\sum_{k=1}^{|C|}Matrix(k,k)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCEN_{Overall}=\\sum_{j=1}^{|C|}P_jMCEN_j$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5189369467580801"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[20]](#ref20) [[27]](#ref27).\n",
    "\n",
    "<a href=\"#SOA6-(Matthews's-benchmark)\">Benchmark</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCC_{Overall}=\\frac{cov(X,Y)}{\\sqrt{cov(X,X)\\times cov(Y,Y)}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(X,Y)=\\sum_{i,j,k=1}^{|C|}\\Big(Matrix(i,i)Matrix(k,j)-Matrix(j,i)Matrix(i,k)\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(X,X) = \\sum_{i=1}^{|C|}\\Bigg[\\Big(\\sum_{j=1}^{|C|}Matrix(j,i)\\Big)\\Big(\\sum_{k,l=1,k\\neq i}^{|C|}Matrix(l,k)\\Big)\\Bigg]$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(Y,Y) = \\sum_{i=1}^{|C|}\\Bigg[\\Big(\\sum_{j=1}^{|C|}Matrix(i,j)\\Big)\\Big(\\sum_{k,l=1,k\\neq i}^{|C|}Matrix(k,l)\\Big)\\Bigg]$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36666666666666664"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RR (Global performance index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[21]](#ref21)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RR=\\frac{1}{|C|}\\sum_{i,j=1}^{|C|}Matrix(i,j)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CBA (Class balance accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As an evaluation tool, CBA creates an overall assessment of\n",
    "model predictive power by scrutinizing measures simultaneously across each class in a conservative manner that guarantees that a model’s ability to recall observations from each class and\n",
    "its ability to do so efficiently won’t fall below the bound [[22]](#ref22) [[51]](#ref51)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CBA=\\frac{\\sum_{i=1}^{|C|}\\frac{Matrix(i,i)}{Max(TOP_i,P_i)}}{|C|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4777777777777778"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CBA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUNU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When dealing with multiclass problems, a global measure of classification performances based on the ROC approach (AUNU) has been proposed as the average of single-class measures [[23]](#ref23)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUNU=\\frac{\\sum_{i=1}^{|C|}AUC_i}{|C|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6785714285714285"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUNU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUNP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another option (AUNP) is that of averaging the $ AUC_i $ values with weights proportional to the number of samples experimentally belonging to each class, that is, the a priori class distribution [[23]](#ref23)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUNP=\\sum_{i=1}^{|C|}\\frac{P_i}{POP}AUC_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6857142857142857"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUNP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RCI (Relative classifier information)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performance of different classifiers on the same domain can be measured by\n",
    "comparing relative classifier information while classifier information (mutual information) can be used for comparison across different decision problems [[32]](#ref32) [[22]](#ref22)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_d=-\\sum_{i=1}^{|C|}\\Big(\\frac{\\sum_{l=1}^{|C|}Matrix(i,l)}{\\sum_{h,k=1}^{|C|}Matrix(h,k)}log_2\\frac{\\sum_{l=1}^{|C|}Matrix(i,l)}{\\sum_{h,k=1}^{|C|}Matrix(h,k)}\\Big)=Entropy_{Reference}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_o=\\sum_{j=1}^{|C|}\\Big(\\frac{\\sum_{k=1}^{|C|}Matrix(k,j)}{\\sum_{h,l=0}^{|C|}Matrix(h,l)}H_{oj}\\Big)=Entropy_{Conditional}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_{oj}=-\\sum_{i=1}^{|C|}\\Big(\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}Matrix(k,j)}log_2\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}Matrix(k,j)}\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RCI=\\frac{H_d-H_o}{H_d}=\\frac{MI}{Entropy_{Reference}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3533932006492363"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pearson's C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The contingency coefficient is a coefficient of association that tells whether two variables or data sets are independent or dependent of/on each other. It is also known as Pearson’s coefficient (not to be confused with Pearson’s coefficient of skewness) [[43]](#ref43) [[44]](#ref44)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$C=\\sqrt{\\frac{\\chi^2}{\\chi^2+POP}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5956833971812706"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CSI (Classification success index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Classification Success Index (CSI) is an overall\n",
    "measure defined by averaging ICSI over all classes [[58]](#ref58)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CSI=\\frac{1}{|C|}\\sum_{i=1}^{|C|}{ICSI_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1777777777777778"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CSI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ARI (Adjusted Rand index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Rand index or Rand measure (named after William M. Rand) in statistics, and in particular in data clustering, is a measure of the similarity between two data clusterings. A form of the Rand index may be defined that is adjusted for the chance grouping of elements, this is the adjusted Rand index. From a mathematical standpoint, Rand index is related to the accuracy, but is applicable even when class labels are not used [[68]](#ref68).\n",
    "\n",
    "The Adjusted Rand Index (ARI) is frequently used in cluster validation since it is a measure of agreement between two partitions: one given by the clustering process and the other defined by external criteria, but it can also be used in supervised learning [[69]](#ref69).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$X=\\frac{\\sum_{i}C_{2}^{P_i}\\times \\sum_{j}C_{2}^{TOP_j}}{C_2^{POP}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ARI=\\frac{\\sum_{i,j}C_{2}^{Matrix(i,j)}-X}{\\frac{1}{2}[\\sum_{i}C_{2}^{P_i} + \\sum_{j}C_{2}^{TOP_j}]-X}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09206349206349207"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ARI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  $ C_{r}^{n} $ is the number of combinations of $ n $ objects taken $ r $</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bangdiwala's B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bangdiwala's B statistic was created by Shrikant Bangdiwala in 1985 and is a measure of inter-rater agreement. While not as commonly used as the kappa statistic the B test has been used by various workers [[72]](#ref72) [[73]](#ref73).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Bangdiwala%27s_B\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$B=\\frac{\\sum_{i=1}^{|C|}TP_i^2}{\\sum_{i=1}^{|C|}TOP_i\\times P_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37254901960784315"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Krippendorff's alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Krippendorff's alpha coefficient, named after academic Klaus Krippendorff, is a statistical measure of the agreement achieved when coding a set of units of analysis in terms of the values of a variable.\n",
    "Krippendorff's alpha generalizes several known statistics, often called measures of inter-coder agreement, inter-rater reliability, reliability of coding given sets of units (as distinct from unitizing) but it also distinguishes itself from statistics that are called reliability coefficients but are unsuitable to the particulars of coding data generated for subsequent analysis [[74]](#ref74).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Krippendorff%27s_alpha\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\epsilon = \\frac{1}{2\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_a=(1-\\epsilon)\\times ACC_{Overall}+\\epsilon$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_e=RACCU_{Overall}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha=\\frac{P_a-P_e}{1-P_e}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3715846994535519"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Weighted Krippendorff's alpha coefficient [[74]](#ref74)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\epsilon = \\frac{1}{2\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$v_{ij}=1-\\frac{w_{ij}}{max(w)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_e=\\sum_{i,j=1}^{|C|}(\\frac{TOP_i \\times P_j}{2 \\times POP})^2\\times v_{ij}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_a^*=\\sum_{i,j=1}^{|C|}\\frac{Matrix(i,j)}{POP}\\times v_{ij}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_a=(1-\\epsilon)\\times P_a^*+\\epsilon$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha_{Weighted}=\\frac{P_a-P_e}{1-P_e}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.374757281553398"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_alpha(\n",
    "    weight={\n",
    "        \"L1\": {\"L1\": 0, \"L2\": 1, \"L3\": 2},\n",
    "        \"L2\": {\"L1\": 1, \"L2\": 0, \"L3\": 1},\n",
    "        \"L3\": {\"L1\": 2, \"L2\": 1, \"L3\": 0}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-4.2-py3.5.egg\\pycm\\cm.py:873: RuntimeWarning: Invalid weight format; the result is for unweighted alpha.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.3715846994535519"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.weighted_alpha()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `weight` : weight matrix (type : `dict`, default : `None`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Weighted alpha`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Aickin's alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aickin's alpha coefficient [[75]](#ref75)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha^{(t+1)}=\\frac{p_a-p_e^{(t)}}{1-p_e^{(t)}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_e^{(t)}=\\sum_{k=1}^{|C|}p_{k|H}^{A(t)}\\times p_{k|H}^{B(t)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_a=ACC_{Overall}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_{k|H}^{A(t+1)}=\\frac{TOP_k}{(1-\\alpha^{(t)})+\\alpha^{(t)}\\times \\frac{p_{k|H}^{B(t)}}{p_e^{(t)}}\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_{k|H}^{B(t+1)}=\\frac{P_k}{(1-\\alpha^{(t)})+\\alpha^{(t)}\\times \\frac{p_{k|H}^{A(t)}}{p_e^{(t)}}\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Stop:|\\alpha^{(t+1)}-\\alpha^{(t)}|<\\epsilon$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.38540577344968524"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.aickin_alpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.38545857383594895"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.aickin_alpha(max_iter=2000, epsilon=0.00003)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `epsilon` : difference threshold (type : `float`, default : `0.0001`)\n",
    "2. `max_iter` : maximum number of iterations (type : `int`, default : `200`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Aickin's alpha`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Brier score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Brier Score is a strictly proper score function or strictly proper scoring rule that measures the accuracy of probabilistic predictions. For unidimensional predictions, it is strictly equivalent to the mean squared error as applied to predicted probabilities [[78]](#ref78) [[79]](#ref79).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Brier_score\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BS=\\frac{1}{N} \\sum_{t=1}^{N}(f_t-o_t)^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in which $f_t$ is the probability that was forecast, $o_t$ the actual outcome of the event at instance $t$ ($0$ if it does not happen and $1$ if it does happen) and $N$ is the number of forecasting instances."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03749999999999999"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_test = ConfusionMatrix([0, 1, 1, 0], [0.1, 0.9, 0.8, 0.3], threshold=lambda x: 1)\n",
    "cm_test.brier_score()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6875"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_test.brier_score(pos_class=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `pos_class` : positive class name (type : `int/str`, default : `None`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Brier score`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  This option only works in binary probability mode</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>pos_class</code> always defaults to the greater class name (i.e. <code>max(classes)</code>), unless, the <code>actual_vector</code> contains string. In that case, <code>pos_class</code> does not have any default value, and it must be explicitly specified or else an error will result.</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In information theory, the cross-entropy between two probability distributions \n",
    "$p$ and $q$ over the same underlying set of events measures the average number of bits needed to identify an event drawn from the set if a coding scheme used for the set is optimized for an estimated probability distribution $q$, rather than the true distribution $p$.\n",
    "This is also known as the log loss (logarithmic loss or logistic loss); the terms \"log loss\" and \"cross-entropy loss\" are used interchangeably. [[30]](#ref30).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cross_entropy\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$L_{\\log}(y, p) = -(y \\log (p) + (1 - y) \\log (1 - p))$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19763488164214868"
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_test.log_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.854645225687032"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm_test.log_loss(pos_class=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `pos_class` : positive class name (type : `int/str`, default : `None`)\n",
    "2. `normalize` : normalization flag (type : `bool`, default : `True`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Log loss`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 3.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  This option only works in binary probability mode</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>pos_class</code> always defaults to the greater class name (i.e. <code>max(classes)</code>), unless, the <code>actual_vector</code> contains string. In that case, <code>pos_class</code> does not have any default value, and it must be explicitly specified or else an error will result.</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       L2       L3       \n",
      "Actual\n",
      "L1       3        0        2        \n",
      "\n",
      "L2       0        1        1        \n",
      "\n",
      "L3       0        2        3        \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Overall Statistics : \n",
      "\n",
      "95% CI                                                            (0.30439,0.86228)\n",
      "ACC Macro                                                         0.72222\n",
      "ARI                                                               0.09206\n",
      "AUNP                                                              0.68571\n",
      "AUNU                                                              0.67857\n",
      "Bangdiwala B                                                      0.37255\n",
      "Bennett S                                                         0.375\n",
      "CBA                                                               0.47778\n",
      "CSI                                                               0.17778\n",
      "Chi-Squared                                                       6.6\n",
      "Chi-Squared DF                                                    4\n",
      "Conditional Entropy                                               0.97579\n",
      "Cramer V                                                          0.5244\n",
      "Cross Entropy                                                     1.58333\n",
      "F1 Macro                                                          0.56515\n",
      "F1 Micro                                                          0.58333\n",
      "FNR Macro                                                         0.43333\n",
      "FNR Micro                                                         0.41667\n",
      "FPR Macro                                                         0.20952\n",
      "FPR Micro                                                         0.20833\n",
      "Gwet AC1                                                          0.38931\n",
      "Hamming Loss                                                      0.41667\n",
      "Joint Entropy                                                     2.45915\n",
      "KL Divergence                                                     0.09998\n",
      "Kappa                                                             0.35484\n",
      "Kappa 95% CI                                                      (-0.07708,0.78675)\n",
      "Kappa No Prevalence                                               0.16667\n",
      "Kappa Standard Error                                              0.22036\n",
      "Kappa Unbiased                                                    0.34426\n",
      "Krippendorff Alpha                                                0.37158\n",
      "Lambda A                                                          0.42857\n",
      "Lambda B                                                          0.16667\n",
      "Mutual Information                                                0.52421\n",
      "NIR                                                               0.41667\n",
      "NPV Macro                                                         0.77778\n",
      "NPV Micro                                                         0.79167\n",
      "Overall ACC                                                       0.58333\n",
      "Overall CEN                                                       0.46381\n",
      "Overall J                                                         (1.225,0.40833)\n",
      "Overall MCC                                                       0.36667\n",
      "Overall MCEN                                                      0.51894\n",
      "Overall RACC                                                      0.35417\n",
      "Overall RACCU                                                     0.36458\n",
      "P-Value                                                           0.18926\n",
      "PPV Macro                                                         0.61111\n",
      "PPV Micro                                                         0.58333\n",
      "Pearson C                                                         0.59568\n",
      "Phi-Squared                                                       0.55\n",
      "RCI                                                               0.35339\n",
      "RR                                                                4.0\n",
      "Reference Entropy                                                 1.48336\n",
      "Response Entropy                                                  1.5\n",
      "SOA1(Landis & Koch)                                               Fair\n",
      "SOA2(Fleiss)                                                      Poor\n",
      "SOA3(Altman)                                                      Fair\n",
      "SOA4(Cicchetti)                                                   Poor\n",
      "SOA5(Cramer)                                                      Relatively Strong\n",
      "SOA6(Matthews)                                                    Weak\n",
      "SOA7(Lambda A)                                                    Moderate\n",
      "SOA8(Lambda B)                                                    Very Weak\n",
      "SOA9(Krippendorff Alpha)                                          Low\n",
      "SOA10(Pearson C)                                                  Strong\n",
      "Scott PI                                                          0.34426\n",
      "Standard Error                                                    0.14232\n",
      "TNR Macro                                                         0.79048\n",
      "TNR Micro                                                         0.79167\n",
      "TPR Macro                                                         0.56667\n",
      "TPR Micro                                                         0.58333\n",
      "Zero-one Loss                                                     5\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AGF(Adjusted F-score)                                             0.72859       0.62869       0.61009       \n",
      "AGM(Adjusted geometric mean)                                      0.85764       0.70861       0.58034       \n",
      "AM(Difference between automatic and manual classification)        -2            1             1             \n",
      "AUC(Area under the ROC curve)                                     0.8           0.65          0.58571       \n",
      "AUCI(AUC value interpretation)                                    Very Good     Fair          Poor          \n",
      "AUPR(Area under the PR curve)                                     0.8           0.41667       0.55          \n",
      "BB(Braun-Blanquet similarity)                                     0.6           0.33333       0.5           \n",
      "BCD(Bray-Curtis dissimilarity)                                    0.08333       0.04167       0.04167       \n",
      "BM(Informedness or bookmaker informedness)                        0.6           0.3           0.17143       \n",
      "CEN(Confusion entropy)                                            0.25          0.49658       0.60442       \n",
      "DOR(Diagnostic odds ratio)                                        None          4.0           2.0           \n",
      "DP(Discriminant power)                                            None          0.33193       0.16597       \n",
      "DPI(Discriminant power interpretation)                            None          Poor          Poor          \n",
      "ERR(Error rate)                                                   0.16667       0.25          0.41667       \n",
      "F0.5(F0.5 score)                                                  0.88235       0.35714       0.51724       \n",
      "F1(F1 score - harmonic mean of precision and sensitivity)         0.75          0.4           0.54545       \n",
      "F2(F2 score)                                                      0.65217       0.45455       0.57692       \n",
      "FDR(False discovery rate)                                         0.0           0.66667       0.5           \n",
      "FN(False negative/miss/type 2 error)                              2             1             2             \n",
      "FNR(Miss rate or false negative rate)                             0.4           0.5           0.4           \n",
      "FOR(False omission rate)                                          0.22222       0.11111       0.33333       \n",
      "FP(False positive/type 1 error/false alarm)                       0             2             3             \n",
      "FPR(Fall-out or false positive rate)                              0.0           0.2           0.42857       \n",
      "G(G-measure geometric mean of precision and sensitivity)          0.7746        0.40825       0.54772       \n",
      "GI(Gini index)                                                    0.6           0.3           0.17143       \n",
      "GM(G-mean geometric mean of specificity and sensitivity)          0.7746        0.63246       0.58554       \n",
      "HD(Hamming distance)                                              2             3             5             \n",
      "IBA(Index of balanced accuracy)                                   0.36          0.28          0.35265       \n",
      "ICSI(Individual classification success index)                     0.6           -0.16667      0.1           \n",
      "IS(Information score)                                             1.26303       1.0           0.26303       \n",
      "J(Jaccard index)                                                  0.6           0.25          0.375         \n",
      "LS(Lift score)                                                    2.4           2.0           1.2           \n",
      "MCC(Matthews correlation coefficient)                             0.68313       0.2582        0.16903       \n",
      "MCCI(Matthews correlation coefficient interpretation)             Moderate      Negligible    Negligible    \n",
      "MCEN(Modified confusion entropy)                                  0.26439       0.5           0.6875        \n",
      "MK(Markedness)                                                    0.77778       0.22222       0.16667       \n",
      "N(Condition negative)                                             7             10            7             \n",
      "NLR(Negative likelihood ratio)                                    0.4           0.625         0.7           \n",
      "NLRI(Negative likelihood ratio interpretation)                    Poor          Negligible    Negligible    \n",
      "NPV(Negative predictive value)                                    0.77778       0.88889       0.66667       \n",
      "OC(Overlap coefficient)                                           1.0           0.5           0.6           \n",
      "OOC(Otsuka-Ochiai coefficient)                                    0.7746        0.40825       0.54772       \n",
      "OP(Optimized precision)                                           0.58333       0.51923       0.55894       \n",
      "P(Condition positive or support)                                  5             2             5             \n",
      "PLR(Positive likelihood ratio)                                    None          2.5           1.4           \n",
      "PLRI(Positive likelihood ratio interpretation)                    None          Poor          Poor          \n",
      "POP(Population)                                                   12            12            12            \n",
      "PPV(Precision or positive predictive value)                       1.0           0.33333       0.5           \n",
      "PRE(Prevalence)                                                   0.41667       0.16667       0.41667       \n",
      "Q(Yule Q - coefficient of colligation)                            None          0.6           0.33333       \n",
      "QI(Yule Q interpretation)                                         None          Moderate      Weak          \n",
      "RACC(Random accuracy)                                             0.10417       0.04167       0.20833       \n",
      "RACCU(Random accuracy unbiased)                                   0.11111       0.0434        0.21007       \n",
      "TN(True negative/correct rejection)                               7             8             4             \n",
      "TNR(Specificity or true negative rate)                            1.0           0.8           0.57143       \n",
      "TON(Test outcome negative)                                        9             9             6             \n",
      "TOP(Test outcome positive)                                        3             3             6             \n",
      "TP(True positive/hit)                                             3             1             3             \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "Y(Youden index)                                                   0.6           0.3           0.17143       \n",
      "dInd(Distance index)                                              0.4           0.53852       0.58624       \n",
      "sInd(Similarity index)                                            0.71716       0.61921       0.58547       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       L2       L3       \n",
      "Actual\n",
      "L1       3        0        2        \n",
      "\n",
      "L2       0        1        1        \n",
      "\n",
      "L3       0        2        3        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': {'L1': 3, 'L2': 0, 'L3': 2},\n",
       " 'L2': {'L1': 0, 'L2': 1, 'L3': 1},\n",
       " 'L3': {'L1': 0, 'L2': 2, 'L3': 3}}"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       ~        \n",
      "Actual\n",
      "L1       3        2        \n",
      "\n",
      "~        0        7        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix(one_vs_all=True, class_name=\"L1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_cm = ConfusionMatrix(matrix={1: {1: 0, 2: 2}, 2: {1: 0, 2: 18}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  2        \n",
      "Actual\n",
      "1        2        \n",
      "\n",
      "2        18       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sparse_cm.print_matrix(sparse=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `one_vs_all` : one-vs-all mode flag (type : `bool`, default : `False`)\n",
    "2. `class_name` : target class name for one-vs-all mode (type : `any valid type`, default : `None`)\n",
    "3. `sparse` : sparse mode printing flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>one_vs_all</code> option, new in <span style=\"color:red;\">version 1.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>matrix()</code> renamed to <code>print_matrix()</code> and <code>matrix</code> return confusion matrix as <code>dict</code> in <span style=\"color:red;\">version 1.5</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>sparse</code> option, new in <span style=\"color:red;\">version 2.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalized matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   L1        L2        L3        \n",
      "Actual\n",
      "L1        0.6       0.0       0.4       \n",
      "\n",
      "L2        0.0       0.5       0.5       \n",
      "\n",
      "L3        0.0       0.4       0.6       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_normalized_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': {'L1': 0.6, 'L2': 0.0, 'L3': 0.4},\n",
       " 'L2': {'L1': 0.0, 'L2': 0.5, 'L3': 0.5},\n",
       " 'L3': {'L1': 0.0, 'L2': 0.4, 'L3': 0.6}}"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   L1        ~         \n",
      "Actual\n",
      "L1        0.6       0.4       \n",
      "\n",
      "~         0.0       1.0       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_normalized_matrix(one_vs_all=True, class_name=\"L1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   2         \n",
      "Actual\n",
      "1         1.0       \n",
      "\n",
      "2         1.0       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sparse_cm.print_normalized_matrix(sparse=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `one_vs_all` : one-vs-all mode flag (type : `bool`, default : `False`)\n",
    "2. `class_name` : target class name for one-vs-all mode (type : `any valid type`, default : `None`)\n",
    "3. `sparse` : sparse mode printing flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>one_vs_all</code> option, new in <span style=\"color:red;\">version 1.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>normalized_matrix()</code> renamed to <code>print_normalized_matrix()</code> and <code>normalized_matrix</code> return normalized confusion matrix as <code>dict</code> in <span style=\"color:red;\">version 1.5</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>sparse</code> option, new in <span style=\"color:red;\">version 2.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "95% CI                                                            (0.30439,0.86228)\n",
      "ACC Macro                                                         0.72222\n",
      "ARI                                                               0.09206\n",
      "AUNP                                                              0.68571\n",
      "AUNU                                                              0.67857\n",
      "Bangdiwala B                                                      0.37255\n",
      "Bennett S                                                         0.375\n",
      "CBA                                                               0.47778\n",
      "CSI                                                               0.17778\n",
      "Chi-Squared                                                       6.6\n",
      "Chi-Squared DF                                                    4\n",
      "Conditional Entropy                                               0.97579\n",
      "Cramer V                                                          0.5244\n",
      "Cross Entropy                                                     1.58333\n",
      "F1 Macro                                                          0.56515\n",
      "F1 Micro                                                          0.58333\n",
      "FNR Macro                                                         0.43333\n",
      "FNR Micro                                                         0.41667\n",
      "FPR Macro                                                         0.20952\n",
      "FPR Micro                                                         0.20833\n",
      "Gwet AC1                                                          0.38931\n",
      "Hamming Loss                                                      0.41667\n",
      "Joint Entropy                                                     2.45915\n",
      "KL Divergence                                                     0.09998\n",
      "Kappa                                                             0.35484\n",
      "Kappa 95% CI                                                      (-0.07708,0.78675)\n",
      "Kappa No Prevalence                                               0.16667\n",
      "Kappa Standard Error                                              0.22036\n",
      "Kappa Unbiased                                                    0.34426\n",
      "Krippendorff Alpha                                                0.37158\n",
      "Lambda A                                                          0.42857\n",
      "Lambda B                                                          0.16667\n",
      "Mutual Information                                                0.52421\n",
      "NIR                                                               0.41667\n",
      "NPV Macro                                                         0.77778\n",
      "NPV Micro                                                         0.79167\n",
      "Overall ACC                                                       0.58333\n",
      "Overall CEN                                                       0.46381\n",
      "Overall J                                                         (1.225,0.40833)\n",
      "Overall MCC                                                       0.36667\n",
      "Overall MCEN                                                      0.51894\n",
      "Overall RACC                                                      0.35417\n",
      "Overall RACCU                                                     0.36458\n",
      "P-Value                                                           0.18926\n",
      "PPV Macro                                                         0.61111\n",
      "PPV Micro                                                         0.58333\n",
      "Pearson C                                                         0.59568\n",
      "Phi-Squared                                                       0.55\n",
      "RCI                                                               0.35339\n",
      "RR                                                                4.0\n",
      "Reference Entropy                                                 1.48336\n",
      "Response Entropy                                                  1.5\n",
      "SOA1(Landis & Koch)                                               Fair\n",
      "SOA2(Fleiss)                                                      Poor\n",
      "SOA3(Altman)                                                      Fair\n",
      "SOA4(Cicchetti)                                                   Poor\n",
      "SOA5(Cramer)                                                      Relatively Strong\n",
      "SOA6(Matthews)                                                    Weak\n",
      "SOA7(Lambda A)                                                    Moderate\n",
      "SOA8(Lambda B)                                                    Very Weak\n",
      "SOA9(Krippendorff Alpha)                                          Low\n",
      "SOA10(Pearson C)                                                  Strong\n",
      "Scott PI                                                          0.34426\n",
      "Standard Error                                                    0.14232\n",
      "TNR Macro                                                         0.79048\n",
      "TNR Micro                                                         0.79167\n",
      "TPR Macro                                                         0.56667\n",
      "TPR Micro                                                         0.58333\n",
      "Zero-one Loss                                                     5\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AGF(Adjusted F-score)                                             0.72859       0.62869       0.61009       \n",
      "AGM(Adjusted geometric mean)                                      0.85764       0.70861       0.58034       \n",
      "AM(Difference between automatic and manual classification)        -2            1             1             \n",
      "AUC(Area under the ROC curve)                                     0.8           0.65          0.58571       \n",
      "AUCI(AUC value interpretation)                                    Very Good     Fair          Poor          \n",
      "AUPR(Area under the PR curve)                                     0.8           0.41667       0.55          \n",
      "BB(Braun-Blanquet similarity)                                     0.6           0.33333       0.5           \n",
      "BCD(Bray-Curtis dissimilarity)                                    0.08333       0.04167       0.04167       \n",
      "BM(Informedness or bookmaker informedness)                        0.6           0.3           0.17143       \n",
      "CEN(Confusion entropy)                                            0.25          0.49658       0.60442       \n",
      "DOR(Diagnostic odds ratio)                                        None          4.0           2.0           \n",
      "DP(Discriminant power)                                            None          0.33193       0.16597       \n",
      "DPI(Discriminant power interpretation)                            None          Poor          Poor          \n",
      "ERR(Error rate)                                                   0.16667       0.25          0.41667       \n",
      "F0.5(F0.5 score)                                                  0.88235       0.35714       0.51724       \n",
      "F1(F1 score - harmonic mean of precision and sensitivity)         0.75          0.4           0.54545       \n",
      "F2(F2 score)                                                      0.65217       0.45455       0.57692       \n",
      "FDR(False discovery rate)                                         0.0           0.66667       0.5           \n",
      "FN(False negative/miss/type 2 error)                              2             1             2             \n",
      "FNR(Miss rate or false negative rate)                             0.4           0.5           0.4           \n",
      "FOR(False omission rate)                                          0.22222       0.11111       0.33333       \n",
      "FP(False positive/type 1 error/false alarm)                       0             2             3             \n",
      "FPR(Fall-out or false positive rate)                              0.0           0.2           0.42857       \n",
      "G(G-measure geometric mean of precision and sensitivity)          0.7746        0.40825       0.54772       \n",
      "GI(Gini index)                                                    0.6           0.3           0.17143       \n",
      "GM(G-mean geometric mean of specificity and sensitivity)          0.7746        0.63246       0.58554       \n",
      "HD(Hamming distance)                                              2             3             5             \n",
      "IBA(Index of balanced accuracy)                                   0.36          0.28          0.35265       \n",
      "ICSI(Individual classification success index)                     0.6           -0.16667      0.1           \n",
      "IS(Information score)                                             1.26303       1.0           0.26303       \n",
      "J(Jaccard index)                                                  0.6           0.25          0.375         \n",
      "LS(Lift score)                                                    2.4           2.0           1.2           \n",
      "MCC(Matthews correlation coefficient)                             0.68313       0.2582        0.16903       \n",
      "MCCI(Matthews correlation coefficient interpretation)             Moderate      Negligible    Negligible    \n",
      "MCEN(Modified confusion entropy)                                  0.26439       0.5           0.6875        \n",
      "MK(Markedness)                                                    0.77778       0.22222       0.16667       \n",
      "N(Condition negative)                                             7             10            7             \n",
      "NLR(Negative likelihood ratio)                                    0.4           0.625         0.7           \n",
      "NLRI(Negative likelihood ratio interpretation)                    Poor          Negligible    Negligible    \n",
      "NPV(Negative predictive value)                                    0.77778       0.88889       0.66667       \n",
      "OC(Overlap coefficient)                                           1.0           0.5           0.6           \n",
      "OOC(Otsuka-Ochiai coefficient)                                    0.7746        0.40825       0.54772       \n",
      "OP(Optimized precision)                                           0.58333       0.51923       0.55894       \n",
      "P(Condition positive or support)                                  5             2             5             \n",
      "PLR(Positive likelihood ratio)                                    None          2.5           1.4           \n",
      "PLRI(Positive likelihood ratio interpretation)                    None          Poor          Poor          \n",
      "POP(Population)                                                   12            12            12            \n",
      "PPV(Precision or positive predictive value)                       1.0           0.33333       0.5           \n",
      "PRE(Prevalence)                                                   0.41667       0.16667       0.41667       \n",
      "Q(Yule Q - coefficient of colligation)                            None          0.6           0.33333       \n",
      "QI(Yule Q interpretation)                                         None          Moderate      Weak          \n",
      "RACC(Random accuracy)                                             0.10417       0.04167       0.20833       \n",
      "RACCU(Random accuracy unbiased)                                   0.11111       0.0434        0.21007       \n",
      "TN(True negative/correct rejection)                               7             8             4             \n",
      "TNR(Specificity or true negative rate)                            1.0           0.8           0.57143       \n",
      "TON(Test outcome negative)                                        9             9             6             \n",
      "TOP(Test outcome positive)                                        3             3             6             \n",
      "TP(True positive/hit)                                             3             1             3             \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "Y(Youden index)                                                   0.6           0.3           0.17143       \n",
      "dInd(Distance index)                                              0.4           0.53852       0.58624       \n",
      "sInd(Similarity index)                                            0.71716       0.61921       0.58547       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "Kappa                                                             0.35484\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AUC(Area under the ROC curve)                                     0.8           0.65          0.58571       \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat(overall_param=[\"Kappa\"], class_param=[\"ACC\", \"AUC\", \"TPR\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "Kappa                                                             0.35484\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.58333       \n",
      "AUC(Area under the ROC curve)                                     0.8           0.58571       \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.6           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat(overall_param=[\"Kappa\"], class_param=[\"ACC\", \"AUC\", \"TPR\"], class_name=[\"L1\", \"L3\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "ACC Macro                                                         0.72222\n",
      "F1 Macro                                                          0.56515\n",
      "FPR Macro                                                         0.20952\n",
      "Kappa                                                             0.35484\n",
      "NPV Macro                                                         0.77778\n",
      "Overall ACC                                                       0.58333\n",
      "PPV Macro                                                         0.61111\n",
      "SOA1(Landis & Koch)                                               Fair\n",
      "TPR Macro                                                         0.56667\n",
      "Zero-one Loss                                                     5\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AUC(Area under the ROC curve)                                     0.8           0.65          0.58571       \n",
      "AUCI(AUC value interpretation)                                    Very Good     Fair          Poor          \n",
      "F1(F1 score - harmonic mean of precision and sensitivity)         0.75          0.4           0.54545       \n",
      "FN(False negative/miss/type 2 error)                              2             1             2             \n",
      "FP(False positive/type 1 error/false alarm)                       0             2             3             \n",
      "FPR(Fall-out or false positive rate)                              0.0           0.2           0.42857       \n",
      "N(Condition negative)                                             7             10            7             \n",
      "P(Condition positive or support)                                  5             2             5             \n",
      "POP(Population)                                                   12            12            12            \n",
      "PPV(Precision or positive predictive value)                       1.0           0.33333       0.5           \n",
      "TN(True negative/correct rejection)                               7             8             4             \n",
      "TON(Test outcome negative)                                        9             9             6             \n",
      "TOP(Test outcome positive)                                        3             3             6             \n",
      "TP(True positive/hit)                                             3             1             3             \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat(summary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `overall_param` : overall parameters list for print (type : `list`, default : `None`)\n",
    "2. `class_param` : class parameters list for print (type : `list`, default : `None`)\n",
    "3. `class_name` : class name (a subset of classes names) (type : `list`, default : `None`)\n",
    "4. `summary` : summary mode flag (type : `bool`, default : `False`)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>cm.params()</code> in prev versions (0.2 >) </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>overall_param</code> & <code>class_param</code> , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>class_name</code> , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>summary</code> , new in <span style=\"color:red;\">version 2.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm2    0.50278           0.58095\n",
      "2     cm3    0.33611           0.52857\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cp.print_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score       Overall-Score\n",
      "1     cm2    0.50278           0.58095\n",
      "2     cm3    0.33611           0.52857\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "if \"Document_files\" not in os.listdir():\n",
    "    os.mkdir(\"Document_files\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### .pycm file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(os.path.join(\"Document_files\", \"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered\"),\n",
    "    overall_param=[\"Kappa\"],\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered2.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered2\"),\n",
    "    overall_param=[\"Kappa\"],\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"],\n",
    "    class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered2.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_summary.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(\n",
    "    os.path.join(\"Document_files\", \"cm1_summary\"),\n",
    "    summary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_summary.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\sparse_cm.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_cm.save_stat(\n",
    "    os.path.join(\"Document_files\", \"sparse_cm\"),\n",
    "    summary=True,\n",
    "    sparse=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\sparse_cm.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.pycm'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : filename (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`, default : `True`)\n",
    "3. `overall_param` : overall parameters list for save (type : `list`, default : `None`)\n",
    "4. `class_param` : class parameters list for save (type : `list`, default : `None`)\n",
    "5. `class_name` : class name (subset of classes names) (type : `list`, default : `None`)\n",
    "6. `summary` : summary mode flag (type : `bool`, default : `False`)\n",
    "7. `sparse` : sparse mode printing flag (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>overall_param</code> & <code>class_param</code> , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>class_name</code> , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>summary</code> , new in <span style=\"color:red;\">version 2.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>sparse</code>, new in <span style=\"color:red;\">version 2.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_files\", \"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered\"),\n",
    "    overall_param=[\"Kappa\"],\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered2.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered2\"),\n",
    "    overall_param=[\"Kappa\"],\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"],\n",
    "    class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered2.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_colored.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_colored\"),\n",
    "    color=(255, 204, 255))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_colored.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_colored2.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_colored2\"),\n",
    "    color=\"Crimson\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_colored2.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_normalized.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_normalized\"),\n",
    "    color=\"Crimson\",\n",
    "    normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_normalized.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_summary.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\n",
    "    os.path.join(\"Document_files\", \"cm1_summary\"),\n",
    "    summary=True,\n",
    "    normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_summary.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.html'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 287,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : filename (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`, default : `True`)\n",
    "3. `overall_param` : overall parameters list for save (type : `list`, default : `None`)\n",
    "4. `class_param` : class parameters list for save (type : `list`, default : `None`)\n",
    "5. `class_name` : class name (subset of classes names) (type : `list`, default : `None`)\n",
    "6. `color` : matrix color in RGB as (R, G, B) (type : `tuple`/`str`, default : `(0,0,0)`), support <a href=\"https://en.wikipedia.org/wiki/X11_color_names\">X11 color names</a>\n",
    "7. `normalize` : save normalize matrix flag (type : `bool`, default : `False`)\n",
    "8. `summary` : summary mode flag (type : `bool`, default : `False`)\n",
    "9. `alt_link` : alternative link for document flag (type : `bool`, default : `False`)\n",
    "10. `shortener` : class name shortener flag (type : `bool`, default : `True`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>overall_param</code> & <code>class_param</code> , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>class_name</code> , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>color</code>, new in <span style=\"color:red;\">version 1.8 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>normalize</code>, new in <span style=\"color:red;\">version 2.0 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>summary</code> and <code>alt_link</code> , new in <span style=\"color:red;\">version 2.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>shortener</code>, new in <span style=\"color:red;\">version 3.2 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  If PyCM website is not available, set <code>alt_link=True</code></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(os.path.join(\"Document_files\", \"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_files\\cm1_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered\"),\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_files\\cm1_filtered_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered2.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered2\"),\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"],\n",
    "    normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered2.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_files\\cm1_filtered2_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_filtered3.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\n",
    "    os.path.join(\"Document_files\", \"cm1_filtered3\"),\n",
    "    class_param=[\"ACC\", \"AUC\", \"TPR\"],\n",
    "    class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_filtered3.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_files\\cm1_filtered3_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_header.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\n",
    "    os.path.join(\"Document_files\", \"cm1_header\"),\n",
    "    header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_header.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_files\\cm1_header_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_summary.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\n",
    "    os.path.join(\"Document_files\", \"cm1_summary\"),\n",
    "    summary=True,\n",
    "    matrix_save=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_summary.csv\">Open Stat File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.csv'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : filename (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`, default : `True`)\n",
    "4. `class_param` : class parameters list for save (type : `list`, default : `None`)\n",
    "5. `class_name` : class name (subset of classes names) (type : `list`, default : `None`)\n",
    "6. `matrix_save` : save matrix flag (type : `bool`, default : `True`)\n",
    "7. `normalize` : save normalize matrix flag (type : `bool`, default : `False`)\n",
    "8. `summary` : summary mode flag (type : `bool`, default : `False`)\n",
    "9. `header` : add headers to .csv file (type : `bool`, default : `False`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  <code>class_param</code> , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>class_name</code> , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>matrix_save</code> and <code>normalize</code>, new in <span style=\"color:red;\">version 1.9 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>summary</code> , new in <span style=\"color:red;\">version 2.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>header</code> , new in <span style=\"color:red;\">version 2.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OBJ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1.obj',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(os.path.join(\"Document_files\", \"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1.obj\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_stat.obj',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(\n",
    "    os.path.join(\"Document_files\", \"cm1_stat\"),\n",
    "    save_stat=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_stat.obj\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cm1_no_vectors.obj',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(\n",
    "    os.path.join(\"Document_files\", \"cm1_no_vectors\"),\n",
    "    save_vector=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cm1_no_vectors.obj\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.obj'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : filename (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`, default : `True`)\n",
    "3. `save_stat` : save statistics flag (type : `bool`, default : `False`)\n",
    "4. `save_vector` : save vectors flag (type : `bool`, default : `True`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : <code>save_vector</code> and <code>save_stat</code>, new in <span style=\"color:red;\">version 2.3 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  For more information visit <a href=\"#Example-4-(File)\">Example 4</a></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### comp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_files\\\\cp.comp',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.save_report(os.path.join(\"Document_files\", \"cp\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_files\\cp.comp\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.comp'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.save_report(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : filename (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`, default : `True`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must be provided as a list or a NumPy array.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm2 = ConfusionMatrix(y_actu, 2)\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have the same length.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm3 = ConfusionMatrix(y_actu, [1, 2, 3])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must not be empty.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_4 = ConfusionMatrix([], [])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have the same length.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_5 = ConfusionMatrix([1, 1, 1, ], [1, 1, 1, 1])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid input confusion matrix format.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm3 = ConfusionMatrix(matrix={})\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All input matrix classes must be of the same type.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_4 = ConfusionMatrix(matrix={1: {1: 2, \"1\": 2}, \"1\": {1: 2, \"1\": 3}})\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of classes must be at least 2.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_5 = ConfusionMatrix(matrix={1: {1: 2}})\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input must be provided as a dictionary.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare([cm2, cm3])\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All ConfusionMatrix objects must have the same domain (same sample size and number of classes).\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare({\"cm1\": cm, \"cm2\": cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input must be a dictionary containing pycm.ConfusionMatrix objects.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare({\"cm1\": [], \"cm2\": cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At least 2 confusion matrices are required for comparison.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare({\"cm2\": cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`class_weight` must be a dictionary and specified for all classes.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare(\n",
    "        {\"cm1\": cm2, \"cm2\": cm3},\n",
    "        by_class=True,\n",
    "        class_weight={1: 2, 2: 0})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`class_benchmark_weight` must be a dictionary and specified for all class benchmarks.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare(\n",
    "        {\"cm1\":cm2, \"cm2\":cm3},\n",
    "        class_benchmark_weight={1: 2, 2: 0})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`overall_benchmark_weight` must be a dictionary and specified for all overall benchmarks.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp = Compare(\n",
    "        {\"cm1\": cm2, \"cm2\": cm3},\n",
    "        overall_benchmark_weight={1: 2, 2: 0})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confidence interval calculation for this parameter is not supported in this version of pycm.\n",
      " Supported parameters are: TPR, TNR, PPV, NPV, ACC, PLR, NLR, FPR, FNR, AUC, PRE, Kappa, Overall ACC\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.CI(\"MCC\")\n",
    "except pycmCIError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input must be provided as a string.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.CI(2)\n",
    "except pycmCIError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid parameter!\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.average(\"AXY\")\n",
    "except pycmAverageError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid parameter!\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.weighted_average(\"AXY\")\n",
    "except pycmAverageError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`weight` must be a dictionary and specified for all classes."
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.weighted_average(\"AUC\", weight={1: 22})\n",
    "except pycmAverageError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This option is only available in vector mode."
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.position()\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input must be an instance of pycm.ConfusionMatrix.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm.combine(2)\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of classes must be at least 2.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm6 = ConfusionMatrix([1, 1, 1, 1], [1, 1, 1, 1], classes=[])\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classes` must contain unique labels with no duplicates.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm7 = ConfusionMatrix([1, 1, 1, 1], [1, 1, 1, 1], classes=[1, 1, 2])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must be provided as a list or a NumPy array.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve([1, 2, 2, 1], {1, 2, 2, 1}, classes=[1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have the same length.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve([1, 2, 2, 1],[[0.1, 0.9]], classes=[1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The sum of the probability values must equal 1.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.2, 0.9]],\n",
    "        classes=[1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classes` must be provided as a list.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.1, 0.9]],\n",
    "        classes={1, 2})\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classes` does not match the actual vector.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.1, 0.9]],\n",
    "        classes=[1, 2, 3])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of classes must be at least 2.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 1, 1, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.1, 0.9]],\n",
    "        classes=[1])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Probability vector elements must be numeric.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.2, \"s\"]],\n",
    "        classes=[1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`thresholds` must be provided as a list or a NumPy array.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9],[0.1, 0.9], [0.1, 0.9], [0.2, 0.8]],\n",
    "        classes=[1, 2],\n",
    "        thresholds={1, 2})\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of thresholds must be at least 2.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.2, 0.8]],\n",
    "        classes=[1, 2],\n",
    "        thresholds=[0.1])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`thresholds` must contain only numeric values.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.2, 0.8]],\n",
    "        classes=[1, 2],\n",
    "        thresholds=[0.1, \"q\"])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classes` must contain unique labels with no duplicates.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.9], [0.2, 0.8]],\n",
    "        classes=[1, 1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All elements of the probability vector must have the same length and match the number of classes.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        [1, 2, 2, 1],\n",
    "        [[0.1, 0.9], [0.1, 0.9], [0.1, 0.8, 0.1], [0.2, 0.8]],\n",
    "        classes=[1, 2])\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The integral method must be either 'trapezoidal' or 'midpoint'.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    crv = ROCCurve(\n",
    "        actual_vector=numpy.array([1, 1, 2, 2]),\n",
    "        probs=numpy.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]),\n",
    "        classes=[2, 1])\n",
    "    crv.area(method=\"trpz\")\n",
    "except pycmCurveError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to extract classes from input. Input vectors should be a list of sets with unified types.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM([[0, 1], [1, 1]], [[1, 0], [1, 0]])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The specified class name is not among the confusion matrix's classes.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM([{'dog'}, {'cat', 'dog'}], [{'cat'}, {'cat'}])\n",
    "    mlcm.get_cm_by_class(1)\n",
    "except pycmMultiLabelError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index is out of range for the given vector.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm.get_cm_by_sample(2)\n",
    "except pycmMultiLabelError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must be provided as a list or a NumPy array."
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM(2, [{1, 0}, {1, 0}])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have the same length.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM([{1, 0}, {1, 0}, {1,1}], [{1, 0}, {1, 0}])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must not be empty.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM([], [])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "`classes` must contain unique labels with no duplicates.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mlcm = MultiLabelCM([{1, 0}, {1, 0}], [{1, 0}, {1, 0}], classes=[1,0,1])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  updated in <span style=\"color:red;\">version 4.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Example-1 (Comparison of three different classifiers)\t\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example1.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example1.html)\n",
    "\n",
    "### Example-2 (How to plot via matplotlib)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example2.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example2.html)\n",
    "\n",
    "### Example-3 (Activation threshold)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example3.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example3.html)\n",
    "\n",
    "### Example-4 (File)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example4.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example4.html)\n",
    "\n",
    "### Example-5 (Sample weights)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example5.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example5.html)\n",
    "\n",
    "### Example-6 (Unbalanced data)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example6.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example6.html)\n",
    "\n",
    "### Example-7 (How to plot via seaborn+pandas)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example7.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example7.html)\n",
    "\n",
    "### Example-8 (Confidence interval)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example8.ipynb)\n",
    "- [HTML](http://www.pycm.io/doc/Example8.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cite"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you use PyCM in your research, we would appreciate citations to the following paper :"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>Haghighi, S., Jasemi, M., Hessabi, S. and Zolanvari, A. (2018). PyCM: Multiclass confusion matrix library in Python.<br/>Journal of Open Source Software, 3(25), p.729.</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "@article{Haghighi2018,\n",
    "  doi = {10.21105/joss.00729},\n",
    "  url = {https://doi.org/10.21105/joss.00729},\n",
    "  year  = {2018},\n",
    "  month = {may},\n",
    "  publisher = {The Open Journal},\n",
    "  volume = {3},\n",
    "  number = {25},\n",
    "  pages = {729},\n",
    "  author = {Sepand Haghighi and Masoomeh Jasemi and Shaahin Hessabi and Alireza Zolanvari},\n",
    "  title = {{PyCM}: Multiclass confusion matrix library in Python},\n",
    "  journal = {Journal of Open Source Software}\n",
    "}\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download <a href=\"http://www.pycm.io/PYCM.bib\">PyCM.bib</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<blockquote id=\"ref1\">1- J. R. Landis and G. G. Koch, \"The measurement of observer agreement for categorical data,\" <i>biometrics</i>, pp. 159-174, 1977.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref2\">2- D. M. Powers, \"Evaluation: from precision, recall and F-measure to ROC, informedness, markedness and correlation,\" <i>arXiv preprint arXiv:2010.16061</i>, 2020.</blockquote>\n",
    "\n",
    "\n",
    "<blockquote id=\"ref3\">3- C. Sammut and G. I. Webb, <i>Encyclopedia of machine learning</i>. Springer Science & Business Media, 2011.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref4\">4- J. L. Fleiss, \"Measuring nominal scale agreement among many raters,\" <i>Psychological bulletin</i>, vol. 76, no. 5, p. 378, 1971.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref5\">5- D. G. Altman, <i>Practical statistics for medical research</i>. CRC press, 1990.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref6\">6- K. L. Gwet, \"Computing inter-rater reliability and its variance in the presence of high agreement,\" <i>British Journal of Mathematical and Statistical Psychology</i>, vol. 61, no. 1, pp. 29-48, 2008.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref7\">7- W. A. Scott, \"Reliability of content analysis: The case of nominal scale coding,\" <i>Public opinion quarterly</i>, pp. 321-325, 1955.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref8\">8- E. M. Bennett, R. Alpert, and A. Goldstein, \"Communications through limited-response questioning,\" <i>Public Opinion Quarterly</i>, vol. 18, no. 3, pp. 303-308, 1954.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref9\">9- D. V. Cicchetti, \"Guidelines, criteria, and rules of thumb for evaluating normed and standardized assessment instruments in psychology,\" <i>Psychological assessment</i>, vol. 6, no. 4, p. 284, 1994.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref10\">10- R. B. Davies, \"Algorithm AS 155: The distribution of a linear combination of χ2 random variables,\" <i>Applied Statistics</i>, pp. 323-333, 1980.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref11\">11- S. Kullback and R. A. Leibler, \"On information and sufficiency,\" <i>The annals of mathematical statistics</i>, vol. 22, no. 1, pp. 79-86, 1951.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref12\">12- L. A. Goodman and W. H. Kruskal, \"Measures of association for cross classifications, IV: Simplification of asymptotic variances,\" <i>Journal of the American Statistical Association</i>, vol. 67, no. 338, pp. 415-421, 1972.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref13\">13- L. A. Goodman and W. H. Kruskal, \"Measures of association for cross classifications III: Approximate sampling theory,\" <i>Journal of the American Statistical Association</i>, vol. 58, no. 302, pp. 310-364, 1963.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref14\">14- T. Byrt, J. Bishop, and J. B. Carlin, \"Bias, prevalence and kappa,\" <i>Journal of clinical epidemiology</i>, vol. 46, no. 5, pp. 423-429, 1993.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref15\">15- M. Shepperd, D. Bowes, and T. Hall, \"Researcher bias: The use of machine learning in software defect prediction,\" <i>IEEE Transactions on Software Engineering</i>, vol. 40, no. 6, pp. 603-616, 2014.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref16\">16- X. Deng, Q. Liu, Y. Deng, and S. Mahadevan, \"An improved method to construct basic probability assignment based on the confusion matrix for classification problem,\" <i>Information Sciences</i>, vol. 340, pp. 250-261, 2016.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref17\">17- J.-M. Wei, X.-J. Yuan, Q.-H. Hu, and S.-Q. Wang, \"A novel measure for evaluating classifiers,\" <i>Expert Systems with Applications</i>, vol. 37, no. 5, pp. 3799-3809, 2010.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref18\">18- I. Kononenko and I. Bratko, \"Information-based evaluation criterion for classifier's performance,\" <i>Machine learning</i>, vol. 6, no. 1, pp. 67-80, 1991.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref19\">19- R. Delgado and J. D. Núnez-González, \"Enhancing confusion entropy as measure for evaluating classifiers,\" in <i>The 13th International Conference on Soft Computing Models in Industrial and Environmental Applications</i>, 2018: Springer, pp. 79-89.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref20\">20- J. Gorodkin, \"Comparing two K-category assignments by a K-category correlation coefficient,\" <i>Computational biology and chemistry</i>, vol.28, no. 5-6, pp. 367-374, 2004.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref21\">21- C. O. Freitas, J. M. De Carvalho, J. Oliveira, S. B. Aires, and R. Sabourin, \"Confusion matrix disagreement for multiple classifiers,\" in <i>Iberoamerican Congress on Pattern Recognition</i>, 2007: Springer, pp. 387-396.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref22\">22- P. Branco, L. Torgo, and R. P. Ribeiro, \"Relevance-based evaluation metrics for multi-class imbalanced domains,\" in <i>Pacific-Asia Conference on Knowledge Discovery and Data Mining</i>, 2017: Springer, pp. 698-710.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref23\">23- D. Ballabio, F. Grisoni, and R. Todeschini, \"Multivariate comparison of classification performance measures,\" <i>Chemometrics and Intelligent Laboratory Systems</i>, vol. 174, pp. 33-44, 2018.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref24\">24- J. Cohen, \"A coefficient of agreement for nominal scales,\" <i>Educational and psychological measurement</i>, vol. 20, no. 1, pp. 37-46, 1960.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref25\">25- S. Siegel, \"Nonparametric statistics for the behavioral sciences,\" 1956.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref26\">26- H. Cramér, <i>Mathematical methods of statistics</i>. Princeton university press, 1999.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref27\">27- B. W. Matthews, \"Comparison of the predicted and observed secondary structure of T4 phage lysozyme,\" <i>Biochimica et Biophysica Acta (BBA)-Protein Structure</i>, vol. 405, no. 2, pp. 442-451, 1975.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref28\">28- J. A. Swets, \"The relative operating characteristic in psychology: a technique for isolating effects of response bias finds wide use in the study of perception and cognition,\" <i>Science</i>, vol. 182, no. 4116, pp. 990-1000, 1973.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref29\">29- P. Jaccard, \"Étude comparative de la distribution florale dans une portion des Alpes et des Jura,\" <i>Bull Soc Vaudoise Sci Nat</i>, vol. 37, pp. 547-579, 1901.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref30\">30- T. M. Cover and J. A. Thomas, <i>Elements of Information Theory</i>. John Wiley & Sons, 2012.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref31\">31- E. S. Keeping, <i>Introduction to statistical inference</i>. Courier Corporation, 1995.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref32\">32- V. Sindhwani, P. Bhattacharya, and S. Rakshit, \"Information theoretic feature crediting in multiclass support vector machines,\" in <i>Proceedings of the 2001 SIAM International Conference on Data Mining</i>, 2001: SIAM, pp. 1-18.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref33\">33- M. Bekkar, H. K. Djemaa, and T. A. Alitouche, \"Evaluation measures for models assessment over imbalanced data sets,\" <i>J Inf Eng Appl</i>, vol. 3, no. 10, 2013.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref34\">34- W. J. Youden, \"Index for rating diagnostic tests,\" <i>Cancer</i>, vol. 3, no. 1, pp. 32-35, 1950.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref35\">35- S. Brin, R. Motwani, J. D. Ullman, and S. Tsur, \"Dynamic itemset counting and implication rules for market basket data,\" in <i>Proceedings of the 1997 ACM SIGMOD international conference on Management of data</i>, 1997, pp. 255-264.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref36\">36- S. Raschka, \"MLxtend: Providing machine learning and data science utilities and extensions to Python's scientific computing stack,\" <i>Journal of open source software</i>, vol. 3, no. 24, p. 638, 2018.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref37\">37- J. R. Bray and J. T. Curtis, \"An ordination of the upland forest communities of southern Wisconsin,\" Ecological monographs, vol. 27, no. 4, pp. 325-349, 1957.</blockquote>\t\n",
    "\n",
    "<blockquote id=\"ref38\">38- J. L. Fleiss, J. Cohen, and B. S. Everitt, \"Large sample standard errors of kappa and weighted kappa,\" <i>Psychological bulletin</i>, vol. 72, no. 5, p. 323, 1969.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref39\">39- M. Felkin, \"Comparing classification results between n-ary and binary problems,\" in <i>Quality Measures in Data Mining</i>: Springer, 2007, pp. 277-301.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref40\">40- R. Ranawana and V. Palade, \"Optimized precision-a new measure for classifier performance evaluation,\" in <i>2006 IEEE International Conference on Evolutionary Computation</i>, 2006: IEEE, pp. 2254-2261.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref41\">41- V. García, R. A. Mollineda, and J. S. Sánchez, \"Index of balanced accuracy: A performance measure for skewed class distributions,\" in <i>Iberian conference on pattern recognition and image analysis</i>, 2009: Springer, pp. 441-448.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref42\">42- P. Branco, L. Torgo, and R. P. Ribeiro, \"A survey of predictive modeling on imbalanced domains,\" <i>ACM Computing Surveys (CSUR)</i>, vol. 49, no. 2, pp. 1-50, 2016.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref43\">43- K. Pearson, \"Notes on Regression and Inheritance in the Case of Two Parents,\" in <i>Proceedings of the Royal Society of London</i>, p. 240-242, 1895.</blockquote> \n",
    "\n",
    "<blockquote id=\"ref44\">44- W. J. Conover, <i>Practical nonparametric statistics</i>. John Wiley & Sons, 1998.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref45\">45- G. U. Yule, \"On the methods of measuring association between two attributes,\" <i>Journal of the Royal Statistical Society</i>, vol. 75, no. 6, pp. 579-652, 1912.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref46\">46- R. Batuwita and V. Palade, \"A new performance measure for class imbalance learning. application to bioinformatics problems,\" in <i>2009 International Conference on Machine Learning and Applications</i>, 2009: IEEE, pp. 545-550.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref47\">47- D. K. Lee, \"Alternatives to P value: confidence interval and effect size,\" <i>Korean journal of anesthesiology</i>, vol. 69, no. 6, p. 555, 2016.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref48\">48- M. A. Raslich, R. J. Markert, and S. A. Stutes, \"Selecting and interpreting diagnostic tests,\" <i>Biochemia Medica</i>, vol. 17, no. 2, pp. 151-161, 2007.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref49\">49- D. E. Hinkle, W. Wiersma, and S. G. Jurs, <i>Applied statistics for the behavioral sciences</i>. Houghton Mifflin College Division, 2003.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref50\">50- A. Maratea, A. Petrosino, and M. Manzo, \"Adjusted F-measure and kernel scaling for imbalanced data learning,\" <i>Information Sciences</i>, vol. 257, pp. 331-341, 2014.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref51\">51- L. Mosley, \"A balanced approach to the multi-class imbalance problem,\" 2013.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref52\">52- M. Vijaymeena and K. Kavitha, \"A survey on similarity measures in text mining,\" <i>Machine Learning and Applications: An International Journal</i>, vol. 3, no. 2, pp. 19-28, 2016.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref53\">53- Y. Otsuka, \"The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan,\" <i>Biogeograph Soc Japan</i>, vol. 6, no. 16, pp. 165-170, 1936.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref54\">54- A. Tversky, \"Features of similarity,\" <i>Psychological review</i>, vol. 84, no. 4, p. 327, 1977.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref55\">55- K. Boyd, K. H. Eng, and C. D. Page, \"Area under the precision-recall curve: point estimates and confidence intervals,\" in <i>Joint European conference on machine learning and knowledge discovery in databases</i>, 2013: Springer, pp. 451-466.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref56\">56- J. Davis and M. Goadrich, \"The relationship between Precision-Recall and ROC curves,\" in <i>Proceedings of the 23rd international conference on Machine learning</i>, 2006, pp. 233-240.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref57\">57- M. Kuhn, \"Building predictive models in R using the caret package,\" <i>J Stat Softw</i>, vol. 28, no. 5, pp. 1-26, 2008.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref58\">58- V. Labatut and H. Cherifi, \"Accuracy measures for the comparison of classifiers,\" <i>arXiv preprint arXiv:1207.3790</i>, 2012.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref59\">59- S. Wallis, \"Binomial confidence intervals and contingency tests: mathematical fundamentals and the evaluation of alternative methods,\" <i>Journal of Quantitative Linguistics</i>, vol. 20, no. 3, pp. 178-208, 2013.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref60\">60- D. Altman, D. Machin, T. Bryant, and M. Gardner, <i>Statistics with confidence: confidence intervals and statistical guidelines</i>. John Wiley & Sons, 2013.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref61\">61- J. A. Hanley and B. J. McNeil, \"The meaning and use of the area under a receiver operating characteristic (ROC) curve,\" <i>Radiology</i>, vol. 143, no. 1, pp. 29-36, 1982.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref62\">62- E. B. Wilson, \"Probable inference, the law of succession, and statistical inference,\" <i>Journal of the American Statistical Association</i>, vol. 22, no. 158, pp. 209-212, 1927.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref63\">63- A. Agresti and B. A. Coull, \"Approximate is better than “exact” for interval estimation of binomial proportions,\" <i>The American Statistician</i>, vol. 52, no. 2, pp. 119-126, 1998.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref64\">64- C. S. Peirce, \"The numerical measure of the success of predictions,\" <i>Science</i>, no. 93, pp. 453-454, 1884.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref65\">65- E. W. Steyerberg, B. Van Calster, and M. J. Pencina, \"Performance measures for prediction models and markers: evaluation of predictions and classifications,\" <i>Revista Española de Cardiología (English Edition)</i>, vol. 64, no. 9, pp. 788-794, 2011.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref66\">66- A. J. Vickers and E. B. Elkin, \"Decision curve analysis: a novel method for evaluating prediction models,\" <i>Medical Decision Making</i>, vol. 26, no. 6, pp. 565-574, 2006.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref67\">67- G. W. Bohrnstedt and D. Knoke,\"Statistics for social data analysis,\" 1982.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref68\">68- W. M. Rand, \"Objective criteria for the evaluation of clustering methods,\" <i>Journal of the American Statistical association</i>, vol. 66, no. 336, pp. 846-850, 1971.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref69\">69- J. M. Santos and M. Embrechts, \"On the use of the adjusted rand index as a metric for evaluating supervised classification,\" in <i>International conference on artificial neural networks</i>, 2009: Springer, pp. 175-184.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref70\">70- J. Cohen, \"Weighted kappa: nominal scale agreement provision for scaled disagreement or partial credit,\" <i>Psychological bulletin</i>, vol. 70, no. 4, p. 213, 1968.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref71\">71- R. Bakeman and J. M. Gottman, <i>Observing interaction: An introduction to sequential analysis</i>. Cambridge university press, 1997.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref72\">72- S. Bangdiwala, \"A graphical test for observer agreement,\" in <i>45th International Statistical Institute Meeting</i>, 1985, vol. 1985, p. 307.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref73\">73- K. Bangdiwala and H. Bryan, \"Using SAS software graphical procedures for the observer agreement chart,\" in <i>Proceedings of the SAS Users Group International Conference</i>, 1987, vol. 12, pp. 1083-1088.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref74\">74- A. F. Hayes and K. Krippendorff, \"Answering the call for a standard reliability measure for coding data,\" <i>Communication methods and measures</i>, vol. 1, no. 1, pp. 77-89, 2007.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref75\">75- M. Aickin, \"Maximum likelihood estimation of agreement in the constant predictive probability model, and its relation to Cohen's kappa,\" <i>Biometrics</i>, pp. 293-302, 1990.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref76\">76- N. A. Macmillan and C. D. Creelman, <i>Detectiontheory: A user's guide</i>. Psychology press, 2004.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref77\">77- D. J. Hand, P. Christen, and N. Kirielle, \"F*: an interpretable transformation of the F-measure,\" <i>Machine Learning</i>, vol. 110, no. 3, pp. 451-456, 2021.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref78\">78- G. W. Brier, \"Verification of forecasts expressed in terms of probability,\" <i>Monthly weather review</i>, vol. 78, no. 1, pp. 1-3, 1950.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref79\">79- L. Buitinck et al., \"API design for machine learning software: experiences from the scikit-learn project,\" <i>arXiv preprint arXiv:1309.0238</i>, 2013.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref80\">80- R. W. Hamming, \"Error detecting and error correcting codes,\" The Bell system technical journal, vol. 29, no. 2, pp. 147-160, 1950.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref81\">81- S. S. Choi, S. H. Cha, and C. C. Tappert, \"A survey of binary similarity and distance measures,\" Journal of systemics, cybernetics and informatics, vol. 8, no. 1, pp. 43-48, 2010.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref82\">82- J. Braun-Blanquet, \"Plant sociology. The study of plant communities,\" Plant sociology. The study of plant communities. First ed., 1932.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref83\">83- C. C. Little, \"Abydos Documentation,\" 2020.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref84\">84- K. Villela, A. Silva, T. Vale, and E. S. de Almeida, \"A survey on software variability management approaches,\" in <i>Proceedings of the 18th International Software Product Line Conference-Volume 1</i>, 2014, pp. 147-156.</blockquote>\n",
    "    \n",
    "<blockquote id=\"ref85\">85- J. R. Saura, A. Reyes-Menendez, and P. Palos-Sanchez, \"Are black Friday deals worth it? Mining Twitter users’ sentiment and behavior response,\" <i>Journal of Open Innovation: Technology, Market, and Complexity</i>, vol. 5, no. 3, p. 58, 2019.</blockquote>\n",
    "\n",
    "<blockquote id=\"ref86\">86- P. Schubert and U. Leimstoll, \"Importance and use of information technology in small and medium‐sized companies,\" <i>Electronic Markets</i>, vol. 17, no. 1, pp. 38-55, 2007.</blockquote>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<script type=\"text/javascript\">\n",
    "var sc_project=12845182; \n",
    "var sc_invisible=1; \n",
    "var sc_security=\"001dde8f\"; \n",
    "</script>\n",
    "<script type=\"text/javascript\"\n",
    "src=\"https://www.statcounter.com/counter/counter.js\" async></script>\n",
    "<noscript><div class=\"statcounter\"><a title=\"Web Analytics\"\n",
    "href=\"https://statcounter.com/\" target=\"_blank\"><img class=\"statcounter\"\n",
    "src=\"https://c.statcounter.com/12845182/0/001dde8f/1/\" alt=\"Web Analytics\"\n",
    "referrerPolicy=\"no-referrer-when-downgrade\"></a></div></noscript>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "PyCM Document",
   "title_sidebar": "PyCM Document",
   "toc_cell": true,
   "toc_position": {
    "height": "382px",
    "left": "10px",
    "top": "10px",
    "width": "256px"
   },
   "toc_section_display": false,
   "toc_window_display": true
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}