From 0960c45243147cd348391e266020f21d8b1e6570 Mon Sep 17 00:00:00 2001 From: Joshua We <80349780+joshuawe@users.noreply.github.com> Date: Wed, 6 Dec 2023 21:28:19 +0100 Subject: [PATCH] add multiclass prob histogram (#22) --- notebooks/multiclass_classification.ipynb | 131 ++++++++++++++++++++++ plotsandgraphs/__init__.py | 1 + plotsandgraphs/multiclass_classifier.py | 73 ++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 notebooks/multiclass_classification.ipynb create mode 100644 plotsandgraphs/multiclass_classifier.py diff --git a/notebooks/multiclass_classification.ipynb b/notebooks/multiclass_classification.ipynb new file mode 100644 index 0000000..194af92 --- /dev/null +++ b/notebooks/multiclass_classification.ipynb @@ -0,0 +1,131 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Multiclass Classification\n", + "\n", + "This notebook explores multiclass classification" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import plotsandgraphs as pandg\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create some dummy data" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[9.09443337e-01, 9.03393597e-01, 8.31210285e-01],\n", + " [2.21902873e-02, 2.76896844e-01, 1.17129033e-02],\n", + " [3.17239348e-01, 9.98593024e-01, 1.63289150e-02],\n", + " ...,\n", + " [2.58836187e-02, 2.28105168e-01, 5.37207598e-01],\n", + " [2.17134178e-01, 5.08693900e-01, 2.65985367e-01],\n", + " [2.86897406e-15, 7.97772016e-01, 3.77128950e-02]])" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# True labels\n", + "y_true = np.random.choice([0, 1, 2], p=[0.3, 0.5, 0.2], size=1000)\n", + "# one hot encoding\n", + "y_true_one_hot = np.eye(3)[y_true] \n", + "\n", + "# Predicted labels\n", + "y_pred = np.ones(y_true_one_hot.shape)\n", + "\n", + "a0, b0 = [0.1, 0.6, 0.3, 0.4], [0.4, 1.2, 0.8, 1]\n", + "a1, b1 = [0.9, 0.8, 0.9, 1.2], [0.4, 0.1, 0.5, 0.3]\n", + "# iterate through all the columns/labels\n", + "for i in range(y_pred.shape[1]):\n", + " y = y_pred[:, i]\n", + " y_t = y_true_one_hot[:, i]\n", + " y[y_t==0] = np.random.beta(a0[i], b0[i], size=y[y_t==0].shape)\n", + " y[y_t==1] = np.random.beta(a1[i], b1[i], size=y[y_t==1].shape)\n", + "y_pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Histogram of probabilities" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAI4CAYAAAD6VFg7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABILElEQVR4nO3deZhedX3//+d7skIWImEJJpRhcwG0SgMGqVbFWsQWaLUIbqDYuKBfW1qQ1rZS/X0VqUWxLjSKX0EriLiA+wIoCkQUpGxiCTCRRJJAICtJSDLv3x/3GbgZkpl7JnPOPXOf5+O65ppzn/O5z+f9uWdy5pWzRmYiSZKk+uhqdwGSJEmqlgFQkiSpZgyAkiRJNWMAlCRJqhkDoCRJUs0YACVJkmrGACjVQER8ISL+v2L6RRHx24r6zYg4oIJ+uou+xg/z/dutMyJeHxE/3FbbiLggIv5leFU/pZ+zI+JLAyy/IyJeMhJ9SZIBUBolIqInIjZExLqIWF6Etqkj3U9m/iwzn9lCPadExM9Huv+xJjP/OzNfsZ1lb8/MDwJExEsiYkmJdRycmT8ZqM2OBmFJ9WEAlEaXv8jMqcChwFzgn/s3qOMf9zqOeTTy5yB1DgOgNApl5lLge8Ah8Phhx9Mi4m7g7mLen0fELRGxKiKuj4jn9r0/Ip4fETdHxNqI+AowuWnZk/ZURcTeEfH1iHgwIlZGxCcj4tnABcARxR7JVUXbSRHx0Yj4XbGX8oKI2KlpXWdExAMR8fuIeMtAY4yIn0TEhyPixohYExFXRMSuxbK+PVmnRsTvgKsjoisi/jkiFkfEioi4OCJ26bfatxR9PxAR/9DU1+ERcUPxWT1QjHFiv/ceExH3RsRDEfHvEdFVvHe7e0L7Dq1HxJTi5/X04vNaFxFPj4hHI2JmU/tDi895wnY+lonFuNYWh3znNr23JyJe3jSeXxWf2/KIOK9odm3xfVVRwxGDfW4R8aZi2cqI+Jd+/ZwdEZdHxJciYg1wymCfZfFze2dE3F2M44MRsX/xO7omIi7bxmcvqWIGQGkUioi9gWOAXzfNPh54AXBQRDwf+DzwNmAm8F/AlUVAmwh8E/gisCvwVeDV2+lnHPBtYDHQDcwGLs3M3wBvB27IzKmZOaN4yznAM4DnAQcU7f+1WNfRwD8AfwocCLy8haG+CXgLsBewBfhEv+V/Ajwb+DPglOLrpcB+wFTgk/3av7To+xXAe/uCDLAV+DtgN+AI4Cjgnf3e+5c09roeChxX1NWSzFwPvBL4ffF5Tc3M3wM/AU5oavpGGp/v5u2s6ljgUmAGcOU2xtfnfOD8zJwO7A9cVsx/cfF9RlHDDQzwuUXEQcCngdfT+BnsQuNn2uw44PKipv+mtc/yz4A/AuYBZwILgDcAe9P4T81J2xmXpIoYAKXR5ZvF3rafAz8FPtS07MOZ+XBmbgDmA/+Vmb/IzK2ZeRGwicYf3HnABODjmbk5My8Hfrmd/g4Hng6ckZnrM3NjZm5vb1cU/f5dUcfaor4TiyYnAP8vM28vAtHZLYz3i03t/wU4oQilfc4u6tpAI6Scl5n3ZuY64B+BE+PJhyX/rWh/G/D/KIJGZt6UmQszc0tm9tAIzH/Sr5aPFOP6HfBxRiakXEQj+PSF7ZNoBPPt+Xlmfjcztxbt/nA77TYDB0TEbpm5LjMXDrDOgT631wDfysyfZ+ZjNMJ8/wfE35CZ38zM3szc0OJneW5mrsnMO4DbgR8W/a+msaf0+QPUK6kCns8hjS7HZ+aPt7Ps/qbpfYCTI+LdTfMm0ghzCSzNzOY/5Iu3s869gcWZuaWF2nYHdgZuamRBAALoC2xPB25qoc9mzWNaTCO47rad5U/vt87FNLZhew6wvucARMQzgPNo7OHbuXhfc63beu/TW6h/MFcAF0TEvsAzgdWZeeMA7Zc1TT8KTI6I8dv4+ZwKfAC4KyLuoxF8v72ddQ70uT2dpnFn5qMRsbLf+5s/l1Y/y+VN0xu28XrWdmqVVBH3AEpjR3Ogux/4v5k5o+lr58y8BHgAmB1NKQ34g+2s837gD2LbJ/f33xP0EI0/3gc39blLcdEKRb97t9Bns/7tNxf9bKuG39MIvs3tt/DkcNF/fb8vpj8D3AUcWBw2/Sca4XWgWn7P0PT/vMjMjTQOz76BxuHfgfb+td5R5t2ZeRKwB/AR4PLiPMSn1MDAn9sDwJy+BcX5nDN5sv7rbOWzlDTKGQClsemzwNsj4gXRMCUiXhUR04AbaPyB/z8RMSEi/orGod5tuZFGCDinWMfkiDiyWLYcmNN3wn5m9hb9fiwi9gCIiNkR8WdF+8toXCRwUETsDLy/hXG8oan9B4DLi8Of23IJ8HcRsW80bo/zIeAr/faO/UtE7BwRBwNvBr5SzJ8GrAHWRcSzgHdsY/1nRMTTivMv39P03lYtB2bGUy9MuZjGOXjHMkIBMCLeEBG7Fz+TVcXsXuDB4vt+Tc0H+twuB/4iIl5Y/JzPZvAw18pnKWmUMwBKY1Bm/gr4Gxon8z8CLKIRMijO5fqr4vXDwGuBr29nPVuBv6BxQcfvgCVFe4CrgTuAZRHRt1fuvUVfC4urQn9M49Ammfk9GufOXV20ubqFoXwR+AKNQ5+Tgf8zQNvPF+2vBe4DNgLv7tfmp0XfVwEfzcy+Gzj/A/A6YC2NELutcHcFjUOZtwDfAS5sof7HZeZdNMLWvcUVsk8v5l9HI5TdnJmtHBZvxdHAHRGxjsYFIScW5+c9Cvxf4LqihnkM8LkV5+i9m8aFJw8A64AVNM4n3Z5WPktJo1w8+TQhSapGRPwE+FJmfq7dtZQtIq4Gvjzax1rsIVxF4/DufW0uR1KJ3AMoSSWKiMNo3FpmVO4pi4i/KA6bTwE+CtwG9LS3KkllMwBKUkki4iIah8n/trhtzmh0HI0LRX5P4x6KJ6aHhqSO5yFgSZKkmnEPoCRJUs0YACVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZgyAkiRJNWMAlCRJqhkDoCRJUs0YADXmRMTZEfGldtchSa1yu6XRxgCoUSkiXhcRv4qIdRHxQER8LyL+uE21dEfENRHxaETcFREvb0cdkka3Ubbd+mBE3BYRWyLi7HbUoNHNAKhRJyJOBz4OfAjYE/gD4NM0HlrfDpcAvwZmAu8DLo+I3dtUi6RRaBRutxYBZwLfaVP/GuUMgBpVImIX4APAaZn59cxcn5mbM/NbmXnGdt7z1YhYFhGrI+LaiDi4adkxEXFnRKyNiKUR8Q/F/N0i4tsRsSoiHo6In0XEU/49RMQzgEOB92fmhsz8GnAb8Ooyxi9p7Blt2y2AzLwoM78HrC1hyOoABkCNNkcAk4FvDOE93wMOBPYAbgb+u2nZhcDbMnMacAhwdTH/74ElwO40/rf+T0BuY90HA/dmZvNG9H+K+ZIEo2+7JQ1qfLsLkPqZCTyUmVtafUNmfr5vujjX5ZGI2CUzVwObgYMi4n8y8xHgkaLpZmAvYJ/MXAT8bDurnwqs7jdvNTC71fokdbzRtt2SBuUeQI02K4HdIqKl/5xExLiIOCci7omINUBPsWi34vurgWOAxRHx04g4opj/7zTOkflhRNwbEWdtp4t1wPR+86bjYRVJTxht2y1pUAZAjTY3AJuA41ts/zoaJ1m/HNgF6C7mB0Bm/jIzj6NxmOWbwGXF/LWZ+feZuR9wLHB6RBy1jfXfAewXEdOa5v1hMV+SYPRtt6RBGQA1qhSHP/4V+FREHB8RO0fEhIh4ZUScu423TKOx4V0J7EzjCjwAImJiRLy+OKyyGVgD9BbL/jwiDoiIoHFId2vfsn71/C9wC/D+iJgcEX8JPBf42ggOW9IYNtq2W0XbCRExmcbf+fHF9mvcyI1aY50BUKNOZv4HcDrwz8CDwP3Au2j8T7i/i4HFwFLgTmBhv+VvBHqKwyxvB15fzD8Q+DGNQ7w3AJ/OzGu2U9KJwFwa5+GcA7wmMx8cztgkdaZRuN36LLABOInG7as2FOuVAIhMLyCSJEmqE/cASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNjOlHwe222245e/ZsJkyY0O5SSrF58+aOHRt09vgc29h00003PZSZu5fZR6dvt6Czf0cc29jUyWMb7nZrTAfA7u5uLr/8crq7u9tdSil6eno6dmzQ2eNzbGNTRCwuu49O325BZ/+OOLaxqZPHNtztloeAJUmSasYAKEmSVDMGQEmSpJoZ0+cAStpxmzdvZsmSJWzcuLGl9lu2bOE3v/lNyVWVa/LkycyZM6djTwqXpMEYAKWaW7JkCdOmTaO7u5uIGLT9pk2bmDRpUgWVlSMzWblyJUuWLGHfffdtdzmS1BalHgKOiJ6IuC0ibomIXxXzdo2IH0XE3cX3pxXzIyI+ERGLIuLWiDi0zNokNWzcuJGZM2e2FP46QUQwc+bMlvd4SlInquIcwJdm5vMyc27x+izgqsw8ELiqeA3wSuDA4ms+8JkKapMEtQl/feo2Xknqrx0XgRwHXFRMXwQc3zT/4mxYCMyIiL3aUJ8kSVJHK/scwAR+GBEJ/FdmLgD2zMwHiuXLgD2L6dnA/U3vXVLMe4DtWLLkES644Bp6e6eMfOUDOPfcEyrtT5IkaSSVHQD/ODOXRsQewI8i4q7mhZmZRThsWUTMp3GImOnTZ7Jk1cOs27h25CoexLNmTaOnp6eSvpYtW1ZJP+3SyeMbS2PbsmULmzZtarn95s2bS6vl9ttv51WvehXf+c53OOSQQ0rrBxrjrurfcvN2a/bs2WPq92M4Onl8jm1s6uSxDVepATAzlxbfV0TEN4DDgeURsVdmPlAc4l1RNF8K7N309jnFvP7rXAAsAJg1a/9ct3ECzz72pWUO43E3XHIdvb1TKn2cTKc+uqZPJ49vrIztN7/5zZCv6i3rKuCPfvSjXH/99bzvfe/jkksuKaWPPuPHj6/sZ9S83Zo7d27OmjVrzPx+DFcnj8+xjU2dPLbhKC0ARsQUoCsz1xbTrwA+AFwJnAycU3y/onjLlcC7IuJS4AXA6qZDxZIqcOaZlw3aZuvWrYwbN25I6231tIm+0Lcj4e/73/8+73nPe9i6dStvfetbOeusswZ/kyTVTJl7APcEvlFcbTce+HJmfj8ifglcFhGnAouBvr8M3wWOARYBjwJvLrE2Sdtx430PD7g8M4d0Fe3h++66oyW1bOvWrZx22mn86Ec/Ys6cORx22GEce+yxHHTQQZXVIEljQWkBMDPvBf5wG/NXAkdtY34Cp5VVj6TWHXHSkdtd1rt1K10t7gG84ZLrWmp3++23M3/+fK6//noAbr75Zs444wyuuuqqlt7f58Ybb+SAAw5gv/32A+DEE0/kiiuuMABKUj8+CURS2x100EHce++9jx9ePv300znvvPOe1OZFL3oRa9c+9YKvj370o7z85S8HYOnSpey99xOnEs+ZM4df/OIX5RYvSWOQAVBS23V1dXHwwQdzxx13cPfdd7PPPvtw6KFPfhjQz372szZVJ0mdxwAoaVSYN28e1113HZ/+9Kf5/ve//5TlrewBnD17Nvff/8TtRJcsWcLs2bPLK1qSxigDoKRRYd68eZxyyimcdtpp2wxtrewBPOyww7j77ru57777mD17Npdeeilf/vKXyyhXksY0A6Ckpxjo4o2hXgXcqmc961lMmjSJ9773vcNex/jx4/nkJz/Jn/3Zn7F161be8pa3cPDBB49glZLUGQyAkp5ksNu2DOc+gK04//zz+fCHP8yUKTv2aMdjjjmGY445ZoSqkqTOZACU9LhWbti8adOmEX0SyD333MOrXvUqjjzySE4++eQRW68kafsMgJLaav/99+euu+4avKEkacR0tbsASZIkVcsAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJY0at912G7NmzeK2225rdymS1NEMgJJGjQ996ENcf/31fOhDH2p3KZLU0QyAkkaNSy65hP32249LLrlk2Ot4y1vewh577MEhhxwygpVJUmfxWcCSHnfmmZcN2mbr1q2MGzduSOs999wThlvSkJ1yyim8613v4k1velNlfUrSWGMAlPRki68fcHFX9kIM4eDBPi8ctMntt9/O/Pnzuf76Rt8333wzZ5xxBldddVXr/RRe/OIX09PTM+T3SVKdGAAlPcW5b8jtLtu6tZdx46Kl9Zz5pdbaHXTQQdx7772P7108/fTTOe+8857U5kUvehFr1659yns/+tGP8vKXv7ylfiRJDQZASW3X1dXFwQcfzB133MHdd9/NPvvsw6GHHvqkNj/72c/aVJ0kdR4DoKRRYd68eVx33XV8+tOf5vvf//5TlrsHUJJGjgFQ0qgwb948TjnlFE477TRmz579lOXuAZSkkeNtYCSNCs961rOYNGkS733ve3doPSeddBJHHHEEv/3tb5kzZw4XXnjhCFUoSZ3DPYCSnmKgizcyu4ho7eKOoTj//PP58Ic/zJQpU3ZoPTtyD0FJqgsDoKQnG+S2Lb3DuA/gQO655x5e9apXceSRR3LyySeP2HolSdtnAJT0uFZu2Lxp0yYmTZo0Yn3uv//+3HXXXSO2PknS4DwHUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNlB4AI2JcRPw6Ir5dvN43In4REYsi4isRMbGYP6l4vahY3l12bZIaMrf/7N9OVLfxSlJ/VewBfA/wm6bXHwE+lpkHAI8ApxbzTwUeKeZ/rGgnqWSTJ09m5cqVtQlFmcnKlSuZPHlyu0uRpLYp9TYwETEHeBXwf4HTo3H32JcBryuaXAScDXwGOK6YBrgc+GRERNblr5LUJnPmzGHJkiU8+OCDLbXfsmUL48eP7TtITZ48mTlz5rS7DElqm7K34h8HzgSmFa9nAqsyc0vxegnQ99DP2cD9AJm5JSJWF+0fKrlGqdYmTJjAvvvu23L7np4euru7yytIklS60gJgRPw5sCIzb4qIl4zgeucD8wGmT5/J1Mmb6VrX2p6LHTVjymN0da2np6enkv6WLVtWST/t0snjc2xq1rzdmj17dsd/hp08Psc2NnXy2IarzD2ARwLHRsQxwGRgOnA+MCMixhd7AecAS4v2S4G9gSURMR7YBVjZf6WZuQBYADBr1v65buMEeqfuXuIwnrBq/UR6e6dUuvej0/e0dPL4HJv6NG+35s6dm7Nmzer4z7CTx+fYxqZOHttwlHYRSGb+Y2bOycxu4ETg6sx8PXAN8Jqi2cnAFcX0lcVriuVXe/6fJEnSyGvHfQDfS+OCkEU0zvG7sJh/ITCzmH86cFYbapMkSep4lVzKl5k/AX5STN8LHL6NNhuBv66iHkmSpDob2/dykCRJGiXOPPOydpfQMgOgJEnSSFl8fbsraIkBUJIkaQSd+4ZqrmE980sx7Pe24yIQSZIktZEBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaqZ8QMtjIi/amEdGzPzuyNUjyRJkko2YAAEPgtcAcQAbV4MGAAlSZLGiMEC4Pcy8y0DNYiIL21n/mTgWmBS0c/lmfn+iNgXuBSYCdwEvDEzH4uIScDFwB8BK4HXZmbPUAYjSZKkwQ14DmBmvmGwFQzQZhPwssz8Q+B5wNERMQ/4CPCxzDwAeAQ4tWh/KvBIMf9jRTtJkiSNsCFfBBIRC1pplw3ripcTiq8EXgZcXsy/CDi+mD6ueE2x/KiIGOjQsyRJkoZhOFcBz221YUSMi4hbgBXAj4B7gFWZuaVosgSYXUzPBu4HKJavpnGYWJIkSSNosHMAt2VFqw0zcyvwvIiYAXwDeNYw+nuSiJgPzAeYPn0mUydvpmvdgzu62pbMmPIYXV3r6enpqaS/ZcuWVdJPu3Ty+BybmjVvt2bPnt3xn2Enj8+xjU1Vja2raz3sPJ6etVlNfzsP/0DpkANgZh49jPesiohrgCOAGRExvtjLNwdYWjRbCuwNLImI8cAuNC4G6b+uBcACgFmz9s91GyfQO3X3oZY0LKvWT6S3dwrd3d2V9AdU2lc7dPL4HJv6NG+35s6dm7Nmzer4z7CTx+fYxqYqxtbbOwUe3UL3tGoCYO+jww+AAx4CjoizB1vB9tpExO7Fnj8iYifgT4HfANcArymanUzjNjMAVxavKZZfnZnVfIKSJEk1MtgewLdGxJoBlgdwInD2NpbtBVwUEeNoBM3LMvPbEXEncGlE/H/Ar4ELi/YXAl+MiEXAw8V6JUmSNMJauRH0tBbaPEVm3go8fxvz7wUO38b8jcBfD9KXJEmSdtCAATAz/62qQiRJklSN4dwGRpIkSWOYAVCSJKlmWgqAEXFkK/MkSZI0+rW6B/A/W5wnSZKkUW7Ai0Ai4gjghcDuEXF606LpwLgyC5MkSVI5BrsNzERgatGu+XYwa3jiZs6SJEkaQwa7DcxPgZ9GxBcyc3FFNUmSJKlErT4LeFJELAC6m9+TmS8royhJkiSVp9UA+FXgAuBzwNbyypEkSVLZWg2AWzLzM6VWIkmSpEq0ehuYb0XEOyNir4jYte+r1MokSZJUilb3AJ5cfD+jaV4C+41sOZIkSSpbSwEwM/ctuxBJkiRVo6UAGBFv2tb8zLx4ZMuRJElS2Vo9BHxY0/Rk4CjgZsAAKEmSNMa0egj43c2vI2IGcGkZBUmSJKlcrV4F3N96wPMCJUmSxqBWzwH8Fo2rfgHGAc8GLiurKEmSJJWn1XMAP9o0vQVYnJlLSqhHkiRJJWvpEHBm/hS4C5gGPA14rMyiJEmSVJ6WAmBEnADcCPw1cALwi4h4TZmFSZIkqRytHgJ+H3BYZq4AiIjdgR8Dl5dVmCRJksrR6lXAXX3hr7ByCO+VJEnSKNLqHsDvR8QPgEuK168FvldOSZIkSSpTqzeCPiMi/gr442LWgsz8RnllSZIkqSwDBsCIOADYMzOvy8yvA18v5v9xROyfmfdUUaQkSZJGzmDn8X0cWLON+auLZZIkSRpjBguAe2bmbf1nFvO6S6lIkiRJpRosAM4YYNlOI1iHJEmSKjJYAPxVRPxN/5kR8VbgpnJKkiRJUpkGuwr4b4FvRMTreSLwzQUmAn9ZYl2SJEkqyYABMDOXAy+MiJcChxSzv5OZV5demSRJkkrR6n0ArwGuGcqKI2Jv4GJgTyBp3Dvw/IjYFfgKjYtIeoATMvORiAjgfOAY4FHglMy8eSh9SpIkaXBlPs5tC/D3mXkQMA84LSIOAs4CrsrMA4GritcArwQOLL7mA58psTZJkqTaKi0AZuYDfXvwMnMt8BtgNnAccFHR7CLg+GL6OODibFgIzIiIvcqqT5Ikqa5afRbwDomIbuD5wC9o3FvwgWLRMhqHiKERDu9vetuSYt4DTfOIiPk09hAyffpMpk7eTNe6B8srvsmMKY/R1bWenp6eSvpbtmxZJf20SyePz7GpWfN2a/bs2R3/GXby+Bzb2FTV2Lq61sPO4+lZm9X0t3MM+72lB8CImAp8DfjbzFzTONWvITMzIob0KWXmAmABwKxZ++e6jRPonbr7SJa8XavWT6S3dwrd3d2V9AdU2lc7dPL4HJv6NG+35s6dm7Nmzer4z7CTx+fYxqYqxtbbOwUe3UL3tGoCYO+jww+AZZ4DSERMoBH+/rt4ljDA8r5Du8X3FcX8pcDeTW+fU8yTJEnSCCotABZX9V4I/CYzz2tadCVwcjF9MnBF0/w3RcM8YHXToWJJkiSNkDIPAR8JvBG4LSJuKeb9E3AOcFlEnAosBk4oln2Xxi1gFtG4DcybS6xNkiSptkoLgJn5c2B7B6eP2kb7BE4rqx5JkiQ1lHoOoCRJkkYfA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWZKfxawJOkJS5Y8wgUXXNN4ZmiFzj33hMEbSaoNA6AkVeyuZWtZtX5TZf0dvu+ulfUlaWwwAEpSGxxx0pGV9HPDJddV0o+kscVzACVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZgyAkiRJNWMAlCRJqhkDoCRJUs0YACVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZgyAkiRJNWMAlCRJqhkDoCRJUs0YACVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZgyAkiRJNVNaAIyIz0fEioi4vWnerhHxo4i4u/j+tGJ+RMQnImJRRNwaEYeWVZckSVLdlbkH8AvA0f3mnQVclZkHAlcVrwFeCRxYfM0HPlNiXZIkSbVWWgDMzGuBh/vNPg64qJi+CDi+af7F2bAQmBERe5VVmyRJUp2Nr7i/PTPzgWJ6GbBnMT0buL+p3ZJi3gNIqqUzz7ys3SVIUseqOgA+LjMzInKo74uI+TQOEzN9+kymTt5M17oHR7y+bZkx5TG6utbT09NTSX/Lli2rpJ926eTxObYd19W1HpbfUUlfZavTdgv8/R+rHNuO6+paDzuPp2ftkOPN8PrbOYb93qoD4PKI2CszHygO8a4o5i8F9m5qN6eY9xSZuQBYADBr1v65buMEeqfuXmbNj1u1fiK9vVPo7u6upD+g0r7aoZPH59h2TG/vFHh0C+e+oZoN6ZlfGv6GdDB1226Bv/9jlWPbMX3bre5p1Wy3eh8d/nar6tvAXAmcXEyfDFzRNP9NxdXA84DVTYeKJUmSNIJK2wMYEZcALwF2i4glwPuBc4DLIuJUYDFwQtH8u8AxwCLgUeDNZdUlSZJUd6UFwMw8aTuLjtpG2wROK6sWSZIkPcEngUiSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNVMaU8CkSS13/JFy1i4/BHOPPOyyvp85zsPr6wvScNjAJSkDvdQjOPG+x6upK/D9921kn4k7RgDoKRRaeHCe9i3aw0LFz5YSX/Ll+9eST/tcsRJR5bexw2XXFd6H5JGhgFQ0qi1cfNWVqzdWFlfklQXBkBJo9qeB8yqpqObqummk/Wdb9jVtZ7e3imV9HnuuSdU0o/UaQyAkqQR81CM465la1m1flPpfXm+oTR8BkBJ0og6+GWH0Du13HMqPd9Q2jHeB1CSJKlmDICSJEk1YwCUJEmqGc8BlCSNOe14wolXHKuTGAAlSWOSTziRhs8AKKllZ555WWX3eFuxYg27T9+CZ6poID7hRBoeA6CkoVl+Bzy6pfRupsVjpfchSXVlAJRGWJV7yfpUfW7SuW/I0vs47MzSu5Ba0ny+Ydn/thcuvAeAefP2L62PbfH8xvoxAEplqGgv2cK7gal7VnYi/MKF9/DsXdcDEyvpTxot+s43nDHlsVKfcvLg2k3E9J0rO7dx+aJl7D5tUseG23PPPaHS/5QvXHgP+3atAaaV3teOMgAOQdVXnXV1reecc95cSV8aeVXsJXvx+4N9u+6GxctL7wuAdYDnwqumjjjpSLrWPVjqU06++cGvPd5XFb75wa91ZLhtDraN/7j+jo2ryn084Z57Tm9sI6eX2s2IMQAOUZVXnc3bf1Il/WjsqyJsAhx25mYee2wLCxcuKb2vrVtnk73VjEuqu04Lt83B9sG1m9h/ei+vfu79pfS1Yc0Gdpowjj32mM41t+7NxomPsXDhvaX01WfevP12eB0GwGHwqrOxpcr7hEHnHybd2pusWLux9H7MflLnKvtv3IY1G2DDZtjjid1xex4wq5S+em6+j40EK9ZuZGtvsqXkbeQe0yaPyHrGfAB8ZOnDlYWlx3+hOlDVIQmqO8S9cOE9PLi23F3/zTas2cD+U7fQqQEQytuQSqqP7vW3lLbumTPXQlcX09av5NEtE0rrp9meB8xi/DXjGT9hfGnbyOWLlo3YusZ8AIRyf4mazZy5lpvWdFfS1/JFy7hz04RKT+5/KBq7sKuwfNEyDty7mvGtWLGGg2csYdqu1VyVe+ujUUk/kjTWvePo9aWst+fmxcSE8ezznL1592dnlNLHWNcRARDK+yVqdu6XSu/iSVas2sDdP7qzkr7WPtT431JVAXDDmg2sWLWFNRWcT7lxcy9Qze8IwDs+NbWSfiRJGq6OCYBVuOeRaWx57LFKDjlvWLOBqROD5+58PzvtslPp/d2yDvbbZQ3T168tvS+AVV2wddOMSvra8tgWcstWYFwl/fX2Jlu39pZ+EjDAhg2NE44laUfdcfXtrFpf3qkrfadR3XDJdSxftIw9d95QWl8anAFwiA7ffWklIWnmzLWsnjATspo9V2+7u3F4tKq9ZG+7ewrPmvEg49avKr2vVSNzvuyQJFRyocTmLb1s2VJN2ITiytz06oyRUtV/Jvv+6EJ1txfpVFWHJCj3nNvm/nLVw8x69L7Sdjq047w8bd+oCoARcTRwPo1dNZ/LzHPaXNI2VRGSem5ezGcXzSy9n3arMtxWraoLJaoKm+CVuWUo+xzmvj+699+1hrXjZ5YaOvvCxM+/eC0rHqouuJQdlNoZkvadvoad1o/cif8D9bds686A5+XVxagJgBExDvgU8KfAEuCXEXFlZlZzEpw0hnlV7thW5n+E+v7ofnThDJ67832VhIn1W3fnuTuvrCy4lB2U2h2Sqvj92Oc5e/P+b+5cWj8afUZNAAQOBxZl5r0AEXEpcBxgAJSkEVJFmPj8PbtX1lcVQcmQpE4Uo+V8noh4DXB0Zr61eP1G4AWZ+a7tvWfWrP3zmc95I7Meva+SGn+7Yif222UNEyaVf+7Cls1bWD1xJrs8tpLxE8rP6VWOra+/5+y3mcfWlf+8XMc2sv116tgALrv+4psyc26ZfcyatX/OO/o9rFo/keWLlvHcncvdfm3ZvAW6urjvkansu8uaUrcnfX2tmbAr0zc9VElf48d1cfdDO5c6tua+Hu6aUerYqhxX//46aWz9+zpk381sWlvOfXyr/plNnjCu8di5wr9fdv6wtltjLgBGxHxgfvHymcBK4KEqa63QbnTu2KCzx+fYxqZnZuaIP8W9Ztst6OzfEcc2NnXy2Ia13RpNh4CXAns3vZ5TzHuSzFwALOh7HRG/Kvt/7O3SyWODzh6fYxubIuJXZay3Ttst6OzxObaxqdPHNpz3dY10ITvgl8CBEbFvREwETgSubHNNkiRJHWfU7AHMzC0R8S7gBzRuA/P5zLyjzWVJkiR1nFETAAEy87vAd4f4tgWDNxmzOnls0Nnjc2xjU1Vj6+TPEDp7fI5tbHJs/Yyai0AkSZJUjdF0DqAkSZIqMGYCYEQcHRG/jYhFEXHWNpZPioivFMt/ERHdbShzWFoY2+kRcWdE3BoRV0XEPu2oczgGG1tTu1dHREbEmLlKq5WxRcQJxc/ujoj4ctU1DlcLv5N/EBHXRMSvi9/LY9pR53BExOcjYkVE3L6d5RERnyjGfmtEHLoDfbndcrs1qrjdcrv1uMwc9V80Lgq5B9gPmAj8D3BQvzbvBC4opk8EvtLuukdwbC8Fdi6m39FJYyvaTQOuBRYCc9td9wj+3A4Efg08rXi9R7vrHsGxLQDeUUwfBPS0u+4hjO/FwKHA7dtZfgzwPSCAecAvSvwc3W6Nsi+3W263RuNXGdutsbIH8PHHxGXmY0DfY+KaHQdcVExfDhwVEVFhjcM16Ngy85rMfLR4uZDGPRLHglZ+bgAfBD4CbKyyuB3Uytj+BvhUZj4CkJkrKq5xuFoZWwJ9t6LfBfh9hfXtkMy8Fnh4gCbHARdnw0JgRkTsNYyu3G653Rpt3G653XrcWAmAs4H7m14vKeZts01mbgFWAzMrqW7HtDK2ZqfSSPljwaBjK3ZT752Z36mysBHQys/tGcAzIuK6iFgYEUdXVt2OaWVsZwNviIglNK7cf3c1pVViqP8md2Q9brdGH7dbbrfGoiFvt0bVbWA0sIh4AzAX+JN21zISIqILOA84pc2llGU8jcMpL6Gx9+PaiHhOZq5qZ1Ej5CTgC5n5HxFxBPDFiDgkM3vbXZhGF7dbY47brZoYK3sAW3lM3ONtImI8jd27Kyupbse09Ai8iHg58D7g2MzcVFFtO2qwsU0DDgF+EhE9NM5buHKMnFDdys9tCXBlZm7OzPuA/6WxYR3tWhnbqcBlAJl5AzCZxrM2O0FL/yZHaD1ut0Yft1tut8aioW+32n1iY4snP44H7gX25YmTOw/u1+Y0nnwy9WXtrnsEx/Z8Gie3Htjuekd6bP3a/4SxczJ1Kz+3o4GLiundaOyen9nu2kdobN8DTimmn03jXJpod+1DGGM32z+Z+lU8+WTqG0v8HN1ujbIvt1tut0br10hvt9o+oCEM/Bga/xO5B3hfMe8DNP5nCY0k/1VgEXAjsF+7ax7Bsf0YWA7cUnxd2e6aR2ps/dqOmQ1piz+3oHGo6E7gNuDEdtc8gmM7CLiu2MjeAryi3TUPYWyXAA8Am2ns7TgVeDvw9qaf26eKsd+2I7+Tbrfcbo22L7dbbrf6vnwSiCRJUs2MlXMAJUmSNEIMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAA15kTE2RHxpXbXIUnSWGUA1KgUEa+LiF9FxLqIeCAivhcRf9yGOvaIiEsi4vcRsToirouIF1RdhyRJI8kAqFEnIk4HPg58CNgT+APg08BxbShnKvBL4I+AXYGLgO9ExNQ21CJJ0ogwAGpUiYhdgA8Ap2Xm1zNzfWZuzsxvZeYZ23nPVyNiWbGH7tqIOLhp2TERcWdErI2IpRHxD8X83SLi2xGxKiIejoifRcRT/j1k5r2ZeV5mPpCZWzNzATAReGY5n4AkSeUzAGq0OQKYDHxjCO/5HnAgsAdwM/DfTcsuBN6WmdOAQ4Cri/l/DywBdqexl/GfgByso4h4Ho0AuGgI9UmSNKqMb3cBUj8zgYcyc0urb8jMz/dNR8TZwCMRsUtmrgY2AwdFxP9k5iPAI0XTzcBewD6ZuQj42WD9RMR04IvAvxXrliRpTHIPoEablcBuEdHSf04iYlxEnBMR90TEGqCnWLRb8f3VwDHA4oj4aUQcUcz/dxp78X4YEfdGxFmD9LMT8C1gYWZ+eGhDkiRpdDEAarS5AdgEHN9i+9fRuDjk5cAuQHcxPwAy85eZeRyNw8PfBC4r5q/NzL/PzP2AY4HTI+KobXUQEZOK9y4B3jbUAUmSNNoYADWqFIdW/xX4VEQcHxE7R8SEiHhlRJy7jbdMoxEYVwI707hyGICImBgRry8OB28G1gC9xbI/j4gDIiKA1cDWvmXNImICcDmwATg5M5/SRpKkscYAqFEnM/8DOB34Z+BB4H7gXTT2wvV3MbAYWArcCSzst/yNQE9xePjtwOuL+QcCPwbW0djr+OnMvGYb638h8OfAK4BVxX0J10XEi4Y9QEmS2iwyB73wUZIkSR3EPYCSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDNj+lFwu+22W86ePZsJEya0u5RSbN68uWPHBp09Psc2Nt10000PZebu7a5Dkso2pgNgd3c3l19+Od3d3e0upRQ9PT0dOzbo7PE5trEpIha3uwZJqoKHgCVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZkoNgBHRExG3RcQtEfGrYt6uEfGjiLi7+P60Yn5ExCciYlFE3BoRh5ZZmyRJUl1VsQfwpZn5vMycW7w+C7gqMw8EripeA7wSOLD4mg98poLaJEmSaqcdh4CPAy4qpi8Cjm+af3E2LARmRMRebahPkiSpo5UdABP4YUTcFBHzi3l7ZuYDxfQyYM9iejZwf9N7lxTzJEmSNILKfhLIH2fm0ojYA/hRRNzVvDAzMyJyKCssguR8gNmzZ7Ns2bKRq3aU6eSxQWePz7FJkkazUgNgZi4tvq+IiG8AhwPLI2KvzHygOMS7omi+FNi76e1zinn917kAWAAwd+7cnDVrVsc+lgro6LFBZ4/PsUmSRqvSDgFHxJSImNY3DbwCuB24Eji5aHYycEUxfSXwpuJq4HnA6qZDxZIkSRohZe4B3BP4RkT09fPlzPx+RPwSuCwiTgUWAycU7b8LHAMsAh4F3lxibZIkSbVVWgDMzHuBP9zG/JXAUduYn8BpZdUjSZKkBp8EIkmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDOlB8CIGBcRv46Ibxev942IX0TEooj4SkRMLOZPKl4vKpZ3l12bJElSHVWxB/A9wG+aXn8E+FhmHgA8ApxazD8VeKSY/7GinSRJkkZYqQEwIuYArwI+V7wO4GXA5UWTi4Dji+njitcUy48q2kuSJGkElb0H8OPAmUBv8XomsCoztxSvlwCzi+nZwP0AxfLVRXtJkiSNoPFlrTgi/hxYkZk3RcRLRnC984H5ALNnz2bZsmUjtepRp5PHBp09PscmSRrNSguAwJHAsRFxDDAZmA6cD8yIiPHFXr45wNKi/VJgb2BJRIwHdgFW9l9pZi4AFgDMnTs3Z82aRXd3d4nDaK9OHht09vgcmyRptCrtEHBm/mNmzsnMbuBE4OrMfD1wDfCaotnJwBXF9JXFa4rlV2dmllWfJElSXbXjPoDvBU6PiEU0zvG7sJh/ITCzmH86cFYbapMkSep4ZR4Cflxm/gT4STF9L3D4NtpsBP66inokSZLqzCeBSJIk1UwlewDLsmTJI1xwwTX09k6ptN9zzz2h0v4kSZJG0pgOgAB3LVvLqvWbKuvv8H13rawvSZKkMoz5AAhwxElHVtLPDZdcV0k/kiRJZfIcQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDICSJEk1YwCUJEmqGQOgJElSzRgAJUmSasYAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmxg+0MCL+qoV1bMzM745QPZIkSSrZgAEQ+CxwBRADtHkx8JQAGBGTgWuBSUU/l2fm+yNiX+BSYCZwE/DGzHwsIiYBFwN/BKwEXpuZPUMbjiRJkgYzWAD8Xma+ZaAGEfGl7SzaBLwsM9dFxATg5xHxPeB04GOZeWlEXACcCnym+P5IZh4QEScCHwFeO5TBSJIkaXADngOYmW8YbAXba5MN64qXE4qvBF4GXF7Mvwg4vpg+rnhNsfyoiBhoz6MkSZKGYcgXgUTEgiG0HRcRtwArgB8B9wCrMnNL0WQJMLuYng3cD1AsX03jMLEkSZJG0GCHgLdlbqsNM3Mr8LyImAF8A3jWMPp7koiYD8wHmD59JlMnb6Zr3YM7utqWzJjyGF1d6+np6amkv2XLllXST7t08vgcmyRpNBtOAFwx1Ddk5qqIuAY4ApgREeOLvXxzgKVFs6XA3sCSiBgP7ELjYpD+61oALACYNWv/XLdxAr1Tdx/GMIZu1fqJ9PZOobu7u5L+gEr7aodOHp9jkySNVkM+BJyZR7fSLiJ2L/b8ERE7AX8K/Aa4BnhN0exkGlcZA1xZvKZYfnVm5lDrkyRJ0sAGDIARcfZgKxigzV7ANRFxK/BL4EeZ+W3gvcDpEbGIxjl+FxbtLwRmFvNPB85qZQCSJEkamsEOAb81ItYMsDyAE4Gz+y/IzFuB529j/r3A4duYvxH460HqkSRJ0g5q5UbQ01poI0mSpDFiwACYmf9WVSGSJEmqxpAvApEkSdLYZgCUJEmqmZYCYEQc2co8SZIkjX6t7gH8zxbnSZIkaZQb8CKQiDgCeCGwe0Sc3rRoOjCuzMIkSZJUjsFuAzMRmFq0a74dzBqeeJqHJEmSxpDBbgPzU+CnEfGFzFxcUU2SJEkq0WB7APtMiogFQHfzezLzZWUUJUmSpPK0GgC/ClwAfA7YWl45kiRJKlurAXBLZn6m1EokSZJUiVZvA/OtiHhnROwVEbv2fZVamSRJkkrR6h7Ak4vvZzTNS2C/kS1HkiRJZWspAGbmvmUXIkmSpGq0FAAj4k3bmp+ZF49sOZIkSSpbq4eAD2uangwcBdwMGAAlSZLGmFYPAb+7+XVEzAAuLaMgSZIklavVq4D7Ww94XqAkSdIY1Oo5gN+icdUvwDjg2cBlZRUlSZKk8rR6DuBHm6a3AIszc0kJ9UiSJKlkLR0CzsyfAncB04CnAY+VWZQkSZLK01IAjIgTgBuBvwZOAH4REa8pszBJkiSVo9VDwO8DDsvMFQARsTvwY+DysgqTJElSOVq9CrirL/wVVg7hvZIkSRpFWt0D+P2I+AFwSfH6tcD3yilJkiRJZWr1RtBnRMRfAX9czFqQmd8oryxJkiSVZcAAGBEHAHtm5nWZ+XXg68X8P46I/TPzniqKlCRJ0sgZ7Dy+jwNrtjF/dbFMkiRJY8xgAXDPzLyt/8xiXncpFUmSJKlUgwXAGQMs22kE65AkSVJFBguAv4qIv+k/MyLeCtxUTkmSJEkq02BXAf8t8I2IeD1PBL65wETgL0usS5IkSSUZcA9gZi7PzBcC/wb0FF//lplHZOaygd4bEXtHxDURcWdE3BER7ynm7xoRP4qIu4vvTyvmR0R8IiIWRcStEXHoSAxQkiRJT9bqfQCvAa4Z4rq3AH+fmTdHxDTgpoj4EXAKcFVmnhMRZwFnAe8FXgkcWHy9APhM8V2SJEkjqLTHuWXmA5l5czG9FvgNMBs4DrioaHYRcHwxfRxwcTYsBGZExF5l1SdJklRXlTzPNyK6gecDv6Bxa5kHikXLgD2L6dnA/U1vW1LMkyRJ0ghq9VnAwxYRU4GvAX+bmWsi4vFlmZkRkUNc33xgPsD06TOZOnkzXeseHMmSt2vGlMfo6lpPT09PJf0tWzbgaZZjXiePz7FJkkazUgNgREygEf7+u3iUHMDyiNgrMx8oDvGuKOYvBfZuevucYt6TZOYCYAHArFn757qNE+iduntpY2i2av1Eenun0N3dXUl/QKV9tUMnj8+xSZJGq9IOAUdjV9+FwG8y87ymRVcCJxfTJwNXNM1/U3E18DxgddOhYkmSJI2QMvcAHgm8EbgtIm4p5v0TcA5wWUScCiwGTiiWfRc4BlgEPAq8ucTaJEmSaqu0AJiZPwdiO4uP2kb7BE4rqx5JkiQ1VHIVsCRJkkYPA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaoZA6AkSVLNGAAlSZJqxgAoSZJUMwZASZKkmjEASpIk1YwBUJIkqWYMgJIkSTVjAJQkSaqZ0gJgRHw+IlZExO1N83aNiB9FxN3F96cV8yMiPhERiyLi1og4tKy6JEmS6q7MPYBfAI7uN+8s4KrMPBC4qngN8ErgwOJrPvCZEuuSJEmqtdICYGZeCzzcb/ZxwEXF9EXA8U3zL86GhcCMiNirrNokSZLqrOpzAPfMzAeK6WXAnsX0bOD+pnZLinmSJEkaYePb1XFmZkTkUN8XEfNpHCZm+vSZTJ28ma51D454fdsyY8pjdHWtp6enp5L+li1bVkk/7dLJ43NskqTRrOoAuDwi9srMB4pDvCuK+UuBvZvazSnmPUVmLgAWAMyatX+u2ziB3qm7l1nz41atn0hv7xS6u7sr6Q+otK926OTxOTZJ0mhV9SHgK4GTi+mTgSua5r+puBp4HrC66VCxJEmSRlBpewAj4hLgJcBuEbEEeD9wDnBZRJwKLAZOKJp/FzgGWAQ8Cry5rLokSZLqrrQAmJknbWfRUdtom8BpZdUyUpYvWsbC5Y9w5pmXVdJfV9d6zjnHLCxJkkZW2y4CGaseinHceF//u9uUY97+kyrpR5Ik1YsBcBiOOOnI0vu44ZLrSu9DkiTVk88CliRJqhkDoCRJUs0YACVJkmrGAChJklQzBkBJkqSaMQBKkiTVjAFQkiSpZgyAkiRJNWMAlCRJqhkDoCRJUs34KLhRavmiZdy5aQJnnnlZpf2ee+4JlfYnbU/Vv/uSVCcGwFFsdXRx430PV9bf4fvuWllfUksWX9/uCiSpIxkAR7kjTjqykn5uuOS6SvqRhurcN2Ql/Zz5paikH0kaDTwHUJIkqWYMgJIkSTVjAJQkSaoZzwEU0LjqeOHyRyq98vKd7zy8sr46VTuulPXnJkljnwFQj3soxlV21bFXHI8crxSXJA2VAVBPUsVVx1VfcVyHvWSd+HNbuPAe9u1aw8KFD1bS3/Llu1fSjySNBgZAVa7vcHNX13p6e6eU3t/ChffwUIxjjz2ml97X8kXL2H3apMrH1qk2bt7KirUb212GJHUcA6Da4qEYx13L1rJq/abS+3pw7SZi+s6V7CX75ge/1paxVWH5omVcedti7rrrblatKv/6scWLV7Lb7pvZ84BZpfe1fNGy0vuQpNHEAKi2Ofhlh9A7tfzDbt/84NdK76O/Th3bRrqY2fs7dtpa/l65NRN3Kr0PSaorA6CkIZk0ZRJvf8Wjpffztv8svQtJqi3vAyhJklQzBkBJkqSa8RDwED26fHUlt8PYsGYDmx7Zwg2XXFfJxQuSJKk+DIDD0L3+ltL7mDlzLVOmTuF3zCm9L0mSVC8GwGF6x9HrS11/z82L+c6Kg0rto06q3HPLhs3ccfXtPPvYl5benyRJw2EA1OOqDElbHl7Pz794LTGj/Hu8bVizgS1betnz9zew0y7l3lpk5sy13LNqGo8snVDJZ7n2obXw8Prqfm5betlUwf0NJUnlMgCOYr9fM5nlDyyr9I97lSFpv0lryPXl34C3rz+oZs/th68/hP0mPVDJ2FZNhv12WcP09WtL76v5c5QkjW0dEQCrekZpX0jasHpDJf0BPHfn+9ipQ0NSFX31769KVYztbXdPqayvdn2OkqSRN6oCYEQcDZwPjAM+l5nntPreqi7MaMceEP+4S5KkkTRqAmBEjAM+BfwpsAT4ZURcmZl3trqOKvdcSZIkjVWj6UbQhwOLMvPezHwMuBQ4rs01SZIkdZxRswcQmA3c3/R6CfCCVt64fNEy9tx5A+d9pdw8u2XzPsX3LZz3lXI/ui2b94GJ1fT1eH9UODZg65atpf/MmvvrxLH1qfJz7MSxwdOZPKGCbiRplIjMbHcNAETEa4CjM/Otxes3Ai/IzHf1azcfmF+8fCawEnioylortBudOzbo7PE5trHpmZnppc6SOt5o2gO4FNi76fWcYt6TZOYCYEHf64j4VWbOLb+86nXy2KCzx+fYxqaI+FW7a5CkKoymcwB/CRwYEftGxETgRODKNtckSZLUcUbNHsDM3BIR7wJ+QOM2MJ/PzDvaXJYkSVLHGTUBECAzvwt8d4hvWzB4kzGrk8cGnT0+xzY2dfLYJOlxo+YiEEmSJFVjNJ0DKEmSpAqMmQAYEUdHxG8jYlFEnLWN5ZMi4ivF8l9ERHcbyhyWFsZ2ekTcGRG3RsRVEbFPO+ocjsHG1tTu1RGRETFmri5tZWwRcULxs7sjIr5cdY3D1cLv5B9ExDUR8evi9/KYdtQ5HBHx+YhYERG3b2d5RMQnirHfGhGHVl2jJJVtTATApsfEvRI4CDgpIg7q1+xU4JHMPAD4GPCRaqscnhbH9mtgbmY+F7gcOLfaKoenxbEREdOA9wC/qLbC4WtlbBFxIPCPwJGZeTDwt1XXORwt/tz+GbgsM59P44r9T1db5Q75AnD0AMtfCRxYfM0HPlNBTZJUqTERAGntMXHHARcV05cDR0VEVFjjcA06tsy8JjMfLV4upHGPxLGg1cf7fZBGYN9YZXE7qJWx/Q3wqcx8BCAzV1Rc43C1MrYEphfTuwC/r7C+HZKZ1wIPD9DkOODibFgIzIiIvaqpTpKqMVYC4LYeEzd7e20ycwuwGphZSXU7ppWxNTsV+F6pFY2cQcdWHF7bOzO/U2VhI6CVn9szgGdExHURsTAiBtrrNJq0MrazgTdExBIaV+6/u5rSKjHUf5OSNOaMqtvAaGAR8QZgLvAn7a5lJEREF3AecEqbSynLeBqHEV9CY6/ttRHxnMxc1c6iRshJwBcy8z8i4gjgixFxSGb2trswSdLgxsoewFYeE/d4m4gYT+Ow1MpKqtsxLT0CLyJeDrwPODYzN1VU244abGzTgEOAn0REDzAPuHKMXAjSys9tCXBlZm7OzPuA/6URCEe7VsZ2KnAZQGbeAEym8YzgTtDSv0lJGsvGSgBs5TFxVwInF9OvAa7OsXGTw0HHFhHPB/6LRvgbK+eRwSBjy8zVmblbZnZnZjeN8xuPzcyx8DzWVn4nv0lj7x8RsRuNQ8L3VljjcLUytt8BRwFExLNpBMAHK62yPFcCbyquBp4HrM7MB9pdlCSNpDFxCHh7j4mLiA8Av8rMK4ELaRyGWkTjBO8T21dx61oc278DU4GvFte1/C4zj21b0S1qcWxjUotj+wHwioi4E9gKnJGZo36vdItj+3vgsxHxdzQuCDlljPyHi4i4hEYw3604h/H9wASAzLyAxjmNxwCLgEeBN7enUkkqj08CkSRJqpmxcghYkiRJI8QAKEmSVDMGQEmSpJoxAEqSJNWMAVCSJKlmDIA1FhFbI+KWiLg9Ir4aETvvwLq+EBGvKaY/FxEHDdD2JRHxwmH00VPcT29EDXW9EXF2RPzDNuY/PSIuL6ZfEhHfLqaPjYiziunjB/psttPfTyLitxHxlFv/RMROxc/wsTI+G0lSZzIA1tuGzHxeZh4CPAa8vXlh8USVIcvMt2bmnQM0eQkw5AC4I4Y7lqHIzN9n5mu2Mf/KzDyneHk8MKQAWHj9tu6bmJkbMvN5wO+HsU5JUk0ZANXnZ8ABxZ6rn0XElcCdETEuIv49In4ZEbdGxNsAiqckfLLYM/VjYI++FRV7rOYW00dHxM0R8T8RcVVEdNMImn9X7Ll6UUTsHhFfK/r4ZUQcWbx3ZkT8MCLuiIjPAbGtwiNiXUR8rGh3VUTs3lTHxyPiV8B7IuKoiPh1RNwWEZ+PiElNqzmzmH9jRBxQvP8vIuIXxXt+HBF7NrX/w4i4ISLujoi/Kdp3R8Tt26jvlOKzeiFwLPDvxdj3j4ibm9od2PxakqSyGADVt3fslcBtxaxDgfdk5jNoPPN1dWYeBhwG/E1E7Av8JfBMGnuz3sQ29ugVQeyzwKsz8w+Bv87MHuAC4GPF3sefAecXrw8DXg18rljF+4GfZ+bBwDeAP9jOEKbQeELFwcBPi/f1mZiZc4FPAV8AXpuZz6HxFJx3NLVbXcz/JPDxYt7PgXmZ+XzgUuDMpvbPBV4GHAH8a0Q8fTu1PS4zr6fxmLEzirHfA6yOiOcVTd4M/L/B1iNJ0o4aE4+CU2l2iohbiumf0Xic3guBGzPzvmL+K4Dn9p3fB+wCHAi8GLgkM7cCv4+Iq7ex/nnAtX3rysyHt1PHy4GDisfcAUyPiKlFH39VvPc7EfHIdt7fC3ylmP4S8PWmZX3znwncl5n/W7y+CDiNJ8LeJU3fP1ZMzwG+EhF7AROBvs8E4IrM3ABsiIhrgMOBW7ZT30A+B7w5Ik4HXlusR5KkUhkA663v/LHHFSFsffMs4N2Z+YN+7Y4ZwTq6aOxp27iNWoaj+fmG67fbavvv6Zv+T+C8zLwyIl4CnL2d9tt63aqv0dhjeTVw02DPCo6IvYFvFS8vKJ5dK0nSkHgIWIP5AfCOiJgAEBHPiIgpwLXAa4tzBPcCXrqN9y4EXlwcMiYidi3mrwWmNbX7IfDuvhdNh0SvBV5XzHsl8LTt1NgF9O2hfB2NQ7f9/Rbo7ju/D3gjjcPFfV7b9P2GYnoXYGkxfXK/9R0XEZMjYiaNi1p+uZ3a+nvS2IvQ+wPgM7Rw+Dcz7y8OHz/P8CdJGi4DoAbzOeBO4ObiAof/orHn+BvA3cWyi3kiND0uMx8E5gNfj4j/4YnDsd8C/rLvIhDg/wBzi4tM7uSJq5H/jUaAvIPGoeDfbafG9cDhRX0vAz6wjVo20jjH7qsRcRuNw8bNAeppEXEr8B7g74p5ZxftbwIe6rfKW4FraITcD2Zmq1fhXgqcUVxYsn8x77+Len7Y4jokSdohkTncI1fS6BAR6zJzarvrGK5o3FNwl8z8l+0s/wnwD5n5qwHW0QPMzcz+QVWSpKdwD6DURhHxDRpXUZ8/QLOHgS/EADeCBibQ2IsoSdKg3AMoSZJUM+4BlCRJqhkDoCRJUs0YACVJkmrGAChJklQzBkBJkqSaMQBKkiTVzP8P78UjA4mN+O8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig = pandg.multiclass_classifier.plot_y_prob_histogram(y_true_one_hot, y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "DAISE", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/plotsandgraphs/__init__.py b/plotsandgraphs/__init__.py index 1d36456..4f9626e 100644 --- a/plotsandgraphs/__init__.py +++ b/plotsandgraphs/__init__.py @@ -1,2 +1,3 @@ from . import binary_classifier from . import compare_distributions +from . import multiclass_classifier diff --git a/plotsandgraphs/multiclass_classifier.py b/plotsandgraphs/multiclass_classifier.py new file mode 100644 index 0000000..b69d532 --- /dev/null +++ b/plotsandgraphs/multiclass_classifier.py @@ -0,0 +1,73 @@ +from pathlib import Path +from typing import Optional +import matplotlib.pyplot as plt +from matplotlib.colors import to_rgba +from matplotlib.figure import Figure +import numpy as np +import pandas as pd +from sklearn.metrics import ( + confusion_matrix, + classification_report, + ConfusionMatrixDisplay, + roc_curve, + auc, + accuracy_score, + precision_recall_curve, +) +from sklearn.calibration import calibration_curve +from sklearn.utils import resample +from tqdm import tqdm + + +def plot_y_prob_histogram(y_true: np.ndarray, y_prob: Optional[np.ndarray] = None, save_fig_path=None) -> Figure: + + plot_len = np.ceil(np.sqrt(y_true.shape[-1])).astype(int) # Number of plots in a row/column + fig, axes = plt.subplots(nrows=plot_len, ncols=plot_len, figsize=(plot_len*4+1, plot_len*4), sharey=True) + alpha = 0.6 + plt.suptitle("Predicted probability histogram") + + for i, ax in enumerate(axes.flat): + if i >= y_true.shape[-1]: + ax.axis("off") + continue + + if y_prob is not None: + y_true_i = y_true[:, i] + y_prob_i = y_prob[:, i] + ax.hist(y_prob_i[y_true_i==0], + bins=10, + label="$\\hat{y} = 0$", + alpha=alpha, + edgecolor="midnightblue", + linewidth=2, + rwidth=1,) + ax.hist(y_prob_i[y_true_i==1], + bins=10, + label="$\\hat{y} = 1$", + alpha=alpha, + edgecolor="midnightblue", + linewidth=2, + rwidth=1,) + ax.set_title(f"Class {i}") + ax.set_xlim((-0.005, 1.0)) + # if subplot in first column + if i % plot_len == 0: + ax.set_ylabel("Count [-]") + # if subplot in last row + if i >= plot_len*(plot_len-1): + ax.set_xlabel("Predicted probability [-]") + # ax.spines[:].set_visible(False) + ax.grid(True, linestyle="-", linewidth=0.5, color="grey", alpha=0.5) + ax.set_xticks(np.arange(0, 1.1, 0.2)) + # only first subplot should have legends + if i == 0: + ax.legend() + + plt.tight_layout() + + # save plot + if save_fig_path is not None: + path = Path(save_fig_path) + path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(save_fig_path, bbox_inches="tight") + return fig