diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e636b5..56f55af0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,52 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.8] - 2023-02-01 +### Added +- `distance` method +- `__contains__` method +- `__getitem__` method +- Goodman-Kruskal's Lambda A benchmark +- Goodman-Kruskal's Lambda B benchmark +- Krippendorff's Alpha benchmark +- Pearson's C benchmark +- 30 new distance/similarity + 1. AMPLE + 2. Anderberg's D + 3. Andres & Marzo's Delta + 4. Baroni-Urbani & Buser I + 5. Baroni-Urbani & Buser II + 6. Batagelj & Bren + 7. Baulieu I + 8. Baulieu II + 9. Baulieu III + 10. Baulieu IV + 11. Baulieu V + 12. Baulieu VI + 13. Baulieu VII + 14. Baulieu VIII + 15. Baulieu IX + 16. Baulieu X + 17. Baulieu XI + 18. Baulieu XII + 19. Baulieu XIII + 20. Baulieu XIV + 21. Baulieu XV + 22. Benini I + 23. Benini II + 24. Canberra + 25. Clement + 26. Consonni & Todeschini I + 27. Consonni & Todeschini II + 28. Consonni & Todeschini III + 29. Consonni & Todeschini IV + 30. Consonni & Todeschini V +### Changed +- `relabel` method sort bug fixed +- `README.md` modified +- `Compare` overall benchmarks default weights updated +- Document modified +- Test system modified ## [3.7] - 2022-12-15 ### Added - `Curve` class @@ -606,7 +652,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - TPR - documents and `README.md` -[Unreleased]: https://github.com/sepandhaghighi/pycm/compare/v3.7...dev +[Unreleased]: https://github.com/sepandhaghighi/pycm/compare/v3.8...dev +[3.8]: https://github.com/sepandhaghighi/pycm/compare/v3.7...v3.8 [3.7]: https://github.com/sepandhaghighi/pycm/compare/v3.6...v3.7 [3.6]: https://github.com/sepandhaghighi/pycm/compare/v3.5...v3.6 [3.5]: https://github.com/sepandhaghighi/pycm/compare/v3.4...v3.5 diff --git a/Document/Distance.ipynb b/Document/Distance.ipynb new file mode 100644 index 00000000..2cd9f832 --- /dev/null +++ b/Document/Distance.ipynb @@ -0,0 +1,1652 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Please cite us if you use the software

" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Distance/Similarity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "PyCM's `distance` method provides users with a wide range of string distance/similarity metrics to evaluate a confusion matrix by measuring its distance to a perfect confusion matrix. Distance/Similarity metrics measure the distance between two vectors of numbers. Small distances between two objects indicate similarity. In the PyCM's `distance` method, a distance measure can be chosen from `DistanceType`. The measures' names are chosen based on the namig style suggested in [[1]](#ref1)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pycm import ConfusionMatrix, DistanceType" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "cm = ConfusionMatrix(matrix={0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$TP \\rightarrow True Positive$$\n", + "$$TN \\rightarrow True Negative$$\n", + "$$FP \\rightarrow False Positive$$\n", + "$$FN \\rightarrow False Negative$$\n", + "$$POP \\rightarrow Population$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## AMPLE" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "AMPLE similarity [[2]](#ref2) [[3]](#ref3)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{AMPLE}=|\\frac{TP}{TP+FP}-\\frac{FN}{FN+TN}|$$" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.6, 1: 0.3, 2: 0.17142857142857143}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.AMPLE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Anderberg's D" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Anderberg's D [[4]](#ref4)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{Anderberg} =\n", + "\\frac{(max(TP,FP)+max(FN,TN)+max(TP,FN)+max(FP,TN))-\n", + "(max(TP+FP,FP+TN)+max(TP+FP,FN+TN))}{2\\times POP}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.16666666666666666, 1: 0.0, 2: 0.041666666666666664}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.Anderberg)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Andres & Marzo's Delta" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Andres & Marzo's Delta correlation [[5]](#ref5)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$corr_{AndresMarzo_\\Delta} = \\Delta =\n", + "\\frac{TP+TN-2 \\times \\sqrt{FP \\times FN}}{POP}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.8333333333333334, 1: 0.5142977396044842, 2: 0.17508504286947035}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.AndresMarzoDelta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baroni-Urbani & Buser I" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baroni-Urbani & Buser I similarity [[6]](#ref6)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{BaroniUrbaniBuserI} =\n", + "\\frac{\\sqrt{TP\\times TN}+TP}{\\sqrt{TP\\times TN}+TP+FP+FN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.79128784747792, 1: 0.5606601717798213, 2: 0.5638559245324765}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaroniUrbaniBuserI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baroni-Urbani & Buser II" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baroni-Urbani & Buser II correlation [[6]](#ref6)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$corr_{BaroniUrbaniBuserII} =\n", + "\\frac{\\sqrt{TP \\times TN}+TP-FP-FN}{\\sqrt{TP \\times TN}+TP+FP+FN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.58257569495584, 1: 0.12132034355964261, 2: 0.1277118490649528}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaroniUrbaniBuserII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batagelj & Bren" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Batagelj & Bren distance [[7]](#ref7)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BatageljBren} =\n", + "\\frac{FP \\times FN}{TP \\times TN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.0, 1: 0.25, 2: 0.5}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BatageljBren)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu I" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu I distance [[8]](#ref8)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{BaulieuI} =\n", + "\\frac{(TP+FP) \\times (TP+FN)-TP^2}{(TP+FP) \\times (TP+FN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.4, 1: 0.8333333333333334, 2: 0.7}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu II" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu II similarity [[8]](#ref8)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{BaulieuII} =\n", + "\\frac{TP^2 \\times TN^2}{(TP+FP) \\times (TP+FN) \\times (FP+TN) \\times (FN+TN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.4666666666666667, 1: 0.11851851851851852, 2: 0.11428571428571428}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu III" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu III distance [[8]](#ref8)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{BaulieuIII} =\n", + "\\frac{POP^2 - 4 \\times (TP \\times TN-FP \\times FN)}{2 \\times POP^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.20833333333333334, 1: 0.4166666666666667, 2: 0.4166666666666667}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuIII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu IV" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu IV distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuIV} = \\frac{FP+FN-(TP+\\frac{1}{2})\\times(TN+\\frac{1}{2})\\times TN \\times k}{POP}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: -41.45702383161246, 1: -22.855395541901885, 2: -13.85431293274332}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuIV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* The default value of k is Euler's number $e$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu V" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu V distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuV} = \\frac{FP+FN+1}{TP+FP+FN+1}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.5, 1: 0.8, 2: 0.6666666666666666}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu VI" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu VI distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuVI} = \\frac{FP+FN}{TP+FP+FN+1}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.3333333333333333, 1: 0.6, 2: 0.5555555555555556}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuVI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu VII" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu VII distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuVII} = \\frac{FP+FN}{POP + TP \\times (TP-4)^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.13333333333333333, 1: 0.14285714285714285, 2: 0.3333333333333333}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuVII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu VIII" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu VIII distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuVIII} = \\frac{(FP-FN)^2}{POP^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.027777777777777776, 1: 0.006944444444444444, 2: 0.006944444444444444}" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuVIII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu IX" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu IX distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuIX} = \\frac{FP+2 \\times FN}{TP+FP+2 \\times FN+TN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.16666666666666666, 1: 0.35714285714285715, 2: 0.5333333333333333}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuIX)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu X" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu X distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuX} = \\frac{FP+FN+max(FP,FN)}{POP+max(FP,FN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.2857142857142857, 1: 0.35714285714285715, 2: 0.5333333333333333}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuX)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu XI" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu XI distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuXI} = \\frac{FP+FN}{FP+FN+TN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.2222222222222222, 1: 0.2727272727272727, 2: 0.5555555555555556}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuXI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu XII" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu XII distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuXII} = \\frac{FP+FN}{TP+FP+FN-1}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.5, 1: 1.0, 2: 0.7142857142857143}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuXII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu XIII" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu XIII distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuXIII} = \\frac{FP+FN}{TP+FP+FN+TP \\times (TP-4)^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.25, 1: 0.23076923076923078, 2: 0.45454545454545453}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuXIII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu XIV" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu XIV distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuXIV} = \\frac{FP+2 \\times FN}{TP+FP+2 \\times FN}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.4, 1: 0.8333333333333334, 2: 0.7272727272727273}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuXIV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Baulieu XV" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Baulieu XV distance [[9]](#ref9)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$dist_{BaulieuXV} = \\frac{FP+FN+max(FP, FN)}{TP+FP+FN+max(FP, FN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.5714285714285714, 1: 0.8333333333333334, 2: 0.7272727272727273}" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BaulieuXV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Benini I" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Benini I correlation [[10]](#ref10)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$corr_{BeniniI} = \\frac{TP \\times TN-FP \\times FN}{(TP+FN)\\times(FN+TN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 1.0, 1: 0.2, 2: 0.14285714285714285}" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BeniniI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Benini II" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Benini II correlation [[10]](#ref10)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$corr_{BeniniII} = \\frac{TP \\times TN-FP \\times FN}{min((TP+FN)\\times(FN+TN), (TP+FP)\\times(FP+TN))}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 1.0, 1: 0.3333333333333333, 2: 0.2}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.BeniniII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Canberra" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Canberra distance [[11]](#ref11) [[12]](#ref12)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{Canberra} =\n", + "\\frac{FP+FN}{(TP+FP)+(TP+FN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.25, 1: 0.6, 2: 0.45454545454545453}" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.Canberra)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clement" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clement similarity [[13]](#ref13)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{Clement} =\n", + "\\frac{TP}{TP+FP}\\times\\Big(1 - \\frac{TP+FP}{POP}\\Big) +\n", + "\\frac{TN}{FN+TN}\\times\\Big(1 - \\frac{FN+TN}{POP}\\Big)$$" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.7666666666666666, 1: 0.55, 2: 0.588095238095238}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.Clement)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Consonni & Todeschini I" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consonni & Todeschini I similarity [[14]](#ref14)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{ConsonniTodeschiniI} =\n", + "\\frac{log(1+TP+TN)}{log(1+POP)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.9348704159880586, 1: 0.8977117175026231, 2: 0.8107144632819592}" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.ConsonniTodeschiniI)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Consonni & Todeschini II" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consonni & Todeschini II similarity [[14]](#ref14)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{ConsonniTodeschiniII} =\n", + "\\frac{log(1+POP)-log(1+FP+FN)}{log(1+POP)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.5716826589686053, 1: 0.4595236911453605, 2: 0.3014445045412856}" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.ConsonniTodeschiniII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Consonni & Todeschini III" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consonni & Todeschini III similarity [[14]](#ref14)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{ConsonniTodeschiniIII} =\n", + "\\frac{log(1+TP)}{log(1+POP)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.5404763088546395, 1: 0.27023815442731974, 2: 0.5404763088546395}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.ConsonniTodeschiniIII)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Consonni & Todeschini IV" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consonni & Todeschini IV similarity [[14]](#ref14)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$sim_{ConsonniTodeschiniIV} =\n", + "\\frac{log(1+TP)}{log(1+TP+FP+FN)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.7737056144690831, 1: 0.43067655807339306, 2: 0.6309297535714574}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.ConsonniTodeschiniIV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Consonni & Todeschini V" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Consonni & Todeschini V correlation [[14]](#ref14)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$corr_{ConsonniTodeschiniV} =\n", + "\\frac{log(1+TP \\times TN)-log(1+FP \\times FN)}{log(1+\\frac{POP^2}{4})}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 0.8560267854703983, 1: 0.30424737289682985, 2: 0.17143541431350617}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.distance(metric=DistanceType.ConsonniTodeschiniV)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
1- C. C. Little, \"Abydos Documentation,\" 2018.
\n", + "\n", + "
2- V. Dallmeier, C. Lindig, and A. Zeller, \"Lightweight defect localization for Java,\" in European conference on object-oriented programming, 2005: Springer, pp. 528-550.
\n", + "\n", + "
3- R. Abreu, P. Zoeteweij, and A. J. Van Gemund, \"An evaluation of similarity coefficients for software fault localization,\" in 2006 12th Pacific Rim International Symposium on Dependable Computing (PRDC'06), 2006: IEEE, pp. 39-46.
\n", + "\n", + "
4- M. R. Anderberg, Cluster analysis for applications: probability and mathematical statistics: a series of monographs and textbooks. Academic press, 2014.
\n", + "\n", + "
5- A. M. Andrés and P. F. Marzo, \"Delta: A new measure of agreement between two raters,\" British journal of mathematical and statistical psychology, vol. 57, no. 1, pp. 1-19, 2004.
\n", + "\n", + "
6- C. Baroni-Urbani and M. W. Buser, \"Similarity of binary data,\" Systematic Zoology, vol. 25, no. 3, pp. 251-259, 1976.
\n", + "\n", + "
7- V. Batagelj and M. Bren, \"Comparing resemblance measures,\" Journal of classification, vol. 12, no. 1, pp. 73-90, 1995.
\n", + "\n", + "
8- F. B. Baulieu, \"A classification of presence/absence based dissimilarity coefficients,\" Journal of Classification, vol. 6, no. 1, pp. 233-246, 1989.
\n", + "\n", + "
9- F. B. Baulieu, \"Two variant axiom systems for presence/absence based dissimilarity coefficients,\" Journal of Classification, vol. 14, no. 1, pp. 0159-0170, 1997.
\n", + "\n", + "
10- R. Benini, Principii di demografia. Barbera, 1901.
\n", + "\n", + "
11- G. N. Lance and W. T. Williams, \"Computer programs for hierarchical polythetic classification (“similarity analyses”),\" The Computer Journal, vol. 9, no. 1, pp. 60-64, 1966.
\n", + "\n", + "
12- G. N. Lance and W. T. Williams, \"Mixed-Data Classificatory Programs I - Agglomerative Systems,\" Australian Computer Journal, vol. 1, no. 1, pp. 15-20, 1967.
\n", + "\n", + "
13- P. W. Clement, \"A formula for computing inter-observer agreement,\" Psychological Reports, vol. 39, no. 1, pp. 257-258, 1976.
\n", + "\n", + "
14- V. Consonni and R. Todeschini, \"New similarity coefficients for binary data,\" Match-Communications in Mathematical and Computer Chemistry, vol. 68, no. 2, p. 581, 2012.
" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": false, + "sideBar": true, + "skip_h1_title": true, + "title_cell": "Table of Contents", + "title_sidebar": "Distance/Similarity", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 59f19ddd..633f2a17 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -18,7 +18,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Version : 3.7 " + "### Version : 3.8 " ] }, { @@ -71,6 +71,7 @@ "
  • To Array
    • \n", "
  • Combine
    • \n", "
  • Plot
    • \n", + "
  • Distance/Similarity
    • \n", "
  • Parameter Recommender
    • \n", "
  • Compare
    • \n", "
  • ROC Curve
    • \n", @@ -186,6 +187,10 @@ "
  • Cicchetti's Benchmark
    • \n", "
  • Cramer's Benchmark
    • \n", "
  • Matthews's Benchmark
    • \n", + "
  • Goodman-Kruskal's Lambda A Benchmark
    • \n", + "
  • Goodman-Kruskal's Lambda B Benchmark
    • \n", + "
  • Krippendorff's Alpha Benchmark
    • \n", + "
  • Pearson's C Benchmark
    • \n", "
  • Overall Accuracy
    • \n", "
  • Overall Random Accuracy
    • \n", "
  • Overall Random Accuracy Unbiased
    • \n", @@ -302,7 +307,7 @@ "metadata": {}, "source": [ "### Source code\n", - "- Download [Version 3.7](https://github.com/sepandhaghighi/pycm/archive/v3.7.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n", + "- Download [Version 3.8](https://github.com/sepandhaghighi/pycm/archive/v3.8.zip) or [Latest Source](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n", "- Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access)\n", "- Run `python3 setup.py install` or `python setup.py install` (Need root access)" ] @@ -315,7 +320,7 @@ "\n", "\n", "- Check [Python Packaging User Guide](https://packaging.python.org/installing/) \n", - "- Run `pip install pycm==3.7` or `pip3 install pycm==3.7` (Need root access)" + "- Run `pip install pycm==3.8` or `pip3 install pycm==3.8` (Need root access)" ] }, { @@ -682,11 +687,15 @@ " 'Reference Entropy': 1.5,\n", " 'Response Entropy': 1.4833557549816874,\n", " 'SOA1(Landis & Koch)': 'Fair',\n", + " 'SOA10(Pearson C)': 'Strong',\n", " 'SOA2(Fleiss)': 'Poor',\n", " 'SOA3(Altman)': 'Fair',\n", " 'SOA4(Cicchetti)': 'Poor',\n", " 'SOA5(Cramer)': 'Relatively Strong',\n", " 'SOA6(Matthews)': 'Weak',\n", + " 'SOA7(Lambda A)': 'Very Weak',\n", + " 'SOA8(Lambda B)': 'Moderate',\n", + " 'SOA9(Krippendorff Alpha)': 'Low',\n", " 'Scott PI': 0.34426229508196726,\n", " 'Standard Error': 0.14231876063832777,\n", " 'TNR Macro': 0.7777777777777777,\n", @@ -948,7 +957,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.7-py3.5.egg\\pycm\\pycm_util.py:398: RuntimeWarning: Used classes is not a subset of classes in actual and predict vectors.\n" + "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.8-py3.5.egg\\pycm\\pycm_util.py:399: RuntimeWarning: Used classes is not a subset of classes in actual and predict vectors.\n" ] } ], @@ -990,6 +999,75 @@ "" ] }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1 in cm" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "10 in cm" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm[1][1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -999,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -1008,7 +1086,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -1017,7 +1095,7 @@ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, - "execution_count": 26, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1028,7 +1106,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -1037,7 +1115,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -1046,7 +1124,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -1055,7 +1133,7 @@ "[0, 1, 2]" ] }, - "execution_count": 29, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -1066,7 +1144,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -1149,7 +1227,7 @@ " 'sInd': {0: 0.8428651597363228, 1: 0.5220930407198541, 2: 0.5750817072006014}}" ] }, - "execution_count": 30, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1160,7 +1238,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -1217,11 +1295,15 @@ " 'Reference Entropy': 1.5,\n", " 'Response Entropy': 1.4833557549816874,\n", " 'SOA1(Landis & Koch)': 'Fair',\n", + " 'SOA10(Pearson C)': 'Strong',\n", " 'SOA2(Fleiss)': 'Poor',\n", " 'SOA3(Altman)': 'Fair',\n", " 'SOA4(Cicchetti)': 'Poor',\n", " 'SOA5(Cramer)': 'Relatively Strong',\n", " 'SOA6(Matthews)': 'Weak',\n", + " 'SOA7(Lambda A)': 'Very Weak',\n", + " 'SOA8(Lambda B)': 'Moderate',\n", + " 'SOA9(Krippendorff Alpha)': 'Low',\n", " 'Scott PI': 0.34426229508196726,\n", " 'Standard Error': 0.14231876063832777,\n", " 'TNR Macro': 0.7777777777777777,\n", @@ -1231,7 +1313,7 @@ " 'Zero-one Loss': 5}" ] }, - "execution_count": 31, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1253,7 +1335,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1262,7 +1344,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1271,7 +1353,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -1280,7 +1362,7 @@ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, - "execution_count": 34, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -1291,7 +1373,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 38, "metadata": {}, "outputs": [ { @@ -1300,7 +1382,7 @@ "[0, 1, 2]" ] }, - "execution_count": 35, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1320,7 +1402,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -1329,7 +1411,7 @@ "{0: {0: 1, 1: 2, 2: 3}, 1: {0: 4, 1: 6, 2: 1}, 2: {0: 1, 1: 2, 2: 3}}" ] }, - "execution_count": 36, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1340,7 +1422,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -1349,7 +1431,7 @@ "{0: {0: 1, 1: 2, 2: 3}, 1: {0: 4, 1: 6, 2: 1}, 2: {0: 1, 1: 2, 2: 3}}" ] }, - "execution_count": 37, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -1360,7 +1442,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -1371,7 +1453,7 @@ " 2: {0: 0.16667, 1: 0.33333, 2: 0.5}}" ] }, - "execution_count": 38, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -1382,7 +1464,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 42, "metadata": {}, "outputs": [ { @@ -1393,7 +1475,7 @@ " 2: {0: 0.16667, 1: 0.33333, 2: 0.5}}" ] }, - "execution_count": 39, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -1404,7 +1486,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -1429,7 +1511,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -1438,7 +1520,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -1447,7 +1529,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1456,7 +1538,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 47, "metadata": {}, "outputs": [ { @@ -1465,7 +1547,7 @@ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, - "execution_count": 44, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -1476,7 +1558,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -1485,7 +1567,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -1494,7 +1576,7 @@ "pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3'])" ] }, - "execution_count": 46, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -1519,16 +1601,16 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 50, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "L2 {'L2': 6, 'L1': 4, 'L3': 1}\n", - "L1 {'L2': 2, 'L1': 1, 'L3': 3}\n", - "L3 {'L2': 2, 'L1': 1, 'L3': 3}\n" + "L1 {'L1': 1, 'L3': 3, 'L2': 2}\n", + "L3 {'L1': 1, 'L3': 3, 'L2': 2}\n", + "L2 {'L1': 4, 'L3': 1, 'L2': 6}\n" ] } ], @@ -1539,16 +1621,16 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 51, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "('L2', {'L1': 4, 'L2': 6, 'L3': 1})" + "('L1', {'L1': 1, 'L2': 2, 'L3': 3})" ] }, - "execution_count": 48, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -1560,7 +1642,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 52, "metadata": {}, "outputs": [ { @@ -1571,7 +1653,7 @@ " 'L3': {'L1': 1, 'L2': 2, 'L3': 3}}" ] }, - "execution_count": 49, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } @@ -1583,18 +1665,18 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 53, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[('L2', {'L1': 4, 'L2': 6, 'L3': 1}),\n", - " ('L1', {'L1': 1, 'L2': 2, 'L3': 3}),\n", - " ('L3', {'L1': 1, 'L2': 2, 'L3': 3})]" + "[('L1', {'L1': 1, 'L2': 2, 'L3': 3}),\n", + " ('L3', {'L1': 1, 'L2': 2, 'L3': 3}),\n", + " ('L2', {'L1': 4, 'L2': 6, 'L3': 1})]" ] }, - "execution_count": 50, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -1706,7 +1788,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ @@ -1715,7 +1797,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 55, "metadata": {}, "outputs": [ { @@ -1764,7 +1846,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -1773,7 +1855,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 57, "metadata": {}, "outputs": [ { @@ -1782,7 +1864,7 @@ "pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3'])" ] }, - "execution_count": 54, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -1830,7 +1912,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 58, "metadata": {}, "outputs": [ { @@ -1841,7 +1923,7 @@ " 2: {'FN': [0, 3, 7], 'FP': [5, 10], 'TN': [1, 4, 6, 9], 'TP': [2, 8, 11]}}" ] }, - "execution_count": 55, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -1885,7 +1967,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 59, "metadata": {}, "outputs": [ { @@ -1896,7 +1978,7 @@ " [0, 2, 3]])" ] }, - "execution_count": 56, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } @@ -1907,7 +1989,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 60, "metadata": {}, "outputs": [ { @@ -1918,7 +2000,7 @@ " [0. , 0.4, 0.6]])" ] }, - "execution_count": 57, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -1929,7 +2011,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -1939,7 +2021,7 @@ " [0. , 1. ]])" ] }, - "execution_count": 58, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -2003,7 +2085,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 62, "metadata": {}, "outputs": [ { @@ -2080,7 +2162,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ @@ -2092,16 +2174,16 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 61, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" }, @@ -2124,16 +2206,16 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 62, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" }, @@ -2156,16 +2238,16 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 66, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 63, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" }, @@ -2188,16 +2270,16 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 64, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" }, @@ -2220,16 +2302,16 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 65, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" }, @@ -2293,6 +2375,30 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Distance/Similarity" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Distance.ipynb)\n", + "- [HTML](http://www.pycm.io/doc/Distance.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 3.8
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2340,7 +2446,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 69, "metadata": {}, "outputs": [ { @@ -2349,7 +2455,7 @@ "False" ] }, - "execution_count": 66, + "execution_count": 69, "metadata": {}, "output_type": "execute_result" } @@ -2360,7 +2466,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 70, "metadata": {}, "outputs": [ { @@ -2369,7 +2475,7 @@ "False" ] }, - "execution_count": 67, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -2380,7 +2486,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 71, "metadata": {}, "outputs": [ { @@ -2402,7 +2508,7 @@ " 'Zero-one Loss']" ] }, - "execution_count": 68, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -2420,7 +2526,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 72, "metadata": {}, "outputs": [ { @@ -2429,7 +2535,7 @@ "True" ] }, - "execution_count": 69, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -2441,7 +2547,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -2450,7 +2556,7 @@ "False" ] }, - "execution_count": 70, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -2500,7 +2606,7 @@ "source": [ "In `version 2.0`, a method for comparing several confusion matrices is introduced. This option is a combination of several overall and class-based benchmarks. Each of the benchmarks evaluates the performance of the classification algorithm from good to poor and give them a numeric score. The score of good and poor performances are 1 and 0, respectively.\n", "\n", - "After that, two scores are calculated for each confusion matrices, overall and class-based. The overall score is the average of the score of six overall benchmarks which are Landis & Koch, Fleiss, Altman, Cicchetti, Cramer, and Matthews. In the same manner, the class-based score is the average of the score of six class-based benchmarks which are Positive Likelihood Ratio Interpretation, Negative Likelihood Ratio Interpretation, Discriminant Power Interpretation, AUC value Interpretation, Matthews Correlation Coefficient Interpretation and Yule's Q Interpretation. It should be noticed that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminated in total averaging. If the user sets weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average.\n", + "After that, two scores are calculated for each confusion matrices, overall and class-based. The overall score is the average of the score of seven overall benchmarks which are Landis & Koch, Cramer, Matthews, Goodman-Kruskal's Lambda A, Goodman-Kruskal's Lambda B, Krippendorff's Alpha, and Pearson's C. In the same manner, the class-based score is the average of the score of six class-based benchmarks which are Positive Likelihood Ratio Interpretation, Negative Likelihood Ratio Interpretation, Discriminant Power Interpretation, AUC value Interpretation, Matthews Correlation Coefficient Interpretation and Yule's Q Interpretation. It should be noticed that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminated in total averaging. If the user sets weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average.\n", "\n", "If the user sets the value of `by_class` boolean input `True`, the best confusion matrix is the one with the maximum class-based score. Otherwise, if a confusion matrix obtains the maximum of both overall and class-based scores, that will be reported as the best confusion matrix, but in any other case, the compared object doesn’t select the best confusion matrix." ] @@ -2559,7 +2665,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -2569,7 +2675,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -2578,7 +2684,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -2588,8 +2694,8 @@ "Best : cm2\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 cm2 0.50278 0.425\n", - "2 cm3 0.33611 0.33056\n", + "1 cm2 0.50278 0.58095\n", + "2 cm3 0.33611 0.52857\n", "\n" ] } @@ -2600,17 +2706,17 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 77, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'cm2': {'class': 0.50278, 'overall': 0.425},\n", - " 'cm3': {'class': 0.33611, 'overall': 0.33056}}" + "{'cm2': {'class': 0.50278, 'overall': 0.58095},\n", + " 'cm3': {'class': 0.33611, 'overall': 0.52857}}" ] }, - "execution_count": 74, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -2621,7 +2727,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -2630,7 +2736,7 @@ "['cm2', 'cm3']" ] }, - "execution_count": 75, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -2641,7 +2747,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -2650,7 +2756,7 @@ "pycm.ConfusionMatrix(classes: [0, 1, 2])" ] }, - "execution_count": 76, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" } @@ -2661,7 +2767,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -2670,7 +2776,7 @@ "'cm2'" ] }, - "execution_count": 77, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -2681,7 +2787,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -2690,7 +2796,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 82, "metadata": {}, "outputs": [ { @@ -2700,8 +2806,8 @@ "Best : cm3\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 cm3 0.45357 0.33056\n", - "2 cm2 0.34881 0.425\n", + "1 cm3 0.45357 0.52857\n", + "2 cm2 0.34881 0.58095\n", "\n" ] } @@ -2712,7 +2818,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -2721,7 +2827,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 84, "metadata": {}, "outputs": [ { @@ -2731,8 +2837,8 @@ "Best : cm2\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 cm2 0.46667 0.425\n", - "2 cm3 0.33333 0.33056\n", + "1 cm2 0.46667 0.58095\n", + "2 cm3 0.33333 0.52857\n", "\n" ] } @@ -2743,16 +2849,16 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ - "cp4 = Compare({\"cm2\":cm2,\"cm3\":cm3},overall_benchmark_weight={\"SOA1\":1,\"SOA2\":0,\"SOA3\":0,\"SOA4\":0,\"SOA5\":0,\"SOA6\":1})" + "cp4 = Compare({\"cm2\":cm2,\"cm3\":cm3},overall_benchmark_weight={\"SOA1\":1,\"SOA2\":0,\"SOA3\":0,\"SOA4\":0,\"SOA5\":0,\"SOA6\":1,\"SOA7\":0,\"SOA8\":0,\"SOA9\":0,\"SOA10\":0})" ] }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 86, "metadata": {}, "outputs": [ { @@ -2781,7 +2887,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 87, "metadata": {}, "outputs": [ { @@ -2789,7 +2895,7 @@ "output_type": "stream", "text": [ "['AUCI', 'DPI', 'MCCI', 'NLRI', 'PLRI', 'QI']\n", - "['SOA1', 'SOA2', 'SOA3', 'SOA4', 'SOA5', 'SOA6']\n" + "['SOA1', 'SOA10', 'SOA2', 'SOA3', 'SOA4', 'SOA5', 'SOA6', 'SOA7', 'SOA8', 'SOA9']\n" ] } ], @@ -2808,6 +2914,15 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : From version 3.8, Goodman-Kruskal's Lambda A, Goodman-Kruskal's Lambda B, Krippendorff's Alpha, and Pearson's C benchmarks are considered in the overall score and default weights of the overall benchmarks are modified accordingly.
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2825,7 +2940,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 88, "metadata": {}, "outputs": [ { @@ -2834,7 +2949,7 @@ "0.75" ] }, - "execution_count": 85, + "execution_count": 88, "metadata": {}, "output_type": "execute_result" } @@ -2850,16 +2965,16 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 89, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 86, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" }, @@ -2906,14 +3021,14 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 90, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.7-py3.5.egg\\pycm\\pycm_curve.py:377: RuntimeWarning: The curve axes contain non-numerical value(s).\n" + "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.8-py3.5.egg\\pycm\\pycm_curve.py:379: RuntimeWarning: The curve axes contain non-numerical value(s).\n" ] }, { @@ -2922,7 +3037,7 @@ "0.29166666666666663" ] }, - "execution_count": 87, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -2938,16 +3053,16 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 91, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 88, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" }, @@ -3022,7 +3137,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 92, "metadata": {}, "outputs": [ { @@ -3143,22 +3258,26 @@ "108-SOA4(Cicchetti)\n", "109-SOA5(Cramer)\n", "110-SOA6(Matthews)\n", - "111-Scott PI\n", - "112-Standard Error\n", - "113-TN\n", - "114-TNR\n", - "115-TNR Macro\n", - "116-TNR Micro\n", - "117-TON\n", - "118-TOP\n", - "119-TP\n", - "120-TPR\n", - "121-TPR Macro\n", - "122-TPR Micro\n", - "123-Y\n", - "124-Zero-one Loss\n", - "125-dInd\n", - "126-sInd\n" + "111-SOA7(Lambda A)\n", + "112-SOA8(Lambda B)\n", + "113-SOA9(Krippendorff Alpha)\n", + "114-SOA10(Pearson C)\n", + "115-Scott PI\n", + "116-Standard Error\n", + "117-TN\n", + "118-TNR\n", + "119-TNR Macro\n", + "120-TNR Micro\n", + "121-TON\n", + "122-TOP\n", + "123-TP\n", + "124-TPR\n", + "125-TPR Macro\n", + "126-TPR Micro\n", + "127-Y\n", + "128-Zero-one Loss\n", + "129-dInd\n", + "130-sInd\n" ] } ], @@ -3286,7 +3405,7 @@ "2. `probs` : python `list` or numpy `array`\n", "3. `classes` : python `list`\n", "4. `thresholds`: python `list` or numpy `array` \n", - "5. `sample_weight`: python `list`" + "5. `sample_weight`: python `list` or numpy `array`" ] }, { @@ -3311,7 +3430,7 @@ "2. `probs` : python `list` or numpy `array`\n", "3. `classes` : python `list`\n", "4. `thresholds`: python `list` or numpy `array` \n", - "5. `sample_weight`: python `list`" + "5. `sample_weight`: python `list` or numpy `array`" ] }, { @@ -3345,7 +3464,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 93, "metadata": {}, "outputs": [ { @@ -3354,7 +3473,7 @@ "{'L1': 3, 'L2': 1, 'L3': 3}" ] }, - "execution_count": 90, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } @@ -3380,7 +3499,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 94, "metadata": {}, "outputs": [ { @@ -3389,7 +3508,7 @@ "{'L1': 7, 'L2': 8, 'L3': 4}" ] }, - "execution_count": 91, + "execution_count": 94, "metadata": {}, "output_type": "execute_result" } @@ -3415,7 +3534,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 95, "metadata": {}, "outputs": [ { @@ -3424,7 +3543,7 @@ "{'L1': 0, 'L2': 2, 'L3': 3}" ] }, - "execution_count": 92, + "execution_count": 95, "metadata": {}, "output_type": "execute_result" } @@ -3450,7 +3569,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 96, "metadata": {}, "outputs": [ { @@ -3459,7 +3578,7 @@ "{'L1': 2, 'L2': 1, 'L3': 2}" ] }, - "execution_count": 93, + "execution_count": 96, "metadata": {}, "output_type": "execute_result" } @@ -3492,7 +3611,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 97, "metadata": {}, "outputs": [ { @@ -3501,7 +3620,7 @@ "{'L1': 5, 'L2': 2, 'L3': 5}" ] }, - "execution_count": 94, + "execution_count": 97, "metadata": {}, "output_type": "execute_result" } @@ -3533,7 +3652,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 98, "metadata": {}, "outputs": [ { @@ -3542,7 +3661,7 @@ "{'L1': 7, 'L2': 10, 'L3': 7}" ] }, - "execution_count": 95, + "execution_count": 98, "metadata": {}, "output_type": "execute_result" } @@ -3574,7 +3693,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 99, "metadata": {}, "outputs": [ { @@ -3583,7 +3702,7 @@ "{'L1': 3, 'L2': 3, 'L3': 6}" ] }, - "execution_count": 96, + "execution_count": 99, "metadata": {}, "output_type": "execute_result" } @@ -3615,7 +3734,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 100, "metadata": {}, "outputs": [ { @@ -3624,7 +3743,7 @@ "{'L1': 9, 'L2': 9, 'L3': 6}" ] }, - "execution_count": 97, + "execution_count": 100, "metadata": {}, "output_type": "execute_result" } @@ -3656,7 +3775,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 101, "metadata": {}, "outputs": [ { @@ -3665,7 +3784,7 @@ "{'L1': 12, 'L2': 12, 'L3': 12}" ] }, - "execution_count": 98, + "execution_count": 101, "metadata": {}, "output_type": "execute_result" } @@ -3713,7 +3832,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 102, "metadata": {}, "outputs": [ { @@ -3722,7 +3841,7 @@ "{'L1': 0.6, 'L2': 0.5, 'L3': 0.6}" ] }, - "execution_count": 99, + "execution_count": 102, "metadata": {}, "output_type": "execute_result" } @@ -3756,7 +3875,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 103, "metadata": {}, "outputs": [ { @@ -3765,7 +3884,7 @@ "{'L1': 1.0, 'L2': 0.8, 'L3': 0.5714285714285714}" ] }, - "execution_count": 100, + "execution_count": 103, "metadata": {}, "output_type": "execute_result" } @@ -3800,7 +3919,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 104, "metadata": {}, "outputs": [ { @@ -3809,7 +3928,7 @@ "{'L1': 1.0, 'L2': 0.3333333333333333, 'L3': 0.5}" ] }, - "execution_count": 101, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -3844,7 +3963,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 105, "metadata": {}, "outputs": [ { @@ -3853,7 +3972,7 @@ "{'L1': 0.7777777777777778, 'L2': 0.8888888888888888, 'L3': 0.6666666666666666}" ] }, - "execution_count": 102, + "execution_count": 105, "metadata": {}, "output_type": "execute_result" } @@ -3887,7 +4006,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 106, "metadata": {}, "outputs": [ { @@ -3896,7 +4015,7 @@ "{'L1': 0.4, 'L2': 0.5, 'L3': 0.4}" ] }, - "execution_count": 103, + "execution_count": 106, "metadata": {}, "output_type": "execute_result" } @@ -3932,7 +4051,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 107, "metadata": {}, "outputs": [ { @@ -3941,7 +4060,7 @@ "{'L1': 0.0, 'L2': 0.19999999999999996, 'L3': 0.4285714285714286}" ] }, - "execution_count": 104, + "execution_count": 107, "metadata": {}, "output_type": "execute_result" } @@ -3975,7 +4094,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 108, "metadata": {}, "outputs": [ { @@ -3984,7 +4103,7 @@ "{'L1': 0.0, 'L2': 0.6666666666666667, 'L3': 0.5}" ] }, - "execution_count": 105, + "execution_count": 108, "metadata": {}, "output_type": "execute_result" } @@ -4018,7 +4137,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 109, "metadata": {}, "outputs": [ { @@ -4029,7 +4148,7 @@ " 'L3': 0.33333333333333337}" ] }, - "execution_count": 106, + "execution_count": 109, "metadata": {}, "output_type": "execute_result" } @@ -4063,7 +4182,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 110, "metadata": {}, "outputs": [ { @@ -4072,7 +4191,7 @@ "{'L1': 0.8333333333333334, 'L2': 0.75, 'L3': 0.5833333333333334}" ] }, - "execution_count": 107, + "execution_count": 110, "metadata": {}, "output_type": "execute_result" } @@ -4104,7 +4223,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 111, "metadata": {}, "outputs": [ { @@ -4113,7 +4232,7 @@ "{'L1': 0.16666666666666663, 'L2': 0.25, 'L3': 0.41666666666666663}" ] }, - "execution_count": 108, + "execution_count": 111, "metadata": {}, "output_type": "execute_result" } @@ -4157,7 +4276,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 112, "metadata": {}, "outputs": [ { @@ -4166,7 +4285,7 @@ "{'L1': 0.75, 'L2': 0.4, 'L3': 0.5454545454545454}" ] }, - "execution_count": 109, + "execution_count": 112, "metadata": {}, "output_type": "execute_result" } @@ -4177,7 +4296,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 113, "metadata": {}, "outputs": [ { @@ -4186,7 +4305,7 @@ "{'L1': 0.8823529411764706, 'L2': 0.35714285714285715, 'L3': 0.5172413793103449}" ] }, - "execution_count": 110, + "execution_count": 113, "metadata": {}, "output_type": "execute_result" } @@ -4197,7 +4316,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 114, "metadata": {}, "outputs": [ { @@ -4206,7 +4325,7 @@ "{'L1': 0.6521739130434783, 'L2': 0.45454545454545453, 'L3': 0.5769230769230769}" ] }, - "execution_count": 111, + "execution_count": 114, "metadata": {}, "output_type": "execute_result" } @@ -4217,7 +4336,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 115, "metadata": {}, "outputs": [ { @@ -4226,7 +4345,7 @@ "{'L1': 0.6144578313253012, 'L2': 0.4857142857142857, 'L3': 0.5930232558139535}" ] }, - "execution_count": 112, + "execution_count": 115, "metadata": {}, "output_type": "execute_result" } @@ -4299,7 +4418,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 116, "metadata": {}, "outputs": [ { @@ -4308,7 +4427,7 @@ "{'L1': 0.6831300510639732, 'L2': 0.25819888974716115, 'L3': 0.1690308509457033}" ] }, - "execution_count": 113, + "execution_count": 116, "metadata": {}, "output_type": "execute_result" } @@ -4342,7 +4461,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 117, "metadata": {}, "outputs": [ { @@ -4353,7 +4472,7 @@ " 'L3': 0.17142857142857126}" ] }, - "execution_count": 114, + "execution_count": 117, "metadata": {}, "output_type": "execute_result" } @@ -4385,7 +4504,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 118, "metadata": {}, "outputs": [ { @@ -4394,7 +4513,7 @@ "{'L1': 0.7777777777777777, 'L2': 0.2222222222222221, 'L3': 0.16666666666666652}" ] }, - "execution_count": 115, + "execution_count": 118, "metadata": {}, "output_type": "execute_result" } @@ -4430,7 +4549,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 119, "metadata": {}, "outputs": [ { @@ -4439,7 +4558,7 @@ "{'L1': 'None', 'L2': 2.5000000000000004, 'L3': 1.4}" ] }, - "execution_count": 116, + "execution_count": 119, "metadata": {}, "output_type": "execute_result" } @@ -4484,7 +4603,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 120, "metadata": {}, "outputs": [ { @@ -4493,7 +4612,7 @@ "{'L1': 0.4, 'L2': 0.625, 'L3': 0.7000000000000001}" ] }, - "execution_count": 117, + "execution_count": 120, "metadata": {}, "output_type": "execute_result" } @@ -4536,7 +4655,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 121, "metadata": {}, "outputs": [ { @@ -4545,7 +4664,7 @@ "{'L1': 'None', 'L2': 4.000000000000001, 'L3': 1.9999999999999998}" ] }, - "execution_count": 118, + "execution_count": 121, "metadata": {}, "output_type": "execute_result" } @@ -4579,7 +4698,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 122, "metadata": {}, "outputs": [ { @@ -4588,7 +4707,7 @@ "{'L1': 0.4166666666666667, 'L2': 0.16666666666666666, 'L3': 0.4166666666666667}" ] }, - "execution_count": 119, + "execution_count": 122, "metadata": {}, "output_type": "execute_result" } @@ -4622,7 +4741,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 123, "metadata": {}, "outputs": [ { @@ -4631,7 +4750,7 @@ "{'L1': 0.7745966692414834, 'L2': 0.408248290463863, 'L3': 0.5477225575051661}" ] }, - "execution_count": 120, + "execution_count": 123, "metadata": {}, "output_type": "execute_result" } @@ -4663,7 +4782,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 124, "metadata": {}, "outputs": [ { @@ -4674,7 +4793,7 @@ " 'L3': 0.20833333333333334}" ] }, - "execution_count": 121, + "execution_count": 124, "metadata": {}, "output_type": "execute_result" } @@ -4715,7 +4834,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 125, "metadata": {}, "outputs": [ { @@ -4726,7 +4845,7 @@ " 'L3': 0.21006944444444442}" ] }, - "execution_count": 122, + "execution_count": 125, "metadata": {}, "output_type": "execute_result" } @@ -4771,7 +4890,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 126, "metadata": {}, "outputs": [ { @@ -4780,7 +4899,7 @@ "{'L1': 0.6, 'L2': 0.25, 'L3': 0.375}" ] }, - "execution_count": 123, + "execution_count": 126, "metadata": {}, "output_type": "execute_result" } @@ -4822,7 +4941,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 127, "metadata": {}, "outputs": [ { @@ -4831,7 +4950,7 @@ "{'L1': 1.2630344058337937, 'L2': 0.9999999999999998, 'L3': 0.26303440583379367}" ] }, - "execution_count": 124, + "execution_count": 127, "metadata": {}, "output_type": "execute_result" } @@ -4888,7 +5007,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 128, "metadata": {}, "outputs": [ { @@ -4897,7 +5016,7 @@ "{'L1': 0.25, 'L2': 0.49657842846620864, 'L3': 0.6044162769630221}" ] }, - "execution_count": 125, + "execution_count": 128, "metadata": {}, "output_type": "execute_result" } @@ -4961,7 +5080,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 129, "metadata": {}, "outputs": [ { @@ -4970,7 +5089,7 @@ "{'L1': 0.2643856189774724, 'L2': 0.5, 'L3': 0.6875}" ] }, - "execution_count": 126, + "execution_count": 129, "metadata": {}, "output_type": "execute_result" } @@ -5014,7 +5133,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 130, "metadata": {}, "outputs": [ { @@ -5023,7 +5142,7 @@ "{'L1': 0.8, 'L2': 0.65, 'L3': 0.5857142857142856}" ] }, - "execution_count": 127, + "execution_count": 130, "metadata": {}, "output_type": "execute_result" } @@ -5065,7 +5184,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 131, "metadata": {}, "outputs": [ { @@ -5074,7 +5193,7 @@ "{'L1': 0.4, 'L2': 0.5385164807134504, 'L3': 0.5862367008195198}" ] }, - "execution_count": 128, + "execution_count": 131, "metadata": {}, "output_type": "execute_result" } @@ -5115,7 +5234,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 132, "metadata": {}, "outputs": [ { @@ -5124,7 +5243,7 @@ "{'L1': 0.717157287525381, 'L2': 0.6192113447068046, 'L3': 0.5854680534700882}" ] }, - "execution_count": 129, + "execution_count": 132, "metadata": {}, "output_type": "execute_result" } @@ -5182,7 +5301,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 133, "metadata": {}, "outputs": [ { @@ -5191,7 +5310,7 @@ "{'L1': 'None', 'L2': 0.33193306999649924, 'L3': 0.1659665349982495}" ] }, - "execution_count": 130, + "execution_count": 133, "metadata": {}, "output_type": "execute_result" } @@ -5241,7 +5360,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 134, "metadata": {}, "outputs": [ { @@ -5252,7 +5371,7 @@ " 'L3': 0.17142857142857126}" ] }, - "execution_count": 131, + "execution_count": 134, "metadata": {}, "output_type": "execute_result" } @@ -5316,7 +5435,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 135, "metadata": {}, "outputs": [ { @@ -5325,7 +5444,7 @@ "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}" ] }, - "execution_count": 132, + "execution_count": 135, "metadata": {}, "output_type": "execute_result" } @@ -5389,7 +5508,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 136, "metadata": {}, "outputs": [ { @@ -5398,7 +5517,7 @@ "{'L1': 'Poor', 'L2': 'Negligible', 'L3': 'Negligible'}" ] }, - "execution_count": 133, + "execution_count": 136, "metadata": {}, "output_type": "execute_result" } @@ -5462,7 +5581,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 137, "metadata": {}, "outputs": [ { @@ -5471,7 +5590,7 @@ "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}" ] }, - "execution_count": 134, + "execution_count": 137, "metadata": {}, "output_type": "execute_result" } @@ -5538,7 +5657,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 138, "metadata": {}, "outputs": [ { @@ -5547,7 +5666,7 @@ "{'L1': 'Very Good', 'L2': 'Fair', 'L3': 'Poor'}" ] }, - "execution_count": 135, + "execution_count": 138, "metadata": {}, "output_type": "execute_result" } @@ -5616,7 +5735,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 139, "metadata": {}, "outputs": [ { @@ -5625,7 +5744,7 @@ "{'L1': 'Moderate', 'L2': 'Negligible', 'L3': 'Negligible'}" ] }, - "execution_count": 136, + "execution_count": 139, "metadata": {}, "output_type": "execute_result" } @@ -5698,7 +5817,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 140, "metadata": {}, "outputs": [ { @@ -5707,7 +5826,7 @@ "{'L1': 'None', 'L2': 'Moderate', 'L3': 'Weak'}" ] }, - "execution_count": 137, + "execution_count": 140, "metadata": {}, "output_type": "execute_result" } @@ -5752,7 +5871,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 141, "metadata": {}, "outputs": [ { @@ -5763,7 +5882,7 @@ " 'L3': 0.17142857142857126}" ] }, - "execution_count": 138, + "execution_count": 141, "metadata": {}, "output_type": "execute_result" } @@ -5804,7 +5923,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 142, "metadata": {}, "outputs": [ { @@ -5813,7 +5932,7 @@ "{'L1': 2.4, 'L2': 2.0, 'L3': 1.2}" ] }, - "execution_count": 139, + "execution_count": 142, "metadata": {}, "output_type": "execute_result" } @@ -5854,7 +5973,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 143, "metadata": {}, "outputs": [ { @@ -5863,7 +5982,7 @@ "{'L1': -2, 'L2': 1, 'L3': 1}" ] }, - "execution_count": 140, + "execution_count": 143, "metadata": {}, "output_type": "execute_result" } @@ -5906,7 +6025,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 144, "metadata": {}, "outputs": [ { @@ -5917,7 +6036,7 @@ " 'L3': 0.041666666666666664}" ] }, - "execution_count": 141, + "execution_count": 144, "metadata": {}, "output_type": "execute_result" } @@ -5961,7 +6080,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 145, "metadata": {}, "outputs": [ { @@ -5970,7 +6089,7 @@ "{'L1': 0.5833333333333334, 'L2': 0.5192307692307692, 'L3': 0.5589430894308943}" ] }, - "execution_count": 142, + "execution_count": 145, "metadata": {}, "output_type": "execute_result" } @@ -6012,7 +6131,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 146, "metadata": {}, "outputs": [ { @@ -6021,7 +6140,7 @@ "{'L1': 0.36, 'L2': 0.27999999999999997, 'L3': 0.35265306122448975}" ] }, - "execution_count": 143, + "execution_count": 146, "metadata": {}, "output_type": "execute_result" } @@ -6032,7 +6151,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 147, "metadata": {}, "outputs": [ { @@ -6041,7 +6160,7 @@ "{'L1': 0.48, 'L2': 0.34, 'L3': 0.3477551020408163}" ] }, - "execution_count": 144, + "execution_count": 147, "metadata": {}, "output_type": "execute_result" } @@ -6052,7 +6171,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 148, "metadata": {}, "outputs": [ { @@ -6061,7 +6180,7 @@ "{'L1': 0.576, 'L2': 0.388, 'L3': 0.34383673469387754}" ] }, - "execution_count": 145, + "execution_count": 148, "metadata": {}, "output_type": "execute_result" } @@ -6130,7 +6249,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 149, "metadata": {}, "outputs": [ { @@ -6139,7 +6258,7 @@ "{'L1': 0.7745966692414834, 'L2': 0.6324555320336759, 'L3': 0.5855400437691198}" ] }, - "execution_count": 146, + "execution_count": 149, "metadata": {}, "output_type": "execute_result" } @@ -6191,7 +6310,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 150, "metadata": {}, "outputs": [ { @@ -6200,7 +6319,7 @@ "{'L1': 'None', 'L2': 0.6, 'L3': 0.3333333333333333}" ] }, - "execution_count": 147, + "execution_count": 150, "metadata": {}, "output_type": "execute_result" } @@ -6255,7 +6374,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 151, "metadata": {}, "outputs": [ { @@ -6264,7 +6383,7 @@ "{'L1': 0.8576400016262, 'L2': 0.708612108382005, 'L3': 0.5803410802752335}" ] }, - "execution_count": 148, + "execution_count": 151, "metadata": {}, "output_type": "execute_result" } @@ -6319,7 +6438,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 152, "metadata": {}, "outputs": [ { @@ -6328,7 +6447,7 @@ "{'L1': 0.7285871475307653, 'L2': 0.6286946134619315, 'L3': 0.610088876086563}" ] }, - "execution_count": 149, + "execution_count": 152, "metadata": {}, "output_type": "execute_result" } @@ -6371,7 +6490,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 153, "metadata": {}, "outputs": [ { @@ -6380,7 +6499,7 @@ "{'L1': 1.0, 'L2': 0.5, 'L3': 0.6}" ] }, - "execution_count": 150, + "execution_count": 153, "metadata": {}, "output_type": "execute_result" } @@ -6421,7 +6540,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 154, "metadata": {}, "outputs": [ { @@ -6430,7 +6549,7 @@ "{'L1': 0.6, 'L2': 0.3333333333333333, 'L3': 0.5}" ] }, - "execution_count": 151, + "execution_count": 154, "metadata": {}, "output_type": "execute_result" } @@ -6473,7 +6592,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 155, "metadata": {}, "outputs": [ { @@ -6482,7 +6601,7 @@ "{'L1': 0.7745966692414834, 'L2': 0.4082482904638631, 'L3': 0.5477225575051661}" ] }, - "execution_count": 152, + "execution_count": 155, "metadata": {}, "output_type": "execute_result" } @@ -6525,7 +6644,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 156, "metadata": {}, "outputs": [ { @@ -6534,7 +6653,7 @@ "{'L1': 0.42857142857142855, 'L2': 0.1111111111111111, 'L3': 0.1875}" ] }, - "execution_count": 153, + "execution_count": 156, "metadata": {}, "output_type": "execute_result" } @@ -6605,7 +6724,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 157, "metadata": {}, "outputs": [ { @@ -6614,7 +6733,7 @@ "{'L1': 0.8, 'L2': 0.41666666666666663, 'L3': 0.55}" ] }, - "execution_count": 154, + "execution_count": 157, "metadata": {}, "output_type": "execute_result" } @@ -6662,7 +6781,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 158, "metadata": {}, "outputs": [ { @@ -6673,7 +6792,7 @@ " 'L3': 0.10000000000000009}" ] }, - "execution_count": 155, + "execution_count": 158, "metadata": {}, "output_type": "execute_result" } @@ -6793,7 +6912,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 159, "metadata": {}, "outputs": [ { @@ -6804,7 +6923,7 @@ " 'L3': [0.21908902300206645, (0.17058551491594975, 1.0294144850840503)]}" ] }, - "execution_count": 156, + "execution_count": 159, "metadata": {}, "output_type": "execute_result" } @@ -6815,7 +6934,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 160, "metadata": {}, "outputs": [ { @@ -6826,7 +6945,7 @@ " 'L3': [0.21908902300206645, (-0.2769850810763853, 1.0769850810763852)]}" ] }, - "execution_count": 157, + "execution_count": 160, "metadata": {}, "output_type": "execute_result" } @@ -6837,7 +6956,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 161, "metadata": {}, "outputs": [ { @@ -6848,7 +6967,7 @@ " 'L3': [0.14231876063832774, (0.19325746190524654, 0.6804926643446272)]}" ] }, - "execution_count": 158, + "execution_count": 161, "metadata": {}, "output_type": "execute_result" } @@ -6859,7 +6978,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 162, "metadata": {}, "outputs": [ { @@ -6868,7 +6987,7 @@ "[0.14231876063832777, (0.2805568916340536, 0.8343177950165198)]" ] }, - "execution_count": 159, + "execution_count": 162, "metadata": {}, "output_type": "execute_result" } @@ -6879,7 +6998,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 163, "metadata": {}, "outputs": [ { @@ -6888,7 +7007,7 @@ "[0.14231876063832777, (0.30438856248221097, 0.8622781041844558)]" ] }, - "execution_count": 160, + "execution_count": 163, "metadata": {}, "output_type": "execute_result" } @@ -6989,7 +7108,7 @@ }, { "cell_type": "code", - "execution_count": 161, + "execution_count": 164, "metadata": {}, "outputs": [ { @@ -6998,7 +7117,7 @@ "{'L1': 0.25, 'L2': 0.0735, 'L3': 0.23525}" ] }, - "execution_count": 161, + "execution_count": 164, "metadata": {}, "output_type": "execute_result" } @@ -7062,7 +7181,7 @@ }, { "cell_type": "code", - "execution_count": 162, + "execution_count": 165, "metadata": {}, "outputs": [ { @@ -7071,7 +7190,7 @@ "0.6111111111111112" ] }, - "execution_count": 162, + "execution_count": 165, "metadata": {}, "output_type": "execute_result" } @@ -7082,7 +7201,7 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 166, "metadata": {}, "outputs": [ { @@ -7091,7 +7210,7 @@ "0.5651515151515151" ] }, - "execution_count": 163, + "execution_count": 166, "metadata": {}, "output_type": "execute_result" } @@ -7102,7 +7221,7 @@ }, { "cell_type": "code", - "execution_count": 164, + "execution_count": 167, "metadata": {}, "outputs": [ { @@ -7111,7 +7230,7 @@ "3.0000000000000004" ] }, - "execution_count": 164, + "execution_count": 167, "metadata": {}, "output_type": "execute_result" } @@ -7178,16 +7297,16 @@ }, { "cell_type": "code", - "execution_count": 165, + "execution_count": 168, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.6805555555555557" + "0.6805555555555555" ] }, - "execution_count": 165, + "execution_count": 168, "metadata": {}, "output_type": "execute_result" } @@ -7198,16 +7317,16 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 169, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.606439393939394" + "0.6064393939393938" ] }, - "execution_count": 166, + "execution_count": 169, "metadata": {}, "output_type": "execute_result" } @@ -7218,7 +7337,7 @@ }, { "cell_type": "code", - "execution_count": 167, + "execution_count": 170, "metadata": {}, "outputs": [ { @@ -7227,7 +7346,7 @@ "2.5714285714285716" ] }, - "execution_count": 167, + "execution_count": 170, "metadata": {}, "output_type": "execute_result" } @@ -7238,7 +7357,7 @@ }, { "cell_type": "code", - "execution_count": 168, + "execution_count": 171, "metadata": {}, "outputs": [ { @@ -7247,7 +7366,7 @@ "0.7152097902097903" ] }, - "execution_count": 168, + "execution_count": 171, "metadata": {}, "output_type": "execute_result" } @@ -7328,7 +7447,7 @@ }, { "cell_type": "code", - "execution_count": 169, + "execution_count": 172, "metadata": {}, "outputs": [ { @@ -7337,7 +7456,7 @@ "{'L1': 'None', 'L2': 0.8416212335729143, 'L3': 0.4333594729285047}" ] }, - "execution_count": 169, + "execution_count": 172, "metadata": {}, "output_type": "execute_result" } @@ -7401,7 +7520,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 173, "metadata": {}, "outputs": [ { @@ -7410,7 +7529,7 @@ "{'L1': 2, 'L2': 3, 'L3': 5}" ] }, - "execution_count": 170, + "execution_count": 173, "metadata": {}, "output_type": "execute_result" } @@ -7465,7 +7584,7 @@ }, { "cell_type": "code", - "execution_count": 171, + "execution_count": 174, "metadata": {}, "outputs": [ { @@ -7474,7 +7593,7 @@ "0.35483870967741943" ] }, - "execution_count": 171, + "execution_count": 174, "metadata": {}, "output_type": "execute_result" } @@ -7517,7 +7636,7 @@ }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 175, "metadata": {}, "outputs": [ { @@ -7526,7 +7645,7 @@ "0.34426229508196726" ] }, - "execution_count": 172, + "execution_count": 175, "metadata": {}, "output_type": "execute_result" } @@ -7567,7 +7686,7 @@ }, { "cell_type": "code", - "execution_count": 173, + "execution_count": 176, "metadata": {}, "outputs": [ { @@ -7576,7 +7695,7 @@ "0.16666666666666674" ] }, - "execution_count": 173, + "execution_count": 176, "metadata": {}, "output_type": "execute_result" } @@ -7638,7 +7757,7 @@ }, { "cell_type": "code", - "execution_count": 174, + "execution_count": 177, "metadata": {}, "outputs": [ { @@ -7647,7 +7766,7 @@ "0.39130434782608675" ] }, - "execution_count": 174, + "execution_count": 177, "metadata": {}, "output_type": "execute_result" } @@ -7658,14 +7777,14 @@ }, { "cell_type": "code", - "execution_count": 175, + "execution_count": 178, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.7-py3.5.egg\\pycm\\pycm_obj.py:802: RuntimeWarning: The weight format is wrong, the result is for unweighted kappa.\n" + "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.8-py3.5.egg\\pycm\\pycm_obj.py:839: RuntimeWarning: The weight format is wrong, the result is for unweighted kappa.\n" ] }, { @@ -7674,7 +7793,7 @@ "0.35483870967741943" ] }, - "execution_count": 175, + "execution_count": 178, "metadata": {}, "output_type": "execute_result" } @@ -7743,7 +7862,7 @@ }, { "cell_type": "code", - "execution_count": 176, + "execution_count": 179, "metadata": {}, "outputs": [ { @@ -7752,7 +7871,7 @@ "0.2203645326012817" ] }, - "execution_count": 176, + "execution_count": 179, "metadata": {}, "output_type": "execute_result" } @@ -7793,7 +7912,7 @@ }, { "cell_type": "code", - "execution_count": 177, + "execution_count": 180, "metadata": {}, "outputs": [ { @@ -7802,7 +7921,7 @@ "(-0.07707577422109269, 0.7867531935759315)" ] }, - "execution_count": 177, + "execution_count": 180, "metadata": {}, "output_type": "execute_result" } @@ -7852,7 +7971,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 181, "metadata": {}, "outputs": [ { @@ -7861,7 +7980,7 @@ "6.6000000000000005" ] }, - "execution_count": 178, + "execution_count": 181, "metadata": {}, "output_type": "execute_result" } @@ -7902,7 +8021,7 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 182, "metadata": {}, "outputs": [ { @@ -7911,7 +8030,7 @@ "4" ] }, - "execution_count": 179, + "execution_count": 182, "metadata": {}, "output_type": "execute_result" } @@ -7954,7 +8073,7 @@ }, { "cell_type": "code", - "execution_count": 180, + "execution_count": 183, "metadata": {}, "outputs": [ { @@ -7963,7 +8082,7 @@ "0.55" ] }, - "execution_count": 180, + "execution_count": 183, "metadata": {}, "output_type": "execute_result" } @@ -8008,7 +8127,7 @@ }, { "cell_type": "code", - "execution_count": 181, + "execution_count": 184, "metadata": {}, "outputs": [ { @@ -8017,7 +8136,7 @@ "0.5244044240850758" ] }, - "execution_count": 181, + "execution_count": 184, "metadata": {}, "output_type": "execute_result" } @@ -8060,7 +8179,7 @@ }, { "cell_type": "code", - "execution_count": 182, + "execution_count": 185, "metadata": {}, "outputs": [ { @@ -8069,7 +8188,7 @@ "0.14231876063832777" ] }, - "execution_count": 182, + "execution_count": 185, "metadata": {}, "output_type": "execute_result" } @@ -8112,7 +8231,7 @@ }, { "cell_type": "code", - "execution_count": 183, + "execution_count": 186, "metadata": {}, "outputs": [ { @@ -8121,7 +8240,7 @@ "(0.30438856248221097, 0.8622781041844558)" ] }, - "execution_count": 183, + "execution_count": 186, "metadata": {}, "output_type": "execute_result" } @@ -8181,7 +8300,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 187, "metadata": {}, "outputs": [ { @@ -8190,7 +8309,7 @@ "0.37500000000000006" ] }, - "execution_count": 184, + "execution_count": 187, "metadata": {}, "output_type": "execute_result" } @@ -8243,7 +8362,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 188, "metadata": {}, "outputs": [ { @@ -8252,7 +8371,7 @@ "0.34426229508196726" ] }, - "execution_count": 185, + "execution_count": 188, "metadata": {}, "output_type": "execute_result" } @@ -8307,7 +8426,7 @@ }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 189, "metadata": {}, "outputs": [ { @@ -8316,7 +8435,7 @@ "0.3893129770992367" ] }, - "execution_count": 186, + "execution_count": 189, "metadata": {}, "output_type": "execute_result" } @@ -8371,7 +8490,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 190, "metadata": {}, "outputs": [ { @@ -8380,7 +8499,7 @@ "1.4833557549816874" ] }, - "execution_count": 187, + "execution_count": 190, "metadata": {}, "output_type": "execute_result" } @@ -8435,7 +8554,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 191, "metadata": {}, "outputs": [ { @@ -8444,7 +8563,7 @@ "1.5" ] }, - "execution_count": 188, + "execution_count": 191, "metadata": {}, "output_type": "execute_result" } @@ -8508,7 +8627,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 192, "metadata": {}, "outputs": [ { @@ -8517,7 +8636,7 @@ "1.5833333333333335" ] }, - "execution_count": 189, + "execution_count": 192, "metadata": {}, "output_type": "execute_result" } @@ -8572,7 +8691,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 193, "metadata": {}, "outputs": [ { @@ -8581,7 +8700,7 @@ "2.4591479170272446" ] }, - "execution_count": 190, + "execution_count": 193, "metadata": {}, "output_type": "execute_result" } @@ -8638,7 +8757,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 194, "metadata": {}, "outputs": [ { @@ -8647,7 +8766,7 @@ "0.9757921620455572" ] }, - "execution_count": 191, + "execution_count": 194, "metadata": {}, "output_type": "execute_result" } @@ -8704,7 +8823,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 195, "metadata": {}, "outputs": [ { @@ -8713,7 +8832,7 @@ "0.09997757835164581" ] }, - "execution_count": 192, + "execution_count": 195, "metadata": {}, "output_type": "execute_result" } @@ -8785,7 +8904,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 196, "metadata": {}, "outputs": [ { @@ -8794,7 +8913,7 @@ "0.5242078379544428" ] }, - "execution_count": 193, + "execution_count": 196, "metadata": {}, "output_type": "execute_result" } @@ -8837,7 +8956,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 197, "metadata": {}, "outputs": [ { @@ -8846,7 +8965,7 @@ "0.42857142857142855" ] }, - "execution_count": 194, + "execution_count": 197, "metadata": {}, "output_type": "execute_result" } @@ -8889,7 +9008,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 198, "metadata": {}, "outputs": [ { @@ -8898,7 +9017,7 @@ "0.16666666666666666" ] }, - "execution_count": 195, + "execution_count": 198, "metadata": {}, "output_type": "execute_result" } @@ -8970,7 +9089,7 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 199, "metadata": {}, "outputs": [ { @@ -8979,7 +9098,7 @@ "'Fair'" ] }, - "execution_count": 196, + "execution_count": 199, "metadata": {}, "output_type": "execute_result" } @@ -9039,7 +9158,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 200, "metadata": {}, "outputs": [ { @@ -9048,7 +9167,7 @@ "'Poor'" ] }, - "execution_count": 197, + "execution_count": 200, "metadata": {}, "output_type": "execute_result" } @@ -9116,7 +9235,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 201, "metadata": {}, "outputs": [ { @@ -9125,7 +9244,7 @@ "'Fair'" ] }, - "execution_count": 198, + "execution_count": 201, "metadata": {}, "output_type": "execute_result" } @@ -9189,7 +9308,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 202, "metadata": {}, "outputs": [ { @@ -9198,7 +9317,7 @@ "'Poor'" ] }, - "execution_count": 199, + "execution_count": 202, "metadata": {}, "output_type": "execute_result" } @@ -9270,7 +9389,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 203, "metadata": {}, "outputs": [ { @@ -9279,7 +9398,7 @@ "'Relatively Strong'" ] }, - "execution_count": 200, + "execution_count": 203, "metadata": {}, "output_type": "execute_result" } @@ -9348,7 +9467,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 204, "metadata": {}, "outputs": [ { @@ -9357,7 +9476,7 @@ "'Weak'" ] }, - "execution_count": 201, + "execution_count": 204, "metadata": {}, "output_type": "execute_result" } @@ -9388,48 +9507,71 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Overall_ACC" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more information visit [[3]](#ref3)." + "### SOA7 (Goodman & Kruskal's lambda A benchmark)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "$$ACC_{Overall}=\\frac{\\sum_{i=1}^{|C|}TP_i}{POP}$$" + "For more information visit [[84]](#ref84)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Equals to [TPR Micro](#TPR_Micro), [F1 Micro](#F1_Micro) and [PPV Micro](#PPV_Micro)" + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    Lambda AStrength of Association
    0 - 0.2Very Weak
    0.2 - 0.4Weak
    0.4 - 0.6Moderate
    0.6 - 0.8Strong
    0.8 - 1.0Very Strong
    1.0Perfect
    " ] }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 205, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.5833333333333334" + "'Moderate'" ] }, - "execution_count": 202, + "execution_count": 205, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "cm.Overall_ACC" + "cm.SOA7" ] }, { @@ -9437,7 +9579,7 @@ "metadata": {}, "source": [ "
      \n", - "
    • Notice : new in version 0.4
      • \n", + "
    • Notice : new in version 3.8
      • \n", "
    " ] }, @@ -9445,35 +9587,310 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Overall_RACC" + "### SOA8 (Goodman & Kruskal's lambda B benchmark)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "For more information visit [[24]](#ref24)." + "For more information visit [[84]](#ref84)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "$$RACC_{Overall}=\\sum_{i=1}^{|C|}RACC_i$$" + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    Lambda BStrength of Association
    0 - 0.2Very Weak
    0.2 - 0.4Weak
    0.4 - 0.6Moderate
    0.6 - 0.8Strong
    0.8 - 1.0Very Strong
    1.0Perfect
    " ] }, { "cell_type": "code", - "execution_count": 203, + "execution_count": 206, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "0.3541666666666667" + "'Very Weak'" ] }, - "execution_count": 203, + "execution_count": 206, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.SOA8" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 3.8
      • \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SOA9 (Krippendorff's alpha benchmark)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more information visit [[85]](#ref85)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    AlphaStrength of Agreement
    0.667 >Low
    0.667 - 0.8Tentative
    0.8 < High
    " + ] + }, + { + "cell_type": "code", + "execution_count": 207, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Low'" + ] + }, + "execution_count": 207, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.SOA9" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 3.8
      • \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SOA10 (Pearson's C benchmark)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more information visit [[86]](#ref86)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    CStrength of Association
    0 - 0.1Not Appreciable
    0.1 - 0.2Weak
    0.2 - 0.3Medium
    0.3 < Strong
    " + ] + }, + { + "cell_type": "code", + "execution_count": 208, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Strong'" + ] + }, + "execution_count": 208, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.SOA10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 3.8
      • \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Overall_ACC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more information visit [[3]](#ref3)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$ACC_{Overall}=\\frac{\\sum_{i=1}^{|C|}TP_i}{POP}$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Equals to [TPR Micro](#TPR_Micro), [F1 Micro](#F1_Micro) and [PPV Micro](#PPV_Micro)" + ] + }, + { + "cell_type": "code", + "execution_count": 209, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5833333333333334" + ] + }, + "execution_count": 209, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.Overall_ACC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 0.4
      • \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Overall_RACC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more information visit [[24]](#ref24)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$RACC_{Overall}=\\sum_{i=1}^{|C|}RACC_i$$" + ] + }, + { + "cell_type": "code", + "execution_count": 210, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3541666666666667" + ] + }, + "execution_count": 210, "metadata": {}, "output_type": "execute_result" } @@ -9514,7 +9931,7 @@ }, { "cell_type": "code", - "execution_count": 204, + "execution_count": 211, "metadata": {}, "outputs": [ { @@ -9523,7 +9940,7 @@ "0.3645833333333333" ] }, - "execution_count": 204, + "execution_count": 211, "metadata": {}, "output_type": "execute_result" } @@ -9571,7 +9988,7 @@ }, { "cell_type": "code", - "execution_count": 205, + "execution_count": 212, "metadata": {}, "outputs": [ { @@ -9580,7 +9997,7 @@ "0.5833333333333334" ] }, - "execution_count": 205, + "execution_count": 212, "metadata": {}, "output_type": "execute_result" } @@ -9628,7 +10045,7 @@ }, { "cell_type": "code", - "execution_count": 206, + "execution_count": 213, "metadata": {}, "outputs": [ { @@ -9637,7 +10054,7 @@ "0.5833333333333334" ] }, - "execution_count": 206, + "execution_count": 213, "metadata": {}, "output_type": "execute_result" } @@ -9678,7 +10095,7 @@ }, { "cell_type": "code", - "execution_count": 207, + "execution_count": 214, "metadata": {}, "outputs": [ { @@ -9687,7 +10104,7 @@ "0.7916666666666666" ] }, - "execution_count": 207, + "execution_count": 214, "metadata": {}, "output_type": "execute_result" } @@ -9728,7 +10145,7 @@ }, { "cell_type": "code", - "execution_count": 208, + "execution_count": 215, "metadata": {}, "outputs": [ { @@ -9737,7 +10154,7 @@ "0.20833333333333337" ] }, - "execution_count": 208, + "execution_count": 215, "metadata": {}, "output_type": "execute_result" } @@ -9778,7 +10195,7 @@ }, { "cell_type": "code", - "execution_count": 209, + "execution_count": 216, "metadata": {}, "outputs": [ { @@ -9787,7 +10204,7 @@ "0.41666666666666663" ] }, - "execution_count": 209, + "execution_count": 216, "metadata": {}, "output_type": "execute_result" } @@ -9835,7 +10252,7 @@ }, { "cell_type": "code", - "execution_count": 210, + "execution_count": 217, "metadata": {}, "outputs": [ { @@ -9844,7 +10261,7 @@ "0.5833333333333334" ] }, - "execution_count": 210, + "execution_count": 217, "metadata": {}, "output_type": "execute_result" } @@ -9885,7 +10302,7 @@ }, { "cell_type": "code", - "execution_count": 211, + "execution_count": 218, "metadata": {}, "outputs": [ { @@ -9894,7 +10311,7 @@ "0.611111111111111" ] }, - "execution_count": 211, + "execution_count": 218, "metadata": {}, "output_type": "execute_result" } @@ -9935,7 +10352,7 @@ }, { "cell_type": "code", - "execution_count": 212, + "execution_count": 219, "metadata": {}, "outputs": [ { @@ -9944,7 +10361,7 @@ "0.5666666666666668" ] }, - "execution_count": 212, + "execution_count": 219, "metadata": {}, "output_type": "execute_result" } @@ -9985,7 +10402,7 @@ }, { "cell_type": "code", - "execution_count": 213, + "execution_count": 220, "metadata": {}, "outputs": [ { @@ -9994,7 +10411,7 @@ "0.7904761904761904" ] }, - "execution_count": 213, + "execution_count": 220, "metadata": {}, "output_type": "execute_result" } @@ -10035,7 +10452,7 @@ }, { "cell_type": "code", - "execution_count": 214, + "execution_count": 221, "metadata": {}, "outputs": [ { @@ -10044,7 +10461,7 @@ "0.20952380952380956" ] }, - "execution_count": 214, + "execution_count": 221, "metadata": {}, "output_type": "execute_result" } @@ -10085,7 +10502,7 @@ }, { "cell_type": "code", - "execution_count": 215, + "execution_count": 222, "metadata": {}, "outputs": [ { @@ -10094,7 +10511,7 @@ "0.43333333333333324" ] }, - "execution_count": 215, + "execution_count": 222, "metadata": {}, "output_type": "execute_result" } @@ -10135,7 +10552,7 @@ }, { "cell_type": "code", - "execution_count": 216, + "execution_count": 223, "metadata": {}, "outputs": [ { @@ -10144,7 +10561,7 @@ "0.5651515151515151" ] }, - "execution_count": 216, + "execution_count": 223, "metadata": {}, "output_type": "execute_result" } @@ -10185,7 +10602,7 @@ }, { "cell_type": "code", - "execution_count": 217, + "execution_count": 224, "metadata": {}, "outputs": [ { @@ -10194,7 +10611,7 @@ "0.7222222222222223" ] }, - "execution_count": 217, + "execution_count": 224, "metadata": {}, "output_type": "execute_result" } @@ -10249,7 +10666,7 @@ }, { "cell_type": "code", - "execution_count": 218, + "execution_count": 225, "metadata": {}, "outputs": [ { @@ -10258,7 +10675,7 @@ "(1.225, 0.4083333333333334)" ] }, - "execution_count": 218, + "execution_count": 225, "metadata": {}, "output_type": "execute_result" } @@ -10299,7 +10716,7 @@ }, { "cell_type": "code", - "execution_count": 219, + "execution_count": 226, "metadata": {}, "outputs": [ { @@ -10308,7 +10725,7 @@ "0.41666666666666663" ] }, - "execution_count": 219, + "execution_count": 226, "metadata": {}, "output_type": "execute_result" } @@ -10349,7 +10766,7 @@ }, { "cell_type": "code", - "execution_count": 220, + "execution_count": 227, "metadata": {}, "outputs": [ { @@ -10358,7 +10775,7 @@ "5" ] }, - "execution_count": 220, + "execution_count": 227, "metadata": {}, "output_type": "execute_result" } @@ -10399,7 +10816,7 @@ }, { "cell_type": "code", - "execution_count": 221, + "execution_count": 228, "metadata": {}, "outputs": [ { @@ -10408,7 +10825,7 @@ "0.4166666666666667" ] }, - "execution_count": 221, + "execution_count": 228, "metadata": {}, "output_type": "execute_result" } @@ -10476,7 +10893,7 @@ }, { "cell_type": "code", - "execution_count": 222, + "execution_count": 229, "metadata": {}, "outputs": [ { @@ -10485,7 +10902,7 @@ "0.18926430237560654" ] }, - "execution_count": 222, + "execution_count": 229, "metadata": {}, "output_type": "execute_result" } @@ -10533,7 +10950,7 @@ }, { "cell_type": "code", - "execution_count": 223, + "execution_count": 230, "metadata": {}, "outputs": [ { @@ -10542,7 +10959,7 @@ "0.4638112995385119" ] }, - "execution_count": 223, + "execution_count": 230, "metadata": {}, "output_type": "execute_result" } @@ -10597,7 +11014,7 @@ }, { "cell_type": "code", - "execution_count": 224, + "execution_count": 231, "metadata": {}, "outputs": [ { @@ -10606,7 +11023,7 @@ "0.5189369467580801" ] }, - "execution_count": 224, + "execution_count": 231, "metadata": {}, "output_type": "execute_result" } @@ -10670,7 +11087,7 @@ }, { "cell_type": "code", - "execution_count": 225, + "execution_count": 232, "metadata": {}, "outputs": [ { @@ -10679,7 +11096,7 @@ "0.36666666666666664" ] }, - "execution_count": 225, + "execution_count": 232, "metadata": {}, "output_type": "execute_result" } @@ -10720,7 +11137,7 @@ }, { "cell_type": "code", - "execution_count": 226, + "execution_count": 233, "metadata": {}, "outputs": [ { @@ -10729,7 +11146,7 @@ "4.0" ] }, - "execution_count": 226, + "execution_count": 233, "metadata": {}, "output_type": "execute_result" } @@ -10772,7 +11189,7 @@ }, { "cell_type": "code", - "execution_count": 227, + "execution_count": 234, "metadata": {}, "outputs": [ { @@ -10781,7 +11198,7 @@ "0.4777777777777778" ] }, - "execution_count": 227, + "execution_count": 234, "metadata": {}, "output_type": "execute_result" } @@ -10822,7 +11239,7 @@ }, { "cell_type": "code", - "execution_count": 228, + "execution_count": 235, "metadata": {}, "outputs": [ { @@ -10831,7 +11248,7 @@ "0.6785714285714285" ] }, - "execution_count": 228, + "execution_count": 235, "metadata": {}, "output_type": "execute_result" } @@ -10872,7 +11289,7 @@ }, { "cell_type": "code", - "execution_count": 229, + "execution_count": 236, "metadata": {}, "outputs": [ { @@ -10881,7 +11298,7 @@ "0.6857142857142857" ] }, - "execution_count": 229, + "execution_count": 236, "metadata": {}, "output_type": "execute_result" } @@ -10944,7 +11361,7 @@ }, { "cell_type": "code", - "execution_count": 230, + "execution_count": 237, "metadata": {}, "outputs": [ { @@ -10953,7 +11370,7 @@ "0.3533932006492363" ] }, - "execution_count": 230, + "execution_count": 237, "metadata": {}, "output_type": "execute_result" } @@ -10994,7 +11411,7 @@ }, { "cell_type": "code", - "execution_count": 231, + "execution_count": 238, "metadata": {}, "outputs": [ { @@ -11003,7 +11420,7 @@ "0.5956833971812706" ] }, - "execution_count": 231, + "execution_count": 238, "metadata": {}, "output_type": "execute_result" } @@ -11045,7 +11462,7 @@ }, { "cell_type": "code", - "execution_count": 232, + "execution_count": 239, "metadata": {}, "outputs": [ { @@ -11054,7 +11471,7 @@ "0.1777777777777778" ] }, - "execution_count": 232, + "execution_count": 239, "metadata": {}, "output_type": "execute_result" } @@ -11106,7 +11523,7 @@ }, { "cell_type": "code", - "execution_count": 233, + "execution_count": 240, "metadata": {}, "outputs": [ { @@ -11115,7 +11532,7 @@ "0.09206349206349207" ] }, - "execution_count": 233, + "execution_count": 240, "metadata": {}, "output_type": "execute_result" } @@ -11167,7 +11584,7 @@ }, { "cell_type": "code", - "execution_count": 234, + "execution_count": 241, "metadata": {}, "outputs": [ { @@ -11176,7 +11593,7 @@ "0.37254901960784315" ] }, - "execution_count": 234, + "execution_count": 241, "metadata": {}, "output_type": "execute_result" } @@ -11241,7 +11658,7 @@ }, { "cell_type": "code", - "execution_count": 235, + "execution_count": 242, "metadata": {}, "outputs": [ { @@ -11250,7 +11667,7 @@ "0.3715846994535519" ] }, - "execution_count": 235, + "execution_count": 242, "metadata": {}, "output_type": "execute_result" } @@ -11326,7 +11743,7 @@ }, { "cell_type": "code", - "execution_count": 236, + "execution_count": 243, "metadata": {}, "outputs": [ { @@ -11335,7 +11752,7 @@ "0.374757281553398" ] }, - "execution_count": 236, + "execution_count": 243, "metadata": {}, "output_type": "execute_result" } @@ -11346,14 +11763,14 @@ }, { "cell_type": "code", - "execution_count": 237, + "execution_count": 244, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.7-py3.5.egg\\pycm\\pycm_obj.py:824: RuntimeWarning: The weight format is wrong, the result is for unweighted alpha.\n" + "C:\\Users\\Sepkjaer\\AppData\\Local\\Programs\\Python\\Python35-32\\lib\\site-packages\\pycm-3.8-py3.5.egg\\pycm\\pycm_obj.py:861: RuntimeWarning: The weight format is wrong, the result is for unweighted alpha.\n" ] }, { @@ -11362,7 +11779,7 @@ "0.3715846994535519" ] }, - "execution_count": 237, + "execution_count": 244, "metadata": {}, "output_type": "execute_result" } @@ -11466,7 +11883,7 @@ }, { "cell_type": "code", - "execution_count": 238, + "execution_count": 245, "metadata": {}, "outputs": [ { @@ -11475,7 +11892,7 @@ "0.38540577344968524" ] }, - "execution_count": 238, + "execution_count": 245, "metadata": {}, "output_type": "execute_result" } @@ -11486,7 +11903,7 @@ }, { "cell_type": "code", - "execution_count": 239, + "execution_count": 246, "metadata": {}, "outputs": [ { @@ -11495,7 +11912,7 @@ "0.38545857383594895" ] }, - "execution_count": 239, + "execution_count": 246, "metadata": {}, "output_type": "execute_result" } @@ -11574,7 +11991,7 @@ }, { "cell_type": "code", - "execution_count": 240, + "execution_count": 247, "metadata": {}, "outputs": [ { @@ -11583,7 +12000,7 @@ "0.03749999999999999" ] }, - "execution_count": 240, + "execution_count": 247, "metadata": {}, "output_type": "execute_result" } @@ -11595,7 +12012,7 @@ }, { "cell_type": "code", - "execution_count": 241, + "execution_count": 248, "metadata": {}, "outputs": [ { @@ -11604,7 +12021,7 @@ "0.6875" ] }, - "execution_count": 241, + "execution_count": 248, "metadata": {}, "output_type": "execute_result" } @@ -11684,7 +12101,7 @@ }, { "cell_type": "code", - "execution_count": 242, + "execution_count": 249, "metadata": {}, "outputs": [ { @@ -11761,6 +12178,10 @@ "SOA4(Cicchetti) Poor\n", "SOA5(Cramer) Relatively Strong\n", "SOA6(Matthews) Weak\n", + "SOA7(Lambda A) Moderate\n", + "SOA8(Lambda B) Very Weak\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.34426\n", "Standard Error 0.14232\n", "TNR Macro 0.79048\n", @@ -11852,7 +12273,7 @@ }, { "cell_type": "code", - "execution_count": 243, + "execution_count": 250, "metadata": {}, "outputs": [ { @@ -11877,7 +12298,7 @@ }, { "cell_type": "code", - "execution_count": 244, + "execution_count": 251, "metadata": {}, "outputs": [ { @@ -11888,7 +12309,7 @@ " 'L3': {'L1': 0, 'L2': 2, 'L3': 3}}" ] }, - "execution_count": 244, + "execution_count": 251, "metadata": {}, "output_type": "execute_result" } @@ -11899,7 +12320,7 @@ }, { "cell_type": "code", - "execution_count": 245, + "execution_count": 252, "metadata": {}, "outputs": [ { @@ -11922,7 +12343,7 @@ }, { "cell_type": "code", - "execution_count": 246, + "execution_count": 253, "metadata": {}, "outputs": [], "source": [ @@ -11931,7 +12352,7 @@ }, { "cell_type": "code", - "execution_count": 247, + "execution_count": 254, "metadata": {}, "outputs": [ { @@ -12004,7 +12425,7 @@ }, { "cell_type": "code", - "execution_count": 248, + "execution_count": 255, "metadata": {}, "outputs": [ { @@ -12029,7 +12450,7 @@ }, { "cell_type": "code", - "execution_count": 249, + "execution_count": 256, "metadata": {}, "outputs": [ { @@ -12040,7 +12461,7 @@ " 'L3': {'L1': 0.0, 'L2': 0.4, 'L3': 0.6}}" ] }, - "execution_count": 249, + "execution_count": 256, "metadata": {}, "output_type": "execute_result" } @@ -12051,7 +12472,7 @@ }, { "cell_type": "code", - "execution_count": 250, + "execution_count": 257, "metadata": {}, "outputs": [ { @@ -12074,7 +12495,7 @@ }, { "cell_type": "code", - "execution_count": 251, + "execution_count": 258, "metadata": {}, "outputs": [ { @@ -12147,7 +12568,7 @@ }, { "cell_type": "code", - "execution_count": 252, + "execution_count": 259, "metadata": {}, "outputs": [ { @@ -12212,6 +12633,10 @@ "SOA4(Cicchetti) Poor\n", "SOA5(Cramer) Relatively Strong\n", "SOA6(Matthews) Weak\n", + "SOA7(Lambda A) Moderate\n", + "SOA8(Lambda B) Very Weak\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.34426\n", "Standard Error 0.14232\n", "TNR Macro 0.79048\n", @@ -12296,7 +12721,7 @@ }, { "cell_type": "code", - "execution_count": 253, + "execution_count": 260, "metadata": {}, "outputs": [ { @@ -12323,7 +12748,7 @@ }, { "cell_type": "code", - "execution_count": 254, + "execution_count": 261, "metadata": {}, "outputs": [ { @@ -12350,7 +12775,7 @@ }, { "cell_type": "code", - "execution_count": 255, + "execution_count": 262, "metadata": {}, "outputs": [ { @@ -12458,7 +12883,7 @@ }, { "cell_type": "code", - "execution_count": 256, + "execution_count": 263, "metadata": {}, "outputs": [ { @@ -12468,8 +12893,8 @@ "Best : cm2\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 cm2 0.50278 0.425\n", - "2 cm3 0.33611 0.33056\n", + "1 cm2 0.50278 0.58095\n", + "2 cm3 0.33611 0.52857\n", "\n" ] } @@ -12480,7 +12905,7 @@ }, { "cell_type": "code", - "execution_count": 257, + "execution_count": 264, "metadata": {}, "outputs": [ { @@ -12490,8 +12915,8 @@ "Best : cm2\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 cm2 0.50278 0.425\n", - "2 cm3 0.33611 0.33056\n", + "1 cm2 0.50278 0.58095\n", + "2 cm3 0.33611 0.52857\n", "\n" ] } @@ -12509,7 +12934,7 @@ }, { "cell_type": "code", - "execution_count": 258, + "execution_count": 265, "metadata": {}, "outputs": [], "source": [ @@ -12527,7 +12952,7 @@ }, { "cell_type": "code", - "execution_count": 259, + "execution_count": 266, "metadata": {}, "outputs": [ { @@ -12537,7 +12962,7 @@ " 'Status': True}" ] }, - "execution_count": 259, + "execution_count": 266, "metadata": {}, "output_type": "execute_result" } @@ -12555,7 +12980,7 @@ }, { "cell_type": "code", - "execution_count": 260, + "execution_count": 267, "metadata": {}, "outputs": [ { @@ -12565,7 +12990,7 @@ " 'Status': True}" ] }, - "execution_count": 260, + "execution_count": 267, "metadata": {}, "output_type": "execute_result" } @@ -12583,7 +13008,7 @@ }, { "cell_type": "code", - "execution_count": 261, + "execution_count": 268, "metadata": {}, "outputs": [ { @@ -12593,7 +13018,7 @@ " 'Status': True}" ] }, - "execution_count": 261, + "execution_count": 268, "metadata": {}, "output_type": "execute_result" } @@ -12611,7 +13036,7 @@ }, { "cell_type": "code", - "execution_count": 262, + "execution_count": 269, "metadata": {}, "outputs": [ { @@ -12621,7 +13046,7 @@ " 'Status': True}" ] }, - "execution_count": 262, + "execution_count": 269, "metadata": {}, "output_type": "execute_result" } @@ -12639,7 +13064,7 @@ }, { "cell_type": "code", - "execution_count": 263, + "execution_count": 270, "metadata": {}, "outputs": [ { @@ -12649,7 +13074,7 @@ " 'Status': True}" ] }, - "execution_count": 263, + "execution_count": 270, "metadata": {}, "output_type": "execute_result" } @@ -12667,7 +13092,7 @@ }, { "cell_type": "code", - "execution_count": 264, + "execution_count": 271, "metadata": {}, "outputs": [ { @@ -12677,7 +13102,7 @@ " 'Status': False}" ] }, - "execution_count": 264, + "execution_count": 271, "metadata": {}, "output_type": "execute_result" } @@ -12760,7 +13185,7 @@ }, { "cell_type": "code", - "execution_count": 265, + "execution_count": 272, "metadata": {}, "outputs": [ { @@ -12770,7 +13195,7 @@ " 'Status': True}" ] }, - "execution_count": 265, + "execution_count": 272, "metadata": {}, "output_type": "execute_result" } @@ -12788,7 +13213,7 @@ }, { "cell_type": "code", - "execution_count": 266, + "execution_count": 273, "metadata": {}, "outputs": [ { @@ -12798,7 +13223,7 @@ " 'Status': True}" ] }, - "execution_count": 266, + "execution_count": 273, "metadata": {}, "output_type": "execute_result" } @@ -12816,7 +13241,7 @@ }, { "cell_type": "code", - "execution_count": 267, + "execution_count": 274, "metadata": {}, "outputs": [ { @@ -12826,7 +13251,7 @@ " 'Status': True}" ] }, - "execution_count": 267, + "execution_count": 274, "metadata": {}, "output_type": "execute_result" } @@ -12844,7 +13269,7 @@ }, { "cell_type": "code", - "execution_count": 268, + "execution_count": 275, "metadata": {}, "outputs": [ { @@ -12854,7 +13279,7 @@ " 'Status': True}" ] }, - "execution_count": 268, + "execution_count": 275, "metadata": {}, "output_type": "execute_result" } @@ -12872,7 +13297,7 @@ }, { "cell_type": "code", - "execution_count": 269, + "execution_count": 276, "metadata": {}, "outputs": [ { @@ -12882,7 +13307,7 @@ " 'Status': True}" ] }, - "execution_count": 269, + "execution_count": 276, "metadata": {}, "output_type": "execute_result" } @@ -12900,7 +13325,7 @@ }, { "cell_type": "code", - "execution_count": 270, + "execution_count": 277, "metadata": {}, "outputs": [ { @@ -12910,7 +13335,7 @@ " 'Status': True}" ] }, - "execution_count": 270, + "execution_count": 277, "metadata": {}, "output_type": "execute_result" } @@ -12928,7 +13353,7 @@ }, { "cell_type": "code", - "execution_count": 271, + "execution_count": 278, "metadata": {}, "outputs": [ { @@ -12938,7 +13363,7 @@ " 'Status': True}" ] }, - "execution_count": 271, + "execution_count": 278, "metadata": {}, "output_type": "execute_result" } @@ -12956,7 +13381,7 @@ }, { "cell_type": "code", - "execution_count": 272, + "execution_count": 279, "metadata": {}, "outputs": [ { @@ -12966,7 +13391,7 @@ " 'Status': False}" ] }, - "execution_count": 272, + "execution_count": 279, "metadata": {}, "output_type": "execute_result" } @@ -13079,7 +13504,7 @@ }, { "cell_type": "code", - "execution_count": 273, + "execution_count": 280, "metadata": {}, "outputs": [ { @@ -13089,7 +13514,7 @@ " 'Status': True}" ] }, - "execution_count": 273, + "execution_count": 280, "metadata": {}, "output_type": "execute_result" } @@ -13109,7 +13534,7 @@ }, { "cell_type": "code", - "execution_count": 274, + "execution_count": 281, "metadata": {}, "outputs": [ { @@ -13119,7 +13544,7 @@ " 'Status': True}" ] }, - "execution_count": 274, + "execution_count": 281, "metadata": {}, "output_type": "execute_result" } @@ -13139,7 +13564,7 @@ }, { "cell_type": "code", - "execution_count": 275, + "execution_count": 282, "metadata": {}, "outputs": [ { @@ -13149,7 +13574,7 @@ " 'Status': True}" ] }, - "execution_count": 275, + "execution_count": 282, "metadata": {}, "output_type": "execute_result" } @@ -13169,7 +13594,7 @@ }, { "cell_type": "code", - "execution_count": 276, + "execution_count": 283, "metadata": {}, "outputs": [ { @@ -13179,7 +13604,7 @@ " 'Status': True}" ] }, - "execution_count": 276, + "execution_count": 283, "metadata": {}, "output_type": "execute_result" } @@ -13199,7 +13624,7 @@ }, { "cell_type": "code", - "execution_count": 277, + "execution_count": 284, "metadata": {}, "outputs": [ { @@ -13209,7 +13634,7 @@ " 'Status': True}" ] }, - "execution_count": 277, + "execution_count": 284, "metadata": {}, "output_type": "execute_result" } @@ -13229,7 +13654,7 @@ }, { "cell_type": "code", - "execution_count": 278, + "execution_count": 285, "metadata": {}, "outputs": [ { @@ -13239,7 +13664,7 @@ " 'Status': True}" ] }, - "execution_count": 278, + "execution_count": 285, "metadata": {}, "output_type": "execute_result" } @@ -13257,7 +13682,7 @@ }, { "cell_type": "code", - "execution_count": 279, + "execution_count": 286, "metadata": {}, "outputs": [ { @@ -13267,7 +13692,7 @@ " 'Status': False}" ] }, - "execution_count": 279, + "execution_count": 286, "metadata": {}, "output_type": "execute_result" } @@ -13360,7 +13785,7 @@ }, { "cell_type": "code", - "execution_count": 280, + "execution_count": 287, "metadata": {}, "outputs": [ { @@ -13370,7 +13795,7 @@ " 'Status': True}" ] }, - "execution_count": 280, + "execution_count": 287, "metadata": {}, "output_type": "execute_result" } @@ -13388,7 +13813,7 @@ }, { "cell_type": "code", - "execution_count": 281, + "execution_count": 288, "metadata": {}, "outputs": [ { @@ -13398,7 +13823,7 @@ " 'Status': True}" ] }, - "execution_count": 281, + "execution_count": 288, "metadata": {}, "output_type": "execute_result" } @@ -13416,7 +13841,7 @@ }, { "cell_type": "code", - "execution_count": 282, + "execution_count": 289, "metadata": {}, "outputs": [ { @@ -13426,7 +13851,7 @@ " 'Status': True}" ] }, - "execution_count": 282, + "execution_count": 289, "metadata": {}, "output_type": "execute_result" } @@ -13444,7 +13869,7 @@ }, { "cell_type": "code", - "execution_count": 283, + "execution_count": 290, "metadata": {}, "outputs": [ { @@ -13454,7 +13879,7 @@ " 'Status': False}" ] }, - "execution_count": 283, + "execution_count": 290, "metadata": {}, "output_type": "execute_result" } @@ -13516,7 +13941,7 @@ }, { "cell_type": "code", - "execution_count": 284, + "execution_count": 291, "metadata": {}, "outputs": [ { @@ -13526,7 +13951,7 @@ " 'Status': True}" ] }, - "execution_count": 284, + "execution_count": 291, "metadata": {}, "output_type": "execute_result" } @@ -13544,7 +13969,7 @@ }, { "cell_type": "code", - "execution_count": 285, + "execution_count": 292, "metadata": {}, "outputs": [ { @@ -13554,7 +13979,7 @@ " 'Status': False}" ] }, - "execution_count": 285, + "execution_count": 292, "metadata": {}, "output_type": "execute_result" } @@ -13596,7 +14021,7 @@ }, { "cell_type": "code", - "execution_count": 286, + "execution_count": 293, "metadata": {}, "outputs": [ { @@ -13616,7 +14041,7 @@ }, { "cell_type": "code", - "execution_count": 287, + "execution_count": 294, "metadata": { "scrolled": true }, @@ -13638,7 +14063,7 @@ }, { "cell_type": "code", - "execution_count": 288, + "execution_count": 295, "metadata": {}, "outputs": [ { @@ -13658,7 +14083,7 @@ }, { "cell_type": "code", - "execution_count": 289, + "execution_count": 296, "metadata": {}, "outputs": [ { @@ -13678,7 +14103,7 @@ }, { "cell_type": "code", - "execution_count": 290, + "execution_count": 297, "metadata": {}, "outputs": [ { @@ -13698,7 +14123,7 @@ }, { "cell_type": "code", - "execution_count": 291, + "execution_count": 298, "metadata": {}, "outputs": [ { @@ -13718,7 +14143,7 @@ }, { "cell_type": "code", - "execution_count": 292, + "execution_count": 299, "metadata": {}, "outputs": [ { @@ -13738,7 +14163,7 @@ }, { "cell_type": "code", - "execution_count": 293, + "execution_count": 300, "metadata": {}, "outputs": [ { @@ -13758,7 +14183,7 @@ }, { "cell_type": "code", - "execution_count": 294, + "execution_count": 301, "metadata": {}, "outputs": [ { @@ -13778,7 +14203,7 @@ }, { "cell_type": "code", - "execution_count": 295, + "execution_count": 302, "metadata": {}, "outputs": [ { @@ -13798,7 +14223,7 @@ }, { "cell_type": "code", - "execution_count": 296, + "execution_count": 303, "metadata": {}, "outputs": [ { @@ -13818,7 +14243,7 @@ }, { "cell_type": "code", - "execution_count": 297, + "execution_count": 304, "metadata": {}, "outputs": [ { @@ -13838,7 +14263,7 @@ }, { "cell_type": "code", - "execution_count": 298, + "execution_count": 305, "metadata": {}, "outputs": [ { @@ -13858,7 +14283,7 @@ }, { "cell_type": "code", - "execution_count": 299, + "execution_count": 306, "metadata": {}, "outputs": [ { @@ -13878,7 +14303,7 @@ }, { "cell_type": "code", - "execution_count": 300, + "execution_count": 307, "metadata": {}, "outputs": [ { @@ -13899,7 +14324,7 @@ }, { "cell_type": "code", - "execution_count": 301, + "execution_count": 308, "metadata": {}, "outputs": [ { @@ -13919,7 +14344,7 @@ }, { "cell_type": "code", - "execution_count": 302, + "execution_count": 309, "metadata": {}, "outputs": [ { @@ -13939,7 +14364,7 @@ }, { "cell_type": "code", - "execution_count": 303, + "execution_count": 310, "metadata": {}, "outputs": [ { @@ -13959,7 +14384,7 @@ }, { "cell_type": "code", - "execution_count": 304, + "execution_count": 311, "metadata": {}, "outputs": [ { @@ -13979,7 +14404,7 @@ }, { "cell_type": "code", - "execution_count": 305, + "execution_count": 312, "metadata": {}, "outputs": [ { @@ -13999,7 +14424,7 @@ }, { "cell_type": "code", - "execution_count": 306, + "execution_count": 313, "metadata": {}, "outputs": [ { @@ -14019,7 +14444,7 @@ }, { "cell_type": "code", - "execution_count": 307, + "execution_count": 314, "metadata": {}, "outputs": [ { @@ -14039,7 +14464,7 @@ }, { "cell_type": "code", - "execution_count": 308, + "execution_count": 315, "metadata": {}, "outputs": [ { @@ -14059,7 +14484,7 @@ }, { "cell_type": "code", - "execution_count": 309, + "execution_count": 316, "metadata": {}, "outputs": [ { @@ -14079,7 +14504,7 @@ }, { "cell_type": "code", - "execution_count": 310, + "execution_count": 317, "metadata": {}, "outputs": [ { @@ -14099,7 +14524,7 @@ }, { "cell_type": "code", - "execution_count": 311, + "execution_count": 318, "metadata": {}, "outputs": [ { @@ -14119,7 +14544,7 @@ }, { "cell_type": "code", - "execution_count": 312, + "execution_count": 319, "metadata": {}, "outputs": [ { @@ -14139,7 +14564,7 @@ }, { "cell_type": "code", - "execution_count": 313, + "execution_count": 320, "metadata": {}, "outputs": [ { @@ -14159,7 +14584,7 @@ }, { "cell_type": "code", - "execution_count": 314, + "execution_count": 321, "metadata": {}, "outputs": [ { @@ -14179,7 +14604,7 @@ }, { "cell_type": "code", - "execution_count": 315, + "execution_count": 322, "metadata": {}, "outputs": [ { @@ -14199,7 +14624,7 @@ }, { "cell_type": "code", - "execution_count": 316, + "execution_count": 323, "metadata": {}, "outputs": [ { @@ -14219,7 +14644,7 @@ }, { "cell_type": "code", - "execution_count": 317, + "execution_count": 324, "metadata": {}, "outputs": [ { @@ -14239,7 +14664,7 @@ }, { "cell_type": "code", - "execution_count": 318, + "execution_count": 325, "metadata": {}, "outputs": [ { @@ -14259,7 +14684,7 @@ }, { "cell_type": "code", - "execution_count": 319, + "execution_count": 326, "metadata": {}, "outputs": [ { @@ -14279,7 +14704,7 @@ }, { "cell_type": "code", - "execution_count": 320, + "execution_count": 327, "metadata": {}, "outputs": [ { @@ -14299,7 +14724,7 @@ }, { "cell_type": "code", - "execution_count": 321, + "execution_count": 328, "metadata": {}, "outputs": [ { @@ -14605,7 +15030,13 @@ "\n", "
    82- J. Braun-Blanquet, \"Plant sociology. The study of plant communities,\" Plant sociology. The study of plant communities. First ed., 1932.
    \n", "\n", - "
    83- C. C. Little, \"Abydos Documentation,\" 2020.
    \n" + "
    83- C. C. Little, \"Abydos Documentation,\" 2020.
    \n", + "\n", + "
    84- K. Villela, A. Silva, T. Vale, and E. S. de Almeida, \"A survey on software variability management approaches,\" in Proceedings of the 18th International Software Product Line Conference-Volume 1, 2014, pp. 147-156.
    \n", + " \n", + "
    85- J. R. Saura, A. Reyes-Menendez, and P. Palos-Sanchez, \"Are black Friday deals worth it? Mining Twitter users’ sentiment and behavior response,\" Journal of Open Innovation: Technology, Market, and Complexity, vol. 5, no. 3, p. 58, 2019.
    \n", + "\n", + "
    86- P. Schubert and U. Leimstoll, \"Importance and use of information technology in small and medium‐sized companies,\" Electronic Markets, vol. 17, no. 1, pp. 38-55, 2007.
    \n" ] } ], diff --git a/Document/Document_Files/cm1.html b/Document/Document_Files/cm1.html index d0bdbf2d..cb79547e 100644 --- a/Document/Document_Files/cm1.html +++ b/Document/Document_Files/cm1.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Weak +SOA7(Lambda A) +Moderate + + +SOA8(Lambda B) +Very Weak + + +SOA9(Krippendorff Alpha) +Low + + +SOA10(Pearson C) +Strong + + Scott PI 0.34426 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1.obj b/Document/Document_Files/cm1.obj index 5d651302..34f91b7c 100644 --- a/Document/Document_Files/cm1.obj +++ b/Document/Document_Files/cm1.obj @@ -1 +1 @@ -{"Digit": 5, "Prob-Vector": null, "Sample-Weight": null, "Imbalanced": false, "Transpose": true, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Actual-Vector": null, "Predict-Vector": null} \ No newline at end of file +{"Prob-Vector": null, "Transpose": true, "Imbalanced": false, "Predict-Vector": null, "Matrix": [["L1", [["L1", 3], ["L3", 2], ["L2", 0]]], ["L2", [["L1", 0], ["L3", 1], ["L2", 1]]], ["L3", [["L1", 0], ["L3", 3], ["L2", 2]]]], "Actual-Vector": null, "Sample-Weight": null, "Digit": 5} \ No newline at end of file diff --git a/Document/Document_Files/cm1.pycm b/Document/Document_Files/cm1.pycm index 7ea48225..4e805d68 100644 --- a/Document/Document_Files/cm1.pycm +++ b/Document/Document_Files/cm1.pycm @@ -80,6 +80,10 @@ SOA3(Altman) Fair SOA4(Cicchetti) Poor SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Moderate +SOA8(Lambda B) Very Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.34426 Standard Error 0.14232 TNR Macro 0.79048 diff --git a/Document/Document_Files/cm1_colored.html b/Document/Document_Files/cm1_colored.html index 797f5768..322a4d58 100644 --- a/Document/Document_Files/cm1_colored.html +++ b/Document/Document_Files/cm1_colored.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Weak +SOA7(Lambda A) +Moderate + + +SOA8(Lambda B) +Very Weak + + +SOA9(Krippendorff Alpha) +Low + + +SOA10(Pearson C) +Strong + + Scott PI 0.34426 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1_colored2.html b/Document/Document_Files/cm1_colored2.html index 9af44eb3..1e123316 100644 --- a/Document/Document_Files/cm1_colored2.html +++ b/Document/Document_Files/cm1_colored2.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Weak +SOA7(Lambda A) +Moderate + + +SOA8(Lambda B) +Very Weak + + +SOA9(Krippendorff Alpha) +Low + + +SOA10(Pearson C) +Strong + + Scott PI 0.34426 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1_filtered.html b/Document/Document_Files/cm1_filtered.html index 5fd7713f..25bc01e3 100644 --- a/Document/Document_Files/cm1_filtered.html +++ b/Document/Document_Files/cm1_filtered.html @@ -95,6 +95,6 @@

    Class Statistics :

    Sensitivity, recall, hit rate, or true positive rate -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1_filtered2.html b/Document/Document_Files/cm1_filtered2.html index 5ac43395..f4e02cf7 100644 --- a/Document/Document_Files/cm1_filtered2.html +++ b/Document/Document_Files/cm1_filtered2.html @@ -87,6 +87,6 @@

    Class Statistics :

    Sensitivity, recall, hit rate, or true positive rate -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1_no_vectors.obj b/Document/Document_Files/cm1_no_vectors.obj index 5d651302..34f91b7c 100644 --- a/Document/Document_Files/cm1_no_vectors.obj +++ b/Document/Document_Files/cm1_no_vectors.obj @@ -1 +1 @@ -{"Digit": 5, "Prob-Vector": null, "Sample-Weight": null, "Imbalanced": false, "Transpose": true, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Actual-Vector": null, "Predict-Vector": null} \ No newline at end of file +{"Prob-Vector": null, "Transpose": true, "Imbalanced": false, "Predict-Vector": null, "Matrix": [["L1", [["L1", 3], ["L3", 2], ["L2", 0]]], ["L2", [["L1", 0], ["L3", 1], ["L2", 1]]], ["L3", [["L1", 0], ["L3", 3], ["L2", 2]]]], "Actual-Vector": null, "Sample-Weight": null, "Digit": 5} \ No newline at end of file diff --git a/Document/Document_Files/cm1_normalized.html b/Document/Document_Files/cm1_normalized.html index 3a093b80..ae62104b 100644 --- a/Document/Document_Files/cm1_normalized.html +++ b/Document/Document_Files/cm1_normalized.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Weak +SOA7(Lambda A) +Moderate + + +SOA8(Lambda B) +Very Weak + + +SOA9(Krippendorff Alpha) +Low + + +SOA10(Pearson C) +Strong + + Scott PI 0.34426 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cm1_stat.obj b/Document/Document_Files/cm1_stat.obj index 9ab8ab41..fbcf00f5 100644 --- a/Document/Document_Files/cm1_stat.obj +++ b/Document/Document_Files/cm1_stat.obj @@ -1 +1 @@ -{"Digit": 5, "Class-Stat": {"IS": {"L2": 0.9999999999999998, "L1": 1.2630344058337937, "L3": 0.26303440583379367}, "GM": {"L2": 0.6324555320336759, "L1": 0.7745966692414834, "L3": 0.5855400437691198}, "G": {"L2": 0.408248290463863, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "MCC": {"L2": 0.25819888974716115, "L1": 0.6831300510639732, "L3": 0.1690308509457033}, "F2": {"L2": 0.45454545454545453, "L1": 0.6521739130434783, "L3": 0.5769230769230769}, "AUCI": {"L2": "Fair", "L1": "Very Good", "L3": "Poor"}, "RACC": {"L2": 0.041666666666666664, "L1": 0.10416666666666667, "L3": 0.20833333333333334}, "PRE": {"L2": 0.16666666666666666, "L1": 0.4166666666666667, "L3": 0.4166666666666667}, "OOC": {"L2": 0.4082482904638631, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "PLR": {"L2": 2.5000000000000004, "L1": "None", "L3": 1.4}, "F0.5": {"L2": 0.35714285714285715, "L1": 0.8823529411764706, "L3": 0.5172413793103449}, "FOR": {"L2": 0.11111111111111116, "L1": 0.2222222222222222, "L3": 0.33333333333333337}, "FPR": {"L2": 0.19999999999999996, "L1": 0.0, "L3": 0.4285714285714286}, "BM": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "MCCI": {"L2": "Negligible", "L1": "Moderate", "L3": "Negligible"}, "TN": {"L2": 8, "L1": 7, "L3": 4}, "PLRI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "F1": {"L2": 0.4, "L1": 0.75, "L3": 0.5454545454545454}, "DOR": {"L2": 4.000000000000001, "L1": "None", "L3": 1.9999999999999998}, "AGM": {"L2": 0.708612108382005, "L1": 0.8576400016262, "L3": 0.5803410802752335}, "GI": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "QI": {"L2": "Moderate", "L1": "None", "L3": "Weak"}, "NPV": {"L2": 0.8888888888888888, "L1": 0.7777777777777778, "L3": 0.6666666666666666}, "TOP": {"L2": 3, "L1": 3, "L3": 6}, "FN": {"L2": 1, "L1": 2, "L3": 2}, "NLRI": {"L2": "Negligible", "L1": "Poor", "L3": "Negligible"}, "IBA": {"L2": 0.27999999999999997, "L1": 0.36, "L3": 0.35265306122448975}, "POP": {"L2": 12, "L1": 12, "L3": 12}, "HD": {"L2": 3, "L1": 2, "L3": 5}, "TNR": {"L2": 0.8, "L1": 1.0, "L3": 0.5714285714285714}, "N": {"L2": 10, "L1": 7, "L3": 7}, "FP": {"L2": 2, "L1": 0, "L3": 3}, "OP": {"L2": 0.5192307692307692, "L1": 0.5833333333333334, "L3": 0.5589430894308943}, "LS": {"L2": 2.0, "L1": 2.4, "L3": 1.2}, "ACC": {"L2": 0.75, "L1": 0.8333333333333334, "L3": 0.5833333333333334}, "FNR": {"L2": 0.5, "L1": 0.4, "L3": 0.4}, "AGF": {"L2": 0.6286946134619315, "L1": 0.7285871475307653, "L3": 0.610088876086563}, "BB": {"L2": 0.3333333333333333, "L1": 0.6, "L3": 0.5}, "NLR": {"L2": 0.625, "L1": 0.4, "L3": 0.7000000000000001}, "AM": {"L2": 1, "L1": -2, "L3": 1}, "Q": {"L2": 0.6, "L1": "None", "L3": 0.3333333333333333}, "TP": {"L2": 1, "L1": 3, "L3": 3}, "P": {"L2": 2, "L1": 5, "L3": 5}, "TON": {"L2": 9, "L1": 9, "L3": 6}, "MK": {"L2": 0.2222222222222221, "L1": 0.7777777777777777, "L3": 0.16666666666666652}, "MCEN": {"L2": 0.5, "L1": 0.2643856189774724, "L3": 0.6875}, "ERR": {"L2": 0.25, "L1": 0.16666666666666663, "L3": 0.41666666666666663}, "RACCU": {"L2": 0.04340277777777778, "L1": 0.1111111111111111, "L3": 0.21006944444444442}, "OC": {"L2": 0.5, "L1": 1.0, "L3": 0.6}, "DP": {"L2": 0.33193306999649924, "L1": "None", "L3": 0.1659665349982495}, "FDR": {"L2": 0.6666666666666667, "L1": 0.0, "L3": 0.5}, "dInd": {"L2": 0.5385164807134504, "L1": 0.4, "L3": 0.5862367008195198}, "AUPR": {"L2": 0.41666666666666663, "L1": 0.8, "L3": 0.55}, "BCD": {"L2": 0.041666666666666664, "L1": 0.08333333333333333, "L3": 0.041666666666666664}, "TPR": {"L2": 0.5, "L1": 0.6, "L3": 0.6}, "CEN": {"L2": 0.49657842846620864, "L1": 0.25, "L3": 0.6044162769630221}, "sInd": {"L2": 0.6192113447068046, "L1": 0.717157287525381, "L3": 0.5854680534700882}, "DPI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "AUC": {"L2": 0.65, "L1": 0.8, "L3": 0.5857142857142856}, "PPV": {"L2": 0.3333333333333333, "L1": 1.0, "L3": 0.5}, "ICSI": {"L2": -0.16666666666666674, "L1": 0.6000000000000001, "L3": 0.10000000000000009}, "Y": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "J": {"L2": 0.25, "L1": 0.6, "L3": 0.375}}, "Prob-Vector": null, "Sample-Weight": null, "Imbalanced": false, "Transpose": true, "Overall-Stat": {"Standard Error": 0.14231876063832777, "P-Value": 0.18926430237560654, "TNR Macro": 0.7904761904761904, "FPR Macro": 0.20952380952380956, "CBA": 0.4777777777777778, "Chi-Squared DF": 4, "Kappa 95% CI": [-0.07707577422109269, 0.7867531935759315], "TPR Macro": 0.5666666666666668, "Chi-Squared": 6.6000000000000005, "SOA1(Landis & Koch)": "Fair", "KL Divergence": 0.09997757835164581, "Overall MCEN": 0.5189369467580801, "Gwet AC1": 0.3893129770992367, "AUNP": 0.6857142857142857, "Overall RACC": 0.3541666666666667, "Bangdiwala B": 0.37254901960784315, "SOA2(Fleiss)": "Poor", "Overall MCC": 0.36666666666666664, "Zero-one Loss": 5, "95% CI": [0.30438856248221097, 0.8622781041844558], "CSI": 0.1777777777777778, "Scott PI": 0.34426229508196726, "F1 Macro": 0.5651515151515151, "ARI": 0.09206349206349207, "TPR Micro": 0.5833333333333334, "PPV Macro": 0.611111111111111, "PPV Micro": 0.5833333333333334, "Bennett S": 0.37500000000000006, "Joint Entropy": 2.4591479170272446, "Krippendorff Alpha": 0.3715846994535519, "Pearson C": 0.5956833971812706, "Phi-Squared": 0.55, "Cross Entropy": 1.5833333333333335, "AUNU": 0.6785714285714285, "Overall RACCU": 0.3645833333333333, "Cramer V": 0.5244044240850758, "Kappa Standard Error": 0.2203645326012817, "Overall ACC": 0.5833333333333334, "SOA6(Matthews)": "Weak", "FNR Macro": 0.43333333333333324, "RR": 4.0, "Conditional Entropy": 0.9757921620455572, "Kappa Unbiased": 0.34426229508196726, "SOA4(Cicchetti)": "Poor", "TNR Micro": 0.7916666666666666, "Response Entropy": 1.5, "Lambda B": 0.16666666666666666, "FNR Micro": 0.41666666666666663, "SOA5(Cramer)": "Relatively Strong", "F1 Micro": 0.5833333333333334, "Mutual Information": 0.5242078379544428, "Reference Entropy": 1.4833557549816874, "Overall J": [1.225, 0.4083333333333334], "Lambda A": 0.42857142857142855, "Kappa No Prevalence": 0.16666666666666674, "FPR Micro": 0.20833333333333337, "NIR": 0.4166666666666667, "Kappa": 0.35483870967741943, "SOA3(Altman)": "Fair", "Overall CEN": 0.4638112995385119, "Hamming Loss": 0.41666666666666663, "ACC Macro": 0.7222222222222223, "RCI": 0.3533932006492363}, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Actual-Vector": null, "Predict-Vector": null} \ No newline at end of file +{"Prob-Vector": null, "Transpose": true, "Imbalanced": false, "Predict-Vector": null, "Class-Stat": {"BCD": {"L1": 0.08333333333333333, "L3": 0.041666666666666664, "L2": 0.041666666666666664}, "TP": {"L1": 3, "L3": 3, "L2": 1}, "P": {"L1": 5, "L3": 5, "L2": 2}, "FNR": {"L1": 0.4, "L3": 0.4, "L2": 0.5}, "GI": {"L1": 0.6000000000000001, "L3": 0.17142857142857126, "L2": 0.30000000000000004}, "RACC": {"L1": 0.10416666666666667, "L3": 0.20833333333333334, "L2": 0.041666666666666664}, "BB": {"L1": 0.6, "L3": 0.5, "L2": 0.3333333333333333}, "FOR": {"L1": 0.2222222222222222, "L3": 0.33333333333333337, "L2": 0.11111111111111116}, "DP": {"L1": "None", "L3": 0.1659665349982495, "L2": 0.33193306999649924}, "FDR": {"L1": 0.0, "L3": 0.5, "L2": 0.6666666666666667}, "DPI": {"L1": "None", "L3": "Poor", "L2": "Poor"}, "AUCI": {"L1": "Very Good", "L3": "Poor", "L2": "Fair"}, "TOP": {"L1": 3, "L3": 6, "L2": 3}, "F2": {"L1": 0.6521739130434783, "L3": 0.5769230769230769, "L2": 0.45454545454545453}, "QI": {"L1": "None", "L3": "Weak", "L2": "Moderate"}, "AUC": {"L1": 0.8, "L3": 0.5857142857142856, "L2": 0.65}, "TPR": {"L1": 0.6, "L3": 0.6, "L2": 0.5}, "BM": {"L1": 0.6000000000000001, "L3": 0.17142857142857126, "L2": 0.30000000000000004}, "OC": {"L1": 1.0, "L3": 0.6, "L2": 0.5}, "F1": {"L1": 0.75, "L3": 0.5454545454545454, "L2": 0.4}, "AUPR": {"L1": 0.8, "L3": 0.55, "L2": 0.41666666666666663}, "ACC": {"L1": 0.8333333333333334, "L3": 0.5833333333333334, "L2": 0.75}, "MCEN": {"L1": 0.2643856189774724, "L3": 0.6875, "L2": 0.5}, "F0.5": {"L1": 0.8823529411764706, "L3": 0.5172413793103449, "L2": 0.35714285714285715}, "TN": {"L1": 7, "L3": 4, "L2": 8}, "OOC": {"L1": 0.7745966692414834, "L3": 0.5477225575051661, "L2": 0.4082482904638631}, "NLR": {"L1": 0.4, "L3": 0.7000000000000001, "L2": 0.625}, "ICSI": {"L1": 0.6000000000000001, "L3": 0.10000000000000009, "L2": -0.16666666666666674}, "PLRI": {"L1": "None", "L3": "Poor", "L2": "Poor"}, "PLR": {"L1": "None", "L3": 1.4, "L2": 2.5000000000000004}, "J": {"L1": 0.6, "L3": 0.375, "L2": 0.25}, "Y": {"L1": 0.6000000000000001, "L3": 0.17142857142857126, "L2": 0.30000000000000004}, "MCCI": {"L1": "Moderate", "L3": "Negligible", "L2": "Negligible"}, "PRE": {"L1": 0.4166666666666667, "L3": 0.4166666666666667, "L2": 0.16666666666666666}, "AGM": {"L1": 0.8576400016262, "L3": 0.5803410802752335, "L2": 0.708612108382005}, "POP": {"L1": 12, "L3": 12, "L2": 12}, "FP": {"L1": 0, "L3": 3, "L2": 2}, "NLRI": {"L1": "Poor", "L3": "Negligible", "L2": "Negligible"}, "AGF": {"L1": 0.7285871475307653, "L3": 0.610088876086563, "L2": 0.6286946134619315}, "Q": {"L1": "None", "L3": 0.3333333333333333, "L2": 0.6}, "HD": {"L1": 2, "L3": 5, "L2": 3}, "OP": {"L1": 0.5833333333333334, "L3": 0.5589430894308943, "L2": 0.5192307692307692}, "DOR": {"L1": "None", "L3": 1.9999999999999998, "L2": 4.000000000000001}, "IS": {"L1": 1.2630344058337937, "L3": 0.26303440583379367, "L2": 0.9999999999999998}, "sInd": {"L1": 0.717157287525381, "L3": 0.5854680534700882, "L2": 0.6192113447068046}, "AM": {"L1": -2, "L3": 1, "L2": 1}, "FN": {"L1": 2, "L3": 2, "L2": 1}, "G": {"L1": 0.7745966692414834, "L3": 0.5477225575051661, "L2": 0.408248290463863}, "N": {"L1": 7, "L3": 7, "L2": 10}, "dInd": {"L1": 0.4, "L3": 0.5862367008195198, "L2": 0.5385164807134504}, "IBA": {"L1": 0.36, "L3": 0.35265306122448975, "L2": 0.27999999999999997}, "GM": {"L1": 0.7745966692414834, "L3": 0.5855400437691198, "L2": 0.6324555320336759}, "PPV": {"L1": 1.0, "L3": 0.5, "L2": 0.3333333333333333}, "TON": {"L1": 9, "L3": 6, "L2": 9}, "TNR": {"L1": 1.0, "L3": 0.5714285714285714, "L2": 0.8}, "FPR": {"L1": 0.0, "L3": 0.4285714285714286, "L2": 0.19999999999999996}, "LS": {"L1": 2.4, "L3": 1.2, "L2": 2.0}, "MK": {"L1": 0.7777777777777777, "L3": 0.16666666666666652, "L2": 0.2222222222222221}, "NPV": {"L1": 0.7777777777777778, "L3": 0.6666666666666666, "L2": 0.8888888888888888}, "CEN": {"L1": 0.25, "L3": 0.6044162769630221, "L2": 0.49657842846620864}, "ERR": {"L1": 0.16666666666666663, "L3": 0.41666666666666663, "L2": 0.25}, "RACCU": {"L1": 0.1111111111111111, "L3": 0.21006944444444442, "L2": 0.04340277777777778}, "MCC": {"L1": 0.6831300510639732, "L3": 0.1690308509457033, "L2": 0.25819888974716115}}, "Matrix": [["L1", [["L1", 3], ["L3", 2], ["L2", 0]]], ["L2", [["L1", 0], ["L3", 1], ["L2", 1]]], ["L3", [["L1", 0], ["L3", 3], ["L2", 2]]]], "Actual-Vector": null, "Sample-Weight": null, "Overall-Stat": {"PPV Macro": 0.611111111111111, "Mutual Information": 0.5242078379544428, "Pearson C": 0.5956833971812706, "Kappa": 0.35483870967741943, "ACC Macro": 0.7222222222222223, "ARI": 0.09206349206349207, "Kappa Unbiased": 0.34426229508196726, "PPV Micro": 0.5833333333333334, "CBA": 0.4777777777777778, "Overall RACCU": 0.3645833333333333, "Overall J": [1.225, 0.4083333333333334], "SOA3(Altman)": "Fair", "TPR Micro": 0.5833333333333334, "Overall ACC": 0.5833333333333334, "Cramer V": 0.5244044240850758, "Phi-Squared": 0.55, "Lambda B": 0.16666666666666666, "TPR Macro": 0.5666666666666668, "Kappa 95% CI": [-0.07707577422109269, 0.7867531935759315], "SOA5(Cramer)": "Relatively Strong", "Lambda A": 0.42857142857142855, "FPR Macro": 0.20952380952380956, "SOA2(Fleiss)": "Poor", "AUNP": 0.6857142857142857, "Overall CEN": 0.4638112995385119, "Krippendorff Alpha": 0.3715846994535519, "Chi-Squared DF": 4, "Gwet AC1": 0.3893129770992367, "SOA8(Lambda B)": "Very Weak", "F1 Micro": 0.5833333333333334, "SOA1(Landis & Koch)": "Fair", "FPR Micro": 0.20833333333333337, "P-Value": 0.18926430237560654, "RR": 4.0, "SOA4(Cicchetti)": "Poor", "Overall MCC": 0.36666666666666664, "SOA7(Lambda A)": "Moderate", "F1 Macro": 0.5651515151515151, "FNR Micro": 0.41666666666666663, "Overall RACC": 0.3541666666666667, "Response Entropy": 1.5, "SOA10(Pearson C)": "Strong", "Conditional Entropy": 0.9757921620455572, "Overall MCEN": 0.5189369467580801, "TNR Macro": 0.7904761904761904, "Standard Error": 0.14231876063832777, "SOA6(Matthews)": "Weak", "FNR Macro": 0.43333333333333324, "Kappa No Prevalence": 0.16666666666666674, "RCI": 0.3533932006492363, "Hamming Loss": 0.41666666666666663, "KL Divergence": 0.09997757835164581, "Bennett S": 0.37500000000000006, "Bangdiwala B": 0.37254901960784315, "Chi-Squared": 6.6000000000000005, "95% CI": [0.30438856248221097, 0.8622781041844558], "AUNU": 0.6785714285714285, "NIR": 0.4166666666666667, "Kappa Standard Error": 0.2203645326012817, "Cross Entropy": 1.5833333333333335, "SOA9(Krippendorff Alpha)": "Low", "TNR Micro": 0.7916666666666666, "Joint Entropy": 2.4591479170272446, "Zero-one Loss": 5, "Scott PI": 0.34426229508196726, "Reference Entropy": 1.4833557549816874, "CSI": 0.1777777777777778}, "Digit": 5} \ No newline at end of file diff --git a/Document/Document_Files/cm1_summary.html b/Document/Document_Files/cm1_summary.html index 5cf6c2f4..7aeda609 100644 --- a/Document/Document_Files/cm1_summary.html +++ b/Document/Document_Files/cm1_summary.html @@ -218,6 +218,6 @@

    Class Statistics :

    Test outcome negative -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Document_Files/cp.comp b/Document/Document_Files/cp.comp index 4d0bc5a9..e236be3a 100644 --- a/Document/Document_Files/cp.comp +++ b/Document/Document_Files/cp.comp @@ -1,5 +1,5 @@ Best : cm2 Rank Name Class-Score Overall-Score -1 cm2 0.50278 0.425 -2 cm3 0.33611 0.33056 +1 cm2 0.50278 0.58095 +2 cm3 0.33611 0.52857 diff --git a/Document/Example1.ipynb b/Document/Example1.ipynb index 9e257d33..f35a66a6 100644 --- a/Document/Example1.ipynb +++ b/Document/Example1.ipynb @@ -785,9 +785,9 @@ "Best : Decision tree\n", "\n", "Rank Name Class-Score Overall-Score\n", - "1 Decision tree 0.55556 1.0\n", - "2 AdaBoost 0.48333 0.96667\n", - "3 C-Support vector 0.44444 0.90556\n", + "1 Decision tree 0.55556 0.95238\n", + "2 AdaBoost 0.48333 0.92381\n", + "3 C-Support vector 0.44444 0.80476\n", "\n" ] } diff --git a/Document/Example1_Files/cm1.html b/Document/Example1_Files/cm1.html index b62dc3e6..4dce3d7d 100644 --- a/Document/Example1_Files/cm1.html +++ b/Document/Example1_Files/cm1.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Strong +SOA7(Lambda A) +Strong + + +SOA8(Lambda B) +Strong + + +SOA9(Krippendorff Alpha) +Tentative + + +SOA10(Pearson C) +Strong + + Scott PI 0.76299 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Example1_Files/cm2.html b/Document/Example1_Files/cm2.html index aea011a5..6ccf6d9d 100644 --- a/Document/Example1_Files/cm2.html +++ b/Document/Example1_Files/cm2.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Very Strong +SOA7(Lambda A) +Very Strong + + +SOA8(Lambda B) +Very Strong + + +SOA9(Krippendorff Alpha) +High + + +SOA10(Pearson C) +Strong + + Scott PI 0.95977 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Example1_Files/cm3.html b/Document/Example1_Files/cm3.html index 62e395a8..aa5cc848 100644 --- a/Document/Example1_Files/cm3.html +++ b/Document/Example1_Files/cm3.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Strong +SOA7(Lambda A) +Very Strong + + +SOA8(Lambda B) +Very Strong + + +SOA9(Krippendorff Alpha) +High + + +SOA10(Pearson C) +Strong + + Scott PI 0.83514 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Document/Example1_Files/cp.comp b/Document/Example1_Files/cp.comp index bb75b0e1..76ed309a 100644 --- a/Document/Example1_Files/cp.comp +++ b/Document/Example1_Files/cp.comp @@ -1,6 +1,6 @@ Best : Decision tree Rank Name Class-Score Overall-Score -1 Decision tree 0.55556 1.0 -2 AdaBoost 0.48333 0.96667 -3 C-Support vector 0.44444 0.90556 +1 Decision tree 0.55556 0.95238 +2 AdaBoost 0.48333 0.92381 +3 C-Support vector 0.44444 0.80476 diff --git a/Document/Example3.ipynb b/Document/Example3.ipynb index 3929ee20..e1bba8bc 100644 --- a/Document/Example3.ipynb +++ b/Document/Example3.ipynb @@ -174,6 +174,10 @@ "SOA4(Cicchetti) Fair\n", "SOA5(Cramer) Strong\n", "SOA6(Matthews) Moderate\n", + "SOA7(Lambda A) Moderate\n", + "SOA8(Lambda B) None\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.55556\n", "Standard Error 0.15215\n", "TNR Macro 0.75\n", @@ -386,6 +390,10 @@ "SOA4(Cicchetti) Excellent\n", "SOA5(Cramer) Very Strong\n", "SOA6(Matthews) Strong\n", + "SOA7(Lambda A) Strong\n", + "SOA8(Lambda B) Strong\n", + "SOA9(Krippendorff Alpha) Tentative\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.74468\n", "Standard Error 0.15215\n", "TNR Macro 0.91667\n", diff --git a/Document/Example4.ipynb b/Document/Example4.ipynb index fe4f398f..3580011d 100644 --- a/Document/Example4.ipynb +++ b/Document/Example4.ipynb @@ -166,6 +166,10 @@ "SOA4(Cicchetti) Poor\n", "SOA5(Cramer) None\n", "SOA6(Matthews) Negligible\n", + "SOA7(Lambda A) None\n", + "SOA8(Lambda B) None\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) None\n", "Scott PI -0.12554\n", "Standard Error 0.10665\n", "TNR Macro 0.78529\n", @@ -448,6 +452,10 @@ "SOA4(Cicchetti) Poor\n", "SOA5(Cramer) None\n", "SOA6(Matthews) Negligible\n", + "SOA7(Lambda A) None\n", + "SOA8(Lambda B) None\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) None\n", "Scott PI -0.12554\n", "Standard Error 0.10665\n", "TNR Macro 0.78529\n", @@ -546,7 +554,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"Actual-Vector\": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], \"Digit\": 5, \"Prob-Vector\": null, \"Imbalanced\": true, \"Transpose\": false, \"Sample-Weight\": null, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], \"Predict-Vector\": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]}\n" + "{\"Digit\": 5, \"Predict-Vector\": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200], \"Imbalanced\": true, \"Actual-Vector\": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], \"Sample-Weight\": null, \"Prob-Vector\": null, \"Transpose\": false, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]]}\n" ] } ], @@ -563,7 +571,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"Actual-Vector\": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], \"Digit\": 5, \"Prob-Vector\": null, \"Class-Stat\": {\"FNR\": {\"200\": 0.625, \"500\": 0.6666666666666667, \"100\": \"None\", \"600\": 1.0}, \"HD\": {\"200\": 11, \"500\": 3, \"100\": 11, \"600\": 1}, \"NLR\": {\"200\": 0.8333333333333334, \"500\": 0.7083333333333334, \"100\": \"None\", \"600\": 1.0}, \"AUPR\": {\"200\": 0.6160714285714286, \"500\": 0.41666666666666663, \"100\": \"None\", \"600\": \"None\"}, \"Y\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"MK\": {\"200\": 0.08791208791208782, \"500\": 0.38888888888888884, \"100\": 0.0, \"600\": \"None\"}, \"PLRI\": {\"200\": \"Poor\", \"500\": \"Fair\", \"100\": \"None\", \"600\": \"None\"}, \"F1\": {\"200\": 0.5217391304347826, \"500\": 0.4, \"100\": 0.0, \"600\": 0.0}, \"BCD\": {\"200\": 0.225, \"500\": 0.025, \"100\": 0.275, \"600\": 0.025}, \"IS\": {\"200\": 0.09953567355091428, \"500\": 1.736965594166206, \"100\": \"None\", \"600\": \"None\"}, \"GI\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"N\": {\"200\": 4, \"500\": 17, \"100\": 20, \"600\": 19}, \"MCCI\": {\"200\": \"Negligible\", \"500\": \"Weak\", \"100\": \"None\", \"600\": \"None\"}, \"DPI\": {\"200\": \"Poor\", \"500\": \"Poor\", \"100\": \"None\", \"600\": \"None\"}, \"ACC\": {\"200\": 0.45, \"500\": 0.85, \"100\": 0.45, \"600\": 0.95}, \"FP\": {\"200\": 1, \"100\": 11, \"500\": 1, \"600\": 0}, \"OP\": {\"200\": 0.1166666666666667, \"500\": 0.373076923076923, \"100\": \"None\", \"600\": -0.050000000000000044}, \"OOC\": {\"200\": 0.5669467095138409, \"500\": 0.4082482904638631, \"100\": \"None\", \"600\": \"None\"}, \"TOP\": {\"200\": 7, \"500\": 2, \"100\": 11, \"600\": 0}, \"TP\": {\"200\": 6, \"100\": 0, \"500\": 1, \"600\": 0}, \"RACC\": {\"200\": 0.28, \"500\": 0.015, \"100\": 0.0, \"600\": 0.0}, \"POP\": {\"200\": 20, \"500\": 20, \"100\": 20, \"600\": 20}, \"OC\": {\"200\": 0.8571428571428571, \"500\": 0.5, \"100\": \"None\", \"600\": \"None\"}, \"GM\": {\"200\": 0.5303300858899106, \"500\": 0.5601120336112039, \"100\": \"None\", \"600\": 0.0}, \"FN\": {\"200\": 10, \"100\": 0, \"500\": 2, \"600\": 1}, \"J\": {\"200\": 0.35294117647058826, \"500\": 0.25, \"100\": 0.0, \"600\": 0.0}, \"NPV\": {\"200\": 0.23076923076923078, \"500\": 0.8888888888888888, \"100\": 1.0, \"600\": 0.95}, \"Q\": {\"200\": 0.28571428571428575, \"500\": 0.7777777777777778, \"100\": \"None\", \"600\": \"None\"}, \"MCC\": {\"200\": 0.10482848367219183, \"500\": 0.32673201960653564, \"100\": \"None\", \"600\": \"None\"}, \"IBA\": {\"200\": 0.17578125, \"500\": 0.1230296039984621, \"100\": \"None\", \"600\": 0.0}, \"FDR\": {\"200\": 0.1428571428571429, \"500\": 0.5, \"100\": 1.0, \"600\": \"None\"}, \"FPR\": {\"200\": 0.25, \"500\": 0.05882352941176472, \"100\": 0.55, \"600\": 0.0}, \"P\": {\"200\": 16, \"500\": 3, \"100\": 0, \"600\": 1}, \"PRE\": {\"200\": 0.8, \"500\": 0.15, \"100\": 0.0, \"600\": 0.05}, \"PLR\": {\"200\": 1.5, \"500\": 5.666666666666665, \"100\": \"None\", \"600\": \"None\"}, \"TPR\": {\"200\": 0.375, \"500\": 0.3333333333333333, \"100\": \"None\", \"600\": 0.0}, \"CEN\": {\"200\": 0.3570795472009597, \"500\": 0.5389466410223563, \"100\": 0.3349590631259315, \"600\": 0.0}, \"PPV\": {\"200\": 0.8571428571428571, \"500\": 0.5, \"100\": 0.0, \"600\": \"None\"}, \"NLRI\": {\"200\": \"Negligible\", \"500\": \"Negligible\", \"100\": \"None\", \"600\": \"Negligible\"}, \"ERR\": {\"200\": 0.55, \"500\": 0.15000000000000002, \"100\": 0.55, \"600\": 0.050000000000000044}, \"AUCI\": {\"200\": \"Poor\", \"500\": \"Fair\", \"100\": \"None\", \"600\": \"Poor\"}, \"FOR\": {\"200\": 0.7692307692307692, \"500\": 0.11111111111111116, \"100\": 0.0, \"600\": 0.050000000000000044}, \"BB\": {\"200\": 0.375, \"500\": 0.3333333333333333, \"100\": 0.0, \"600\": 0.0}, \"TON\": {\"200\": 13, \"500\": 18, \"100\": 9, \"600\": 20}, \"DOR\": {\"200\": 1.7999999999999998, \"500\": 7.999999999999997, \"100\": \"None\", \"600\": \"None\"}, \"DP\": {\"200\": 0.1407391082701595, \"500\": 0.49789960499474867, \"100\": \"None\", \"600\": \"None\"}, \"dInd\": {\"200\": 0.673145600891813, \"500\": 0.6692567908186672, \"100\": \"None\", \"600\": 1.0}, \"F2\": {\"200\": 0.4225352112676056, \"500\": 0.35714285714285715, \"100\": 0.0, \"600\": 0.0}, \"AUC\": {\"200\": 0.5625, \"500\": 0.6372549019607843, \"100\": \"None\", \"600\": 0.5}, \"sInd\": {\"200\": 0.5240141808835057, \"500\": 0.5267639848569737, \"100\": \"None\", \"600\": 0.29289321881345254}, \"TN\": {\"200\": 3, \"100\": 9, \"500\": 16, \"600\": 19}, \"BM\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"RACCU\": {\"200\": 0.33062499999999995, \"500\": 0.015625, \"100\": 0.07562500000000001, \"600\": 0.0006250000000000001}, \"MCEN\": {\"200\": 0.3739448088748241, \"500\": 0.5802792108518123, \"100\": 0.3349590631259315, \"600\": 0.0}, \"LS\": {\"200\": 1.0714285714285714, \"500\": 3.3333333333333335, \"100\": \"None\", \"600\": \"None\"}, \"ICSI\": {\"200\": 0.2321428571428572, \"500\": -0.16666666666666674, \"100\": \"None\", \"600\": \"None\"}, \"AM\": {\"200\": -9, \"500\": -1, \"100\": 11, \"600\": -1}, \"F0.5\": {\"200\": 0.6818181818181818, \"500\": 0.45454545454545453, \"100\": 0.0, \"600\": 0.0}, \"G\": {\"200\": 0.5669467095138409, \"500\": 0.408248290463863, \"100\": \"None\", \"600\": \"None\"}, \"AGM\": {\"200\": 0.5669417382415922, \"500\": 0.7351956938438939, \"100\": \"None\", \"600\": 0}, \"AGF\": {\"200\": 0.33642097801219245, \"500\": 0.5665926996700735, \"100\": 0.0, \"600\": 0.0}, \"TNR\": {\"200\": 0.75, \"500\": 0.9411764705882353, \"100\": 0.45, \"600\": 1.0}, \"QI\": {\"200\": \"Weak\", \"500\": \"Strong\", \"100\": \"None\", \"600\": \"None\"}}, \"Imbalanced\": true, \"Transpose\": false, \"Sample-Weight\": null, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], \"Overall-Stat\": {\"Scott PI\": -0.12554112554112543, \"Bennett S\": 0.1333333333333333, \"Chi-Squared\": \"None\", \"Overall ACC\": 0.35, \"Overall MCC\": 0.1264200803632855, \"Lambda A\": 0.0, \"TPR Macro\": \"None\", \"Kappa 95% CI\": [-0.21849807698648957, 0.3745264457808156], \"Mutual Information\": 0.10087710767390168, \"Pearson C\": \"None\", \"Bangdiwala B\": 0.3135593220338983, \"Standard Error\": 0.1066536450385077, \"FPR Macro\": 0.2147058823529412, \"RR\": 5.0, \"SOA5(Cramer)\": \"None\", \"F1 Macro\": 0.23043478260869565, \"Kappa Standard Error\": 0.15128176601206766, \"Cramer V\": \"None\", \"Joint Entropy\": 2.119973094021975, \"Zero-one Loss\": 13, \"FPR Micro\": 0.21666666666666667, \"Chi-Squared DF\": 9, \"TNR Macro\": 0.7852941176470588, \"FNR Micro\": 0.65, \"Krippendorff Alpha\": -0.09740259740259723, \"Kappa\": 0.07801418439716304, \"Phi-Squared\": \"None\", \"Cross Entropy\": 1.709947752496911, \"TNR Micro\": 0.7833333333333333, \"F1 Micro\": 0.35, \"KL Divergence\": \"None\", \"CBA\": 0.17708333333333331, \"Hamming Loss\": 0.65, \"RCI\": 0.11409066398451011, \"Overall CEN\": 0.3648028121279775, \"ACC Macro\": 0.675, \"SOA3(Altman)\": \"Poor\", \"AUNP\": \"None\", \"CSI\": \"None\", \"Overall J\": [0.6029411764705883, 0.15073529411764708], \"Reference Entropy\": 0.8841837197791889, \"Overall RACCU\": 0.42249999999999993, \"Overall RACC\": 0.29500000000000004, \"Conditional Entropy\": 1.235789374242786, \"Kappa No Prevalence\": -0.30000000000000004, \"PPV Macro\": \"None\", \"NIR\": 0.8, \"FNR Macro\": \"None\", \"ARI\": 0.02298247455136956, \"Kappa Unbiased\": -0.12554112554112543, \"SOA4(Cicchetti)\": \"Poor\", \"TPR Micro\": 0.35, \"Response Entropy\": 1.3366664819166876, \"Lambda B\": 0.0, \"SOA2(Fleiss)\": \"Poor\", \"Gwet AC1\": 0.19504643962848295, \"SOA1(Landis & Koch)\": \"Slight\", \"Overall MCEN\": 0.3746281299595305, \"AUNU\": \"None\", \"P-Value\": 0.9999981549942787, \"SOA6(Matthews)\": \"Negligible\", \"PPV Micro\": 0.35, \"95% CI\": [0.14095885572452488, 0.559041144275475]}, \"Predict-Vector\": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]}\n" + "{\"Digit\": 5, \"Class-Stat\": {\"MK\": {\"200\": 0.08791208791208782, \"500\": 0.38888888888888884, \"100\": 0.0, \"600\": \"None\"}, \"TPR\": {\"200\": 0.375, \"500\": 0.3333333333333333, \"100\": \"None\", \"600\": 0.0}, \"TN\": {\"200\": 3, \"100\": 9, \"500\": 16, \"600\": 19}, \"TON\": {\"200\": 13, \"500\": 18, \"100\": 9, \"600\": 20}, \"NLR\": {\"200\": 0.8333333333333334, \"500\": 0.7083333333333334, \"100\": \"None\", \"600\": 1.0}, \"POP\": {\"200\": 20, \"500\": 20, \"100\": 20, \"600\": 20}, \"FOR\": {\"200\": 0.7692307692307692, \"500\": 0.11111111111111116, \"100\": 0.0, \"600\": 0.050000000000000044}, \"FPR\": {\"200\": 0.25, \"500\": 0.05882352941176472, \"100\": 0.55, \"600\": 0.0}, \"OOC\": {\"200\": 0.5669467095138409, \"500\": 0.4082482904638631, \"100\": \"None\", \"600\": \"None\"}, \"N\": {\"200\": 4, \"500\": 17, \"100\": 20, \"600\": 19}, \"TNR\": {\"200\": 0.75, \"500\": 0.9411764705882353, \"100\": 0.45, \"600\": 1.0}, \"PLRI\": {\"200\": \"Poor\", \"500\": \"Fair\", \"100\": \"None\", \"600\": \"None\"}, \"DOR\": {\"200\": 1.7999999999999998, \"500\": 7.999999999999997, \"100\": \"None\", \"600\": \"None\"}, \"F1\": {\"200\": 0.5217391304347826, \"500\": 0.4, \"100\": 0.0, \"600\": 0.0}, \"ACC\": {\"200\": 0.45, \"500\": 0.85, \"100\": 0.45, \"600\": 0.95}, \"LS\": {\"200\": 1.0714285714285714, \"500\": 3.3333333333333335, \"100\": \"None\", \"600\": \"None\"}, \"BCD\": {\"200\": 0.225, \"500\": 0.025, \"100\": 0.275, \"600\": 0.025}, \"TP\": {\"200\": 6, \"100\": 0, \"500\": 1, \"600\": 0}, \"PRE\": {\"200\": 0.8, \"500\": 0.15, \"100\": 0.0, \"600\": 0.05}, \"MCEN\": {\"200\": 0.3739448088748241, \"500\": 0.5802792108518123, \"100\": 0.3349590631259315, \"600\": 0.0}, \"AUCI\": {\"200\": \"Poor\", \"500\": \"Fair\", \"100\": \"None\", \"600\": \"Poor\"}, \"ERR\": {\"200\": 0.55, \"500\": 0.15000000000000002, \"100\": 0.55, \"600\": 0.050000000000000044}, \"AGM\": {\"200\": 0.5669417382415922, \"500\": 0.7351956938438939, \"100\": \"None\", \"600\": 0}, \"MCCI\": {\"200\": \"Negligible\", \"500\": \"Weak\", \"100\": \"None\", \"600\": \"None\"}, \"DPI\": {\"200\": \"Poor\", \"500\": \"Poor\", \"100\": \"None\", \"600\": \"None\"}, \"Q\": {\"200\": 0.28571428571428575, \"500\": 0.7777777777777778, \"100\": \"None\", \"600\": \"None\"}, \"RACCU\": {\"200\": 0.33062499999999995, \"500\": 0.015625, \"100\": 0.07562500000000001, \"600\": 0.0006250000000000001}, \"FN\": {\"200\": 10, \"100\": 0, \"500\": 2, \"600\": 1}, \"GI\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"AUPR\": {\"200\": 0.6160714285714286, \"500\": 0.41666666666666663, \"100\": \"None\", \"600\": \"None\"}, \"F2\": {\"200\": 0.4225352112676056, \"500\": 0.35714285714285715, \"100\": 0.0, \"600\": 0.0}, \"QI\": {\"200\": \"Weak\", \"500\": \"Strong\", \"100\": \"None\", \"600\": \"None\"}, \"AM\": {\"200\": -9, \"500\": -1, \"100\": 11, \"600\": -1}, \"AUC\": {\"200\": 0.5625, \"500\": 0.6372549019607843, \"100\": \"None\", \"600\": 0.5}, \"G\": {\"200\": 0.5669467095138409, \"500\": 0.408248290463863, \"100\": \"None\", \"600\": \"None\"}, \"dInd\": {\"200\": 0.673145600891813, \"500\": 0.6692567908186672, \"100\": \"None\", \"600\": 1.0}, \"ICSI\": {\"200\": 0.2321428571428572, \"500\": -0.16666666666666674, \"100\": \"None\", \"600\": \"None\"}, \"J\": {\"200\": 0.35294117647058826, \"500\": 0.25, \"100\": 0.0, \"600\": 0.0}, \"F0.5\": {\"200\": 0.6818181818181818, \"500\": 0.45454545454545453, \"100\": 0.0, \"600\": 0.0}, \"IS\": {\"200\": 0.09953567355091428, \"500\": 1.736965594166206, \"100\": \"None\", \"600\": \"None\"}, \"IBA\": {\"200\": 0.17578125, \"500\": 0.1230296039984621, \"100\": \"None\", \"600\": 0.0}, \"RACC\": {\"200\": 0.28, \"500\": 0.015, \"100\": 0.0, \"600\": 0.0}, \"FDR\": {\"200\": 0.1428571428571429, \"500\": 0.5, \"100\": 1.0, \"600\": \"None\"}, \"BB\": {\"200\": 0.375, \"500\": 0.3333333333333333, \"100\": 0.0, \"600\": 0.0}, \"PLR\": {\"200\": 1.5, \"500\": 5.666666666666665, \"100\": \"None\", \"600\": \"None\"}, \"MCC\": {\"200\": 0.10482848367219183, \"500\": 0.32673201960653564, \"100\": \"None\", \"600\": \"None\"}, \"AGF\": {\"200\": 0.33642097801219245, \"500\": 0.5665926996700735, \"100\": 0.0, \"600\": 0.0}, \"HD\": {\"200\": 11, \"500\": 3, \"100\": 11, \"600\": 1}, \"FP\": {\"200\": 1, \"100\": 11, \"500\": 1, \"600\": 0}, \"GM\": {\"200\": 0.5303300858899106, \"500\": 0.5601120336112039, \"100\": \"None\", \"600\": 0.0}, \"CEN\": {\"200\": 0.3570795472009597, \"500\": 0.5389466410223563, \"100\": 0.3349590631259315, \"600\": 0.0}, \"P\": {\"200\": 16, \"500\": 3, \"100\": 0, \"600\": 1}, \"NPV\": {\"200\": 0.23076923076923078, \"500\": 0.8888888888888888, \"100\": 1.0, \"600\": 0.95}, \"NLRI\": {\"200\": \"Negligible\", \"500\": \"Negligible\", \"100\": \"None\", \"600\": \"Negligible\"}, \"BM\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"OC\": {\"200\": 0.8571428571428571, \"500\": 0.5, \"100\": \"None\", \"600\": \"None\"}, \"PPV\": {\"200\": 0.8571428571428571, \"500\": 0.5, \"100\": 0.0, \"600\": \"None\"}, \"TOP\": {\"200\": 7, \"500\": 2, \"100\": 11, \"600\": 0}, \"FNR\": {\"200\": 0.625, \"500\": 0.6666666666666667, \"100\": \"None\", \"600\": 1.0}, \"DP\": {\"200\": 0.1407391082701595, \"500\": 0.49789960499474867, \"100\": \"None\", \"600\": \"None\"}, \"sInd\": {\"200\": 0.5240141808835057, \"500\": 0.5267639848569737, \"100\": \"None\", \"600\": 0.29289321881345254}, \"Y\": {\"200\": 0.125, \"500\": 0.27450980392156854, \"100\": \"None\", \"600\": 0.0}, \"OP\": {\"200\": 0.1166666666666667, \"500\": 0.373076923076923, \"100\": \"None\", \"600\": -0.050000000000000044}}, \"Predict-Vector\": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200], \"Imbalanced\": true, \"Actual-Vector\": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], \"Sample-Weight\": null, \"Prob-Vector\": null, \"Transpose\": false, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], \"Overall-Stat\": {\"Phi-Squared\": \"None\", \"Overall ACC\": 0.35, \"Conditional Entropy\": 1.235789374242786, \"ARI\": 0.02298247455136956, \"Kappa\": 0.07801418439716304, \"F1 Micro\": 0.35, \"Kappa No Prevalence\": -0.30000000000000004, \"SOA5(Cramer)\": \"None\", \"FPR Micro\": 0.21666666666666667, \"Gwet AC1\": 0.19504643962848295, \"Chi-Squared\": \"None\", \"Bangdiwala B\": 0.3135593220338983, \"FNR Macro\": \"None\", \"FPR Macro\": 0.2147058823529412, \"Joint Entropy\": 2.119973094021975, \"NIR\": 0.8, \"SOA3(Altman)\": \"Poor\", \"RR\": 5.0, \"ACC Macro\": 0.675, \"CBA\": 0.17708333333333331, \"Krippendorff Alpha\": -0.09740259740259723, \"SOA9(Krippendorff Alpha)\": \"Low\", \"Kappa Unbiased\": -0.12554112554112543, \"F1 Macro\": 0.23043478260869565, \"Zero-one Loss\": 13, \"KL Divergence\": \"None\", \"Kappa 95% CI\": [-0.21849807698648957, 0.3745264457808156], \"Bennett S\": 0.1333333333333333, \"Response Entropy\": 1.3366664819166876, \"CSI\": \"None\", \"Overall MCEN\": 0.3746281299595305, \"SOA10(Pearson C)\": \"None\", \"Cramer V\": \"None\", \"SOA6(Matthews)\": \"Negligible\", \"Standard Error\": 0.1066536450385077, \"SOA7(Lambda A)\": \"None\", \"95% CI\": [0.14095885572452488, 0.559041144275475], \"AUNP\": \"None\", \"PPV Micro\": 0.35, \"SOA4(Cicchetti)\": \"Poor\", \"SOA1(Landis & Koch)\": \"Slight\", \"Scott PI\": -0.12554112554112543, \"Chi-Squared DF\": 9, \"Hamming Loss\": 0.65, \"Kappa Standard Error\": 0.15128176601206766, \"TPR Macro\": \"None\", \"Lambda A\": 0.0, \"SOA8(Lambda B)\": \"None\", \"Overall CEN\": 0.3648028121279775, \"Overall RACC\": 0.29500000000000004, \"SOA2(Fleiss)\": \"Poor\", \"Cross Entropy\": 1.709947752496911, \"PPV Macro\": \"None\", \"Overall MCC\": 0.1264200803632855, \"Mutual Information\": 0.10087710767390168, \"TNR Micro\": 0.7833333333333333, \"TNR Macro\": 0.7852941176470588, \"Overall RACCU\": 0.42249999999999993, \"Lambda B\": 0.0, \"FNR Micro\": 0.65, \"P-Value\": 0.9999981549942787, \"RCI\": 0.11409066398451011, \"Pearson C\": \"None\", \"Reference Entropy\": 0.8841837197791889, \"AUNU\": \"None\", \"TPR Micro\": 0.35, \"Overall J\": [0.6029411764705883, 0.15073529411764708]}}\n" ] } ], @@ -580,7 +588,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{\"Actual-Vector\": null, \"Digit\": 5, \"Prob-Vector\": null, \"Imbalanced\": true, \"Transpose\": false, \"Sample-Weight\": null, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], \"Predict-Vector\": null}\n" + "{\"Digit\": 5, \"Predict-Vector\": null, \"Imbalanced\": true, \"Actual-Vector\": null, \"Sample-Weight\": null, \"Prob-Vector\": null, \"Transpose\": false, \"Matrix\": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]]}\n" ] } ], diff --git a/Document/Example4_Files/cm.obj b/Document/Example4_Files/cm.obj index 0f4bdf1b..442c32d6 100644 --- a/Document/Example4_Files/cm.obj +++ b/Document/Example4_Files/cm.obj @@ -1 +1 @@ -{"Actual-Vector": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], "Digit": 5, "Prob-Vector": null, "Imbalanced": true, "Transpose": false, "Sample-Weight": null, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], "Predict-Vector": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]} \ No newline at end of file +{"Digit": 5, "Predict-Vector": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200], "Imbalanced": true, "Actual-Vector": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], "Sample-Weight": null, "Prob-Vector": null, "Transpose": false, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]]} \ No newline at end of file diff --git a/Document/Example4_Files/cm_no_vectors.obj b/Document/Example4_Files/cm_no_vectors.obj index 1d33e773..df37bfa8 100644 --- a/Document/Example4_Files/cm_no_vectors.obj +++ b/Document/Example4_Files/cm_no_vectors.obj @@ -1 +1 @@ -{"Actual-Vector": null, "Digit": 5, "Prob-Vector": null, "Imbalanced": true, "Transpose": false, "Sample-Weight": null, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], "Predict-Vector": null} \ No newline at end of file +{"Digit": 5, "Predict-Vector": null, "Imbalanced": true, "Actual-Vector": null, "Sample-Weight": null, "Prob-Vector": null, "Transpose": false, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]]} \ No newline at end of file diff --git a/Document/Example4_Files/cm_stat.obj b/Document/Example4_Files/cm_stat.obj index 66d9575a..2e58fb1d 100644 --- a/Document/Example4_Files/cm_stat.obj +++ b/Document/Example4_Files/cm_stat.obj @@ -1 +1 @@ -{"Actual-Vector": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], "Digit": 5, "Prob-Vector": null, "Class-Stat": {"FNR": {"200": 0.625, "500": 0.6666666666666667, "100": "None", "600": 1.0}, "HD": {"200": 11, "500": 3, "100": 11, "600": 1}, "NLR": {"200": 0.8333333333333334, "500": 0.7083333333333334, "100": "None", "600": 1.0}, "AUPR": {"200": 0.6160714285714286, "500": 0.41666666666666663, "100": "None", "600": "None"}, "Y": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "MK": {"200": 0.08791208791208782, "500": 0.38888888888888884, "100": 0.0, "600": "None"}, "PLRI": {"200": "Poor", "500": "Fair", "100": "None", "600": "None"}, "F1": {"200": 0.5217391304347826, "500": 0.4, "100": 0.0, "600": 0.0}, "BCD": {"200": 0.225, "500": 0.025, "100": 0.275, "600": 0.025}, "IS": {"200": 0.09953567355091428, "500": 1.736965594166206, "100": "None", "600": "None"}, "GI": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "N": {"200": 4, "500": 17, "100": 20, "600": 19}, "MCCI": {"200": "Negligible", "500": "Weak", "100": "None", "600": "None"}, "DPI": {"200": "Poor", "500": "Poor", "100": "None", "600": "None"}, "ACC": {"200": 0.45, "500": 0.85, "100": 0.45, "600": 0.95}, "FP": {"200": 1, "100": 11, "500": 1, "600": 0}, "OP": {"200": 0.1166666666666667, "500": 0.373076923076923, "100": "None", "600": -0.050000000000000044}, "OOC": {"200": 0.5669467095138409, "500": 0.4082482904638631, "100": "None", "600": "None"}, "TOP": {"200": 7, "500": 2, "100": 11, "600": 0}, "TP": {"200": 6, "100": 0, "500": 1, "600": 0}, "RACC": {"200": 0.28, "500": 0.015, "100": 0.0, "600": 0.0}, "POP": {"200": 20, "500": 20, "100": 20, "600": 20}, "OC": {"200": 0.8571428571428571, "500": 0.5, "100": "None", "600": "None"}, "GM": {"200": 0.5303300858899106, "500": 0.5601120336112039, "100": "None", "600": 0.0}, "FN": {"200": 10, "100": 0, "500": 2, "600": 1}, "J": {"200": 0.35294117647058826, "500": 0.25, "100": 0.0, "600": 0.0}, "NPV": {"200": 0.23076923076923078, "500": 0.8888888888888888, "100": 1.0, "600": 0.95}, "Q": {"200": 0.28571428571428575, "500": 0.7777777777777778, "100": "None", "600": "None"}, "MCC": {"200": 0.10482848367219183, "500": 0.32673201960653564, "100": "None", "600": "None"}, "IBA": {"200": 0.17578125, "500": 0.1230296039984621, "100": "None", "600": 0.0}, "FDR": {"200": 0.1428571428571429, "500": 0.5, "100": 1.0, "600": "None"}, "FPR": {"200": 0.25, "500": 0.05882352941176472, "100": 0.55, "600": 0.0}, "P": {"200": 16, "500": 3, "100": 0, "600": 1}, "PRE": {"200": 0.8, "500": 0.15, "100": 0.0, "600": 0.05}, "PLR": {"200": 1.5, "500": 5.666666666666665, "100": "None", "600": "None"}, "TPR": {"200": 0.375, "500": 0.3333333333333333, "100": "None", "600": 0.0}, "CEN": {"200": 0.3570795472009597, "500": 0.5389466410223563, "100": 0.3349590631259315, "600": 0.0}, "PPV": {"200": 0.8571428571428571, "500": 0.5, "100": 0.0, "600": "None"}, "NLRI": {"200": "Negligible", "500": "Negligible", "100": "None", "600": "Negligible"}, "ERR": {"200": 0.55, "500": 0.15000000000000002, "100": 0.55, "600": 0.050000000000000044}, "AUCI": {"200": "Poor", "500": "Fair", "100": "None", "600": "Poor"}, "FOR": {"200": 0.7692307692307692, "500": 0.11111111111111116, "100": 0.0, "600": 0.050000000000000044}, "BB": {"200": 0.375, "500": 0.3333333333333333, "100": 0.0, "600": 0.0}, "TON": {"200": 13, "500": 18, "100": 9, "600": 20}, "DOR": {"200": 1.7999999999999998, "500": 7.999999999999997, "100": "None", "600": "None"}, "DP": {"200": 0.1407391082701595, "500": 0.49789960499474867, "100": "None", "600": "None"}, "dInd": {"200": 0.673145600891813, "500": 0.6692567908186672, "100": "None", "600": 1.0}, "F2": {"200": 0.4225352112676056, "500": 0.35714285714285715, "100": 0.0, "600": 0.0}, "AUC": {"200": 0.5625, "500": 0.6372549019607843, "100": "None", "600": 0.5}, "sInd": {"200": 0.5240141808835057, "500": 0.5267639848569737, "100": "None", "600": 0.29289321881345254}, "TN": {"200": 3, "100": 9, "500": 16, "600": 19}, "BM": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "RACCU": {"200": 0.33062499999999995, "500": 0.015625, "100": 0.07562500000000001, "600": 0.0006250000000000001}, "MCEN": {"200": 0.3739448088748241, "500": 0.5802792108518123, "100": 0.3349590631259315, "600": 0.0}, "LS": {"200": 1.0714285714285714, "500": 3.3333333333333335, "100": "None", "600": "None"}, "ICSI": {"200": 0.2321428571428572, "500": -0.16666666666666674, "100": "None", "600": "None"}, "AM": {"200": -9, "500": -1, "100": 11, "600": -1}, "F0.5": {"200": 0.6818181818181818, "500": 0.45454545454545453, "100": 0.0, "600": 0.0}, "G": {"200": 0.5669467095138409, "500": 0.408248290463863, "100": "None", "600": "None"}, "AGM": {"200": 0.5669417382415922, "500": 0.7351956938438939, "100": "None", "600": 0}, "AGF": {"200": 0.33642097801219245, "500": 0.5665926996700735, "100": 0.0, "600": 0.0}, "TNR": {"200": 0.75, "500": 0.9411764705882353, "100": 0.45, "600": 1.0}, "QI": {"200": "Weak", "500": "Strong", "100": "None", "600": "None"}}, "Imbalanced": true, "Transpose": false, "Sample-Weight": null, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], "Overall-Stat": {"Scott PI": -0.12554112554112543, "Bennett S": 0.1333333333333333, "Chi-Squared": "None", "Overall ACC": 0.35, "Overall MCC": 0.1264200803632855, "Lambda A": 0.0, "TPR Macro": "None", "Kappa 95% CI": [-0.21849807698648957, 0.3745264457808156], "Mutual Information": 0.10087710767390168, "Pearson C": "None", "Bangdiwala B": 0.3135593220338983, "Standard Error": 0.1066536450385077, "FPR Macro": 0.2147058823529412, "RR": 5.0, "SOA5(Cramer)": "None", "F1 Macro": 0.23043478260869565, "Kappa Standard Error": 0.15128176601206766, "Cramer V": "None", "Joint Entropy": 2.119973094021975, "Zero-one Loss": 13, "FPR Micro": 0.21666666666666667, "Chi-Squared DF": 9, "TNR Macro": 0.7852941176470588, "FNR Micro": 0.65, "Krippendorff Alpha": -0.09740259740259723, "Kappa": 0.07801418439716304, "Phi-Squared": "None", "Cross Entropy": 1.709947752496911, "TNR Micro": 0.7833333333333333, "F1 Micro": 0.35, "KL Divergence": "None", "CBA": 0.17708333333333331, "Hamming Loss": 0.65, "RCI": 0.11409066398451011, "Overall CEN": 0.3648028121279775, "ACC Macro": 0.675, "SOA3(Altman)": "Poor", "AUNP": "None", "CSI": "None", "Overall J": [0.6029411764705883, 0.15073529411764708], "Reference Entropy": 0.8841837197791889, "Overall RACCU": 0.42249999999999993, "Overall RACC": 0.29500000000000004, "Conditional Entropy": 1.235789374242786, "Kappa No Prevalence": -0.30000000000000004, "PPV Macro": "None", "NIR": 0.8, "FNR Macro": "None", "ARI": 0.02298247455136956, "Kappa Unbiased": -0.12554112554112543, "SOA4(Cicchetti)": "Poor", "TPR Micro": 0.35, "Response Entropy": 1.3366664819166876, "Lambda B": 0.0, "SOA2(Fleiss)": "Poor", "Gwet AC1": 0.19504643962848295, "SOA1(Landis & Koch)": "Slight", "Overall MCEN": 0.3746281299595305, "AUNU": "None", "P-Value": 0.9999981549942787, "SOA6(Matthews)": "Negligible", "PPV Micro": 0.35, "95% CI": [0.14095885572452488, 0.559041144275475]}, "Predict-Vector": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]} \ No newline at end of file +{"Digit": 5, "Class-Stat": {"MK": {"200": 0.08791208791208782, "500": 0.38888888888888884, "100": 0.0, "600": "None"}, "TPR": {"200": 0.375, "500": 0.3333333333333333, "100": "None", "600": 0.0}, "TN": {"200": 3, "100": 9, "500": 16, "600": 19}, "TON": {"200": 13, "500": 18, "100": 9, "600": 20}, "NLR": {"200": 0.8333333333333334, "500": 0.7083333333333334, "100": "None", "600": 1.0}, "POP": {"200": 20, "500": 20, "100": 20, "600": 20}, "FOR": {"200": 0.7692307692307692, "500": 0.11111111111111116, "100": 0.0, "600": 0.050000000000000044}, "FPR": {"200": 0.25, "500": 0.05882352941176472, "100": 0.55, "600": 0.0}, "OOC": {"200": 0.5669467095138409, "500": 0.4082482904638631, "100": "None", "600": "None"}, "N": {"200": 4, "500": 17, "100": 20, "600": 19}, "TNR": {"200": 0.75, "500": 0.9411764705882353, "100": 0.45, "600": 1.0}, "PLRI": {"200": "Poor", "500": "Fair", "100": "None", "600": "None"}, "DOR": {"200": 1.7999999999999998, "500": 7.999999999999997, "100": "None", "600": "None"}, "F1": {"200": 0.5217391304347826, "500": 0.4, "100": 0.0, "600": 0.0}, "ACC": {"200": 0.45, "500": 0.85, "100": 0.45, "600": 0.95}, "LS": {"200": 1.0714285714285714, "500": 3.3333333333333335, "100": "None", "600": "None"}, "BCD": {"200": 0.225, "500": 0.025, "100": 0.275, "600": 0.025}, "TP": {"200": 6, "100": 0, "500": 1, "600": 0}, "PRE": {"200": 0.8, "500": 0.15, "100": 0.0, "600": 0.05}, "MCEN": {"200": 0.3739448088748241, "500": 0.5802792108518123, "100": 0.3349590631259315, "600": 0.0}, "AUCI": {"200": "Poor", "500": "Fair", "100": "None", "600": "Poor"}, "ERR": {"200": 0.55, "500": 0.15000000000000002, "100": 0.55, "600": 0.050000000000000044}, "AGM": {"200": 0.5669417382415922, "500": 0.7351956938438939, "100": "None", "600": 0}, "MCCI": {"200": "Negligible", "500": "Weak", "100": "None", "600": "None"}, "DPI": {"200": "Poor", "500": "Poor", "100": "None", "600": "None"}, "Q": {"200": 0.28571428571428575, "500": 0.7777777777777778, "100": "None", "600": "None"}, "RACCU": {"200": 0.33062499999999995, "500": 0.015625, "100": 0.07562500000000001, "600": 0.0006250000000000001}, "FN": {"200": 10, "100": 0, "500": 2, "600": 1}, "GI": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "AUPR": {"200": 0.6160714285714286, "500": 0.41666666666666663, "100": "None", "600": "None"}, "F2": {"200": 0.4225352112676056, "500": 0.35714285714285715, "100": 0.0, "600": 0.0}, "QI": {"200": "Weak", "500": "Strong", "100": "None", "600": "None"}, "AM": {"200": -9, "500": -1, "100": 11, "600": -1}, "AUC": {"200": 0.5625, "500": 0.6372549019607843, "100": "None", "600": 0.5}, "G": {"200": 0.5669467095138409, "500": 0.408248290463863, "100": "None", "600": "None"}, "dInd": {"200": 0.673145600891813, "500": 0.6692567908186672, "100": "None", "600": 1.0}, "ICSI": {"200": 0.2321428571428572, "500": -0.16666666666666674, "100": "None", "600": "None"}, "J": {"200": 0.35294117647058826, "500": 0.25, "100": 0.0, "600": 0.0}, "F0.5": {"200": 0.6818181818181818, "500": 0.45454545454545453, "100": 0.0, "600": 0.0}, "IS": {"200": 0.09953567355091428, "500": 1.736965594166206, "100": "None", "600": "None"}, "IBA": {"200": 0.17578125, "500": 0.1230296039984621, "100": "None", "600": 0.0}, "RACC": {"200": 0.28, "500": 0.015, "100": 0.0, "600": 0.0}, "FDR": {"200": 0.1428571428571429, "500": 0.5, "100": 1.0, "600": "None"}, "BB": {"200": 0.375, "500": 0.3333333333333333, "100": 0.0, "600": 0.0}, "PLR": {"200": 1.5, "500": 5.666666666666665, "100": "None", "600": "None"}, "MCC": {"200": 0.10482848367219183, "500": 0.32673201960653564, "100": "None", "600": "None"}, "AGF": {"200": 0.33642097801219245, "500": 0.5665926996700735, "100": 0.0, "600": 0.0}, "HD": {"200": 11, "500": 3, "100": 11, "600": 1}, "FP": {"200": 1, "100": 11, "500": 1, "600": 0}, "GM": {"200": 0.5303300858899106, "500": 0.5601120336112039, "100": "None", "600": 0.0}, "CEN": {"200": 0.3570795472009597, "500": 0.5389466410223563, "100": 0.3349590631259315, "600": 0.0}, "P": {"200": 16, "500": 3, "100": 0, "600": 1}, "NPV": {"200": 0.23076923076923078, "500": 0.8888888888888888, "100": 1.0, "600": 0.95}, "NLRI": {"200": "Negligible", "500": "Negligible", "100": "None", "600": "Negligible"}, "BM": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "OC": {"200": 0.8571428571428571, "500": 0.5, "100": "None", "600": "None"}, "PPV": {"200": 0.8571428571428571, "500": 0.5, "100": 0.0, "600": "None"}, "TOP": {"200": 7, "500": 2, "100": 11, "600": 0}, "FNR": {"200": 0.625, "500": 0.6666666666666667, "100": "None", "600": 1.0}, "DP": {"200": 0.1407391082701595, "500": 0.49789960499474867, "100": "None", "600": "None"}, "sInd": {"200": 0.5240141808835057, "500": 0.5267639848569737, "100": "None", "600": 0.29289321881345254}, "Y": {"200": 0.125, "500": 0.27450980392156854, "100": "None", "600": 0.0}, "OP": {"200": 0.1166666666666667, "500": 0.373076923076923, "100": "None", "600": -0.050000000000000044}}, "Predict-Vector": [100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200], "Imbalanced": true, "Actual-Vector": [600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200], "Sample-Weight": null, "Prob-Vector": null, "Transpose": false, "Matrix": [[100, [[200, 0], [500, 0], [100, 0], [600, 0]]], [200, [[200, 6], [500, 1], [100, 9], [600, 0]]], [500, [[200, 1], [500, 1], [100, 1], [600, 0]]], [600, [[200, 0], [500, 0], [100, 1], [600, 0]]]], "Overall-Stat": {"Phi-Squared": "None", "Overall ACC": 0.35, "Conditional Entropy": 1.235789374242786, "ARI": 0.02298247455136956, "Kappa": 0.07801418439716304, "F1 Micro": 0.35, "Kappa No Prevalence": -0.30000000000000004, "SOA5(Cramer)": "None", "FPR Micro": 0.21666666666666667, "Gwet AC1": 0.19504643962848295, "Chi-Squared": "None", "Bangdiwala B": 0.3135593220338983, "FNR Macro": "None", "FPR Macro": 0.2147058823529412, "Joint Entropy": 2.119973094021975, "NIR": 0.8, "SOA3(Altman)": "Poor", "RR": 5.0, "ACC Macro": 0.675, "CBA": 0.17708333333333331, "Krippendorff Alpha": -0.09740259740259723, "SOA9(Krippendorff Alpha)": "Low", "Kappa Unbiased": -0.12554112554112543, "F1 Macro": 0.23043478260869565, "Zero-one Loss": 13, "KL Divergence": "None", "Kappa 95% CI": [-0.21849807698648957, 0.3745264457808156], "Bennett S": 0.1333333333333333, "Response Entropy": 1.3366664819166876, "CSI": "None", "Overall MCEN": 0.3746281299595305, "SOA10(Pearson C)": "None", "Cramer V": "None", "SOA6(Matthews)": "Negligible", "Standard Error": 0.1066536450385077, "SOA7(Lambda A)": "None", "95% CI": [0.14095885572452488, 0.559041144275475], "AUNP": "None", "PPV Micro": 0.35, "SOA4(Cicchetti)": "Poor", "SOA1(Landis & Koch)": "Slight", "Scott PI": -0.12554112554112543, "Chi-Squared DF": 9, "Hamming Loss": 0.65, "Kappa Standard Error": 0.15128176601206766, "TPR Macro": "None", "Lambda A": 0.0, "SOA8(Lambda B)": "None", "Overall CEN": 0.3648028121279775, "Overall RACC": 0.29500000000000004, "SOA2(Fleiss)": "Poor", "Cross Entropy": 1.709947752496911, "PPV Macro": "None", "Overall MCC": 0.1264200803632855, "Mutual Information": 0.10087710767390168, "TNR Micro": 0.7833333333333333, "TNR Macro": 0.7852941176470588, "Overall RACCU": 0.42249999999999993, "Lambda B": 0.0, "FNR Micro": 0.65, "P-Value": 0.9999981549942787, "RCI": 0.11409066398451011, "Pearson C": "None", "Reference Entropy": 0.8841837197791889, "AUNU": "None", "TPR Micro": 0.35, "Overall J": [0.6029411764705883, 0.15073529411764708]}} \ No newline at end of file diff --git a/Document/Example5.ipynb b/Document/Example5.ipynb index 85e58e4d..52305751 100644 --- a/Document/Example5.ipynb +++ b/Document/Example5.ipynb @@ -160,6 +160,10 @@ "SOA4(Cicchetti) Poor\n", "SOA5(Cramer) Relatively Strong\n", "SOA6(Matthews) Weak\n", + "SOA7(Lambda A) Very Weak\n", + "SOA8(Lambda B) Moderate\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.34426\n", "Standard Error 0.14232\n", "TNR Macro 0.77778\n", @@ -361,6 +365,10 @@ "SOA4(Cicchetti) Good\n", "SOA5(Cramer) Strong\n", "SOA6(Matthews) Moderate\n", + "SOA7(Lambda A) Moderate\n", + "SOA8(Lambda B) Moderate\n", + "SOA9(Krippendorff Alpha) Low\n", + "SOA10(Pearson C) Strong\n", "Scott PI 0.63481\n", "Standard Error 0.03347\n", "TNR Macro 0.88519\n", diff --git a/Document/Example6.ipynb b/Document/Example6.ipynb index 8272208e..baa4dff8 100644 --- a/Document/Example6.ipynb +++ b/Document/Example6.ipynb @@ -88,11 +88,11 @@ "Class2 0.04762 0.95238 \n", "\n", "\n", - "ACC: {'Class1': 0.9976333515383216, 'Class2': 0.9976333515383216}\n", - "MCC: {'Class1': 0.9378574017402594, 'Class2': 0.9378574017402594}\n", - "CEN: {'Class1': 0.012858728415908176, 'Class2': 0.30489006849060607}\n", - "MCEN: {'Class1': 0.023280122318969122, 'Class2': 0.46949279678726225}\n", - "DP: {'Class1': 2.276283896527635, 'Class2': 2.276283896527635}\n", + "ACC: {'Class2': 0.9976333515383216, 'Class1': 0.9976333515383216}\n", + "MCC: {'Class2': 0.9378574017402594, 'Class1': 0.9378574017402594}\n", + "CEN: {'Class2': 0.30489006849060607, 'Class1': 0.012858728415908176}\n", + "MCEN: {'Class2': 0.46949279678726225, 'Class1': 0.023280122318969122}\n", + "DP: {'Class2': 2.276283896527635, 'Class1': 2.276283896527635}\n", "Kappa: 0.9377606597584491\n", "RCI: 0.8682877002417864\n", "SOA1: Almost Perfect\n" @@ -142,11 +142,11 @@ "Class2 0.95238 0.04762 \n", "\n", "\n", - "ACC: {'Class1': 0.982098458478369, 'Class2': 0.982098458478369}\n", - "MCC: {'Class1': 0.13048897476798949, 'Class2': 0.13048897476798949}\n", - "CEN: {'Class1': 0.06481573363174531, 'Class2': 0.4655917826576813}\n", - "MCEN: {'Class1': 0.11078640690031397, 'Class2': 0.4264929996758212}\n", - "DP: {'Class1': 0.864594924328404, 'Class2': 0.864594924328404}\n", + "ACC: {'Class2': 0.982098458478369, 'Class1': 0.982098458478369}\n", + "MCC: {'Class2': 0.13048897476798949, 'Class1': 0.13048897476798949}\n", + "CEN: {'Class2': 0.4655917826576813, 'Class1': 0.06481573363174531}\n", + "MCEN: {'Class2': 0.4264929996758212, 'Class1': 0.11078640690031397}\n", + "DP: {'Class2': 0.864594924328404, 'Class1': 0.864594924328404}\n", "Kappa: 0.08122239707598865\n", "RCI: 0.022375346499017443\n", "SOA1: Slight\n" @@ -196,11 +196,11 @@ "Class2 0.04762 0.95238 \n", "\n", "\n", - "ACC: {'Class1': 0.019661387220098307, 'Class2': 0.019661387220098307}\n", - "MCC: {'Class1': -0.13000800945464058, 'Class2': -0.13000800945464058}\n", - "CEN: {'Class1': 0.014927427128936136, 'Class2': 0.06103563616795208}\n", - "MCEN: {'Class1': 0.01281422838054554, 'Class2': 0.03655796690365652}\n", - "DP: {'Class1': -0.8416930356875597, 'Class2': -0.8416930356875597}\n", + "ACC: {'Class2': 0.019661387220098307, 'Class1': 0.019661387220098307}\n", + "MCC: {'Class2': -0.13000800945464058, 'Class1': -0.13000800945464058}\n", + "CEN: {'Class2': 0.06103563616795208, 'Class1': 0.014927427128936136}\n", + "MCEN: {'Class2': 0.03655796690365652, 'Class1': 0.01281422838054554}\n", + "DP: {'Class2': -0.8416930356875597, 'Class1': -0.8416930356875597}\n", "Kappa: -0.0017678372492452412\n", "RCI: 0.02192606003351106\n", "SOA1: Poor\n" @@ -261,13 +261,13 @@ "Class4 0.0 0.0 2e-05 0.99998 \n", "\n", "\n", - "ACC: {'Class1': 0.9999750099960016, 'Class4': 0.9999500199920032, 'Class2': 0.9999500199920032, 'Class3': 0.9999250299880048}\n", - "MCC: {'Class1': 0.8944160139432883, 'Class4': 0.9333083339583177, 'Class2': 0.7999750068731099, 'Class3': 0.7302602381427055}\n", - "CEN: {'Class1': 0.13625493172565745, 'Class4': 0.0001575200922489127, 'Class2': 0.25701944178769376, 'Class3': 0.3649884090288471}\n", - "MCEN: {'Class1': 0.17964888034078544, 'Class4': 0.00029569133318617423, 'Class2': 0.3333333333333333, 'Class3': 0.4654427710721536}\n", - "DP: {'Class1': 'None', 'Class4': 3.1691421556058055, 'Class2': 2.869241573973406, 'Class3': 2.7032690544190636}\n", + "ACC: {'Class2': 0.9999500199920032, 'Class3': 0.9999250299880048, 'Class1': 0.9999750099960016, 'Class4': 0.9999500199920032}\n", + "MCC: {'Class2': 0.7999750068731099, 'Class3': 0.7302602381427055, 'Class1': 0.8944160139432883, 'Class4': 0.9333083339583177}\n", + "CEN: {'Class2': 0.25701944178769376, 'Class3': 0.3649884090288471, 'Class1': 0.13625493172565745, 'Class4': 0.0001575200922489127}\n", + "MCEN: {'Class2': 0.3333333333333333, 'Class3': 0.4654427710721536, 'Class1': 0.17964888034078544, 'Class4': 0.00029569133318617423}\n", + "DP: {'Class2': 2.869241573973406, 'Class3': 2.7032690544190636, 'Class1': 'None', 'Class4': 3.1691421556058055}\n", "Kappa: 0.8666333383326446\n", - "RCI: 0.8711441699127427\n", + "RCI: 0.8711441699127425\n", "SOA1: Almost Perfect\n" ] } @@ -320,11 +320,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class1': 0.625, 'Class4': 0.625, 'Class2': 0.625, 'Class3': 0.625}\n", - "MCC: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class1': 0.8704188162777186, 'Class4': 0.8704188162777186, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class1': 0.9308855421443073, 'Class4': 0.9308855421443073, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class2': 0.625, 'Class3': 0.625, 'Class1': 0.625, 'Class4': 0.625}\n", + "MCC: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186, 'Class1': 0.8704188162777186, 'Class4': 0.8704188162777186}\n", + "MCEN: {'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073, 'Class1': 0.9308855421443073, 'Class4': 0.9308855421443073}\n", + "DP: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" @@ -379,13 +379,13 @@ "Class4 0.76923 0.07692 0.07692 0.07692 \n", "\n", "\n", - "ACC: {'Class1': 0.4, 'Class4': 0.4, 'Class2': 0.76, 'Class3': 0.76}\n", - "MCC: {'Class1': -0.2358640882624316, 'Class4': -0.2358640882624316, 'Class2': 0.10714285714285714, 'Class3': 0.10714285714285714}\n", - "CEN: {'Class1': 0.6392779429225794, 'Class4': 0.6392779429225796, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class1': 0.647512271542988, 'Class4': 0.647512271542988, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class1': -0.33193306999649924, 'Class4': -0.3319330699964992, 'Class2': 0.16596653499824943, 'Class3': 0.16596653499824943}\n", - "Kappa: -0.07361963190184051\n", - "RCI: 0.1160303056449364\n", + "ACC: {'Class2': 0.76, 'Class3': 0.76, 'Class1': 0.4, 'Class4': 0.4}\n", + "MCC: {'Class2': 0.10714285714285714, 'Class3': 0.10714285714285714, 'Class1': -0.2358640882624316, 'Class4': -0.2358640882624316}\n", + "CEN: {'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186, 'Class1': 0.6392779429225794, 'Class4': 0.6392779429225796}\n", + "MCEN: {'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073, 'Class1': 0.647512271542988, 'Class4': 0.647512271542988}\n", + "DP: {'Class2': 0.16596653499824943, 'Class3': 0.16596653499824943, 'Class1': -0.33193306999649924, 'Class4': -0.3319330699964992}\n", + "Kappa: -0.07361963190184047\n", + "RCI: 0.11603030564493641\n", "SOA1: Poor\n" ] } @@ -438,12 +438,12 @@ "Class4 0.76923 0.07692 0.07692 0.07692 \n", "\n", "\n", - "ACC: {'Class1': 0.000998502246630055, 'Class4': 0.000998502246630055, 'Class2': 0.999400898652022, 'Class3': 0.999400898652022}\n", - "MCC: {'Class1': -0.43266656861311537, 'Class4': -0.43266656861311537, 'Class2': 0.24970032963739885, 'Class3': 0.24970032963739885}\n", - "CEN: {'Class1': 0.0029588592520426657, 'Class4': 0.0029588592520426657, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class1': 0.002903385725603509, 'Class4': 0.002903385725603509, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class1': -1.9423127303715728, 'Class4': -1.9423127303715728, 'Class2': 1.6794055876913858, 'Class3': 1.6794055876913858}\n", - "Kappa: -0.0003990813465900263\n", + "ACC: {'Class2': 0.999400898652022, 'Class3': 0.999400898652022, 'Class1': 0.000998502246630055, 'Class4': 0.000998502246630055}\n", + "MCC: {'Class2': 0.24970032963739885, 'Class3': 0.24970032963739885, 'Class1': -0.43266656861311537, 'Class4': -0.43266656861311537}\n", + "CEN: {'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186, 'Class1': 0.0029588592520426657, 'Class4': 0.0029588592520426657}\n", + "MCEN: {'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073, 'Class1': 0.002903385725603509, 'Class4': 0.002903385725603509}\n", + "DP: {'Class2': 1.6794055876913858, 'Class3': 1.6794055876913858, 'Class1': -1.9423127303715728, 'Class4': -1.9423127303715728}\n", + "Kappa: -0.0003990813465900262\n", "RCI: 0.5536610475678805\n", "SOA1: Poor\n" ] @@ -497,11 +497,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class1': 0.7115384615384616, 'Class4': 0.36538461538461536, 'Class2': 0.7115384615384616, 'Class3': 0.7115384615384616}\n", - "MCC: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class1': 0.6392779429225794, 'Class4': 0.6522742127953861, 'Class2': 0.6392779429225794, 'Class3': 0.6392779429225794}\n", - "MCEN: {'Class1': 0.647512271542988, 'Class4': 0.7144082229288313, 'Class2': 0.647512271542988, 'Class3': 0.647512271542988}\n", - "DP: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class2': 0.7115384615384616, 'Class3': 0.7115384615384616, 'Class1': 0.7115384615384616, 'Class4': 0.36538461538461536}\n", + "MCC: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class2': 0.6392779429225794, 'Class3': 0.6392779429225794, 'Class1': 0.6392779429225794, 'Class4': 0.6522742127953861}\n", + "MCEN: {'Class2': 0.647512271542988, 'Class3': 0.647512271542988, 'Class1': 0.647512271542988, 'Class4': 0.7144082229288313}\n", + "DP: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" @@ -556,11 +556,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class1': 0.7499500149955014, 'Class4': 0.25014995501349596, 'Class2': 0.7499500149955014, 'Class3': 0.7499500149955014}\n", - "MCC: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class1': 0.0029588592520426657, 'Class4': 0.539296694603886, 'Class2': 0.0029588592520426657, 'Class3': 0.0029588592520426657}\n", - "MCEN: {'Class1': 0.002903385725603509, 'Class4': 0.580710610324597, 'Class2': 0.002903385725603509, 'Class3': 0.002903385725603509}\n", - "DP: {'Class1': 0.0, 'Class4': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class2': 0.7499500149955014, 'Class3': 0.7499500149955014, 'Class1': 0.7499500149955014, 'Class4': 0.25014995501349596}\n", + "MCC: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class2': 0.0029588592520426657, 'Class3': 0.0029588592520426657, 'Class1': 0.0029588592520426657, 'Class4': 0.539296694603886}\n", + "MCEN: {'Class2': 0.002903385725603509, 'Class3': 0.002903385725603509, 'Class1': 0.002903385725603509, 'Class4': 0.580710610324597}\n", + "DP: {'Class2': 0.0, 'Class3': 0.0, 'Class1': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" diff --git a/Document/README.md b/Document/README.md index c91b74b9..d0d9d936 100644 --- a/Document/README.md +++ b/Document/README.md @@ -8,7 +8,12 @@ ## Document - [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Document.ipynb) -- [HTML](http://www.pycm.io/doc/) +- [HTML](http://www.pycm.io/doc/) + +## Distance + +- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Distance.ipynb) +- [HTML](http://www.pycm.io/doc/Distance.html) ## Example-1 (Comparison of three different classifiers) diff --git a/Otherfiles/doc_run.bat b/Otherfiles/doc_run.bat index 3aea127b..3edbdac3 100644 --- a/Otherfiles/doc_run.bat +++ b/Otherfiles/doc_run.bat @@ -1,5 +1,6 @@ @echo off call python -m art text "Doc Run Script" +call python setup.py install for %%f in (Document\*.ipynb) do ( echo %%f is running! call python -m nbconvert %%f --execute --clear-output --log-level=ERROR @@ -12,3 +13,5 @@ copy Document\Document_Files\cm1.html Otherfiles\test.html copy Document\Document_Files\cm1.obj Otherfiles\test.obj copy Document\Document_Files\cm1.pycm Otherfiles\test.pycm copy Document\Document_Files\cp.comp Otherfiles\test.comp + +call python Otherfiles\version_check.py diff --git a/Otherfiles/doc_to_html.bat b/Otherfiles/doc_to_html.bat index 2d7dc4e9..b3df01d8 100644 --- a/Otherfiles/doc_to_html.bat +++ b/Otherfiles/doc_to_html.bat @@ -1,6 +1,7 @@ @echo off call python -m art text "Doc 2 HTML" +call python setup.py install if not exist "doc_html" mkdir doc_html for %%f in (doc_html\*) do (del %%f) copy Document\*.ipynb doc_html @@ -12,6 +13,11 @@ move Document.html index.html if ERRORLEVEL 1 (echo Document.ipynb Failed! & cd .. & exit /b 0) else (echo Document.ipynb Done!) echo -------------------------- del Document.ipynb +echo Distance.ipynb is running! +call python -m nbconvert --to html_toc --execute Distance.ipynb --no-prompt --TagRemovePreprocessor.enabled=True --TagRemovePreprocessor.remove_cell_tags={\"html_hide\"} --log-level=ERROR +if ERRORLEVEL 1 (echo Distance.ipynb Failed! & cd .. & exit /b 0) else (echo Distance.ipynb Done!) +echo -------------------------- +del Distance.ipynb for %%f in (*.ipynb) do ( echo %%f is running! call python -m nbconvert --to html --execute %%f --no-prompt --log-level=ERROR @@ -20,3 +26,4 @@ echo -------------------------- del %%f ) cd .. +call python Otherfiles\version_check.py diff --git a/Otherfiles/meta.yaml b/Otherfiles/meta.yaml index 179d8ecc..e0e3e70e 100644 --- a/Otherfiles/meta.yaml +++ b/Otherfiles/meta.yaml @@ -1,5 +1,5 @@ {% set name = "pycm" %} -{% set version = "3.7" %} +{% set version = "3.8" %} package: name: {{ name|lower }} diff --git a/Otherfiles/notebook_check.py b/Otherfiles/notebook_check.py index 8545f8ab..a8f8ae75 100644 --- a/Otherfiles/notebook_check.py +++ b/Otherfiles/notebook_check.py @@ -7,6 +7,7 @@ NOTEBOOKS_LIST = [ "Document", + "Distance", "Example1", "Example2", "Example3", diff --git a/Otherfiles/test.comp b/Otherfiles/test.comp index 4d0bc5a9..e236be3a 100644 --- a/Otherfiles/test.comp +++ b/Otherfiles/test.comp @@ -1,5 +1,5 @@ Best : cm2 Rank Name Class-Score Overall-Score -1 cm2 0.50278 0.425 -2 cm3 0.33611 0.33056 +1 cm2 0.50278 0.58095 +2 cm3 0.33611 0.52857 diff --git a/Otherfiles/test.html b/Otherfiles/test.html index d0bdbf2d..cb79547e 100644 --- a/Otherfiles/test.html +++ b/Otherfiles/test.html @@ -284,6 +284,22 @@

    Overall Statistics :

    Weak +SOA7(Lambda A) +Moderate + + +SOA8(Lambda B) +Very Weak + + +SOA9(Krippendorff Alpha) +Low + + +SOA10(Pearson C) +Strong + + Scott PI 0.34426 @@ -763,6 +779,6 @@

    Class Statistics :

    Similarity index -

    Generated By PyCM Version 3.7

    +

    Generated By PyCM Version 3.8

    diff --git a/Otherfiles/test.obj b/Otherfiles/test.obj index 5d651302..34f91b7c 100644 --- a/Otherfiles/test.obj +++ b/Otherfiles/test.obj @@ -1 +1 @@ -{"Digit": 5, "Prob-Vector": null, "Sample-Weight": null, "Imbalanced": false, "Transpose": true, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Actual-Vector": null, "Predict-Vector": null} \ No newline at end of file +{"Prob-Vector": null, "Transpose": true, "Imbalanced": false, "Predict-Vector": null, "Matrix": [["L1", [["L1", 3], ["L3", 2], ["L2", 0]]], ["L2", [["L1", 0], ["L3", 1], ["L2", 1]]], ["L3", [["L1", 0], ["L3", 3], ["L2", 2]]]], "Actual-Vector": null, "Sample-Weight": null, "Digit": 5} \ No newline at end of file diff --git a/Otherfiles/test.pycm b/Otherfiles/test.pycm index 7ea48225..4e805d68 100644 --- a/Otherfiles/test.pycm +++ b/Otherfiles/test.pycm @@ -80,6 +80,10 @@ SOA3(Altman) Fair SOA4(Cicchetti) Poor SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Moderate +SOA8(Lambda B) Very Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.34426 Standard Error 0.14232 TNR Macro 0.79048 diff --git a/Otherfiles/version_check.py b/Otherfiles/version_check.py index 5afca101..b2a4c4f4 100644 --- a/Otherfiles/version_check.py +++ b/Otherfiles/version_check.py @@ -4,7 +4,7 @@ import sys import codecs Failed = 0 -PYCM_VERSION = "3.7" +PYCM_VERSION = "3.8" SETUP_ITEMS = [ diff --git a/README.md b/README.md index 82aa9377..38a023ed 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,7 @@ - -## Table of contents +## Table of contents * [Overview](https://github.com/sepandhaghighi/pycm#overview) * [Installation](https://github.com/sepandhaghighi/pycm#installation) * [Usage](https://github.com/sepandhaghighi/pycm#usage) @@ -38,7 +37,7 @@ ## Overview

    -PyCM is a multi-class confusion matrix library written in Python that supports both input data vectors and direct matrix, and a proper tool for post-classification model evaluation that supports most classes and overall statistics parameters. +PyCM is a multi-class confusion matrix library written in Python that supports both input data vectors and direct matrix, and a proper tool for post-classification model evaluation that supports most classes and overall statistics parameters. PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scientists that need a broad array of metrics for predictive models and accurate evaluation of a large variety of classifiers.

    @@ -48,11 +47,10 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie

    Fig1. ConfusionMatrix Block Diagram

    - - + - + @@ -69,8 +67,8 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie
    Open Hub
    PyPI Counter
    - - + + @@ -83,30 +81,27 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie
    Branchmasterdevmasterdev
    CI
    - - - + + +
    Code QualityCodeFactorcodebeat badgeCodeFactorcodebeat badge
    - - -## Installation +## Installation ⚠️ PyCM 2.4 is the last version to support **Python 2.7** & **Python 3.4** -⚠️ Plotting capability requires **Matplotlib (>= 3.0.0)** or **Seaborn (>= 0.9.1)** +⚠️ Plotting capability requires **Matplotlib (>= 3.0.0)** or **Seaborn (>= 0.9.1)** ### Source code -- Download [Version 3.7](https://github.com/sepandhaghighi/pycm/archive/v3.7.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip) +- Download [Version 3.8](https://github.com/sepandhaghighi/pycm/archive/v3.8.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip) - Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access) -- Run `python3 setup.py install` or `python setup.py install` (Need root access) +- Run `python3 setup.py install` or `python setup.py install` (Need root access) ### PyPI - -- Check [Python Packaging User Guide](https://packaging.python.org/installing/) -- Run `pip install pycm==3.7` or `pip3 install pycm==3.7` (Need root access) +- Check [Python Packaging User Guide](https://packaging.python.org/installing/) +- Run `pip install pycm==3.8` or `pip3 install pycm==3.8` (Need root access) ### Conda @@ -121,21 +116,22 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie ### MATLAB - Download and install [MATLAB](https://www.mathworks.com/products/matlab.html) (>=8.5, 64/32 bit) -- Download and install [Python3.x](https://www.python.org/downloads/) (>=3.5, 64/32 bit) +- Download and install [Python3.x](https://www.python.org/downloads/) (>=3.5, 64/32 bit) - [x] Select `Add to PATH` option - [x] Select `Install pip` option - Run `pip install pycm` or `pip3 install pycm` (Need root access) - Configure Python interpreter + ```matlab >> pyversion PYTHON_EXECUTABLE_FULL_PATH ``` -- Visit [MATLAB Examples](https://github.com/sepandhaghighi/pycm/tree/master/MATLAB) +- Visit [MATLAB Examples](https://github.com/sepandhaghighi/pycm/tree/master/MATLAB) ## Usage - ### From vector + ```pycon >>> from pycm import * >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] @@ -197,7 +193,9 @@ TP(True positive/hit) 3 TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 0.33333 0.5 ``` + ### Direct CM + ```pycon >>> from pycm import * >>> cm2 = ConfusionMatrix(matrix={"Class1": {"Class1": 1, "Class2":2}, "Class2": {"Class1": 0, "Class2": 5}}) @@ -253,28 +251,33 @@ TP(True positive/hit) 1 TPR(Sensitivity, recall, hit rate, or true positive rate) 0.33333 1.0 ``` -* `matrix()` and `normalized_matrix()` renamed to `print_matrix()` and `print_normalized_matrix()` in `version 1.5` + +* `matrix()` and `normalized_matrix()` renamed to `print_matrix()` and `print_normalized_matrix()` in `version 1.5` ### Activation threshold -`threshold` is added in `version 0.9` for real value prediction. - + +`threshold` is added in `version 0.9` for real value prediction. For more information visit [Example3](http://www.pycm.io/doc/Example3.html "Example3") -### Load from file +### Load from file + `file` is added in `version 0.9.5` in order to load saved confusion matrix with `.obj` format generated by `save_obj` method. For more information visit [Example4](http://www.pycm.io/doc/Example4.html "Example4") ### Sample weights + `sample_weight` is added in `version 1.2` For more information visit [Example5](http://www.pycm.io/doc/Example5.html "Example5") ### Transpose + `transpose` is added in `version 1.2` in order to transpose input matrix (only in `Direct CM` mode) -### Relabel -`relabel` method is added in `version 1.5` in order to change ConfusionMatrix classnames. +### Relabel + +`relabel` method is added in `version 1.5` in order to change ConfusionMatrix classnames. ```pycon >>> cm.relabel(mapping={0:"L1",1:"L2",2:"L3"}) @@ -282,8 +285,9 @@ For more information visit [Example5](http://www.pycm.io/doc/Example5.html "Exam pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3']) ``` -### Position -`position` method is added in `version 2.8` in order to find the indexes of observations in `predict_vector` which made TP, TN, FP, FN. +### Position + +`position` method is added in `version 2.8` in order to find the indexes of observations in `predict_vector` which made TP, TN, FP, FN. ```pycon >>> cm.position() @@ -291,6 +295,7 @@ pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3']) ``` ### To array + `to_array` method is added in `version 2.9` in order to returns the confusion matrix in the form of a NumPy array. This can be helpful to apply different operations over the confusion matrix for different purposes such as aggregation, normalization, and combination. ```pycon @@ -302,13 +307,14 @@ array([[3, 0, 0], array([[1. , 0. , 0. ], [0. , 0.33333, 0.66667], [0.33333, 0.16667, 0.5 ]]) ->>> cm.to_array(normalized=True,one_vs_all=True, class_name="L1") +>>> cm.to_array(normalized=True, one_vs_all=True, class_name="L1") array([[1. , 0. ], [0.22222, 0.77778]]) ``` ### Combine -`combine` method is added in `version 3.0` in order to merge two confusion matrices. This option will be useful in mini-batch learning. + +`combine` method is added in `version 3.0` in order to merge two confusion matrices. This option will be useful in mini-batch learning. ```pycon >>> cm_combined = cm2.combine(cm3) @@ -322,61 +328,63 @@ Class2 0 10 ``` ### Plot + `plot` method is added in `version 3.0` in order to plot a confusion matrix using Matplotlib or Seaborn. ```pycon >>> cm.plot() ``` - + + ```pycon >>> from matplotlib import pyplot as plt ->>> cm.plot(cmap=plt.cm.Greens,number_label=True,plot_lib="matplotlib") -``` +>>> cm.plot(cmap=plt.cm.Greens, number_label=True, plot_lib="matplotlib") +``` ```pycon ->>> cm.plot(cmap=plt.cm.Reds,normalized=True,number_label=True,plot_lib="seaborn") -``` +>>> cm.plot(cmap=plt.cm.Reds, normalized=True, number_label=True, plot_lib="seaborn") +``` ### ROC curve -`ROCCurve`, added in `version 3.7`, is devised to compute the Receiver Operating Characteristic (ROC) or simply ROC curve. In ROC curves, the Y axis represents the True Positive Rate, and the X axis represents the False Positive Rate. Thus, the ideal point is located at the top left of the curve, and a larger area under the curve represents better performance. ROC curve is a graphical representation of binary classifiers' performance. In PyCM, `ROCCurve` binarizes the output based on the "One vs. Rest" strategy to provide an extension of ROC for multi-class classifiers. Getting the actual labels vector, the target probability estimates of the positive classes, and the list of ordered labels of classes, this method is able to compute and plot TPR-FPR pairs for different discrimination thresholds and compute the area under the ROC curve. +`ROCCurve`, added in `version 3.7`, is devised to compute the Receiver Operating Characteristic (ROC) or simply ROC curve. In ROC curves, the Y axis represents the True Positive Rate, and the X axis represents the False Positive Rate. Thus, the ideal point is located at the top left of the curve, and a larger area under the curve represents better performance. ROC curve is a graphical representation of binary classifiers' performance. In PyCM, `ROCCurve` binarizes the output based on the "One vs. Rest" strategy to provide an extension of ROC for multi-class classifiers. Getting the actual labels vector, the target probability estimates of the positive classes, and the list of ordered labels of classes, this method is able to compute and plot TPR-FPR pairs for different discrimination thresholds and compute the area under the ROC curve. ```pycon - >>> crv = ROCCurve(actual_vector = np.array([1, 1, 2, 2]), probs = np.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]), classes=[2, 1]) - >>> crv.thresholds - [0.1, 0.2, 0.35, 0.4, 0.6, 0.65, 0.8, 0.9] - >>> auc_trp = crv.area() - >>> auc_trp[1] - 0.75 - >>> auc_trp[2] - 0.75 -``` +>>> crv = ROCCurve(actual_vector=np.array([1, 1, 2, 2]), probs=np.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]), classes=[2, 1]) +>>> crv.thresholds +[0.1, 0.2, 0.35, 0.4, 0.6, 0.65, 0.8, 0.9] +>>> auc_trp = crv.area() +>>> auc_trp[1] +0.75 +>>> auc_trp[2] +0.75 +``` ### Precision-Recall curve `PRCurve`, added in `version 3.7`, is devised to compute the Precision-Recall curve in which the Y axis represents the Precision, and the X axis represents the Recall of a classifier. Thus, the ideal point is located at the top right of the curve, and a larger area under the curve represents better performance. Precision-Recall curve is a graphical representation of binary classifiers' performance. In PyCM, `PRCurve` binarizes the output based on the "One vs. Rest" strategy to provide an extension of this curve for multi-class classifiers. Getting the actual labels vector, the target probability estimates of the positive classes, and the list of ordered labels of classes, this method is able to compute and plot Precision-Recall pairs for different discrimination thresholds and compute the area under the curve. ```pycon - >>> crv = PRCurve(actual_vector = np.array([1, 1, 2, 2]), probs = np.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]), classes=[2, 1]) - >>> crv.thresholds - [0.1, 0.2, 0.35, 0.4, 0.6, 0.65, 0.8, 0.9] - >>> auc_trp = crv.area() - >>> auc_trp[1] - 0.29166666666666663 - >>> auc_trp[2] - 0.29166666666666663 -``` +>>> crv = PRCurve(actual_vector=np.array([1, 1, 2, 2]), probs=np.array([[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]]), classes=[2, 1]) +>>> crv.thresholds +[0.1, 0.2, 0.35, 0.4, 0.6, 0.65, 0.8, 0.9] +>>> auc_trp = crv.area() +>>> auc_trp[1] +0.29166666666666663 +>>> auc_trp[2] +0.29166666666666663 +``` ### Parameter recommender -This option has been added in `version 1.9` to recommend the most related parameters considering the characteristics of the input dataset. +This option has been added in `version 1.9` to recommend the most related parameters considering the characteristics of the input dataset. The suggested parameters are selected according to some characteristics of the input such as being balance/imbalance and binary/multi-class. -All suggestions can be categorized into three main groups: imbalanced dataset, binary classification for a balanced dataset, and multi-class classification for a balanced dataset. +All suggestions can be categorized into three main groups: imbalanced dataset, binary classification for a balanced dataset, and multi-class classification for a balanced dataset. The recommendation lists have been gathered according to the respective paper of each parameter and the capabilities which had been claimed by the paper. ```pycon @@ -398,27 +406,26 @@ True >>> cm = ConfusionMatrix(y_actu, y_pred, is_imbalanced = False) >>> cm.imbalance False -``` +``` ### Compare In `version 2.0`, a method for comparing several confusion matrices is introduced. This option is a combination of several overall and class-based benchmarks. Each of the benchmarks evaluates the performance of the classification algorithm from good to poor and give them a numeric score. The score of good and poor performances are 1 and 0, respectively. -After that, two scores are calculated for each confusion matrices, overall and class-based. The overall score is the average of the score of six overall benchmarks which are Landis & Koch, Fleiss, Altman, Cicchetti, Cramer, and Matthews. In the same manner, the class-based score is the average of the score of six class-based benchmarks which are Positive Likelihood Ratio Interpretation, Negative Likelihood Ratio Interpretation, Discriminant Power Interpretation, AUC value Interpretation, Matthews Correlation Coefficient Interpretation and Yule's Q Interpretation. It should be noticed that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminated in total averaging. If the user sets weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average. +After that, two scores are calculated for each confusion matrices, overall and class-based. The overall score is the average of the score of seven overall benchmarks which are Landis & Koch, Cramer, Matthews, Goodman-Kruskal's Lambda A, Goodman-Kruskal's Lambda B, Krippendorff's Alpha, and Pearson's C. In the same manner, the class-based score is the average of the score of six class-based benchmarks which are Positive Likelihood Ratio Interpretation, Negative Likelihood Ratio Interpretation, Discriminant Power Interpretation, AUC value Interpretation, Matthews Correlation Coefficient Interpretation and Yule's Q Interpretation. It should be noticed that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminated in total averaging. If the user sets weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average. If the user sets the value of `by_class` boolean input `True`, the best confusion matrix is the one with the maximum class-based score. Otherwise, if a confusion matrix obtains the maximum of both overall and class-based scores, that will be reported as the best confusion matrix, but in any other case, the compared object doesn’t select the best confusion matrix. - ```pycon ->>> cm2 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) ->>> cm3 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) ->>> cp = Compare({"cm2":cm2,"cm3":cm3}) +>>> cm2 = ConfusionMatrix(matrix={0:{0:2, 1:50, 2:6}, 1:{0:5, 1:50, 2:3}, 2:{0:1, 1:7, 2:50}}) +>>> cm3 = ConfusionMatrix(matrix={0:{0:50, 1:2, 2:6}, 1:{0:50, 1:5, 2:3}, 2:{0:1, 1:55, 2:2}}) +>>> cp = Compare({"cm2":cm2, "cm3":cm3}) >>> print(cp) Best : cm2 Rank Name Class-Score Overall-Score -1 cm2 0.50278 0.425 -2 cm3 0.33611 0.33056 +1 cm2 0.50278 0.58095 +2 cm3 0.33611 0.52857 >>> cp.best pycm.ConfusionMatrix(classes: [0, 1, 2]) @@ -426,34 +433,31 @@ pycm.ConfusionMatrix(classes: [0, 1, 2]) ['cm2', 'cm3'] >>> cp.best_name 'cm2' -``` +``` ### Online help `online_help` function is added in `version 1.1` in order to open each statistics definition in web browser - ```pycon - >>> from pycm import online_help >>> online_help("J") >>> online_help("SOA1(Landis & Koch)") >>> online_help(2) - ``` -* List of items are available by calling `online_help()` (without argument) + +* List of items are available by calling `online_help()` (without argument) * If PyCM website is not available, set `alt_link = True` (new in `version 2.4`) -### Acceptable data types +### Acceptable data types **ConfusionMatrix** - 1. `actual_vector` : python `list` or numpy `array` of any stringable objects 2. `predict_vector` : python `list` or numpy `array` of any stringable objects 3. `matrix` : `dict` -4. `digit`: `int` -5. `threshold` : `FunctionType (function or lambda)` +4. `digit`: `int` +5. `threshold` : `FunctionType (function or lambda)` 6. `file` : `File object` 7. `sample_weight` : python `list` or numpy `array` of numbers 8. `transpose` : `bool` @@ -467,8 +471,8 @@ pycm.ConfusionMatrix(classes: [0, 1, 2]) 1. `cm_dict` : python `dict` of `ConfusionMatrix` object (`str` : `ConfusionMatrix`) 2. `by_class` : `bool` 3. `class_weight` : python `dict` of class weights (`class_name` : `float`) -4. `class_benchmark_weight`: python `dict` of class benchmark weights (`class_benchmark_name` : `float`) -5. `overall_benchmark_weight`: python `dict` of overall benchmark weights (`overall_benchmark_name` : `float`) +4. `class_benchmark_weight`: python `dict` of class benchmark weights (`class_benchmark_name` : `float`) +5. `overall_benchmark_weight`: python `dict` of overall benchmark weights (`overall_benchmark_name` : `float`) 6. `digit`: `int` * Run `help(Compare)` for `Compare` object details @@ -478,43 +482,43 @@ pycm.ConfusionMatrix(classes: [0, 1, 2]) 1. `actual_vector` : python `list` or numpy `array` of any stringable objects 2. `probs` : python `list` or numpy `array` 3. `classes` : python `list` -4. `thresholds`: python `list` or numpy `array` -5. `sample_weight`: python `list` +4. `thresholds`: python `list` or numpy `array` +5. `sample_weight`: python `list` or numpy `array` **PRCurve** 1. `actual_vector` : python `list` or numpy `array` of any stringable objects 2. `probs` : python `list` or numpy `array` 3. `classes` : python `list` -4. `thresholds`: python `list` or numpy `array` -5. `sample_weight`: python `list` +4. `thresholds`: python `list` or numpy `array` +5. `sample_weight`: python `list` or numpy `array` For more information visit [here](https://github.com/sepandhaghighi/pycm/tree/master/Document "Document")
    - +
    ## Try PyCM in your browser! + PyCM can be used online in interactive Jupyter Notebooks via the Binder or Colab services! Try it out now! : [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/sepandhaghighi/pycm/master) [![Google Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/sepandhaghighi/pycm/blob/master) -* Check `Examples` in `Document` folder +* Check `Examples` in `Document` folder -## Issues & bug reports +## Issues & bug reports -1. Fill an issue and describe it. We'll check it ASAP! +1. Fill an issue and describe it. We'll check it ASAP! - Please complete the issue template 2. Discord : [https://discord.com/invite/zqpU2b3J3f](https://discord.com/invite/zqpU2b3J3f) -3. Website : [https://www.pycm.io](https://www.pycm.io) +3. Website : [https://www.pycm.io](https://www.pycm.io) 4. Mailing List : [https://mail.python.org/mailman3/lists/pycm.python.org/](https://mail.python.org/mailman3/lists/pycm.python.org/) 5. Email : [info@pycm.io](mailto:info@pycm.io "info@pycm.io") - ## Acknowledgments [NLnet foundation](https://nlnet.nl) has supported the PyCM project from version **3.6** to **4.0** through the [NGI Assure](https://nlnet.nl/assure) Fund. This fund is set up by [NLnet foundation](https://nlnet.nl) with funding from the European Commission's [Next Generation Internet program](https://ngi.eu), administered by DG Communications Networks, Content, and Technology under grant agreement [**No 957073**](https://nlnet.nl/project/PyCM/). @@ -548,16 +552,14 @@ Haghighi, S., Jasemi, M., Hessabi, S. and Zolanvari, A. (2018). PyCM: Multiclass journal = {Journal of Open Source Software} } - -Download [PyCM.bib](http://www.pycm.io/PYCM.bib) - +Download [PyCM.bib](http://www.pycm.io/PYCM.bib) - + @@ -575,11 +577,8 @@ Download [PyCM.bib](http://www.pycm.io/PYCM.bib) Give a ⭐️ if this project helped you! - ### Donate to our project If you do like our project and we hope that you do, can you please support us? Our project is not and is never going to be working for profit. We need the money just so we can continue doing what we do ;-) . PyCM Donation - - diff --git a/Test/compare_test.py b/Test/compare_test.py index a8dd703d..9c0ee2e1 100644 --- a/Test/compare_test.py +++ b/Test/compare_test.py @@ -18,7 +18,7 @@ True >>> cp pycm.Compare(classes: [0, 1, 2]) ->>> cp.scores == {'model2': {'overall': 0.33056, 'class': 0.33611}, 'model1': {'overall': 0.425, 'class': 0.50278}} +>>> cp.scores == {'model2': {'overall': 0.52857, 'class': 0.33611}, 'model1': {'overall': 0.58095, 'class': 0.50278}} True >>> cp.best pycm.ConfusionMatrix(classes: [0, 1, 2]) @@ -27,16 +27,16 @@ >>> print(cp) Best : model1 -Rank Name Class-Score Overall-Score -1 model1 0.50278 0.425 -2 model2 0.33611 0.33056 +Rank Name Class-Score Overall-Score +1 model1 0.50278 0.58095 +2 model2 0.33611 0.52857 >>> cp.print_report() Best : model1 -Rank Name Class-Score Overall-Score -1 model1 0.50278 0.425 -2 model2 0.33611 0.33056 +Rank Name Class-Score Overall-Score +1 model1 0.50278 0.58095 +2 model2 0.33611 0.52857 >>> class_weight = {0:5,1:1,2:1} >>> class_weight_copy = {0:5,1:1,2:1} @@ -46,9 +46,9 @@ >>> print(cp) Best : model2 -Rank Name Class-Score Overall-Score -1 model2 0.45357 0.33056 -2 model1 0.34881 0.425 +Rank Name Class-Score Overall-Score +1 model2 0.45357 0.52857 +2 model1 0.34881 0.58095 >>> cp.best pycm.ConfusionMatrix(classes: [0, 1, 2]) @@ -56,7 +56,7 @@ 'model2' >>> with warns(RuntimeWarning, match='Confusion matrices are too close'): ... cp2 = Compare({"model1":cm_comp1,"model2":cm_comp1}) ->>> cp2.scores == {'model2': {'class': 0.50278, 'overall': 0.425}, 'model1': {'class': 0.50278, 'overall': 0.425}} +>>> cp2.scores == {'model2': {'class': 0.50278, 'overall': 0.58095}, 'model1': {'class': 0.50278, 'overall': 0.58095}} True >>> cp2.best >>> cp2.best_name @@ -66,9 +66,9 @@ >>> print(cp3) Best : cm2 -Rank Name Class-Score Overall-Score -1 cm2 0.70556 0.96667 -2 cm1 0.55 0.74722 +Rank Name Class-Score Overall-Score +1 cm2 0.70556 0.92381 +2 cm1 0.55 0.75238 >>> cp4 = Compare({"cm1":cm1,"cm2":cm2},class_weight={0:0,1:1,2:1}) >>> cp4.class_weight == {0:0,1:1,2:1} @@ -76,9 +76,9 @@ >>> print(cp4) Best : cm2 -Rank Name Class-Score Overall-Score -1 cm2 0.825 0.96667 -2 cm1 0.575 0.74722 +Rank Name Class-Score Overall-Score +1 cm2 0.825 0.92381 +2 cm1 0.575 0.75238 >>> class_benchmark_weight = {"PLRI":0,"NLRI":0,"DPI":0,"AUCI":1,"MCCI":0,"QI":0} >>> cp5 = Compare({"cm1":cm1,"cm2":cm2},class_benchmark_weight=class_benchmark_weight) @@ -88,10 +88,10 @@ Best : cm2 Rank Name Class-Score Overall-Score -1 cm2 0.93333 0.96667 -2 cm1 0.73333 0.74722 +1 cm2 0.93333 0.92381 +2 cm1 0.73333 0.75238 ->>> overall_benchmark_weight = {"SOA1":1,"SOA2":0,"SOA3":0,"SOA4":0,"SOA5":0,"SOA6":1} +>>> overall_benchmark_weight = {"SOA1":1,"SOA2":0,"SOA3":0,"SOA4":0,"SOA5":0,"SOA6":1,"SOA7":0,"SOA8":0,"SOA9":0,"SOA10":0} >>> cp6 = Compare({"cm1":cm1,"cm2":cm2},class_benchmark_weight=class_benchmark_weight,overall_benchmark_weight=overall_benchmark_weight) >>> cp6.overall_benchmark_weight == overall_benchmark_weight True @@ -109,9 +109,9 @@ >>> print(cp7) Best : None -Rank Name Class-Score Overall-Score -1 cm1 0.50074 0.74722 -2 cm2 0.47021 0.96667 +Rank Name Class-Score Overall-Score +1 cm1 0.50074 0.75238 +2 cm2 0.47021 0.92381 >>> cp7.best >>> cp7.best_name @@ -119,6 +119,7 @@ >>> y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] >>> cm = ConfusionMatrix(y_actu, y_pred) >>> cm.relabel({0:"L1",1:"L2",2:"L3"}) +>>> cm_null = ConfusionMatrix(matrix={0:{0:0,1:0,2:0},1:{0:0,1:0,2:0},2:{0:0,1:0,2:0}}) >>> with warns(RuntimeWarning, match='Confusion matrices are too close'): ... cp8 = Compare({"cm1":cm,"cm2":cm},class_weight={'L3': 6, 'L1': 3, 'L2': 3}) >>> with warns(RuntimeWarning, match='The class_weight format is wrong, the result is for unweighted mode.'): @@ -126,7 +127,9 @@ >>> class_benchmark_weight = {"PLRI":0,"NLRI":0,"DPI":0,"AUCI":0,"MCCI":0,"QI":0} >>> with warns(RuntimeWarning, match='The class_benchmark_weight format is wrong, the result is for unweighted mode.'): ... cp10 = Compare({"cm1":cm1,"cm2":cm2},class_benchmark_weight=class_benchmark_weight) ->>> overall_benchmark_weight = {"SOA1":0,"SOA2":0,"SOA3":0,"SOA4":0,"SOA5":0,"SOA6":0} +>>> overall_benchmark_weight = {"SOA1":0,"SOA2":0,"SOA3":0,"SOA4":0,"SOA5":0,"SOA6":0,"SOA7":0,"SOA8":0,"SOA9":0,"SOA10":0} >>> with warns(RuntimeWarning, match='The overall_benchmark_weight format is wrong, the result is for unweighted mode.'): ... cp11 = Compare({"cm1":cm1,"cm2":cm2},overall_benchmark_weight=overall_benchmark_weight) +>>> with warns(RuntimeWarning, match='Confusion matrices are too close'): +... cp12 = Compare({"cm1":cm_null,"cm2":cm_null}) """ diff --git a/Test/error_test.py b/Test/error_test.py index b57b7fe2..512b5b7e 100644 --- a/Test/error_test.py +++ b/Test/error_test.py @@ -53,6 +53,10 @@ >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] >>> y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] >>> cm = ConfusionMatrix(y_actu,y_pred) +>>> cm.distance(metric = 2) +Traceback (most recent call last): + ... +pycm.pycm_error.pycmMatrixError: The metric type must be DistanceType >>> cm.relabel([1,2,3]) Traceback (most recent call last): ... diff --git a/Test/function_test.py b/Test/function_test.py index 116e9d4f..1769f6b0 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """ >>> from pycm import * +>>> from pycm.pycm_distance import DISTANCE_MAPPER >>> import os >>> import json >>> import numpy as np @@ -154,22 +155,26 @@ 108-SOA4(Cicchetti) 109-SOA5(Cramer) 110-SOA6(Matthews) -111-Scott PI -112-Standard Error -113-TN -114-TNR -115-TNR Macro -116-TNR Micro -117-TON -118-TOP -119-TP -120-TPR -121-TPR Macro -122-TPR Micro -123-Y -124-Zero-one Loss -125-dInd -126-sInd +111-SOA7(Lambda A) +112-SOA8(Lambda B) +113-SOA9(Krippendorff Alpha) +114-SOA10(Pearson C) +115-Scott PI +116-Standard Error +117-TN +118-TNR +119-TNR Macro +120-TNR Micro +121-TON +122-TOP +123-TP +124-TPR +125-TPR Macro +126-TPR Micro +127-Y +128-Zero-one Loss +129-dInd +130-sInd >>> online_help("J") ... >>> online_help("J",alt_link=True) @@ -413,6 +418,9 @@ [5, 10] >>> POS['L3']['FN'] [0, 3, 7] +>>> POS = cm.position() +>>> POS == cm.positions +True >>> y_actu = [0, 0, 1, 1, 0] >>> y_pred = [0, 1, 1, 0, 0] >>> cm2 = ConfusionMatrix(actual_vector=y_actu, predict_vector=y_pred) @@ -567,6 +575,36 @@ 'Excellent' >>> kappa_analysis_cicchetti(1.2) 'None' +>>> lambda_analysis(0) +'None' +>>> lambda_analysis(0.1) +'Very Weak' +>>> lambda_analysis(0.3) +'Weak' +>>> lambda_analysis(0.5) +'Moderate' +>>> lambda_analysis(0.7) +'Strong' +>>> lambda_analysis(0.9) +'Very Strong' +>>> lambda_analysis(1) +'Perfect' +>>> alpha_analysis(0) +'Low' +>>> alpha_analysis(0.667) +'Tentative' +>>> alpha_analysis(0.8) +'High' +>>> pearson_C_analysis(0) +'None' +>>> pearson_C_analysis(0.05) +'Not Appreciable' +>>> pearson_C_analysis(0.1) +'Weak' +>>> pearson_C_analysis(0.2) +'Medium' +>>> pearson_C_analysis(0.3) +'Strong' >>> PLR_analysis("None") 'None' >>> PLR_analysis(1) @@ -717,4 +755,9 @@ >>> cm4.to_array() array([[3, 1], [0, 0]]) +>>> result = [] +>>> for item in DISTANCE_MAPPER.values(): +... result.append(item(TP=2, TN=2, FP=1, FN="2")) +>>> all(list(map(lambda x: x=="None", result))) +True """ diff --git a/Test/output_test.py b/Test/output_test.py index 510c5a5e..c19961bd 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -123,136 +123,140 @@ Overall Statistics : -95% CI (0.14096,0.55904) -ACC Macro 0.675 -ARI 0.02298 -AUNP None -AUNU None -Bangdiwala B 0.31356 -Bennett S 0.13333 -CBA 0.17708 -CSI None -Chi-Squared None -Chi-Squared DF 9 -Conditional Entropy 1.23579 -Cramer V None -Cross Entropy 1.70995 -F1 Macro 0.23043 -F1 Micro 0.35 -FNR Macro None -FNR Micro 0.65 -FPR Macro 0.21471 -FPR Micro 0.21667 -Gwet AC1 0.19505 -Hamming Loss 0.65 -Joint Entropy 2.11997 -KL Divergence None -Kappa 0.07801 -Kappa 95% CI (-0.2185,0.37453) -Kappa No Prevalence -0.3 -Kappa Standard Error 0.15128 -Kappa Unbiased -0.12554 -Krippendorff Alpha -0.0974 -Lambda A 0.0 -Lambda B 0.0 -Mutual Information 0.10088 -NIR 0.8 -Overall ACC 0.35 -Overall CEN 0.3648 -Overall J (0.60294,0.15074) -Overall MCC 0.12642 -Overall MCEN 0.37463 -Overall RACC 0.295 -Overall RACCU 0.4225 -P-Value 1.0 -PPV Macro None -PPV Micro 0.35 -Pearson C None -Phi-Squared None -RCI 0.11409 -RR 5.0 -Reference Entropy 0.88418 -Response Entropy 1.33667 -SOA1(Landis & Koch) Slight -SOA2(Fleiss) Poor -SOA3(Altman) Poor -SOA4(Cicchetti) Poor -SOA5(Cramer) None -SOA6(Matthews) Negligible -Scott PI -0.12554 -Standard Error 0.10665 -TNR Macro 0.78529 -TNR Micro 0.78333 -TPR Macro None -TPR Micro 0.35 -Zero-one Loss 13 +95% CI (0.14096,0.55904) +ACC Macro 0.675 +ARI 0.02298 +AUNP None +AUNU None +Bangdiwala B 0.31356 +Bennett S 0.13333 +CBA 0.17708 +CSI None +Chi-Squared None +Chi-Squared DF 9 +Conditional Entropy 1.23579 +Cramer V None +Cross Entropy 1.70995 +F1 Macro 0.23043 +F1 Micro 0.35 +FNR Macro None +FNR Micro 0.65 +FPR Macro 0.21471 +FPR Micro 0.21667 +Gwet AC1 0.19505 +Hamming Loss 0.65 +Joint Entropy 2.11997 +KL Divergence None +Kappa 0.07801 +Kappa 95% CI (-0.2185,0.37453) +Kappa No Prevalence -0.3 +Kappa Standard Error 0.15128 +Kappa Unbiased -0.12554 +Krippendorff Alpha -0.0974 +Lambda A 0.0 +Lambda B 0.0 +Mutual Information 0.10088 +NIR 0.8 +Overall ACC 0.35 +Overall CEN 0.3648 +Overall J (0.60294,0.15074) +Overall MCC 0.12642 +Overall MCEN 0.37463 +Overall RACC 0.295 +Overall RACCU 0.4225 +P-Value 1.0 +PPV Macro None +PPV Micro 0.35 +Pearson C None +Phi-Squared None +RCI 0.11409 +RR 5.0 +Reference Entropy 0.88418 +Response Entropy 1.33667 +SOA1(Landis & Koch) Slight +SOA2(Fleiss) Poor +SOA3(Altman) Poor +SOA4(Cicchetti) Poor +SOA5(Cramer) None +SOA6(Matthews) Negligible +SOA7(Lambda A) None +SOA8(Lambda B) None +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) None +Scott PI -0.12554 +Standard Error 0.10665 +TNR Macro 0.78529 +TNR Micro 0.78333 +TPR Macro None +TPR Micro 0.35 +Zero-one Loss 13 Class Statistics : -Classes 100 200 500 600 -ACC(Accuracy) 0.45 0.45 0.85 0.95 -AGF(Adjusted F-score) 0.0 0.33642 0.56659 0.0 -AGM(Adjusted geometric mean) None 0.56694 0.7352 0 -AM(Difference between automatic and manual classification) 11 -9 -1 -1 -AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 -AUCI(AUC value interpretation) None Poor Fair Poor -AUPR(Area under the PR curve) None 0.61607 0.41667 None -BB(Braun-Blanquet similarity) 0.0 0.375 0.33333 0.0 -BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 -BM(Informedness or bookmaker informedness) None 0.125 0.27451 0.0 -CEN(Confusion entropy) 0.33496 0.35708 0.53895 0.0 -DOR(Diagnostic odds ratio) None 1.8 8.0 None -DP(Discriminant power) None 0.14074 0.4979 None -DPI(Discriminant power interpretation) None Poor Poor None -ERR(Error rate) 0.55 0.55 0.15 0.05 -F0.5(F0.5 score) 0.0 0.68182 0.45455 0.0 -F1(F1 score - harmonic mean of precision and sensitivity) 0.0 0.52174 0.4 0.0 -F2(F2 score) 0.0 0.42254 0.35714 0.0 -FDR(False discovery rate) 1.0 0.14286 0.5 None -FN(False negative/miss/type 2 error) 0 10 2 1 -FNR(Miss rate or false negative rate) None 0.625 0.66667 1.0 -FOR(False omission rate) 0.0 0.76923 0.11111 0.05 -FP(False positive/type 1 error/false alarm) 11 1 1 0 -FPR(Fall-out or false positive rate) 0.55 0.25 0.05882 0.0 -G(G-measure geometric mean of precision and sensitivity) None 0.56695 0.40825 None -GI(Gini index) None 0.125 0.27451 0.0 -GM(G-mean geometric mean of specificity and sensitivity) None 0.53033 0.56011 0.0 -HD(Hamming distance) 11 11 3 1 -IBA(Index of balanced accuracy) None 0.17578 0.12303 0.0 -ICSI(Individual classification success index) None 0.23214 -0.16667 None -IS(Information score) None 0.09954 1.73697 None -J(Jaccard index) 0.0 0.35294 0.25 0.0 -LS(Lift score) None 1.07143 3.33333 None -MCC(Matthews correlation coefficient) None 0.10483 0.32673 None -MCCI(Matthews correlation coefficient interpretation) None Negligible Weak None -MCEN(Modified confusion entropy) 0.33496 0.37394 0.58028 0.0 -MK(Markedness) 0.0 0.08791 0.38889 None -N(Condition negative) 20 4 17 19 -NLR(Negative likelihood ratio) None 0.83333 0.70833 1.0 -NLRI(Negative likelihood ratio interpretation) None Negligible Negligible Negligible -NPV(Negative predictive value) 1.0 0.23077 0.88889 0.95 -OC(Overlap coefficient) None 0.85714 0.5 None -OOC(Otsuka-Ochiai coefficient) None 0.56695 0.40825 None -OP(Optimized precision) None 0.11667 0.37308 -0.05 -P(Condition positive or support) 0 16 3 1 -PLR(Positive likelihood ratio) None 1.5 5.66667 None -PLRI(Positive likelihood ratio interpretation) None Poor Fair None -POP(Population) 20 20 20 20 -PPV(Precision or positive predictive value) 0.0 0.85714 0.5 None -PRE(Prevalence) 0.0 0.8 0.15 0.05 -Q(Yule Q - coefficient of colligation) None 0.28571 0.77778 None -QI(Yule Q interpretation) None Weak Strong None -RACC(Random accuracy) 0.0 0.28 0.015 0.0 -RACCU(Random accuracy unbiased) 0.07563 0.33062 0.01562 0.00063 -TN(True negative/correct rejection) 9 3 16 19 -TNR(Specificity or true negative rate) 0.45 0.75 0.94118 1.0 -TON(Test outcome negative) 9 13 18 20 -TOP(Test outcome positive) 11 7 2 0 -TP(True positive/hit) 0 6 1 0 -TPR(Sensitivity, recall, hit rate, or true positive rate) None 0.375 0.33333 0.0 -Y(Youden index) None 0.125 0.27451 0.0 -dInd(Distance index) None 0.67315 0.66926 1.0 -sInd(Similarity index) None 0.52401 0.52676 0.29289 +Classes 100 200 500 600 +ACC(Accuracy) 0.45 0.45 0.85 0.95 +AGF(Adjusted F-score) 0.0 0.33642 0.56659 0.0 +AGM(Adjusted geometric mean) None 0.56694 0.7352 0 +AM(Difference between automatic and manual classification) 11 -9 -1 -1 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 +AUCI(AUC value interpretation) None Poor Fair Poor +AUPR(Area under the PR curve) None 0.61607 0.41667 None +BB(Braun-Blanquet similarity) 0.0 0.375 0.33333 0.0 +BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 +BM(Informedness or bookmaker informedness) None 0.125 0.27451 0.0 +CEN(Confusion entropy) 0.33496 0.35708 0.53895 0.0 +DOR(Diagnostic odds ratio) None 1.8 8.0 None +DP(Discriminant power) None 0.14074 0.4979 None +DPI(Discriminant power interpretation) None Poor Poor None +ERR(Error rate) 0.55 0.55 0.15 0.05 +F0.5(F0.5 score) 0.0 0.68182 0.45455 0.0 +F1(F1 score - harmonic mean of precision and sensitivity) 0.0 0.52174 0.4 0.0 +F2(F2 score) 0.0 0.42254 0.35714 0.0 +FDR(False discovery rate) 1.0 0.14286 0.5 None +FN(False negative/miss/type 2 error) 0 10 2 1 +FNR(Miss rate or false negative rate) None 0.625 0.66667 1.0 +FOR(False omission rate) 0.0 0.76923 0.11111 0.05 +FP(False positive/type 1 error/false alarm) 11 1 1 0 +FPR(Fall-out or false positive rate) 0.55 0.25 0.05882 0.0 +G(G-measure geometric mean of precision and sensitivity) None 0.56695 0.40825 None +GI(Gini index) None 0.125 0.27451 0.0 +GM(G-mean geometric mean of specificity and sensitivity) None 0.53033 0.56011 0.0 +HD(Hamming distance) 11 11 3 1 +IBA(Index of balanced accuracy) None 0.17578 0.12303 0.0 +ICSI(Individual classification success index) None 0.23214 -0.16667 None +IS(Information score) None 0.09954 1.73697 None +J(Jaccard index) 0.0 0.35294 0.25 0.0 +LS(Lift score) None 1.07143 3.33333 None +MCC(Matthews correlation coefficient) None 0.10483 0.32673 None +MCCI(Matthews correlation coefficient interpretation) None Negligible Weak None +MCEN(Modified confusion entropy) 0.33496 0.37394 0.58028 0.0 +MK(Markedness) 0.0 0.08791 0.38889 None +N(Condition negative) 20 4 17 19 +NLR(Negative likelihood ratio) None 0.83333 0.70833 1.0 +NLRI(Negative likelihood ratio interpretation) None Negligible Negligible Negligible +NPV(Negative predictive value) 1.0 0.23077 0.88889 0.95 +OC(Overlap coefficient) None 0.85714 0.5 None +OOC(Otsuka-Ochiai coefficient) None 0.56695 0.40825 None +OP(Optimized precision) None 0.11667 0.37308 -0.05 +P(Condition positive or support) 0 16 3 1 +PLR(Positive likelihood ratio) None 1.5 5.66667 None +PLRI(Positive likelihood ratio interpretation) None Poor Fair None +POP(Population) 20 20 20 20 +PPV(Precision or positive predictive value) 0.0 0.85714 0.5 None +PRE(Prevalence) 0.0 0.8 0.15 0.05 +Q(Yule Q - coefficient of colligation) None 0.28571 0.77778 None +QI(Yule Q interpretation) None Weak Strong None +RACC(Random accuracy) 0.0 0.28 0.015 0.0 +RACCU(Random accuracy unbiased) 0.07563 0.33062 0.01562 0.00063 +TN(True negative/correct rejection) 9 3 16 19 +TNR(Specificity or true negative rate) 0.45 0.75 0.94118 1.0 +TON(Test outcome negative) 9 13 18 20 +TOP(Test outcome positive) 11 7 2 0 +TP(True positive/hit) 0 6 1 0 +TPR(Sensitivity, recall, hit rate, or true positive rate) None 0.375 0.33333 0.0 +Y(Youden index) None 0.125 0.27451 0.0 +dInd(Distance index) None 0.67315 0.66926 1.0 +sInd(Similarity index) None 0.52401 0.52676 0.29289 >>> cm_stat_file=ConfusionMatrix(file=open("test_stat.obj","r")) >>> cm_no_vectors_file=ConfusionMatrix(file=open("test_no_vectors.obj","r")) @@ -345,136 +349,140 @@ >>> cm_file_3.stat() Overall Statistics : -95% CI (0.41134,0.82675) -ACC Macro 0.74603 -ARI 0.15323 -AUNP 0.7 -AUNU 0.70556 -Bangdiwala B 0.44242 -Bennett S 0.42857 -CBA 0.47778 -CSI 0.17222 -Chi-Squared 10.44167 -Chi-Squared DF 4 -Conditional Entropy 0.96498 -Cramer V 0.49861 -Cross Entropy 1.50249 -F1 Macro 0.56111 -F1 Micro 0.61905 -FNR Macro 0.38889 -FNR Micro 0.38095 -FPR Macro 0.2 -FPR Micro 0.19048 -Gwet AC1 0.45277 -Hamming Loss 0.38095 -Joint Entropy 2.34377 -KL Divergence 0.1237 -Kappa 0.3913 -Kappa 95% CI (0.05943,0.72318) -Kappa No Prevalence 0.2381 -Kappa Standard Error 0.16932 -Kappa Unbiased 0.37313 -Krippendorff Alpha 0.38806 -Lambda A 0.22222 -Lambda B 0.36364 -Mutual Information 0.47618 -NIR 0.57143 -Overall ACC 0.61905 -Overall CEN 0.43947 -Overall J (1.22857,0.40952) -Overall MCC 0.41558 -Overall MCEN 0.50059 -Overall RACC 0.37415 -Overall RACCU 0.39229 -P-Value 0.41709 -PPV Macro 0.56111 -PPV Micro 0.61905 -Pearson C 0.57628 -Phi-Squared 0.49722 -RCI 0.34536 -RR 7.0 -Reference Entropy 1.37878 -Response Entropy 1.44117 -SOA1(Landis & Koch) Fair -SOA2(Fleiss) Poor -SOA3(Altman) Fair -SOA4(Cicchetti) Poor -SOA5(Cramer) Relatively Strong -SOA6(Matthews) Weak -Scott PI 0.37313 -Standard Error 0.10597 -TNR Macro 0.8 -TNR Micro 0.80952 -TPR Macro 0.61111 -TPR Micro 0.61905 -Zero-one Loss 8 +95% CI (0.41134,0.82675) +ACC Macro 0.74603 +ARI 0.15323 +AUNP 0.7 +AUNU 0.70556 +Bangdiwala B 0.44242 +Bennett S 0.42857 +CBA 0.47778 +CSI 0.17222 +Chi-Squared 10.44167 +Chi-Squared DF 4 +Conditional Entropy 0.96498 +Cramer V 0.49861 +Cross Entropy 1.50249 +F1 Macro 0.56111 +F1 Micro 0.61905 +FNR Macro 0.38889 +FNR Micro 0.38095 +FPR Macro 0.2 +FPR Micro 0.19048 +Gwet AC1 0.45277 +Hamming Loss 0.38095 +Joint Entropy 2.34377 +KL Divergence 0.1237 +Kappa 0.3913 +Kappa 95% CI (0.05943,0.72318) +Kappa No Prevalence 0.2381 +Kappa Standard Error 0.16932 +Kappa Unbiased 0.37313 +Krippendorff Alpha 0.38806 +Lambda A 0.22222 +Lambda B 0.36364 +Mutual Information 0.47618 +NIR 0.57143 +Overall ACC 0.61905 +Overall CEN 0.43947 +Overall J (1.22857,0.40952) +Overall MCC 0.41558 +Overall MCEN 0.50059 +Overall RACC 0.37415 +Overall RACCU 0.39229 +P-Value 0.41709 +PPV Macro 0.56111 +PPV Micro 0.61905 +Pearson C 0.57628 +Phi-Squared 0.49722 +RCI 0.34536 +RR 7.0 +Reference Entropy 1.37878 +Response Entropy 1.44117 +SOA1(Landis & Koch) Fair +SOA2(Fleiss) Poor +SOA3(Altman) Fair +SOA4(Cicchetti) Poor +SOA5(Cramer) Relatively Strong +SOA6(Matthews) Weak +SOA7(Lambda A) Weak +SOA8(Lambda B) Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong +Scott PI 0.37313 +Standard Error 0.10597 +TNR Macro 0.8 +TNR Micro 0.80952 +TPR Macro 0.61111 +TPR Micro 0.61905 +Zero-one Loss 8 Class Statistics : -Classes 0 1 2 -ACC(Accuracy) 0.80952 0.80952 0.61905 -AGF(Adjusted F-score) 0.90694 0.54433 0.55442 -AGM(Adjusted geometric mean) 0.80509 0.70336 0.66986 -AM(Difference between automatic and manual classification) 4 0 -4 -AUC(Area under the ROC curve) 0.86667 0.61111 0.63889 -AUCI(AUC value interpretation) Very Good Fair Fair -AUPR(Area under the PR curve) 0.8 0.33333 0.625 -BB(Braun-Blanquet similarity) 0.6 0.33333 0.5 -BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 -BM(Informedness or bookmaker informedness) 0.73333 0.22222 0.27778 -CEN(Confusion entropy) 0.25 0.52832 0.56439 -DOR(Diagnostic odds ratio) None 4.0 3.5 -DP(Discriminant power) None 0.33193 0.29996 -DPI(Discriminant power interpretation) None Poor Poor -ERR(Error rate) 0.19048 0.19048 0.38095 -F0.5(F0.5 score) 0.65217 0.33333 0.68182 -F1(F1 score - harmonic mean of precision and sensitivity) 0.75 0.33333 0.6 -F2(F2 score) 0.88235 0.33333 0.53571 -FDR(False discovery rate) 0.4 0.66667 0.25 -FN(False negative/miss/type 2 error) 0 2 6 -FNR(Miss rate or false negative rate) 0.0 0.66667 0.5 -FOR(False omission rate) 0.0 0.11111 0.46154 -FP(False positive/type 1 error/false alarm) 4 2 2 -FPR(Fall-out or false positive rate) 0.26667 0.11111 0.22222 -G(G-measure geometric mean of precision and sensitivity) 0.7746 0.33333 0.61237 -GI(Gini index) 0.73333 0.22222 0.27778 -GM(G-mean geometric mean of specificity and sensitivity) 0.85635 0.54433 0.62361 -HD(Hamming distance) 4 4 8 -IBA(Index of balanced accuracy) 0.92889 0.13169 0.28086 -ICSI(Individual classification success index) 0.6 -0.33333 0.25 -IS(Information score) 1.07039 1.22239 0.39232 -J(Jaccard index) 0.6 0.2 0.42857 -LS(Lift score) 2.1 2.33333 1.3125 -MCC(Matthews correlation coefficient) 0.66332 0.22222 0.28307 -MCCI(Matthews correlation coefficient interpretation) Moderate Negligible Negligible -MCEN(Modified confusion entropy) 0.26439 0.52877 0.65924 -MK(Markedness) 0.6 0.22222 0.28846 -N(Condition negative) 15 18 9 -NLR(Negative likelihood ratio) 0.0 0.75 0.64286 -NLRI(Negative likelihood ratio interpretation) Good Negligible Negligible -NPV(Negative predictive value) 1.0 0.88889 0.53846 -OC(Overlap coefficient) 1.0 0.33333 0.75 -OOC(Otsuka-Ochiai coefficient) 0.7746 0.33333 0.61237 -OP(Optimized precision) 0.65568 0.35498 0.40166 -P(Condition positive or support) 6 3 12 -PLR(Positive likelihood ratio) 3.75 3.0 2.25 -PLRI(Positive likelihood ratio interpretation) Poor Poor Poor -POP(Population) 21 21 21 -PPV(Precision or positive predictive value) 0.6 0.33333 0.75 -PRE(Prevalence) 0.28571 0.14286 0.57143 -Q(Yule Q - coefficient of colligation) None 0.6 0.55556 -QI(Yule Q interpretation) None Moderate Moderate -RACC(Random accuracy) 0.13605 0.02041 0.21769 -RACCU(Random accuracy unbiased) 0.14512 0.02041 0.22676 -TN(True negative/correct rejection) 11 16 7 -TNR(Specificity or true negative rate) 0.73333 0.88889 0.77778 -TON(Test outcome negative) 11 18 13 -TOP(Test outcome positive) 10 3 8 -TP(True positive/hit) 6 1 6 -TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 0.33333 0.5 -Y(Youden index) 0.73333 0.22222 0.27778 -dInd(Distance index) 0.26667 0.67586 0.54716 -sInd(Similarity index) 0.81144 0.52209 0.6131 +Classes 0 1 2 +ACC(Accuracy) 0.80952 0.80952 0.61905 +AGF(Adjusted F-score) 0.90694 0.54433 0.55442 +AGM(Adjusted geometric mean) 0.80509 0.70336 0.66986 +AM(Difference between automatic and manual classification) 4 0 -4 +AUC(Area under the ROC curve) 0.86667 0.61111 0.63889 +AUCI(AUC value interpretation) Very Good Fair Fair +AUPR(Area under the PR curve) 0.8 0.33333 0.625 +BB(Braun-Blanquet similarity) 0.6 0.33333 0.5 +BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 +BM(Informedness or bookmaker informedness) 0.73333 0.22222 0.27778 +CEN(Confusion entropy) 0.25 0.52832 0.56439 +DOR(Diagnostic odds ratio) None 4.0 3.5 +DP(Discriminant power) None 0.33193 0.29996 +DPI(Discriminant power interpretation) None Poor Poor +ERR(Error rate) 0.19048 0.19048 0.38095 +F0.5(F0.5 score) 0.65217 0.33333 0.68182 +F1(F1 score - harmonic mean of precision and sensitivity) 0.75 0.33333 0.6 +F2(F2 score) 0.88235 0.33333 0.53571 +FDR(False discovery rate) 0.4 0.66667 0.25 +FN(False negative/miss/type 2 error) 0 2 6 +FNR(Miss rate or false negative rate) 0.0 0.66667 0.5 +FOR(False omission rate) 0.0 0.11111 0.46154 +FP(False positive/type 1 error/false alarm) 4 2 2 +FPR(Fall-out or false positive rate) 0.26667 0.11111 0.22222 +G(G-measure geometric mean of precision and sensitivity) 0.7746 0.33333 0.61237 +GI(Gini index) 0.73333 0.22222 0.27778 +GM(G-mean geometric mean of specificity and sensitivity) 0.85635 0.54433 0.62361 +HD(Hamming distance) 4 4 8 +IBA(Index of balanced accuracy) 0.92889 0.13169 0.28086 +ICSI(Individual classification success index) 0.6 -0.33333 0.25 +IS(Information score) 1.07039 1.22239 0.39232 +J(Jaccard index) 0.6 0.2 0.42857 +LS(Lift score) 2.1 2.33333 1.3125 +MCC(Matthews correlation coefficient) 0.66332 0.22222 0.28307 +MCCI(Matthews correlation coefficient interpretation) Moderate Negligible Negligible +MCEN(Modified confusion entropy) 0.26439 0.52877 0.65924 +MK(Markedness) 0.6 0.22222 0.28846 +N(Condition negative) 15 18 9 +NLR(Negative likelihood ratio) 0.0 0.75 0.64286 +NLRI(Negative likelihood ratio interpretation) Good Negligible Negligible +NPV(Negative predictive value) 1.0 0.88889 0.53846 +OC(Overlap coefficient) 1.0 0.33333 0.75 +OOC(Otsuka-Ochiai coefficient) 0.7746 0.33333 0.61237 +OP(Optimized precision) 0.65568 0.35498 0.40166 +P(Condition positive or support) 6 3 12 +PLR(Positive likelihood ratio) 3.75 3.0 2.25 +PLRI(Positive likelihood ratio interpretation) Poor Poor Poor +POP(Population) 21 21 21 +PPV(Precision or positive predictive value) 0.6 0.33333 0.75 +PRE(Prevalence) 0.28571 0.14286 0.57143 +Q(Yule Q - coefficient of colligation) None 0.6 0.55556 +QI(Yule Q interpretation) None Moderate Moderate +RACC(Random accuracy) 0.13605 0.02041 0.21769 +RACCU(Random accuracy unbiased) 0.14512 0.02041 0.22676 +TN(True negative/correct rejection) 11 16 7 +TNR(Specificity or true negative rate) 0.73333 0.88889 0.77778 +TON(Test outcome negative) 11 18 13 +TOP(Test outcome positive) 10 3 8 +TP(True positive/hit) 6 1 6 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 0.33333 0.5 +Y(Youden index) 0.73333 0.22222 0.27778 +dInd(Distance index) 0.26667 0.67586 0.54716 +sInd(Similarity index) 0.81144 0.52209 0.6131 >>> cm = ConfusionMatrix(matrix={1:{1:13182,2:30516},2:{1:5108,2:295593}},transpose=True) # Verified Case >>> save_obj = cm.save_obj("test4",address=False) >>> save_obj=={'Status': True, 'Message': None} diff --git a/Test/overall_test.py b/Test/overall_test.py index a50c4fd9..9088713d 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -84,6 +84,10 @@ SOA4(Cicchetti) Poor SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Very Weak +SOA8(Lambda B) Moderate +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.34426 Standard Error 0.14232 TNR Macro 0.77778 @@ -237,6 +241,10 @@ SOA4(Cicchetti) Poor SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Very Weak +SOA8(Lambda B) Moderate +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.34426 Standard Error 0.14232 TNR Macro 0.77778 @@ -405,6 +413,10 @@ SOA4(Cicchetti) Poor SOA5(Cramer) None SOA6(Matthews) Negligible +SOA7(Lambda A) None +SOA8(Lambda B) None +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) None Scott PI -0.12554 Standard Error 0.10665 TNR Macro 0.78529 @@ -539,6 +551,10 @@ SOA4(Cicchetti) Poor SOA5(Cramer) None SOA6(Matthews) Negligible +SOA7(Lambda A) None +SOA8(Lambda B) None +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) None Scott PI -0.12554 Standard Error 0.10665 TNR Macro 0.78529 @@ -831,6 +847,10 @@ SOA4(Cicchetti) Fair SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Moderate +SOA8(Lambda B) Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.47346 Standard Error 0.09072 TNR Macro 0.82116 @@ -982,6 +1002,10 @@ SOA4(Cicchetti) Fair SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Moderate +SOA8(Lambda B) Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.47346 Standard Error 0.09072 TNR Macro 0.82116 @@ -1135,6 +1159,10 @@ SOA4(Cicchetti) Poor SOA5(Cramer) Relatively Strong SOA6(Matthews) Weak +SOA7(Lambda A) Weak +SOA8(Lambda B) Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) Strong Scott PI 0.37313 Standard Error 0.10597 TNR Macro 0.8 @@ -1415,7 +1443,28 @@ >>> actual = [1,0,0,0,1,2,0,2,1] >>> predict = [1,0,1,1,1,2,0,2,0] >>> cm = ConfusionMatrix(actual,predict) ->>> cm.relabel({0:1,1:2,2:3}) +>>> cm.print_matrix() +Predict 0 1 2 +Actual +0 2 2 0 + +1 1 2 0 + +2 0 0 2 + + +>>> cm.relabel({0:"Z", 1:"A", 2:"B"}) +>>> cm.print_matrix() +Predict Z A B +Actual +Z 2 2 0 + +A 1 2 0 + +B 0 0 2 + + +>>> cm.relabel({"Z":1, "A":2, "B":3}) >>> cm pycm.ConfusionMatrix(classes: [1, 2, 3]) >>> cm.label_map[0] @@ -1552,4 +1601,19 @@ 2 1 2 3 +>>> cm = ConfusionMatrix([1,1,1,1,0],[1,1,1,1,1]) +>>> 1 in cm +True +>>> 0 in cm +True +>>> -1 in cm +False +>>> cm[0][0] +0 +>>> cm[0][1] +1 +>>> cm[1][0] +0 +>>> cm[1][1] +4 """ diff --git a/Test/verified_test.py b/Test/verified_test.py index 3c963640..ffb959d9 100644 --- a/Test/verified_test.py +++ b/Test/verified_test.py @@ -427,4 +427,127 @@ True >>> abs(crv.area(method="midpoint")[2] - 0.2916) < 0.001 True +>>> cm1 = ConfusionMatrix(matrix = {1:{1:2,0:2},0:{0:778,1:2}}) # Verified Case -- (https://bit.ly/3vVMWRT) +>>> cm2 = ConfusionMatrix(matrix = {1:{1:2,0:3},0:{0:775,1:4}}) # Verified Case -- (https://bit.ly/3vVMWRT) +>>> cm1.distance(metric=DistanceType.AMPLE)[1] +0.49743589743589745 +>>> cm2.distance(metric=DistanceType.AMPLE)[1] +0.32947729220222793 +>>> cm1.distance(metric=DistanceType.Anderberg)[1] +0.0 +>>> cm2.distance(metric=DistanceType.Anderberg)[1] +0.0 +>>> cm1.distance(metric=DistanceType.AndresMarzoDelta)[1] +0.9897959183673469 +>>> cm2.distance(metric=DistanceType.AndresMarzoDelta)[1] +0.9822344346552608 +>>> cm1.distance(metric=DistanceType.BaroniUrbaniBuserI)[1] +0.9119837740878104 +>>> cm2.distance(metric=DistanceType.BaroniUrbaniBuserI)[1] +0.8552823175014205 +>>> cm1.distance(metric=DistanceType.BaroniUrbaniBuserII)[1] +0.8239675481756209 +>>> cm2.distance(metric=DistanceType.BaroniUrbaniBuserII)[1] +0.7105646350028408 +>>> cm1.distance(metric=DistanceType.BatageljBren)[1] +0.002570694087403599 +>>> cm2.distance(metric=DistanceType.BatageljBren)[1] +0.007741935483870968 +>>> cm1.distance(metric=DistanceType.BaulieuI)[1] +0.75 +>>> cm2.distance(metric=DistanceType.BaulieuI)[1] +0.8666666666666667 +>>> cm1.distance(metric=DistanceType.BaulieuII)[1] +0.24871959237343852 +>>> cm2.distance(metric=DistanceType.BaulieuII)[1] +0.13213719608444902 +>>> cm1.distance(metric=DistanceType.BaulieuIII)[1] +0.4949500208246564 +>>> cm2.distance(metric=DistanceType.BaulieuIII)[1] +0.4949955747605165 +>>> cm1.distance(metric=DistanceType.BaulieuIV)[1] +-5249.96272285802 +>>> cm2.distance(metric=DistanceType.BaulieuIV)[1] +-5209.561726488335 +>>> cm1.distance(metric=DistanceType.BaulieuV)[1] +0.7142857142857143 +>>> cm2.distance(metric=DistanceType.BaulieuV)[1] +0.8 +>>> cm1.distance(metric=DistanceType.BaulieuVI)[1] +0.5714285714285714 +>>> cm2.distance(metric=DistanceType.BaulieuVI)[1] +0.7 +>>> cm1.distance(metric=DistanceType.BaulieuVII)[1] +0.005050505050505051 +>>> cm2.distance(metric=DistanceType.BaulieuVII)[1] +0.008838383838383838 +>>> cm1.distance(metric=DistanceType.BaulieuVIII)[1] +0.0 +>>> cm2.distance(metric=DistanceType.BaulieuVIII)[1] +1.6269262807163682e-06 +>>> cm1.distance(metric=DistanceType.BaulieuIX)[1] +0.007633587786259542 +>>> cm2.distance(metric=DistanceType.BaulieuIX)[1] +0.012706480304955527 +>>> cm1.distance(metric=DistanceType.BaulieuX)[1] +0.007633587786259542 +>>> cm2.distance(metric=DistanceType.BaulieuX)[1] +0.013959390862944163 +>>> cm1.distance(metric=DistanceType.BaulieuXI)[1] +0.005115089514066497 +>>> cm2.distance(metric=DistanceType.BaulieuXI)[1] +0.008951406649616368 +>>> cm1.distance(metric=DistanceType.BaulieuXII)[1] +0.8 +>>> cm2.distance(metric=DistanceType.BaulieuXII)[1] +0.875 +>>> cm1.distance(metric=DistanceType.BaulieuXIII)[1] +0.2857142857142857 +>>> cm2.distance(metric=DistanceType.BaulieuXIII)[1] +0.4117647058823529 +>>> cm1.distance(metric=DistanceType.BaulieuXIV)[1] +0.75 +>>> cm2.distance(metric=DistanceType.BaulieuXIV)[1] +0.8333333333333334 +>>> cm1.distance(metric=DistanceType.BaulieuXV)[1] +0.75 +>>> cm2.distance(metric=DistanceType.BaulieuXV)[1] +0.8461538461538461 +>>> cm1.distance(metric=DistanceType.BeniniI)[1] +0.49743589743589745 +>>> cm2.distance(metric=DistanceType.BeniniI)[1] +0.3953727506426735 +>>> cm1.distance(metric=DistanceType.BeniniII)[1] +0.49743589743589745 +>>> cm2.distance(metric=DistanceType.BeniniII)[1] +0.3953727506426735 +>>> cm1.distance(metric=DistanceType.Canberra)[1] +0.5 +>>> cm2.distance(metric=DistanceType.Canberra)[1] +0.6363636363636364 +>>> cm1.distance(metric=DistanceType.Clement)[1] +0.5025379382522239 +>>> cm2.distance(metric=DistanceType.Clement)[1] +0.33840586363079933 +>>> cm1.distance(metric=DistanceType.ConsonniTodeschiniI)[1] +0.9992336018090547 +>>> cm2.distance(metric=DistanceType.ConsonniTodeschiniI)[1] +0.998656222829757 +>>> cm1.distance(metric=DistanceType.ConsonniTodeschiniII)[1] +0.7585487129939101 +>>> cm2.distance(metric=DistanceType.ConsonniTodeschiniII)[1] +0.6880377723094788 +>>> cm1.distance(metric=DistanceType.ConsonniTodeschiniIII)[1] +0.16481614417697044 +>>> cm2.distance(metric=DistanceType.ConsonniTodeschiniIII)[1] +0.16481614417697044 +>>> cm1.distance(metric=DistanceType.ConsonniTodeschiniIV)[1] +0.5645750340535797 +>>> cm2.distance(metric=DistanceType.ConsonniTodeschiniIV)[1] +0.47712125471966244 +>>> cm1.distance(metric=DistanceType.ConsonniTodeschiniV)[1] +0.48072545510682463 +>>> cm2.distance(metric=DistanceType.ConsonniTodeschiniV)[1] +0.4003930264973547 + """ diff --git a/Test/warning_test.py b/Test/warning_test.py index 48f7d8d8..efdef909 100644 --- a/Test/warning_test.py +++ b/Test/warning_test.py @@ -111,6 +111,10 @@ SOA4(Cicchetti) Excellent SOA5(Cramer) Very Strong SOA6(Matthews) Strong +SOA7(Lambda A) Strong +SOA8(Lambda B) Strong +SOA9(Krippendorff Alpha) Tentative +SOA10(Pearson C) Strong Scott PI 0.74172 Standard Error 0.11685 TNR Macro 0.97424 @@ -272,6 +276,10 @@ SOA4(Cicchetti) Excellent SOA5(Cramer) Very Strong SOA6(Matthews) Strong +SOA7(Lambda A) Strong +SOA8(Lambda B) Strong +SOA9(Krippendorff Alpha) Tentative +SOA10(Pearson C) Strong Scott PI 0.74172 Standard Error 0.11685 TNR Macro 0.97424 @@ -453,6 +461,10 @@ SOA4(Cicchetti) Fair SOA5(Cramer) None SOA6(Matthews) Weak +SOA7(Lambda A) Moderate +SOA8(Lambda B) Weak +SOA9(Krippendorff Alpha) Low +SOA10(Pearson C) None Scott PI 0.47346 Standard Error 0.09072 TNR Macro 0.86587 diff --git a/dev-requirements.txt b/dev-requirements.txt index 5133d41a..14339570 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ art==5.8 -numpy==1.23.5 +numpy==1.24.1 codecov>=2.0.15 pytest>=4.3.1 pytest-cov>=2.6.1 diff --git a/pycm/__init__.py b/pycm/__init__.py index 7074fd4f..6d185842 100644 --- a/pycm/__init__.py +++ b/pycm/__init__.py @@ -3,6 +3,7 @@ from .pycm_param import PYCM_VERSION, OVERALL_BENCHMARK_LIST, CLASS_BENCHMARK_LIST from .pycm_error import * from .pycm_output import pycm_help, online_help +from .pycm_distance import DistanceType from .pycm_obj import ConfusionMatrix from .pycm_compare import Compare from .pycm_curve import Curve, ROCCurve, PRCurve diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index a4d07b43..3b0f7284 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -314,7 +314,7 @@ def __compare_assign_handler__( compare.classes = list(cm_dict.values())[0].classes compare.class_weight = {k: 1 for k in compare.classes} compare.class_benchmark_weight = {k: 1 for k in CLASS_BENCHMARK_LIST} - compare.overall_benchmark_weight = {k: 1 for k in OVERALL_BENCHMARK_LIST} + compare.overall_benchmark_weight = {k: 0 if k in KAPPA_BENCHMARK_LIST[1:] else 1 for k in OVERALL_BENCHMARK_LIST} compare.digit = digit compare.best = None compare.best_name = None diff --git a/pycm/pycm_curve.py b/pycm/pycm_curve.py index ac712e3e..9a187aa0 100644 --- a/pycm/pycm_curve.py +++ b/pycm/pycm_curve.py @@ -53,7 +53,7 @@ def __init__( :param thresholds: thresholds list :type thresholds: list or numpy array :param sample_weight: sample weights list - :type sample_weight: list + :type sample_weight: list or numpy array """ self.data = {} self.thresholds = [] @@ -93,8 +93,10 @@ def area(self, method="trapezoidal"): dx = numpy.diff(x) if numpy.any(dx < 0) and numpy.any(dx > 0): sort_indices = numpy.argsort(x, kind="mergesort") - self.data[c][self.plot_x_axis] = x = numpy.array(x)[sort_indices].tolist() - self.data[c][self.plot_y_axis] = y = numpy.array(y)[sort_indices].tolist() + self.data[c][self.plot_x_axis] = x = numpy.array(x)[ + sort_indices].tolist() + self.data[c][self.plot_y_axis] = y = numpy.array(y)[ + sort_indices].tolist() if method == "trapezoidal": self.auc[c] = __trapezoidal_numeric_integral__(x, y) elif method == "midpoint": @@ -362,7 +364,7 @@ def __curve_data_filter__(curve): :return: None """ none_warning = False - for c_index, c in enumerate(curve.classes): + for c in curve.classes: data_temp = {curve.plot_x_axis: [], curve.plot_y_axis: []} x_data = curve.data[c][curve.plot_x_axis] y_data = curve.data[c][curve.plot_y_axis] diff --git a/pycm/pycm_distance.py b/pycm/pycm_distance.py new file mode 100644 index 00000000..19298b1b --- /dev/null +++ b/pycm/pycm_distance.py @@ -0,0 +1,713 @@ +# -*- coding: utf-8 -*- +"""Distance/Similarity functions.""" +from __future__ import division +from enum import Enum +import math + + +class DistanceType(Enum): + """ + Distance metric type class. + + >>> pycm.DistanceType.AMPLE + """ + + AMPLE = "AMPLE" + Anderberg = "Anderberg" + AndresMarzoDelta = "AndresMarzoDelta" + BaroniUrbaniBuserI = "BaroniUrbaniBuserI" + BaroniUrbaniBuserII = "BaroniUrbaniBuserII" + BatageljBren = "BatageljBren" + BaulieuI = "BaulieuI" + BaulieuII = "BaulieuII" + BaulieuIII = "BaulieuIII" + BaulieuIV = "BaulieuIV" + BaulieuV = "BaulieuV" + BaulieuVI = "BaulieuVI" + BaulieuVII = "BaulieuVII" + BaulieuVIII = "BaulieuVIII" + BaulieuIX = "BaulieuIX" + BaulieuX = "BaulieuX" + BaulieuXI = "BaulieuXI" + BaulieuXII = "BaulieuXII" + BaulieuXIII = "BaulieuXIII" + BaulieuXIV = "BaulieuXIV" + BaulieuXV = "BaulieuXV" + BeniniI = "BeniniI" + BeniniII = "BeniniII" + Canberra = "Canberra" + Clement = "Clement" + ConsonniTodeschiniI = "ConsonniTodeschiniI" + ConsonniTodeschiniII = "ConsonniTodeschiniII" + ConsonniTodeschiniIII = "ConsonniTodeschiniIII" + ConsonniTodeschiniIV = "ConsonniTodeschiniIV" + ConsonniTodeschiniV = "ConsonniTodeschiniV" + + +def AMPLE_calc(TP, FP, FN, TN): + """ + Calculate AMPLE. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: AMPLE as float + """ + try: + part1 = TP / (TP + FP) + part2 = FN / (FN + TN) + return abs(part1 - part2) + except Exception: + return "None" + + +def Anderberg_calc(TP, FP, FN, TN): + """ + Calculate Anderberg's D. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Anderberg's D as float + """ + try: + part1 = max(TP, FP) + max(FN, TN) + max(TP, FN) + max(FP, TN) + part2 = max(TP + FP, FP + TN) + max(TP + FP, FN + TN) + n = TP + FP + FN + TN + return (part1 - part2) / (2 * n) + except Exception: + return "None" + + +def AndresMarzoDelta_calc(TP, FP, FN, TN): + """ + Calculate Andres & Marzo's Delta. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Andres & Marzo's Delta as float + """ + try: + part1 = TP + TN - 2 * math.sqrt(FP * FN) + n = TP + FP + FN + TN + return part1 / n + except Exception: + return "None" + + +def BaroniUrbaniBuserI_calc(TP, FP, FN, TN): + """ + Calculate Baroni-Urbani & Buser I. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baroni-Urbani & Buser I as float + """ + try: + part1 = math.sqrt(TP * TN) + TP + part2 = part1 + FP + FN + return part1 / part2 + except Exception: + return "None" + + +def BaroniUrbaniBuserII_calc(TP, FP, FN, TN): + """ + Calculate Baroni-Urbani & Buser II. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baroni-Urbani & Buser II as float + """ + try: + part1 = math.sqrt(TP * TN) + TP - FP - FN + part2 = math.sqrt(TP * TN) + TP + FP + FN + return part1 / part2 + except Exception: + return "None" + + +def BatageljBren_calc(TP, FP, FN, TN): + """ + Calculate Batagelj & Bren. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Batagelj & Bren as float + """ + try: + return (FP * FN) / (TP * TN) + except Exception: + return "None" + + +def BaulieuI_calc(TP, FP, FN, TN): + """ + Calculate Baulieu I. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu I as float + """ + try: + part1 = (TP + FP) * (TP + FN) + return (part1 - TP * TP) / part1 + except Exception: + return "None" + + +def BaulieuII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu II. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu II as float + """ + try: + part1 = TP * TP * TN * TN + part2 = (TP + FP) * (TP + FN) * (FP + TN) * (FN + TN) + return part1 / part2 + except Exception: + return "None" + + +def BaulieuIII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu III. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu III as float + """ + try: + n = TP + FP + FN + TN + part1 = n * n - 4 * (TP * TN - FP * FN) + return part1 / (2 * n * n) + except Exception: + return "None" + + +def BaulieuIV_calc(TP, FP, FN, TN): + """ + Calculate Baulieu IV. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu IV as float + """ + try: + n = TP + FP + FN + TN + part1 = FP + FN - (TP + 0.5) * (TN + 0.5) * TN * math.e + return part1 / n + except Exception: + return "None" + + +def BaulieuV_calc(TP, FP, FN, TN): + """ + Calculate Baulieu V. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu V as float + """ + try: + return (FP + FN + 1) / (TP + FP + FN + 1) + except Exception: + return "None" + + +def BaulieuVI_calc(TP, FP, FN, TN): + """ + Calculate Baulieu VI. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu VI as float + """ + try: + return (FP + FN) / (TP + FP + FN + 1) + except Exception: + return "None" + + +def BaulieuVII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu VII. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu VII as float + """ + try: + n = TP + FP + FN + TN + return (FP + FN) / (n + TP * (TP - 4) * (TP - 4)) + except Exception: + return "None" + + +def BaulieuVIII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu VIII. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu VIII as float + """ + try: + n = TP + FP + FN + TN + return ((FP - FN) * (FP - FN)) / (n * n) + except Exception: + return "None" + + +def BaulieuIX_calc(TP, FP, FN, TN): + """ + Calculate Baulieu IX. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu IX as float + """ + try: + return (FP + 2 * FN) / (TP + FP + 2 * FN + TN) + except Exception: + return "None" + + +def BaulieuX_calc(TP, FP, FN, TN): + """ + Calculate Baulieu X. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu X as float + """ + try: + n = TP + FP + FN + TN + max_bc = max(FP, FN) + return (FP + FN + max_bc) / (n + max_bc) + except Exception: + return "None" + + +def BaulieuXI_calc(TP, FP, FN, TN): + """ + Calculate Baulieu XI. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu XI as float + """ + try: + return (FP + FN) / (FP + FN + TN) + except Exception: + return "None" + + +def BaulieuXII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu XII. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu XII as float + """ + try: + return (FP + FN) / (TP + FP + FN - 1) + except Exception: + return "None" + + +def BaulieuXIII_calc(TP, FP, FN, TN): + """ + Calculate Baulieu XIII. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu XIII as float + """ + try: + part2 = TP + FP + FN + TP * (TP - 4) * (TP - 4) + return (FP + FN) / part2 + except Exception: + return "None" + + +def BaulieuXIV_calc(TP, FP, FN, TN): + """ + Calculate Baulieu XIV. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu XIV as float + """ + try: + return (FP + 2 * FN) / (TP + FP + 2 * FN) + except Exception: + return "None" + + +def BaulieuXV_calc(TP, FP, FN, TN): + """ + Calculate Baulieu XV. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Baulieu XV as float + """ + try: + max_bc = max(FP, FN) + return (FP + FN + max_bc) / (TP + FP + FN + max_bc) + except Exception: + return "None" + + +def BeniniI_calc(TP, FP, FN, TN): + """ + Calculate Benini I correlation. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Benini I correlation as float + """ + try: + return (TP * TN - FP * FN) / ((TP + FN) * (FN + TN)) + except Exception: + return "None" + + +def BeniniII_calc(TP, FP, FN, TN): + """ + Calculate Benini II correlation. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Benini II correlation as float + """ + try: + part2 = min((TP + FN) * (FN + TN), (TP + FP) * (FP + TN)) + return (TP * TN - FP * FN) / part2 + except Exception: + return "None" + + +def Canberra_calc(TP, FP, FN, TN): + """ + Calculate Canberra distance. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Canberra distance as float + """ + try: + return (FP + FN) / ((TP + FP) + (TP + FN)) + except Exception: + return "None" + + +def Clement_calc(TP, FP, FN, TN): + """ + Calculate Clement similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Clement similarity as float + """ + try: + n = TP + FP + FN + TN + term1 = (TP / (TP + FP)) * (1 - (TP + FP) / n) + term2 = (TN / (FN + TN)) * (1 - (FN + TN) / n) + return term1 + term2 + except Exception: + return "None" + + +def ConsonniTodeschiniI_calc(TP, FP, FN, TN): + """ + Calculate Consonni & Todeschini I similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Consonni & Todeschini I similarity as float + """ + try: + n = TP + FP + FN + TN + return math.log(1 + TP + TN) / math.log(1 + n) + except Exception: + return "None" + + +def ConsonniTodeschiniII_calc(TP, FP, FN, TN): + """ + Calculate Consonni & Todeschini II similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Consonni & Todeschini II similarity as float + """ + try: + n = TP + FP + FN + TN + part1 = math.log(1 + n) - math.log(1 + FP + FN) + return part1 / math.log(1 + n) + except Exception: + return "None" + + +def ConsonniTodeschiniIII_calc(TP, FP, FN, TN): + """ + Calculate Consonni & Todeschini III similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Consonni & Todeschini III similarity as float + """ + try: + n = TP + FP + FN + TN + return math.log(1 + TP) / math.log(1 + n) + except Exception: + return "None" + + +def ConsonniTodeschiniIV_calc(TP, FP, FN, TN): + """ + Calculate Consonni & Todeschini IV similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Consonni & Todeschini IV similarity as float + """ + try: + return math.log(1 + TP) / math.log(1 + TP + FP + FN) + except Exception: + return "None" + + +def ConsonniTodeschiniV_calc(TP, FP, FN, TN): + """ + Calculate Consonni & Todeschini V similarity. + + :param TP: true positive + :type TP: int + :param TN: true negative + :type TN: int + :param FP: false positive + :type FP: int + :param FN: false negative + :type FN: int + :return: Consonni & Todeschini V similarity as float + """ + try: + n = TP + FP + FN + TN + part1 = math.log(1 + TP * TN) - math.log(1 + FP * FN) + part2 = math.log(1 + n * n / 4) + return part1 / part2 + except Exception: + return "None" + + +DISTANCE_MAPPER = { + DistanceType.AMPLE: AMPLE_calc, + DistanceType.Anderberg: Anderberg_calc, + DistanceType.AndresMarzoDelta: AndresMarzoDelta_calc, + DistanceType.BaroniUrbaniBuserI: BaroniUrbaniBuserI_calc, + DistanceType.BaroniUrbaniBuserII: BaroniUrbaniBuserII_calc, + DistanceType.BatageljBren: BatageljBren_calc, + DistanceType.BaulieuI: BaulieuI_calc, + DistanceType.BaulieuII: BaulieuII_calc, + DistanceType.BaulieuIII: BaulieuIII_calc, + DistanceType.BaulieuIV: BaulieuIV_calc, + DistanceType.BaulieuV: BaulieuV_calc, + DistanceType.BaulieuVI: BaulieuVI_calc, + DistanceType.BaulieuVII: BaulieuVII_calc, + DistanceType.BaulieuVIII: BaulieuVIII_calc, + DistanceType.BaulieuIX: BaulieuIX_calc, + DistanceType.BaulieuX: BaulieuX_calc, + DistanceType.BaulieuXI: BaulieuXI_calc, + DistanceType.BaulieuXII: BaulieuXII_calc, + DistanceType.BaulieuXIII: BaulieuXIII_calc, + DistanceType.BaulieuXIV: BaulieuXIV_calc, + DistanceType.BaulieuXV: BaulieuXV_calc, + DistanceType.BeniniI: BeniniI_calc, + DistanceType.BeniniII: BeniniII_calc, + DistanceType.Canberra: Canberra_calc, + DistanceType.Clement: Clement_calc, + DistanceType.ConsonniTodeschiniI: ConsonniTodeschiniI_calc, + DistanceType.ConsonniTodeschiniII: ConsonniTodeschiniII_calc, + DistanceType.ConsonniTodeschiniIII: ConsonniTodeschiniIII_calc, + DistanceType.ConsonniTodeschiniIV: ConsonniTodeschiniIV_calc, + DistanceType.ConsonniTodeschiniV: ConsonniTodeschiniV_calc, +} diff --git a/pycm/pycm_handler.py b/pycm/pycm_handler.py index 7736731f..87274f24 100644 --- a/pycm/pycm_handler.py +++ b/pycm/pycm_handler.py @@ -151,6 +151,10 @@ def __overall_stat_init__(cm): cm.ARI = cm.overall_stat["ARI"] cm.B = cm.overall_stat["Bangdiwala B"] cm.Alpha = cm.overall_stat["Krippendorff Alpha"] + cm.SOA7 = cm.overall_stat["SOA7(Lambda A)"] + cm.SOA8 = cm.overall_stat["SOA8(Lambda B)"] + cm.SOA9 = cm.overall_stat["SOA9(Krippendorff Alpha)"] + cm.SOA10 = cm.overall_stat["SOA10(Pearson C)"] def __obj_assign_handler__(cm, matrix_param): diff --git a/pycm/pycm_interpret.py b/pycm/pycm_interpret.py index 8e3e1fe3..accaf645 100644 --- a/pycm/pycm_interpret.py +++ b/pycm/pycm_interpret.py @@ -253,3 +253,71 @@ def kappa_analysis_altman(kappa): return "None" except Exception: # pragma: no cover return "None" + + +def lambda_analysis(lambda_): + """ + Analysis of lambda (A or B) value with interpretation table. + + :param lambda_: lambda (A or B) value + :type lambda_ : float + :return: interpretation result as str + """ + try: + if 0 < lambda_ < 0.2: + return "Very Weak" + if 0.2 <= lambda_ < 0.4: + return "Weak" + if 0.4 <= lambda_ < 0.6: + return "Moderate" + if 0.6 <= lambda_ < 0.8: + return "Strong" + if 0.8 <= lambda_ < 1: + return "Very Strong" + if lambda_ == 1: + return "Perfect" + return "None" + except Exception: # pragma: no cover + return "None" + + +def alpha_analysis(alpha): + """ + Analysis of Krippendorff's alpha value with interpretation table. + + :param alpha: Krippendorff's alpha value + :type alpha: float + :return: interpretation result as str + """ + try: + if alpha < 0.667: + return "Low" + if 0.667 <= alpha < 0.8: + return "Tentative" + if alpha >= 0.8: + return "High" + return "None" + except Exception: # pragma: no cover + return "None" + + +def pearson_C_analysis(pearson_C): + """ + Analysis of Pearson's coefficient value with interpretation table. + + :param pearson_C: Pearson's coefficient value + :type pearson_C: float + :return: interpretation result as str + """ + try: + if 0 < pearson_C < 0.1: + return "Not Appreciable" + if 0.1 <= pearson_C < 0.2: + return "Weak" + if 0.2 <= pearson_C < 0.3: + return "Medium" + if pearson_C >= 0.3: + return "Strong" + return "None" + except Exception: # pragma: no cover + return "None" diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index e2072e2f..5bcad7af 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -6,6 +6,7 @@ from .pycm_handler import __obj_assign_handler__, __obj_file_handler__, __obj_matrix_handler__, __obj_vector_handler__, __obj_array_handler__ from .pycm_class_func import F_calc, IBA_calc, TI_calc, NB_calc, sensitivity_index_calc from .pycm_overall_func import weighted_kappa_calc, weighted_alpha_calc, alpha2_calc, brier_score_calc +from .pycm_distance import DistanceType, DISTANCE_MAPPER from .pycm_output import * from .pycm_util import * from .pycm_param import * @@ -217,6 +218,26 @@ def __iter__(self): for key in self.matrix.keys(): yield key, self.matrix[key] + def __contains__(self, class_name): + """ + Check if the confusion matrix contains the given class name. + + :param class_name: given class name + :param class_name: any valid type + :return: True if class_name is a class in confusion matrix + """ + return class_name in self.classes + + def __getitem__(self, class_name): + """ + Return the element(s) in the matrix corresponding to the given class name. + + :param class_name: given class name + :type class_name: any valid type + :return: the element(s) in matrix corresponding to the given class name + """ + return self.matrix[class_name] + def save_stat( self, name, @@ -591,6 +612,22 @@ def NB(self, w=1): except Exception: return {} + def distance(self, metric): + """ + Calculate distance/similarity for all classes. + + :param metric: metric + :type metric: DistanceType + :return: result as dict + """ + distance_dict = {} + if not isinstance(metric, DistanceType): + raise pycmMatrixError(DISTANCE_METRIC_TYPE_ERROR) + for i in self.classes: + distance_dict[i] = DISTANCE_MAPPER[metric]( + TP=self.TP[i], FP=self.FP[i], FN=self.FN[i], TN=self.TN[i]) + return distance_dict + def CI( self, param, @@ -731,7 +768,7 @@ def relabel(self, mapping): temp_label_map[prime_label] = mapping[new_label] self.label_map = temp_label_map self.positions = None - self.classes = sorted(list(mapping.values())) + self.classes = [mapping[x] for x in self.classes] self.TP = self.class_stat["TP"] self.TN = self.class_stat["TN"] self.FP = self.class_stat["FP"] diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 206bde86..e3e1a541 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -3,7 +3,7 @@ from __future__ import division from functools import partial from .pycm_param import * -from .pycm_util import rounder +from .pycm_util import rounder, sort_char_num import webbrowser @@ -171,10 +171,10 @@ def html_overall_stat( result = "" result += "

    Overall Statistics :

    \n" result += '
    JOSS
    Zenodo
    \n' - overall_stat_keys = sorted(overall_stat.keys()) + overall_stat_keys = sort_char_num(overall_stat.keys()) if isinstance(overall_param, list): if set(overall_param) <= set(overall_stat_keys): - overall_stat_keys = sorted(overall_param) + overall_stat_keys = sort_char_num(overall_param) if len(overall_stat_keys) < 1: return "" for i in overall_stat_keys: @@ -423,11 +423,11 @@ def stat_print( """ shift = max(map(len, PARAMS_DESCRIPTION.values())) + 5 classes_len = len(classes) - overall_stat_keys = sorted(overall_stat.keys()) + overall_stat_keys = sort_char_num(overall_stat.keys()) result = "" if isinstance(overall_param, list): if set(overall_param) <= set(overall_stat_keys): - overall_stat_keys = sorted(overall_param) + overall_stat_keys = sort_char_num(overall_param) if len(overall_stat_keys) > 0: result = "Overall Statistics : " + "\n\n" for key in overall_stat_keys: @@ -508,7 +508,7 @@ def online_help(param=None, alt_link=False): document_link = DOCUMENT_ADR if alt_link: document_link = DOCUMENT_ADR_ALT - params_link_keys = sorted(PARAMS_LINK.keys()) + params_link_keys = sort_char_num(PARAMS_LINK.keys()) if param in params_link_keys: webbrowser.open_new_tab(document_link + PARAMS_LINK[param]) elif param in range(1, len(params_link_keys) + 1): @@ -519,5 +519,5 @@ def online_help(param=None, alt_link=False): print('Example : online_help("J") or online_help(2)\n') for index, item in enumerate(params_link_keys): print(str(index + 1) + "-" + item) - except Exception: + except Exception: # pragma: no cover print("Error in online help") diff --git a/pycm/pycm_overall_func.py b/pycm/pycm_overall_func.py index 1a6d2b2f..c83bdcbc 100644 --- a/pycm/pycm_overall_func.py +++ b/pycm/pycm_overall_func.py @@ -1034,6 +1034,10 @@ def overall_statistics(**kwargs): result["SOA4(Cicchetti)"] = kappa_analysis_cicchetti(result["Kappa"]) result["SOA5(Cramer)"] = V_analysis(result["Cramer V"]) result["SOA6(Matthews)"] = MCC_analysis(result["Overall MCC"]) + result["SOA7(Lambda A)"] = lambda_analysis(result["Lambda A"]) + result["SOA8(Lambda B)"] = lambda_analysis(result["Lambda B"]) + result["SOA9(Krippendorff Alpha)"] = alpha_analysis(result["Krippendorff Alpha"]) + result["SOA10(Pearson C)"] = pearson_C_analysis(result["Pearson C"]) result["FPR Macro"] = complement(result["TNR Macro"]) result["FNR Macro"] = complement(result["TPR Macro"]) result["PPV Macro"] = macro_calc(kwargs["PPV"]) diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index f22c99a2..54b4849e 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Parameters and constants.""" -PYCM_VERSION = "3.7" +PYCM_VERSION = "3.8" OVERVIEW = ''' @@ -117,6 +117,8 @@ CURVE_NONE_WARNING = "The curve axes contain non-numerical value(s)." +DISTANCE_METRIC_TYPE_ERROR = "The metric type must be DistanceType" + CLASS_NUMBER_THRESHOLD = 10 BALANCE_RATIO_THRESHOLD = 3 @@ -366,6 +368,42 @@ "None": "None"} +SOA7_SCORE = { + "Perfect": 6, + "Very Strong": 5, + "Strong": 4, + "Moderate": 3, + "Weak": 2, + "Very Weak": 1, + "None": "None" +} + +SOA8_SCORE = { + "Perfect": 6, + "Very Strong": 5, + "Strong": 4, + "Moderate": 3, + "Weak": 2, + "Very Weak": 1, + "None": "None" +} + +SOA9_SCORE = { + "High": 3, + "Tentative": 2, + "Low": 1, + "None": "None" +} + +SOA10_SCORE = { + "Strong": 4, + "Medium": 3, + "Weak": 2, + "Not Appreciable": 1, + "None": "None" +} + + CLASS_BENCHMARK_SCORE_DICT = { "PLRI": PLRI_SCORE, "NLRI": NLRI_SCORE, @@ -382,9 +420,14 @@ "SOA3": SOA3_SCORE, "SOA4": SOA4_SCORE, "SOA5": SOA5_SCORE, - "SOA6": SOA6_SCORE} + "SOA6": SOA6_SCORE, + "SOA7": SOA7_SCORE, + "SOA8": SOA8_SCORE, + "SOA9": SOA9_SCORE, + "SOA10": SOA10_SCORE} OVERALL_BENCHMARK_LIST = sorted(list(OVERALL_BENCHMARK_SCORE_DICT.keys())) +KAPPA_BENCHMARK_LIST = ["SOA1", "SOA2", "SOA3", "SOA4"] OVERALL_BENCHMARK_MAP = { "SOA1": "SOA1(Landis & Koch)", @@ -392,7 +435,12 @@ "SOA3": "SOA3(Altman)", "SOA4": "SOA4(Cicchetti)", "SOA5": "SOA5(Cramer)", - "SOA6": "SOA6(Matthews)"} + "SOA6": "SOA6(Matthews)", + "SOA7": "SOA7(Lambda A)", + "SOA8": "SOA8(Lambda B)", + "SOA9": "SOA9(Krippendorff Alpha)", + "SOA10": "SOA10(Pearson C)" +} RECOMMEND_BACKGROUND_COLOR = "aqua" DEFAULT_BACKGROUND_COLOR = "transparent" @@ -599,7 +647,12 @@ "Bangdiwala B": "Bangdiwala's-B", "Krippendorff Alpha": "Krippendorff's-alpha", "HD": "HD-(Hamming-distance)", - "BB": "BB-(Braun-Blanquet-similarity)"} + "BB": "BB-(Braun-Blanquet-similarity)", + "SOA7(Lambda A)": "SOA7-(Goodman-&-Kruskal's-lambda-A-benchmark)", + "SOA8(Lambda B)": "SOA8-(Goodman-&-Kruskal's-lambda-B-benchmark)", + "SOA9(Krippendorff Alpha)": "SOA9-(Krippendorff's-alpha-benchmark)", + "SOA10(Pearson C)": "SOA10-(Pearson's-C-benchmark)" +} CAPITALIZE_FILTER = [ "BCD", @@ -691,7 +744,35 @@ "Moderate": "Yellow", "Strong": "LawnGreen", "Very Strong": "Green", - "None": "White"}} + "None": "White"}, + "SOA7(Lambda A)": { + "Very Weak": "Red", + "Weak": "OrangeRed", + "Moderate": "Orange", + "Strong": "Yellow", + "Very Strong": "LawnGreen", + "Perfect": "Green", + "None": "White"}, + "SOA8(Lambda B)": { + "Very Weak": "Red", + "Weak": "OrangeRed", + "Moderate": "Orange", + "Strong": "Yellow", + "Very Strong": "LawnGreen", + "Perfect": "Green", + "None": "White"}, + "SOA9(Krippendorff Alpha)": { + "Low": "Red", + "Tentative": "LawnGreen", + "High": "Green", + "None": "White"}, + "SOA10(Pearson C)": { + "Not Appreciable": "Red", + "Weak": "Orange", + "Medium": "LawnGreen", + "Strong": "Green", + "None": "White"} + } BENCHMARK_LIST = list(BENCHMARK_COLOR.keys()) diff --git a/pycm/pycm_util.py b/pycm/pycm_util.py index 4f138bf2..6252ca51 100644 --- a/pycm/pycm_util.py +++ b/pycm/pycm_util.py @@ -4,6 +4,7 @@ import sys import math import numpy +import re from .pycm_param import * from .pycm_error import pycmMatrixError from warnings import warn @@ -716,3 +717,18 @@ def thresholds_calc(probs): thresholds = numpy.ravel(probs) thresholds = sorted(set(thresholds)) return thresholds + + +def sort_char_num(input_list): + """ + Sort a list of strings first alphabetically and then numerically. + + :param input_list: input list of strings + :type input_list: iterable + :return: a sorted list of strings + """ + sort_by = lambda x: [(x, False, False) if not re.findall(r'\d+', x) + else (x[:re.search(r'\d+', x).start()], + int(re.findall(r'\d+', x)[0]), + x[re.search(r'\d+', x).end():])] + return sorted(input_list, key=sort_by) diff --git a/setup.py b/setup.py index 301ecc6f..f69c4ae9 100644 --- a/setup.py +++ b/setup.py @@ -36,14 +36,14 @@ def read_description(): setup( name='pycm', packages=['pycm'], - version='3.7', + version='3.8', description='Multi-class confusion matrix library in Python', long_description=read_description(), long_description_content_type='text/markdown', author='PyCM Development Team', author_email='info@pycm.io', url='https://github.com/sepandhaghighi/pycm', - download_url='https://github.com/sepandhaghighi/pycm/tarball/v3.7', + download_url='https://github.com/sepandhaghighi/pycm/tarball/v3.8', keywords="confusion-matrix python3 python machine_learning ML", project_urls={ 'Webpage': 'https://www.pycm.io',