10-Classification-Performance-Measures/08-Confusion-Matrix-in-Multiclass-Classification/08-Confusion-Matrix-in-Multiclass-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多分类问题中的混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93115438108484005"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Target is multiclass but average='binary'. Please choose another average setting.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-1a98a718d908>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprecision_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprecision_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_predict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/yuanzhang/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py\u001b[0m in \u001b[0;36mprecision_score\u001b[0;34m(y_true, y_pred, labels, pos_label, average, sample_weight)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                                                  \u001b[0maverage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maverage\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                                                  \u001b[0mwarn_for\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'precision'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m                                                  sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m   1240\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/yuanzhang/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py\u001b[0m in \u001b[0;36mprecision_recall_fscore_support\u001b[0;34m(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight)\u001b[0m\n\u001b[1;32m   1016\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m             raise ValueError(\"Target is %s but average='binary'. Please \"\n\u001b[0;32m-> 1018\u001b[0;31m                              \"choose another average setting.\" % y_type)\n\u001b[0m\u001b[1;32m   1019\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mpos_label\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m         warnings.warn(\"Note that pos_label (set to %r) is ignored when \"\n",
      "\u001b[0;31mValueError\u001b[0m: Target is multiclass but average='binary'. Please choose another average setting."
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93115438108484005"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_predict, average=\"micro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[147,   0,   1,   0,   0,   1,   0,   0,   0,   0],\n",
       "       [  0, 123,   1,   2,   0,   0,   0,   3,   4,  10],\n",
       "       [  0,   0, 134,   1,   0,   0,   0,   0,   1,   0],\n",
       "       [  0,   0,   0, 138,   0,   5,   0,   1,   5,   0],\n",
       "       [  2,   5,   0,   0, 139,   0,   0,   3,   0,   1],\n",
       "       [  1,   3,   1,   0,   0, 146,   0,   0,   1,   0],\n",
       "       [  0,   2,   0,   0,   0,   1, 131,   0,   2,   0],\n",
       "       [  0,   0,   0,   1,   0,   0,   0, 132,   1,   2],\n",
       "       [  1,   9,   2,   3,   2,   4,   0,   0, 115,   4],\n",
       "       [  0,   1,   0,   5,   0,   3,   0,   2,   2, 134]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACttJREFUeJzt3U2IXfUZx/HfLzOjY2JjhATETOhkUSwqlMhQokIWRqSt\nUjddWFCom9m0GkUQ7cZlNyK6KEKIdVOxi5hFkaIW1EUXhk5eQJOxINrmVZ0u4qgQMi9PF/cGX5LO\nPQP3uWdunu8HhMx48ufJ5H7n3Htzzn8cEQJQy7q2BwAweIQPFET4QEGEDxRE+EBBhA8U1Fr4tn9m\n+1+2P7L9VFtzNGV7m+13bB+3fcz2nrZnasL2iO0jtl9ve5YmbG+yvd/2h7Znbd/e9ky92H68+5j4\nwPartsfbnqmXVsK3PSLpj5J+LulmSb+2fXMbs6zCoqQnIuJmSTsl/XYIZpakPZJm2x5iFV6Q9EZE\n/FjST7TGZ7e9VdKjkqYi4lZJI5IeaHeq3to64/9U0kcR8XFEXJD0F0n3tzRLIxFxNiIOd3/9pToP\nyK3tTrUy2xOS7pW0r+1ZmrB9naRdkl6SpIi4EBHn2p2qkVFJ19gelbRe0pmW5+mprfC3Sjr5rY9P\naY1H9G22JyXtkHSw3Ul6el7Sk5KW2x6koe2S5iS93H15ss/2hraHWklEnJb0rKQTks5K+iIi3mp3\nqt54c2+VbF8r6TVJj0XEfNvz/D+275P0eUQcanuWVRiVdJukFyNih6SvJa3p939sX6/Os9Xtkm6U\ntMH2g+1O1Vtb4Z+WtO1bH090P7em2R5TJ/pXIuJA2/P0cKekX9r+tzovpe6y/ed2R+rplKRTEXHx\nmdR+db4RrGV3S/okIuYiYkHSAUl3tDxTT22F/09JP7K93fZV6rwZ8teWZmnEttV57TkbEc+1PU8v\nEfF0RExExKQ6X9+3I2JNn4ki4lNJJ23f1P3UbknHWxypiROSdtpe332M7NYaf0NS6jy1GriIWLT9\nO0lvqvMu6J8i4lgbs6zCnZIekvS+7aPdz/0+Iv7W4kxXokckvdI9IXws6eGW51lRRBy0vV/SYXX+\n5eeIpL3tTtWbuS0XqIc394CCCB8oiPCBgggfKIjwgYJaD9/2dNszrMawzSsx8yAM27ythy9pqL5g\nGr55JWYehKGady2ED2DAUi7g2bx5c0xOTjY6dm5uTlu2bGl07KFDw3S/CdCOiHCvY1Iu2Z2cnNTM\nzEzf1+1cCp0ja+1hvDKSr8WVj6f6QEGEDxRE+EBBhA8URPhAQY3CH7Y98AGsrGf4Q7oHPoAVNDnj\nD90e+ABW1iT8od4DH8Cl+vbmnu1p2zO2Z+bm5vq1LIAETcJvtAd+ROyNiKmImGp67T2AdjQJf+j2\nwAewsp436QzpHvgAVtDo7rzuD43gB0cAVwiu3AMKInygIMIHCiJ8oCDCBwpK2WzTdsrmaseP5/2o\n9FtuuSVl3XXr8r63Li0tpa09bMbGxlLWzdznccOGDX1fc35+XouLiz2H5owPFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGE\nDxRE+EBBQ7W9dqbPPvssZd0bbrghZV1Jyvi7w3dlbq+d9fcXEWyvDeBShA8URPhAQYQPFET4QEGE\nDxRE+EBBPcO3vc32O7aP2z5me88gBgOQZ7TBMYuSnoiIw7Z/IOmQ7b9HRN4PqweQqucZPyLORsTh\n7q+/lDQraWv2YADyrOo1vu1JSTskHcwYBsBgNHmqL0myfa2k1yQ9FhHzl/n/05Km+zgbgCSNbtKx\nPSbpdUlvRsRzDY4furtHuEkHl1P2Jh13/uQvSZptEj2Ata/Ja/w7JT0k6S7bR7v//SJ5LgCJer7G\nj4h/SMp7vgNg4LhyDyiI8IGCCB8oiPCBgggfKIhddpOdO3cube1NmzalrDs+Pp6y7vnz51PWlfIu\ntLn66qtT1pXyvh7ssgvgsggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCB\ngggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHygobXvtkZGRvq87NjbW9zUvytz6Ocv8/HzK\nuhs3bkxZN1PWY2NhYSFlXSlnS/CIYHttAJdH+EBBhA8URPhAQYQPFET4QEGEDxTUOHzbI7aP2H49\ncyAA+VZzxt8jaTZrEACD0yh82xOS7pW0L3ccAIPQ9Iz/vKQnJS0nzgJgQHqGb/s+SZ9HxKEex03b\nnrE907fpAKToeZOO7T9IekjSoqRxSRslHYiIB1f4PdykMwDcpPMNbtLp6NtNOhHxdERMRMSkpAck\nvb1S9ADWPv4dHyhodDUHR8S7kt5NmQTAwHDGBwoifKAgwgcKInygIMIHCkrbZTfj4oTR0VX9I8Sq\nLC4upqyb8fXNtrycc2X2unXDd57JeBxflPXYYJddAJdF+EBBhA8URPhAQYQPFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8UlLZtbcYOolm7v0rD\nuRtulowfcS5JJ0+eTFlXkrZt25aybubOwEtLS2lr98IZHyiI8IGCCB8oiPCBgggfKIjwgYIIHyio\nUfi2N9neb/tD27O2b88eDECephfwvCDpjYj4le2rJK1PnAlAsp7h275O0i5Jv5GkiLgg6ULuWAAy\nNXmqv13SnKSXbR+xvc/2huS5ACRqEv6opNskvRgROyR9Lemp7x9ke9r2jO2ZPs8IoM+ahH9K0qmI\nONj9eL863wi+IyL2RsRUREz1c0AA/dcz/Ij4VNJJ2zd1P7Vb0vHUqQCkavqu/iOSXum+o/+xpIfz\nRgKQrVH4EXFUEk/hgSsEV+4BBRE+UBDhAwURPlAQ4QMFET5QkDO2lbbNXtVdttPWZkvwb5w+fTpl\n3YmJiZR1pZytu5eWlhQRPR90nPGBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGC\nCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYLSdtnN2F1248aNfV/zoq+++ipl3Yyd\nVC9aXl5OWXdkZCRl3QsXLqSsm+m9995LW3vXrl19X3NhYUHLy8vssgvgUoQPFET4QEGEDxRE+EBB\nhA8URPhAQY3Ct/247WO2P7D9qu3x7MEA5OkZvu2tkh6VNBURt0oakfRA9mAA8jR9qj8q6Rrbo5LW\nSzqTNxKAbD3Dj4jTkp6VdELSWUlfRMRb2YMByNPkqf71ku6XtF3SjZI22H7wMsdN256xPdP/MQH0\nU5On+ndL+iQi5iJiQdIBSXd8/6CI2BsRUxEx1e8hAfRXk/BPSNppe707t9ztljSbOxaATE1e4x+U\ntF/SYUnvd3/P3uS5ACQabXJQRDwj6ZnkWQAMCFfuAQURPlAQ4QMFET5QEOEDBRE+UFDa9tp9X7Sz\nbsaykqSMr4MkjY/n3cF8/vz5lHXHxsZS1l1YWEhZV8rbEjzTmTP9v9ftnnvu0dGjR9leG8ClCB8o\niPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI\n8IGCCB8oiPCBgrJ22Z2T9J+Gh2+W9N++D5Fn2OaVmHkQ1sq8P4yILb0OSgl/NWzPRMRUq0OswrDN\nKzHzIAzbvDzVBwoifKCgtRD+3rYHWKVhm1di5kEYqnlbf40PYPDWwhkfwIARPlAQ4QMFET5QEOED\nBf0PDFK2BTKTWFsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11954b0f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cfm = confusion_matrix(y_test, y_predict)\n",
    "plt.matshow(cfm, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACvZJREFUeJzt3U+IXfUZxvHnSWaCJhYVppqZRJosikHEkjDWPwEXxkXb\nBEXShQWFuskm1SiCaEQFwZ2IgkUYkrox6CLJokixFtRFN6HJRIjJKIiamPmDyaIq2WTGebuYG4gm\nnXNGzu+ee32/HxDMeHx5CfPNOffm3DOOCAHIZVnbCwDoPsIHEiJ8ICHCBxIifCAhwgcSai1827+z\n/antz2w/1dYeddm+wfYHtk/YPm57V9s71WF7ue2jtt9pe5c6bF9je7/tT2xP2L6j7Z2q2H688z3x\nse23bF/R9k5VWgnf9nJJf5X0e0k3SfqT7Zva2GUJ5iQ9ERE3Sbpd0s4+2FmSdkmaaHuJJXhV0rsR\nsUHSb9Tju9teI+lRSaMRcbOk5ZIeaHeram2d8X8r6bOI+Dwizkt6W9J9Le1SS0RMR8R459+/08I3\n5Jp2t1qc7bWStkra0/Yuddi+WtJdkvZKUkScj4j/trtVLQOSrrQ9IGmlpKmW96nUVvhrJH110a9P\nq8cjupjtdZI2SjrU7iaVXpH0pKT5thepab2kM5Le6Lw82WN7VdtLLSYiJiW9JOmUpGlJ30TEe+1u\nVY0395bI9lWSDkh6LCK+bXuf/8f2NklfR8SRtndZggFJmyS9HhEbJZ2T1NPv/9i+VgtXq+sljUha\nZfvBdreq1lb4k5JuuOjXaztf62m2B7UQ/b6IONj2PhU2S7rX9pdaeCl1t+03212p0mlJpyPiwpXU\nfi38QdDL7pH0RUSciYhZSQcl3dnyTpXaCv8/kn5te73tFVp4M+TvLe1Si21r4bXnRES83PY+VSLi\n6YhYGxHrtPD7+35E9PSZKCJmJH1l+8bOl7ZIOtHiSnWcknS77ZWd75Et6vE3JKWFS6uui4g523+R\n9E8tvAv6t4g43sYuS7BZ0kOSjtn+qPO13RHxjxZ3+jl6RNK+zgnhc0kPt7zPoiLikO39ksa18Dc/\nRyWNtbtVNfOxXCAf3twDEiJ8ICHCBxIifCAhwgcSaj182zva3mEp+m1fiZ27od/2bT18SX31G6b+\n21di527oq317IXwAXVbkBh7bfXdX0ODgYK3j5ufntWxZ/T8vZ2dnf+pKrRkYKHND59zcXJG5+KGI\ncNUxrdyy24uuu+66InMnJ3v+s0eXGBoaKjJ3ZmamyFwsHZf6QEKEDyRE+EBChA8kRPhAQrXC77dn\n4ANYXGX4ffoMfACLqHPG77tn4ANYXJ3w+/oZ+AAu1dide51PJ/XVBxWArOqEX+sZ+BExps7TRfvx\nXn0gkzqX+n33DHwAi6s84/fpM/ABLKLWa/zOD43gB0cAPxPcuQckRPhAQoQPJET4QEKEDyTEM/cK\n27RpU7HZ4+PjReb247Pxtm/fXmTu6tWri8yVpNdee63xmaOjo7WO44wPJET4QEKEDyRE+EBChA8k\nRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE\n+EBCjmj+R9nbbn5onxoeHi42e3p6uthsLBgZGSk2e2pqqsjciHDVMZzxgYQIH0iI8IGECB9IiPCB\nhAgfSIjwgYQqw7d9g+0PbJ+wfdz2rm4sBqCcgRrHzEl6IiLGbf9C0hHb/4qIE4V3A1BI5Rk/IqYj\nYrzz799JmpC0pvRiAMpZ0mt82+skbZR0qMQyALqjzqW+JMn2VZIOSHosIr69zH/fIWlHg7sBKKRW\n+LYHtRD9vog4eLljImJM0ljneD6kA/SwOu/qW9JeSRMR8XL5lQCUVuc1/mZJD0m62/ZHnX/+UHgv\nAAVVXupHxL8lVX6+F0D/4M49ICHCBxIifCAhwgcSInwgIZ6yi0s888wzRea++OKLReZK0po1ZT4+\nsnPnziJzJWn37t1F5vKUXQCXRfhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBC\nhA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEJFHq+9atWq2LBhQ+Nz77///sZnXvDs\ns88Wm43ytm/fXmTugQMHisyVpOHh4cZnnj17VufPn+fx2gAuRfhAQoQPJET4QEKEDyRE+EBChA8k\nVDt828ttH7X9TsmFAJS3lDP+LkkTpRYB0D21wre9VtJWSXvKrgOgG+qe8V+R9KSk+YK7AOiSyvBt\nb5P0dUQcqThuh+3Dtg/Pzc01tiCA5tU542+WdK/tLyW9Lelu22/++KCIGIuI0YgYHRgYaHhNAE2q\nDD8ino6ItRGxTtIDkt6PiAeLbwagGP4eH0hoSdfkEfGhpA+LbAKgazjjAwkRPpAQ4QMJET6QEOED\nCRV5yu6KFStiaGio8bm33npr4zMvOHJk0RsTf7LJyckic9EdIyMjxWZPTU0VmRsRPGUXwKUIH0iI\n8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjw\ngYQIH0iI8IGEivw869nZWU1PTzc+t9RTSSWehnux1atXF5k7MzNTZG5J/fiU3To44wMJET6QEOED\nCRE+kBDhAwkRPpAQ4QMJ1Qrf9jW299v+xPaE7TtKLwagnLo38Lwq6d2I+KPtFZJWFtwJQGGV4du+\nWtJdkv4sSRFxXtL5smsBKKnOpf56SWckvWH7qO09tlcV3gtAQXXCH5C0SdLrEbFR0jlJT/34INs7\nbB+2fbjhHQE0rE74pyWdjohDnV/v18IfBD8QEWMRMRoRo00uCKB5leFHxIykr2zf2PnSFkknim4F\noKi67+o/Imlf5x39zyU9XG4lAKXVCj8iPpLEJTzwM8Gde0BChA8kRPhAQoQPJET4QEKEDyTkiGh+\nqN380D41PDxcbHaJR5jjh0o+XrvE98bExITOnTvnquM44wMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJ\nET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCdX9ablLMjg4\nqKGhocbnHjt2rPGZF2zdurXI3BK/DxeUesruxo0bi8zdu3dvkbklTU1NFZu9bdu2xmeePHmy1nGc\n8YGECB9IiPCBhAgfSIjwgYQIH0iI8IGEaoVv+3Hbx21/bPst21eUXgxAOZXh214j6VFJoxFxs6Tl\nkh4ovRiAcupe6g9IutL2gKSVksrdzgSguMrwI2JS0kuSTkmalvRNRLxXejEA5dS51L9W0n2S1ksa\nkbTK9oOXOW6H7cO2D8/Pzze/KYDG1LnUv0fSFxFxJiJmJR2UdOePD4qIsYgYjYjRZcv4ywKgl9Up\n9JSk222vtG1JWyRNlF0LQEl1XuMfkrRf0rikY53/Z6zwXgAKqvV5/Ih4XtLzhXcB0CW8GAcSInwg\nIcIHEiJ8ICHCBxIifCChIo/Xnp2dLfLo51tuuaXxmReUeozyc889V2SuJL3wwgtF5l5//fVF5pZ0\n2223FZn7/fffF5krSWNj7d0OwxkfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9I\niPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0jIEdH8UPuMpJM1Dx+SdLbxJcrpt30l\ndu6GXtn3VxHxy6qDioS/FLYPR8Roq0ssQb/tK7FzN/TbvlzqAwkRPpBQL4Tf3o8T+Wn6bV+Jnbuh\nr/Zt/TU+gO7rhTM+gC4jfCAhwgcSInwgIcIHEvofNUSAm+o1BD4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11d974320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "row_sums = np.sum(cfm, axis=1)\n",
    "err_matrix = cfm / row_sums\n",
    "np.fill_diagonal(err_matrix, 0)\n",
    "\n",
    "plt.matshow(err_matrix, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}