{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 16:Training neural networks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.1 Classifying data with neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.1 Classifying images of handwritten digits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.1 Building the 64-dimensional image vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "digits = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  5., 13.,  9.,  1.,  0.,  0.],\n",
       "       [ 0.,  0., 13., 15., 10., 15.,  5.,  0.],\n",
       "       [ 0.,  3., 15.,  2.,  0., 11.,  8.,  0.],\n",
       "       [ 0.,  4., 12.,  0.,  0.,  8.,  8.,  0.],\n",
       "       [ 0.,  5.,  8.,  0.,  0.,  9.,  8.,  0.],\n",
       "       [ 0.,  4., 11.,  0.,  1., 12.,  7.,  0.],\n",
       "       [ 0.,  2., 14.,  5., 10., 12.,  0.,  0.],\n",
       "       [ 0.,  0.,  6., 13., 10.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.images[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2b926a3da20>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAACtlJREFUeJzt3V9onfUdx/HPZ1HZ/FOsazekqYsBKchgtoaCFITVZdQpuospLShMBr1SlA2s7m53eiPuYghSdYKd0lQFEacTVJywOZO226ypo60dzapryir+GaxUv7vIKXRdtjzp+T1/ztf3C4L5c8jve4jvPs85OXl+jggByOlLbQ8AoD4EDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiZ9XxTZctWxYjIyN1fOtWHTt2rNH1ZmZmGltryZIlja01PDzc2FpDQ0ONrdWkgwcP6ujRo17odrUEPjIyosnJyTq+dasmJiYaXW/Lli2NrTU+Pt7YWvfdd19jay1durSxtZo0NjZW6XacogOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWKXAbW+w/a7tfbbvqXsoAGUsGLjtIUm/kHStpMslbbJ9ed2DAehflSP4Wkn7IuJARByX9JSkG+sdC0AJVQJfIenQKR/P9D4HoOOqBD7fX6z818XUbW+2PWl7cnZ2tv/JAPStSuAzklae8vGwpMOn3ygiHo6IsYgYW758ean5APShSuBvSbrM9qW2z5G0UdJz9Y4FoIQF/x48Ik7Yvl3SS5KGJD0aEXtqnwxA3ypd8CEiXpD0Qs2zACiMV7IBiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kFgtO5tk1eROI5L03nvvNbZWk9syXXTRRY2ttX379sbWkqSbbrqp0fUWwhEcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEisys4mj9o+YvvtJgYCUE6VI/gvJW2oeQ4ANVgw8Ih4XdI/GpgFQGE8BgcSKxY4WxcB3VMscLYuArqHU3QgsSq/JntS0u8krbI9Y/tH9Y8FoIQqe5NtamIQAOVxig4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgO/ddHU1FRjazW5lZAk7d+/v7G1RkdHG1trfHy8sbWa/P9DYusiAA0icCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgsSoXXVxp+1Xb07b32L6zicEA9K/Ka9FPSPpJROy0fYGkKdsvR8Q7Nc8GoE9V9iZ7PyJ29t7/WNK0pBV1Dwagf4t6DG57RNJqSW/O8zW2LgI6pnLgts+X9LSkuyLio9O/ztZFQPdUCtz22ZqLe1tEPFPvSABKqfIsuiU9Imk6Ih6ofyQApVQ5gq+TdKuk9bZ3996+V/NcAAqosjfZG5LcwCwACuOVbEBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kNvB7kx07dqyxtdasWdPYWlKz+4U16corr2x7hC8MjuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGJVLrr4Zdt/sP3H3tZFP2tiMAD9q/JS1X9JWh8Rn/Qun/yG7V9HxO9rng1An6pcdDEkfdL78OzeW9Q5FIAyqm58MGR7t6Qjkl6OCLYuAgZApcAj4rOIuELSsKS1tr85z23YugjomEU9ix4RH0p6TdKGWqYBUFSVZ9GX276w9/5XJH1H0t66BwPQvyrPol8s6XHbQ5r7B2F7RDxf71gASqjyLPqfNLcnOIABwyvZgMQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMrYsWYXx8vLG1MmvyZ7Z06dLG1uoijuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGKVA+9dG32Xba7HBgyIxRzB75Q0XdcgAMqrurPJsKTrJG2tdxwAJVU9gj8o6W5Jn9c4C4DCqmx8cL2kIxExtcDt2JsM6JgqR/B1km6wfVDSU5LW237i9BuxNxnQPQsGHhH3RsRwRIxI2ijplYi4pfbJAPSN34MDiS3qii4R8ZrmdhcFMAA4ggOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQ2MBvXdTk1jRTU//3720GWpPbCU1OTja21s0339zYWl3EERxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSKzSK9l6V1T9WNJnkk5ExFidQwEoYzEvVf12RBytbRIAxXGKDiRWNfCQ9BvbU7Y31zkQgHKqnqKvi4jDtr8m6WXbeyPi9VNv0At/syRdcsklhccEcCYqHcEj4nDvv0ckPStp7Ty3YesioGOqbD54nu0LTr4v6buS3q57MAD9q3KK/nVJz9o+eftfRcSLtU4FoIgFA4+IA5K+1cAsAArj12RAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJDbwWxeNjo42tlaTW+5I0sTERMq1mrRly5a2R2gVR3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwILFKgdu+0PYO23ttT9u+qu7BAPSv6ktVfy7pxYj4ge1zJJ1b40wAClkwcNtLJF0t6YeSFBHHJR2vdywAJVQ5RR+VNCvpMdu7bG/tXR8dQMdVCfwsSWskPRQRqyV9Kume029ke7PtSduTs7OzhccEcCaqBD4jaSYi3ux9vENzwf8Hti4CumfBwCPiA0mHbK/qfeoaSe/UOhWAIqo+i36HpG29Z9APSLqtvpEAlFIp8IjYLWms5lkAFMYr2YDECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxNibbBHuv//+xtaSmt1Xa2ysuRcqTk1NNbbWFx1HcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgsQUDt73K9u5T3j6yfVcTwwHoz4IvVY2IdyVdIUm2hyT9TdKzNc8FoIDFnqJfI2l/RPy1jmEAlLXYwDdKenK+L7B1EdA9lQPvbXpwg6SJ+b7O1kVA9yzmCH6tpJ0R8fe6hgFQ1mIC36T/cXoOoJsqBW77XEnjkp6pdxwAJVXdm+yfkr5a8ywACuOVbEBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4k5ogo/03tWUmL/ZPSZZKOFh+mG7LeN+5Xe74REQv+VVctgZ8J25MR0dwGWQ3Ket+4X93HKTqQGIEDiXUp8IfbHqBGWe8b96vjOvMYHEB5XTqCAyisE4Hb3mD7Xdv7bN/T9jwl2F5p+1Xb07b32L6z7ZlKsj1ke5ft59uepSTbF9reYXtv72d3Vdsz9aP1U/Tetdb/orkrxsxIekvSpoh4p9XB+mT7YkkXR8RO2xdImpL0/UG/XyfZ/rGkMUlLIuL6tucpxfbjkn4bEVt7Fxo9NyI+bHuuM9WFI/haSfsi4kBEHJf0lKQbW56pbxHxfkTs7L3/saRpSSvanaoM28OSrpO0te1ZSrK9RNLVkh6RpIg4PshxS90IfIWkQ6d8PKMkIZxke0TSaklvtjtJMQ9KulvS520PUtiopFlJj/Uefmy1fV7bQ/WjC4F7ns+leWrf9vmSnpZ0V0R81PY8/bJ9vaQjETHV9iw1OEvSGkkPRcRqSZ9KGujnhLoQ+Iyklad8PCzpcEuzFGX7bM3FvS0islyRdp2kG2wf1NzDqfW2n2h3pGJmJM1ExMkzrR2aC35gdSHwtyRdZvvS3pMaGyU91/JMfbNtzT2Wm46IB9qep5SIuDcihiNiRHM/q1ci4paWxyoiIj6QdMj2qt6nrpE00E+KVrpscp0i4oTt2yW9JGlI0qMRsaflsUpYJ+lWSX+2vbv3uZ9GxAstzoSF3SFpW+9gc0DSbS3P05fWf00GoD5dOEUHUBMCBxIjcCAxAgcSI3AgMQIHEiNwIDECBxL7NyyRs2/TGgiSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(digits.images[0], cmap=plt.cm.gray_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FYI, a way to overlay numbers on the pixels showing their brightness values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12864 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(digits.images[0], cmap=plt.cm.gray_r)\n",
    "for i in range(0,8):\n",
    "    for j in range(0,8):\n",
    "        plt.gca().text(i-0.15,j,int(digits.images[0][i][j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  5., 13.,  9.,  1.,  0.,  0.,  0.,  0., 13., 15., 10.,\n",
       "       15.,  5.,  0.,  0.,  3., 15.,  2.,  0., 11.,  8.,  0.,  0.,  4.,\n",
       "       12.,  0.,  0.,  8.,  8.,  0.,  0.,  5.,  8.,  0.,  0.,  9.,  8.,\n",
       "        0.,  0.,  4., 11.,  0.,  1., 12.,  7.,  0.,  0.,  2., 14.,  5.,\n",
       "       10., 12.,  0.,  0.,  0.,  0.,  6., 13., 10.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.matrix.flatten(digits.images[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.33333333, 0.86666667, 0.6       ,\n",
       "       0.06666667, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.86666667, 1.        , 0.66666667, 1.        , 0.33333333,\n",
       "       0.        , 0.        , 0.2       , 1.        , 0.13333333,\n",
       "       0.        , 0.73333333, 0.53333333, 0.        , 0.        ,\n",
       "       0.26666667, 0.8       , 0.        , 0.        , 0.53333333,\n",
       "       0.53333333, 0.        , 0.        , 0.33333333, 0.53333333,\n",
       "       0.        , 0.        , 0.6       , 0.53333333, 0.        ,\n",
       "       0.        , 0.26666667, 0.73333333, 0.        , 0.06666667,\n",
       "       0.8       , 0.46666667, 0.        , 0.        , 0.13333333,\n",
       "       0.93333333, 0.33333333, 0.66666667, 0.8       , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.4       , 0.86666667,\n",
       "       0.66666667, 0.        , 0.        , 0.        ])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.matrix.flatten(digits.images[0]) / 15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.2 Building a random digit classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_classifier(input_vector):\n",
    "    return np.random.rand(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09609494, 0.75660412, 0.78242862, 0.43881297, 0.33470396,\n",
       "       0.92745083, 0.9695724 , 0.12471611, 0.01307655, 0.61261849])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = np.matrix.flatten(digits.images[0]) / 15.\n",
    "result = random_classifier(v)\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE** because this is random, you will get a different digit result when you re-run the code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(result).index(max(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.target[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.3 Measuring performance of the digit classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_digit_classify(classifier,test_count=1000):\n",
    "    correct = 0 #<1>\n",
    "    for img, target in zip(digits.images[:test_count], digits.target[:test_count]): #<2>\n",
    "        v = np.matrix.flatten(img) / 15. #<3>\n",
    "        output = classifier(v) #<4>\n",
    "        answer = list(output).index(max(output)) #<5>\n",
    "        if answer == target:\n",
    "            correct += 1 #<6>\n",
    "    return (correct/test_count) #<7>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.089"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(random_classifier)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.1.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Suppose a digit classifier function outputs the following NumPy array.  What digit does it think the image represents?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.00512567e-06, 3.94168539e-05, 5.57124430e-09, 9.31981207e-09,\n",
       "       9.98060276e-01, 9.10328786e-07, 1.56262695e-03, 1.82976466e-04,\n",
       "       1.48519455e-04, 2.54354113e-07])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([5.00512567e-06, 3.94168539e-05, 5.57124430e-09, 9.31981207e-09,\n",
    "       9.98060276e-01, 9.10328786e-07, 1.56262695e-03, 1.82976466e-04,\n",
    "       1.48519455e-04, 2.54354113e-07])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini project:** Find the average of all the images of nines in the data set, in the same way we took averages of images in Chapter 6.  Plot the resulting image. What does it look like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_img(i):\n",
    "    imgs = [img for img,target in zip(digits.images[1000:], digits.target[1000:]) if target==i]\n",
    "    return sum(imgs) / len(imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2b926891c50>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 4112 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(average_img(9), cmap=plt.cm.gray_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini project:** Build a better classifier than a random one by finding the average image of each kind of digit in the test data set, and comparing a target image with all of the averages.  Specifically, return a vector of the dot products of the target image with each average digit image.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_digits = [np.matrix.flatten(average_img(i)) for i in range(10)]\n",
    "def compare_to_avg(v):\n",
    "    return [np.dot(v,avg_digits[i]) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.853"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(compare_to_avg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.2 Designing a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.1 Organizing neurons and connections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.2 Data flow through a neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.3 Calculating activations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.4 Calculating activations in matrix notation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.2.5 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.3 Building a neural network in Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.1 Implementing an MLP class in Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP():\n",
    "    def __init__(self,layer_sizes): #<1>\n",
    "        self.layer_sizes = layer_sizes\n",
    "        self.weights = [\n",
    "            np.random.rand(n,m) #<2>\n",
    "            for m,n in zip(layer_sizes[:-1],layer_sizes[1:]) #<3>\n",
    "        ]\n",
    "        self.biases = [np.random.rand(n) for n in layer_sizes[1:]] #<4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([2,3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE** these numbers are randomly initialized, so your results below will vary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0.04098182, 0.61121605],\n",
       "        [0.99402839, 0.43278994],\n",
       "        [0.14826355, 0.39370373]])]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.62871743, 0.55151304, 0.91445791])]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.biases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.2 Evaluating the MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp\n",
    "def sigmoid(x):\n",
    "    return 1 / (1+exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP():\n",
    "    def __init__(self,layer_sizes): #<1>\n",
    "        self.layer_sizes = layer_sizes\n",
    "        self.weights = [\n",
    "            np.random.rand(n,m) #<2>\n",
    "            for m,n in zip(layer_sizes[:-1],layer_sizes[1:]) #<3>\n",
    "        ]\n",
    "        self.biases = [np.random.rand(n) for n in layer_sizes[1:]] #<4>\n",
    "    def feedforward(self,v):\n",
    "        activations = [] #<1>\n",
    "        a = v\n",
    "        activations.append(a) #<2>\n",
    "        for w,b in zip(self.weights, self.biases): #<3>\n",
    "            z = w @ a + b #<4>\n",
    "            a = [sigmoid(x) for x in z] #<5>\n",
    "            activations.append(a) #<6>\n",
    "        return activations\n",
    "    def evaluate(self,v):\n",
    "        return np.array(self.feedforward(v)[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.3 Testing the classification performance of an MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([64,16,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.matrix.flatten(digits.images[0]) / 15."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.99996588, 0.99829151, 0.99979785, 0.9998958 , 0.99991507,\n",
       "       0.99982444, 0.9999176 , 0.99863889, 0.99977906, 0.99887847])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.evaluate(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.099"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(nn.evaluate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.3.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.4 Training a neural network using gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.1 Framing training as a minimization problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.2 Calculating gradients with backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.3 Automatic training with scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([np.matrix.flatten(img) for img in digits.images[:1000]]) / 15.0\n",
    "y = digits.target[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(16,), #<1>\n",
    "                    activation='logistic', #<2>\n",
    "                    max_iter=100, #<3>\n",
    "                    verbose=10, #<4>\n",
    "                    random_state=1, #<5>\n",
    "                    learning_rate_init=.1) #<6>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 2.21958598\n",
      "Iteration 2, loss = 1.56912978\n",
      "Iteration 3, loss = 0.98970277\n",
      "Iteration 4, loss = 0.57473464\n",
      "Iteration 5, loss = 0.34048448\n",
      "Iteration 6, loss = 0.21495855\n",
      "Iteration 7, loss = 0.14366771\n",
      "Iteration 8, loss = 0.11077020\n",
      "Iteration 9, loss = 0.08764273\n",
      "Iteration 10, loss = 0.07193546\n",
      "Iteration 11, loss = 0.06020348\n",
      "Iteration 12, loss = 0.04961899\n",
      "Iteration 13, loss = 0.03979645\n",
      "Iteration 14, loss = 0.03334502\n",
      "Iteration 15, loss = 0.02996006\n",
      "Iteration 16, loss = 0.02603968\n",
      "Iteration 17, loss = 0.02355514\n",
      "Iteration 18, loss = 0.02137348\n",
      "Iteration 19, loss = 0.01967878\n",
      "Iteration 20, loss = 0.01751214\n",
      "Iteration 21, loss = 0.01617330\n",
      "Iteration 22, loss = 0.01460386\n",
      "Iteration 23, loss = 0.01408517\n",
      "Iteration 24, loss = 0.01270504\n",
      "Iteration 25, loss = 0.01191634\n",
      "Iteration 26, loss = 0.01114222\n",
      "Iteration 27, loss = 0.01045989\n",
      "Iteration 28, loss = 0.00983648\n",
      "Iteration 29, loss = 0.00920912\n",
      "Iteration 30, loss = 0.00890851\n",
      "Iteration 31, loss = 0.00843426\n",
      "Iteration 32, loss = 0.00796039\n",
      "Iteration 33, loss = 0.00749839\n",
      "Iteration 34, loss = 0.00726271\n",
      "Iteration 35, loss = 0.00673963\n",
      "Iteration 36, loss = 0.00655405\n",
      "Iteration 37, loss = 0.00626207\n",
      "Iteration 38, loss = 0.00600639\n",
      "Iteration 39, loss = 0.00581857\n",
      "Iteration 40, loss = 0.00557529\n",
      "Iteration 41, loss = 0.00533573\n",
      "Iteration 42, loss = 0.00519479\n",
      "Iteration 43, loss = 0.00505128\n",
      "Iteration 44, loss = 0.00490121\n",
      "Iteration 45, loss = 0.00469161\n",
      "Iteration 46, loss = 0.00459590\n",
      "Iteration 47, loss = 0.00464844\n",
      "Iteration 48, loss = 0.00445157\n",
      "Iteration 49, loss = 0.00425515\n",
      "Iteration 50, loss = 0.00424934\n",
      "Iteration 51, loss = 0.00397800\n",
      "Iteration 52, loss = 0.00399927\n",
      "Iteration 53, loss = 0.00383932\n",
      "Iteration 54, loss = 0.00372439\n",
      "Iteration 55, loss = 0.00361744\n",
      "Iteration 56, loss = 0.00356447\n",
      "Iteration 57, loss = 0.00345899\n",
      "Iteration 58, loss = 0.00336792\n",
      "Iteration 59, loss = 0.00330330\n",
      "Iteration 60, loss = 0.00321734\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='logistic', alpha=0.0001, batch_size='auto',\n",
       "       beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(16,), learning_rate='constant',\n",
       "       learning_rate_init=0.1, max_iter=100, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=1, shuffle=True,\n",
       "       solver='adam', tol=0.0001, validation_fraction=0.1, verbose=10,\n",
       "       warm_start=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.fit(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.99766643e-01, 8.43331208e-11, 3.47867059e-06, 1.49956270e-07,\n",
       "       1.88677660e-06, 3.44652605e-05, 6.23829017e-06, 1.09043503e-04,\n",
       "       1.11195821e-07, 7.79837557e-05])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp._predict(x)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sklearn_trained_classify(v):\n",
    "    return mlp._predict([v])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(sklearn_trained_classify)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.4.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Modify the `test_digit_classify` function to work on a custom range of examples in the test set.  How does it do on the next 500 examples after the 1,000 training examples?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_digit_classify(classifier,start=0,test_count=1000):\n",
    "    correct = 0\n",
    "    end = start + test_count #<1>\n",
    "    for img, target in zip(digits.images[start:end], digits.target[start:end]): #<2>\n",
    "        v = np.matrix.flatten(img) / 15.\n",
    "        output = classifier(v)\n",
    "        answer = list(output).index(max(output))\n",
    "        if answer == target:\n",
    "            correct += 1\n",
    "    return (correct/test_count) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.962"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(sklearn_trained_classify,start=1000,test_count=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Using the squared distance cost function, what is the cost of your randomly-generated MLP for the first 1,000 training examples?  What is the cost of the scikit-learn MLP?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def y_vec(digit):\n",
    "    return np.array([1 if i == digit else 0 for i in range(0,10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_one(classifier,x,i):\n",
    "    return sum([(classifier(x)[j] - y_vec(i)[j])**2 for j in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_cost(classifier):\n",
    "    return sum([cost_one(classifier,x[j],y[j]) for j in range(1000)])/1000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.990834701722013"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_cost(nn.evaluate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.670512721637246e-05"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_cost(sklearn_trained_classify)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** Extract the MLPClassifier weights and biases using its properties called `coefs_` and `intercepts_`, respectively.  Plug these weights and biases into the MLP class we built from scratch and show that your resulting MLP performs well on digit classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = MLP([64,16,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.weights = [w.T for w in mlp.coefs_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.biases = mlp.intercepts_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.962"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_digit_classify(nn.evaluate,start=1000,test_count=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 16.5 Calculating gradients with backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.1 Finding the cost in terms of the last layer weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.1 Finding the cost in terms of the last layer weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.2 Calculating the partial derivatives for the last layer weights using the chain rule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 16.5.3 Exercises "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** Use SymPy or your own code from chapter 10 to automatically find the derivative of the sigmoid function:\n",
    "\n",
    "$$\\sigma(x) = \\frac{1}{1+e^{-x}}$$\n",
    "\n",
    "Show that the answer you get is equal to $\\sigma(x)(1-\\sigma(x))$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exp(-x)/(1 + exp(-x))**2"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sympy import *\n",
    "X = symbols('x')\n",
    "diff(1 / (1+exp(-X)),X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
