{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 14: Fitting functions to data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_function(f,xmin,xmax,**kwargs):\n",
    "    ts = np.linspace(xmin,xmax,1000)\n",
    "    plt.plot(ts,[f(t) for t in ts],**kwargs)\n",
    "    \n",
    "def scalar_field_heatmap(f,xmin,xmax,ymin,ymax,xsteps=100,ysteps=100):\n",
    "    fig = plt.figure()\n",
    "    fig.set_size_inches(7,7)\n",
    "    fv = np.vectorize(f)\n",
    "    X = np.linspace(xmin, xmax, xsteps)\n",
    "    Y = np.linspace(ymin, ymax, ysteps)\n",
    "    X, Y = np.meshgrid(X, Y)\n",
    "    z = fv(X,Y)\n",
    "    fig, ax = plt.subplots()\n",
    "    c = ax.pcolormesh(X,Y,z, cmap='plasma')\n",
    "    ax.axis([X.min(), X.max(), Y.min(), Y.max()])\n",
    "    fig.colorbar(c, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vectors import length \n",
    "def secant_slope(f,xmin,xmax):\n",
    "    return (f(xmax) - f(xmin)) / (xmax - xmin)\n",
    "\n",
    "def approx_derivative(f,x,dx=1e-6):\n",
    "    return secant_slope(f,x-dx,x+dx)\n",
    "\n",
    "def approx_gradient(f,x0,y0,dx=1e-6):\n",
    "    partial_x = approx_derivative(lambda x:f(x,y0),x0,dx=dx)\n",
    "    partial_y = approx_derivative(lambda y:f(x0,y),y0,dx=dx)\n",
    "    return (partial_x,partial_y)\n",
    "    \n",
    "def gradient_descent(f,xstart,ystart,tolerance=1e-6):\n",
    "    x = xstart\n",
    "    y = ystart\n",
    "    grad = approx_gradient(f,x,y)\n",
    "    while length(grad) > tolerance:\n",
    "        x -= 0.01 * grad[0]\n",
    "        y -= 0.01 * grad[1]\n",
    "        grad = approx_gradient(f,x,y)\n",
    "    return x,y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import car data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from car_data import priuses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 14.1 Measuring the quality of fit for a function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.1.1 Measuring distance from a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = [\n",
    "     (-1.0, -2.0137862606487387),\n",
    "     (-0.9, -1.7730222478628337),\n",
    "     (-0.8, -1.5510125944820812),\n",
    "     (-0.7, -1.6071832453434687),\n",
    "     (-0.6, -0.7530149734137868),\n",
    "     (-0.5, -1.4185018340443283),\n",
    "     (-0.4, -0.6055579756271128),\n",
    "     (-0.3, -1.0067254915961406),\n",
    "     (-0.2, -0.4382360549665138),\n",
    "     (-0.1, -0.17621952751051906),\n",
    "     (0.0, -0.12218090884626329),\n",
    "     (0.1, 0.07428573423209717),\n",
    "     (0.2, 0.4268795998864943),\n",
    "     (0.3, 0.7254661223608084),\n",
    "     (0.4, 0.04798697977420063),\n",
    "     (0.5, 1.1578103735448106),\n",
    "     (0.6, 1.5684111061340824),\n",
    "     (0.7, 1.157745051031345),\n",
    "     (0.8, 2.1744401978240675),\n",
    "     (0.9, 1.6380001974121732),\n",
    "     (1.0, 2.538951262545233)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12568 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter([t[0] for t in test_data],[t[1] for t in test_data])\n",
    "plot_function(lambda x: 2*x,-1,1,c='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's the drawing routine I used to illustrate the absolute value cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_cost(h,points):\n",
    "    xs = [t[0] for t in points]\n",
    "    ys = [t[1] for t in points]\n",
    "    plt.scatter(xs,ys)\n",
    "    plot_function(h,min(xs),max(xs),c='k')\n",
    "    for (x,y) in points:\n",
    "        plt.plot([x,x],[y,h(x)],c='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13060 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_cost(lambda x: 2*x, test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_error(f,data):\n",
    "    errors = [abs(f(x) - y) for (x,y) in data]\n",
    "    return sum(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x): \n",
    "    return 2*x\n",
    "\n",
    "def g(x): \n",
    "    return 1-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.021727176394801"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_error(f,test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38.47711311130152"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_error(g,test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.1.2 Summing the squares of the errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_squared_error(f,data):\n",
    "    squared_errors = [(f(x) - y)**2 for (x,y) in data]\n",
    "    return sum(squared_errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "here's a function to picture the sum_squared_error for a line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_square_cost(h,points):\n",
    "    xs = [t[0] for t in points]\n",
    "    ys = [t[1] for t in points]\n",
    "    plt.scatter(xs,ys)\n",
    "    plot_function(h,min(xs),max(xs),c='k')\n",
    "    for (x,y) in points:\n",
    "        e = abs(y - h(x))\n",
    "        plt.plot([x,x],[y,h(x)],c='r')\n",
    "        plt.fill([x,x,x+e,x+e],[h(x),y,y,h(x)],c='r',alpha=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12848 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_square_cost(lambda x: 2*x, test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.105175107540148"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(f,test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.1078879283203"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "97.1078879283203"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.1.3 Calculating cost for car price functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_mileage_price(cars):\n",
    "    prices = [c.price for c in cars]\n",
    "    mileages = [c.mileage for c in cars]\n",
    "    plt.scatter(mileages, prices, alpha=0.5)\n",
    "    plt.ylabel(\"Price ($)\",fontsize=16)\n",
    "    plt.xlabel(\"Odometer (mi)\",fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def p1(x):\n",
    "    return 25000 - 0.2 * x\n",
    "\n",
    "def p2(x):\n",
    "    return 25000 - 0.1 * x\n",
    "\n",
    "def p3(x):\n",
    "    return 22500 - 0.1 * x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47020 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plot_function(p1,0,125000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48864 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plot_function(p2,0,250000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47276 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plot_function(p3,0,225000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "prius_mileage_price = [(p.mileage, p.price) for p in priuses]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88782506640.23999"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(p1, prius_mileage_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34723507681.55999"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(p2, prius_mileage_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22997230681.560013"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(p3, prius_mileage_price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.1.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Create a set of data points lying on a line and demonstrate that the `sum_error` and `sum_squared_error` cost functions both return exactly zero for the appropriate linear function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def line(x):\n",
    "    return 3*x-2\n",
    "points = [(x,line(x)) for x in range(0,10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_error(line,points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(line,points)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Calculate the value of the cost for the two linear functions $x + 0.5$ and $2x - 1$.  Which one produces a lower sum squared error relative to the `test_data` and what does that say about the quality of the fits?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23.1942461283472"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(lambda x:2*x-1,test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.607900877665685"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(lambda x:x+0.5,test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Find a linear function `p4` that fits the data even better than `p1`, `p2`, or `p3`.  Demonstrate that it is a better fit by showing the cost function is lower than for `p1`, `p2`, or `p3`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def p4(x):\n",
    "    return 20000 - 0.1 * x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18958453681.56001"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_squared_error(p4, prius_mileage_price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.2 Exploring spaces of functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.2.1 Picturing cost for lines through the origin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_data_coefficient_cost(a):\n",
    "    def f(x):\n",
    "        return a * x\n",
    "    return sum_squared_error(f,test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plots from the chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'a')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7240 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "some_slopes = [-1,0,1,2,3]\n",
    "plt.scatter(some_slopes,[test_data_coefficient_cost(a) for a in some_slopes])\n",
    "plt.ylabel(\"cost\", fontsize=16)\n",
    "plt.xlabel(\"a\", fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'x')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 22364 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter([t[0] for t in test_data],[t[1] for t in test_data])\n",
    "for a in some_slopes:\n",
    "    plot_function(lambda x: a*x, -1, 1, c='k')\n",
    "plt.ylabel(\"y\", fontsize=16)\n",
    "plt.xlabel(\"x\", fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'a')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 14296 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_function(test_data_coefficient_cost, -5, 5)\n",
    "plt.ylabel(\"cost\", fontsize=16)\n",
    "plt.xlabel(\"a\", fontsize=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.2.2 The space of all linear functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coefficient_cost(a,b):\n",
    "    def p(x):\n",
    "        return a * x + b\n",
    "    return sum_squared_error(p,prius_mileage_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def coefficient_cost(a,b):\n",
    "    def p(x):\n",
    "        return a * x + b\n",
    "    return sum_squared_error(p,prius_mileage_price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plots from the chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 51180 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plot_function(lambda x: 25000 - 0.20*x,0,125000,c='k')\n",
    "plot_function(lambda x: 5000 + 0.05*x,0,350000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'a')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 504x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24756 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scalar_field_heatmap(coefficient_cost,-0.5,0.5,-50000,50000)\n",
    "plt.ylabel(\"b\",fontsize=16)\n",
    "plt.xlabel(\"a\",fontsize=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.2.3 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 14.3 Finding the line of best fit using gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.3.1 Rescaling the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scaled_cost_function(c,d):\n",
    "    return coefficient_cost(0.5*c,50000*d)/1e13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "mileages = [p.mileage for p in priuses]\n",
    "prices = [p.price for p in priuses]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scaled_cost_function(c,d):\n",
    "    return coefficient_cost(0.5*c,50000*d)/1e13"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.3.2 Finding and plotting the line of best fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "c,d = gradient_descent(scaled_cost_function,0,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.12111901781185214, 0.314954228880389)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(c,d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.06055950890592607, 15747.71144401945)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = 0.5*c\n",
    "b = 50000*d\n",
    "(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47160 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "\n",
    "def p(x):\n",
    "    return a*x + b\n",
    "\n",
    "plot_function(p,0,250000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14536218169.403488"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coefficient_cost(a,b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.3.3 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Use gradient descent to find the linear function that best fits the `test_data`.  Your resulting function should be close to $2x + 0$ but not exactly, because the data was randomly generated around that line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_data_linear_cost(a,b):\n",
    "    def f(x):\n",
    "        return a*x+b\n",
    "    return sum_squared_error(f,test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 504x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 26024 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scalar_field_heatmap(test_data_linear_cost,-0,4,-2,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.103718204728344, 0.0021207385859157535)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient_descent(test_data_linear_cost,1,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 14.4 Fitting a nonlinear function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.4.1 Understanding the behavior of exponential functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47872 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plt.ylim(-5000,35000)\n",
    "plt.xlim(0,375000)\n",
    "plot_function(lambda x:0, -50000, 400000, c='gray')\n",
    "q,r = (16133.220556990309, -5.951793936498175e-06)\n",
    "plot_function(lambda x: q*exp(r*x),0,375000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49440 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plt.ylim(-5000,35000)\n",
    "plt.xlim(0,375000)\n",
    "plot_function(lambda x:0, -50000, 400000, c='gray')\n",
    "\n",
    "plot_function(lambda x: a*x + b,0,375000,c='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.4.2 Finding the exponential function of best fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exp_coefficient_cost(q,r):\n",
    "    def f(x):\n",
    "        return q*exp(r*x) #<1>\n",
    "    return sum_squared_error(f,prius_mileage_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 504x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 19396 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def scaled_exp_coefficient_cost(s,t):\n",
    "    return exp_coefficient_cost(30000*s,1e-4*t) / 1e11\n",
    "\n",
    "scalar_field_heatmap(scaled_exp_coefficient_cost,0,1,-1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6235404892898214, -0.07686877731180546)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s,t = gradient_descent(scaled_exp_coefficient_cost,0,0)\n",
    "(s,t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18706.214678694643, -7.686877731180546e-06)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q,r = 30000*s,1e-4*t\n",
    "(q,r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47840 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "q,r = (16133.220556990309, -5.951793936498175e-06)\n",
    "plot_function(lambda x: q*exp(r*x),0,375000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14071654468.280859"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp_coefficient_cost(q,r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14.4.3 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Confirm, by choosing a sample value of $r$, that $e^{-rx}$ decreases by a factor of $e$ every time $x$ increases by $1/r$ units."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(x):\n",
    "    return exp(-3*x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.36787944117144233, 0.36787944117144233)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(1/3), test(0)/e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1353352832366127, 0.1353352832366127)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.1353352832366127, 0.1353352832366127)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.049787068367863944, 0.04978706836786395)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.049787068367863944, 0.04978706836786395)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** According to the exponential function of best fit, what percent of the value of a prius is lost every 10,000 miles?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9422186306357088"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp(r * 10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Asserting that the retail price (the price at zero miles) is $\\$25,000$ what is the exponential function that best fits the data?  In other words, fixing $q = 25,000$, what is the value of $r$ yielding the best fit for $qe^{rx}$?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exponential_cost2(r):\n",
    "    def f(x):\n",
    "        return 25000 * exp(r*x)\n",
    "    return sum_squared_error(f,prius_mileage_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12956 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_function(exponential_cost2,-1e-4,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16157457680.500858, 16156919168.114105, 16161508051.813206)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# note that r = -1.12e-5 yields a smaller cost function than some values around it.\n",
    "exponential_cost2(-1.11e-5), exponential_cost2(-1.12e-5), exponential_cost2(-1.13e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 47676 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_mileage_price(priuses)\n",
    "plot_function(lambda x: 25000 * exp(-1.12e-5 * x), 0, 350000, c='k')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
