{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 15: Classifying data with logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib import cm\n",
    "from mpl_toolkits.mplot3d import Axes3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_function(f,xmin,xmax,**kwargs):\n",
    "    ts = np.linspace(xmin,xmax,1000)\n",
    "    plt.plot(ts,[f(t) for t in ts],**kwargs)\n",
    "    \n",
    "def scalar_field_heatmap(f,xmin,xmax,ymin,ymax,xsteps=100,ysteps=100):\n",
    "    fig = plt.figure()\n",
    "    fig.set_size_inches(7,7)\n",
    "    fv = np.vectorize(f)\n",
    "    X = np.linspace(xmin, xmax, xsteps)\n",
    "    Y = np.linspace(ymin, ymax, ysteps)\n",
    "    X, Y = np.meshgrid(X, Y)\n",
    "    z = fv(X,Y)\n",
    "    fig, ax = plt.subplots()\n",
    "    c = ax.pcolormesh(X,Y,z, cmap='plasma')\n",
    "    ax.axis([X.min(), X.max(), Y.min(), Y.max()])\n",
    "    fig.colorbar(c, ax=ax)\n",
    "    \n",
    "def plot_scalar_field(f,xmin,xmax,ymin,ymax,xsteps=100,ysteps=100,c=None,cmap=cm.coolwarm,alpha=1,antialiased=False, zorder=0):\n",
    "    fig = plt.gcf()\n",
    "    fig.set_size_inches(7,7)\n",
    "    ax = fig.gca(projection='3d')\n",
    "\n",
    "    fv = np.vectorize(f)\n",
    "    \n",
    "    # Make data.\n",
    "    X = np.linspace(xmin, xmax, xsteps)\n",
    "    Y = np.linspace(ymin, ymax, ysteps)\n",
    "    X, Y = np.meshgrid(X, Y)\n",
    "    Z = fv(X,Y)\n",
    "\n",
    "    # Plot the surface.\n",
    "    surf = ax.plot_surface(X, Y, Z, cmap=cmap,color=c,alpha=alpha,\n",
    "                           linewidth=0, antialiased=antialiased, zorder=zorder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.1 Testing a classification function on real data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bmw_finder(mileage,price):\n",
    "    if price > 25000:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.1.1 Loading the car data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from car_data import bmws, priuses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_car_data = []\n",
    "for bmw in bmws:\n",
    "    all_car_data.append((bmw.mileage,bmw.price,1))\n",
    "for prius in priuses:\n",
    "    all_car_data.append((prius.mileage,prius.price,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(93404.0, 13999.0, 1),\n",
       " (110890.0, 13995.0, 1),\n",
       " (94133.0, 13982.0, 1),\n",
       " (46778.0, 14599.0, 1),\n",
       " (53106.0, 22500.0, 1),\n",
       " (58761.0, 24998.0, 1),\n",
       " (108816.0, 24947.0, 1),\n",
       " (81100.0, 13995.0, 1),\n",
       " (90000.0, 8400.0, 1),\n",
       " (68613.0, 14995.0, 1),\n",
       " (94000.0, 11995.0, 1),\n",
       " (92500.0, 10995.0, 1),\n",
       " (112081.0, 11995.0, 1),\n",
       " (105121.0, 11500.0, 1),\n",
       " (92000.0, 2013.0, 1),\n",
       " (107953.0, 12999.0, 1),\n",
       " (56000.0, 18995.0, 1),\n",
       " (101191.0, 18900.0, 1),\n",
       " (64365.0, 16998.0, 1),\n",
       " (66000.0, 10000.0, 1),\n",
       " (76675.0, 12500.0, 1),\n",
       " (93015.0, 15995.0, 1),\n",
       " (80917.0, 14970.0, 1),\n",
       " (96000.0, 16795.0, 1),\n",
       " (70000.0, 12999.0, 1),\n",
       " (107000.0, 11950.0, 1),\n",
       " (78000.0, 18995.0, 1),\n",
       " (78000.0, 15000.0, 1),\n",
       " (92000.0, 2013.0, 1),\n",
       " (57624.0, 21963.0, 1),\n",
       " (77854.0, 16995.0, 1),\n",
       " (48310.0, 22998.0, 1),\n",
       " (51656.0, 20998.0, 1),\n",
       " (62410.0, 19991.0, 1),\n",
       " (39332.0, 29995.0, 1),\n",
       " (31420.0, 21000.0, 1),\n",
       " (41267.0, 22450.0, 1),\n",
       " (73000.0, 19999.0, 1),\n",
       " (94608.0, 11995.0, 1),\n",
       " (67000.0, 24964.0, 1),\n",
       " (50000.0, 18985.0, 1),\n",
       " (73601.0, 16999.0, 1),\n",
       " (50000.0, 18985.0, 1),\n",
       " (95229.0, 15980.0, 1),\n",
       " (69000.0, 27247.0, 1),\n",
       " (73309.0, 13790.0, 1),\n",
       " (41573.0, 21970.0, 1),\n",
       " (42172.0, 25842.0, 1),\n",
       " (41085.0, 18750.0, 1),\n",
       " (81625.0, 13990.0, 1),\n",
       " (38000.0, 24900.0, 1),\n",
       " (65000.0, 12000.0, 1),\n",
       " (103000.0, 16495.0, 1),\n",
       " (76000.0, 18500.0, 1),\n",
       " (34000.0, 19950.0, 1),\n",
       " (66000.0, 29500.0, 1),\n",
       " (36756.0, 18999.0, 1),\n",
       " (42681.0, 26697.0, 1),\n",
       " (29000.0, 29997.0, 1),\n",
       " (51000.0, 26968.0, 1),\n",
       " (26000.0, 25980.0, 1),\n",
       " (88262.0, 18338.0, 1),\n",
       " (48321.0, 21322.0, 1),\n",
       " (93762.0, 16991.0, 1),\n",
       " (79972.0, 16499.0, 1),\n",
       " (54177.0, 20774.0, 1),\n",
       " (43294.0, 21800.0, 1),\n",
       " (55736.0, 24985.0, 1),\n",
       " (74896.0, 22599.0, 1),\n",
       " (72282.0, 20770.0, 1),\n",
       " (58328.0, 22000.0, 1),\n",
       " (49856.0, 20998.0, 1),\n",
       " (42229.0, 21998.0, 1),\n",
       " (38126.0, 25998.0, 1),\n",
       " (46000.0, 21900.0, 1),\n",
       " (95000.0, 18500.0, 1),\n",
       " (30000.0, 19899.0, 1),\n",
       " (63000.0, 21500.0, 1),\n",
       " (39835.0, 27629.0, 1),\n",
       " (51043.0, 25964.0, 1),\n",
       " (27000.0, 30998.0, 1),\n",
       " (39591.0, 25000.0, 1),\n",
       " (37703.0, 35990.0, 1),\n",
       " (41467.0, 30254.0, 1),\n",
       " (83000.0, 15900.0, 1),\n",
       " (58623.0, 21998.0, 1),\n",
       " (57276.0, 21998.0, 1),\n",
       " (27620.0, 23998.0, 1),\n",
       " (40237.0, 21998.0, 1),\n",
       " (52961.0, 22998.0, 1),\n",
       " (36707.0, 34888.0, 1),\n",
       " (42363.0, 29891.0, 1),\n",
       " (44943.0, 23697.0, 1),\n",
       " (90000.0, 11000.0, 1),\n",
       " (90000.0, 13000.0, 1),\n",
       " (42470.0, 30977.0, 1),\n",
       " (27000.0, 20999.0, 1),\n",
       " (37000.0, 31990.0, 1),\n",
       " (63000.0, 2016.0, 1),\n",
       " (51000.0, 2016.0, 1),\n",
       " (59014.0, 13995.0, 0),\n",
       " (70000.0, 9900.0, 0),\n",
       " (84507.0, 14998.0, 0),\n",
       " (76000.0, 8300.0, 0),\n",
       " (41868.0, 15998.0, 0),\n",
       " (40631.0, 13991.0, 0),\n",
       " (74000.0, 10532.0, 0),\n",
       " (69703.0, 12900.0, 0),\n",
       " (37000.0, 15850.0, 0),\n",
       " (33000.0, 13995.0, 0),\n",
       " (75000.0, 9500.0, 0),\n",
       " (35000.0, 11100.0, 0),\n",
       " (97109.0, 11991.0, 0),\n",
       " (128882.0, 8359.0, 0),\n",
       " (39000.0, 13759.0, 0),\n",
       " (65000.0, 13990.0, 0),\n",
       " (37000.0, 12950.0, 0),\n",
       " (130000.0, 9500.0, 0),\n",
       " (63430.0, 12995.0, 0),\n",
       " (120000.0, 7600.0, 0),\n",
       " (74000.0, 9000.0, 0),\n",
       " (46000.0, 13982.0, 0),\n",
       " (73428.0, 14991.0, 0),\n",
       " (200000.0, 7950.0, 0),\n",
       " (88180.0, 12998.0, 0),\n",
       " (89466.0, 14599.0, 0),\n",
       " (76985.0, 11990.0, 0),\n",
       " (65000.0, 13990.0, 0),\n",
       " (63621.0, 12395.0, 0),\n",
       " (96000.0, 7900.0, 0),\n",
       " (74000.0, 9000.0, 0),\n",
       " (75000.0, 11000.0, 0),\n",
       " (143000.0, 8999.0, 0),\n",
       " (112000.0, 8900.0, 0),\n",
       " (74000.0, 9000.0, 0),\n",
       " (185000.0, 5950.0, 0),\n",
       " (80424.0, 13000.0, 0),\n",
       " (48655.0, 17998.0, 0),\n",
       " (102000.0, 9997.0, 0),\n",
       " (67508.0, 14991.0, 0),\n",
       " (39395.0, 18998.0, 0),\n",
       " (44531.0, 16577.0, 0),\n",
       " (68000.0, 12495.0, 0),\n",
       " (35000.0, 11999.0, 0),\n",
       " (68000.0, 12495.0, 0),\n",
       " (115000.0, 10900.0, 0),\n",
       " (66293.0, 14472.0, 0),\n",
       " (141000.0, 10000.0, 0),\n",
       " (109496.0, 9677.0, 0),\n",
       " (82000.0, 7500.0, 0),\n",
       " (106361.0, 10953.0, 0),\n",
       " (187000.0, 8800.0, 0),\n",
       " (76912.0, 11993.0, 0),\n",
       " (117000.0, 9450.0, 0),\n",
       " (113963.0, 11494.0, 0),\n",
       " (72000.0, 12490.0, 0),\n",
       " (71218.0, 11994.0, 0),\n",
       " (48924.0, 13786.0, 0),\n",
       " (100000.0, 7800.0, 0),\n",
       " (98531.0, 10979.0, 0),\n",
       " (43015.0, 16977.0, 0),\n",
       " (29695.0, 16599.0, 0),\n",
       " (44640.0, 16900.0, 0),\n",
       " (56478.0, 13990.0, 0),\n",
       " (114959.0, 11060.0, 0),\n",
       " (83861.0, 13000.0, 0),\n",
       " (91314.0, 11911.0, 0),\n",
       " (37000.0, 16375.0, 0),\n",
       " (46000.0, 14500.0, 0),\n",
       " (45358.0, 16991.0, 0),\n",
       " (68432.0, 16998.0, 0),\n",
       " (37739.0, 19998.0, 0),\n",
       " (89352.0, 12995.0, 0),\n",
       " (65000.0, 13500.0, 0),\n",
       " (61000.0, 13000.0, 0),\n",
       " (35385.0, 13988.0, 0),\n",
       " (65000.0, 13500.0, 0),\n",
       " (57000.0, 16791.0, 0),\n",
       " (90019.0, 14790.0, 0),\n",
       " (136095.0, 11595.0, 0),\n",
       " (27000.0, 16235.0, 0),\n",
       " (54480.0, 17910.0, 0),\n",
       " (100000.0, 16000.0, 0),\n",
       " (65899.0, 16482.0, 0),\n",
       " (27000.0, 17734.0, 0),\n",
       " (144000.0, 9800.0, 0),\n",
       " (26989.0, 17925.0, 0),\n",
       " (67662.0, 13400.0, 0),\n",
       " (38401.0, 16998.0, 0),\n",
       " (80982.0, 13999.0, 0),\n",
       " (43000.0, 17492.0, 0),\n",
       " (87428.0, 11794.0, 0),\n",
       " (34028.0, 17491.0, 0),\n",
       " (37000.0, 15800.0, 0),\n",
       " (32000.0, 20995.0, 0),\n",
       " (89000.0, 12999.0, 0),\n",
       " (64000.0, 15000.0, 0),\n",
       " (45000.0, 16900.0, 0),\n",
       " (38000.0, 13500.0, 0),\n",
       " (71000.0, 12500.0, 0)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_car_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.1.1 Testing the classification function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_classifier(classifier, data):\n",
    "    trues = 0\n",
    "    falses = 0\n",
    "    for mileage, price, is_bmw in data:\n",
    "        if classifier(mileage, price) == is_bmw: #1\n",
    "            trues += 1\n",
    "        else:\n",
    "            falses += 1 #1\n",
    "    return trues / (trues + falses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(bmw_finder, all_car_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.1.3 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Update the `test_classifier` function to print the number of true positives, true negatives, false positives, and false negatives.  Printing these for the `bmw_finder` classifier, what can you tell about the performance of the classifier?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_classifier(classifier, data, verbose=False): #1\n",
    "    true_positives = 0 #2\n",
    "    true_negatives = 0\n",
    "    false_positives = 0\n",
    "    false_negatives = 0\n",
    "    for mileage, price, is_bmw in data:\n",
    "        predicted = classifier(mileage,price)\n",
    "        if predicted and is_bmw: #3\n",
    "            true_positives += 1\n",
    "        elif predicted:\n",
    "            false_positives += 1\n",
    "        elif is_bmw:\n",
    "            false_negatives += 1\n",
    "        else:\n",
    "            true_negatives += 1\n",
    "            \n",
    "    if verbose:        \n",
    "        print(\"true positives %f\" % true_positives) #4\n",
    "        print(\"true negatives %f\" % true_negatives)\n",
    "        print(\"false positives %f\" % false_positives)\n",
    "        print(\"false negatives %f\" % false_negatives)\n",
    "            \n",
    "    return (true_positives + true_negatives) / len(data) #5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true positives 18.000000\n",
      "true negatives 100.000000\n",
      "false positives 0.000000\n",
      "false negatives 82.000000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.59"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(bmw_finder,all_car_data,verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Find a way to update the `bmw_finder` function slightly to improve its performance, and use the `test_classifier` function to confirm that your improved function has better than 59% accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bmw_finder2(mileage,price):\n",
    "    if price > 20000:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.735"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(bmw_finder2, all_car_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.2 Picturing a decision boundary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.2.1 Picturing the space of cars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_data(ds):\n",
    "    plt.scatter([d[0] for d in ds if d[2]==0],[d[1] for d in ds if d[2]==0],c='C1')\n",
    "    plt.scatter([d[0] for d in ds if d[2]==1],[d[1] for d in ds if d[2]==1],c='C0',marker='x')\n",
    "    plt.ylabel(\"Price ($)\",fontsize=16)\n",
    "    plt.xlabel(\"Odometer (mi)\",fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23300 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(all_car_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23536 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(all_car_data)\n",
    "plot_function(lambda x: 25000, 0, 200000, c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 27116 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(all_car_data)\n",
    "plot_function(lambda x: 21000 - 0.07 * x, 0, 200000, c='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.2.3 Implementing the classification function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decision_boundary_classify(mileage,price):\n",
    "    if price > 21000 - 0.07 * mileage:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.805"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(decision_boundary_classify, all_car_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.2.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** What is the decision boundary of the form $p = constant$ giving the best classification accuracy on the test data set?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def constant_price_classifier(cutoff_price):\n",
    "    def c(x,p):\n",
    "        if p > cutoff_price:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cutoff_accuracy(cutoff_price):\n",
    "    c = constant_price_classifier(cutoff_price)\n",
    "    return test_classifier(c,all_car_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17998.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_prices = [price for (mileage,price,is_bmw) in all_car_data]\n",
    "max(all_prices,key=cutoff_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.795"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(constant_price_classifier(17998.0), all_car_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.3 Framing classification as a regression problem\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.1 Scaling the raw car data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_scale(data):\n",
    "    min_val = min(data) #1\n",
    "    max_val = max(data)\n",
    "    def scale(x): #2\n",
    "        return (x-min_val) / (max_val - min_val)\n",
    "    def unscale(y): #3\n",
    "        return y * (max_val - min_val) + min_val\n",
    "    return scale, unscale #4\n",
    "\n",
    "price_scale, price_unscale = make_scale([x[1] for x in all_car_data]) #5\n",
    "mileage_scale, mileage_unscale = make_scale([x[0] for x in all_car_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_car_data = [(mileage_scale(mileage), price_scale(price), is_bmw) \n",
    "                    for mileage,price,is_bmw in all_car_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 20432 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(scaled_car_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.2 Measuring BMWness of a car"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 22724 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(scaled_car_data)\n",
    "plot_function(lambda x: 0.56 - 0.35*x,0,1,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 504x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24028 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scalar_field_heatmap(lambda x,p: p + 0.35*x - 0.56, 0, 1,0,1)\n",
    "plt.ylabel('Price', fontsize=16)\n",
    "plt.xlabel('Mileage', fontsize=16)\n",
    "plot_function(lambda x: 0.56-0.35*x,0,1,c='k')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.3 Introducing the sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp\n",
    "def sigmoid(x):\n",
    "    return 1 / (1+exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10896 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_function(sigmoid,-5,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.4 Composing the sigmoid function with other functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x,p):\n",
    "    return p + 0.35 * x - 0.56\n",
    "\n",
    "def l(x,p):\n",
    "    return sigmoid(f(x,p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 504x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24848 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scalar_field_heatmap(l, 0, 1,0,1)\n",
    "# plot_data(scaled_car_data,white=True)\n",
    "plt.ylabel('Price', fontsize=16)\n",
    "plt.xlabel('Mileage', fontsize=16)\n",
    "plot_function(lambda x: 0.56-0.35*x,0,1,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55392 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(l, -5, 5, -5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55900 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(f,-5,5,-5,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.3.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Find a function $h(x)$ such that large positive values of $x$ cause $h(x)$ to be very close to $0$, large negative values of $x$ cause $h(x)$ to be very close to $1$, and $h(3) = 0.5$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def h(x):\n",
    "    return sigmoid(3-x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 11264 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_function(h,-5,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.4 Exploring possible logistic functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.4.1 Parametrizing logistic functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_logistic(a,b,c):\n",
    "    def l(x,p):\n",
    "        return sigmoid(a*x + b*p - c)\n",
    "    return l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.4.2 Measuring the quality of fit for a logistic function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_logistic_cost(a,b,c):\n",
    "    l = make_logistic(a,b,c)\n",
    "    errors = [abs(is_bmw-l(x,p)) \n",
    "              for x,p,is_bmw in scaled_car_data]\n",
    "    return sum(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10428 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from math import log\n",
    "plot_function(lambda x: -log(x),0.01,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.605170185988091"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-log(0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.907755278982137"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-log(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def point_cost(l,x,p,is_bmw): #1\n",
    "    wrong = 1 - is_bmw\n",
    "    return -log(abs(wrong - l(x,p)))\n",
    "\n",
    "def logistic_cost(a,b,c):\n",
    "    l = make_logistic(a,b,c)\n",
    "    errors = [point_cost(l,x,p,is_bmw) #2 \n",
    "              for x,p,is_bmw in scaled_car_data]\n",
    "    return sum(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.4.3 Testing different logistic functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_line(acoeff,bcoeff,ccoeff,**kwargs):\n",
    "    a,b,c = acoeff, bcoeff, ccoeff\n",
    "    \n",
    "    # black by default\n",
    "    if 'c' not in kwargs:\n",
    "        kwargs['c'] = 'k'\n",
    "        \n",
    "    if b == 0:\n",
    "        plt.plot([c/a,c/a],[0,1])\n",
    "    else:\n",
    "        def y(x):\n",
    "            return (c-a*x)/b\n",
    "        plt.plot([0,1],[y(0),y(1)],**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29500 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(scaled_car_data)\n",
    "plot_line(0.35,1,0.56,)\n",
    "plot_line(1,1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "130.92490748700456"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_cost(0.35,1,0.56)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135.56446830870456"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_cost(1,1,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.4.4 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Implement the function `plot_line(a,b,c)` referenced in the chapter that plots the line $ax + by = c$ where $0 \\leq x \\leq 1$ and $0 \\leq y \\leq 1$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Solution:** (see implementation above)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** What does the graph of $k(x,y) = \\sigma(x^2 + y^2 - 1)$ look like? What does the decision boundary look like, meaning the set of points where $k(x,y) = 0.5$?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 66564 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def k(x,y):\n",
    "    return sigmoid(x**2 + y**2 - 1)\n",
    "plot_scalar_field(k,-3,3,-3,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini-project:** Two equations $2x + y = 1$ and $4x + 2y = 2$ define the same line, and therefore the same decision boundary.  Are the logistic functions $\\sigma(2x + y - 1)$ and $\\sigma(4x + 2y - 2)$ the same?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56792 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(lambda x,y:sigmoid(2*x+y-1),-3,3,-3,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50904 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(lambda x,y:sigmoid(4*x+2*y-2),-3,3,-3,3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 15.5 Finding the best logistic function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.5.1 Gradient descent in three dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import sqrt\n",
    "def length(v):\n",
    "    return sqrt(sum([vi*vi for vi in v]))\n",
    "\n",
    "def secant_slope(f,xmin,xmax):\n",
    "    return (f(xmax) - f(xmin)) / (xmax - xmin)\n",
    "\n",
    "def approx_derivative(f,x,dx=1e-6):\n",
    "    return secant_slope(f,x-dx,x+dx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def approx_gradient(f,x0,y0,dx=1e-6):\n",
    "    partial_x = approx_derivative(lambda x:f(x,y0),x0,dx=dx)\n",
    "    partial_y = approx_derivative(lambda y:f(x0,y),y0,dx=dx)\n",
    "    return (partial_x,partial_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def approx_gradient3(f,x0,y0,z0,dx=1e-6):\n",
    "    partial_x = approx_derivative(lambda x:f(x,y0,z0),x0,dx=dx)\n",
    "    partial_y = approx_derivative(lambda y:f(x0,y,z0),y0,dx=dx)\n",
    "    partial_z = approx_derivative(lambda z:f(x0,y0,z),z0,dx=dx)\n",
    "    return (partial_x,partial_y,partial_z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent3(f,xstart,ystart,zstart,\n",
    "                      tolerance=1e-6,max_steps=1000):\n",
    "    x = xstart\n",
    "    y = ystart\n",
    "    z = zstart\n",
    "    grad = approx_gradient3(f,x,y,z)\n",
    "    steps = 0\n",
    "    while length(grad) > tolerance and steps < max_steps:\n",
    "        x -= 0.01 * grad[0]\n",
    "        y -= 0.01 * grad[1]\n",
    "        z -= 0.01 * grad[2]\n",
    "        grad = approx_gradient3(f,x,y,z)\n",
    "        steps += 1\n",
    "    return x,y,z"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.5.2 Using gradient descent to find the best fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.21114493148496014, 5.045439728207488, 2.1260122566471376)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient_descent3(logistic_cost,1,1,1,max_steps=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8845715265377516, 6.65754318727634, 2.9550572849988455)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient_descent3(logistic_cost,1,1,1,max_steps=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To make a cool graph of the decision boundary moving as gradient descent runs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 28620 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(scaled_car_data)\n",
    "for i in range(0,1000,100):\n",
    "    a,b,c = gradient_descent3(logistic_cost,1,1,1,max_steps=i)\n",
    "    plot_line(a,b,c,alpha=i/1000,c='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.7167003088210118, 11.422062406779267, 5.596878365032182)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient_descent3(logistic_cost,1,1,1,max_steps=8000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24912 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data(scaled_car_data)\n",
    "plot_line(0.35,1,0.56)\n",
    "plot_line(3.7167003153580045, 11.422062409195114, 5.596878367305919)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.5.3 Testing an understanding the best logistic classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_logistic_classifier(x,p):\n",
    "    l = make_logistic(3.7167003153580045, 11.422062409195114, 5.596878367305919)\n",
    "    if l(x,p) > 0.5:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_classifier(best_logistic_classifier,scaled_car_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 69964 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(make_logistic(0.35,1,0.56), -2, 2, -2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59724 characters]\n",
      "text/plain": [
       "<Figure size 504x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_scalar_field(make_logistic(3.7167003153580045, 11.422062409195114, 5.596878367305919), -2, 2, -2, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 15.5.5 Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** Modify the `gradient_descent3` function to print out the total number of steps taken before it returns its result.  How many steps does the gradient descent take to converge for `logistic_cost`?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent3(f,xstart,ystart,zstart,tolerance=1e-6,max_steps=1000):\n",
    "    x = xstart\n",
    "    y = ystart\n",
    "    z = zstart\n",
    "    grad = approx_gradient3(f,x,y,z)\n",
    "    steps = 0\n",
    "    while length(grad) > tolerance and steps < max_steps:\n",
    "        x -= 0.01 * grad[0]\n",
    "        y -= 0.01 * grad[1]\n",
    "        z -= 0.01 * grad[2]\n",
    "        grad = approx_gradient3(f,x,y,z)\n",
    "        steps += 1\n",
    "    print(steps)\n",
    "    return x,y,z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7243\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3.7167003088210118, 11.422062406779267, 5.596878365032182)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient_descent3(logistic_cost,1,1,1,max_steps=8000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mini project:** Write an `approx_gradient` function that can calculate the gradient of a function in any number of dimensions.  Then write a `gradient_descent` function that works in any number of dimensions.  To test your `gradient_descent` on an $n$-dimensional function, you can try a function like $f(x_1,x_2, \\ldots ,x_n) = (x_1 - 1)^2 + (x_2 - 1)^ + \\ldots + (x_n - 1)^2$ where $x_1, x_2, \\ldots , x_n$ are the $n$ input variables to the function $f$.  The minimum of this function should be $(1,1, \\ldots , 1)$, an $n$-dimensional vector with 1 in every entry."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def partial_derivative(f,i,v,**kwargs):\n",
    "    def cross_section(x):\n",
    "        arg = [(vj if j != i else x) for j,vj in enumerate(v)]\n",
    "        return f(*arg)\n",
    "    return approx_derivative(cross_section, v[i], **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def approx_gradient(f,v,dx=1e-6):\n",
    "    return [partial_derivative(f,i,v) for i in range(0,len(v))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(f,vstart,tolerance=1e-6,max_steps=1000):\n",
    "    v = vstart\n",
    "    grad = approx_gradient(f,v)\n",
    "    steps = 0\n",
    "    while length(grad) > tolerance and steps < max_steps:\n",
    "        v = [(vi - 0.01 * dvi) for vi,dvi in zip(v,grad)]\n",
    "        grad = approx_gradient(f,v)\n",
    "        steps += 1\n",
    "    return v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_squares(*v):\n",
    "    return sum([(x-1)**2 for x in v])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.0000002235452137,\n",
       " 1.0000002235452137,\n",
       " 1.0000002235452137,\n",
       " 1.0000002235452137,\n",
       " 1.0000002235452137]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = [2,2,2,2,2]\n",
    "gradient_descent(sum_squares,v)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
