{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_data(N=30, err=0.8, rseed=1):\n",
    "    rng = np.random.RandomState(rseed)\n",
    "    X = rng.rand(N, 1) ** 2\n",
    "    y = 10 - 1. / (X.ravel() + 0.1)\n",
    "    if err > 0:\n",
    "        y += err * rng.randn(N)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PolynomialRegression(degree=2, **kwargs):\n",
    "    return make_pipeline(PolynomialFeatures(degree), LinearRegression(**kwargs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31332 characters]\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model1  = PolynomialRegression(1).fit(X, y)\n",
    "model20 = PolynomialRegression(20).fit(X, y)\n",
    "fig, ax = plt.subplots(1, 2, figsize=(10, 5))\n",
    "#fig.subplots_adjust(left=0.0625, right=0.95, wspace=0.1)\n",
    "\n",
    "xfit = np.linspace(-0.1, 1.0, 1000)[:, None]\n",
    "\n",
    "ax[0].scatter(X.ravel(), y, s=40)\n",
    "ax[0].plot(xfit.ravel(), model1.predict(xfit), color='g')\n",
    "ax[0].axis([-0.1, 1.0, -2, 14])\n",
    "ax[0].set_title('High-bias model: Underfits the data', size=13)\n",
    "\n",
    "\n",
    "ax[1].scatter(X.ravel(), y, s=40)\n",
    "ax[1].plot(xfit.ravel(), model20.predict(xfit), color='g')\n",
    "ax[1].axis([-0.1, 1.0, -2, 14])\n",
    "ax[1].set_title('High-variance model: Overfits the data');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41204 characters]\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1, 2, figsize=(10, 5))\n",
    "fig.subplots_adjust(left=0.0625, right=0.95, wspace=0.1)\n",
    "\n",
    "X2, y2 = make_data(10, rseed=42)\n",
    "\n",
    "ax[0].scatter(X.ravel(), y, s=40, c='blue')\n",
    "ax[0].plot(xfit.ravel(), model1.predict(xfit), color='gray')\n",
    "ax[0].axis([-0.1, 1.0, -2, 14])\n",
    "ax[0].set_title('High-bias model: Underfits the data', size=14)\n",
    "ax[0].scatter(X2.ravel(), y2, s=40, c='red')\n",
    "\n",
    "ax[0].text(0.02, 0.98, \"training score: $R^2$ = {0:.2f}\".format(model1.score(X, y)),\n",
    "           ha='left', va='top', transform=ax[0].transAxes, size=14, color='blue')\n",
    "\n",
    "ax[0].text(0.02, 0.91, \"validation score: $R^2$ = {0:.2f}\".format(model1.score(X2, y2)),\n",
    "           ha='left', va='top', transform=ax[0].transAxes, size=14, color='red')\n",
    "\n",
    "ax[1].scatter(X.ravel(), y, s=40, c='blue')\n",
    "ax[1].plot(xfit.ravel(), model20.predict(xfit), color='gray')\n",
    "ax[1].axis([-0.1, 1.0, -2, 14])\n",
    "ax[1].set_title('High-variance model: Overfits the data', size=14)\n",
    "ax[1].scatter(X2.ravel(), y2, s=40, c='red')\n",
    "\n",
    "ax[1].text(0.02, 0.98, \"training score: $R^2$ = {0:.2g}\".format(model20.score(X, y)),\n",
    "           ha='left', va='top', transform=ax[1].transAxes, size=14, color='blue')\n",
    "\n",
    "ax[1].text(0.02, 0.91, \"validation score: $R^2$ = {0:.2g}\".format(model20.score(X2, y2)),\n",
    "           ha='left', va='top', transform=ax[1].transAxes, size=14, color='red');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Validation Curve Schematic')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 34272 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0, 1, 1000)\n",
    "y1 = -(x - 0.5) ** 2\n",
    "y2 = y1 - 0.33 + np.exp(x - 1)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(x, y2, lw=10, alpha=0.5, color='blue')\n",
    "ax.plot(x, y1, lw=10, alpha=0.5, color='red')\n",
    "\n",
    "ax.text(0.15, 0.2, \"training score\", rotation=45, size=16, color='blue')\n",
    "ax.text(0.2, -0.05, \"validation score\", rotation=20, size=16, color='red')\n",
    "\n",
    "ax.text(0.02, 0.1, r'$\\longleftarrow$ High Bias', size=18, rotation=90, va='center')\n",
    "ax.text(0.98, 0.1, r'$\\longleftarrow$ High Variance $\\longrightarrow$', size=18, rotation=90, ha='right', va='center')\n",
    "ax.text(0.48, -0.12, 'Best$\\\\longrightarrow$\\nModel', size=18, rotation=90, va='center')\n",
    "\n",
    "ax.set_xlim(0, 1)\n",
    "ax.set_ylim(-0.3, 0.5)\n",
    "\n",
    "ax.set_xlabel(r'model complexity $\\longrightarrow$', size=14)\n",
    "ax.set_ylabel(r'model score $\\longrightarrow$', size=14)\n",
    "\n",
    "ax.xaxis.set_major_formatter(plt.NullFormatter())\n",
    "ax.yaxis.set_major_formatter(plt.NullFormatter())\n",
    "\n",
    "ax.set_title(\"Validation Curve Schematic\", size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Learning Curve Schematic')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 28192 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "N = np.linspace(0, 1, 1000)\n",
    "y1 = 0.75 + 0.2 * np.exp(-4 * N)\n",
    "y2 = 0.7 - 0.6 * np.exp(-4 * N)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(x, y1, lw=10, alpha=0.5, color='blue')\n",
    "ax.plot(x, y2, lw=10, alpha=0.5, color='red')\n",
    "\n",
    "ax.text(0.2, 0.88, \"training score\", rotation=-10, size=16, color='blue')\n",
    "ax.text(0.2, 0.5, \"validation score\", rotation=30, size=16, color='red')\n",
    "\n",
    "ax.text(0.98, 0.45, r'Good Fit $\\longrightarrow$', size=18, rotation=90, ha='right', va='center')\n",
    "ax.text(0.02, 0.57, r'$\\longleftarrow$ High Variance $\\longrightarrow$', size=18, rotation=90, va='center')\n",
    "\n",
    "ax.set_xlim(0, 1)\n",
    "ax.set_ylim(0, 1)\n",
    "\n",
    "ax.set_xlabel(r'training set size $\\longrightarrow$', size=14)\n",
    "ax.set_ylabel(r'model score $\\longrightarrow$', size=14)\n",
    "\n",
    "ax.xaxis.set_major_formatter(plt.NullFormatter())\n",
    "ax.yaxis.set_major_formatter(plt.NullFormatter())\n",
    "\n",
    "ax.set_title(\"Learning Curve Schematic\", size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
