W1-Definitivo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "# Workbook analyzing DIGITS dataset from scikit-learn using K Nearest Neighbours Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Authors:\n",
    "\n",
    "        Fco Rafael Tendero Jiménez \n",
    "        Fco Javier Rodríguez Sánchez"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "1. Election of dataset: Digits dataset is going to be used.\n",
    "This is a classification dataset where each data-point is a 8x8 image image of a digit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "digits = load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAC85JREFUeJzt3d+LXPUZx/HPxzXBX9FItUGMuBVKQISaIKESkDRRiVXSXPQiAYVIS3rRSmILor0p/gOyvShCiBrBGNFopEhrDWgQodUmcdXoxmJCxO2qa5AkaiHB5OnFnMgatu7Z7X6/O7PP+wVDZnbPzvPMbj5zzpk5cx5HhADkcs5MNwCgPoIPJETwgYQIPpAQwQcSIvhAQl0RfNurbL9v+wPb9xeu9ajtUdv7S9YZU+8q26/YHrL9ru2NheudZ/sN22819R4sWa+p2Wf7TdsvlK7V1Dts+x3bg7b3FK413/YO2weav+GNBWstah7Tmctx25uKFIuIGb1I6pN0UNI1kuZKekvStQXr3SRpiaT9lR7fFZKWNNfnSfpX4cdnSRc11+dIel3Sjws/xt9KelLSC5V+p4clXVap1uOSftlcnytpfqW6fZI+kXR1ifvvhjX+UkkfRMShiDgp6SlJPytVLCJelfR5qfsfp97HEbGvuf6FpCFJVxasFxHxZXNzTnMpdpSW7YWSbpe0pVSNmWL7YnVWFI9IUkScjIijlcqvlHQwIj4scefdEPwrJX005vawCgZjJtnul7RYnbVwyTp9tgcljUraFREl6w1Iuk/S6YI1zhaSXrK91/aGgnWukfSZpMeaXZktti8sWG+stZK2l7rzbgi+x/narDuO2PZFkp6VtCkijpesFRGnIuJ6SQslLbV9XYk6tu+QNBoRe0vc/3dYFhFLJN0m6de2bypU51x1dgsfjojFkr6SVPQ1KEmyPVfSaknPlKrRDcEflnTVmNsLJY3MUC9F2J6jTui3RcRzteo2m6W7Ja0qVGKZpNW2D6uzi7bC9hOFan0jIkaaf0cl7VRnd7GEYUnDY7aYdqjzRFDabZL2RcSnpQp0Q/D/KemHtn/QPNOtlfTnGe5p2ti2OvuIQxHxUIV6l9ue31w/X9LNkg6UqBURD0TEwojoV+fv9nJE3Fmi1hm2L7Q978x1SbdKKvIOTUR8Iukj24uaL62U9F6JWmdZp4Kb+VJnU2ZGRcTXtn8j6W/qvJL5aES8W6qe7e2Slku6zPawpD9ExCOl6qmzVrxL0jvNfrck/T4i/lKo3hWSHrfdp84T+9MRUeVttkoWSNrZeT7VuZKejIgXC9a7R9K2ZqV0SNLdBWvJ9gWSbpH0q6J1mrcOACTSDZv6ACoj+EBCBB9IiOADCRF8IKGuCn7hwy9nrBb1qNdt9boq+JJq/nKr/iGpR71uqtdtwQdQQZEDeGzP6qOCLrnkkkn/zMmTJzV37twp1VuwYMGkf+bYsWNT6lOSDh48OOmfOX36tM45Z2rrkVOnTk3p5zC+iBjvg2/fMuOH7Pai5cuXV6137733Vq23Zs2aqvWOHq31EXecwaY+kBDBBxIi+EBCBB9IiOADCRF8ICGCDyRE8IGEWgW/5ogrAOVNGPzmpI1/UueUv9dKWmf72tKNASinzRq/6ogrAOW1CX6aEVdAFm0+pNNqxFVz4oDan1kGMAVtgt9qxFVEbJa0WZr9H8sFel2bTf1ZPeIKyGjCNX7tEVcAymt1Io5mzlupWW8AKuPIPSAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IiOADCTFCawpqT36pXW9gYGBW15vt2ozQYo0PJETwgYQIPpAQwQcSIvhAQgQfSIjgAwkRfCAhgg8kRPCBhNqM0HrU9qjt/TUaAlBemzX+VkmrCvcBoKIJgx8Rr0r6vEIvACphHx9IqNV59dtgdh7QO6Yt+MzOA3oHm/pAQm3eztsu6e+SFtketv2L8m0BKKnN0Mx1NRoBUA+b+kBCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEmJ23hQMDg5Wrbd8+fKq9Z5//vmq9Wo/vtmO2XkAxkXwgYQIPpAQwQcSIvhAQgQfSIjgAwkRfCAhgg8kRPCBhNqcbPMq26/YHrL9ru2NNRoDUE6b8+p/Lel3EbHP9jxJe23vioj3CvcGoJA2s/M+joh9zfUvJA1JurJ0YwDKmdQ+vu1+SYslvV6iGQB1tB6hZfsiSc9K2hQRx8f5PrPzgB7RKvi256gT+m0R8dx4yzA7D+gdbV7Vt6RHJA1FxEPlWwJQWpt9/GWS7pK0wvZgc/lp4b4AFNRmdt5rkiY8lQ+A3sGRe0BCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEmr9IZ1u1t/fX7Ve7dl5R48erVqv9u8T9bHGBxIi+EBCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEiL4QEJtzrJ7nu03bL/VzM57sEZjAMppc6z+CUkrIuLL5vz6r9n+a0T8o3BvAAppc5bdkPRlc3NOc2FgBtDDWu3j2+6zPShpVNKuiGB2HtDDWgU/Ik5FxPWSFkpaavu6s5exvcH2Htt7prtJANNrUq/qR8RRSbslrRrne5sj4oaIuGGaegNQSJtX9S+3Pb+5fr6kmyUdKN0YgHLavKp/haTHbfep80TxdES8ULYtACW1eVX/bUmLK/QCoBKO3AMSIvhAQgQfSIjgAwkRfCAhgg8kRPCBhAg+kJA7n7qd5ju1Z/XHdmvPljt8+HDVeiX+T3yXSy+9tGq92rMIa4sIT7QMa3wgIYIPJETwgYQIPpAQwQcSIvhAQgQfSIjgAwkRfCAhgg8k1Dr4zVCNN21zok2gx01mjb9R0lCpRgDU03aE1kJJt0vaUrYdADW0XeMPSLpP0umCvQCopM0knTskjUbE3gmWY3Ye0CParPGXSVpt+7CkpyStsP3E2QsxOw/oHRMGPyIeiIiFEdEvaa2klyPizuKdASiG9/GBhNoMzfxGROxWZ0w2gB7GGh9IiOADCRF8ICGCDyRE8IGECD6QEMEHEiL4QEKTOoAHHbVn2a1fv75qvWPHjlWtN9tn2XUj1vhAQgQfSIjgAwkRfCAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IqNUhu82ptb+QdErS15xCG+htkzlW/ycRcaRYJwCqYVMfSKht8EPSS7b32t5QsiEA5bXd1F8WESO2vy9pl+0DEfHq2AWaJwSeFIAe0GqNHxEjzb+jknZKWjrOMszOA3pEm2m5F9qed+a6pFsl7S/dGIBy2mzqL5C00/aZ5Z+MiBeLdgWgqAmDHxGHJP2oQi8AKuHtPCAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IiOADCTkipv9O7em/0y4yMDBQtd7GjRur1qs9O2/r1q1V69We1Vfz8Y2MjOjEiROeaDnW+EBCBB9IiOADCRF8ICGCDyRE8IGECD6QEMEHEiL4QEIEH0ioVfBtz7e9w/YB20O2byzdGIBy2g7U+KOkFyPi57bnSrqgYE8ACpsw+LYvlnSTpPWSFBEnJZ0s2xaAktps6l8j6TNJj9l+0/aWZrDGt9jeYHuP7T3T3iWAadUm+OdKWiLp4YhYLOkrSfefvRAjtIDe0Sb4w5KGI+L15vYOdZ4IAPSoCYMfEZ9I+sj2ouZLKyW9V7QrAEW1fVX/Hknbmlf0D0m6u1xLAEprFfyIGJTEvjswS3DkHpAQwQcSIvhAQgQfSIjgAwkRfCAhgg8kRPCBhNoeuYcxas966+/vr1pvcHCwar01a9ZUrVd7dt7u3bur1Tpy5Eir5VjjAwkRfCAhgg8kRPCBhAg+kBDBBxIi+EBCBB9IiOADCU0YfNuLbA+OuRy3valGcwDKmPCQ3Yh4X9L1kmS7T9K/Je0s3BeAgia7qb9S0sGI+LBEMwDqmGzw10raXqIRAPW0Dn5zTv3Vkp75H99ndh7QIybzsdzbJO2LiE/H+2ZEbJa0WZJsxzT0BqCQyWzqrxOb+cCs0Cr4ti+QdIuk58q2A6CGtiO0/iPpe4V7AVAJR+4BCRF8ICGCDyRE8IGECD6QEMEHEiL4QEIEH0iI4AMJOWL6P09j+zNJU/nM/mWS2g3/+v/VrEU96tWqd3VEXD7RQkWCP1W290TEDbOtFvWo12312NQHEiL4QELdFvzNs7QW9ajXVfW6ah8fQB3dtsYHUAHBBxIi+EBCBB9IiOADCf0X6MSztJibpesAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# An example of the data:\n",
    "import matplotlib.pyplot as plt\n",
    "plt.gray()\n",
    "plt.matshow(digits.images[12])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Note that num features is 64 because it's a 8x8 image, now we print then number of observations and features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(number of observations, number of features): (1797, 64)\n",
      "Num of digit '0' samples: 178\n",
      "Num of digit '1' samples: 182\n",
      "Num of digit '2' samples: 177\n",
      "Num of digit '3' samples: 183\n",
      "Num of digit '4' samples: 181\n",
      "Num of digit '5' samples: 182\n",
      "Num of digit '6' samples: 181\n",
      "Num of digit '7' samples: 179\n",
      "Num of digit '8' samples: 174\n",
      "Num of digit '9' samples: 180\n"
     ]
    }
   ],
   "source": [
    "# Data is a dictionary. We can then navigate through the dataset using the keys\n",
    "digits.keys()\n",
    "X = digits['data']\n",
    "y = digits['target']\n",
    "\n",
    "print(\"(number of observations, number of features): \"+ str(X.shape))\n",
    "\n",
    "# Classes: (each one of the 10 class is a digit from 0 to 9\n",
    "classes = digits['target_names']\n",
    "\n",
    "# Number of samples per class\n",
    "for i in classes:\n",
    "    print(\"Num of digit '\" + str(i) + \"' samples: \" + str(sum(y == i)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Now, we split the dataset where 80% of the data is used for training, and 20% for test. K Nearest Neighbors Classifier is going to be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=1, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 1}\n",
      "\n",
      "Training score: 0.9895615866388309\n",
      "Test score: 0.9861111111111112\n"
     ]
    }
   ],
   "source": [
    "# the dataset is split so that 80% of the data is used for training, and 20% for test.\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=13)\n",
    "\n",
    "# K Nearest Neighbors Classifier is going to be used\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "myKNN = KNeighborsClassifier()\n",
    "\n",
    "# required imports for performing grid search cross validation and stratified k fold technique.\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# A param grid dictionary is created with the parameters to try in the cross-validation process\n",
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13, 15]}\n",
    "\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "myGSCV = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                      cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred = myGSCV.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Using the best parameters for KNeighborsClassifier. About the training data we obtain as accuracy 0.98956 and on the test data we obtain as accuracy 0.98611. Therefore, we obtain a high precision."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "4. Using Leave one out validation.\n",
    "Now, in each iteration of the cross validation process, all observations except one will be used for training.\n",
    "This means that, in the first iteration, observation #1 is out, in second iteration, observation #2 is out, and so on.\n",
    "Thus, there will be as many iterations as training samples (length of X_train), which means a high computational cost.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1437 folds for each of 8 candidates, totalling 11496 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    8.8s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   19.3s\n",
      "[Parallel(n_jobs=-1)]: Done 357 tasks      | elapsed:   38.7s\n",
      "[Parallel(n_jobs=-1)]: Done 640 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1005 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1450 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1977 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 2584 tasks      | elapsed:  7.8min\n",
      "[Parallel(n_jobs=-1)]: Done 3273 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done 4042 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 4893 tasks      | elapsed: 19.4min\n",
      "[Parallel(n_jobs=-1)]: Done 5824 tasks      | elapsed: 22.8min\n",
      "[Parallel(n_jobs=-1)]: Done 6837 tasks      | elapsed: 26.1min\n",
      "[Parallel(n_jobs=-1)]: Done 7930 tasks      | elapsed: 29.7min\n",
      "[Parallel(n_jobs=-1)]: Done 9105 tasks      | elapsed: 33.8min\n",
      "[Parallel(n_jobs=-1)]: Done 10360 tasks      | elapsed: 38.1min\n",
      "[Parallel(n_jobs=-1)]: Done 11496 out of 11496 | elapsed: 42.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=1, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 1}\n",
      "\n",
      "Training score: 0.9895615866388309\n",
      "Test score: 0.9861111111111112\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import LeaveOneOut\n",
    "myLOO = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy', cv=LeaveOneOut(), verbose=2, n_jobs=-1)\n",
    "\n",
    "# Training of the model\n",
    "myLOO.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_predLOO = myLOO.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myLOO.best_estimator_))   # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myLOO.best_params_))    # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myLOO.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myLOO.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Using Leave one out validation about the training data we obtain as accuracy 0.98956 and on the test data we obtain as accuracy 0.98611. Therefore, we obtain a high precision."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "5. Stratified K Fold.\n",
    "There is a similar number of samples per each class, so, if this distribution is preserved when\n",
    "splitting into training (X_train) and (X_test), stratification is not indispensable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes distribution in Training\n",
      "Num of digit '0' samples: 148\n",
      "Num of digit '1' samples: 145\n",
      "Num of digit '2' samples: 139\n",
      "Num of digit '3' samples: 146\n",
      "Num of digit '4' samples: 143\n",
      "Num of digit '5' samples: 137\n",
      "Num of digit '6' samples: 139\n",
      "Num of digit '7' samples: 153\n",
      "Num of digit '8' samples: 140\n",
      "Num of digit '9' samples: 147\n",
      "\n",
      "Classes distribution in Test\n",
      "Num of digit '0' samples: 30\n",
      "Num of digit '1' samples: 37\n",
      "Num of digit '2' samples: 38\n",
      "Num of digit '3' samples: 37\n",
      "Num of digit '4' samples: 38\n",
      "Num of digit '5' samples: 45\n",
      "Num of digit '6' samples: 42\n",
      "Num of digit '7' samples: 26\n",
      "Num of digit '8' samples: 34\n",
      "Num of digit '9' samples: 33\n",
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=1, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 1}\n",
      "\n",
      "Training score: 0.9895615866388309\n",
      "Test score: 0.9861111111111112\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('Classes distribution in Training')\n",
    "for i in classes:\n",
    "    print(\"Num of digit '\" + str(i) + \"' samples: \" + str(sum(y_train == i)))\n",
    "\n",
    "print('\\nClasses distribution in Test')\n",
    "for i in classes:\n",
    "    print(\"Num of digit '\" + str(i) + \"' samples: \" + str(sum(y_test == i)))\n",
    "\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "myGSCV_noStrat = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                              cv=KFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_noStrat.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred_noStrat = myGSCV_noStrat.predict(X_test)\n",
    "\n",
    "# Results. It's shown how the score is not affected when not using Stratified KFOLD\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_noStrat.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_noStrat.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV_noStrat.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV_noStrat.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Also, we get a high precission."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "6. Distance Weights\n",
    "uniform weights (default): all points in each neighborhood have same weight (used in exercise 3)\n",
    "distance weights: points closer to the evaluated will have more influence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=3, p=2,\n",
      "           weights='distance')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 3, 'weights': 'distance'}\n",
      "\n",
      "Training score: 0.9902574808629089\n",
      "Test score: 0.9805555555555555\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13, 15], 'weights': ['uniform', 'distance']}\n",
    "\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "\n",
    "myGSCV_w = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "\n",
    "                        cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_w.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "\n",
    "y_pred_w = myGSCV_w.predict(X_test)\n",
    "\n",
    "\n",
    "# Results. Note that distance weights are preferred\n",
    "\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_w.best_estimator_))  # best estimator\n",
    "\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_w.best_params_))   # parameters of the best estimator\n",
    "\n",
    "print(\"\\nTraining score: \" + str(myGSCV_w.best_score_))  # training score for achieved with the best estimator\n",
    "\n",
    "print(\"Test score: \" + str(myGSCV_w.score(X_test, y_test)))  # test score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "7. Testing different metrics.\n",
    "The default metric used is minkowski. Now euclidean and manhattan metrics will be taken into account, thus, they will\n",
    "be included in the param grid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=1, p=3,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 1, 'p': 3, 'weights': 'uniform'}\n",
      "\n",
      "Training score: 0.9902574808629089\n",
      "Test score: 0.9833333333333333\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Also, the value 1 and 2 for the parameter 'p' are manhattan and euclidean metrics and for minkowski, a parameter 'p' can be adjusted, so the different values will also be included.\n",
    "\"\"\"\n",
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13],\n",
    "              'weights': ['uniform', 'distance'],\n",
    "              'p': [1, 2, 3, 4, 5, 6, 7, 8, 9]}\n",
    "\n",
    "myGSCV_m = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                        cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_m.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred_m = myGSCV_m.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_m.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_m.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV_m.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV_m.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "The parameters of the best stimator are 1 neihbors and p = '3' with minkowski metrc and weights uniform. Finally we get a 0.98 about test data and 0.99 for training data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "8. Now, let's repeat the previous sections with the following data.\n",
    "\n",
    "QSAR DATASET: https://archive.ics.uci.edu/ml/datasets/QSAR+biodegradation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(number of observations, number of features): (1055, 41)\n",
      "Num of Classes 'RB' samples: 356\n",
      "Num of Classes 'NRB' samples: 699\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import metrics\n",
    "\n",
    "# Importing csv dataset into a numpy structure (2)\n",
    "X = np.genfromtxt('data/biodeg.csv', delimiter=\";\", skip_header=0, usecols=range(0, 41))    #1055x41\n",
    "y = np.genfromtxt('data/biodeg.csv', delimiter=\";\", skip_header=0, usecols=-1, dtype=str)   #1055x1\n",
    "\n",
    "\n",
    "# (number of observations, number of features)\n",
    "print(\"(number of observations, number of features): \"+ str(X.shape))\n",
    "\n",
    "# classes:\n",
    "classes = [\"RB\", \"NRB\"]   # Ready-Biodegradable, Not Ready-Biodegradable\n",
    "\n",
    "\n",
    "# number of samples per class\n",
    "for i in classes:\n",
    "    print(\"Num of Classes '\" + str(i) + \"' samples: \" + str(sum(y == i)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "The dataset is split so that 80% of the data is used for training, and 20% for test and K Nearest Neighbors Classifier is going to be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=7, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 7}\n",
      "\n",
      "Training score: 0.8139810426540285\n",
      "Test score: 0.8293838862559242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/master/Aplicaciones/anaconda/lib/python3.6/site-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=13)\n",
    "\n",
    "# K Nearest Neighbors Classifier is going to be used\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "myKNN = KNeighborsClassifier()\n",
    "\n",
    "# required imports for performing grid search cross validation and stratified k fold technique.\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# A param grid dictionary is created with the parameters to try in the cross-validation process\n",
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13, 15]}\n",
    "\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "myGSCV = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                      cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred = myGSCV.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Through KNeighborsClassifier we get over training data 0.81 accuracy and over the test data 0.83. The parameters over the best configurations are 7 neighbors."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Using Leave one out validation.\n",
    "Now, in each iteration of the cross validation process, all observations except one will be used for training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 844 folds for each of 8 candidates, totalling 6752 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    4.4s\n",
      "[Parallel(n_jobs=-1)]: Done 493 tasks      | elapsed:   12.5s\n",
      "[Parallel(n_jobs=-1)]: Done 1305 tasks      | elapsed:   29.6s\n",
      "[Parallel(n_jobs=-1)]: Done 2437 tasks      | elapsed:   60.0s\n",
      "[Parallel(n_jobs=-1)]: Done 3897 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 5677 tasks      | elapsed:  2.6min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=3, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 3}\n",
      "\n",
      "Training score: 0.8104265402843602\n",
      "Test score: 0.8246445497630331\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 6752 out of 6752 | elapsed:  3.2min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import LeaveOneOut\n",
    "myLOO = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy', cv=LeaveOneOut(), verbose=2, n_jobs=-1)\n",
    "\n",
    "# Training of the model\n",
    "myLOO.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_predLOO = myLOO.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myLOO.best_estimator_))   # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myLOO.best_params_))    # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myLOO.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myLOO.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "We get over training data 0.81 accuracy and over the test data 0.82. The parameters over the best configurations are 3 neighbors, with uniforms weights, p = 2 and minkowski metric."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Stratified K Fold.\n",
    "There is a similar number of samples per each class, so, if this distribution is preserved when\n",
    "splitting into training (X_train) and (X_test), stratification is not indispensable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes distribution in Training\n",
      "Num of digit 'RB' samples: 291\n",
      "Num of digit 'NRB' samples: 553\n",
      "\n",
      "Classes distribution in Test\n",
      "Num of digit 'RB' samples: 65\n",
      "Num of digit 'NRB' samples: 146\n",
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=3, p=2,\n",
      "           weights='uniform')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 3}\n",
      "\n",
      "Training score: 0.8127962085308057\n",
      "Test score: 0.8246445497630331\n"
     ]
    }
   ],
   "source": [
    "print('Classes distribution in Training')\n",
    "for i in classes:\n",
    "    print(\"Num of digit '\" + str(i) + \"' samples: \" + str(sum(y_train == i)))\n",
    "\n",
    "print('\\nClasses distribution in Test')\n",
    "for i in classes:\n",
    "    print(\"Num of digit '\" + str(i) + \"' samples: \" + str(sum(y_test == i)))\n",
    "\n",
    "# Let's try then with KFold (no Stratified)\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "myGSCV_noStrat = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                              cv=KFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_noStrat.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred_noStrat = myGSCV_noStrat.predict(X_test)\n",
    "\n",
    "# Results. It's shown how the score is not affected when not using Stratified KFOLD\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_noStrat.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_noStrat.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV_noStrat.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV_noStrat.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "We get over training data 0.81 accuracy and over the test data 0.82. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Distance Weights uniform weights (default): all points in each neighborhood have same weight (used in exercise 3)\n",
    "distance weights: points closer to the evaluated will have more influence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=7, p=2,\n",
      "           weights='distance')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'n_neighbors': 7, 'weights': 'distance'}\n",
      "\n",
      "Training score: 0.8246445497630331\n",
      "Test score: 0.8246445497630331\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/master/Aplicaciones/anaconda/lib/python3.6/site-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13, 15], 'weights': ['uniform', 'distance']}\n",
    "# KNN estimator, accuracy as scoring, and the data will be split into 10 chunks, thus, it will take 10 iterations\n",
    "myGSCV_w = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                        cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_w.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred_w = myGSCV_w.predict(X_test)\n",
    "\n",
    "# Results. Note that distance weights are preferred\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_w.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_w.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV_w.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV_w.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "We get over training data 0.82 accuracy and over the test data 0.82."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "Testing different metrics.\n",
    "The default metric used is minkowski. Now euclidean and manhattan metrics will be taken into account, thus, they will\n",
    "be included in the param grid.\n",
    "Also, for minkowski, a parameter 'p' can be adjusted, so the different values will also be included."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {}
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best Estimator:\n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='manhattan',\n",
      "           metric_params=None, n_jobs=None, n_neighbors=7, p=2,\n",
      "           weights='distance')\n",
      "\n",
      "Parameters of best estimator:\n",
      "{'metric': 'manhattan', 'n_neighbors': 7, 'p': 2, 'weights': 'distance'}\n",
      "\n",
      "Training score: 0.8376777251184834\n",
      "Test score: 0.8341232227488151\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/master/Aplicaciones/anaconda/lib/python3.6/site-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'n_neighbors': [1, 3, 5, 7, 9, 11, 13],\n",
    "              'weights': ['uniform', 'distance'],\n",
    "              'metric': ['euclidean', 'manhattan', 'minkowski'],\n",
    "              'p': [2, 3, 4, 5, 6, 7, 8, 9]}\n",
    "\n",
    "myGSCV_m = GridSearchCV(estimator=myKNN, param_grid=param_grid, scoring='accuracy',\n",
    "                        cv=StratifiedKFold(n_splits=10, random_state=3))\n",
    "\n",
    "# Training of the model\n",
    "myGSCV_m.fit(X_train, y_train)\n",
    "\n",
    "# prediction (using best_estimator_ by default)\n",
    "y_pred_m = myGSCV_m.predict(X_test)\n",
    "\n",
    "# Results\n",
    "print(\"\\nBest Estimator:\\n\" + str(myGSCV_m.best_estimator_))  # best estimator\n",
    "print(\"\\nParameters of best estimator:\\n\" + str(myGSCV_m.best_params_))   # parameters of the best estimator\n",
    "print(\"\\nTraining score: \" + str(myGSCV_m.best_score_))  # training score for achieved with the best estimator\n",
    "print(\"Test score: \" + str(myGSCV_m.score(X_test, y_test)))  # test score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {}
   },
   "source": [
    "We get over training data 0.84 accuracy and over the test data 0.83."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}