1-keras-and-tensorflow/Keras_CNN_best_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras and Tensorflow model for Aerial Cactus Detection in Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing tensorflow and keras packages for NN\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Conv2D, Dense, Flatten, BatchNormalization, Dropout, LeakyReLU, Flatten\n",
    "from keras import regularizers\n",
    "from keras.optimizers import Adam\n",
    "from keras.optimizers import SGD\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.constraints import maxnorm\n",
    "from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n",
    "\n",
    "#Importing the scikitlearn packages for ML metrics and grid search\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import RandomizedSearchCV,KFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "#Importing tqdm for progress bars\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "\n",
    "#Importing pandas and numpy for array manipulation and cv to read in the images as arrays\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import cv2 as cv\n",
    "\n",
    "#For paths\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting seed for reproducibility\n",
    "seed = 12345\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Navigating to directory with data\n",
    "os.listdir('../input')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Setting the path for training and test images\n",
    "training_path = '../input/train/train/'\n",
    "test_path = '../input/test/test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading in training labels\n",
    "train_data = pd.read_csv('../input/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>has_cactus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0004be2cfeaba1c0361d39e2b000257b.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000c8a36845c0208e833c79c1bffedd1.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000d1e9a533f62e55c289303b072733d.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0011485b40695e9138e92d0b3fb55128.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0014d7a11e90b62848904c1418fc8cf2.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  has_cactus\n",
       "0  0004be2cfeaba1c0361d39e2b000257b.jpg           1\n",
       "1  000c8a36845c0208e833c79c1bffedd1.jpg           1\n",
       "2  000d1e9a533f62e55c289303b072733d.jpg           1\n",
       "3  0011485b40695e9138e92d0b3fb55128.jpg           1\n",
       "4  0014d7a11e90b62848904c1418fc8cf2.jpg           1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70ae6c31a50e4dd790c1508b339431ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=17500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Convert training images to numpy arrays \n",
    "\n",
    "images_train = []\n",
    "labels_train = []\n",
    "\n",
    "images = train_data['id'].values\n",
    "for image_id in tqdm_notebook(images):     #<- tqdm_notebook adds a progress bar in jupyter notebook\n",
    "    \n",
    "    image = np.array(cv.imread(training_path + image_id))\n",
    "    label = train_data[train_data['id'] == image_id]['has_cactus'].values[0]\n",
    "    \n",
    "    images_train.append(image)             #<- Add original image\n",
    "    labels_train.append(label)\n",
    "    \n",
    "    # Data Augmentation\n",
    "    images_train.append(np.flip(image))    #<- Add flipped up down and left-right image\n",
    "    labels_train.append(label)\n",
    "    \n",
    "    images_train.append(np.flipud(image))  #<- Add flipped up down image\n",
    "    labels_train.append(label)\n",
    "    \n",
    "    images_train.append(np.fliplr(image))  #<- Add flipped left right image\n",
    "    labels_train.append(label)\n",
    "    \n",
    "    \n",
    "images_train = np.asarray(images_train)    #<- Convert combined list to np array\n",
    "images_train = images_train.astype('float32')\n",
    "images_train /= 255.                       #<- Normalize          \n",
    "  \n",
    "labels_train = np.asarray(labels_train)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebdd3e6cd6494dd781e7e522b570e788",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Convert test images to numpy arrays \n",
    "\n",
    "test_images_names = []\n",
    "\n",
    "for filename in os.listdir(test_path):\n",
    "    test_images_names.append(filename)\n",
    "    \n",
    "test_images_names.sort()\n",
    "\n",
    "images_test = []\n",
    "\n",
    "for image_id in tqdm_notebook(test_images_names):\n",
    "    images_test.append(np.array(cv.imread(test_path + image_id)))\n",
    "    \n",
    "images_test = np.asarray(images_test)\n",
    "images_test = images_test.astype('float32')\n",
    "images_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into test and train using Stratified sampling\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(images_train, labels_train, test_size = 0.2, stratify = labels_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define AUC as metric since Kaggle competition is evaluated on AUC\n",
    "def auroc(y_true, y_pred):\n",
    "    return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The hyperparameter values being passed to create the model below are the parameters of the best model from a grid search\n",
    "#The code for the grid search is here. \n",
    "\n",
    "def create_model(optimizer='rmsprop', filters_1=16,filters_2=2,filters_3=4,filters_4=32,\n",
    "                 nn1=300, nn2=100, nn3 = 150, lr=0.01, l2=0.0001, l1=0,\n",
    "                activation = 'relu', dropout_1=0.4, dropout_2=0, dropout_3=0.2, dropout_4=0.2): \n",
    "    \n",
    "    #Apply l1 L2 regularization to the NN layers\n",
    "    reg = regularizers.l1_l2(l1=l1, l2=l2)\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    #Our input images are 32*32 pixels and have 3 color channels \n",
    "    model.add(Conv2D(filters = filters_1, kernel_size = 3, activation = activation, input_shape = (32, 32, 3)))\n",
    "    \n",
    "    model.add(Conv2D(filters = filters_2, kernel_size = 3, activation = activation))\n",
    "    #Normalizing intermediate layers imrpoves training speed\n",
    "    model.add(BatchNormalization())\n",
    "    #Dropout reduces training accuracy but improves test and validation accuracy\n",
    "    model.add(Dropout(dropout_1))\n",
    "    \n",
    "    model.add(Conv2D(filters = filters_3, kernel_size = 1, activation = activation))\n",
    "    model.add(Conv2D(filters = filters_4, kernel_size = 1, activation = activation))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(dropout_2))\n",
    "   \n",
    "    #Output from convolutional layers converted to a flat array \n",
    "    model.add(Flatten())\n",
    "    \n",
    "    model.add(Dense(nn1, activation = activation,kernel_regularizer=reg))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(dropout_3))\n",
    "    \n",
    "    model.add(Dense(nn2, activation = activation,kernel_regularizer=reg))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(dropout_4))\n",
    "    \n",
    "    model.add(Dense(nn3, activation = 'tanh', kernel_regularizer=reg))\n",
    "    \n",
    "    #Output layer\n",
    "    model.add(Dense(1, activation = 'sigmoid'))\n",
    "    \n",
    "    #Define optimizer \n",
    "    if optimizer =='sgd':\n",
    "        optimizer = SGD(lr=lr)\n",
    "    elif optimizer == 'adam':\n",
    "        optimizer = Adam(lr=lr)\n",
    "    elif optimizer == 'rmspop':\n",
    "        optimizer = RMSprop(lr=lr)\n",
    "        \n",
    "    #Compile model\n",
    "    #We use mean_squared_error instead of the more commonly used binary_crossentropy \n",
    "    #because it significantly improved the accuracy\n",
    "    model.compile(optimizer = optimizer , loss= \"mean_squared_error\", metrics = ['accuracy',auroc])\n",
    "    \n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From <ipython-input-12-1d61a90f3982>:3: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "tf.py_func is deprecated in TF V2. Instead, use\n",
      "    tf.py_function, which takes a python function which manipulates tf eager\n",
      "    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to\n",
      "    an ndarray (just call tensor.numpy()) but having access to eager tensors\n",
      "    means `tf.py_function`s can use accelerators such as GPUs as well as\n",
      "    being differentiable using a gradient tape.\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "model = create_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define where weights will be stored in case we want to use them later.\n",
    "\n",
    "file_path = 'weights-aerial-cactus.h5'\n",
    "\n",
    "#Callbacks for training. \n",
    "callbacks = [\n",
    "        #Save the model after every epoch\n",
    "        ModelCheckpoint(file_path, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max'),\n",
    "        #Reduce learning rate when loss has stopped improving.\n",
    "        ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 1, mode = 'min', min_lr = 0.00001),\n",
    "        #Stop training when loss has stopped improving by delta\n",
    "        EarlyStopping(monitor = 'val_loss', min_delta = 1e-10, patience = 15, verbose = 1, restore_best_weights = True)\n",
    "        ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 56000 samples, validate on 14000 samples\n",
      "Epoch 1/80\n",
      "56000/56000 [==============================] - 180s 3ms/step - loss: 0.1163 - acc: 0.9138 - auroc: 0.9527 - val_loss: 0.3995 - val_acc: 0.5753 - val_auroc: 0.9238\n",
      "Epoch 2/80\n",
      "56000/56000 [==============================] - 168s 3ms/step - loss: 0.0734 - acc: 0.9487 - auroc: 0.9843 - val_loss: 0.0913 - val_acc: 0.9238 - val_auroc: 0.9926\n",
      "Epoch 3/80\n",
      "56000/56000 [==============================] - 159s 3ms/step - loss: 0.0548 - acc: 0.9638 - auroc: 0.9916 - val_loss: 0.0550 - val_acc: 0.9576 - val_auroc: 0.9982\n",
      "Epoch 4/80\n",
      "56000/56000 [==============================] - 165s 3ms/step - loss: 0.0343 - acc: 0.9791 - auroc: 0.9971 - val_loss: 0.0319 - val_acc: 0.9801 - val_auroc: 0.9984\n",
      "Epoch 5/80\n",
      "56000/56000 [==============================] - 163s 3ms/step - loss: 0.0278 - acc: 0.9838 - auroc: 0.9980 - val_loss: 0.0266 - val_acc: 0.9839 - val_auroc: 0.9984\n",
      "Epoch 6/80\n",
      "56000/56000 [==============================] - 163s 3ms/step - loss: 0.0244 - acc: 0.9857 - auroc: 0.9985 - val_loss: 0.2004 - val_acc: 0.7639 - val_auroc: 0.9824\n",
      "Epoch 7/80\n",
      "56000/56000 [==============================] - 165s 3ms/step - loss: 0.0238 - acc: 0.9858 - auroc: 0.9981 - val_loss: 0.0302 - val_acc: 0.9766 - val_auroc: 0.9952\n",
      "Epoch 8/80\n",
      "56000/56000 [==============================] - 177s 3ms/step - loss: 0.0215 - acc: 0.9871 - auroc: 0.9987 - val_loss: 0.0359 - val_acc: 0.9698 - val_auroc: 0.9972\n",
      "Epoch 9/80\n",
      "56000/56000 [==============================] - 179s 3ms/step - loss: 0.0199 - acc: 0.9877 - auroc: 0.9988 - val_loss: 0.0250 - val_acc: 0.9821 - val_auroc: 0.9978\n",
      "Epoch 10/80\n",
      "56000/56000 [==============================] - 169s 3ms/step - loss: 0.0185 - acc: 0.9887 - auroc: 0.9988 - val_loss: 0.2096 - val_acc: 0.7863 - val_auroc: 0.9305\n",
      "Epoch 11/80\n",
      "56000/56000 [==============================] - 167s 3ms/step - loss: 0.0179 - acc: 0.9887 - auroc: 0.9987 - val_loss: 0.0732 - val_acc: 0.9269 - val_auroc: 0.9914\n",
      "Epoch 12/80\n",
      "56000/56000 [==============================] - 167s 3ms/step - loss: 0.0163 - acc: 0.9896 - auroc: 0.9991 - val_loss: 0.0560 - val_acc: 0.9438 - val_auroc: 0.9905\n",
      "Epoch 13/80\n",
      "56000/56000 [==============================] - 158s 3ms/step - loss: 0.0159 - acc: 0.9897 - auroc: 0.9988 - val_loss: 0.0577 - val_acc: 0.9419 - val_auroc: 0.9871\n",
      "Epoch 14/80\n",
      "56000/56000 [==============================] - 165s 3ms/step - loss: 0.0152 - acc: 0.9905 - auroc: 0.9990 - val_loss: 0.0501 - val_acc: 0.9492 - val_auroc: 0.9853\n",
      "Epoch 15/80\n",
      "56000/56000 [==============================] - 171s 3ms/step - loss: 0.0144 - acc: 0.9904 - auroc: 0.9988 - val_loss: 0.0643 - val_acc: 0.9349 - val_auroc: 0.9962\n",
      "Epoch 16/80\n",
      "56000/56000 [==============================] - 164s 3ms/step - loss: 0.0138 - acc: 0.9911 - auroc: 0.9991 - val_loss: 0.1091 - val_acc: 0.8784 - val_auroc: 0.9144\n",
      "Epoch 17/80\n",
      "56000/56000 [==============================] - 152s 3ms/step - loss: 0.0130 - acc: 0.9919 - auroc: 0.9991 - val_loss: 0.0183 - val_acc: 0.9861 - val_auroc: 0.9988\n",
      "Epoch 18/80\n",
      "56000/56000 [==============================] - 175s 3ms/step - loss: 0.0130 - acc: 0.9915 - auroc: 0.9992 - val_loss: 0.0489 - val_acc: 0.9500 - val_auroc: 0.9951\n",
      "Epoch 19/80\n",
      "56000/56000 [==============================] - 144s 3ms/step - loss: 0.0126 - acc: 0.9914 - auroc: 0.9991 - val_loss: 0.0641 - val_acc: 0.9356 - val_auroc: 0.9972\n",
      "Epoch 20/80\n",
      "56000/56000 [==============================] - 142s 3ms/step - loss: 0.0128 - acc: 0.9910 - auroc: 0.9990 - val_loss: 0.0255 - val_acc: 0.9757 - val_auroc: 0.9940\n",
      "Epoch 21/80\n",
      "56000/56000 [==============================] - 144s 3ms/step - loss: 0.0126 - acc: 0.9910 - auroc: 0.9989 - val_loss: 0.0227 - val_acc: 0.9784 - val_auroc: 0.9972\n",
      "Epoch 22/80\n",
      "56000/56000 [==============================] - 144s 3ms/step - loss: 0.0118 - acc: 0.9915 - auroc: 0.9993 - val_loss: 0.0781 - val_acc: 0.9151 - val_auroc: 0.9966\n",
      "Epoch 23/80\n",
      "56000/56000 [==============================] - 175s 3ms/step - loss: 0.0113 - acc: 0.9917 - auroc: 0.9992 - val_loss: 0.0314 - val_acc: 0.9659 - val_auroc: 0.9975\n",
      "Epoch 24/80\n",
      "56000/56000 [==============================] - 172s 3ms/step - loss: 0.0112 - acc: 0.9919 - auroc: 0.9991 - val_loss: 0.0160 - val_acc: 0.9861 - val_auroc: 0.9983\n",
      "Epoch 25/80\n",
      "56000/56000 [==============================] - 171s 3ms/step - loss: 0.0110 - acc: 0.9918 - auroc: 0.9993 - val_loss: 0.0428 - val_acc: 0.9564 - val_auroc: 0.9945\n",
      "Epoch 26/80\n",
      "56000/56000 [==============================] - 182s 3ms/step - loss: 0.0107 - acc: 0.9921 - auroc: 0.9992 - val_loss: 0.0313 - val_acc: 0.9669 - val_auroc: 0.9954\n",
      "Epoch 27/80\n",
      "56000/56000 [==============================] - 183s 3ms/step - loss: 0.0098 - acc: 0.9930 - auroc: 0.9992 - val_loss: 0.0331 - val_acc: 0.9670 - val_auroc: 0.9979\n",
      "Epoch 28/80\n",
      "56000/56000 [==============================] - 197s 4ms/step - loss: 0.0102 - acc: 0.9922 - auroc: 0.9992 - val_loss: 0.0341 - val_acc: 0.9652 - val_auroc: 0.9987\n",
      "Epoch 29/80\n",
      "56000/56000 [==============================] - 177s 3ms/step - loss: 0.0101 - acc: 0.9924 - auroc: 0.9994 - val_loss: 0.0472 - val_acc: 0.9426 - val_auroc: 0.9925\n",
      "Epoch 30/80\n",
      "56000/56000 [==============================] - 171s 3ms/step - loss: 0.0096 - acc: 0.9929 - auroc: 0.9994 - val_loss: 0.0351 - val_acc: 0.9644 - val_auroc: 0.9964\n",
      "Epoch 31/80\n",
      "56000/56000 [==============================] - 167s 3ms/step - loss: 0.0098 - acc: 0.9924 - auroc: 0.9994 - val_loss: 0.0256 - val_acc: 0.9715 - val_auroc: 0.9969\n",
      "Epoch 32/80\n",
      "56000/56000 [==============================] - 168s 3ms/step - loss: 0.0097 - acc: 0.9923 - auroc: 0.9994 - val_loss: 0.0384 - val_acc: 0.9603 - val_auroc: 0.9940\n",
      "Epoch 33/80\n",
      "56000/56000 [==============================] - 181s 3ms/step - loss: 0.0095 - acc: 0.9926 - auroc: 0.9993 - val_loss: 0.0291 - val_acc: 0.9670 - val_auroc: 0.9946\n",
      "Epoch 34/80\n",
      "56000/56000 [==============================] - 205s 4ms/step - loss: 0.0096 - acc: 0.9924 - auroc: 0.9993 - val_loss: 0.0556 - val_acc: 0.9419 - val_auroc: 0.9839\n",
      "Epoch 35/80\n",
      "56000/56000 [==============================] - 200s 4ms/step - loss: 0.0094 - acc: 0.9926 - auroc: 0.9995 - val_loss: 0.0469 - val_acc: 0.9501 - val_auroc: 0.9968\n",
      "Epoch 36/80\n",
      "56000/56000 [==============================] - 160s 3ms/step - loss: 0.0090 - acc: 0.9931 - auroc: 0.9995 - val_loss: 0.0254 - val_acc: 0.9737 - val_auroc: 0.9973\n",
      "Epoch 37/80\n",
      "56000/56000 [==============================] - 150s 3ms/step - loss: 0.0089 - acc: 0.9930 - auroc: 0.9995 - val_loss: 0.0299 - val_acc: 0.9684 - val_auroc: 0.9986\n",
      "Epoch 38/80\n",
      "56000/56000 [==============================] - 149s 3ms/step - loss: 0.0086 - acc: 0.9935 - auroc: 0.9995 - val_loss: 0.0410 - val_acc: 0.9568 - val_auroc: 0.9982\n",
      "Epoch 39/80\n",
      "56000/56000 [==============================] - 164s 3ms/step - loss: 0.0090 - acc: 0.9931 - auroc: 0.9992 - val_loss: 0.0371 - val_acc: 0.9603 - val_auroc: 0.9975\n",
      "Epoch 40/80\n",
      "56000/56000 [==============================] - 155s 3ms/step - loss: 0.0088 - acc: 0.9933 - auroc: 0.9993 - val_loss: 0.0338 - val_acc: 0.9651 - val_auroc: 0.9927\n",
      "Epoch 41/80\n",
      "56000/56000 [==============================] - 169s 3ms/step - loss: 0.0089 - acc: 0.9928 - auroc: 0.9993 - val_loss: 0.0456 - val_acc: 0.9519 - val_auroc: 0.9982\n",
      "Epoch 42/80\n",
      "56000/56000 [==============================] - 164s 3ms/step - loss: 0.0083 - acc: 0.9934 - auroc: 0.9995 - val_loss: 0.0267 - val_acc: 0.9719 - val_auroc: 0.9960\n",
      "Epoch 43/80\n",
      "56000/56000 [==============================] - 151s 3ms/step - loss: 0.0090 - acc: 0.9928 - auroc: 0.9991 - val_loss: 0.0237 - val_acc: 0.9744 - val_auroc: 0.9968\n",
      "Epoch 44/80\n",
      "56000/56000 [==============================] - 152s 3ms/step - loss: 0.0083 - acc: 0.9936 - auroc: 0.9995 - val_loss: 0.0243 - val_acc: 0.9764 - val_auroc: 0.9940\n",
      "Epoch 45/80\n",
      "56000/56000 [==============================] - 155s 3ms/step - loss: 0.0085 - acc: 0.9931 - auroc: 0.9994 - val_loss: 0.0205 - val_acc: 0.9788 - val_auroc: 0.9962\n",
      "Epoch 46/80\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56000/56000 [==============================] - 163s 3ms/step - loss: 0.0085 - acc: 0.9935 - auroc: 0.9994 - val_loss: 0.0374 - val_acc: 0.9614 - val_auroc: 0.9973\n",
      "Epoch 47/80\n",
      "56000/56000 [==============================] - 174s 3ms/step - loss: 0.0081 - acc: 0.9937 - auroc: 0.9994 - val_loss: 0.0426 - val_acc: 0.9536 - val_auroc: 0.9953\n",
      "Epoch 48/80\n",
      "56000/56000 [==============================] - 175s 3ms/step - loss: 0.0080 - acc: 0.9938 - auroc: 0.9996 - val_loss: 0.0643 - val_acc: 0.9309 - val_auroc: 0.9726\n",
      "Epoch 49/80\n",
      "56000/56000 [==============================] - 208s 4ms/step - loss: 0.0082 - acc: 0.9932 - auroc: 0.9996 - val_loss: 0.0391 - val_acc: 0.9566 - val_auroc: 0.9884\n",
      "Epoch 50/80\n",
      "56000/56000 [==============================] - 189s 3ms/step - loss: 0.0081 - acc: 0.9934 - auroc: 0.9996 - val_loss: 0.0679 - val_acc: 0.9291 - val_auroc: 0.9945\n",
      "Epoch 51/80\n",
      "56000/56000 [==============================] - 202s 4ms/step - loss: 0.0078 - acc: 0.9938 - auroc: 0.9995 - val_loss: 0.0432 - val_acc: 0.9531 - val_auroc: 0.9924\n",
      "Epoch 52/80\n",
      "56000/56000 [==============================] - 170s 3ms/step - loss: 0.0080 - acc: 0.9933 - auroc: 0.9996 - val_loss: 0.0186 - val_acc: 0.9809 - val_auroc: 0.9976\n",
      "Epoch 53/80\n",
      "56000/56000 [==============================] - 152s 3ms/step - loss: 0.0075 - acc: 0.9943 - auroc: 0.9997 - val_loss: 0.0438 - val_acc: 0.9546 - val_auroc: 0.9977\n",
      "Epoch 54/80\n",
      "56000/56000 [==============================] - 152s 3ms/step - loss: 0.0078 - acc: 0.9938 - auroc: 0.9996 - val_loss: 0.0850 - val_acc: 0.9104 - val_auroc: 0.9939\n",
      "Epoch 55/80\n",
      "56000/56000 [==============================] - 151s 3ms/step - loss: 0.0082 - acc: 0.9934 - auroc: 0.9993 - val_loss: 0.0299 - val_acc: 0.9681 - val_auroc: 0.9908\n",
      "Epoch 56/80\n",
      "56000/56000 [==============================] - 153s 3ms/step - loss: 0.0077 - acc: 0.9938 - auroc: 0.9995 - val_loss: 0.0280 - val_acc: 0.9694 - val_auroc: 0.9939\n",
      "Epoch 57/80\n",
      "56000/56000 [==============================] - 170s 3ms/step - loss: 0.0078 - acc: 0.9937 - auroc: 0.9995 - val_loss: 0.0354 - val_acc: 0.9617 - val_auroc: 0.9956\n",
      "Epoch 58/80\n",
      "56000/56000 [==============================] - 155s 3ms/step - loss: 0.0079 - acc: 0.9934 - auroc: 0.9995 - val_loss: 0.0287 - val_acc: 0.9691 - val_auroc: 0.9936\n",
      "Epoch 59/80\n",
      "56000/56000 [==============================] - 149s 3ms/step - loss: 0.0081 - acc: 0.9932 - auroc: 0.9995 - val_loss: 0.0198 - val_acc: 0.9796 - val_auroc: 0.9970\n",
      "Epoch 60/80\n",
      "56000/56000 [==============================] - 148s 3ms/step - loss: 0.0078 - acc: 0.9937 - auroc: 0.9994 - val_loss: 0.1568 - val_acc: 0.8242 - val_auroc: 0.9644\n",
      "Epoch 61/80\n",
      "56000/56000 [==============================] - 148s 3ms/step - loss: 0.0077 - acc: 0.9938 - auroc: 0.9994 - val_loss: 0.0353 - val_acc: 0.9634 - val_auroc: 0.9942\n",
      "Epoch 62/80\n",
      "56000/56000 [==============================] - 186s 3ms/step - loss: 0.0071 - acc: 0.9943 - auroc: 0.9994 - val_loss: 0.0357 - val_acc: 0.9619 - val_auroc: 0.9929\n",
      "Epoch 63/80\n",
      "56000/56000 [==============================] - 191s 3ms/step - loss: 0.0078 - acc: 0.9935 - auroc: 0.9995 - val_loss: 0.0545 - val_acc: 0.9386 - val_auroc: 0.9862\n",
      "Epoch 64/80\n",
      "56000/56000 [==============================] - 169s 3ms/step - loss: 0.0075 - acc: 0.9941 - auroc: 0.9996 - val_loss: 0.0240 - val_acc: 0.9761 - val_auroc: 0.9983\n",
      "Epoch 65/80\n",
      "56000/56000 [==============================] - 180s 3ms/step - loss: 0.0073 - acc: 0.9941 - auroc: 0.9995 - val_loss: 0.0332 - val_acc: 0.9645 - val_auroc: 0.9877\n",
      "Epoch 66/80\n",
      "56000/56000 [==============================] - 189s 3ms/step - loss: 0.0076 - acc: 0.9936 - auroc: 0.9996 - val_loss: 0.0734 - val_acc: 0.9201 - val_auroc: 0.9937\n",
      "Epoch 67/80\n",
      "56000/56000 [==============================] - 171s 3ms/step - loss: 0.0073 - acc: 0.9939 - auroc: 0.9995 - val_loss: 0.0526 - val_acc: 0.9433 - val_auroc: 0.9839\n",
      "Epoch 68/80\n",
      "56000/56000 [==============================] - 169s 3ms/step - loss: 0.0075 - acc: 0.9938 - auroc: 0.9995 - val_loss: 0.0313 - val_acc: 0.9659 - val_auroc: 0.9926\n",
      "Epoch 69/80\n",
      "56000/56000 [==============================] - 170s 3ms/step - loss: 0.0071 - acc: 0.9941 - auroc: 0.9996 - val_loss: 0.1250 - val_acc: 0.8671 - val_auroc: 0.9947\n",
      "Epoch 70/80\n",
      "56000/56000 [==============================] - 179s 3ms/step - loss: 0.0072 - acc: 0.9941 - auroc: 0.9995 - val_loss: 0.0828 - val_acc: 0.9141 - val_auroc: 0.9965\n",
      "Epoch 71/80\n",
      "56000/56000 [==============================] - 173s 3ms/step - loss: 0.0076 - acc: 0.9937 - auroc: 0.9995 - val_loss: 0.0516 - val_acc: 0.9464 - val_auroc: 0.9932\n",
      "Epoch 72/80\n",
      "56000/56000 [==============================] - 172s 3ms/step - loss: 0.0072 - acc: 0.9943 - auroc: 0.9996 - val_loss: 0.0368 - val_acc: 0.9606 - val_auroc: 0.9899\n",
      "Epoch 73/80\n",
      "56000/56000 [==============================] - 175s 3ms/step - loss: 0.0071 - acc: 0.9941 - auroc: 0.9996 - val_loss: 0.0251 - val_acc: 0.9731 - val_auroc: 0.9958\n",
      "Epoch 74/80\n",
      "56000/56000 [==============================] - 177s 3ms/step - loss: 0.0070 - acc: 0.9942 - auroc: 0.9996 - val_loss: 0.0449 - val_acc: 0.9528 - val_auroc: 0.9960\n",
      "Epoch 75/80\n",
      "56000/56000 [==============================] - 177s 3ms/step - loss: 0.0071 - acc: 0.9943 - auroc: 0.9994 - val_loss: 0.0229 - val_acc: 0.9759 - val_auroc: 0.9949\n",
      "Epoch 76/80\n",
      "56000/56000 [==============================] - 178s 3ms/step - loss: 0.0073 - acc: 0.9938 - auroc: 0.9996 - val_loss: 0.0234 - val_acc: 0.9748 - val_auroc: 0.9951\n",
      "Epoch 77/80\n",
      "56000/56000 [==============================] - 188s 3ms/step - loss: 0.0073 - acc: 0.9938 - auroc: 0.9996 - val_loss: 0.0357 - val_acc: 0.9611 - val_auroc: 0.9838\n",
      "Epoch 78/80\n",
      "56000/56000 [==============================] - 182s 3ms/step - loss: 0.0072 - acc: 0.9941 - auroc: 0.9996 - val_loss: 0.0520 - val_acc: 0.9460 - val_auroc: 0.9944\n",
      "Epoch 79/80\n",
      "56000/56000 [==============================] - 181s 3ms/step - loss: 0.0073 - acc: 0.9941 - auroc: 0.9994 - val_loss: 0.0400 - val_acc: 0.9579 - val_auroc: 0.9948\n",
      "Epoch 80/80\n",
      "56000/56000 [==============================] - 179s 3ms/step - loss: 0.0076 - acc: 0.9935 - auroc: 0.9994 - val_loss: 0.0421 - val_acc: 0.9540 - val_auroc: 0.9875\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a2e0b1ba8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fitting the model. The epochs and Batch size were decided based on multiple runs.\n",
    "model.fit(x_train, \n",
    "            y_train, \n",
    "            batch_size = 128, \n",
    "            epochs = 80, \n",
    "            validation_data = (x_valid, y_valid),\n",
    "            verbose = 1    #<- Shows results while running, useful to see progress of fitting\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4000/4000 [==============================] - 3s 803us/step\n"
     ]
    }
   ],
   "source": [
    "#Predict on test data \n",
    "predictions = model.predict(images_test, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Confusion Matrix \n",
    "\n",
    "y_pred_probability = model.predict_proba(x_train)\n",
    "\n",
    "y_pred = model.predict_classes(x_train)\n",
    "conf_matrix = confusion_matrix(y_train, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[13273,   692],\n",
       "       [ 1987, 40048]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9923824],\n",
       "       [0.9980409],\n",
       "       [0.2532274],\n",
       "       ...,\n",
       "       [0.955483 ],\n",
       "       [0.9977274],\n",
       "       [0.9979794]], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Look at predictions\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Generate submission file \n",
    "\n",
    "test_df = pd.read_csv('../input/sample_submission.csv')\n",
    "X_test = []\n",
    "images_test = test_df['id'].values\n",
    "\n",
    "for img_id in tqdm_notebook(images_test):\n",
    "    X_test.append(cv.imread(test_path + img_id))\n",
    "    \n",
    "X_test = np.asarray(X_test)\n",
    "X_test = X_test.astype('float32')\n",
    "X_test /= 255\n",
    "\n",
    "y_test_pred = model.predict_proba(X_test)\n",
    "\n",
    "test_df['has_cactus'] = y_test_pred\n",
    "test_df.to_csv('aerial-cactus-submission.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}