lab-12-2-many-to-one-stacking-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-2 many to one stacking\n",
    "### sentence sentiment classification \n",
    "* many to one\n",
    "* variable input sequence length\n",
    "* stacking\n",
    "* drop out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = ['What I cannot create, I do not understand.',\n",
    "             'Intellecuals solve problems, geniuses prevent them',\n",
    "             'A person who never made a mistake never tied anything new.',\n",
    "             'The same equations have the same solutions.']\n",
    "y_data = [1,0,0,1] # 1: richard feynman, 0: albert einstein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', ',', '.', 'A', 'I', 'T', 'W', 'a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'y']\n",
      "{0: '<pad>', 1: ' ', 2: ',', 3: '.', 4: 'A', 5: 'I', 6: 'T', 7: 'W', 8: 'a', 9: 'b', 10: 'c', 11: 'd', 12: 'e', 13: 'g', 14: 'h', 15: 'i', 16: 'k', 17: 'l', 18: 'm', 19: 'n', 20: 'o', 21: 'p', 22: 'q', 23: 'r', 24: 's', 25: 't', 26: 'u', 27: 'v', 28: 'w', 29: 'y'}\n",
      "{'<pad>': 0, ' ': 1, ',': 2, '.': 3, 'A': 4, 'I': 5, 'T': 6, 'W': 7, 'a': 8, 'b': 9, 'c': 10, 'd': 11, 'e': 12, 'g': 13, 'h': 14, 'i': 15, 'k': 16, 'l': 17, 'm': 18, 'n': 19, 'o': 20, 'p': 21, 'q': 22, 'r': 23, 's': 24, 't': 25, 'u': 26, 'v': 27, 'w': 28, 'y': 29}\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary\n",
    "char_set = ['<pad>'] + sorted(list(set(''.join(sentences))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7, 14, 8, 25, 1, 5, 1, 10, 8, 19, 19, 20, 25, 1, 10, 23, 12, 8, 25, 12, 2, 1, 5, 1, 11, 20, 1, 19, 20, 25, 1, 26, 19, 11, 12, 23, 24, 25, 8, 19, 11, 3], [5, 19, 25, 12, 17, 17, 12, 10, 26, 8, 17, 24, 1, 24, 20, 17, 27, 12, 1, 21, 23, 20, 9, 17, 12, 18, 24, 2, 1, 13, 12, 19, 15, 26, 24, 12, 24, 1, 21, 23, 12, 27, 12, 19, 25, 1, 25, 14, 12, 18], [4, 1, 21, 12, 23, 24, 20, 19, 1, 28, 14, 20, 1, 19, 12, 27, 12, 23, 1, 18, 8, 11, 12, 1, 8, 1, 18, 15, 24, 25, 8, 16, 12, 1, 19, 12, 27, 12, 23, 1, 25, 15, 12, 11, 1, 8, 19, 29, 25, 14, 15, 19, 13, 1, 19, 12, 28, 3], [6, 14, 12, 1, 24, 8, 18, 12, 1, 12, 22, 26, 8, 25, 15, 20, 19, 24, 1, 14, 8, 27, 12, 1, 25, 14, 12, 1, 24, 8, 18, 12, 1, 24, 20, 17, 26, 25, 15, 20, 19, 24, 3]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "x_data = list(map(lambda sentence : [char2idx.get(char) for char in sentence], sentences))\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 7 14  8 25  1  5  1 10  8 19 19 20 25  1 10 23 12  8 25 12  2  1  5  1\n",
      "  11 20  1 19 20 25  1 26 19 11 12 23 24 25  8 19 11  3  0  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]\n",
      " [ 5 19 25 12 17 17 12 10 26  8 17 24  1 24 20 17 27 12  1 21 23 20  9 17\n",
      "  12 18 24  2  1 13 12 19 15 26 24 12 24  1 21 23 12 27 12 19 25  1 25 14\n",
      "  12 18  0  0  0  0  0]\n",
      " [ 4  1 21 12 23 24 20 19  1 28 14 20  1 19 12 27 12 23  1 18  8 11 12  1\n",
      "   8  1 18 15 24 25  8 16 12  1 19 12 27 12 23  1 25 15 12 11  1  8 19 29\n",
      "  25 14 15 19 13  1 19]\n",
      " [ 6 14 12  1 24  8 18 12  1 12 22 26  8 25 15 20 19 24  1 14  8 27 12  1\n",
      "  25 14 12  1 24  8 18 12  1 24 20 17 26 25 15 20 19 24  3  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# padding the sequence of indices\n",
    "max_sequence = 55\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
    "                       padding = 'post', truncating = 'post')\n",
    "\n",
    "# checking data\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating stacked rnn for \"many to one\" classification with dropout\n",
    "num_classes = 2\n",
    "hidden_dims = [10,10]\n",
    "\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[0], return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dropout(rate = .2)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[1]))\n",
    "model.add(layers.Dropout(rate = .2))\n",
    "model.add(layers.Dense(units=num_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 55, 30)            900       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 55, 10)            410       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 55, 10)            0         \n",
      "_________________________________________________________________\n",
      "simple_rnn_1 (SimpleRNN)     (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 1,542\n",
      "Trainable params: 642\n",
      "Non-trainable params: 900\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, training):\n",
    "    return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x, training))\n",
    "\n",
    "# creating and optimizer\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 55), (?,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size=batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.319\n",
      "epoch :  10, tr_loss : 0.084\n",
      "epoch :  15, tr_loss : 0.038\n",
      "epoch :  20, tr_loss : 0.015\n",
      "epoch :  25, tr_loss : 0.016\n",
      "epoch :  30, tr_loss : 0.010\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, training=True)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('accuracy : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f08103852e8>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd0XOd95vHvDwMMOjDonU0E2ElQhinZMiW5yJJtWcWJbDHRHmfj2MkmynFiJxtns1YcJY7juolPFGeV5rgqconFKIxkrU1Vm6RAixR7ryhEI3rHvPvHDCgIQhmSQwzuxfM5B4czdy7u/C4v+cyd9733fc05h4iI+EtSogsQEZH4U7iLiPiQwl1ExIcU7iIiPqRwFxHxIYW7iIgPKdxFRHxI4S4i4kMKdxERH0pO1BsXFha6JUuWJOrtRUQ8affu3W3OuaLZ1ktYuC9ZsoT6+vpEvb2IiCeZ2ZlY1lOzjIiIDyncRUR8SOEuIuJDCncRER9SuIuI+JDCXUTEhxTuIiI+5Llwrz/dweefOoymBxQRmZ7nwn1/Qxdfe/YEF7qHEl2KiMi85blwX1eZC8C+hq4EVyIiMn95LtxXleWQZAp3EZGZeC7cM4LJXFeUxX6Fu4jItDwX7gDrKnIV7iIiM/BkuK+tyKWlZ4iW7sFElyIiMi95NtxB7e4iItPxZLivKc/BDPY3dCe6FBGRecmT4Z6ZmsyywkyduYuITMOT4Q6Rphl1qoqITM2z4b6uIpfm7kFae3SnqojIZJ4N9/FO1f2NOnsXEZnMs+G+pjwHgP3nFe4iIpN5Ntyz01JYqk5VEZEpeTbcIdI0c6BRl0OKiEzm6XBfV5FDQ+cAHX3DiS5FRGRe8XS4ry3XnaoiIlPxdLivGb9iRuEuIvI6ng733PQUFhdkKNxFRCaJKdzN7A4zO2Jmx83sU1O8vsjMtpvZK2b2qpm9N/6lTm1tea6aZUREJpk13M0sADwCvAdYDWwxs9WTVvvfwOPOuY3A/cDfxbvQ6aytyOX8xQE6+9WpKiIyLpYz903AcefcSefcMPAYcPekdRyQE32cCzTGr8SZrbvU7q5LIkVExsUS7hXAuQnPz0eXTfQZ4AEzOw9sA343LtXFYG1F5DNFTTMiIq+JV4fqFuDrzrlK4L3AN83sDds2s4+ZWb2Z1be2tsbljUMZQSrz0tWpKiIyQSzh3gBUTXheGV020UeAxwGccz8H0oDCyRtyzj3qnKtzztUVFRVdWcVTWFeRqwHEREQmiCXcXwaqzWypmQWJdJhunbTOWeCdAGa2iki4x+fUPAZrK3I5095P18DIXL2liMi8Nmu4O+dGgQeBp4FDRK6KOWBmD5vZXdHVPgl81Mz2At8Ffs05565V0ZOND/97QE0zIiIAJMeyknNuG5GO0onLHprw+CBwU3xLi926CWO7v3X5G1qDREQWHE/foTouPzNIRSidfbocUkQE8Em4Q2TyDl0xIyIS4ZtwX1eRy6m2PnoG1akqIuKbcF9bGe1U1eQdIiI+CvdyDf8rIjLON+FelJ1KaU6awl1EBB+FO0Sud9cYMyIiPgv3dRW5nGzro3doNNGliIgklK/CfW1FDs7BQXWqisgC56twX6c5VUVEAJ+Fe3FOGsXZqQp3EVnwfBXuoE5VERHwabifaO2lf1idqiKycPku3NdV5BJ2cKhJnaoisnD5MtwB9p1X04yILFy+C/eSnFQKs4Ls1+WQIrKA+S7czYy1Fbm6YkZEFjTfhTtEmmaOtfQyODKW6FJERBLCl+G+pjyXsbDjoDpVRWSB8mW4r6vUhNkisrD5MtzLc9PIzwzqZiYRWbB8Ge5mxpryHE2YLSILli/DHaKdqhd61KkqIguSr8N9NOw40tyT6FJEROacb8N97fidqmp3F5EFyLfhXpmXTnF2Kj893JLoUkRE5pxvw93M+NCbq9h+pIVzHf2JLkdEZE75NtwBtmxahAHf3XU20aWIiMwpX4d7eSidd6ws4fH6cwyPhhNdjojInPF1uAM8cOMi2nqHeepAc6JLERGZM74P95uri1iUn8G3fn4m0aWIiMwZ34d7UpLxKzcsYtfpDl3zLiILhu/DHeC+N1USDCTx7Z06exeRhWFBhHtBVirvW1/GD3/RQN+QJs4WEf9bEOEOkY7V3qFRntjTmOhSRESuuZjC3czuMLMjZnbczD41zTofNLODZnbAzL4T3zKv3vWL8lhZms23dpzBOZfockRErqlZw93MAsAjwHuA1cAWM1s9aZ1q4I+Bm5xza4Dfuwa1XhUz44EbF3OwqZtXznUmuhwRkWsqljP3TcBx59xJ59ww8Bhw96R1Pgo84py7COCcm5cDutyzsYLMYIBv7VDHqoj4WyzhXgGcm/D8fHTZRDVAjZm9ZGY7zOyOeBUYT1mpydx7fQVPvtrExb7hRJcjInLNxKtDNRmoBm4FtgD/YGahySuZ2cfMrN7M6ltbW+P01pfngRsXMzwa5nu7z82+soiIR8US7g1A1YTnldFlE50HtjrnRpxzp4CjRML+dZxzjzrn6pxzdUVFRVda81VZWZrDm5fk8e2dZwmH1bEqIv4US7i/DFSb2VIzCwL3A1snrfMjImftmFkhkWaak3GsM64euHExZ9r7efF4W6JLERG5JmYNd+fcKPAg8DRwCHjcOXfAzB42s7uiqz0NtJvZQWA78IfOufZrVfTVumNtKQWZQXWsiohvJceyknNuG7Bt0rKHJjx2wCeiP/NeanKA++qqePT5EzR1DVCWm57okkRE4mrB3KE62a/esAgHfHeXOlZFxH8WbLhX5WdwS00Rj+06y8iYJvIQEX9ZsOEO8MANi2npGeKZgxcSXYqISFwt6HB/+8piKkLp6lgVEd9Z0OEeiE7k8bMT7Zxo7U10OSIicbOgwx3gg3VVpASMb+84m+hSRETiZsGHe1F2KrevKeX7u8+pY1VEfGPBhzvA7WtK6R4c1RyrIuIbCnegtioyxpnGeRcRv1C4A5V56RRkBtlzVuEuIv6gcCcyS1NtVYi95xXuIuIPCveo2qoQJ1p76R4cSXQpIiJXTeEetaEqhHOw73xXoksREblqCveoDZWRTtU96lQVER9QuEflZqSwrDBT4S4ivqBwn6C2KsSec51EhqcXEfEuhfsEG6pCtPYM0dg1mOhSRESuisJ9gvGbmfaqaUZEPE7hPsGqshyCgSS1u4uI5yncJwgmJ7G6PEfhLiKep3CfpLYqxL7zXYxqhEgR8TCF+yS1VSEGRsY41qLJO0TEuxTuk4x3qqppRkS8TOE+yeKCDEIZKRohUkQ8TeE+iZmxoVIjRIqItyncp1BbFeLohR76hkYTXYqIyBVRuE+htipE2MG+Bo0QKSLepHCfwgZ1qoqIxyncp5CfGWRRfoaGIRARz1K4T2N8hEgRES9SuE+jtipEU9cgF7o1QqSIeI/CfRpqdxcRL1O4T2NNeQ7JSaZwFxFPUrhPIy0lwKqyHHWqiognKdxnUFsV4tXzXYyFNe2eiHhLTOFuZneY2REzO25mn5phvV8yM2dmdfErMXFqq0L0Do1yslUjRIqIt8wa7mYWAB4B3gOsBraY2eop1ssGPg7sjHeRiTLeqfqKmmZExGNiOXPfBBx3zp10zg0DjwF3T7HenwOfB3xz7eCywkyy05LVqSoinhNLuFcA5yY8Px9ddomZXQ9UOef+M461JVxSUnSESIW7iHjMVXeomlkS8BXgkzGs+zEzqzez+tbW1qt96zlRWxXicHMPA8NjiS5FRCRmsYR7A1A14XlldNm4bGAt8KyZnQZuBLZO1anqnHvUOVfnnKsrKiq68qrn0IaqEGNhx4FGjRApIt4RS7i/DFSb2VIzCwL3A1vHX3TOdTnnCp1zS5xzS4AdwF3OufprUvEc07R7IuJFs4a7c24UeBB4GjgEPO6cO2BmD5vZXde6wEQryk6lIpSucBcRT0mOZSXn3DZg26RlD02z7q1XX9b8ohEiRcRrdIdqDGqrQpy/OEBb71CiSxERiYnCPQbjNzPpkkgR8QqFewzWVeQS0AiRIuIhCvcYpAcDrCjJVriLiGco3GO0oSpyp2pYI0SKiAco3GO0sSpE9+Aop9v7El2KiMisFO4x0rR7IuIlCvcYLS/OIjMY0BUzIuIJCvcYBZKM9ZW6mUlEvEHhfhk2VIU42NTN4IhGiBSR+U3hfhlqq0KMjDkONXUnuhQRkRnFNLaMRIyPEPnnTx5kTXkueRkphDKC5GcGCWWkkJcRjPxkppCVmoyZJbhiEVmoFO6XoTQ3jQ9cX8Ges508+WojnQMjuGkue08JRNrov/mRTWQE9dcsInNLqXOZvvLB2kuPx8KO7oERLvYPc7F/hM7+YTr6hunsH6Gxa4B/eek03/z5GX7zlusSWLGILEQK96sQSDLyMoPkZQanfP1Eax9//9wJfvXGxWSl6q9aROaOOlSvoU/cVsPF/hG+/tKpRJciIguMwv0aqq0K8a5VxTz6/Em6BkYSXY6ILCAK92vs92+roXtwlH96UWfvIjJ3FO7X2JryXN6ztpR/fvEUF/uGE12OiCwQCvc58Pu31dA3PMqjL5xMdCkiskAo3OdATUk2719fztdfOq15WEVkTijc58jH31XN0OgYf//siUSXIiILgMJ9jlxXlMW9Gyv55o4zXOgeTHQ5IuJzCvc59PF3VjMWdvzd9uOJLkVEfE7hPocWFWRwX10l3911jobOgUSXIyI+pnCfYw++oxqAv/3p/Dh7rz/dwWe2HsBNNwKaiHiSwn2OVYTSuX9TFd+rP8fZ9v5El8NXf3qcr//sNMdaehNdiojEkcI9AX7n7csJJBlf/emxhNbR2jPES8fbAHj+aGtCaxGR+FK4J0BJThoP3LiYH/7iPCdbE3fGvG1fE2NhR256Ci8ca0tYHSISfwr3BPkft15HanKAv/lJ4s7en9jTwMrSbO7dWMHOU+2aG1bERxTuCVKYlcqH37qErXsbOdLcM+fvf7a9n1+c7eTu2go2VxcyOBJm95mLc16HiFwbCvcE+s2bl5EZTOav/9/ROX/vrXsbAHj/hjJuXFZASsB4/pja3UX8QuGeQHmZQX79piX81/5mDjR2zdn7Ouf40Z5GNi3JpzIvg8zUZK5flMcLR9XuLuIXCvcE+8jmZeSmp/BnWw8SDs/NteaHmno43tLLXbXll5bdXFPEwaZuWns0sJmIHyjcEyw3PYU/ee8qdp3u4LGXz83Jez6xp4HkJOO968ouLdtcXQhw6dJIEfG2mMLdzO4wsyNmdtzMPjXF658ws4Nm9qqZ/cTMFse/VP+6r66Stywr4HPbDl3zQcXCYcfWvY3cXFNE/oSJvdeW55KXkaJ2dxGfmDXczSwAPAK8B1gNbDGz1ZNWewWoc86tB74PfCHehfqZmfGXH1jH8FiYP33iwDV9r5dPd9DUNcjdE5pkAJKSjLdVF/HCsTYNRSDiA7GcuW8CjjvnTjrnhoHHgLsnruCc2+6cG7+XfgdQGd8y/W9pYSYff1c1Tx1o5qn9zdfsfZ7Y20h6SoDbVpe84bXN1YW09gxx5MLcX5opIvEVS7hXABMbg89Hl03nI8B/XU1RC9VHNy9jVVkODz2xn66Bkbhvf3g0zLZ9Tbx7TQkZweQ3vD7e7q6hCES8L64dqmb2AFAHfHGa1z9mZvVmVt/aqgCZLCWQxOd/aR1tvUN8/qnDcd/+80db6ewfeUOTzLiy3HSqi7M0FIGID8QS7g1A1YTnldFlr2Nm7wL+BLjLOTfl9XTOuUedc3XOubqioqIrqdf31leG+PWblvKdnWfZdaojrtt+Ym8jeRkpbK6e/u9+c3URO091aCgCEY+LJdxfBqrNbKmZBYH7ga0TVzCzjcD/JRLsLfEvc2H5xLtrqMxL549/+GrcQrZvaJRnDjbzvvVlpASmP+ybawoZHg3H/YNFRObWrOHunBsFHgSeBg4BjzvnDpjZw2Z2V3S1LwJZwPfMbI+ZbZ1mcxKDjGAyn713HSda++I2Jd+PDzYzOBLm7tqZukvghqX5BANJvKBLIkU87Y29alNwzm0Dtk1a9tCEx++Kc10L3i01Rdy7sYKvPXeC960vZ0Vp9lVt74k9jVSE0nnTorwZ18sIJlO3JE/t7iIepztU57FP37ma7LQUPvXDVxm7iqEJ2nuHeOFYG+/fUE5Sks26/ubqIg4399ByjW+oEpFrR+E+j+VnBvn0nat45Wwn39px5oq3Mz4pxz0bp75KZrLxSyJ19i7iXQr3ee6e2gpuriniC08dprFz4Iq28aM9jawoyWZlaU5M668uy6EgM6h2dxEPU7jPc2bGZ+9ZS9jBp3+0/7KHBjjX0c/uMxdfNwLkbCJDERTy4vG2ORupUkTiS+HuAVX5GXzy3TX85HALT77adFm/u3VvIwB3bYg93CHS7t7WO8yh5u7L+j0RmR8U7h7x329ayvrKXP7k3/fxby+fjbmDdeueRuoW51GVn3FZ76d2dxFvU7h7RCDJ+Or9G6kuyeaPfrCPu/72RXaebJ/xdw41dXPkQs+0ww3MpCQnjRUl2Wp3F/EohbuHLCnM5Pu/9Ra+umUjF/uG+dCjO/jtb+/mXEf/lOs/saeRwKRJOS7HzTWFvHzqIgPDGopAxGsU7h5jZty1oZyf/sGtfPK2GrYfbuWdX36Ozz91mN6h0UvrhcOO/9jbyObqQgqyUq/ovTZXFzE8FmbnqZm/IYjI/KNw96i0lAC/+85qtv/Brdy5oYyvPXuCW7/4LI+/fI6xsKP+zEUaOge4Z5bhBmayaWk+weQktbuLeFBMww/I/FWam8ZXPljLh9+yhIefPMj//MGr/OvPT5OfGSQtJWnKSTlilZYS4Ial+Wp3F/Egnbn7xIaq0Ova41841sZtq0vJTL26z+/N1YUcvdBLc5eGIhDxEp25+8h4e/y7V5fw76808LblhVe9zcjY74d54Vgr99VVzbq+iMwPOnP3obSUAFs2Lbrsa9unsrI0m8KsVLW7i3iMwl1mZGbcrKEIRDxH4S6z2lxTSEffMAebNBSBiFco3GVWN0Xb7p87qqtmRLxC4S6zKs5OY1VZji6JFPEQhbvE5ObqQnafuUjfhLtg4+Fsez8jY+G4blNEFO4So83VRYyMOf5rf/Nljyk/Wd/QKI/tOsvdj7zEzV/czq9//WUGRzR+jUg82dX+R71SdXV1rr6+PiHvLZdvcGSMW764nQvdQ1TmpfPu1aXcsbaUNy3OIxDDvKwA+8538Z1dZ9m6p4G+4TFqSrK4cVkB3/j5Gd61qpivPfAmUgI63xCZiZntds7Vzbqewl1i1dE3zDMHm3n6wAVePNbG8FiYgswgt60u4fY1pbx1eQGpyYHX/U734AhP7GnksV1nOdDYTVpKEneuL2fLpiquX5SHmfHNHWf49I/28751ZfzN/bUkK+BFpqVwl2uqd2iUZ4+08NT+Zp490krv0ChZqcm8fWUxt68poTg7je/Vn+PJV5sYGBljVVkOv7KpirtqK8hNT3nD9v7xhZP8xX8e4gMbK/jSfRtIivHbgMhCE2u4a/gBuSJZqcncub6cO9eXMzQ6xs+Ot/P0gWaeOXiB/4hO7ZcZDHDPxnLuf/Mi1lfmYjZ9YP/G5mUMDI/x5WeOkpoS4C/vXTvj+iIyM4W7XLXU5ABvX1nM21cW89l7HbvPXKSpa4B3rioh6zIGLnvwHcsZGBnj7549QVpKEg/duVoBL3KFFO4SV4EkY9PS/Cv6XTPjD29fwcDIGP/y0mkyggH+8PaVca5QZGFQuMu8YmY8dOdqBkfCPLL9BOkpAR58R3WiyxLxHIW7zDtmxmfvWcvQyBhf+vFR0lIC/MbmZYkuS8RTFO4yLyUlGV/45fUMjYb5i/88RGpyEv/tLUsSXdaUugdHqD/dwY6THew82U5H/zBvW17ILTXFvK268LL6HUTiRf/qZN5KDiTxfz5Uy9DoGJ9+4gAjY447N5RRlJV61R2tzjkauwZp7x0iLyNIYVYq6cHA7L8IdPWPsOt0JMh3nGrnYGM3YQfBQBK1VSFWlebw5N4mvrvrHCkB481L8rl1RRFvX1HM8uIsdRLLnNB17jLvDY6M8dFv1F+aMCQ/M8iKkmxWlmWzsjSbFaU51JRkkRF847mKc47WniGOXOjh6IVejjb3cLSlh2MXeumdNE5OekqAgqwgBZlBCrJSyc8MXnqem57C4eYedp7s4FBzN85BMDmJ6xeFuGFpATcsy+f6RXmkpUQ+IEbGwuw+c5HtR1p47kgrh5t7AKgIpV8K+rcuL5iyZpGZ6CYm8ZXRsTC7TndwpLmHI809HGru4WhzDwPRMWnMYHF+BitKs6kuzuZi/zDHLvRy5EIPXQMjl7aTnxmkpiSLmpJsqkuyKclOpbN/hLa+ITp6h2nvi/70DtHRN0x77zDD0YHN0lKSuH5RHjcuK+CGpflsqApdCvPZNHYO8OyRVp490sKLx9voHx4jGEiiuiSL64oiP8uLs7iuOJMlBZkxb1cWHoW7+F447Djb0c/haOAfbu7mSHMPp9v7yEpNjgR9STY1xVnUlGZTUxKZMvByOOfoHRqls3+Ekpw0gslXPzTC0OgY9acv8vyxVo4093C8pZeGzgHG/ysmGVTlZ7C8KIvrirO4riiT6pJsVpflKPRF4S4L1/BomJSAeapte2B4jJNtvZxo7eN4Sy8nWns50dLLybY+hkcj3xwCSUZ1cRbrK3NZVxliXUUuK0uzZw388aap4y29HGvp5Vi0WWpoNMxNywu4paaYjYtCGrTNIxTuIj4wFnY0XBzgUHM3+xu6ePV8F/sauujoGwYgOclYUZrN+spc1lbksrosh66BkUiQX+jleGsvxy700D34Wv9Cdloy1dGO3T3nOhkLO7JTk7lpeSG3rCji5poiKkLps9bmnKOlZ4gjzT0cvdDD4eYeugdGWFyQwZLCTJYWZLKkMJPSnLTLGitoYHiMxq4BGjsHaLg4gAPqFuepMzoqruFuZncAfwMEgH90zv3VpNdTgW8AbwLagQ85507PtE2Fu8iVcc7R0DnwurDf19BFZ//I69YrzApyXVEW1SVZVBdns7w4i+riLIqyX7vaqGtghJ8db+O5o608d7SVpq5BAKqLs7ilpohbVhTx5iX5DI2GORYN8KMT/pz4noVZqYQyUjjX0c/Q6GsTsKQmJ0UCvyCTpYWZLC7IZFF+Br1DI5y/OEBj52AkyDsjgd4e/eCaLD8zyKYl+dywLJ8blhawsjT7mg8wNzoWJpAU32+B45PTXOk3pbiFu5kFgKPAbcB54GVgi3Pu4IR1fhtY75z7LTO7H7jXOfehmbarcBeJH+cc5y8OcLCpm7yMIMuLs8jPDF72No619PLckUjQ7zrVwfBYpIlrZOy1nMhKTaamJIsVpTmsKIn0Z6woyaYg2p8RDjuauwc53dbHqfY+Trf1cbq9n9NtfZzp6L/UzDQuIxigIpROefSnMi+d8lAaFaEMykNpjI45dp3qYOepDnaeauf8xQEActKS2bQ0/9LVSqvLci5ruOixcKS5qqFzgKauAZo6By99Y2jqGqSxc5C23iECSUZWajLZaclkp6VE/pz8PC2FlIDRMzga/Rmhd+i1xz0THg+OhPncB9axZdOiyzo+4+IZ7m8BPuOcuz36/I8BnHOfm7DO09F1fm5myUAzUORm2LjCXWR+6x8eZefJDnacbCc3I4WV0U7pilD6FZ/JhsOOpu5Bzrb3k52WTGVeOrnpKZe1vYbOAXaebL8U+Kfa+oDIh0ROWgpJFrnL2SxyFVWSGcaEZcDgSJgL3YOMhl8fUZnBAOWhdMpC6ZTnplGSk8ZY2EUCenCU7vGwHhylZ2jkUpiPhV//4Zedlvy6D4SstGRyxh+nJvOOlcWsrci9or/DeA75WwGcm/D8PHDDdOs450bNrAsoANpiK1dE5puMYPKl0T7jJSnJqAilx9SmP52KUDofuL6SD1xfCcCF7kF2nepg95mLDAyPEXYOB4SdA8eE55FvJ85FmorKQmmRbwu56ZSF0ijLTScnLfmyP7iccwyMjDEy5shKTY55ZrJrbU7voDCzjwEfA1i06Mq+koiITFSSk8b7N5Tz/g3lCXl/M5uXN6PF0kDVAFRNeF4ZXTblOtFmmVwiHauv45x71DlX55yrKyoqurKKRURkVrGE+8tAtZktNbMgcD+wddI6W4EPRx//MvDTmdrbRUTk2pr1u0S0Df1B4Gkil0L+s3PugJk9DNQ757YC/wR808yOAx1EPgBERCRBYmoocs5tA7ZNWvbQhMeDwH3xLU1ERK6U7jcWEfEhhbuIiA8p3EVEfEjhLiLiQwkbFdLMWoEzV/jrhfjv7le/7ZPf9gf8t09+2x/w3z5NtT+LnXOz3iiUsHC/GmZWH8vYCl7it33y2/6A//bJb/sD/tunq9kfNcuIiPiQwl1ExIe8Gu6PJrqAa8Bv++S3/QH/7ZPf9gf8t09XvD+ebHMXEZGZefXMXUREZuC5cDezO8zsiJkdN7NPJbqeq2Vmp81sn5ntMTNPTk1lZv9sZi1mtn/Csnwze8bMjkX/zEtkjZdjmv35jJk1RI/THjN7byJrvFxmVmVm283soJkdMLOPR5d78jjNsD+ePU5mlmZmu8xsb3Sf/iy6fKmZ7Yxm3r9FR+edfXteapaJZT5XrzGz00Cdc86z1+aa2c1AL/AN59za6LIvAB3Oub+KfgjnOef+KJF1xmqa/fkM0Ouc+1Iia7tSZlYGlDnnfmFm2cBu4B7g1/DgcZphfz6IR4+TRaaAynTO9ZpZCvAi8HHgE8APnXOPmdnfA3udc1+bbXteO3PfBBx3zp10zg0DjwF3J7imBc859zyRoZ4nuhv41+jjfyXyH88TptkfT3PONTnnfhF93AMcIjI9pieP0wz741kuojf6NCX644B3AN+PLo/5GHkt3Keaz9XTB5TIwfuxme2OTkPoFyXOuabo42agJJHFxMmDZvZqtNnGE80XUzGzJcBGYCc+OE6T9gc8fJzMLGBme4AW4BngBNDpnBuNrhJz5nkt3P3obc6564H3AL8TbRLwleisXN5p/5va14DrgFqgCfhyYsu5MmaWBfwA+D3nXPfE17x4nKbAsTHEAAABd0lEQVTYH08fJ+fcmHOulsh0ppuAlVe6La+FeyzzuXqKc64h+mcL8O9EDqgfXIi2i463j7YkuJ6r4py7EP2PFwb+AQ8ep2g77g+Abzvnfhhd7NnjNNX++OE4ATjnOoHtwFuAUHRuariMzPNauMcyn6tnmFlmtDMIM8sE3g3sn/m3PGPivLofBp5IYC1XbTwAo+7FY8cp2ln3T8Ah59xXJrzkyeM03f54+TiZWZGZhaKP04lcOHKISMj/cnS1mI+Rp66WAYhe2vTXvDaf62cTXNIVM7NlRM7WITLl4Xe8uD9m9l3gViIj2F0A/hT4EfA4sIjI6J8fdM55opNymv25lchXfQecBn5zQlv1vGdmbwNeAPYB4eji/0Wkndpzx2mG/dmCR4+Tma0n0mEaIHLi/bhz7uFoTjwG5AOvAA8454Zm3Z7Xwl1ERGbntWYZERGJgcJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR8SOEuIuJDCncRER/6/zHnFYh3p7nhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}