nmt-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string, os, io\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import contractions\n",
    "from tensorflow.keras.utils import Sequence, to_categorical\n",
    "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Activation\n",
    "from tensorflow.keras.layers import Concatenate, Dropout, Bidirectional, Dot\n",
    "from tensorflow.keras.callbacks import CSVLogger, ModelCheckpoint, LearningRateScheduler\n",
    "from tensorflow.keras.optimizers import RMSprop, Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.initializers import Constant\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eng</th>\n",
       "      <th>french</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Va !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Cours !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Run!</td>\n",
       "      <td>Courez !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Wow!</td>\n",
       "      <td>Ça alors !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fire!</td>\n",
       "      <td>Au feu !</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     eng      french\n",
       "0    Go.        Va !\n",
       "1   Run!     Cours !\n",
       "2   Run!    Courez !\n",
       "3   Wow!  Ça alors !\n",
       "4  Fire!    Au feu !"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df = pd.read_csv('en-fr.txt', sep='\\t', header=None, names=['eng','french'], nrows=20000)\n",
    "corpus_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng = []\n",
    "for sentence in corpus_df['eng']:\n",
    "    x = sentence.strip()\n",
    "    x = contractions.fix(x).lower()\n",
    "    eng.append(x)\n",
    "\n",
    "french = []\n",
    "for sentence in corpus_df['french']:\n",
    "    x = sentence.strip().lower()\n",
    "    x = \"<BOS> \" + x + \" <EOS>\"\n",
    "    french.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_train, en_test, fr_train, fr_test = train_test_split(eng, french, test_size=.1, random_state=67)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum sentence length(i.e n words) English: 6\n",
      "Maximum sentence length(i.e n words) French: 14\n",
      "Size of the vocabulary (English) : 4729\n",
      "Size of the vocabulary (French) : 9051\n"
     ]
    }
   ],
   "source": [
    "def get_vocab_idx(data):\n",
    "    tokenizer = Tokenizer(filters='')\n",
    "    tokenizer.fit_on_texts(data)\n",
    "    max_len = max(map(lambda x: len(x.split()), data))\n",
    "    vocab_size = len(tokenizer.word_index) + 1 # for zero padding.\n",
    "    return tokenizer, vocab_size, max_len\n",
    "\n",
    "en_tokenizer, input_vocab, max_len_en = get_vocab_idx(en_train)\n",
    "fr_tokenizer, target_vocab, max_len_fr = get_vocab_idx(fr_train)\n",
    "\n",
    "print(f'Maximum sentence length(i.e n words) English: {max_len_en}')\n",
    "print(f'Maximum sentence length(i.e n words) French: {max_len_fr}')\n",
    "print(f'Size of the vocabulary (English) : {input_vocab}')\n",
    "print(f'Size of the vocabulary (French) : {target_vocab}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataGenerator(Sequence): \n",
    "    def __init__(self, en_train, fr_train, batch_size, shuffle):        \n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.en_train = en_train\n",
    "        self.fr_train = fr_train\n",
    "        self.on_epoch_end()\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.en_train) // self.batch_size\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
    "        en_batch = [self.en_train[i] for i in indexes]\n",
    "        fr_batch = [self.fr_train[i] for i in indexes]\n",
    "        input_data, decoder_target = self.__data_generation(en_batch, fr_batch)\n",
    "        return input_data, decoder_target\n",
    "    \n",
    "    def on_epoch_end(self):\n",
    "        self.indexes = np.arange(len(self.en_train))\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "               \n",
    "    def __data_generation(self, en_batch, fr_batch):\n",
    "        encoder_input = en_tokenizer.texts_to_sequences(en_batch)\n",
    "        decoder_input = fr_tokenizer.texts_to_sequences(fr_batch)\n",
    "        # zero padding\n",
    "        encoder_input = pad_sequences(encoder_input, max_len_en, padding='post')\n",
    "        decoder_input = pad_sequences(decoder_input, max_len_fr, padding='post')\n",
    "        # decoder target\n",
    "        decoder_target = np.zeros((len(fr_batch), max_len_fr, target_vocab))\n",
    "        for x, seq in enumerate(decoder_input):\n",
    "            for y, s in enumerate(seq):\n",
    "                if y > 0:\n",
    "                    decoder_target[x, y - 1, s] = 1.  \n",
    "        return [encoder_input, decoder_input], decoder_target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 300\n",
    "lstm_units = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "encoder_emb_layer = Embedding(input_dim=input_vocab, output_dim=embed_size)\n",
    "encoder_embeddings = encoder_emb_layer(encoder_inputs)\n",
    "encoder_lstm_layer1 = Bidirectional(LSTM(lstm_units, return_sequences=True, return_state=True))\n",
    "\n",
    "encoder_outputs, fstate_h, fstate_c, bstate_h, bstate_c = encoder_lstm_layer1(encoder_embeddings)\n",
    "state_h = Concatenate()([fstate_h, bstate_h])\n",
    "state_c = Concatenate()([fstate_c, bstate_c])\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "# Decoder with attention\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "decoder_emb_layer = Embedding(input_dim=target_vocab, output_dim=embed_size)\n",
    "decoder_embeddings = decoder_emb_layer(decoder_inputs)\n",
    "decoder_lstm_layer1 = LSTM(lstm_units*2, return_sequences=True, return_state=True)\n",
    "decoder_outputs, _, _ = decoder_lstm_layer1(decoder_embeddings, initial_state=encoder_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "class AttentionLayer(tf.keras.layers.Layer):\n",
    "    \"\"\"\n",
    "    This class implements Bahdanau attention (https://arxiv.org/pdf/1409.0473.pdf).\n",
    "    There are three sets of weights introduced W_a, U_a, and V_a\n",
    "     \"\"\"\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert isinstance(input_shape, list)\n",
    "        # Create a trainable weight variable for this layer.\n",
    "\n",
    "        self.W_a = self.add_weight(name='W_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.U_a = self.add_weight(name='U_a',\n",
    "                                   shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.V_a = self.add_weight(name='V_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], 1)),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "\n",
    "        super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end\n",
    "\n",
    "    def call(self, inputs, verbose=False):\n",
    "        \"\"\"\n",
    "        inputs: [encoder_output_sequence, decoder_output_sequence]\n",
    "        \"\"\"\n",
    "        assert type(inputs) == list\n",
    "        encoder_out_seq, decoder_out_seq = inputs\n",
    "        if verbose:\n",
    "            print('encoder_out_seq>', encoder_out_seq.shape)\n",
    "            print('decoder_out_seq>', decoder_out_seq.shape)\n",
    "\n",
    "        def energy_step(inputs, states):\n",
    "            \"\"\" Step function for computing energy for a single decoder state \"\"\"\n",
    "\n",
    "            assert_msg = \"States must be a list. However states {} is of type {}\".format(states, type(states))\n",
    "            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n",
    "\n",
    "            \"\"\" Some parameters required for shaping tensors\"\"\"\n",
    "            en_seq_len, en_hidden = encoder_out_seq.shape[1], encoder_out_seq.shape[2]\n",
    "            de_hidden = inputs.shape[-1]\n",
    "\n",
    "            \"\"\" Computing S.Wa where S=[s0, s1, ..., si]\"\"\"\n",
    "            # <= batch_size*en_seq_len, latent_dim\n",
    "            reshaped_enc_outputs = K.reshape(encoder_out_seq, (-1, en_hidden))\n",
    "            # <= batch_size*en_seq_len, latent_dim\n",
    "            W_a_dot_s = K.reshape(K.dot(reshaped_enc_outputs, self.W_a), (-1, en_seq_len, en_hidden))\n",
    "            if verbose:\n",
    "                print('wa.s>',W_a_dot_s.shape)\n",
    "\n",
    "            \"\"\" Computing hj.Ua \"\"\"\n",
    "            U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim\n",
    "            if verbose:\n",
    "                print('Ua.h>',U_a_dot_h.shape)\n",
    "\n",
    "            \"\"\" tanh(S.Wa + hj.Ua) \"\"\"\n",
    "            # <= batch_size*en_seq_len, latent_dim\n",
    "            reshaped_Ws_plus_Uh = K.tanh(K.reshape(W_a_dot_s + U_a_dot_h, (-1, en_hidden)))\n",
    "            if verbose:\n",
    "                print('Ws+Uh>', reshaped_Ws_plus_Uh.shape)\n",
    "\n",
    "            \"\"\" softmax(va.tanh(S.Wa + hj.Ua)) \"\"\"\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.reshape(K.dot(reshaped_Ws_plus_Uh, self.V_a), (-1, en_seq_len))\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.softmax(e_i)\n",
    "\n",
    "            if verbose:\n",
    "                print('ei>', e_i.shape)\n",
    "\n",
    "            return e_i, [e_i]\n",
    "\n",
    "        def context_step(inputs, states):\n",
    "            \"\"\" Step function for computing ci using ei \"\"\"\n",
    "            # <= batch_size, hidden_size\n",
    "            c_i = K.sum(encoder_out_seq * K.expand_dims(inputs, -1), axis=1)\n",
    "            if verbose:\n",
    "                print('ci>', c_i.shape)\n",
    "            return c_i, [c_i]\n",
    "\n",
    "        def create_inital_state(inputs, hidden_size):\n",
    "            # We are not using initial states, but need to pass something to K.rnn funciton\n",
    "            fake_state = K.zeros_like(inputs)  # <= (batch_size, enc_seq_len, latent_dim\n",
    "            fake_state = K.sum(fake_state, axis=[1, 2])  # <= (batch_size)\n",
    "            fake_state = K.expand_dims(fake_state)  # <= (batch_size, 1)\n",
    "            fake_state = K.tile(fake_state, [1, hidden_size])  # <= (batch_size, latent_dim\n",
    "            return fake_state\n",
    "\n",
    "        fake_state_c = create_inital_state(encoder_out_seq, encoder_out_seq.shape[-1])\n",
    "        fake_state_e = create_inital_state(encoder_out_seq, encoder_out_seq.shape[1])  # <= (batch_size, enc_seq_len, latent_dim\n",
    "\n",
    "        \"\"\" Computing energy outputs \"\"\"\n",
    "        # e_outputs => (batch_size, de_seq_len, en_seq_len)\n",
    "        last_out, e_outputs, _ = K.rnn(\n",
    "            energy_step, decoder_out_seq, [fake_state_e],\n",
    "        )\n",
    "\n",
    "        \"\"\" Computing context vectors \"\"\"\n",
    "        last_out, c_outputs, _ = K.rnn(\n",
    "            context_step, e_outputs, [fake_state_c],\n",
    "        )\n",
    "\n",
    "        return c_outputs, e_outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        \"\"\" Outputs produced by the layer \"\"\"\n",
    "        return [\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "in converted code:\n\n    <ipython-input-9-bc7c7a274fd3>:93 create_inital_state  *\n        fake_state = K.tile(fake_state, [1, hidden_size])  # <= (batch_size, latent_dim\n    /tmp/tmpop6robw2.py:153 create_inital_state\n        fake_state = ag__.converted_call(K.tile, create_inital_state_scope.callopts, (fake_state, [1, hidden_size]), None, create_inital_state_scope)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py:3014 tile\n        return array_ops.tile(x, n)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py:11311 tile\n        \"Tile\", input=input, multiples=multiples, name=name)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py:530 _apply_op_helper\n        raise err\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py:527 _apply_op_helper\n        preferred_dtype=default_dtype)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1296 internal_convert_to_tensor\n        ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:286 _constant_tensor_conversion_function\n        return constant(v, dtype=dtype, name=name)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:227 constant\n        allow_broadcast=True)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:265 _constant_impl\n        allow_broadcast=allow_broadcast))\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/tensor_util.py:545 make_tensor_proto\n        \"supported type.\" % (type(values), values))\n\n    TypeError: Failed to convert object of type <class 'list'> to Tensor. Contents: [1, None]. Consider casting elements to a supported type.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-c00f2aaab1c5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mattn_layer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAttentionLayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'attention_layer'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mattn_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_energy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattn_layer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mencoder_outputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_outputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    840\u001b[0m                     not base_layer_utils.is_in_eager_or_tf_function()):\n\u001b[1;32m    841\u001b[0m                   \u001b[0;32mwith\u001b[0m \u001b[0mauto_control_deps\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAutomaticControlDependencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0macd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 842\u001b[0;31m                     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    843\u001b[0m                     \u001b[0;31m# Wrap Tensors in `outputs` in `tf.identity` to avoid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    844\u001b[0m                     \u001b[0;31m# circular dependencies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    235\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint:disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ag_error_metadata'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 237\u001b[0;31m           \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mag_error_metadata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m           \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: in converted code:\n\n    <ipython-input-9-bc7c7a274fd3>:93 create_inital_state  *\n        fake_state = K.tile(fake_state, [1, hidden_size])  # <= (batch_size, latent_dim\n    /tmp/tmpop6robw2.py:153 create_inital_state\n        fake_state = ag__.converted_call(K.tile, create_inital_state_scope.callopts, (fake_state, [1, hidden_size]), None, create_inital_state_scope)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py:3014 tile\n        return array_ops.tile(x, n)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py:11311 tile\n        \"Tile\", input=input, multiples=multiples, name=name)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py:530 _apply_op_helper\n        raise err\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py:527 _apply_op_helper\n        preferred_dtype=default_dtype)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1296 internal_convert_to_tensor\n        ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:286 _constant_tensor_conversion_function\n        return constant(v, dtype=dtype, name=name)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:227 constant\n        allow_broadcast=True)\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/constant_op.py:265 _constant_impl\n        allow_broadcast=allow_broadcast))\n    /home/john70/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/tensor_util.py:545 make_tensor_proto\n        \"supported type.\" % (type(values), values))\n\n    TypeError: Failed to convert object of type <class 'list'> to Tensor. Contents: [1, None]. Consider casting elements to a supported type.\n"
     ]
    }
   ],
   "source": [
    "attn_layer = AttentionLayer(name='attention_layer')\n",
    "attn_out, attn_energy = attn_layer([encoder_outputs, decoder_outputs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}