final_behavioral.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from PPO import PPO\n",
    "from PIL import Image\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions import Categorical\n",
    "import time\n",
    "import numpy as np\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_name = \"LunarLander-v2\"\n",
    "# creating environment\n",
    "env = gym.make(env_name)\n",
    "state_dim = env.observation_space.shape[0]\n",
    "action_dim = 4\n",
    "device = \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, state_dim, action_dim, n_latent_var):\n",
    "        super(ActorCritic, self).__init__()\n",
    "\n",
    "        # actor\n",
    "        self.action_layer = nn.Sequential(\n",
    "                nn.Linear(state_dim, n_latent_var),\n",
    "                nn.Tanh(),\n",
    "                nn.Linear(n_latent_var, n_latent_var),\n",
    "                nn.Tanh(),\n",
    "                nn.Linear(n_latent_var, action_dim),\n",
    "                nn.Softmax(dim=-1)\n",
    "                )\n",
    "        \n",
    "        # critic\n",
    "        self.value_layer = nn.Sequential(\n",
    "                nn.Linear(state_dim, n_latent_var),\n",
    "                nn.Tanh(),\n",
    "                nn.Linear(n_latent_var, n_latent_var),\n",
    "                nn.Tanh(),\n",
    "                nn.Linear(n_latent_var, 1)\n",
    "                )\n",
    "        \n",
    "    def forward(self):\n",
    "        raise NotImplementedError\n",
    "        \n",
    "    def act(self, state):\n",
    "        state = torch.from_numpy(state).float().to(device) \n",
    "        action_probs = self.action_layer(state)\n",
    "        dist = Categorical(action_probs)\n",
    "        action = dist.sample()\n",
    "        return action.item()\n",
    "\n",
    "    \n",
    "ppo = ActorCritic(state_dim, action_dim, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "<All keys matched successfully>"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "ppo.load_state_dict(torch.load(\"./PPO_LunarLander-v2.pth\",map_location=torch.device('cpu')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This is only for testing if it works\n",
    "state = env.reset()\n",
    "while True:\n",
    "    action = ppo.act(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    time.sleep(0.01)\n",
    "    if done: break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_sample(indices, batch_size):\n",
    "    indices = np.asarray(np.random.permutation(indices))\n",
    "    batches = indices[:len(indices) // batch_size * batch_size].reshape(-1, batch_size)\n",
    "    for batch in batches:\n",
    "        yield batch\n",
    "    r = len(indices) % batch_size\n",
    "    if r:\n",
    "        yield indices[-r:]\n",
    "\n",
    "def play(model):\n",
    "    state = env.reset()\n",
    "    score = 0\n",
    "    for _ in range(max_len):\n",
    "        action = model.act(state)\n",
    "        env.render()\n",
    "        state, reward, done , _ = env.step(action)\n",
    "        score += reward\n",
    "        time.sleep(0.1) \n",
    "        if done:break\n",
    "    env.close()\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Storage:\n",
    "    def __init__(self):\n",
    "        self.states = deque(maxlen=10000)\n",
    "        self.actions = deque(maxlen=10000)\n",
    "\n",
    "    def reset(self):\n",
    "        del self.states[:]\n",
    "        del self.actions[:]\n",
    "\n",
    "    def return_onehot(self, data, size = action_dim):\n",
    "        final = [0]*size\n",
    "        final[data] += 1\n",
    "        return torch.tensor(final).float().reshape(1,-1).to(device)\n",
    "\n",
    "    def stack(self, data):\n",
    "        return torch.cat(data, dim=0).detach()\n",
    "\n",
    "    def sample(self):\n",
    "        return self.stack(list(self.states)), self.stack(list(self.actions))\n",
    "\n",
    "    def append(self,states, actions):\n",
    "        self.actions += actions\n",
    "        self.states += states\n",
    "\n",
    "    def return_full_data(self):\n",
    "        return (self.states, self.actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 200\n",
    "initial_episode = 5\n",
    "lr = 0.0003             \n",
    "betas = (0.9, 0.999)\n",
    "batch_size = 128\n",
    "training_epoch = 200\n",
    "initial_collection_visizliztion = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Current Iteration 4"
    }
   ],
   "source": [
    "## Data Collection \n",
    "storage = Storage()\n",
    "for k in range(initial_episode):\n",
    "    state = env.reset()\n",
    "    for i in range(max_len):\n",
    "        action = ppo.act(state)\n",
    "        if initial_collection_visizliztion: env.render()\n",
    "        storage.states.append(torch.tensor(state).reshape(1,-1))\n",
    "        storage.actions.append(storage.return_onehot(action))\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        state = next_state\n",
    "        if done: break\n",
    "    print(\"\\rCurrent Iteration {} \".format(k), end = \" \")\n",
    "    if initial_collection_visizliztion: env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training():\n",
    "    ## Training Network for supervised network\n",
    "    for i in range(training_epoch):\n",
    "        loss_sample = []\n",
    "        for sampled_set in random_sample(np.arange(states.size()[0]), batch_size):\n",
    "            y_pred = behavior_cloniing.action_layer(states[sampled_set])\n",
    "            y_actual = actions[sampled_set]\n",
    "            optimizer.zero_grad()\n",
    "            loss = MseLoss(y_pred, y_actual)\n",
    "            loss_sample.append(loss.data)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        if i%50 == 0:\n",
    "            print(\"Iteration {} with Current Loss :{}\".format(i, sum(loss_sample)*(1/len(loss_sample))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Functions is used to do step 2 and 3 of Dagger Algo\n",
    "## this function is used to play using behavioural Network and \n",
    "## rectifing actions inaccordance with perfect network\n",
    "def play_imporve_and_return_storage():\n",
    "    secondary_storage = Storage()\n",
    "    state = env.reset()\n",
    "    while True:\n",
    "        action = behavior_cloniing.act(state)\n",
    "        secondary_storage.states.append(state.reshape(1,-1))\n",
    "        state, _ , done, _ = env.step(action)\n",
    "        action_selected_by_master = ppo.act(state)\n",
    "        secondary_storage.actions.append(secondary_storage.return_onehot(action_selected_by_master))\n",
    "        if done: break\n",
    "    return secondary_storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "1000\n"
    }
   ],
   "source": [
    "states, actions = storage.sample()\n",
    "behavior_cloniing = ActorCritic(state_dim, action_dim, 64).to(device)\n",
    "optimizer = torch.optim.Adam(behavior_cloniing.parameters(), lr=lr, betas=betas)\n",
    "# optimizer = torch.optim.SGD(behavior_cloniing.parameters(), lr = 0.01)\n",
    "MseLoss = nn.MSELoss()\n",
    "print(storage.states.__len__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def DaGGER(iterations = 10):\n",
    "    print(\"Initial Training with Master Network\")\n",
    "    training()\n",
    "    for i in range(iterations):\n",
    "        print(\"Game {} :\".format(i))\n",
    "        secondary_storage = play_imporve_and_return_storage()\n",
    "        new_states, new_actions = secondary_storage.return_full_data()\n",
    "        storage.append(new_states, new_actions)\n",
    "        print(\"Size of new Storage {}\".format(storage.states.__len__()))\n",
    "        training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Initial Training with Master Network\nIteration 0 with Current Loss :0.07327543944120407\nIteration 50 with Current Loss :0.07246550917625427\nIteration 100 with Current Loss :0.07149096578359604\nIteration 150 with Current Loss :0.07127222418785095\nGame 0 :\nSize of new Storage 2447\nIteration 0 with Current Loss :0.070045605301857\nIteration 50 with Current Loss :0.07024196535348892\nIteration 100 with Current Loss :0.06956054270267487\nIteration 150 with Current Loss :0.07026540488004684\nGame 1 :\nSize of new Storage 2570\nIteration 0 with Current Loss :0.06936687231063843\nIteration 50 with Current Loss :0.0693860873579979\nIteration 100 with Current Loss :0.06900467723608017\nIteration 150 with Current Loss :0.06909945607185364\nGame 2 :\nSize of new Storage 2797\nIteration 0 with Current Loss :0.06915541738271713\nIteration 50 with Current Loss :0.06879891455173492\nIteration 100 with Current Loss :0.06848203390836716\nIteration 150 with Current Loss :0.06786541640758514\nGame 3 :\nSize of new Storage 2994\nIteration 0 with Current Loss :0.06788171827793121\nIteration 50 with Current Loss :0.06765644252300262\nIteration 100 with Current Loss :0.06779453158378601\nIteration 150 with Current Loss :0.06735966354608536\nGame 4 :\nSize of new Storage 3219\nIteration 0 with Current Loss :0.06741472333669662\nIteration 50 with Current Loss :0.06735249608755112\nIteration 100 with Current Loss :0.06706640124320984\nIteration 150 with Current Loss :0.06689673662185669\nGame 5 :\nSize of new Storage 3416\nIteration 0 with Current Loss :0.06683149188756943\nIteration 50 with Current Loss :0.06688941270112991\nIteration 100 with Current Loss :0.06626605987548828\nIteration 150 with Current Loss :0.06627359986305237\nGame 6 :\nSize of new Storage 3591\nIteration 0 with Current Loss :0.06597363948822021\nIteration 50 with Current Loss :0.06607558578252792\nIteration 100 with Current Loss :0.06606047600507736\nIteration 150 with Current Loss :0.06590191274881363\nGame 7 :\nSize of new Storage 4031\nIteration 0 with Current Loss :0.0660128965973854\nIteration 50 with Current Loss :0.06545691937208176\nIteration 100 with Current Loss :0.06582439690828323\nIteration 150 with Current Loss :0.0657307431101799\nGame 8 :\nSize of new Storage 4297\nIteration 0 with Current Loss :0.06516876071691513\nIteration 50 with Current Loss :0.06524258852005005\nIteration 100 with Current Loss :0.06493651121854782\nIteration 150 with Current Loss :0.06486330926418304\nGame 9 :\nSize of new Storage 4485\nIteration 0 with Current Loss :0.06490285694599152\nIteration 50 with Current Loss :0.06432691216468811\nIteration 100 with Current Loss :0.06480982899665833\nIteration 150 with Current Loss :0.06475938856601715\n"
    }
   ],
   "source": [
    "DaGGER()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "115.92978978874993"
     },
     "metadata": {},
     "execution_count": 46
    }
   ],
   "source": [
    "play(behavior_cloniing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python36864bitpy36condacd98d047ccbe4577af8016d2df1f7b32",
   "display_name": "Python 3.6.8 64-bit ('py36': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}