Pytorch NN module.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch NN module\n",
    "\n",
    "Deep learning consists of composing linearities with non-linearities in clever ways. The introduction of non-linearities allows for powerful models. In this section, we will play with these core components, make up an objective function, and see how the model is trained.\n",
    "\n",
    "### Affine Maps\n",
    "\n",
    "One of the core workhorses of deep learning is the affine map, which is a function f(x) where\n",
    "\n",
    "                                            f(x)=Ax+b\n",
    "for a matrix A and vectors x,b. The parameters to be learned here are A and b. Often, b is refered to as the bias term.\n",
    "\n",
    "Pytorch and most other deep learning frameworks do things a little differently than traditional linear algebra. **It maps the rows of the input instead of the columns. That is, the i‘th row of the output below is the mapping of the i‘th row of the input under A, plus the bias term. Look at the example below.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 436.4664611816406)\n",
      "(101, 280.26239013671875)\n",
      "(201, 216.25485229492188)\n",
      "(301, 192.68450927734375)\n",
      "(401, 184.41600036621094)\n",
      "(501, 181.48941040039062)\n",
      "(601, 180.35386657714844)\n",
      "(701, 179.81932067871094)\n",
      "(801, 179.4915008544922)\n",
      "(901, 179.2390594482422)\n",
      "CPU times: user 696 ms, sys: 612 ms, total: 1.31 s\n",
      "Wall time: 5.76 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "M, In_size, H_size, Out_size = 1000, 5, 4, 2\n",
    "\n",
    "x = Variable(torch.rand(M, In_size), requires_grad = False)  # Row is taking different example\n",
    "y = Variable(torch.rand(M, Out_size), requires_grad = False) # that's how it's defined in package, \n",
    "                                                            # so operation will be col major\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "        torch.nn.Linear(In_size, H_size),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(H_size, Out_size))\n",
    "\n",
    "loss = torch.nn.MSELoss(size_average = False)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for t in range(1000):\n",
    "    out = model(x)\n",
    "    loss_out = loss(out, y)\n",
    "    if t%100 == 1:\n",
    "        print(t, loss_out.data[0])\n",
    "    model.zero_grad()\n",
    "    \n",
    "    loss_out.backward()\n",
    "    \n",
    "    for param in model.parameters():\n",
    "        param.data -= learning_rate * param.grad.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Optimizer instead of manual update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 495.200439453125)\n",
      "(101, 494.8741455078125)\n",
      "(201, 494.54766845703125)\n",
      "(301, 494.22125244140625)\n",
      "(401, 493.89483642578125)\n",
      "(501, 493.568603515625)\n",
      "(601, 493.2416687011719)\n",
      "(701, 492.9151306152344)\n",
      "(801, 492.5890197753906)\n",
      "(901, 492.2625732421875)\n",
      "CPU times: user 332 ms, sys: 28 ms, total: 360 ms\n",
      "Wall time: 400 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "M, In_size, H_size, Out_size = 1000, 5, 4, 2\n",
    "\n",
    "x = Variable(torch.rand(M, In_size), requires_grad = False)  # Row is taking different example\n",
    "y = Variable(torch.rand(M, Out_size), requires_grad = False) # that's how it's defined in package, \n",
    "                                                            # so operation will be col major\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "        torch.nn.Linear(In_size, H_size),\n",
    "        torch.nn.ReLU(),\n",
    "        torch.nn.Linear(H_size, Out_size))\n",
    "\n",
    "loss = torch.nn.MSELoss(size_average = False)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n",
    "for t in range(1000):\n",
    "    out = model(x)\n",
    "    loss_out = loss(out, y)\n",
    "    if t%100 == 1:\n",
    "        print(t, loss_out.data[0])\n",
    "    model.zero_grad()\n",
    "    \n",
    "    loss_out.backward()\n",
    "    \n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom net from NN module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1700.6502685546875)\n",
      "(101, 1700.131591796875)\n",
      "(201, 1699.613525390625)\n",
      "(301, 1699.096435546875)\n",
      "(401, 1698.577392578125)\n",
      "(501, 1698.0589599609375)\n",
      "(601, 1697.540283203125)\n",
      "(701, 1697.0238037109375)\n",
      "(801, 1696.507080078125)\n",
      "(901, 1695.988525390625)\n"
     ]
    }
   ],
   "source": [
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, In_size, H_size, Out_size):\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(In_size, H_size)\n",
    "        self.linear2 = torch.nn.Linear(H_size, Out_size)\n",
    "    def forward(self, x):\n",
    "        h = self.linear1(x).clamp(min = 0)\n",
    "        out = self.linear2(h)\n",
    "        return out\n",
    "\n",
    "M, In_size, H_size, Out_size = 1000, 5, 4, 2\n",
    "\n",
    "x = Variable(torch.rand(M, In_size), requires_grad = False)  # Row is taking different example\n",
    "y = Variable(torch.rand(M, Out_size), requires_grad = False) # that's how it's defined in package, \n",
    "                                                            # so operation will be col major\n",
    "\n",
    "model = TwoLayerNet(In_size, H_size, Out_size)\n",
    "\n",
    "loss = torch.nn.MSELoss(size_average = False)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n",
    "for t in range(1000):\n",
    "    out = model(x)\n",
    "    loss_out = loss(out, y)\n",
    "    if t%100 == 1:\n",
    "        print(t, loss_out.data[0])\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    loss_out.backward()\n",
    "    \n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}