Build e5d1b46

Author: chsasank

_downloads/dynamic_net.ipynb

@@ -1,54 +1,54 @@
 {
+  "nbformat": 4,
+  "nbformat_minor": 0,
   "metadata": {
-    "kernelspec": {
-      "name": "python3",
-      "language": "python",
-      "display_name": "Python 3"
-    },
     "language_info": {
-      "version": "3.5.2",
       "file_extension": ".py",
+      "version": "3.5.2",
+      "name": "python",
+      "mimetype": "text/x-python",
+      "pygments_lexer": "ipython3",
       "nbconvert_exporter": "python",
       "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "mimetype": "text/x-python",
-      "name": "python",
-      "pygments_lexer": "ipython3"
+        "version": 3,
+        "name": "ipython"
+      }
+    },
+    "kernelspec": {
+      "name": "python3",
+      "language": "python",
+      "display_name": "Python 3"
     }
   },
-  "nbformat": 4,
   "cells": [
     {
+      "source": [
+        "%matplotlib inline"
+      ],
+      "execution_count": null,
       "cell_type": "code",
       "metadata": {
         "collapsed": false
       },
-      "outputs": [],
-      "execution_count": null,
-      "source": [
-        "%matplotlib inline"
-      ]
+      "outputs": []
     },
     {
-      "cell_type": "markdown",
-      "metadata": {},
       "source": [
         "\nPyTorch: Control Flow + Weight Sharing\n--------------------------------------\n\nTo showcase the power of PyTorch dynamic graphs, we will implement a very strange\nmodel: a fully-connected ReLU network that on each forward pass randomly chooses\na number between 1 and 4 and has that many hidden layers, reusing the same\nweights multiple times to compute the innermost hidden layers.\n\n"
-      ]
+      ],
+      "cell_type": "markdown",
+      "metadata": {}
     },
     {
+      "source": [
+        "import random\nimport torch\nfrom torch.autograd import Variable\n\nclass DynamicNet(torch.nn.Module):\n  def __init__(self, D_in, H, D_out):\n    \"\"\"\n    In the constructor we construct three nn.Linear instances that we will use\n    in the forward pass.\n    \"\"\"\n    super(DynamicNet, self).__init__()\n    self.input_linear = torch.nn.Linear(D_in, H)\n    self.middle_linear = torch.nn.Linear(H, H)\n    self.output_linear = torch.nn.Linear(H, D_out)\n\n  def forward(self, x):\n    \"\"\"\n    For the forward pass of the model, we randomly choose either 0, 1, 2, or 3\n    and reuse the middle_linear Module that many times to compute hidden layer\n    representations.\n\n    Since each forward pass builds a dynamic computation graph, we can use normal\n    Python control-flow operators like loops or conditional statements when\n    defining the forward pass of the model.\n\n    Here we also see that it is perfectly safe to reuse the same Module many\n    times when defining a computational graph. This is a big improvement from Lua\n    Torch, where each Module could be used only once.\n    \"\"\"\n    h_relu = self.input_linear(x).clamp(min=0)\n    for _ in range(random.randint(0, 3)):\n      h_relu = self.middle_linear(h_relu).clamp(min=0)\n    y_pred = self.output_linear(h_relu)\n    return y_pred\n\n\n# N is batch size; D_in is input dimension;\n# H is hidden dimension; D_out is output dimension.\nN, D_in, H, D_out = 64, 1000, 100, 10\n\n# Create random Tensors to hold inputs and outputs, and wrap them in Variables\nx = Variable(torch.randn(N, D_in))\ny = Variable(torch.randn(N, D_out), requires_grad=False)\n\n# Construct our model by instantiating the class defined above\nmodel = DynamicNet(D_in, H, D_out)\n\n# Construct our loss function and an Optimizer. Training this strange model with\n# vanilla stochastic gradient descent is tough, so we use momentum\ncriterion = torch.nn.MSELoss(size_average=False)\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\nfor t in range(500):\n  # Forward pass: Compute predicted y by passing x to the model\n  y_pred = model(x)\n\n  # Compute and print loss\n  loss = criterion(y_pred, y)\n  print(t, loss.data[0])\n\n  # Zero gradients, perform a backward pass, and update the weights.\n  optimizer.zero_grad()\n  loss.backward()\n  optimizer.step()"
+      ],
+      "execution_count": null,
       "cell_type": "code",
       "metadata": {
         "collapsed": false
       },
-      "outputs": [],
-      "execution_count": null,
-      "source": [
-        "import random\nimport torch\nfrom torch.autograd import Variable\n\nclass DynamicNet(torch.nn.Module):\n  def __init__(self, D_in, H, D_out):\n    \"\"\"\n    In the constructor we construct three nn.Linear instances that we will use\n    in the forward pass.\n    \"\"\"\n    super(DynamicNet, self).__init__()\n    self.input_linear = torch.nn.Linear(D_in, H)\n    self.middle_linear = torch.nn.Linear(H, H)\n    self.output_linear = torch.nn.Linear(H, D_out)\n\n  def forward(self, x):\n    \"\"\"\n    For the forward pass of the model, we randomly choose either 0, 1, 2, or 3\n    and reuse the middle_linear Module that many times to compute hidden layer\n    representations.\n\n    Since each forward pass builds a dynamic computation graph, we can use normal\n    Python control-flow operators like loops or conditional statements when\n    defining the forward pass of the model.\n\n    Here we also see that it is perfectly safe to reuse the same Module many\n    times when defining a computational graph. This is a big improvement from Lua\n    Torch, where each Module could be used only once.\n    \"\"\"\n    h_relu = self.input_linear(x).clamp(min=0)\n    for _ in range(random.randint(0, 3)):\n      h_relu = self.middle_linear(h_relu).clamp(min=0)\n    y_pred = self.output_linear(h_relu)\n    return y_pred\n\n\n# N is batch size; D_in is input dimension;\n# H is hidden dimension; D_out is output dimension.\nN, D_in, H, D_out = 64, 1000, 100, 10\n\n# Create random Tensors to hold inputs and outputs, and wrap them in Variables\nx = Variable(torch.randn(N, D_in))\ny = Variable(torch.randn(N, D_out), requires_grad=False)\n\n# Construct our model by instantiating the class defined above\nmodel = DynamicNet(D_in, H, D_out)\n\n# Construct our loss function and an Optimizer. Training this strange model with\n# vanilla stochastic gradient descent is tough, so we use momentum\ncriterion = torch.nn.MSELoss(size_average=False)\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\nfor t in range(500):\n  # Forward pass: Compute predicted y by passing x to the model\n  y_pred = model(x)\n\n  # Compute and print loss\n  loss = criterion(y_pred, y)\n  print(t, loss.data[0])\n\n  # Zero gradients, perform a backward pass, and update the weights.\n  optimizer.zero_grad()\n  loss.backward()\n  optimizer.step()"
-      ]
+      "outputs": []
     }
-  ],
-  "nbformat_minor": 0
+  ]
 }