Merge branch 'master' into master

Author: brianjo

.circleci/scripts/build_for_windows.sh

@@ -49,8 +49,9 @@ if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
   python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
   python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
   python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py  beginner_source/audio_preprocessing_tutorial.py || true
-  # Temp remove for mnist download issue.
-  python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
+  python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
 
   export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
   count=0

_static/img/profiler_overview1.PNG

@@ -75,7 +75,7 @@
 </noscript>
 
 <script type="text/javascript">
-  var collapsedSections = ['PyTorch Recipes', 'Image and Video', 'Audio', 'Text', 'Reinforcement Learning', 'Deploying PyTorch Models in Production', 'Code Transforms with FX', 'Frontend APIs', 'Extending PyTorch', 'Model Optimization', 'Parallel and Distributed Training', 'Mobile'];
+  var collapsedSections = ['PyTorch Recipes', 'Learning PyTorch', 'Image and Video', 'Audio', 'Text', 'Reinforcement Learning', 'Deploying PyTorch Models in Production', 'Code Transforms with FX', 'Frontend APIs', 'Extending PyTorch', 'Model Optimization', 'Parallel and Distributed Training', 'Mobile'];
 </script>
 
 <img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>

_static/img/profiler_overview2.PNG

@@ -89,7 +89,6 @@ def forward(self, x):
 class Encoder(nn.Module):
     def __init__(self, ntoken, ninp, dropout=0.5):
         super(Encoder, self).__init__()
-        self.src_mask = None
         self.pos_encoder = PositionalEncoding(ninp, dropout)
         self.encoder = nn.Embedding(ntoken, ninp)
         self.ninp = ninp
@@ -99,17 +98,9 @@ def init_weights(self):
         initrange = 0.1
         self.encoder.weight.data.uniform_(-initrange, initrange)
 
-    def _generate_square_subsequent_mask(self, sz):
-        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
-        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
-        return mask
-
     def forward(self, src):
-        if self.src_mask is None or self.src_mask.size(0) != src.size(0):
-            device = src.device
-            mask = self._generate_square_subsequent_mask(src.size(0)).to(device)
-            self.src_mask = mask
-
+        # Need (S, N) format for encoder.
+        src = src.t()
         src = self.encoder(src) * math.sqrt(self.ninp)
         return self.pos_encoder(src)
 
@@ -125,7 +116,8 @@ def init_weights(self):
         self.decoder.weight.data.uniform_(-initrange, initrange)
 
     def forward(self, inp):
-        return self.decoder(inp)
+        # Need batch dimension first for output of pipeline.
+        return self.decoder(inp).permute(1, 0, 2)
 
 ######################################################################
 # Start multiple processes for training
@@ -245,7 +237,8 @@ def get_batch(source, i):
         seq_len = min(bptt, len(source) - 1 - i)
         data = source[i:i+seq_len]
         target = source[i+1:i+1+seq_len].view(-1)
-        return data, target
+        # Need batch dimension first for pipeline parallelism.
+        return data.t(), target
 
 ######################################################################
 # Model scale and Pipe initialization
@@ -318,8 +311,9 @@ def get_batch(source, i):
     # Need to use 'checkpoint=never' since as of PyTorch 1.8, Pipe checkpointing
     # doesn't work with DDP.
     from torch.distributed.pipeline.sync import Pipe
+    chunks = 8
     model = Pipe(torch.nn.Sequential(
-        *module_list), chunks = 8, checkpoint="never")
+        *module_list), chunks = chunks, checkpoint="never")
 
     # Initialize process group and wrap model in DDP.
     from torch.nn.parallel import DistributedDataParallel

_static/img/profiler_trace_view1.PNG

@@ -50,7 +50,7 @@ See [onnx](https://pytorch.org/docs/stable/onnx.html)
 from torchvision import datasets, models, transforms     # vision datasets, architectures & transforms
 import torchvision.transforms as transforms              # composable transforms
 ```
-See [torchvision](https://pytorch.org/docs/stable/torchvision/index.html)
+See [torchvision](https://pytorch.org/vision/stable/index.html)
 
 ### Distributed Training
 

_static/img/profiler_trace_view2.PNG

@@ -25,7 +25,7 @@
 # PyTorch domain libraries provide a number of pre-loaded datasets (such as FashionMNIST) that 
 # subclass ``torch.utils.data.Dataset`` and implement functions specific to the particular data.
 # They can be used to prototype and benchmark your model. You can find them
-# here: `Image Datasets <https://pytorch.org/docs/stable/torchvision/datasets.html>`_,
+# here: `Image Datasets <https://pytorch.org/vision/stable/datasets.html>`_,
 # `Text Datasets  <https://pytorch.org/text/stable/datasets.html>`_, and
 # `Audio Datasets <https://pytorch.org/audio/stable/datasets.html>`_
 #
@@ -38,7 +38,7 @@
 # Fashion-MNIST is a dataset of Zalando’s article images consisting of of 60,000 training examples and 10,000 test examples.
 # Each example comprises a 28×28 grayscale image and an associated label from one of 10 classes.
 #
-# We load the `FashionMNIST Dataset <https://pytorch.org/docs/stable/torchvision/datasets.html#fashion-mnist>`_ with the following parameters:
+# We load the `FashionMNIST Dataset <https://pytorch.org/vision/stable/datasets.html#fashion-mnist>`_ with the following parameters:
 #  - ``root`` is the path where the train/test data is stored,
 #  - ``train`` specifies training or test dataset,
 #  - ``download=True`` downloads the data from the internet if it's not available at ``root``.

_static/img/profiler_views_list.PNG

@@ -18,7 +18,7 @@
 the `previous section  <autograd_tutorial.html>`_), and **optimizes** these parameters using gradient descent. For a more 
 detailed walkthrough of this process, check out this video on `backpropagation from 3Blue1Brown <https://www.youtube.com/watch?v=tIeHLnjs5U8>`__.
 
-Pre-requisite Code 
+Prerequisite Code 
 -----------------
 We load the code from the previous sections on `Datasets & DataLoaders <data_tutorial.html>`_ 
 and `Build Model  <buildmodel_tutorial.html>`_.

_static/img/tensorboard_pr_curves.png

@@ -35,7 +35,7 @@
 # all of which include datasets. For this tutorial, we  will be using a TorchVision dataset.
 #
 # The ``torchvision.datasets`` module contains ``Dataset`` objects for many real-world vision data like 
-# CIFAR, COCO (`full list here <https://pytorch.org/docs/stable/torchvision/datasets.html>`_). In this tutorial, we
+# CIFAR, COCO (`full list here <https://pytorch.org/vision/stable/datasets.html>`_). In this tutorial, we
 # use the FashionMNIST dataset. Every TorchVision ``Dataset`` includes two arguments: ``transform`` and
 # ``target_transform`` to modify the samples and labels respectively.
 

_templates/layout.html

@@ -18,7 +18,7 @@
 
 All TorchVision datasets have two parameters -``transform`` to modify the features and
 ``target_transform`` to modify the labels - that accept callables containing the transformation logic.
-The `torchvision.transforms <https://pytorch.org/docs/stable/torchvision/transforms.html>`_ module offers 
+The `torchvision.transforms <https://pytorch.org/vision/stable/transforms.html>`_ module offers 
 several commonly-used transforms out of the box.
 
 The FashionMNIST features are in PIL Image format, and the labels are integers.
@@ -41,7 +41,7 @@
 # ToTensor()
 # -------------------------------
 #
-# `ToTensor <https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.ToTensor>`_ 
+# `ToTensor <https://pytorch.org/vision/stable/transforms.html#torchvision.transforms.ToTensor>`_ 
 # converts a PIL image or NumPy ``ndarray`` into a ``FloatTensor``. and scales 
 # the image's pixel intensity values in the range [0., 1.]
 #

advanced_source/ddp_pipeline.py

@@ -73,7 +73,7 @@ Vision
     import torchvision.transforms as transforms              # composable transforms
 
 See
-`torchvision <https://pytorch.org/docs/stable/torchvision/index.html>`__
+`torchvision <https://pytorch.org/vision/stable/index.html>`__
 
 Distributed Training
 --------------------