Merge branch 'master' into patch-1

Author: holly1238

.circleci/scripts/build_for_windows.sh

@@ -49,8 +49,10 @@ if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
   python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
   python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
   python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py  beginner_source/audio_preprocessing_tutorial.py || true
-  # Temp remove for mnist download issue.
-  python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py  beginner_source/dcgan_faces_tutorial.py || true
+  python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
 
   export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
   count=0

README.md

@@ -28,10 +28,10 @@ In case you prefer to write your tutorial in jupyter, you can use [this script](
 - Then you can build using `make docs`. This will download the data, execute the tutorials and build the documentation to `docs/` directory. This will take about 60-120 min for systems with GPUs. If you do not have a GPU installed on your system, then see next step.
 - You can skip the computationally intensive graph generation by running `make html-noplot` to build basic html documentation to `_build/html`. This way, you can quickly preview your tutorial.
 
-> If you get **ModuleNotFoundError: No module named 'pytorch_sphinx_theme' make: *** [html-noplot] Error 2**, from /tutorials/src/pytorch-sphinx-theme run `python setup.py install`. 
+> If you get **ModuleNotFoundError: No module named 'pytorch_sphinx_theme' make: *** [html-noplot] Error 2** from /tutorials/src/pytorch-sphinx-theme or /venv/src/pytorch-sphinx-theme (while using virtualenv), run `python setup.py install`. 
 
 
 ## About contributing to PyTorch Documentation and Tutorials
 * You can find information about contributing to PyTorch documentation in the 
 PyTorch Repo [README.md](https://github.com/pytorch/pytorch/blob/master/README.md) file. 
-* Additional information can be found in [PyTorch CONTRIBUTING.md](https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md).
+* Additional information can be found in [PyTorch CONTRIBUTING.md](https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md).

_static/img/profiler_overview1.png

@@ -75,7 +75,7 @@
 </noscript>
 
 <script type="text/javascript">
-  var collapsedSections = ['PyTorch Recipes', 'Image and Video', 'Audio', 'Text', 'Reinforcement Learning', 'Deploying PyTorch Models in Production', 'Code Transforms with FX', 'Frontend APIs', 'Extending PyTorch', 'Model Optimization', 'Parallel and Distributed Training', 'Mobile'];
+  var collapsedSections = ['PyTorch Recipes', 'Learning PyTorch', 'Image and Video', 'Audio', 'Text', 'Reinforcement Learning', 'Deploying PyTorch Models in Production', 'Code Transforms with FX', 'Frontend APIs', 'Extending PyTorch', 'Model Optimization', 'Parallel and Distributed Training', 'Mobile'];
 </script>
 
 <img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>

_static/img/profiler_overview2.png

@@ -115,7 +115,7 @@ If you need to exclude some methods in your ``nn.Module``
 because they use Python features that TorchScript doesn't support yet,
 you could annotate those with ``@torch.jit.ignore``
 
-``my_module`` is an instance of
+``sm`` is an instance of
 ``ScriptModule`` that is ready for serialization.
 
 Step 2: Serializing Your Script Module to a File
@@ -132,7 +132,7 @@ on the module and pass it a filename::
   traced_script_module.save("traced_resnet_model.pt")
 
 This will produce a ``traced_resnet_model.pt`` file in your working directory.
-If you also would like to serialize ``my_module``, call ``my_module.save("my_module_model.pt")``
+If you also would like to serialize ``sm``, call ``sm.save("my_module_model.pt")``
 We have now officially left the realm of Python and are ready to cross over to the sphere
 of C++.
 

_static/img/profiler_trace_view1.png

@@ -89,7 +89,6 @@ def forward(self, x):
 class Encoder(nn.Module):
     def __init__(self, ntoken, ninp, dropout=0.5):
         super(Encoder, self).__init__()
-        self.src_mask = None
         self.pos_encoder = PositionalEncoding(ninp, dropout)
         self.encoder = nn.Embedding(ntoken, ninp)
         self.ninp = ninp
@@ -99,17 +98,9 @@ def init_weights(self):
         initrange = 0.1
         self.encoder.weight.data.uniform_(-initrange, initrange)
 
-    def _generate_square_subsequent_mask(self, sz):
-        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
-        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
-        return mask
-
     def forward(self, src):
-        if self.src_mask is None or self.src_mask.size(0) != src.size(0):
-            device = src.device
-            mask = self._generate_square_subsequent_mask(src.size(0)).to(device)
-            self.src_mask = mask
-
+        # Need (S, N) format for encoder.
+        src = src.t()
         src = self.encoder(src) * math.sqrt(self.ninp)
         return self.pos_encoder(src)
 
@@ -125,7 +116,8 @@ def init_weights(self):
         self.decoder.weight.data.uniform_(-initrange, initrange)
 
     def forward(self, inp):
-        return self.decoder(inp)
+        # Need batch dimension first for output of pipeline.
+        return self.decoder(inp).permute(1, 0, 2)
 
 ######################################################################
 # Start multiple processes for training
@@ -245,7 +237,8 @@ def get_batch(source, i):
         seq_len = min(bptt, len(source) - 1 - i)
         data = source[i:i+seq_len]
         target = source[i+1:i+1+seq_len].view(-1)
-        return data, target
+        # Need batch dimension first for pipeline parallelism.
+        return data.t(), target
 
 ######################################################################
 # Model scale and Pipe initialization
@@ -318,8 +311,9 @@ def get_batch(source, i):
     # Need to use 'checkpoint=never' since as of PyTorch 1.8, Pipe checkpointing
     # doesn't work with DDP.
     from torch.distributed.pipeline.sync import Pipe
+    chunks = 8
     model = Pipe(torch.nn.Sequential(
-        *module_list), chunks = 8, checkpoint="never")
+        *module_list), chunks = chunks, checkpoint="never")
 
     # Initialize process group and wrap model in DDP.
     from torch.nn.parallel import DistributedDataParallel

_static/img/profiler_trace_view2.png

@@ -50,7 +50,7 @@ See [onnx](https://pytorch.org/docs/stable/onnx.html)
 from torchvision import datasets, models, transforms     # vision datasets, architectures & transforms
 import torchvision.transforms as transforms              # composable transforms
 ```
-See [torchvision](https://pytorch.org/docs/stable/torchvision/index.html)
+See [torchvision](https://pytorch.org/vision/stable/index.html)
 
 ### Distributed Training
 

_static/img/profiler_views_list.png

@@ -47,7 +47,7 @@
 #
 # In this network, ``w`` and ``b`` are **parameters**, which we need to
 # optimize. Thus, we need to be able to compute the gradients of loss
-# function with respect to those variables. In orded to do that, we set
+# function with respect to those variables. In order to do that, we set
 # the ``requires_grad`` property of those tensors.
 
 #######################################################################
@@ -58,7 +58,7 @@
 # A function that we apply to tensors to construct computational graph is
 # in fact an object of class ``Function``. This object knows how to
 # compute the function in the *forward* direction, and also how to compute
-# it's derivative during the *backward propagation* step. A reference to
+# its derivative during the *backward propagation* step. A reference to
 # the backward propagation function is stored in ``grad_fn`` property of a
 # tensor. You can find more information of ``Function`` `in the
 # documentation <https://pytorch.org/docs/stable/autograd.html#function>`__.

_static/img/tensorboard_pr_curves.png

@@ -67,7 +67,7 @@ def forward(self, x):
 
 ##############################################
 # We create an instance of ``NeuralNetwork``, and move it to the ``device``, and print 
-# it's structure.
+# its structure.
 
 model = NeuralNetwork().to(device)
 print(model)
@@ -119,7 +119,7 @@ def forward(self, x):
 # nn.Linear 
 # ^^^^^^^^^^^^^^^^^^^^^^
 # The `linear layer <https://pytorch.org/docs/stable/generated/torch.nn.Linear.html>`_
-# is a module that applies a linear transformation on the input using it's stored weights and biases.
+# is a module that applies a linear transformation on the input using its stored weights and biases.
 #
 layer1 = nn.Linear(in_features=28*28, out_features=20)
 hidden1 = layer1(flat_image)

_static/img/thumbnails/cropped/parametrizations.png

@@ -25,7 +25,7 @@
 # PyTorch domain libraries provide a number of pre-loaded datasets (such as FashionMNIST) that 
 # subclass ``torch.utils.data.Dataset`` and implement functions specific to the particular data.
 # They can be used to prototype and benchmark your model. You can find them
-# here: `Image Datasets <https://pytorch.org/docs/stable/torchvision/datasets.html>`_,
+# here: `Image Datasets <https://pytorch.org/vision/stable/datasets.html>`_,
 # `Text Datasets  <https://pytorch.org/text/stable/datasets.html>`_, and
 # `Audio Datasets <https://pytorch.org/audio/stable/datasets.html>`_
 #
@@ -38,7 +38,7 @@
 # Fashion-MNIST is a dataset of Zalando’s article images consisting of of 60,000 training examples and 10,000 test examples.
 # Each example comprises a 28×28 grayscale image and an associated label from one of 10 classes.
 #
-# We load the `FashionMNIST Dataset <https://pytorch.org/docs/stable/torchvision/datasets.html#fashion-mnist>`_ with the following parameters:
+# We load the `FashionMNIST Dataset <https://pytorch.org/vision/stable/datasets.html#fashion-mnist>`_ with the following parameters:
 #  - ``root`` is the path where the train/test data is stored,
 #  - ``train`` specifies training or test dataset,
 #  - ``download=True`` downloads the data from the internet if it's not available at ``root``.
@@ -225,7 +225,7 @@ def __getitem__(self, idx):
 # --------------------------
 #
 # We have loaded that dataset into the ``Dataloader`` and can iterate through the dataset as needed.
-# Each iteration below returns a batch of ``train_features`` and ``train_labels``(containing ``batch_size=64`` features and labels respectively).
+# Each iteration below returns a batch of ``train_features`` and ``train_labels`` (containing ``batch_size=64`` features and labels respectively).
 # Because we specified ``shuffle=True``, after we iterate over all batches the data is shuffled (for finer-grained control over 
 # the data loading order, take a look at `Samplers <https://pytorch.org/docs/stable/data.html#data-loading-order-and-sampler>`_).
 

_templates/layout.html

@@ -12,13 +12,13 @@
 Optimizing Model Parameters
 ===========================
 
-Now that we have a model and data it's time to train, validate and test our model by optimizing it's parameters on 
+Now that we have a model and data it's time to train, validate and test our model by optimizing its parameters on 
 our data. Training a model is an iterative process; in each iteration (called an *epoch*) the model makes a guess about the output, calculates 
 the error in its guess (*loss*), collects the derivatives of the error with respect to its parameters (as we saw in 
 the `previous section  <autograd_tutorial.html>`_), and **optimizes** these parameters using gradient descent. For a more 
 detailed walkthrough of this process, check out this video on `backpropagation from 3Blue1Brown <https://www.youtube.com/watch?v=tIeHLnjs5U8>`__.
 
-Pre-requisite Code 
+Prerequisite Code 
 -----------------
 We load the code from the previous sections on `Datasets & DataLoaders <data_tutorial.html>`_ 
 and `Build Model  <buildmodel_tutorial.html>`_.

advanced_source/cpp_export.rst

@@ -35,7 +35,7 @@
 # all of which include datasets. For this tutorial, we  will be using a TorchVision dataset.
 #
 # The ``torchvision.datasets`` module contains ``Dataset`` objects for many real-world vision data like 
-# CIFAR, COCO (`full list here <https://pytorch.org/docs/stable/torchvision/datasets.html>`_). In this tutorial, we
+# CIFAR, COCO (`full list here <https://pytorch.org/vision/stable/datasets.html>`_). In this tutorial, we
 # use the FashionMNIST dataset. Every TorchVision ``Dataset`` includes two arguments: ``transform`` and
 # ``target_transform`` to modify the samples and labels respectively.
 

advanced_source/ddp_pipeline.py

@@ -18,7 +18,7 @@
 
 All TorchVision datasets have two parameters -``transform`` to modify the features and
 ``target_transform`` to modify the labels - that accept callables containing the transformation logic.
-The `torchvision.transforms <https://pytorch.org/docs/stable/torchvision/transforms.html>`_ module offers 
+The `torchvision.transforms <https://pytorch.org/vision/stable/transforms.html>`_ module offers 
 several commonly-used transforms out of the box.
 
 The FashionMNIST features are in PIL Image format, and the labels are integers.
@@ -41,7 +41,7 @@
 # ToTensor()
 # -------------------------------
 #
-# `ToTensor <https://pytorch.org/docs/stable/torchvision/transforms.html#torchvision.transforms.ToTensor>`_ 
+# `ToTensor <https://pytorch.org/vision/stable/transforms.html#torchvision.transforms.ToTensor>`_ 
 # converts a PIL image or NumPy ``ndarray`` into a ``FloatTensor``. and scales 
 # the image's pixel intensity values in the range [0., 1.]
 #

beginner_source/PyTorch Cheat.md

@@ -13,12 +13,11 @@ Deep Learning with PyTorch: A 60 Minute Blitz
 	Neural Networks
 	https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html#
 
-4. autograd_tutorial.py
-	Automatic Differentiation 
-	https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html
-
-5. cifar10_tutorial.py
+4. cifar10_tutorial.py
 	Training a Classifier
 	https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html
 
+5. data_parallel_tutorial.py 
+	Optional: Data Parallelism
+	https://pytorch.org/tutorials/beginner/blitz/data_parallel_tutorial.html
 

beginner_source/basics/autogradqs_tutorial.py

@@ -43,15 +43,15 @@
 
 We will do the following steps in order:
 
-1. Load and normalizing the CIFAR10 training and test datasets using
+1. Load and normalize the CIFAR10 training and test datasets using
    ``torchvision``
 2. Define a Convolutional Neural Network
 3. Define a loss function
 4. Train the network on the training data
 5. Test the network on the test data
 
-1. Loading and normalizing CIFAR10
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+1. Load and normalize CIFAR10
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Using ``torchvision``, it’s extremely easy to load CIFAR10.
 """

beginner_source/basics/buildmodel_tutorial.py

@@ -58,7 +58,7 @@ def __init__(self):
     def forward(self, x):
         # Max pooling over a (2, 2) window
         x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
-        # If the size is a square you can only specify a single number
+        # If the size is a square, you can specify with a single number
         x = F.max_pool2d(F.relu(self.conv2(x)), 2)
         x = x.view(-1, self.num_flat_features(x))
         x = F.relu(self.fc1(x))
@@ -176,8 +176,9 @@ def num_flat_features(self, x):
 #           -> loss
 #
 # So, when we call ``loss.backward()``, the whole graph is differentiated
-# w.r.t. the loss, and all Tensors in the graph that have ``requires_grad=True``
-# will have their ``.grad`` Tensor accumulated with the gradient.
+# w.r.t. the neural net parameters, and all Tensors in the graph that have
+# ``requires_grad=True`` will have their ``.grad`` Tensor accumulated with the
+# gradient.
 #
 # For illustration, let us follow a few steps backward:
 

beginner_source/basics/data_tutorial.py

@@ -471,7 +471,7 @@ def trimRareWords(voc, pairs, MIN_COUNT):
 # with mini-batches.
 #
 # Using mini-batches also means that we must be mindful of the variation
-# of sentence length in our batches. To accomodate sentences of different
+# of sentence length in our batches. To accommodate sentences of different
 # sizes in the same batch, we will make our batched input tensor of shape
 # *(max_length, batch_size)*, where sentences shorter than the
 # *max_length* are zero padded after an *EOS_token*.
@@ -615,7 +615,7 @@ def batch2TrainData(voc, pair_batch):
 # in normal sequential order, and one that is fed the input sequence in
 # reverse order. The outputs of each network are summed at each time step.
 # Using a bidirectional GRU will give us the advantage of encoding both
-# past and future context.
+# past and future contexts.
 #
 # Bidirectional RNN:
 #
@@ -700,7 +700,7 @@ def forward(self, input_seq, input_lengths, hidden=None):
 # states to generate the next word in the sequence. It continues
 # generating words until it outputs an *EOS_token*, representing the end
 # of the sentence. A common problem with a vanilla seq2seq decoder is that
-# if we rely soley on the context vector to encode the entire input
+# if we rely solely on the context vector to encode the entire input
 # sequence’s meaning, it is likely that we will have information loss.
 # This is especially the case when dealing with long input sequences,
 # greatly limiting the capability of our decoder.
@@ -950,7 +950,7 @@ def maskNLLLoss(inp, target, mask):
 #   sequence (or batch of sequences). We use the ``GRU`` layer like this in
 #   the ``encoder``. The reality is that under the hood, there is an
 #   iterative process looping over each time step calculating hidden states.
-#   Alternatively, you ran run these modules one time-step at a time. In
+#   Alternatively, you can run these modules one time-step at a time. In
 #   this case, we manually loop over the sequences during the training
 #   process like we must do for the ``decoder`` model. As long as you
 #   maintain the correct conceptual model of these modules, implementing
@@ -1115,7 +1115,7 @@ def trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer, deco
 # softmax value. This decoding method is optimal on a single time-step
 # level.
 #
-# To facilite the greedy decoding operation, we define a
+# To facilitate the greedy decoding operation, we define a
 # ``GreedySearchDecoder`` class. When run, an object of this class takes
 # an input sequence (``input_seq``) of shape *(input_seq length, 1)*, a
 # scalar input length (``input_length``) tensor, and a ``max_length`` to

beginner_source/basics/optimization_tutorial.py

@@ -20,7 +20,7 @@ At the top of the page click **Run in Google Colab**.
 
 The file will open in Colab.
 
-If you choose, **Runtime** then **Run All**, you'll get an error as the
+If you select **Runtime**, and then **Run All**, you'll get an error as the
 file can't be found.
 
 To fix this, we'll copy the required file into our Google Drive account.
@@ -30,7 +30,7 @@ To fix this, we'll copy the required file into our Google Drive account.
    **cornell**.
 3. Visit the Cornell Movie Dialogs Corpus and download the ZIP file.
 4. Unzip the file on your local machine.
-5. Copy the file **movie\_lines.txt** to **data/cornell** folder you
+5. Copy the files **movie\_lines.txt** and **movie\_conversations.txt** to the **data/cornell** folder that you
    created in Google Drive.
 
 Now we'll need to edit the file in\_ \_Colab to point to the file on
@@ -55,12 +55,12 @@ Change the two lines that follow:
 
 We're now pointing to the file we uploaded to Drive.
 
-Now when you click on the **Run cell** button for the code section,
+Now when you click the **Run cell** button for the code section,
 you'll be prompted to authorize Google Drive and you'll get an
 authorization code. Paste the code into the prompt in Colab and you
 should be set.
 
-Rerun the notebook from **Runtime** / **Run All** menu command and
+Rerun the notebook from the **Runtime** / **Run All** menu command and
 you'll see it process. (Note that this tutorial takes a long time to
 run.)