Merge branch 'master' into packed_accessor

Author: ezyang

.jenkins/build.sh

@@ -19,7 +19,11 @@ export PATH=/opt/conda/bin:$PATH
 pip install sphinx==1.8.2 pandas
 
 # For Tensorboard. Until 1.14 moves to the release channel.
-pip install tb-nightly  
+pip install tb-nightly
+
+# Install two language tokenizers for Translation with TorchText tutorial
+python -m spacy download en
+python -m spacy download de
 
 # PyTorch Theme
 rm -rf src

_static/img/thumbnails/german_to_english_translation.png

@@ -1,4 +1,4 @@
-3. Loading a TorchScript Model in C++
+Loading a TorchScript Model in C++
 =====================================
 
 **This tutorial was updated to work with PyTorch 1.2**

_static/img/thumbnails/torchtext.png

@@ -1,5 +1,5 @@
 """
-4. (optional) Exporting a Model from PyTorch to ONNX and Running it using ONNX Runtime
+(optional) Exporting a Model from PyTorch to ONNX and Running it using ONNX Runtime
 ========================================================================
 
 In this tutorial, we describe how to convert a model defined

_static/img/transformer_input_target.png

@@ -1,5 +1,5 @@
 """
-2. Introduction to TorchScript
+Introduction to TorchScript
 ===========================
 
 *James Reed (jamesreed@fb.com), Michael Suo (suo@fb.com)*, rev2
@@ -24,7 +24,7 @@
 -  How to compose both approaches
 -  Saving and loading TorchScript modules
 
-We hope that after you complete this tutorial, you proceed to go through
+We hope that after you complete this tutorial, you will proceed to go through
 `the follow-on tutorial <https://pytorch.org/tutorials/advanced/cpp_export.html>`_
 which will walk you through an example of actually calling a TorchScript
 model from C++.

advanced_source/cpp_export.rst

@@ -20,3 +20,7 @@ Beginner Tutorials
 5. nlp/* and deep_learning_nlp_tutorial.rst
 	Deep Learning for NLP with Pytorch
 	https://pytorch.org/tutorials/beginner/deep_learning_nlp_tutorial.html
+
+6. transformer_translation.py
+	Language Translation with Transformers
+	https://pytorch.org/tutorials/beginner/transformer_translation.html

advanced_source/super_resolution_with_onnxruntime.py

@@ -1,5 +1,5 @@
 """
-4. (advanced) PyTorch 1.0 Distributed Trainer with Amazon AWS
+(advanced) PyTorch 1.0 Distributed Trainer with Amazon AWS
 =============================================================
 
 **Author**: `Nathan Inkawhich <https://github.com/inkawhich>`_

beginner_source/Intro_to_TorchScript_tutorial.py

@@ -537,7 +537,7 @@ def outputVar(l, voc):
     max_target_len = max([len(indexes) for indexes in indexes_batch])
     padList = zeroPadding(indexes_batch)
     mask = binaryMatrix(padList)
-    mask = torch.ByteTensor(mask)
+    mask = torch.BoolTensor(mask)
     padVar = torch.LongTensor(padList)
     return padVar, mask, max_target_len
 

beginner_source/README.txt

@@ -1,23 +1,23 @@
 """
-Text Classification Tutorial
-============================
+Text Classification with TorchText
+==================================
 
-This tutorial shows how to use the text classification datasets,
-including
+This tutorial shows how to use the text classification datasets
+in ``torchtext``, including
 
 ::
 
    - AG_NEWS,
-   - SogouNews, 
-   - DBpedia, 
+   - SogouNews,
+   - DBpedia,
    - YelpReviewPolarity,
-   - YelpReviewFull, 
-   - YahooAnswers, 
+   - YelpReviewFull,
+   - YahooAnswers,
    - AmazonReviewPolarity,
    - AmazonReviewFull
 
-This example shows the application of ``TextClassification`` Dataset for
-supervised learning analysis.
+This example shows how to train a supervised learning algorithm for
+classification using one of these ``TextClassification`` datasets.
 
 Load data with ngrams
 ---------------------
@@ -54,20 +54,20 @@
 ######################################################################
 # Define the model
 # ----------------
-# 
+#
 # The model is composed of the
 # `EmbeddingBag <https://pytorch.org/docs/stable/nn.html?highlight=embeddingbag#torch.nn.EmbeddingBag>`__
 # layer and the linear layer (see the figure below). ``nn.EmbeddingBag``
 # computes the mean value of a “bag” of embeddings. The text entries here
 # have different lengths. ``nn.EmbeddingBag`` requires no padding here
 # since the text lengths are saved in offsets.
-# 
+#
 # Additionally, since ``nn.EmbeddingBag`` accumulates the average across
 # the embeddings on the fly, ``nn.EmbeddingBag`` can enhance the
 # performance and memory efficiency to process a sequence of tensors.
-# 
+#
 # .. image:: ../_static/img/text_sentiment_ngrams_model.png
-# 
+#
 
 import torch.nn as nn
 import torch.nn.functional as F
@@ -83,7 +83,7 @@ def init_weights(self):
         self.embedding.weight.data.uniform_(-initrange, initrange)
         self.fc.weight.data.uniform_(-initrange, initrange)
         self.fc.bias.data.zero_()
-        
+
     def forward(self, text, offsets):
         embedded = self.embedding(text, offsets)
         return self.fc(embedded)
@@ -92,21 +92,21 @@ def forward(self, text, offsets):
 ######################################################################
 # Initiate an instance
 # --------------------
-# 
+#
 # The AG_NEWS dataset has four labels and therefore the number of classes
 # is four.
-# 
+#
 # ::
-# 
+#
 #    1 : World
 #    2 : Sports
 #    3 : Business
 #    4 : Sci/Tec
-# 
+#
 # The vocab size is equal to the length of vocab (including single word
 # and ngrams). The number of classes is equal to the number of labels,
 # which is four in AG_NEWS case.
-# 
+#
 
 VOCAB_SIZE = len(train_dataset.get_vocab())
 EMBED_DIM = 32
@@ -117,7 +117,7 @@ def forward(self, text, offsets):
 ######################################################################
 # Functions used to generate batch
 # --------------------------------
-# 
+#
 
 
 ######################################################################
@@ -129,13 +129,13 @@ def forward(self, text, offsets):
 # mini-batch. Pay attention here and make sure that ``collate_fn`` is
 # declared as a top level def. This ensures that the function is available
 # in each worker.
-# 
+#
 # The text entries in the original data batch input are packed into a list
 # and concatenated as a single tensor as the input of ``nn.EmbeddingBag``.
 # The offsets is a tensor of delimiters to represent the beginning index
 # of the individual sequence in the text tensor. Label is a tensor saving
 # the labels of individual text entries.
-# 
+#
 
 def generate_batch(batch):
     label = torch.tensor([entry[0] for entry in batch])
@@ -144,7 +144,7 @@ def generate_batch(batch):
     # torch.Tensor.cumsum returns the cumulative sum
     # of elements in the dimension dim.
     # torch.Tensor([1.0, 2.0, 3.0]).cumsum(dim=0)
-    
+
     offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)
     text = torch.cat(text)
     return text, offsets, label
@@ -153,7 +153,7 @@ def generate_batch(batch):
 ######################################################################
 # Define functions to train the model and evaluate results.
 # ---------------------------------------------------------
-# 
+#
 
 
 ######################################################################
@@ -163,7 +163,7 @@ def generate_batch(batch):
 # `here <https://pytorch.org/tutorials/beginner/data_loading_tutorial.html>`__).
 # We use ``DataLoader`` here to load AG_NEWS datasets and send it to the
 # model for training/validation.
-# 
+#
 
 from torch.utils.data import DataLoader
 
@@ -186,7 +186,7 @@ def train_func(sub_train_):
 
     # Adjust the learning rate
     scheduler.step()
-    
+
     return train_loss / len(sub_train_), train_acc / len(sub_train_)
 
 def test(data_):
@@ -207,13 +207,13 @@ def test(data_):
 ######################################################################
 # Split the dataset and run the model
 # -----------------------------------
-# 
+#
 # Since the original AG_NEWS has no valid dataset, we split the training
 # dataset into train/valid sets with a split ratio of 0.95 (train) and
 # 0.05 (valid). Here we use
 # `torch.utils.data.dataset.random_split <https://pytorch.org/docs/stable/data.html?highlight=random_split#torch.utils.data.random_split>`__
 # function in PyTorch core library.
-# 
+#
 # `CrossEntropyLoss <https://pytorch.org/docs/stable/nn.html?highlight=crossentropyloss#torch.nn.CrossEntropyLoss>`__
 # criterion combines nn.LogSoftmax() and nn.NLLLoss() in a single class.
 # It is useful when training a classification problem with C classes.
@@ -222,7 +222,7 @@ def test(data_):
 # learning rate is set to 4.0.
 # `StepLR <https://pytorch.org/docs/master/_modules/torch/optim/lr_scheduler.html#StepLR>`__
 # is used here to adjust the learning rate through epochs.
-# 
+#
 
 import time
 from torch.utils.data.dataset import random_split
@@ -250,56 +250,56 @@ def test(data_):
     print('Epoch: %d' %(epoch + 1), " | time in %d minutes, %d seconds" %(mins, secs))
     print(f'\tLoss: {train_loss:.4f}(train)\t|\tAcc: {train_acc * 100:.1f}%(train)')
     print(f'\tLoss: {valid_loss:.4f}(valid)\t|\tAcc: {valid_acc * 100:.1f}%(valid)')
-              
+
 
 ######################################################################
 # Running the model on GPU with the following information:
-# 
+#
 # Epoch: 1 \| time in 0 minutes, 11 seconds
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0263(train)     |       Acc: 84.5%(train)
 #        Loss: 0.0001(valid)     |       Acc: 89.0%(valid)
-#        
-# 
+#
+#
 # Epoch: 2 \| time in 0 minutes, 10 seconds
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0119(train)     |       Acc: 93.6%(train)
 #        Loss: 0.0000(valid)     |       Acc: 89.6%(valid)
-#        
-# 
+#
+#
 # Epoch: 3 \| time in 0 minutes, 9 seconds
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0069(train)     |       Acc: 96.4%(train)
 #        Loss: 0.0000(valid)     |       Acc: 90.5%(valid)
-#        
-# 
+#
+#
 # Epoch: 4 \| time in 0 minutes, 11 seconds
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0038(train)     |       Acc: 98.2%(train)
 #        Loss: 0.0000(valid)     |       Acc: 90.4%(valid)
-#        
-# 
+#
+#
 # Epoch: 5 \| time in 0 minutes, 11 seconds
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0022(train)     |       Acc: 99.0%(train)
-#        Loss: 0.0000(valid)     |       Acc: 91.0%(valid)        
-# 
+#        Loss: 0.0000(valid)     |       Acc: 91.0%(valid)
+#
 
 
 ######################################################################
 # Evaluate the model with test dataset
 # ------------------------------------
-# 
+#
 
 print('Checking the results of test dataset...')
 test_loss, test_acc = test(test_dataset)
@@ -308,21 +308,21 @@ def test(data_):
 
 ######################################################################
 # Checking the results of test dataset…
-# 
+#
 # ::
-# 
+#
 #        Loss: 0.0237(test)      |       Acc: 90.5%(test)
-# 
+#
 
 
 ######################################################################
 # Test on a random news
 # ---------------------
-# 
+#
 # Use the best model so far and test a golf news. The label information is
 # available
 # `here <https://pytorch.org/text/datasets.html?highlight=ag_news#torchtext.datasets.AG_NEWS>`__.
-# 
+#
 
 import re
 from torchtext.data.utils import ngrams_iterator
@@ -360,10 +360,10 @@ def predict(text, model, vocab, ngrams):
 
 ######################################################################
 # This is a Sports news
-# 
+#
 
 
 ######################################################################
 # You can find the code examples displayed in this note
 # `here <https://github.com/pytorch/text/tree/master/examples/text_classification>`__.
-# 
+#