From 0aa32e88184813617460c36de363f386f2eefca6 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 10 Oct 2022 10:02:30 -0700 Subject: [PATCH 01/13] [DO NOT MERGE] 1.13 RC Test --- .jenkins/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 981615672ce..42d44b7e725 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,8 +26,11 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext -# pip uninstall -y torch torchvision torchaudio torchtext -# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext + +# Test enabled for PyTorch 1.13 RC Below +pip uninstall -y torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From ca8860b9cfc9e04ac8625de0c1aad7e76d1e7169 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 11 Oct 2022 10:15:46 -0700 Subject: [PATCH 02/13] Update .jenkins/build.sh Co-authored-by: Nikita Shulga --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 42d44b7e725..2f2141f1dc9 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -30,7 +30,6 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 931b12cd97086d8bb36f99b05273dc5ef635eba9 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 13 Oct 2022 11:57:54 -0700 Subject: [PATCH 03/13] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 2f2141f1dc9..18ff9a325b7 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From d5c243ba39e215181f5bc89af1e8c77ee72c1287 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 14 Oct 2022 11:37:46 -0700 Subject: [PATCH 04/13] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 18ff9a325b7..8ec0427109b 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From fa15873ca60ef14d26114f7bb75f93f8958e2195 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:33:09 -0700 Subject: [PATCH 05/13] Update build.sh --- .jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 8ec0427109b..e9389dd4832 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,8 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 71b2bf927ff9aaf7d0852d9255617ec74a07b0c7 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:42:54 -0700 Subject: [PATCH 06/13] Update build.sh --- .jenkins/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index e9389dd4832..b6d7e30b83a 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,8 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html -pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata +pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 5c346f2ded7d19f7bad8a0088b6400d97a38a734 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:51:19 -0700 Subject: [PATCH 07/13] Update build.sh --- .jenkins/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index b6d7e30b83a..64bd1c88918 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,11 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext +pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From c023280a4733d01bae21459272edc2bbc6a9412b Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:00:37 -0700 Subject: [PATCH 08/13] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 64bd1c88918..76847c420fe 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 785a5f09e9776ce6ddf56870b1104b7f0ad53b5a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:52:52 -0700 Subject: [PATCH 09/13] Disable ax tutorial --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index cc01326b44c..5f9d563475f 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,7 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed + "ax_multiobjective_nas_tutorial", ] From 8aa64c31be8c278913eec3b3b765707741e99d27 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 13:50:38 -0700 Subject: [PATCH 10/13] Update build.sh --- .jenkins/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 76847c420fe..9b9f75f89cc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,6 +49,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 72db57288bf85fa0a4fd2ac48c6b6fe105ed1273 Mon Sep 17 00:00:00 2001 From: George Qi Date: Fri, 23 Sep 2022 23:10:29 -0700 Subject: [PATCH 11/13] [maskedtensor] Advanced semantics [4/4] --- .../maskedtensor_advanced_semantics.py | 170 ++++++++++++++++++ prototype_source/prototype_index.rst | 7 + 2 files changed, 177 insertions(+) create mode 100644 prototype_source/maskedtensor_advanced_semantics.py diff --git a/prototype_source/maskedtensor_advanced_semantics.py b/prototype_source/maskedtensor_advanced_semantics.py new file mode 100644 index 00000000000..7a023304218 --- /dev/null +++ b/prototype_source/maskedtensor_advanced_semantics.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- + +""" +(Prototype) MaskedTensor Advanced Semantics +=========================================== +""" + +###################################################################### +# +# Before working on this tutorial, please make sure to review our +# `MaskedTensor Overview tutorial `. +# +# The purpose of this tutorial is to help users understand how some of the advanced semantics work +# and how they came to be. We will focus on two particular ones: +# +# *. Differences between MaskedTensor and `NumPy's MaskedArray `__ +# *. Reduction semantics +# +# Preparation +# ----------- +# + +import torch +from torch.masked import masked_tensor +import numpy as np +import warnings + +# Disable prototype warnings and such +warnings.filterwarnings(action='ignore', category=UserWarning) + +###################################################################### +# MaskedTensor vs NumPy's MaskedArray +# ----------------------------------- +# +# NumPy's ``MaskedArray`` has a few fundamental semantics differences from MaskedTensor. +# +# *. Their factory function and basic definition inverts the mask (similar to ``torch.nn.MHA``); that is, MaskedTensor +# uses ``True`` to denote "specified" and ``False`` to denote "unspecified", or "valid"/"invalid", +# whereas NumPy does the opposite. We believe that our mask definition is not only more intuitive, +# but it also aligns more with the existing semantics in PyTorch as a whole. +# *. Intersection semantics. In NumPy, if one of two elements are masked out, the resulting element will be +# masked out as well -- in practice, they +# `apply the logical_or operator `__. +# + +data = torch.arange(5.) +mask = torch.tensor([True, True, False, True, False]) +npm0 = np.ma.masked_array(data.numpy(), (~mask).numpy()) +npm1 = np.ma.masked_array(data.numpy(), (mask).numpy()) + +print("npm0:\n", npm0) +print("npm1:\n", npm1) +print("npm0 + npm1:\n", npm0 + npm1) + +###################################################################### +# Meanwhile, MaskedTensor does not support addition or binary operators with masks that don't match -- +# to understand why, please find the :ref:`section on reductions `. +# + +mt0 = masked_tensor(data, mask) +mt1 = masked_tensor(data, ~mask) +print("mt0:\n", mt0) +print("mt1:\n", mt1) + +try: + mt0 + mt1 +except ValueError as e: + print ("mt0 + mt1 failed. Error: ", e) + +###################################################################### +# However, if this behavior is desired, MaskedTensor does support these semantics by giving access to the data and masks +# and conveniently converting a MaskedTensor to a Tensor with masked values filled in using :func:`to_tensor`. +# For example: +# + +t0 = mt0.to_tensor(0) +t1 = mt1.to_tensor(0) +mt2 = masked_tensor(t0 + t1, mt0.get_mask() & mt1.get_mask()) + +print("t0:\n", t0) +print("t1:\n", t1) +print("mt2 (t0 + t1):\n", mt2) + +###################################################################### +# Note that the mask is `mt0.get_mask() & mt1.get_mask()` since :class:`MaskedTensor`'s mask is the inverse of NumPy's. +# +# .. _reduction-semantics: +# +# Reduction Semantics +# ------------------- +# +# Recall in `MaskedTensor's Overview tutorial `__ +# we discussed "Implementing missing torch.nan* ops". Those are examples of reductions -- operators that remove one +# (or more) dimensions from a Tensor and then aggregate the result. In this section, we will use reduction semantics +# to motivate our strict requirements around matching masks from above. +# +# Fundamentally, :class:`MaskedTensor`s perform the same reduction operation while ignoring the masked out +# (unspecified) values. By way of example: +# + +data = torch.arange(12, dtype=torch.float).reshape(3, 4) +mask = torch.randint(2, (3, 4), dtype=torch.bool) +mt = masked_tensor(data, mask) + +print("data:\n", data) +print("mask:\n", mask) +print("mt:\n", mt) + +###################################################################### +# Now, the different reductions (all on dim=1): +# + +print("torch.sum:\n", torch.sum(mt, 1)) +print("torch.mean:\n", torch.mean(mt, 1)) +print("torch.prod:\n", torch.prod(mt, 1)) +print("torch.amin:\n", torch.amin(mt, 1)) +print("torch.amax:\n", torch.amax(mt, 1)) + +###################################################################### +# Of note, the value under a masked out element is not guaranteed to have any specific value, especially if the +# row or column is entirely masked out (the same is true for normalizations). +# For more details on masked semantics, you can find this `RFC `__. +# +# Now, we can revisit the question: why do we enforce the invariant that masks must match for binary operators? +# In other words, why don't we use the same semantics as ``np.ma.masked_array``? Consider the following example: +# + +data0 = torch.arange(10.).reshape(2, 5) +data1 = torch.arange(10.).reshape(2, 5) + 10 +mask0 = torch.tensor([[True, True, False, False, False], [False, False, False, True, True]]) +mask1 = torch.tensor([[False, False, False, True, True], [True, True, False, False, False]]) +npm0 = np.ma.masked_array(data0.numpy(), (mask0).numpy()) +npm1 = np.ma.masked_array(data1.numpy(), (mask1).numpy()) + +print("npm0:", npm0) +print("npm1:", npm1) + +###################################################################### +# Now, let's try addition: +# + +print("(npm0 + npm1).sum(0):\n", (npm0 + npm1).sum(0)) +print("npm0.sum(0) + npm1.sum(0):\n", npm0.sum(0) + npm1.sum(0)) + +###################################################################### +# Sum and addition should clearly be associative, but with NumPy's semantics, they are not, +# which can certainly be confusing for the user. +# +# :class:`MaskedTensor`, on the other hand, will simply not allow this operation since `mask0 != mask1`. +# That being said, if the user wishes, there are ways around this +# (for example, filling in the MaskedTensor's undefined elements with 0 values using :func:`to_tensor` +# like shown below), but the user must now be more explicit with their intentions. +# + +mt0 = masked_tensor(data0, ~mask0) +mt1 = masked_tensor(data1, ~mask1) + +(mt0.to_tensor(0) + mt1.to_tensor(0)).sum(0) + +###################################################################### +# Conclusion +# ---------- +# +# In this tutorial, we have learned about the different design decisions behind MaskedTensor and +# NumPy's MaskedArray, as well as reduction semantics. +# In general, MaskedTensor is designed to avoid ambiguity and confusing semantics (for example, we try to preserve +# the associative property amongst binary operations), which in turn can necessitate the user +# to be more intentional with their code at times, but we believe this to be the better move. +# If you have any thoughts on this, please `let us know `__! +# diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index 4878cfa8d2c..c3b7aa556af 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -148,6 +148,12 @@ Prototype features are not available as part of binary distributions like PyPI o :card_description: Learn about masked tensors, the source of truth for specified and unspecified values :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png :link: ../prototype/maskedtensor_overview.html + +.. customcarditem:: + :header: Masked Tensor Advanced Semantics + :card_description: Learn more about Masked Tensor's advanced semantics (reductions and comparing vs. NumPy's MaskedArray) + :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png + :link: ../prototype/maskedtensor_advanced_semantics.html :tags: MaskedTensor .. End of tutorial card section @@ -182,3 +188,4 @@ Prototype features are not available as part of binary distributions like PyPI o prototype/vulkan_workflow.html prototype/nestedtensor.html prototype/maskedtensor_overview.html + prototype/maskedtensor_advanced_semantics.html From 906587fab4166ff4fa0d2c6fa7fbfb7d546760f0 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 14:07:04 -0700 Subject: [PATCH 12/13] Update --- .jenkins/build.sh | 6 ++---- .jenkins/validate_tutorials_built.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 9b9f75f89cc..f644c6d920b 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,10 +26,8 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext - -# Test enabled for PyTorch 1.13 RC Below -pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext +# pip uninstall -y torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 5f9d563475f..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,7 +51,6 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed - "ax_multiobjective_nas_tutorial", ] From 12df4378ca54f8d60c44525d42cda66493972f35 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 15:50:56 -0700 Subject: [PATCH 13/13] Update --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index cc01326b44c..c8294b82c02 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,7 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed + "text_to_speech_with_torchaudio", ]