From 720d370f2348d05c3376ff77bd3cba2eb0428640 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 10 Oct 2022 10:02:30 -0700 Subject: [PATCH 01/21] [DO NOT MERGE] 1.13 RC Test --- .jenkins/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 3aca5ba2a01..ac7c333bcdc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,8 +26,11 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext -# pip uninstall -y torch torchvision torchaudio torchtext -# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext + +# Test enabled for PyTorch 1.13 RC Below +pip uninstall -y torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From e67331d720d55444a09d97d22e74712a88f86a8b Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 11 Oct 2022 10:15:46 -0700 Subject: [PATCH 02/21] Update .jenkins/build.sh Co-authored-by: Nikita Shulga --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index ac7c333bcdc..f314e2f9610 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -30,7 +30,6 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 38939c46282ed9b04a910a19bbee1b6f167b7511 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 13 Oct 2022 11:57:54 -0700 Subject: [PATCH 03/21] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f314e2f9610..264d4bf6e42 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From c0d5fedbc47dc1f0ad708f8a8bf0569a4f76d040 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 14 Oct 2022 11:37:46 -0700 Subject: [PATCH 04/21] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 264d4bf6e42..15b0ff90b25 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From f509d8e1a3bf9a6de6554e17d8cd2cf359c76d8d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:33:09 -0700 Subject: [PATCH 05/21] Update build.sh --- .jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 15b0ff90b25..23cc9d4dbdc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,8 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From d6e72e015df9b89e5ac31fd6de8710d899976b99 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:42:54 -0700 Subject: [PATCH 06/21] Update build.sh --- .jenkins/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 23cc9d4dbdc..f3c82763e96 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,8 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html -pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata +pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 5fbf500bf90518fe8b881c26b683b85692c3d8e3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:51:19 -0700 Subject: [PATCH 07/21] Update build.sh --- .jenkins/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f3c82763e96..2ae72906c4b 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,11 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext +pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 3c7694f89a125621c7705cae2d4c99c25767286d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:00:37 -0700 Subject: [PATCH 08/21] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 2ae72906c4b..7019766f144 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 3559c44e1b35d16ca0e31daca1f27448f9eb70e1 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:36:00 -0700 Subject: [PATCH 09/21] Remove functorch --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74e2da5fad3..67bec81cd2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ torchvision torchtext torchaudio torchdata -functorch>=0.2.1 networkx PyHamcrest bs4 From 06b98742c32ffc60bf3967b266e30014c69d08e3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 19 Oct 2022 13:28:00 -0700 Subject: [PATCH 10/21] Temporarily disabling fx_numeric_suite_tutorial --- .jenkins/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 7019766f144..fee713f588f 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,6 +49,8 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # Temp remove for 1.13 release. + python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py || true # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 3c0fc31990aa12094a70ca536c3131ff4343a70d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 19 Oct 2022 14:19:51 -0700 Subject: [PATCH 11/21] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index fee713f588f..5263045a4bd 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -50,7 +50,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true # Temp remove for 1.13 release. - python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py || true + python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py beginner_source/fx_numeric_suite_tutorial.py || true # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From a449a551d8b074766b689866297a73f9af782da3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:32:37 -0700 Subject: [PATCH 12/21] Disable in the validate list --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 92570124a4e..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -50,6 +50,7 @@ "recipes/Captum_Recipe", "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", + "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed ] From 047a956b922a6485eff76d3ce46f319f235be173 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:52:52 -0700 Subject: [PATCH 13/21] Disable ax tutorial --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index cc01326b44c..5f9d563475f 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,7 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed + "ax_multiobjective_nas_tutorial", ] From 16b85bd174b7ebc458fde799b94ce5120ee6b58a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 11:19:14 -0700 Subject: [PATCH 14/21] Update build.sh --- .jenkins/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 5263045a4bd..9b9f75f89cc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,9 +49,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # Temp remove for 1.13 release. - python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py beginner_source/fx_numeric_suite_tutorial.py || true - + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 0b58aa6c6e70d5b69b868f05ced13468e49e4570 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 11:19:37 -0700 Subject: [PATCH 15/21] Update build.sh --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 9b9f75f89cc..76847c420fe 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,7 +49,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 12ea81473b900fd4cbd117d5834b6a71f4b46372 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 13:50:38 -0700 Subject: [PATCH 16/21] Update build.sh --- .jenkins/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 76847c420fe..9b9f75f89cc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,6 +49,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 9d44ef82758ec36d6f06a532feb946637fd12520 Mon Sep 17 00:00:00 2001 From: George Qi Date: Fri, 23 Sep 2022 17:47:44 -0700 Subject: [PATCH 17/21] [maskedtensor] Overview tutorial [1/4] --- prototype_source/maskedtensor_overview.py | 310 ++++++++++++++++++++++ prototype_source/prototype_index.rst | 10 + 2 files changed, 320 insertions(+) create mode 100644 prototype_source/maskedtensor_overview.py diff --git a/prototype_source/maskedtensor_overview.py b/prototype_source/maskedtensor_overview.py new file mode 100644 index 00000000000..322b3fb7f44 --- /dev/null +++ b/prototype_source/maskedtensor_overview.py @@ -0,0 +1,310 @@ +# -*- coding: utf-8 -*- + +""" +(Prototype) MaskedTensor Overview +================================= +""" + +###################################################################### +# This tutorial is designed to serve as a starting point for using MaskedTensors +# and discuss its masking semantics. +# +# MaskedTensor serves as an extension to :class:`torch.Tensor` that provides the user with the ability to: +# +# * use any masked semantics (for example, variable length tensors, nan* operators, etc.) +# * differentiation between 0 and NaN gradients +# * various sparse applications (see tutorial below) +# +# For a more detailed introduction on what MaskedTensors are, please find the +# `torch.masked documentation `__. +# +# Using MaskedTensor +# ++++++++++++++++++ +# +# Construction +# ------------ +# +# There are a few different ways to construct a MaskedTensor: +# +# * The first way is to directly invoke the MaskedTensor class +# * The second (and our recommended way) is to use :func:`masked.masked_tensor` and :func:`masked.as_masked_tensor` +# factory functions, which are analogous to :func:`torch.tensor` and :func:`torch.as_tensor` +# +# Throughout this tutorial, we will be assuming the import line: `from torch.masked import masked_tensor`. +# +# Accessing the data and mask +# --------------------------- +# +# The underlying fields in a MaskedTensor can be accessed through: +# +# * the :meth:`MaskedTensor.get_data` function +# * the :meth:`MaskedTensor.get_mask` function. Recall that ``True`` indicates "specified" or "valid" +# while ``False`` indicates "unspecified" or "invalid". +# +# In general, the underlying data that is returned may not be valid in the unspecified entries, so we recommend that +# when users require a Tensor without any masked entries, that they use :meth:`MaskedTensor.to_tensor` (as shown above) to +# return a Tensor with filled values. +# +# Indexing and slicing +# -------------------- +# +# :class:`MaskedTensor` is a Tensor subclass, which means that it inherits the same semantics for indexing and slicing +# as :class:`torch.Tensor`. Below are some examples of common indexing and slicing patterns: +# + +import torch +from torch.masked import masked_tensor, as_masked_tensor + +data = torch.arange(24).reshape(2, 3, 4) +mask = data % 2 == 0 + +print("data\n", data) +print("mask\n", mask) + +# float is used for cleaner visualization when being printed +mt = masked_tensor(data.float(), mask) + +print ("mt[0]:\n", mt[0]) +print ("mt[:, :, 2:4]", mt[:, :, 2:4]) + +###################################################################### +# Why is MaskedTensor useful? +# +++++++++++++++++++++++++++ +# +# Because of :class:`MaskedTensor`'s treatment of specified and unspecified values as a first-class citizen +# instead of an afterthought (with filled values, nans, etc.), it is able to solve for several of the shortcomings +# that regular Tensors are unable to; indeed, :class:`MaskedTensor` was born in a large part due to these recurring issues. +# +# Below, we will discuss some of the most common issues that are still unresolved in PyTorch today +# and illustrate how :class:`MaskedTensor` can solve these problems. +# +# Distinguishing between 0 and NaN gradient +# ----------------------------------------- +# +# One issue that :class:`torch.Tensor` runs into is the inability to distinguish between gradients that are +# undefined (NaN) vs. gradients that are actually 0. Because PyTorch does not have a way of marking a value +# as specified/valid vs. unspecified/invalid, it is forced to rely on NaN or 0 (depending on the use case), leading +# to unreliable semantics since many operations aren't meant to handle NaN values properly. What is even more confusing +# is that sometimes depending on the order of operations, the gradient could vary (for example, depending on how early +# in the chain of operations a NaN value manifests). +# +# :class:`MaskedTensor` is the perfect solution for this! +# +# :func:`torch.where` +# ^^^^^^^^^^^^^^^^^^^ +# +# In `Issue 10729 `__, we notice a case where the order of operations +# can matter when using :func:`torch.where` because we have trouble differentiating between if the 0 is a real 0 +# or one from undefined gradients. Therefore, we remain consistent and mask out the results: +# +# Current result: +# + +x = torch.tensor([-10., -5, 0, 5, 10, 50, 60, 70, 80, 90, 100], requires_grad=True, dtype=torch.float) +y = torch.where(x < 0, torch.exp(x), torch.ones_like(x)) +y.sum().backward() +x.grad + +###################################################################### +# :class:`MaskedTensor` result: +# + +x = torch.tensor([-10., -5, 0, 5, 10, 50, 60, 70, 80, 90, 100]) +mask = x < 0 +mx = masked_tensor(x, mask, requires_grad=True) +my = masked_tensor(torch.ones_like(x), ~mask, requires_grad=True) +y = torch.where(mask, torch.exp(mx), my) +y.sum().backward() +mx.grad + +###################################################################### +# The gradient here is only provided to the selected subset. Effectively, this changes the gradient of `where` +# to mask out elements instead of setting them to zero. +# +# Another :func:`torch.where` +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# `Issue 52248 `__ is another example. +# +# Current result: +# + +a = torch.randn((), requires_grad=True) +b = torch.tensor(False) +c = torch.ones(()) +print("torch.where(b, a/0, c):\n", torch.where(b, a/0, c)) +print("torch.autograd.grad(torch.where(b, a/0, c), a):\n", torch.autograd.grad(torch.where(b, a/0, c), a)) + +###################################################################### +# :class:`MaskedTensor` result: +# + +a = masked_tensor(torch.randn(()), torch.tensor(True), requires_grad=True) +b = torch.tensor(False) +c = torch.ones(()) +print("torch.where(b, a/0, c):\n", torch.where(b, a/0, c)) +print("torch.autograd.grad(torch.where(b, a/0, c), a):\n", torch.autograd.grad(torch.where(b, a/0, c), a)) + +###################################################################### +# This issue is similar (and even links to the next issue below) in that it expresses frustration with +# unexpected behavior because of the inability to differentiate "no gradient" vs "zero gradient", +# which in turn makes working with other ops difficult to reason about. +# +# When using mask, x/0 yields NaN grad +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# In `Issue 4132 `__, the user proposes that +# `x.grad` should be `[0, 1]` instead of the `[nan, 1]`, +# whereas :class:`MaskedTensor` makes this very clear by masking out the gradient altogether. +# +# Current result: +# + +x = torch.tensor([1., 1.], requires_grad=True) +div = torch.tensor([0., 1.]) +y = x/div # => y is [inf, 1] +mask = (div != 0) # => mask is [0, 1] +y[mask].backward() +x.grad + +###################################################################### +# :class:`MaskedTensor` result: +# + +x = torch.tensor([1., 1.], requires_grad=True) +div = torch.tensor([0., 1.]) +y = x/div # => y is [inf, 1] + >>> +mask = (div != 0) # => mask is [0, 1] +loss = as_masked_tensor(y, mask) +loss.sum().backward() +x.grad + +###################################################################### +# :func:`torch.nansum` and :func:`torch.nanmean` +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# In `Issue 67180 `__, +# the gradient isn't calculate properly (a longstanding issue), whereas :class:`MaskedTensor` handles it correctly. +# +# Current result: +# + +a = torch.tensor([1., 2., float('nan')]) +b = torch.tensor(1.0, requires_grad=True) +c = a * b +c1 = torch.nansum(c) +bgrad1, = torch.autograd.grad(c1, b, retain_graph=True) +bgrad1 + +###################################################################### +# :class:`MaskedTensor` result: +# + +a = torch.tensor([1., 2., float('nan')]) +b = torch.tensor(1.0, requires_grad=True) +mt = masked_tensor(a, ~torch.isnan(a)) +c = mt * b +c1 = torch.sum(c) +bgrad1, = torch.autograd.grad(c1, b, retain_graph=True) +bgrad1 + +###################################################################### +# Safe Softmax +# ------------ +# +# Safe softmax is another great example of `an issue `_ +# that arises frequently. In a nutshell, if there is an entire batch that is "masked out" +# or consists entirely of padding (which, in the softmax case, translates to being set `-inf`), +# then this will result in NaNs, which can lead to training divergence. +# +# Luckily, :class:`MaskedTensor` has solved this issue. Consider this setup: +# + +data = torch.randn(3, 3) +mask = torch.tensor([[True, False, False], [True, False, True], [False, False, False]]) +x = data.masked_fill(~mask, float('-inf')) +mt = masked_tensor(data, mask) +print("x:\n", x) +print("mt:\n", mt) + +###################################################################### +# For example, we want to calculate the softmax along `dim=0`. Note that the second column is "unsafe" (i.e. entirely +# masked out), so when the softmax is calculated, the result will yield `0/0 = nan` since `exp(-inf) = 0`. +# However, what we would really like is for the gradients to be masked out since they are unspecified and would be +# invalid for training. +# +# PyTorch result: +# + +x.softmax(0) + +###################################################################### +# :class:`MaskedTensor` result: +# + +mt.softmax(0) + +###################################################################### +# Implementing missing torch.nan* operators +# -------------------------------------------------------------------------------------------------------------- +# +# In `Issue 61474 <`__, +# there is a request to add additional operators to cover the various `torch.nan*` applications, +# such as ``torch.nanmax``, ``torch.nanmin``, etc. +# +# In general, these problems lend themselves more naturally to masked semantics, so instead of introducing additional +# operators, we propose using :class:`MaskedTensor`s instead. Since +# `nanmean has already landed `_, we can use it as a comparison point: +# + +x = torch.arange(16).float() +y = x * x.fmod(4) +z = y.masked_fill(y == 0, float('nan')) # we want to get the mean of y when ignoring the zeros + +print("y:\n, y") +# z is just y with the zeros replaced with nan's +print("z:\n", z) +print("y.mean():\n", y.mean()) +print("z.nanmean():\n", z.nanmean()) +# MaskedTensor successfully ignores the 0's +print("torch.mean(masked_tensor(y, y != 0)):\n", torch.mean(masked_tensor(y, y != 0))) + +###################################################################### +# In the above example, we've constructed a `y` and would like to calculate the mean of the series while ignoring +# the zeros. `torch.nanmean` can be used to do this, but we don't have implementations for the rest of the +# `torch.nan*` operations. :class:`MaskedTensor` solves this issue by being able to use the base operation, +# and we already have support for the other operations listed in the issue. For example: +# + +torch.argmin(masked_tensor(y, y != 0)) + +###################################################################### +# Indeed, the index of the minimum argument when ignoring the 0's is the 1 in index 1. +# +# :class:`MaskedTensor` can also support reductions when the data is fully masked out, which is equivalent +# to the case above when the data Tensor is completely ``nan``. ``nanmean`` would return ``nan`` +# (an ambiguous return value), while MaskedTensor would more accurately indicate a masked out result. +# + +x = torch.empty(16).fill_(float('nan')) +print("x:\n", x) +print("torch.nanmean(x):\n", torch.nanmean(x)) +print("torch.nanmean via maskedtensor:\n", torch.mean(masked_tensor(x, ~torch.isnan(x)))) + +###################################################################### +# This is a similar problem to safe softmax where `0/0 = nan` when what we really want is an undefined value. +# +# Conclusion +# ++++++++++ +# +# In this tutorial, we've introduced what MaskedTensors are, demonstrated how to use them, and motivated their +# value through a series of examples and issues that they've helped resolve. +# +# Further Reading +# +++++++++++++++ +# +# To continue learning more, you can find our +# `MaskedTensor Sparsity tutorial `__ +# to see how MaskedTensor enables sparsity and the different storage formats we currently support. +# diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index b362a38b9e8..4878cfa8d2c 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -141,6 +141,15 @@ Prototype features are not available as part of binary distributions like PyPI o :link: ../prototype/nestedtensor.html :tags: NestedTensor +.. MaskedTensor + +.. customcarditem:: + :header: MaskedTensor Overview + :card_description: Learn about masked tensors, the source of truth for specified and unspecified values + :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png + :link: ../prototype/maskedtensor_overview.html + :tags: MaskedTensor + .. End of tutorial card section .. raw:: html @@ -172,3 +181,4 @@ Prototype features are not available as part of binary distributions like PyPI o prototype/vmap_recipe.html prototype/vulkan_workflow.html prototype/nestedtensor.html + prototype/maskedtensor_overview.html From 16abb37b50af238c5c7e3604af7fb340a72c455f Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 13:06:15 -0700 Subject: [PATCH 18/21] undo requirements.txt changes --- prototype_source/maskedtensor_overview.py | 77 +++++++++++++++-------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/prototype_source/maskedtensor_overview.py b/prototype_source/maskedtensor_overview.py index 322b3fb7f44..28828693674 100644 --- a/prototype_source/maskedtensor_overview.py +++ b/prototype_source/maskedtensor_overview.py @@ -2,7 +2,7 @@ """ (Prototype) MaskedTensor Overview -================================= +********************************* """ ###################################################################### @@ -14,16 +14,33 @@ # * use any masked semantics (for example, variable length tensors, nan* operators, etc.) # * differentiation between 0 and NaN gradients # * various sparse applications (see tutorial below) -# +# # For a more detailed introduction on what MaskedTensors are, please find the # `torch.masked documentation `__. -# +# # Using MaskedTensor -# ++++++++++++++++++ +# ================== +# +# In this section we discuss how to use MaskedTensor including how to construct, access, the data +# and mask, as well as indexing and slicing. +# +# Preparation +# ----------- # +# We'll begin by doing the necessary setup for the tutorial: +# + +import torch +from torch.masked import masked_tensor, as_masked_tensor +import warnings + +# Disable prototype warnings and such +warnings.filterwarnings(action='ignore', category=UserWarning) + +###################################################################### # Construction # ------------ -# +# # There are a few different ways to construct a MaskedTensor: # # * The first way is to directly invoke the MaskedTensor class @@ -52,24 +69,24 @@ # as :class:`torch.Tensor`. Below are some examples of common indexing and slicing patterns: # -import torch -from torch.masked import masked_tensor, as_masked_tensor - data = torch.arange(24).reshape(2, 3, 4) mask = data % 2 == 0 -print("data\n", data) -print("mask\n", mask) +print("data:\n", data) +print("mask:\n", mask) + +###################################################################### +# # float is used for cleaner visualization when being printed mt = masked_tensor(data.float(), mask) -print ("mt[0]:\n", mt[0]) -print ("mt[:, :, 2:4]", mt[:, :, 2:4]) +print("mt[0]:\n", mt[0]) +print("mt[:, :, 2:4]:\n", mt[:, :, 2:4]) ###################################################################### # Why is MaskedTensor useful? -# +++++++++++++++++++++++++++ +# =========================== # # Because of :class:`MaskedTensor`'s treatment of specified and unspecified values as a first-class citizen # instead of an afterthought (with filled values, nans, etc.), it is able to solve for several of the shortcomings @@ -90,8 +107,8 @@ # # :class:`MaskedTensor` is the perfect solution for this! # -# :func:`torch.where` -# ^^^^^^^^^^^^^^^^^^^ +# torch.where +# ^^^^^^^^^^^ # # In `Issue 10729 `__, we notice a case where the order of operations # can matter when using :func:`torch.where` because we have trouble differentiating between if the 0 is a real 0 @@ -121,8 +138,8 @@ # The gradient here is only provided to the selected subset. Effectively, this changes the gradient of `where` # to mask out elements instead of setting them to zero. # -# Another :func:`torch.where` -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Another torch.where +# ^^^^^^^^^^^^^^^^^^^ # # `Issue 52248 `__ is another example. # @@ -174,7 +191,6 @@ x = torch.tensor([1., 1.], requires_grad=True) div = torch.tensor([0., 1.]) y = x/div # => y is [inf, 1] - >>> mask = (div != 0) # => mask is [0, 1] loss = as_masked_tensor(y, mask) loss.sum().backward() @@ -182,7 +198,7 @@ ###################################################################### # :func:`torch.nansum` and :func:`torch.nanmean` -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# ---------------------------------------------- # # In `Issue 67180 `__, # the gradient isn't calculate properly (a longstanding issue), whereas :class:`MaskedTensor` handles it correctly. @@ -213,7 +229,7 @@ # Safe Softmax # ------------ # -# Safe softmax is another great example of `an issue `_ +# Safe softmax is another great example of `an issue `__ # that arises frequently. In a nutshell, if there is an entire batch that is "masked out" # or consists entirely of padding (which, in the softmax case, translates to being set `-inf`), # then this will result in NaNs, which can lead to training divergence. @@ -247,24 +263,31 @@ ###################################################################### # Implementing missing torch.nan* operators -# -------------------------------------------------------------------------------------------------------------- +# ----------------------------------------- # -# In `Issue 61474 <`__, +# In `Issue 61474 `__, # there is a request to add additional operators to cover the various `torch.nan*` applications, # such as ``torch.nanmax``, ``torch.nanmin``, etc. # # In general, these problems lend themselves more naturally to masked semantics, so instead of introducing additional -# operators, we propose using :class:`MaskedTensor`s instead. Since -# `nanmean has already landed `_, we can use it as a comparison point: +# operators, we propose using :class:`MaskedTensor` instead. +# Since `nanmean has already landed `__, +# we can use it as a comparison point: # x = torch.arange(16).float() y = x * x.fmod(4) z = y.masked_fill(y == 0, float('nan')) # we want to get the mean of y when ignoring the zeros -print("y:\n, y") +###################################################################### +# +print("y:\n", y) # z is just y with the zeros replaced with nan's print("z:\n", z) + +###################################################################### +# + print("y.mean():\n", y.mean()) print("z.nanmean():\n", z.nanmean()) # MaskedTensor successfully ignores the 0's @@ -296,13 +319,13 @@ # This is a similar problem to safe softmax where `0/0 = nan` when what we really want is an undefined value. # # Conclusion -# ++++++++++ +# ========== # # In this tutorial, we've introduced what MaskedTensors are, demonstrated how to use them, and motivated their # value through a series of examples and issues that they've helped resolve. # # Further Reading -# +++++++++++++++ +# =============== # # To continue learning more, you can find our # `MaskedTensor Sparsity tutorial `__ From 4fe66e068c1c66492519ac1d632db95d16a9dd55 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 10:39:11 -0700 Subject: [PATCH 19/21] Update build.sh --- .jenkins/build.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 9b9f75f89cc..d94b551d9d7 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,9 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext - -# Test enabled for PyTorch 1.13 RC Below -pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip uninstall -y torch torchvision torchaudio torchtext +# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 37dc59d3ead5933c03fba052eb55b418111177ef Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 10:39:33 -0700 Subject: [PATCH 20/21] Update build.sh --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index d94b551d9d7..981615672ce 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -47,7 +47,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From c8d6cbf8a615989e9f345fafc3694e5a08122304 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 10:40:12 -0700 Subject: [PATCH 21/21] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 5f9d563475f..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,7 +51,6 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed - "ax_multiobjective_nas_tutorial", ]