From 720d370f2348d05c3376ff77bd3cba2eb0428640 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 10 Oct 2022 10:02:30 -0700 Subject: [PATCH 01/20] [DO NOT MERGE] 1.13 RC Test --- .jenkins/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 3aca5ba2a01..ac7c333bcdc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,8 +26,11 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext -# pip uninstall -y torch torchvision torchaudio torchtext -# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext + +# Test enabled for PyTorch 1.13 RC Below +pip uninstall -y torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From e67331d720d55444a09d97d22e74712a88f86a8b Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 11 Oct 2022 10:15:46 -0700 Subject: [PATCH 02/20] Update .jenkins/build.sh Co-authored-by: Nikita Shulga --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index ac7c333bcdc..f314e2f9610 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -30,7 +30,6 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 38939c46282ed9b04a910a19bbee1b6f167b7511 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 13 Oct 2022 11:57:54 -0700 Subject: [PATCH 03/20] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f314e2f9610..264d4bf6e42 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From c0d5fedbc47dc1f0ad708f8a8bf0569a4f76d040 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 14 Oct 2022 11:37:46 -0700 Subject: [PATCH 04/20] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 264d4bf6e42..15b0ff90b25 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From f509d8e1a3bf9a6de6554e17d8cd2cf359c76d8d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:33:09 -0700 Subject: [PATCH 05/20] Update build.sh --- .jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 15b0ff90b25..23cc9d4dbdc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,8 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From d6e72e015df9b89e5ac31fd6de8710d899976b99 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:42:54 -0700 Subject: [PATCH 06/20] Update build.sh --- .jenkins/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 23cc9d4dbdc..f3c82763e96 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,8 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html -pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata +pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 5fbf500bf90518fe8b881c26b683b85692c3d8e3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:51:19 -0700 Subject: [PATCH 07/20] Update build.sh --- .jenkins/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f3c82763e96..2ae72906c4b 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,11 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext +pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 3c7694f89a125621c7705cae2d4c99c25767286d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:00:37 -0700 Subject: [PATCH 08/20] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 2ae72906c4b..7019766f144 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 3559c44e1b35d16ca0e31daca1f27448f9eb70e1 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:36:00 -0700 Subject: [PATCH 09/20] Remove functorch --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74e2da5fad3..67bec81cd2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ torchvision torchtext torchaudio torchdata -functorch>=0.2.1 networkx PyHamcrest bs4 From 06b98742c32ffc60bf3967b266e30014c69d08e3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 19 Oct 2022 13:28:00 -0700 Subject: [PATCH 10/20] Temporarily disabling fx_numeric_suite_tutorial --- .jenkins/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 7019766f144..fee713f588f 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,6 +49,8 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # Temp remove for 1.13 release. + python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py || true # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 3c0fc31990aa12094a70ca536c3131ff4343a70d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 19 Oct 2022 14:19:51 -0700 Subject: [PATCH 11/20] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index fee713f588f..5263045a4bd 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -50,7 +50,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true # Temp remove for 1.13 release. - python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py || true + python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py beginner_source/fx_numeric_suite_tutorial.py || true # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From a449a551d8b074766b689866297a73f9af782da3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:32:37 -0700 Subject: [PATCH 12/20] Disable in the validate list --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 92570124a4e..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -50,6 +50,7 @@ "recipes/Captum_Recipe", "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", + "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed ] From 047a956b922a6485eff76d3ce46f319f235be173 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:52:52 -0700 Subject: [PATCH 13/20] Disable ax tutorial --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index cc01326b44c..5f9d563475f 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,7 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed + "ax_multiobjective_nas_tutorial", ] From 16b85bd174b7ebc458fde799b94ce5120ee6b58a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 11:19:14 -0700 Subject: [PATCH 14/20] Update build.sh --- .jenkins/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 5263045a4bd..9b9f75f89cc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,9 +49,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # Temp remove for 1.13 release. - python $DIR/remove_runnable_code.py beginner_source/fx_numeric_suite_tutorial.py beginner_source/fx_numeric_suite_tutorial.py || true - + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 0b58aa6c6e70d5b69b868f05ced13468e49e4570 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 11:19:37 -0700 Subject: [PATCH 15/20] Update build.sh --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 9b9f75f89cc..76847c420fe 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,7 +49,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 12ea81473b900fd4cbd117d5834b6a71f4b46372 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 13:50:38 -0700 Subject: [PATCH 16/20] Update build.sh --- .jenkins/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 76847c420fe..9b9f75f89cc 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -49,6 +49,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 304f08d8b34a478e4f610b03845713f64bfd21d8 Mon Sep 17 00:00:00 2001 From: George Qi Date: Fri, 23 Sep 2022 22:14:53 -0700 Subject: [PATCH 17/20] tutorial fixes! --- prototype_source/maskedtensor_adagrad.py | 218 +++++++++++++++++++++++ prototype_source/prototype_index.rst | 11 ++ 2 files changed, 229 insertions(+) create mode 100644 prototype_source/maskedtensor_adagrad.py diff --git a/prototype_source/maskedtensor_adagrad.py b/prototype_source/maskedtensor_adagrad.py new file mode 100644 index 00000000000..445da1e0e28 --- /dev/null +++ b/prototype_source/maskedtensor_adagrad.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- + +""" +(Prototype) Efficiently writing "sparse" semantics for Adagrad with MaskedTensor +================================================================================ +""" + +###################################################################### +# Before working through this tutorial, please review the MaskedTensor +# `Overview `__ and +# `Sparsity `__ tutorials. +# +# Introduction and Motivation +# --------------------------- +# `Issue 1369 `__ discussed the additional lines of code +# that were introduced while writing "sparse" semantics for Adagrad, but really, +# the code uses sparsity as a proxy for masked semantics rather than the intended use case of sparsity: +# a compression and optimization technique. +# Previously, we worked around the lack of formal masked semantics by introducing one-off semantics and operators +# while forcing users to be aware of storage details such as indices and values. +# +# Now that we have masked semantics, we are better equipped to point out when sparsity is used as a semantic extension. +# We'll also compare and contrast this with equivalent code written using MaskedTensor. +# In the end the code snippets are repeated without additional comments to show the difference in brevity. +# +# Preparation +# ----------- +# + +import torch +import warnings + +# Disable prototype warnings and such +warnings.filterwarnings(action='ignore', category=UserWarning) + +# Some hyperparameters +eps = 1e-10 +clr = 0.1 + +i = torch.tensor([[0, 1, 1], [2, 0, 2]]) +v = torch.tensor([3, 4, 5], dtype=torch.float32) +grad = torch.sparse_coo_tensor(i, v, [2, 4]) + +###################################################################### +# Simpler Code with MaskedTensor +# ------------------------------ +# +# Before we get too far in the weeds, let's introduce the problem a bit more concretely. We will be taking a look +# into the `Adagrad (functional) `__ +# implementation in PyTorch with the ultimate goal of simplifying and more faithfully representing the masked approach. +# +# For reference, this is the regular, dense code path without masked gradients or sparsity: +# +# .. code-block:: python +# +# state_sum.addcmul_(grad, grad, value=1) +# std = state_sum.sqrt().add_(eps) +# param.addcdiv_(grad, std, value=-clr) +# +# The vanilla tensor implementation for sparse is: +# +# .. code-block:: python +# +# def _make_sparse(grad, grad_indices, values): +# size = grad.size() +# if grad_indices.numel() == 0 or values.numel() == 0: +# return torch.empty_like(grad) +# return torch.sparse_coo_tensor(grad_indices, values, size) +# +# grad = grad.coalesce() # the update is non-linear so indices must be unique +# grad_indices = grad._indices() +# grad_values = grad._values() +# +# state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2))) # a different _make_sparse per layout +# std = state_sum.sparse_mask(grad) +# std_values = std._values().sqrt_().add_(eps) +# param.add_(_make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr) +# +# while :class:`MaskedTensor` minimizes the code to the snippet: +# +# .. code-block:: python +# +# state_sum2 = state_sum2 + masked_grad.pow(2).get_data() +# std2 = masked_tensor(state_sum2.to_sparse(), mask) +# std2 = std2.sqrt().add(eps) +# param2 = param2.add((masked_grad / std2).get_data(), alpha=-clr) +# +# In this tutorial, we will go through each implementation line by line, but at first glance, we can notice +# (1) how much shorter the MaskedTensor implementation is, and +# (2) how it avoids conversions between dense and sparse tensors. +# + +###################################################################### +# Original Sparse Implementation +# ------------------------------ +# +# Now, let's break down the code with some inline comments: +# + +def _make_sparse(grad, grad_indices, values): + size = grad.size() + if grad_indices.numel() == 0 or values.numel() == 0: + return torch.empty_like(grad) + return torch.sparse_coo_tensor(grad_indices, values, size) + +# We don't support sparse gradients +param = torch.arange(8).reshape(2, 4).float() +state_sum = torch.full_like(param, 0.5) # initial value for state sum + +grad = grad.coalesce() # the update is non-linear so indices must be unique +grad_indices = grad._indices() +grad_values = grad._values() +# pow(2) has the same semantics for both sparse and dense memory layouts since 0^2 is zero +state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2))) + +# We take care to make std sparse, even though state_sum clearly is not. +# This means that we're only applying the gradient to parts of the state_sum +# for which it is specified. This further drives the point home that the passed gradient is not sparse, but masked. +# We currently dodge all these concerns using the private method `_values`. +std = state_sum.sparse_mask(grad) +std_values = std._values().sqrt_().add_(eps) + +# Note here that we currently don't support div for sparse Tensors because zero / zero is not well defined, +# so we're forced to perform `grad_values / std_values` outside the sparse semantic and then convert back to a +# sparse tensor with `make_sparse`. +# We'll later see that MaskedTensor will actually handle these operations for us as well as properly denote +# undefined / undefined = undefined! +param.add_(_make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr) + +###################################################################### +# The third to last line -- `std = state_sum.sparse_mask(grad)` -- is where we have a very important divergence. +# +# The addition of eps should technically be applied to all values but instead is only applied to specified values. +# Here we're using sparsity as a semantic extension and to enforce a certain pattern of defined and undefined values. +# If parts of the values of the gradient are zero, they are still included if materialized even though they +# could be compressed by other sparse storage layouts. This is theoretically quite brittle! +# That said, one could argue that eps is always very small, so it might not matter so much in practice. +# +# Moreover, an implementation `add_` for sparsity as a storage layout and compression scheme +# should cause densification, but we force it not to for performance. +# For this one-off case it is fine.. until we want to introduce new compression scheme, such as +# `CSC `__, +# `BSR `__, +# or `BSC `__. +# We will then need to introduce separate Tensor types for each and write variations for gradients compressed +# using different storage formats, which is inconvenient and not quite scalable nor clean. +# +# MaskedTensor Sparse Implementation +# ---------------------------------- +# +# We've been conflating sparsity as an optimization with sparsity as a semantic extension to PyTorch. +# MaskedTensor proposes to disentangle the sparsity optimization from the semantic extension; for example, +# currently we can't have dense semantics with sparse storage or masked semantics with dense storage. +# MaskedTensor enables these ideas by purposefully separating the storage from the semantics. +# +# Consider the above example using a masked gradient: +# + +# Let's now import MaskedTensor! +from torch.masked import masked_tensor + +# Create an entirely new set of parameters to avoid errors +param2 = torch.arange(8).reshape(2, 4).float() +state_sum2 = torch.full_like(param, 0.5) # initial value for state sum + +mask = (grad.to_dense() != 0).to_sparse() +masked_grad = masked_tensor(grad, mask) + +state_sum2 = state_sum2 + masked_grad.pow(2).get_data() +std2 = masked_tensor(state_sum2.to_sparse(), mask) + +# We can add support for in-place operations later. Notice how this doesn't +# need to access any storage internals and is in general a lot shorter +std2 = std2.sqrt().add(eps) + +param2 = param2.add((masked_grad / std2).get_data(), alpha=-clr) + +###################################################################### +# Note that the implementations look quite similar, but the MaskedTensor implementation is shorter and simpler. +# In particular, much of the boilerplate code around ``_make_sparse`` +# (and needing to have a separate implementation per layout) is handled for the user with :class:`MaskedTensor`. +# +# At this point, let's print both this version and original version for easier comparison: +# + +print("state_sum:\n", state_sum) +print("state_sum2:\n", state_sum2) + +###################################################################### +# + +print("std:\n", std) +print("std2:\n", std2) + +###################################################################### +# + +print("param:\n", param) +print("param2:\n", param2) + +###################################################################### +# Conclusion +# ---------- +# +# In this tutorial, we've discussed how native masked semantics can enable a cleaner developer experience for +# Adagrad's existing implementation in PyTorch, which used sparsity as a proxy for writing masked semantics. +# But more importantly, allowing masked semantics to be a first class citizen through MaskedTensor +# removes the reliance on sparsity or unreliable hacks to mimic masking, thereby allowing for proper independence +# and development, while enabling sparse semantics, such as this one. +# +# Further Reading +# --------------- +# +# To continue learning more, you can find our final review (for now) on +# `MaskedTensor Advanced Semantics `__ +# to see some of the differences in design decisions between :class:`MaskedTensor` and NumPy's MaskedArray, as well +# as reduction semantics. +# diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index b362a38b9e8..7e5c51f8b68 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -141,6 +141,16 @@ Prototype features are not available as part of binary distributions like PyPI o :link: ../prototype/nestedtensor.html :tags: NestedTensor +.. MaskedTensor + +.. customcarditem:: + :header: MaskedTensor: Simplifying Adagrad Sparse Semantics + :card_description: See a showcase on how masked tensors can enable sparse semantics and provide for a cleaner dev experience + :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png + :link: ../prototype/maskedtensor_adagrad.html + :tags: MaskedTensor + + .. End of tutorial card section .. raw:: html @@ -172,3 +182,4 @@ Prototype features are not available as part of binary distributions like PyPI o prototype/vmap_recipe.html prototype/vulkan_workflow.html prototype/nestedtensor.html + prototype/maskedtensor_adagrad.html From db0558973aff1758684c516ae14b3d869770d422 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:08:35 -0700 Subject: [PATCH 18/20] rebase to master --- .jenkins/build.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 9b9f75f89cc..981615672ce 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,9 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext - -# Test enabled for PyTorch 1.13 RC Below -pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip uninstall -y torch torchvision torchaudio torchtext +# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm @@ -49,7 +47,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From d018cb738aa9e39ed7fe3abe78ed3e24c45a0429 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:12:25 -0700 Subject: [PATCH 19/20] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 5f9d563475f..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,7 +51,6 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed - "ax_multiobjective_nas_tutorial", ] From f5b402192ac0fe0253085ca6f9cc22a270a957f5 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:20:41 -0700 Subject: [PATCH 20/20] Update prototype_index.rst --- prototype_source/prototype_index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index 2acf906c9eb..ab9ef713448 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -143,12 +143,14 @@ Prototype features are not available as part of binary distributions like PyPI o .. MaskedTensor +.. customcarditem:: :header: MaskedTensor Overview :card_description: Learn about masked tensors, the source of truth for specified and unspecified values :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png :link: ../prototype/maskedtensor_overview.html :tags: MaskedTensor +.. customcarditem:: :header: MaskedTensor: Simplifying Adagrad Sparse Semantics :card_description: See a showcase on how masked tensors can enable sparse semantics and provide for a cleaner dev experience :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png