From 2ec384511ff124a50e641e8267a08ce107f33155 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 10 Oct 2022 10:02:30 -0700 Subject: [PATCH 01/16] [DO NOT MERGE] 1.13 RC Test --- .jenkins/build.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 981615672ce..42d44b7e725 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,8 +26,11 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext -# pip uninstall -y torch torchvision torchaudio torchtext -# pip install -f https://download.pytorch.org/whl/test/cu111/torch_test.html torch torchvision torchaudio torchtext + +# Test enabled for PyTorch 1.13 RC Below +pip uninstall -y torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 59e485e16c901a32adc47158b8a8b38ef9bda2fc Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 11 Oct 2022 10:15:46 -0700 Subject: [PATCH 02/16] Update .jenkins/build.sh Co-authored-by: Nikita Shulga --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 42d44b7e725..2f2141f1dc9 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -30,7 +30,6 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 005e130b666141fd8a3986f2b1abdc9f9a83684e Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 13 Oct 2022 11:57:54 -0700 Subject: [PATCH 03/16] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 2f2141f1dc9..18ff9a325b7 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --extra-index-url https://download.pytorch.org/whl/test/cu116 torch torchvision torchaudio torchtext +pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From a1309cd304455c8318f8412a530143dc956a20e2 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 14 Oct 2022 11:37:46 -0700 Subject: [PATCH 04/16] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 18ff9a325b7..8ec0427109b 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From a138790d46c184e233e71c7e63d9bfdd5cbac480 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:33:09 -0700 Subject: [PATCH 05/16] Update build.sh --- .jenkins/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 8ec0427109b..e9389dd4832 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,8 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 78e48bf3acfdfecb7789f64732e94f0f0643f99e Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:42:54 -0700 Subject: [PATCH 06/16] Update build.sh --- .jenkins/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index e9389dd4832..b6d7e30b83a 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,8 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip3 install --pre torch torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test torchdata https://download.pytorch.org/whl/test/cu116/torch_test.html -pip install --pre --extra-index-url https://download.pytorch.org/whl/test torchdata +pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 6bbc8cef536e2ebcaf534e4f7cc0854ccd159b3c Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 13:51:19 -0700 Subject: [PATCH 07/16] Update build.sh --- .jenkins/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index b6d7e30b83a..64bd1c88918 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -25,11 +25,11 @@ pip install -r $DIR/../requirements.txt # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html # RC Link # pip uninstall -y torch torchvision torchaudio torchtext -# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre -f https://download.pytorch.org/whl/test torch torchvision torchaudio torchtext +pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 88dff2eacbf01f8f8a69b93e5796a230b5f350ff Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 17 Oct 2022 15:00:37 -0700 Subject: [PATCH 08/16] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 64bd1c88918..76847c420fe 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -29,7 +29,7 @@ pip install -r $DIR/../requirements.txt # Test enabled for PyTorch 1.13 RC Below pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre torch torchdata torchvision torchaudio torchtext -f https://download.pytorch.org/whl/test/cu116/torch_test.html +pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From ff156fa6795f9a6de85c6a19c4ca3f11c7c539dd Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 19 Oct 2022 13:28:00 -0700 Subject: [PATCH 09/16] Temporarily disabling fx_numeric_suite_tutorial --- .jenkins/build.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 76847c420fe..65dc6e43fba 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,10 +26,8 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext - -# Test enabled for PyTorch 1.13 RC Below -pip uninstall -y torch torchvision torchaudio torchtext -pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext +#pip uninstall -y torch torchvision torchaudio torchtext +# pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial python -m spacy download en_core_web_sm From 207558db9662b06e8e1f150af68ba10d9fdad7a2 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 08:52:52 -0700 Subject: [PATCH 10/16] Disable ax tutorial --- .jenkins/validate_tutorials_built.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index cc01326b44c..5f9d563475f 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,6 +51,7 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed + "ax_multiobjective_nas_tutorial", ] From 74bc9f915ae0d17c24ad221751a9c32996d88960 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 20 Oct 2022 13:50:38 -0700 Subject: [PATCH 11/16] Update build.sh --- .jenkins/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 65dc6e43fba..1a7b9efdc66 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -47,6 +47,7 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true + # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From 8a8ee42d5c4238ff3d8c599a771cf94676fb551d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:31:45 -0700 Subject: [PATCH 12/16] [maskedtensor] Sparsity tutorial [2/4] --- prototype_source/maskedtensor_sparsity.py | 315 ++++++++++++++++++++++ prototype_source/prototype_index.rst | 10 + 2 files changed, 325 insertions(+) create mode 100644 prototype_source/maskedtensor_sparsity.py diff --git a/prototype_source/maskedtensor_sparsity.py b/prototype_source/maskedtensor_sparsity.py new file mode 100644 index 00000000000..74024f8e229 --- /dev/null +++ b/prototype_source/maskedtensor_sparsity.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- + +""" +(Prototype) MaskedTensor Sparsity +================================= +""" + +###################################################################### +# Before working on this tutorial, please make sure to review our +# `MaskedTensor Overview tutorial `. +# +# Introduction +# ------------ +# +# Sparsity has been an area of rapid growth and importance within PyTorch; if any sparsity terms are confusing below, +# please refer to the `sparsity tutorial `__ for additional details. +# +# Sparse storage formats have been proven to be powerful in a variety of ways. As a primer, the first use case +# most practitioners think about is when the majority of elements are equal to zero (a high degree of sparsity), +# but even in cases of lower sparsity, certain formats (e.g. BSR) can take advantage of substructures within a matrix. +# +# .. note:: +# +# At the moment, MaskedTensor supports COO and CSR tensors with plans to support additional formats +# (such as BSR and CSC) in the future. If you have any requests for additional formats, +# please file a feature request `here `__! +# +# Principles +# ---------- +# +# When creating a :class:`MaskedTensor` with sparse tensors, there are a few principles that must be observed: +# +# 1. ``data`` and ``mask`` must have the same storage format, whether that's :attr:`torch.strided`, :attr:`torch.sparse_coo`, or :attr:`torch.sparse_csr` +# 2. ``data`` and ``mask`` must have the same size, indicated by :func:`size()` +# +# .. _sparse-coo-tensors: +# +# Sparse COO tensors +# ------------------ +# +# In accordance with Principle #1, a sparse COO MaskedTensor is created by passing in two sparse COO tensors, +# which can be initialized by any of its constructors, for example :func:`torch.sparse_coo_tensor`. +# +# As a recap of `sparse COO tensors `__, the COO format +# stands for "coordinate format", where the specified elements are stored as tuples of their indices and the +# corresponding values. That is, the following are provided: +# +# * ``indices``: array of size ``(ndim, nse)`` and dtype ``torch.int64`` +# * ``values``: array of size `(nse,)` with any integer or floating point dtype +# +# where ``ndim`` is the dimensionality of the tensor and ``nse`` is the number of specified elements. +# +# For both sparse COO and CSR tensors, you can construct a :class:`MaskedTensor` by doing either: +# +# 1. ``masked_tensor(sparse_tensor_data, sparse_tensor_mask)`` +# 2. ``dense_masked_tensor.to_sparse_coo()`` or ``dense_masked_tensor.to_sparse_csr()`` +# +# The second method is easier to illustrate so we've shown that below, but for more on the first and the nuances behind +# the approach, please read the :ref:`Sparse COO Appendix `. +# + +import torch +from torch.masked import masked_tensor +import warnings + +# Disable prototype warnings and such +warnings.filterwarnings(action='ignore', category=UserWarning) + +values = torch.tensor([[0, 0, 3], [4, 0, 5]]) +mask = torch.tensor([[False, False, True], [False, False, True]]) +mt = masked_tensor(values, mask) +sparse_coo_mt = mt.to_sparse_coo() + +print("mt:\n", mt) +print("mt (sparse coo):\n", sparse_coo_mt) +print("mt data (sparse coo):\n", sparse_coo_mt.get_data()) + +###################################################################### +# Sparse CSR tensors +# ------------------ +# +# Similarly, :class:`MaskedTensor` also supports the +# `CSR (Compressed Sparse Row) `__ +# sparse tensor format. Instead of storing the tuples of the indices like sparse COO tensors, sparse CSR tensors +# aim to decrease the memory requirements by storing compressed row indices. +# In particular, a CSR sparse tensor consists of three 1-D tensors: +# +# * ``crow_indices``: array of compressed row indices with size ``(size[0] + 1,)``. This array indicates which row +# a given entry in values lives in. The last element is the number of specified elements, +# while `crow_indices[i+1] - crow_indices[i]` indicates the number of specified elements in row i. +# * ``col_indices``: array of size ``(nnz,)``. Indicates the column indices for each value. +# * ``values``: array of size ``(nnz,)``. Contains the values of the CSR tensor. +# +# Of note, both sparse COO and CSR tensors are in a `beta `__ state. +# +# By way of example: +# + +mt_sparse_csr = mt.to_sparse_csr() + +print("mt (sparse csr):\n", mt_sparse_csr) +print("mt data (sparse csr):\n", mt_sparse_csr.get_data()) + +###################################################################### +# Supported Operations +# -------------------- +# +# Unary +# ^^^^^ +# All `unary operators `__ are supported, e.g.: +# + +mt.sin() + +###################################################################### +# Binary +# ^^^^^^ +# `Binary operators `__ are also supported, but the +# input masks from the two masked tensors must match. For more information on why this decision was made, please +# find our `MaskedTensor: Advanced Semantics tutorial `__. +# +# Please find an example below: +# + +i = [[0, 1, 1], + [2, 0, 2]] +v1 = [3, 4, 5] +v2 = [20, 30, 40] +m = torch.tensor([True, False, True]) + +s1 = torch.sparse_coo_tensor(i, v1, (2, 3)) +s2 = torch.sparse_coo_tensor(i, v2, (2, 3)) +mask = torch.sparse_coo_tensor(i, m, (2, 3)) + +mt1 = masked_tensor(s1, mask) +mt2 = masked_tensor(s2, mask) + +print("mt1:\n", mt1) +print("mt2:\n", mt2) + +###################################################################### +# + +print("torch.div(mt2, mt1):\n", torch.div(mt2, mt1)) +print("torch.mul(mt1, mt2):\n", torch.mul(mt1, mt2)) + +###################################################################### +# Reductions +# ^^^^^^^^^^ +# Finally, `reductions `__ are supported: +# + +mt + +###################################################################### +# + +print("mt.sum():\n", mt.sum()) +print("mt.sum(dim=1):\n", mt.sum(dim=1)) +print("mt.amin():\n", mt.amin()) + +###################################################################### +# MaskedTensor Helper Methods +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# For convenience, :class:`MaskedTensor` has a number of methods to help convert between the different layouts +# and identify the current layout: +# +# Setup: +# + +v = [[3, 0, 0], + [0, 4, 5]] +m = [[True, False, False], + [False, True, True]] + +mt = masked_tensor(torch.tensor(v), torch.tensor(m)) +mt + +###################################################################### +# :meth:`MaskedTensor.to_sparse_coo()` / :meth:`MaskedTensor.to_sparse_csr()` / :meth:`MaskedTensor.to_dense()` +# to help convert between the different layouts. +# + +mt_sparse_coo = mt.to_sparse_coo() +mt_sparse_csr = mt.to_sparse_csr() +mt_dense = mt_sparse_coo.to_dense() + +###################################################################### +# :meth:`MaskedTensor.is_sparse()` -- this will check if the :class:`MaskedTensor`'s layout +# matches any of the supported sparse layouts (currently COO and CSR). +# + +print("mt_dense.is_sparse: ", mt_dense.is_sparse()) +print("mt_sparse_coo.is_sparse: ", mt_sparse_coo.is_sparse()) +print("mt_sparse_csr.is_sparse: ", mt_sparse_csr.is_sparse()) + +###################################################################### +# :meth:`MaskedTensor.is_sparse_coo()` +# + +print("mt_dense.is_sparse_coo: ", mt_dense.is_sparse_coo()) +print("mt_sparse_coo.is_sparse_coo: ", mt_sparse_coo.is_sparse_coo()) +print("mt_sparse_csr.is_sparse_coo: ", mt_sparse_csr.is_sparse_coo()) + +###################################################################### +# :meth:`MaskedTensor.is_sparse_csr()` +# + +print("mt_dense.is_sparse_csr: ", mt_dense.is_sparse_csr()) +print("mt_sparse_coo.is_sparse_csr: ", mt_sparse_coo.is_sparse_csr()) +print("mt_sparse_csr.is_sparse_csr: ", mt_sparse_csr.is_sparse_csr()) + +###################################################################### +# Appendix +# -------- +# +# .. _sparse-coo-appendix: +# +# Sparse COO Construction +# ^^^^^^^^^^^^^^^^^^^^^^^ +# +# Recall in our :ref:`original example `, we created a :class:`MaskedTensor` +# and then converted it to a sparse COO MaskedTensor with :meth:`MaskedTensor.to_sparse_coo`. +# +# Alternatively, we can also construct a sparse COO MaskedTensor directly by passing in two sparse COO tensors: +# + +values = torch.tensor([[0, 0, 3], [4, 0, 5]]).to_sparse() +mask = torch.tensor([[False, False, True], [False, False, True]]).to_sparse() +mt = masked_tensor(values, mask) + +print("values:\n", values) +print("mask:\n", mask) +print("mt:\n", mt) + +###################################################################### +# Instead of using :meth:`torch.Tensor.to_sparse`, we can also create the sparse COO tensors directly, +# which brings us to a warning: +# +# .. warning:: +# +# When using a function like :meth:`MaskedTensor.to_sparse_coo` (analogous to :meth:`Tensor.to_sparse`), +# if the user does not specify the indices like in the above example, +# then the 0 values will be "unspecified" by default. +# +# Below, we explicitly specify the 0's: +# + +i = [[0, 1, 1], + [2, 0, 2]] +v = [3, 4, 5] +m = torch.tensor([True, False, True]) +values = torch.sparse_coo_tensor(i, v, (2, 3)) +mask = torch.sparse_coo_tensor(i, m, (2, 3)) +mt2 = masked_tensor(values, mask) + +print("values:\n", values) +print("mask:\n", mask) +print("mt2:\n", mt2) + +###################################################################### +# Note that ``mt`` and ``mt2`` look identical on the surface, and in the vast majority of operations, will yield the same +# result. But this brings us to a detail on the implementation: +# +# ``data`` and ``mask`` -- only for sparse MaskedTensors -- can have a different number of elements (:func:`nnz`) +# **at creation**, but the indices of ``mask`` must then be a subset of the indices of ``data``. In this case, +# ``data`` will assume the shape of ``mask`` by ``data = data.sparse_mask(mask)``; in other words, any of the elements +# in ``data`` that are not ``True`` in ``mask`` (that is, not specified) will be thrown away. +# +# Therefore, under the hood, the data looks slightly different; ``mt2`` has the "4" value masked out and ``mt`` +# is completely without it. Their underlying data has different shapes, +# which would make operations like ``mt + mt2`` invalid. +# + +print("mt data:\n", mt.get_data()) +print("mt2 data:\n", mt2.get_data()) + +###################################################################### +# .. _sparse-csr-appendix: +# +# Sparse CSR Construction +# ^^^^^^^^^^^^^^^^^^^^^^^ +# +# We can also construct a sparse CSR MaskedTensor using sparse CSR tensors, +# and like the example above, this results in a similar treatment under the hood. +# + +crow_indices = torch.tensor([0, 2, 4]) +col_indices = torch.tensor([0, 1, 0, 1]) +values = torch.tensor([1, 2, 3, 4]) +mask_values = torch.tensor([True, False, False, True]) + +csr = torch.sparse_csr_tensor(crow_indices, col_indices, values, dtype=torch.double) +mask = torch.sparse_csr_tensor(crow_indices, col_indices, mask_values, dtype=torch.bool) +mt = masked_tensor(csr, mask) + +print("mt:\n", mt) +print("mt data:\n", mt.get_data()) + +###################################################################### +# Conclusion +# ---------- +# In this tutorial, we have introduced how to use :class:`MaskedTensor` with sparse COO and CSR formats and +# discussed some of the subtleties under the hood in case users decide to access the underlying data structures +# directly. Sparse storage formats and masked semantics indeed have strong synergies, so much so that they are +# sometimes used as proxies for each other (as we will see in the next tutorial). In the future, we certainly plan +# to invest and continue developing in this direction. +# +# Further Reading +# --------------- +# +# To continue learning more, you can find our +# `Efficiently writing "sparse" semantics for Adagrad with MaskedTensor tutorial `__ +# to see an example of how MaskedTensor can simplify existing workflows with native masking semantics. +# diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index 4878cfa8d2c..c980cb24603 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -148,6 +148,12 @@ Prototype features are not available as part of binary distributions like PyPI o :card_description: Learn about masked tensors, the source of truth for specified and unspecified values :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png :link: ../prototype/maskedtensor_overview.html + +.. customcarditem:: + :header: Masked Tensor Sparsity + :card_description: Learn about how to leverage sparse layouts (e.g. COO and CSR) in MaskedTensor + :image: ../_static/img/thumbnails/cropped/generic-pytorch-logo.png + :link: ../prototype/maskedtensor_sparsity.html :tags: MaskedTensor .. End of tutorial card section @@ -181,4 +187,8 @@ Prototype features are not available as part of binary distributions like PyPI o prototype/vmap_recipe.html prototype/vulkan_workflow.html prototype/nestedtensor.html +<<<<<<< HEAD prototype/maskedtensor_overview.html +======= + prototype/maskedtensor_sparsity.html +>>>>>>> 271d22be9 ([maskedtensor] Sparsity tutorial [2/4]) From b0dbb37fc6ec2b71bbb31b15ea29becfa2a73094 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:33:20 -0700 Subject: [PATCH 13/16] Update build.sh --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 1a7b9efdc66..65dc6e43fba 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -47,7 +47,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true # Temp remove for 1.10 release. # python $DIR/remove_runnable_code.py advanced_source/neural_style_tutorial.py advanced_source/neural_style_tutorial.py || true - # TODO: Fix bugs in these tutorials to make them runnable again # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true From c9dac3c5da9facf21f448343486abaee32f87d4a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 12:35:24 -0700 Subject: [PATCH 14/16] Update validate_tutorials_built.py --- .jenkins/validate_tutorials_built.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index 5f9d563475f..cc01326b44c 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -51,7 +51,6 @@ "hyperparameter_tuning_tutorial", "flask_rest_api_tutorial", "fx_numeric_suite_tutorial", # remove when https://github.com/pytorch/tutorials/pull/2089 is fixed - "ax_multiobjective_nas_tutorial", ] From 58aaa98e6e3d1af6108c61f2ca2bfb00315f6d3e Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 12:50:38 -0700 Subject: [PATCH 15/16] Update build.sh --- .jenkins/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 65dc6e43fba..d3713cf87a4 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -53,7 +53,6 @@ if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then # Remove runnable code from tensorboard_profiler_tutorial.py as it frequently crashes, see https://github.com/pytorch/pytorch/issues/74139 # python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true - # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials # IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename. export WORKER_ID=$(echo "${JOB_BASE_NAME}" | tr -dc '0-9') From c1bdb6ea16dfea87e22ce7551dce96a65fb6d999 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 28 Oct 2022 13:36:38 -0700 Subject: [PATCH 16/16] Update --- .jenkins/build.sh | 2 +- prototype_source/prototype_index.rst | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index d3713cf87a4..6e2529a6fb3 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -26,7 +26,7 @@ pip install -r $DIR/../requirements.txt # RC Link # pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext -#pip uninstall -y torch torchvision torchaudio torchtext +# pip uninstall -y torch torchvision torchaudio torchtext # pip install --pre --upgrade -f https://download.pytorch.org/whl/test/cu116/torch_test.html torch torchdata torchvision torchaudio torchtext # Install two language tokenizers for Translation with TorchText tutorial diff --git a/prototype_source/prototype_index.rst b/prototype_source/prototype_index.rst index c980cb24603..ec4bf5c9d9b 100644 --- a/prototype_source/prototype_index.rst +++ b/prototype_source/prototype_index.rst @@ -187,8 +187,5 @@ Prototype features are not available as part of binary distributions like PyPI o prototype/vmap_recipe.html prototype/vulkan_workflow.html prototype/nestedtensor.html -<<<<<<< HEAD prototype/maskedtensor_overview.html -======= prototype/maskedtensor_sparsity.html ->>>>>>> 271d22be9 ([maskedtensor] Sparsity tutorial [2/4])