From e62ee7c024162b9138e5cb1b3b019bd2d497bb69 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Mon, 28 Sep 2020 23:21:59 -0400 Subject: [PATCH 01/12] Update tensor_tutorial.py --- beginner_source/blitz/tensor_tutorial.py | 271 ++++++++++------------- 1 file changed, 121 insertions(+), 150 deletions(-) diff --git a/beginner_source/blitz/tensor_tutorial.py b/beginner_source/blitz/tensor_tutorial.py index 7b339ee225f..2656c2119a5 100644 --- a/beginner_source/blitz/tensor_tutorial.py +++ b/beginner_source/blitz/tensor_tutorial.py @@ -1,195 +1,166 @@ -# -*- coding: utf-8 -*- """ -What is PyTorch? -================ +Tensors - The building blocks of deep learning +-------------------------------------------- -It’s a Python-based scientific computing package targeted at two sets of -audiences: +Tensors are a specialized data structure that are very similar to arrays +and matrices. In PyTorch, we use tensors to encode the inputs and +outputs of a model, as well as the model’s parameters. -- A replacement for NumPy to use the power of GPUs -- a deep learning research platform that provides maximum flexibility - and speed +Tensors are similar to NumPy’s ndarrays, except that tensors can run on +a GPU to accelerate computing. If you’re familiar with ndarrays, you’ll +be right at home with the Tensor API. If not, follow along in this quick +API walkthrough. -Getting Started ---------------- - -Tensors -^^^^^^^ - -Tensors are similar to NumPy’s ndarrays, with the addition being that -Tensors can also be used on a GPU to accelerate computing. """ -from __future__ import print_function import torch +import numpy as np -############################################################### -# .. note:: -# An uninitialized matrix is declared, -# but does not contain definite known -# values before it is used. When an -# uninitialized matrix is created, -# whatever values were in the allocated -# memory at the time will appear as the initial values. -############################################################### -# Construct a 5x3 matrix, uninitialized: +###################################################################### +# Tensor Initialization +# ~~~~~~~~~~~~~~~~~~~~~ +# +# Tensors can be initialized in various ways. Take a look at the following examples +# +# **Directly from data / NumPy arrays:** +# -x = torch.empty(5, 3) -print(x) - -############################################################### -# Construct a randomly initialized matrix: +data = [[1, 2],[3, 4]] +np_array = np.array(data) -x = torch.rand(5, 3) -print(x) +tnsr_from_data = torch.tensor(data) +tnsr_from_np = torch.from_numpy(np_array) -############################################################### -# Construct a matrix filled zeros and of dtype long: +###################################################################### +# **With random or constant values:** +# -x = torch.zeros(5, 3, dtype=torch.long) -print(x) +shape = (2,3,) +rand_tnsr = torch.rand(shape) +ones_tnsr = torch.ones(shape) +zeros_tnsr = torch.zeros(shape) +print(f"Random Tensor:\n{rand_tnsr}\n\nOnes Tensor:\n{ones_tnsr}\n\nZeros Tensor:\n{zeros_tnsr}") -############################################################### -# Construct a tensor directly from data: - -x = torch.tensor([5.5, 3]) -print(x) ############################################################### -# or create a tensor based on an existing tensor. These methods -# will reuse properties of the input tensor, e.g. dtype, unless -# new values are provided by user - -x = x.new_ones(5, 3, dtype=torch.double) # new_* methods take in sizes -print(x) +# **From another tensor:** +# The new tensor retains the properties of the arg tensor, unless explicitly overridden -x = torch.randn_like(x, dtype=torch.float) # override dtype! -print(x) # result has the same size +new_ones_tnsr = torch.ones_like(tnsr_from_data) # 2 x 2 matrix of ones -############################################################### -# Get its size: +try: + new_rand_tnsr = torch.rand_like(tnsr_from_data) # 2 x 2 matrix of random numbers +except RuntimeError as e: + print(f"RuntimeError thrown: {e}") + print() + print(f"Random values in PyTorch are floating points. Datatype passed to torch.rand_like(): {tnsr_from_data.dtype}") -print(x.size()) -############################################################### -# .. note:: -# ``torch.Size`` is in fact a tuple, so it supports all tuple operations. -# -# Operations -# ^^^^^^^^^^ -# There are multiple syntaxes for operations. In the following -# example, we will take a look at the addition operation. +###################################################################### +# This works after we override the dtype property # -# Addition: syntax 1 -y = torch.rand(5, 3) -print(x + y) -############################################################### -# Addition: syntax 2 +new_rand_tnsr = torch.rand_like(tnsr_from_data, dtype=torch.float) +print(new_rand_tnsr) -print(torch.add(x, y)) -############################################################### -# Addition: providing an output tensor as argument -result = torch.empty(5, 3) -torch.add(x, y, out=result) -print(result) -############################################################### -# Addition: in-place +###################################################################### +# -------------- +# -# adds x to y -y.add_(x) -print(y) -############################################################### -# .. note:: -# Any operation that mutates a tensor in-place is post-fixed with an ``_``. -# For example: ``x.copy_(y)``, ``x.t_()``, will change ``x``. +###################################################################### +# Tensor Attributes +# ~~~~~~~~~~~~~~~~~ +# +# Tensors have attributes that describe their contents and functions. They +# also have attributes that autograd uses to keep track of them in the +# computational graph (more on this in the next section). +# +# **Docs Issues** - https://pytorch.org/docs/stable/tensor_attributes.html +# is not comprehensive (missing data, grad, grad_fn, shape). Contains +# ``memory_format`` which is not an attribute # -# You can use standard NumPy-like indexing with all bells and whistles! -print(x[:, 1]) +tnsr = torch.rand(3,4) -############################################################### -# Resizing: If you want to resize/reshape tensor, you can use ``torch.view``: -x = torch.randn(4, 4) -y = x.view(16) -z = x.view(-1, 8) # the size -1 is inferred from other dimensions -print(x.size(), y.size(), z.size()) +print(f"Data stored in tensor:\n{tnsr.data}\n") +print(f"Shape of tensor: {tnsr.shape}") +print(f"Datatype of tensor: {tnsr.dtype}") +print(f"Device tensor lives on: {tnsr.device}") -############################################################### -# If you have a one element tensor, use ``.item()`` to get the value as a -# Python number -x = torch.randn(1) -print(x) -print(x.item()) +print("-------") -############################################################### -# **Read later:** -# -# -# 100+ Tensor operations, including transposing, indexing, slicing, -# mathematical operations, linear algebra, random numbers, etc., -# are described -# `here `_. +# Autograd-related attributes; more on this later +print(f"Is tensor a leaf on computational graph: {tnsr.is_leaf}") +print(f"Does tensor require gradients: {tnsr.requires_grad}") +print(f"Accumulated gradients of tensor: {tnsr.grad}") +print(f"Function that computed this tensor's gradient: {tnsr.grad_fn}") + + + +###################################################################### +# -------------- # -# NumPy Bridge -# ------------ + + +###################################################################### +# Tensor Operations +# ~~~~~~~~~~~~~~~~~ # -# Converting a Torch Tensor to a NumPy array and vice versa is a breeze. +# Over 100 tensor operations, including transposing, indexing, slicing, +# mathematical operations, linear algebra, random sampling, and more are +# comprehensively described +# `here `__. # -# The Torch Tensor and NumPy array will share their underlying memory -# locations (if the Torch Tensor is on CPU), and changing one will change -# the other. +# Each of them can be run on the GPU (at typically higher speeds than on a +# CPU). If you’re using Colab, allocate a GPU by going to Edit > Notebook +# Settings. # -# Converting a Torch Tensor to a NumPy Array -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -a = torch.ones(5) -print(a) +# We move our tensor to the GPU if available +if torch.cuda.is_available: + tnsr.to('cuda') -############################################################### -# -b = a.numpy() -print(b) +###################################################################### +# Try out some of the operations from the list. You may come across multiple +# syntaxes for the same operation. Don’t get confused by the aliases! +# ############################################################### -# See how the numpy array changed in value. +# **Standard numpy-like indexing and slicing:** -a.add_(1) -print(a) -print(b) +tnsr = torch.ones(4, 4) +tnsr[:,1] = 0 +print(tnsr) -############################################################### -# Converting NumPy Array to Torch Tensor -# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# See how changing the np array changed the Torch Tensor automatically +###################################################################### +# **Both of these are joining ops, but they are subtly different.** -import numpy as np -a = np.ones(5) -b = torch.from_numpy(a) -np.add(a, 1, out=a) -print(a) -print(b) +t1 = torch.cat([tnsr, tnsr], dim=1) +t2 = torch.stack([tnsr, tnsr], dim=1) +print(t1) +print() +print(t2) -############################################################### -# All the Tensors on the CPU except a CharTensor support converting to -# NumPy and back. -# -# CUDA Tensors -# ------------ -# -# Tensors can be moved onto any device using the ``.to`` method. - -# let us run this cell only if CUDA is available -# We will use ``torch.device`` objects to move tensors in and out of GPU -if torch.cuda.is_available(): - device = torch.device("cuda") # a CUDA device object - y = torch.ones_like(x, device=device) # directly create a tensor on GPU - x = x.to(device) # or just use strings ``.to("cuda")`` - z = x + y - print(z) - print(z.to("cpu", torch.double)) # ``.to`` can also change dtype together! +###################################################################### +# **Multiply op - multiple syntaxes** + +# both ops compute element-wise product +print(tnsr * tnsr == tnsr.mul(tnsr)) + +# both ops compute matrix product +print(tnsr @ tnsr.T == tnsr.matmul(tnsr.T)) + +###################################################################### +# .. note:: +# Operations that have a '_' suffix are in-place. For example: +# ``x.copy_(y)``, ``x.t_()``, will change ``x``. In-place operations +# don't work well with Autograd, and their use is discouraged. + +print(tnsr) +tnsr.t_() +print(tnsr) From 0e867c791382db78628c6b23c964cee202741942 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Fri, 9 Oct 2020 08:44:17 -0400 Subject: [PATCH 02/12] Update autograd_tutorial.py --- beginner_source/blitz/autograd_tutorial.py | 413 ++++++++++++--------- 1 file changed, 237 insertions(+), 176 deletions(-) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 98e70a251d6..44875d29a35 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -1,198 +1,259 @@ # -*- coding: utf-8 -*- """ -Autograd: Automatic Differentiation -=================================== +A Gentle Introduction to Autograd +--------------------------------- -Central to all neural networks in PyTorch is the ``autograd`` package. -Let’s first briefly visit this, and we will then go to training our -first neural network. +Autograd is PyTorch’s automatic differentiation engine that powers +neural network training. In this section, you will get a conceptual +understanding of how autograd works under the hood. +Skip to advanced readings: -The ``autograd`` package provides automatic differentiation for all operations -on Tensors. It is a define-by-run framework, which means that your backprop is -defined by how your code is run, and that every single iteration can be -different. +- `In-place operations & Multithreaded + Autograd `__ +- `Example implementation of reverse-mode + autodiff `__ -Let us see this in more simple terms with some examples. +""" -Tensor --------- -``torch.Tensor`` is the central class of the package. If you set its attribute -``.requires_grad`` as ``True``, it starts to track all operations on it. When -you finish your computation you can call ``.backward()`` and have all the -gradients computed automatically. The gradient for this tensor will be -accumulated into ``.grad`` attribute. +###################################################################### +# -------------- +# -To stop a tensor from tracking history, you can call ``.detach()`` to detach -it from the computation history, and to prevent future computation from being -tracked. -To prevent tracking history (and using memory), you can also wrap the code block -in ``with torch.no_grad():``. This can be particularly helpful when evaluating a -model because the model may have trainable parameters with -``requires_grad=True``, but for which we don't need the gradients. +###################################################################### +# Background +# ~~~~~~~~~~ +# +# Neural networks (NNs) are a collection of nested functions that are +# executed on some input data. These functions are defined by *parameters* +# (consisting of weights and biases), which in PyTorch are stored in +# tensors. +# +# Training a NN happens in two steps: +# +# **Forward Propagation**: In forward prop, the NN makes its best guess +# about the correct output. It runs the input data through each of its +# functions to make this guess. +# +# **Backward Propagation**: In backprop, the NN adjusts its parameters +# proportionate to the error in its guess. It does this by traversing +# backwards from the output, collecting the derivatives of the error with +# respect to the parameters of the functions (*gradients*), and optimizing +# the parameters using gradient descent. For a more detailed walkthrough +# of backprop, check out this `video from +# 3Blue1Brown `__. +# +# Most deep learning frameworks use automatic differentiation for +# backprop; in PyTorch, it is handled by Autograd. +# -There’s one more class which is very important for autograd -implementation - a ``Function``. -``Tensor`` and ``Function`` are interconnected and build up an acyclic -graph, that encodes a complete history of computation. Each tensor has -a ``.grad_fn`` attribute that references a ``Function`` that has created -the ``Tensor`` (except for Tensors created by the user - their -``grad_fn is None``). +###################################################################### +# -------------- +# -If you want to compute the derivatives, you can call ``.backward()`` on -a ``Tensor``. If ``Tensor`` is a scalar (i.e. it holds a one element -data), you don’t need to specify any arguments to ``backward()``, -however if it has more elements, you need to specify a ``gradient`` -argument that is a tensor of matching shape. -""" -import torch - -############################################################### -# Create a tensor and set ``requires_grad=True`` to track computation with it -x = torch.ones(2, 2, requires_grad=True) -print(x) - -############################################################### -# Do a tensor operation: -y = x + 2 -print(y) - -############################################################### -# ``y`` was created as a result of an operation, so it has a ``grad_fn``. -print(y.grad_fn) - -############################################################### -# Do more operations on ``y`` -z = y * y * 3 -out = z.mean() - -print(z, out) - -################################################################ -# ``.requires_grad_( ... )`` changes an existing Tensor's ``requires_grad`` -# flag in-place. The input flag defaults to ``False`` if not given. -a = torch.randn(2, 2) -a = ((a * 3) / (a - 1)) -print(a.requires_grad) -a.requires_grad_(True) -print(a.requires_grad) -b = (a * a).sum() -print(b.grad_fn) - -############################################################### -# Gradients -# --------- -# Let's backprop now. -# Because ``out`` contains a single scalar, ``out.backward()`` is -# equivalent to ``out.backward(torch.tensor(1.))``. - -out.backward() - -############################################################### -# Print gradients d(out)/dx -# - -print(x.grad) - -############################################################### -# You should have got a matrix of ``4.5``. Let’s call the ``out`` -# *Tensor* “:math:`o`”. -# We have that :math:`o = \frac{1}{4}\sum_i z_i`, -# :math:`z_i = 3(x_i+2)^2` and :math:`z_i\bigr\rvert_{x_i=1} = 27`. -# Therefore, -# :math:`\frac{\partial o}{\partial x_i} = \frac{3}{2}(x_i+2)`, hence -# :math:`\frac{\partial o}{\partial x_i}\bigr\rvert_{x_i=1} = \frac{9}{2} = 4.5`. - -############################################################### -# Mathematically, if you have a vector valued function :math:`\vec{y}=f(\vec{x})`, -# then the gradient of :math:`\vec{y}` with respect to :math:`\vec{x}` -# is a Jacobian matrix: +###################################################################### +# Differentiation in Autograd +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# The ``requires_grad`` flag lets autograd know +# if we need gradients w.r.t. these tensors. If it is ``True``, autograd +# tracks all operations on them. +# + +a = torch.tensor([2., 3.], requires_grad=True) +b = torch.tensor([6., 4.], requires_grad=True) + +Q = 3*a**3 - b**2 +print(Q) + + +###################################################################### +# ``a`` and ``b`` can be viewed as parameters of an NN, with ``Q`` +# analogous to the error. In training we want gradients of the error +# w.r.t. parameters, i.e. # # .. math:: -# J=\left(\begin{array}{ccc} -# \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{1}}{\partial x_{n}}\\ -# \vdots & \ddots & \vdots\\ -# \frac{\partial y_{m}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} -# \end{array}\right) +# +# +# \frac{\partial Q}{\partial a} = 9a^2 +# +# .. math:: +# +# +# \frac{\partial Q}{\partial b} = -2b +# +# Since ``Q`` is a vector, we pass a ``gradient`` argument in +# ``.backward()``. +# +# ``gradient`` is a tensor of the same shape. Here it represents the +# gradient of Q w.r.t. itself, i.e. +# +# .. math:: +# +# +# \frac{dQ}{dQ} = 1 +# + +external_grad = torch.tensor([1., 1.]) +Q.backward(gradient=external_grad) + +# check if autograd's gradients are correct +print(9*a**2 == a.grad) +print(-2*b == b.grad) + + +###################################################################### +# Optional Reading - Vector Calculus in Autograd +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# +# Mathematically, if you have a vector valued function +# :math:`\vec{y}=f(\vec{x})`, then the gradient of :math:`\vec{y}` with +# respect to :math:`\vec{x}` is a Jacobian matrix :math:`J`: +# +# .. math:: +# +# +# J +# = +# \left(\begin{array}{cc} +# \frac{\partial \bf{y}}{\partial x_{1}} & +# ... & +# \frac{\partial \bf{y}}{\partial x_{n}} +# \end{array}\right) +# = +# \left(\begin{array}{ccc} +# \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{1}}{\partial x_{n}}\\ +# \vdots & \ddots & \vdots\\ +# \frac{\partial y_{m}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} +# \end{array}\right) # # Generally speaking, ``torch.autograd`` is an engine for computing -# vector-Jacobian product. That is, given any vector -# :math:`v=\left(\begin{array}{cccc} v_{1} & v_{2} & \cdots & v_{m}\end{array}\right)^{T}`, -# compute the product :math:`v^{T}\cdot J`. If :math:`v` happens to be -# the gradient of a scalar function :math:`l=g\left(\vec{y}\right)`, -# that is, -# :math:`v=\left(\begin{array}{ccc}\frac{\partial l}{\partial y_{1}} & \cdots & \frac{\partial l}{\partial y_{m}}\end{array}\right)^{T}`, +# vector-Jacobian product. That is, given any vector :math:`\vec{v}`, compute the product +# :math:`J^{T}\cdot \vec{v}` +# +# If :math:`v` happens to be the gradient of a scalar function +# +# .. math:: +# +# +# l +# = +# g\left(\vec{y}\right) +# = +# \left(\begin{array}{ccc}\frac{\partial l}{\partial y_{1}} & \cdots & \frac{\partial l}{\partial y_{m}}\end{array}\right)^{T} +# # then by the chain rule, the vector-Jacobian product would be the # gradient of :math:`l` with respect to :math:`\vec{x}`: # # .. math:: -# J^{T}\cdot v=\left(\begin{array}{ccc} -# \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{1}}\\ -# \vdots & \ddots & \vdots\\ -# \frac{\partial y_{1}}{\partial x_{n}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} -# \end{array}\right)\left(\begin{array}{c} -# \frac{\partial l}{\partial y_{1}}\\ -# \vdots\\ -# \frac{\partial l}{\partial y_{m}} -# \end{array}\right)=\left(\begin{array}{c} -# \frac{\partial l}{\partial x_{1}}\\ -# \vdots\\ -# \frac{\partial l}{\partial x_{n}} -# \end{array}\right) -# -# (Note that :math:`v^{T}\cdot J` gives a row vector which can be -# treated as a column vector by taking :math:`J^{T}\cdot v`.) -# -# This characteristic of vector-Jacobian product makes it very -# convenient to feed external gradients into a model that has -# non-scalar output. - -############################################################### -# Now let's take a look at an example of vector-Jacobian product: - -x = torch.randn(3, requires_grad=True) - -y = x * 2 -while y.data.norm() < 1000: - y = y * 2 - -print(y) - -############################################################### -# Now in this case ``y`` is no longer a scalar. ``torch.autograd`` -# could not compute the full Jacobian directly, but if we just -# want the vector-Jacobian product, simply pass the vector to -# ``backward`` as argument: -v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float) -y.backward(v) - -print(x.grad) - -############################################################### -# You can also stop autograd from tracking history on Tensors -# with ``.requires_grad=True`` either by wrapping the code block in -# ``with torch.no_grad():`` -print(x.requires_grad) -print((x ** 2).requires_grad) - -with torch.no_grad(): - print((x ** 2).requires_grad) - -############################################################### -# Or by using ``.detach()`` to get a new Tensor with the same -# content but that does not require gradients: -print(x.requires_grad) -y = x.detach() -print(y.requires_grad) -print(x.eq(y).all()) - - -############################################################### -# **Read Later:** -# -# Document about ``autograd.Function`` is at -# https://pytorch.org/docs/stable/autograd.html#function +# +# +# J^{T}\cdot \vec{v}=\left(\begin{array}{ccc} +# \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{1}}\\ +# \vdots & \ddots & \vdots\\ +# \frac{\partial y_{1}}{\partial x_{n}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} +# \end{array}\right)\left(\begin{array}{c} +# \frac{\partial l}{\partial y_{1}}\\ +# \vdots\\ +# \frac{\partial l}{\partial y_{m}} +# \end{array}\right)=\left(\begin{array}{c} +# \frac{\partial l}{\partial x_{1}}\\ +# \vdots\\ +# \frac{\partial l}{\partial x_{n}} +# \end{array}\right) +# +# This characteristic of vector-Jacobian product is what we use in the above example; +# ``external_grad`` represents :math:`\vec{v}`. +# + + +###################################################################### +# -------------- +# + + +###################################################################### +# Computational Graph +# ~~~~~~~~~~~~~~~~~~~ +# +# Conceptually, autograd keeps a record of data (tensors) & all executed +# operations (along with the resulting new tensors) in a directed acyclic +# graph (DAG) consisting of +# `Function `__ +# objects. In this DAG, leaves are the input tensors, roots are the output +# tensors. By tracing this graph from roots to leaves, you can +# automatically compute the gradients using the chain rule. +# +# In a forward pass, autograd does two things simultaneously: \* run the +# requested operation to compute a resulting tensor, and \* maintain the +# operation’s *gradient function* in the DAG. This is stored in the +# resulting tensor’s .\ ``grad_fn`` attribute. +# +# The backward pass kicks off when ``.backward()`` is called on the DAG +# root. Autograd then \* computes the gradients from each ``.grad_fn``, \* +# accumulates them in the respective tensor’s ``.grad`` attribute, and \* +# using the chain rule, propagates all the way to the leaf tensors. +# +# .. Note:: +# **Autograd DAGs are dynamic in PyTorch** +# An important thing to note is that the graph is recreated from scratch; after each +# ``.backward()`` call, autograd starts populating a new graph. This is +# exactly what allows you to use control flow statements in your model; +# you can change the shape, size and operations at every iteration if +# needed. Autograd does not need you to encode all possible paths +# beforehand. +# + + +###################################################################### +# Exclusion from the DAG +# ^^^^^^^^^^^^^^^^^^^^^^ +# +# Autograd tracks operations on all tensors which have their +# ``requires_grad`` flag set to ``True``. For tensors that don’t require +# gradients, setting this attribute to ``False`` excludes it from the +# gradient computation DAG and increases efficiency. +# +# The output tensor of an operation will require gradients even if only a +# single input tensor has ``requires_grad=True``. +# + +x = torch.rand(5, 5) +y = torch.rand(5, 5) +z = torch.rand((5, 5), requires_grad=True) + +a = x + y +print("Does `a` require gradients?") +print(a.requires_grad==True) +b = x + z +print("Does `b` require gradients?") +print(b.requires_grad==True) + + +###################################################################### +# This is especially useful when you want to freeze part of your model +# (for instance, when you’re fine-tuning a pretrained model), or you know +# in advance that you’re not going to use gradients w.r.t. some +# parameters. +# + +model = torchvision.models.resnet18(pretrained=True) + +for param in model.parameters(): + param.requires_grad = False +# Replace the last fully-connected layer +# Parameters of nn.Module instances have requires_grad=True by default +model.fc = nn.Linear(512, 100) + +# Optimize only the classifier +optimizer = optim.SGD(model.fc.parameters(), lr=1e-2, momentum=0.9) + + +###################################################################### +# The same functionality is available as a context manager in +# `torch.no_grad() `__ +# From edf2495f27fe3d30d4884fc910407c787703e269 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 13 Oct 2020 20:10:18 -0400 Subject: [PATCH 03/12] Fix import --- beginner_source/blitz/autograd_tutorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 44875d29a35..0da83d8845b 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -64,6 +64,7 @@ # tracks all operations on them. # +import torch a = torch.tensor([2., 3.], requires_grad=True) b = torch.tensor([6., 4.], requires_grad=True) From 6d094b7d7317f1b8f8af27107778daafaca1211d Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 13 Oct 2020 20:22:53 -0400 Subject: [PATCH 04/12] Fix import --- beginner_source/blitz/autograd_tutorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 0da83d8845b..ba7008667e3 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -242,6 +242,7 @@ # parameters. # +import torchvision model = torchvision.models.resnet18(pretrained=True) for param in model.parameters(): From cd6f97ac7a7720d4dd7d372f8d07cd6125cec871 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 13 Oct 2020 20:38:56 -0400 Subject: [PATCH 05/12] Fix import --- beginner_source/blitz/autograd_tutorial.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index ba7008667e3..6ac5b2d1bc9 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -65,6 +65,7 @@ # import torch + a = torch.tensor([2., 3.], requires_grad=True) b = torch.tensor([6., 4.], requires_grad=True) @@ -243,6 +244,8 @@ # import torchvision +from torch import nn, optim + model = torchvision.models.resnet18(pretrained=True) for param in model.parameters(): From e0b21e3484cbd9422834288ac3335c2eac67fd10 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 20 Oct 2020 11:24:46 -0400 Subject: [PATCH 06/12] Fixes issues in tensor, updates autograd --- beginner_source/blitz/autograd_tutorial.py | 94 ++++++++++++---------- beginner_source/blitz/tensor_tutorial.py | 93 +++++++++++++-------- 2 files changed, 111 insertions(+), 76 deletions(-) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 6ac5b2d1bc9..ace27c0a91e 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -7,46 +7,52 @@ neural network training. In this section, you will get a conceptual understanding of how autograd works under the hood. -Skip to advanced readings: - -- `In-place operations & Multithreaded - Autograd `__ -- `Example implementation of reverse-mode - autodiff `__ - +Background +~~~~~~~~~~ +Neural networks (NNs) are a collection of nested functions that are +executed on some input data. These functions are defined by *parameters* +(consisting of weights and biases), which in PyTorch are stored in +tensors. + +Training a NN happens in two steps: + +**Forward Propagation**: In forward prop, the NN makes its best guess +about the correct output. It runs the input data through each of its +functions to make this guess. + +**Backward Propagation**: In backprop, the NN adjusts its parameters +proportionate to the error in its guess. It does this by traversing +backwards from the output, collecting the derivatives of the error with +respect to the parameters of the functions (*gradients*), and optimizing +the parameters using **gradient descent**. For a more detailed walkthrough +of backprop, check out this `video from +3Blue1Brown `__. + +Most deep learning frameworks use automatic differentiation for +backprop; in PyTorch, it is handled by Autograd. + + +Usage in PyTorch +~~~~~~~~~~~ +Backward propagation can be kicked off by calling ``.backward()`` on the error tensor. +This collects the gradients for each parameter in the model. """ +import torch, torchvision -###################################################################### -# -------------- -# +model = torchvision.models.resnet18(pretrained=True) +data = torch.rand(1, 3, 64, 64) +labels = torch.rand(1, 1000) +prediction = model(x) # forward pass +loss = prediction - labels +loss.backward() # backward pass +optim = torch.optim.SGD(model.parameters()) +optim.step() #gradient descent ###################################################################### -# Background -# ~~~~~~~~~~ -# -# Neural networks (NNs) are a collection of nested functions that are -# executed on some input data. These functions are defined by *parameters* -# (consisting of weights and biases), which in PyTorch are stored in -# tensors. -# -# Training a NN happens in two steps: -# -# **Forward Propagation**: In forward prop, the NN makes its best guess -# about the correct output. It runs the input data through each of its -# functions to make this guess. -# -# **Backward Propagation**: In backprop, the NN adjusts its parameters -# proportionate to the error in its guess. It does this by traversing -# backwards from the output, collecting the derivatives of the error with -# respect to the parameters of the functions (*gradients*), and optimizing -# the parameters using gradient descent. For a more detailed walkthrough -# of backprop, check out this `video from -# 3Blue1Brown `__. -# -# Most deep learning frameworks use automatic differentiation for -# backprop; in PyTorch, it is handled by Autograd. +# At this point, you have everything you need to build your neural network. +# The below sections detail the workings of autograd - feel free to skip them. # @@ -58,11 +64,11 @@ ###################################################################### # Differentiation in Autograd # ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + # The ``requires_grad`` flag lets autograd know # if we need gradients w.r.t. these tensors. If it is ``True``, autograd # tracks all operations on them. -# + import torch @@ -173,10 +179,6 @@ # -###################################################################### -# -------------- -# - ###################################################################### # Computational Graph @@ -243,7 +245,6 @@ # parameters. # -import torchvision from torch import nn, optim model = torchvision.models.resnet18(pretrained=True) @@ -262,3 +263,14 @@ # The same functionality is available as a context manager in # `torch.no_grad() `__ # + +###################################################################### +# -------------- +# + +###################################################################### +# Further readings: +# ~~~~~~~~~~~~~~~~~~~ +# +# - `In-place operations & Multithreaded Autograd `__ +# - `Example implementation of reverse-mode autodiff `__ diff --git a/beginner_source/blitz/tensor_tutorial.py b/beginner_source/blitz/tensor_tutorial.py index 2656c2119a5..4fc9b13d120 100644 --- a/beginner_source/blitz/tensor_tutorial.py +++ b/beginner_source/blitz/tensor_tutorial.py @@ -1,5 +1,5 @@ """ -Tensors - The building blocks of deep learning +Tensors -------------------------------------------- Tensors are a specialized data structure that are very similar to arrays @@ -24,7 +24,7 @@ # Tensors can be initialized in various ways. Take a look at the following examples # # **Directly from data / NumPy arrays:** -# +# Tensors can be created directly by passing a Python list or sequence using the ``torch.tensor()`` constructor. The data type is automatically inferred from the data. data = [[1, 2],[3, 4]] np_array = np.array(data) @@ -32,20 +32,10 @@ tnsr_from_data = torch.tensor(data) tnsr_from_np = torch.from_numpy(np_array) -###################################################################### -# **With random or constant values:** -# - -shape = (2,3,) -rand_tnsr = torch.rand(shape) -ones_tnsr = torch.ones(shape) -zeros_tnsr = torch.zeros(shape) -print(f"Random Tensor:\n{rand_tnsr}\n\nOnes Tensor:\n{ones_tnsr}\n\nZeros Tensor:\n{zeros_tnsr}") - ############################################################### # **From another tensor:** -# The new tensor retains the properties of the arg tensor, unless explicitly overridden +# The new tensor retains the properties of the arg tensor, unless explicitly overridden. new_ones_tnsr = torch.ones_like(tnsr_from_data) # 2 x 2 matrix of ones @@ -65,6 +55,20 @@ print(new_rand_tnsr) +###################################################################### +# **With random or constant values:** +# + +shape = (2,3,) +rand_tnsr = torch.rand(shape) +ones_tnsr = torch.ones(shape) +zeros_tnsr = torch.zeros(shape) +print(f"Random Tensor:\n{rand_tnsr} \n\n \ + Ones Tensor:\n{ones_tnsr} \n\n \ + Zeros Tensor:\n{zeros_tnsr}") + + + ###################################################################### # -------------- @@ -75,31 +79,14 @@ # Tensor Attributes # ~~~~~~~~~~~~~~~~~ # -# Tensors have attributes that describe their contents and functions. They -# also have attributes that autograd uses to keep track of them in the -# computational graph (more on this in the next section). -# -# **Docs Issues** - https://pytorch.org/docs/stable/tensor_attributes.html -# is not comprehensive (missing data, grad, grad_fn, shape). Contains -# ``memory_format`` which is not an attribute -# +# Tensor attributes describe their shape data type and where they live. tnsr = torch.rand(3,4) -print(f"Data stored in tensor:\n{tnsr.data}\n") print(f"Shape of tensor: {tnsr.shape}") print(f"Datatype of tensor: {tnsr.dtype}") print(f"Device tensor lives on: {tnsr.device}") -print("-------") - -# Autograd-related attributes; more on this later -print(f"Is tensor a leaf on computational graph: {tnsr.is_leaf}") -print(f"Does tensor require gradients: {tnsr.requires_grad}") -print(f"Accumulated gradients of tensor: {tnsr.grad}") -print(f"Function that computed this tensor's gradient: {tnsr.grad_fn}") - - ###################################################################### # -------------- @@ -121,13 +108,12 @@ # # We move our tensor to the GPU if available -if torch.cuda.is_available: - tnsr.to('cuda') +if torch.cuda.is_available(): + tnsr = tnsr.to('cuda') ###################################################################### -# Try out some of the operations from the list. You may come across multiple -# syntaxes for the same operation. Don’t get confused by the aliases! +# Try out some of the operations from the list. If you're familiar with the NumPy API, you'll find the Tensor API a breeze to use. # ############################################################### @@ -164,3 +150,40 @@ print(tnsr) tnsr.t_() print(tnsr) + +###################################################################### +# -------------- +# + + +###################################################################### +# Bridge with NumPy +# ~~~~~~~~~~~~~~~~~ +# Tensors on the CPU and NumPy arrays can share their underlying memory +# locations, and changing one will change the other. + + +###################################################################### +# Tensor to NumPy array +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +a = torch.ones(5) +print(f"a: {a}") +b = a.numpy() +print(f"b: {b}") + +###################################################################### +# A change in ``a`` reflects in ``b`` + +a.add_(1) +print(f"a: {a}") +print(f"b: {b}") + + +###################################################################### +# NumPy array to Tensor +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +a = np.ones(5) +b = torch.from_numpy(a) +np.add(a, 1, out=a) +print(f"a: {a}") +print(f"b: {b}") From bc503028810610a69e133617bef6d22041261fab Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 20 Oct 2020 11:31:01 -0400 Subject: [PATCH 07/12] Adds "what is pytorch" to homepage --- beginner_source/deep_learning_60min_blitz.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beginner_source/deep_learning_60min_blitz.rst b/beginner_source/deep_learning_60min_blitz.rst index d07d34c0077..12daeed6c61 100644 --- a/beginner_source/deep_learning_60min_blitz.rst +++ b/beginner_source/deep_learning_60min_blitz.rst @@ -8,13 +8,18 @@ Deep Learning with PyTorch: A 60 Minute Blitz -Goal of this tutorial: +What is PyTorch? +~~~~~~~~~~~~~~~~~~~~~ +PyTorch a Python-based scientific computing package for two broad purposes: +- A replacement for NumPy to use the power of GPUs +- A deep learning research platform that provides maximum flexibility and speed +Goal of this tutorial: +~~~~~~~~~~~~~~~~~~~~~~~~ - Understand PyTorch’s Tensor library and neural networks at a high level. - Train a small neural network to classify images -*This tutorial assumes that you have a basic familiarity of numpy* .. Note:: Make sure you have the `torch`_ and `torchvision`_ packages installed. From 0d16b7bb7bae829ab3dd51ae86a4621f8747604c Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 20 Oct 2020 11:37:32 -0400 Subject: [PATCH 08/12] Fixes formatting --- beginner_source/deep_learning_60min_blitz.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/beginner_source/deep_learning_60min_blitz.rst b/beginner_source/deep_learning_60min_blitz.rst index 12daeed6c61..7249abb5519 100644 --- a/beginner_source/deep_learning_60min_blitz.rst +++ b/beginner_source/deep_learning_60min_blitz.rst @@ -10,15 +10,15 @@ Deep Learning with PyTorch: A 60 Minute Blitz What is PyTorch? ~~~~~~~~~~~~~~~~~~~~~ -PyTorch a Python-based scientific computing package for two broad purposes: -- A replacement for NumPy to use the power of GPUs -- A deep learning research platform that provides maximum flexibility and speed +PyTorch is a Python-based scientific computing package for two broad purposes: + +- A replacement for NumPy to use the power of GPUs. +- A deep learning research platform that provides maximum flexibility and speed Goal of this tutorial: ~~~~~~~~~~~~~~~~~~~~~~~~ -- Understand PyTorch’s Tensor library and neural networks at a high - level. -- Train a small neural network to classify images +- Understand PyTorch’s Tensor library and neural networks at a high level. +- Train a small neural network to classify images .. Note:: From 6d91b988d5d8c1d076f0cd500f9aff50f1e275e9 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 20 Oct 2020 12:37:34 -0400 Subject: [PATCH 09/12] Fix typo --- beginner_source/blitz/autograd_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index ace27c0a91e..2b3a3a0c298 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -43,7 +43,7 @@ model = torchvision.models.resnet18(pretrained=True) data = torch.rand(1, 3, 64, 64) labels = torch.rand(1, 1000) -prediction = model(x) # forward pass +prediction = model(data) # forward pass loss = prediction - labels loss.backward() # backward pass From 75efe8b2051b1dd394f64655166bc9f7744644bc Mon Sep 17 00:00:00 2001 From: suraj813 Date: Tue, 20 Oct 2020 14:35:40 -0400 Subject: [PATCH 10/12] Fixes typo --- beginner_source/blitz/autograd_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 2b3a3a0c298..2d2c2377ab2 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -44,7 +44,7 @@ data = torch.rand(1, 3, 64, 64) labels = torch.rand(1, 1000) prediction = model(data) # forward pass -loss = prediction - labels +loss = (prediction - labels).sum() loss.backward() # backward pass optim = torch.optim.SGD(model.parameters()) From adee68469f0d19726ec44365bc16c8d780c2d226 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Wed, 21 Oct 2020 18:34:22 -0400 Subject: [PATCH 11/12] fix failing test --- beginner_source/blitz/autograd_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 2d2c2377ab2..00d9e421369 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -47,7 +47,7 @@ loss = (prediction - labels).sum() loss.backward() # backward pass -optim = torch.optim.SGD(model.parameters()) +optim = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9) optim.step() #gradient descent ###################################################################### From 9dc3c79a9c5773510d43a2aa86e4d26acb1d6977 Mon Sep 17 00:00:00 2001 From: suraj813 Date: Wed, 11 Nov 2020 09:57:03 -0500 Subject: [PATCH 12/12] Make suggested changes Adds a DAG figure for autograd, fixes formatting issues, improve readability --- _static/img/dag_autograd.png | Bin 0 -> 17747 bytes beginner_source/blitz/autograd_tutorial.py | 168 +++++++++++------- beginner_source/blitz/tensor_tutorial.py | 151 ++++++++-------- beginner_source/deep_learning_60min_blitz.rst | 6 +- 4 files changed, 191 insertions(+), 134 deletions(-) create mode 100644 _static/img/dag_autograd.png diff --git a/_static/img/dag_autograd.png b/_static/img/dag_autograd.png new file mode 100644 index 0000000000000000000000000000000000000000..cdc50fed625cc9708919ca3d736eaeed287dca87 GIT binary patch literal 17747 zcmeIaWl&wgwl0c0!8N!;Z~_E(cXtmO2<|Sy0|fU#kl+#|xCRLB5Zv9Jptsq3pMCbd z@5ifCb?g0lRIQ@doZYj#M~^Xjd|!{bB9s-SP>~3cARr)6Wu(PbAs`^-z()@O4qVYU zDT4)HgsjEHlx4)k$dsMzEv#+LAt30!m>C;mN;A^^HZd_a{yqGL9?8jFH9S08)!4tQ zW4Hq`Bpu~PM1K!#3?!tWFf|G*A&yUTZwJCRmV=<7b8g@7Im0e;+$;%uFdX z5L&V}j_h$y(7$B*VI`zzDhAj6lS-bZ0W62rb9g{Yn+Kjb8H!wm6-D5IhJ zP4K%7>OC@*B~>BBoKk>o78TW*3l-OtA^NHV7rQ%0{adnC^$8If8wtd4w-J^i#sT?J zIu!?)Q00;!OT!=8F+pDfF~($-J;c=g+aO;_NwU2P(cXUr2bW-xv#JoHea#0KDiDQ_ zf4fZnOZ_VFVF3jtYyt^03Ulq1@k6g_riCqxiLv2oV*^Ye}a9Gsg1Txk33^Ye4- z;%1Oq4|KtP6BL%@I%B=``54+I2s zd?*Aw_>T!b;yF%)`$1qcg9E0L4FQ@PhKoWflsue^hb# zAV8t5piCxa?_^HK$^4d?l|m4SjEs!m>AeN7s<`C8nuBiw6jm-S4!kTZ?(Xi)?i|ea zPL?ceJUl!stn4i8>`b5rle6bX7ZVSrkIt0;e8_)2N8H@m%*oop#oGQO*~@cHOzmA= z1Slw8I{NqDKgVhAVg27deRTfUw7>*eUhc54F|)G#``Mr=|I1ZgWor*}TWxV`J0Kp= zhad+B8~;D*|3B{hcaQ&Uso`wyBxY|18oCJnw|xKA_&;y{pAG-frOtnJ`Ih@XyZnzk z|7yw4@-p-PkiL*lK_JEj zPzam#zYcu4VT+zqT?-i@@=_7veMZ2_z;NBxaRJK<@+` z6&(4?H$u5(XewbSD&t^?&jqN`D56l{+uP(8_?PE&z=8X(#Bk{rzByUM=76@ve7I;*)fT3ly6L;Rvp>Tb;X}CrQyiiN!VxFP;w53($o3Ug>>*XF=C0Hw$AaUh7AQ?g zGIV7a7;+CA=(*ULZZyc?8rYnL`&|$ffohGOVu20BMTe17El+bB-0b$dDm!a4<`_-x zdeQU!;C>SO;Tm&1hrOWnRAI3GJ7?W(KUUrCmggARM)4k5C$UMs_lGsHgFLf5NMP-3 zF~MUb8?%JGCE?sZGTD*%%gSn$l$iS6jB9(YCG1T+_3vy2YAwg(6%J|k?I@(}D*nA| zLM?D8{&6ny?00PTAOeHwQ*(mpuApr>F5-|1UC3G}}Sz+C-JBcd@75h4`A& z_Hbnk@t%3(Z1@*dpcWpTu$aT9S2O&?@8;nD4d44|iE%Qtl}0H#JBQcZB>>h)env=O zeVavfapRpkux_9|pa}sEgMs8H{vY<0Qqf|~?gKe7<@xocpU-zjX^Ne1buDMNQB@n{ zd19;XfA-0oA>mM4wuRKo#U?BQU&5OHGQ0KuRGugopKEFz)+06RM=K$j1g>)Z76*~g zrnc=)Ek3#S>@>Q9>&Lt1*epRuea)eJNv~GU0pDmLFbGp%U*_xs``h>N;}q2ONB+rlKVOk1Z3m5_q8sqLcoP)w#j=E@(L*DKrQ zD&BIi-|#fwdOgEzd`^)%$9=O!;0s<D~4<~${>@O<%@aw+tbe439IRJU1tZgj1YDRd_Je#TNj;s`Oq>2H`3nHkJHvBt2TFpIef zjuV6rv5Yg!4^zmUfmcU|1Vg|G7?OL@Z9y>LKP+O~FPjJy7!D;wS_;t0a9>czgt?o` zZa#!Zf<_AL1;C%mVaW#af>!uSz;WJEdwzKd+(S)6k3HCwFOwiytdw7^A32o;b}lvDxipY3ii^yz+gV`9IDj}VXu`i&$sA-W_>aJ>6{-P;Qnr< z7-438SZr|oX2$=bNqjJWUX#4n0i#k-{3>Yo#;rg$>HSiZYke0xI_M@1MjI9FhdznV zsmKDG_w}JQA0ssAKUF3m+dOXThu4)&u}WTO`5#9|N7WK_dOeeWn6Et08S+HAYSW(g zH^)nmYDKzrHXYZ)0be}hfkxp&9j+qg(#_X>prw?GK}qd`M&n06WinBy@VdKJR+pG?FrVNr>Ug={PpZ$feqz)1&+;%#c$Lw28LNC zzf11QuC=3py<5IitCBhCjn?(tuz7W)xcdb}TJ;pWC~$8z!+oF2Dpv zl+ZFQv!4oO(EZOekN;DPEQj5E?fEeGN>O{25tSs?!)h>^sk(EC0+beF$a_Zrhcl@y z83TopxpsDoA-dZPZhOO_AYl&CcLmD1qHFRo)?b9D5J1IT*DPMweE;VsmB9cRJi`Wu zTx@22?TXfG-Gr&gbY>-?$J!$vp}ayW7I`Qx#1LCXJHtMVPO#;!hx@Q6P_2rCRdh5h z<`@N<)=wjBmS+1t`bFo&!P+Y_x^AZdcl5H4CVY;@0RAEqM6c{n1%3~=X9)V#4RZQz zJ~~ykEWpE%u?B|UgtBN?2@C$+#NzS3F4L=V*_$X{;5h*Xnf@f4N8LR~o|6EFT3UN`vk=dcK`u zp9})v{_Wum^Vtdm_K%D22;Mp5V6zzN6Wh<%79TD)a%@KbmCXSoWB0jr;B+9G?Ymht z>y7^MJ=;rmB#V#M_x?JmkJMlPa`s)@M3py6Z-Lr(MW^i{f@T#8!dJmPCoLC>JWiY8 ztLCts$cO#3UlYN7#I_~^KDV-=uR=zluZRg%mOR)nS-%MX`bfs?#Wv1^Mv1x%YFXd)sL*pFm?a-$%4oJgGhd~N z3~O}6YR%9$iKr6nv)Alqy(n{4Cc!R_Ouu2Fkz((k?oU$oCcis>l9dR5&Xgx(aDDUW z;1xV;)Ru!}l@tt*#(;*fOiN|~$6 z=l%T&v`HyZX1AAB;t`npZ}3?mxLLyc-LVz^ZdOzx-n6qVitpd_jRH$F_ZSQ_uLQj z0qv=BJuL&$=S$9H79-|cT#{h~RtDYcqXpWfX7`j)!FzeXyFFTw#km^GgGrKwJgL}{ zLO90UsB3xrx4Uq@Mm}c}8jnjq%%k{}wB2xO@H?woAI=Ba-~UDz@q^k6saJVRz34h? z5J_+;kwJw*V&w)atA$UZG$_uYE$s+bl>ij9;*CH`pFZqilb? z&d3qKoHnc#q9EZHRzD+$MZ#8wWAcA`urYeR-!H9vKV70xa)fQ2qsu!~SIEZC&SbR_ zLW>j(9r#70g+Zf)PJqPsQl&1^?B~a&DNp<#T1LJay0uohE!wW9?fwn8Ek^ind{Ju? zOp__3JA+`SNm2NCb!?_f3R>hmGSk@2RVC34jmfgc5_1H03gl7-id6EjN%JA|VsPlM zem-XScSDq)H0nI4(d8L43OAS9ioKVh=tmmf$b9=YuTM z@3B)G;g^?*-51xGBWV=U>CBLzBO1`{evKWV1bzwW!1~mIb$`8JuVER=Yy3BCp9Tf9 zp4G2mxlE^K^K`BISeccz={vq8@wC^Hd_W1kSJPUnbzm+Lo2fX3*x$20($W#k%ia%S z5$}p3yKBU+NQn*64Nzw!u}QdEb+xBb!KHjXrW_x#sqE2(3 zqGdPXy6nSX;(H!ruXw}jcFp}$iFt0Ats;PBZYpib_OSXk#O|{s=tqq@)l(dnWZI%UkjfJr8p3D zY-m@06n<;BBn9y+Ff6QujbGeer1Bw=UM(>!!i4uX@_7JL%X{^MD>+C<=)I7OJHjRf z%)Z)+`PFs&d3*9zq?6Z9s~wPa)6b_r0(Y<9`)2B4IuHgHc_jxLA2|B5dT5SMi~Ex3 z;bW@%n@kldWEA!~Qj9D%I%_B$3jn95&~gL0?GtJzYs8Mahiczljh`SiE_1{$+{<1>-u zszrs}Ro=)mM-^dm3LtiQ^-wK+PRIvOo=ia)ZvqEs^H8YjgT=HPrZbiTT{r!^U4BoYi@x9=i%Tu7@B6djRO~qSfk3bSW;D z1ML+&o*lU_a2W<4zR%Yp>GG%LeBjN%2XT0;dBH2?tlL|UBshL!fri%_2MrDHT=k#D znrSD3b|MR5kJvW&OZIf*ClI4o(?nT$h2;j5nWmL4yYfExg7Z_ zKzVZIEgnPK<|UMxknm@mYP}bem~E8*B02+|m;Ch93ppBg9`%fXf>-;rK4Ml$ z%RJ<48Y7y)t9=dfTQ}TJE}qxAiTWz%w?1N{&Kv_QVgdZk7!TIM(;_xajC)r3jlS5~ zlH%*`SH!$@m8!AcAtc<34#5kj6b`Dqxmj2W7;)SyK5D)nBUZ+cjkd_1u$1_7c?(>m zaAR_1AO*HhBiTZbvg}Sv*2XD>u$>Wz7VE@9uMVl)UHr9w?2J&-#u6uyqq)T02E*rz z99+N9COT)``UFcmBC`VSGVB&36j|A?=}0e6-msP=t|H_+dNLp$tD&EC(-$%X@BC@ejo9dfnQHYjuY+XSkn>nKtYQkqSK zFmV$r9MLT8!IljUF<%hMQlGP6?*!$Gq#bY3njvo(U~wq+^oKPy`;$4=f}=$X2DH)S zjuuW52`E=EYduxtr9klM*@B-A^G(utbY^&e9#$^tRS~pT_bjs^-A~tjnt7{z1ZoCN zRU!8>H2y_}u$^lL5WJT%-bTSh(^UB{Xcib%JitQj|6Ta$ycz$YNI7F!i54krg!Fft zU3S7W71D3$2n3J2;|Rrz7j#As6GQUnyLx|W|MTXQ_s@@C=1y9fM&)nif4o;p)fg!8 z?|tTVdv!8rCor_ag)mn04~Dk`2I?;o?T}DOpOV*s%lR$_!LvT;o`6%!rn~FrZTv+A ziIem-zZPxO8BRIdZ;GDS_}$h@a+fLOt|@u7 zN57+QxY5*uJWU#^L^}6VN|{LttpQhnu2ug^_YVCh7a}0yV0w(H(Cwh`h33xYQr>sg zDivJiJyL!CV9T~S2^h$`m6Vbd$Ag)o@~7FWE$%C&y2qo8IY1)cQx_{z#7Zx!z=(1s zr>1rumkAbUH9p}Zsb#BRA{Y}2$lf9>`WWGvGqSyv_U{a?^gK;Tv^8p1JjEFn6O*LI zN!;;8N9fl!{ee870hO8$71ZxZPZfs=Gto?Aa(p#I_t&apQZ-*6ww;(~dA9dvb0hnR zNhv0a@h=kInc%SRYZy`XWNr6!r=$R`{I}= z_fu}RYG+Qp-IV75G&lsP3XjcRBMl%rFVMs>0(MwX6&`(cESDA63E-BlRKf)Iw2mK* zH8B3z|0cFgD3mAa91IuFf4zwhrT-fjKR!ryuh&eU|En9F;LudViqWiYv&dZ?!C2wO z&hSHPJAbSW5v*kYnw3hV3 z0~f#!+VUg_gMc(E?F?Zv9YDJ{Pk>WJ#S6WU4a&J=X8J%neJ}LByb4Di7;^3lsr89u1GIb>3TP{{ z$aI1idjAs`6Wt|#3g98wzzm7iriSPMyG;kmV!3?{pk>U<$wcZm3rR2lQBWp866ggj z9Zt!k}|2S#cJEQnC=t$H*}5M>nryQ>uJnlI-w zkY`lSn;<%kCyiixA=zl-wj<@DXW1pO$cI?c_Cj=GHc$;~#{hW)5EUbc?XUp0 zgMb}CfmYH26RJ#Zo!8Pbhnl$y~oGUF(-jMu)U_ZJsFC<8Brqh=H-ik{%IA z&8(`&anKgl><(shQ16Mtrl+tR$wW+r>N;F*SFz}3%kw|&Pq&l^!0M{ z`&I*f(LCT7=Iql4?P$ZIL0f5ImW77>W>e-Y(yliB;k+|~ zUfHorg`Ns=%;B#fpi^U!1Ry!AVt}O`MvkR5a>BE4V2KEb5>H9LSW1)0!+;QF=O@F6 zMGmWo8?~4j0mY#3G?`|nSjgaRJpxPBOCoGMJc-2x53QrFT?FQ|Wiy40I!L-7=5R|jr% zgcysPs$^aFW78fDp)6|fLO6OyR3m76`~inaH!*ZVfs$Vc&{5qgWz#wEBpzh)#q0!_ zRZ2PDH^;{vR)pU|QXop#OxCT-;m#(1u!@+o_fr_Rf~hIt^6CYOaV4L$KewTf3NFs{ zsCttvmZc&guLP@!1NMS}USMP5g<%OpBc(ZhMp&f`N5*581`zP~FHvIwf=5~6kfQ*9QOZp9Uk0*8W+-akm-U5-Vo zi14J{EJOEg*2rJL)*Oe+jDYL2={xe}95#KAVsY2DGPMe_x!2iFwrT9Z8!!M<2^@RUydbu-p_!b+2xJKoF$yx|Vq77KjAH_1DcB9&m8KVDXlV5x#w*P@1)xE&vB_ zQA*Le5Tc2ARB;)#b1dQvf4q+{wjKi@iF-VHLKDo@_M}|>U>;^ zw4bH@fnwpOeM(Oh9(ljAwj~r+N@P+ z7%%9n{xKo(cay9$Z~^;bNC(E(zzNCZihjy>+~}RCG7;+sykVq1DRi%q_Yr3&*`8bq zb7N)TStGu*;de#iV!fXqwJS#N$3Am23IW1p*=@m&-Q(1FuEn$FF5|6r{K}Be!&%?4 z18a#4fh<*s#4i%hO^i+=e_d7w0wcn;!e6e{;ghLsW}C&|v*Us7$*i>bZUMlp1?}aU zY5bFw8E})bPFsD(lW3eI(Op^Y%k)w)L>TypzX8uOt*yf-0wzZK8?3bX)5G8WH<`+n zmbyOp1sIf)4JV~|h8`lzJ{Jm&kJk%Zfx{}nvy z3gX}OD3+<2yOWDiAw2oC6Q&|&)4)P#_*X@0r=eA`?fJNt6WNLG>0ZbEt=>0YUom|y z9syq&-tI>FtFvsk$n-CkPY^CC>B&>KV9D;?$!34T4?tAk2+r%0Yhw|bwBQMf;hf*B z_~RMfeM?7G!npe;TMj#nRTQ-T3BsQ6+h^)aDNiftAm^e`F~8e2_&V1vp)CJ0czY)M zL^MMh*7us7twn>UW4U4_K39VEbe0aYeN|}Vi{;j9KcCcb&3Bms2Z&4@{nvUdrWg27 zm&2CD4^O0~wMB=YA5Z*c7wFXri@Ih?HVS`u6}m>SOJ>+RkQrm*1UdtVH5do$udZiHG_3*H}ci10D4k7Ntz zN!|Q0rJ>l3Dy|vJjtGN1s+T4;M2Y5~uda#}s;%j&=@h#Mw9{Z5x!9Cg7(CRhIN@ni zHQ98n_2g9D7*}LrNb_-%oW)3H@nz5aHtY2mjsmm?J(F&&G>PA>3EJb+-2qFna&Ax` zoBAxd=E*XpFvDW!AWV=qBnhdZCRiPHyPqq5_vPpHz?ZhBu z1{Oi%Z>RRRG7OeD@^0i;vjK1Im-X$=Sd`N?_}6oZFO#$kG33RUzDXc^9yjjzc=fiR zi7>v4@ER20bp!jt_h_7wATLEh{=+BSus#ZTR$N`>y3p}YoyfJTEw5sa0GUS(q^Qa* z!23m+iI~eaRjQkJ{@q$c*Q&7bdvvsfvaTqf%Wg+kTL*Sl)Eg%{ z>6l-bZx3MXydWG#ZROK2c%&>wK+CFn_NTCXeLs*WLm=Wo`PJ{P_?#q14}>F$;}Xrx zF5(G6*O@8I1`L3m#oR=-4tPUDFPQ)h6AED+fM+Q0G1f|JG-mzx+)#s}c$c*8`*N*y zf|nz?o>e!@>)P_}14(C?0qmoQTkrJb|K#DWy}D z9AQWexT14Cb6~Uk5@?keH)sP;EK)A-Y!KDA5rUn54We(MW}O8|G;>^DW_NzQ$t_HO zu$vN)D~fhwM?fK4jxS>9D9WjNL_`VKX!)FQm07t`^s$r71bGAg1e0S++gsYi*hT5A zGZp)@}X}J&=U)F`l;{eJm^uL(IQ^VCSIl5(d857gOcH%0Jy~CBU8*9IU zBg4=Q(MrNtT~^^Xib-F2#O#W9z|a;|beoB?R3=M)Cfxff!)4I%`@2P;dyJE?H?`s7 zt^>@QDeW;g1SN$Odc%@bL{jtEaa{CA_MTn%@awU&uCR`pkiKw5PWw(Ds0Q4DpMZ$n ze>(tK!g>`q_ ze3D43QLf85?Po@8_~mdBHay@pAvgIG*(sUBWNvzpiB#MR`6?E=tm=Wns*4{3InG^E zH;LF@evc6L&SN9b=Fo1{kz|=8o$NHgIQ3P_2YeOVwX7W7N>a%XDJcjP>QJ@L3}GD9 zZ#;V+j74FYbhYdp9QB1BJUUjdgk4@&%{xAJi@iGIE|bfWhkb5)IG2lkC6z^8mIpJw zoGWNpW1%oDo{Ap6d)t3nM$>Sm~JqCV2PZ?EQ52rVDU*v|>@RZe_j5WGJ6`>4L*J?nj&>6DIzkDMpFb#!=OLk%;4?X zW_o^=S$jUuaKaL?Xp|)K5*H1PD0^E8(V)5JYgO>&VMT=_d46Gh8D)8* z$yw^B2?gJLbEo}5aC8aZa!V0!9z43;C-k?*2k-6(=aGMTE-ggB`68geEDtp8qjtlN z_rfd_dj+d#h}FrWVToQ5OCsBq?tT(?B|yDrz4lNhyXy_x38cm)E7Cm(Qy_MD?2dx6 z?sS$&QuJnc7i-g`VO(XRJ$@fU;;jBib}BpWhB%5HWT?Z$q*t#lG)@!!ZZ-Hpcx*D( z&;ws?Hy}(8nU`6){FfGWv4V1&)pcObkPo6Hbj}h{OQ!;I>RN%#qcD`oxNYz~;s!N& z5v1^?K!MxS{r9lYy_C1O^r{-B|Zy@d%s4>T8e>CmpM!B{`^VOsdj6X>GM9Sm*Y5nVD<_)?I{VG zmBb{*u|k(>CTme{Idio0aAK`$$X$c&N(c%I3nPZw2pK`Ez1YhjnYL5+UGY8<=&T~R zEE4;!_pwo;CvdGpTinC!mJxmLn;1g7V{mtZFg#K;$7R>q>yHg0doeJ^K0Cu1-}!~` zwJ*hi*BXSJe*0E{OaQ?Un|1e;!gT>_{>SQ^ju8<8ax9i-fD|%uv)fTgMesi9JpLE1 z22;}L)hy!x9z*8vuoUEyaO3dHT)b$$KN}`}JT5jlc*J{oeTG7{4Q<4o-Nk&nmr@XJ z>#$gGl<;FuB@5X{`GMOAEq_j(0y~g{t0#yVXWl9o(q)Y2NtLFMrdy05hSS6Ww5=0d z;%3J>DTxG!eZzWexg;5#4iWlyb-CNVrG@Pm!Uvr-oq)68KuP#4k|+Y8hg=Z{Yb0S9 zeZJx6p`E{G`ftdQ;3>6FAJzn4X>|QCCI}r9rpH#Kg6)M2Mdrr(==QLe!LIMMD$)s^ zW3PY7>-`$d!*LhlTriu6*myOk`6IlTzDI=gte%T=z(^@eR?Eh&Z;M9c zE3ur(S;wP~Yn&!Nj3_mgjz8QctFc}wz!>KXxbry%AqL+=NaC`PovaD*3M|IbA-_@c zH!s@wvoufdJe9o4cE2zE!#iurg_L6SbV|kfvSN5{%LLU@+`d60ck>|>auQ82I(mAH ziZ)Bwq_q{^2yT3(8;j21#F_Q?^7^1MMf6XMi-H7qT`!{>AG4J^<%5eqUC7@(Fgtv6 z7HR}W^_RLIcZa8p2n;`)pQL25|BMN!O0ij2Zb8H17?=7sV^NH=3euiHO4m!8Bx53` zD-}qKcp<2Y@E~zsvW{Q@LaJ7lo%#Z(nSDU2#MmzD&jPK1NSa4a3>8RgA-TL87c9_s>3K$qz(eqMr3D)%(*kh0BDT+5L2t z?U;&h*B+7j-HVCEqueD1QG+UQF~niR%Oa02FOx28)Aa0hb6k!>-R9hv&x~Dz0FoVs z2NZ2T1jEC~Nf@Kz`kZ7RwPv`t2RRGGT3TU^EfM)^qcuc;$@y-J1==m{=myuon;oau%7m(3TY zY;kIvmslh~%v?z}nwkhEa`gBB!Na9sBxiS2F%HE)j)|ca6I4N$7kqTkEK2_bk^Dv2 z^Bbc|8=Yn3U$88wER;iSpIv)YsQW~f+n3z-& zV`i;e1{IEjUrk6w&4pmRXIl0v+&s3LqWC7AxB3gRi$s&!>3Up#oZaUo3h8gZxE{Vp2}VHk~kQ@e3KsKqo%*2{AaLUS9nMWN~~&a>B{b`2#>RL)771P3e!-%*1M)n6}!& z>Prb~xY(n`Mm2$a{8+?bohFw({2>Y3QX)LO?*kAUv?_1_@5dgYsCc76=~oRwOC27s zcK$+HKp(1qwILHh1bGVQjxZ1y%I#372xzQcL-h1g5X82tD9fuO*XxO9#7BwNjClr7 z^bWq4;Ts9RU(jTw?8; z8cLq{3GP0s_?`iyia+gFD&9Fqf7oxQ1{|{^!A21q1S#I(C0nQCMIq`YP={m6>gygG3l?XbMR<__r z*=I~Aw0m_UU~^g8lmOnUoTmNazT~QfPR|-i6Y!VrjOhvBmL7tyZ?VH{tr>|`$4)~5 zRxdAf{V-jUB<4iCY`ZQIY=J7j7}ZEmKLU85y`1x2`Gc?iPmSm73$@miDJ({A%E!T@ z`+#$UX<-!;xIHt+;vdwG0V+LMfERMCP4!_SUuIzNaHc#%ztM?0Ts;#D)`!>qxXdH) zHD0u{;+*$RVpMVV51LjQw!%b32Vyj+WWMAC=9z{||{1mxB@4ZA(35;=#y(BxU~@?NsG2+IVm#>o1! z_#nm_k6iqP$&pa!s|AS-SL;!b0A>v!LoI3?_$bZ(k zbpz3M$t3)d%PDKtpENd*P9yU4#r-rVE+7ez?@6J-{@+0=qu|r6*-H)^Ae$sWcG_-{ z2|LJxYh!zhK-3W|3Z)4QQIqi1a~7AKE?^2HGv7Ii^12-sdG!On=_qBO+N@8#FKz`Q zG*enu$!9L)q26j-bc>XTj%rmQ!2J_9M%c-BxfDf<;q(Hqo-g@5{1bSXC|3SES?)er z?8o?*KC6N7KKoZkKUa{>BdibO>^HxPLJ1-^v?=Gr2~$7;KDu!|JzzsN6osIo!2;su z-3mB}_PKWMsS8?50*tg*CWjsToru!z0sAS*KDiO|M$vi1gQ_$cI;79BBsnE9+(ey zR2qr!o#~vbfuuKsZNB$D6AqsCL0s~aFs<`X^9;zijC9Wr7qOee8MANeieqzO{&bpI zb*6s9^@?8r?Ded&=qMn}QomJ%k=VCC*7vo!T7ne0i99JZktaHEKmZ{i$CpS%2p0+y zMcpig;h=PP`c%TfplC z$5g7@?WK_prxt`$(VQ@A)xET06iEDyAWv)#F?-!4r}=|+ z+)>2mFG_sN&XI3Mknw?bP3*2mWIGk9=HU@!aZ5yE%cx>_IG%3Hqoaua+|Zyf`b~On zt%?kaQ&8TWdj_)3qY9|kfz6`sfeI!OeXir)C==XY3~Un2(|b*hKkq|(O*IXo6*uNj3T zGkC?pq(SZVuyq)yL)6$az`5T>M|+wEKJdG?z8}a|lvV%DGCRvc;r%2)6Wnh1;f*_f z3H!#8(EIV|VRo0!R@HKuGx0OOugw~TXwPDbL_7zADa^jQ{mFa@>UR3_B`d0;4BRfR z!C@j#hClS$NNOaLed=ULymvIMm-A%tRZ)#FS9}{Z{n9UE%M5De+ISYU*x$=)3*vc= z5>>0l9P%+)PZoZ$Sb8&fw8Ze~q)VQ@fm6WM)W zw{M&F%^DInb|U8N4wfT+>JCaI6h)E{FP&2kwL6ixwzv?1d^~Ob+D6Uj5t}g?4&> zQY`+xj@De@%aR7psILNz>uk1U&rK5S=EG!CH(Bu*Xd?*~=0b_n$ZjPUau#-)_pJx=vWfRbZGY&9k=kivXEGUY za0uRzwl`3z;GaOhX^ToQgCCJelI&R|!Rs0#ZkEgj8|&fiPL=G=7s3RCwF~gFjt1);{ObWBk67RHG&B zq|9Z@JTk}kCM(QO-E%VXcxBrQBddi}v z74Bm`Ff?-!CkgGGDDSMb3QXT3@7;7BO;`&XczT&~m^%o0`F-KFe}osi-k|kw?s}`0 zrRjcM!eCy48+Q*srPUdkI84oRVg5zXETiOcarS*uyj8HPOs*b4A!}Sn-doxXcFeh#j=Xh+oNf2lqQ&nXniFW38_i@2 zMsl*6MI=sKzz3P}1l^zDr=7P(DJuQlW7)BHQCC`Q+RtZj{n|RSIF|Gd1Xu4MD<@Mf zJ!-xrR%$3$l&eXT5PL=;{pg5Kj!`?fR(0}i?Ft$puQjX4B&8LS*WbCet=Z{kM0;g- zm@U7t`veo~v=d6n5K$cLucqN{SBv3y%R-zVrUgHKJlwb0SU6d55#@??#kjJ}nfsk! znUzXvd1yeoOKqBj(Pd)Uoh(b;`^29mYu-IEIn-`Yu5ZUa(W`ubK?$o@$2-!hre!bt zV2bnXM7`x?P2sL5kV{+k@CCs?vSYijcPykgXe_POoZDr?})#Ntm%5iNY>+7Iig*DVfDW^QE|2{xkTkYz?PQrWleCZz_8FSt-+$0o{7zG} z`K(#Z#@`s#xw+SuJ(%c8H+_KqWOyPRPwfp??C##>lX;+T7oscK7|QDV)a#_-+xA@G zbyqc?&F`k`_iXVYzIDfnJpjdquWtv!^@y`8A^Um;iQrvcUCFmRyYuh;&vd~RpUEZ= z^RY&}2hX|@i6xqbCqddfa*`}Qxx@Ptnb=9znYi`Yt*dt5nm-`J-i5!N#Q^x^Mik~zEO<<}DDl5IO`D&Nprr~UR(qR~baUO3m4 zM-SBL4}yiGP&i5;8P@oK)~2LoamU8Yg~{lc1SW=ow`3nA#mb_T~@U3`Bkk72sC4JI^nd6sj<$=Iq$fq}6mKZ%WPQK)-U8vKAa2{X}%>qLk@ z$s-LrKq}y+zU+HrF#S~M?RfP5VEyml5073rt>#sFb5^aV27^%dXAxP$6eGA{8>Uka zG93f4Xk#B`jmN(i{`t0bK~4%0L=_upQ$y>sOWNYDF*zLOci}ic{Pqn09tYR$C$y#% zu2Ao8^_=Elpgk3}aOd;fJaV$z-8%h!xY}~+;54j>DOCeb(OvfJ16E1qZTjv>PqX>w z4-FG=v+o9*Ny?dN^)+-oGXF+x$627MCySap4Gw%=e?Q$q%;2j&ZGWPvLvW7x{8O6X zxS3?-3arNzb_dFSbH`%*#BNJ%KLmgJCR#Ft@_&0sF zh$Tl>gPttUdo;y0>ZW1diW`|Xa{jWo0HtB3a|G9rEmZ2|C5H3OeluF6Zjd@QyBYzp zna`X%yW05!^Yc8lhlNZ}r>DA>Qp0@Dsz7JL+=YG12LuHRbNy8lPvp=G<9Em<`@244 zVtQq3$~e6p=Pea1EVY83R)(Q)6zzE#Z%e(k~k<+lLpD^HjEjOnV&U}jg( zXt0(ZOAWlnr-QZ7f*?x5o6;i;4<|Iv;pKbo9PZf@!%>0@7WSZMEH3i75)oc)m>k(@ z_339DKbZ#d#U1JocoB!7)#Ao-ma%~JUbLnUkL8T5-8*mM$M}wYGw-Vv0_>8j4g99E z0e312Y>g5oJq2W=gxkH4RsUD66D59B{#$-$-MktLejUL06uAfYT2(>S9b7gnqe1olHO>1>*w%CA8>n;7t|gu1NiQ6SuB qe2D~-;OoF(SrY;F|G!cFLC8Pvp2~{EZ}#QqHDn|d#VbUOgZ>v0X1GQG literal 0 HcmV?d00001 diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 00d9e421369..53659225d82 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- """ -A Gentle Introduction to Autograd +A Gentle Introduction to ``torch.autograd`` --------------------------------- -Autograd is PyTorch’s automatic differentiation engine that powers +``torch.autograd`` is PyTorch’s automatic differentiation engine that powers neural network training. In this section, you will get a conceptual -understanding of how autograd works under the hood. +understanding of how autograd helps a neural network train. Background ~~~~~~~~~~ @@ -24,34 +24,57 @@ proportionate to the error in its guess. It does this by traversing backwards from the output, collecting the derivatives of the error with respect to the parameters of the functions (*gradients*), and optimizing -the parameters using **gradient descent**. For a more detailed walkthrough +the parameters using gradient descent. For a more detailed walkthrough of backprop, check out this `video from 3Blue1Brown `__. -Most deep learning frameworks use automatic differentiation for -backprop; in PyTorch, it is handled by Autograd. + Usage in PyTorch ~~~~~~~~~~~ -Backward propagation can be kicked off by calling ``.backward()`` on the error tensor. -This collects the gradients for each parameter in the model. +Let's take a look at a single training step. +For this example, we load a pretrained resnet18 model from ``torchvision``. +We create a random data tensor to represent a single image with 3 channels, and height & width of 64, +and its corresponding ``label`` initialized to some random values. """ - import torch, torchvision - model = torchvision.models.resnet18(pretrained=True) data = torch.rand(1, 3, 64, 64) labels = torch.rand(1, 1000) + +############################################################ +# Next, we run the input data through the model through each of its layers to make a prediction. +# This is the **forward pass**. +# + prediction = model(data) # forward pass + +############################################################ +# We use the model's prediction and the corresponding label to calculate the error (``loss``). +# The next step is to backpropagate this error through the network. +# Backward propagation is kicked off when we call ``.backward()`` on the error tensor. +# Autograd then calculates and stores the gradients for each model parameter in the parameter's ``.grad`` attribute. +# + loss = (prediction - labels).sum() loss.backward() # backward pass +############################################################ +# Next, we load an optimizer, in this case SGD with a learning rate of 0.01 and momentum of 0.9. +# We register all the parameters of the model in the optimizer. +# + optim = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9) + +###################################################################### +# Finally, we call ``.step()`` to initiate gradient descent. The optimizer adjusts each parameter by its gradient stored in ``.grad``. +# + optim.step() #gradient descent ###################################################################### -# At this point, you have everything you need to build your neural network. +# At this point, you have everything you need to train your neural network. # The below sections detail the workings of autograd - feel free to skip them. # @@ -64,58 +87,62 @@ ###################################################################### # Differentiation in Autograd # ~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# The ``requires_grad`` flag lets autograd know -# if we need gradients w.r.t. these tensors. If it is ``True``, autograd -# tracks all operations on them. - +# Let's take a look at how ``autograd`` collects gradients. We create two tensors ``a`` and ``b`` with +# ``requires_grad=True``. This signals to ``autograd`` that every operation on them should be tracked. +# import torch a = torch.tensor([2., 3.], requires_grad=True) b = torch.tensor([6., 4.], requires_grad=True) +###################################################################### +# We create another tensor ``Q`` from ``a`` and ``b``. +# +# .. math:: +# Q = 3a^3 - b^2 + Q = 3*a**3 - b**2 -print(Q) ###################################################################### -# ``a`` and ``b`` can be viewed as parameters of an NN, with ``Q`` -# analogous to the error. In training we want gradients of the error +# Let's assume ``a`` and ``b`` to be parameters of an NN, and ``Q`` +# to be the error. In NN training, we want gradients of the error # w.r.t. parameters, i.e. # # .. math:: -# -# # \frac{\partial Q}{\partial a} = 9a^2 # # .. math:: -# -# # \frac{\partial Q}{\partial b} = -2b # -# Since ``Q`` is a vector, we pass a ``gradient`` argument in -# ``.backward()``. # -# ``gradient`` is a tensor of the same shape. Here it represents the +# When we call ``.backward()`` on ``Q``, autograd calculates these gradients +# and stores them in the respective tensors' ``.grad`` attribute. +# +# We need to explicitly pass a ``gradient`` argument in ``Q.backward()`` because it is a vector. +# ``gradient`` is a tensor of the same shape as ``Q``, and it represents the # gradient of Q w.r.t. itself, i.e. # # .. math:: -# -# # \frac{dQ}{dQ} = 1 # - +# Equivalently, we can also aggregate Q into a scalar and call backward implicitly, like ``Q.sum().backward()``. +# external_grad = torch.tensor([1., 1.]) Q.backward(gradient=external_grad) -# check if autograd's gradients are correct + +####################################################################### +# Gradients are now deposited in ``a.grad`` and ``b.grad`` + +# check if collected gradients are correct print(9*a**2 == a.grad) print(-2*b == b.grad) ###################################################################### -# Optional Reading - Vector Calculus in Autograd +# Optional Reading - Vector Calculus using ``autograd`` # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # Mathematically, if you have a vector valued function @@ -192,35 +219,39 @@ # tensors. By tracing this graph from roots to leaves, you can # automatically compute the gradients using the chain rule. # -# In a forward pass, autograd does two things simultaneously: \* run the -# requested operation to compute a resulting tensor, and \* maintain the -# operation’s *gradient function* in the DAG. This is stored in the -# resulting tensor’s .\ ``grad_fn`` attribute. +# In a forward pass, autograd does two things simultaneously: +# +# - run the requested operation to compute a resulting tensor, and +# - maintain the operation’s *gradient function* in the DAG. # # The backward pass kicks off when ``.backward()`` is called on the DAG -# root. Autograd then \* computes the gradients from each ``.grad_fn``, \* -# accumulates them in the respective tensor’s ``.grad`` attribute, and \* -# using the chain rule, propagates all the way to the leaf tensors. +# root. ``autograd`` then: +# +# - computes the gradients from each ``.grad_fn``, +# - accumulates them in the respective tensor’s ``.grad`` attribute, and +# - using the chain rule, propagates all the way to the leaf tensors. +# +# Below is a visual representation of the DAG in our example. In the graph, +# the arrows are in the direction of the forward pass. The nodes represent the backward functions +# of each operation in the forward pass. The leaf nodes in blue represent our leaf tensors ``a`` and ``b``. +# +# .. figure:: /_static/img/dag_autograd.png # -# .. Note:: -# **Autograd DAGs are dynamic in PyTorch** +# .. note:: +# **DAGs are dynamic in PyTorch** # An important thing to note is that the graph is recreated from scratch; after each # ``.backward()`` call, autograd starts populating a new graph. This is # exactly what allows you to use control flow statements in your model; # you can change the shape, size and operations at every iteration if -# needed. Autograd does not need you to encode all possible paths -# beforehand. +# needed. # - - -###################################################################### # Exclusion from the DAG # ^^^^^^^^^^^^^^^^^^^^^^ # -# Autograd tracks operations on all tensors which have their +# ``torch.autograd`` tracks operations on all tensors which have their # ``requires_grad`` flag set to ``True``. For tensors that don’t require # gradients, setting this attribute to ``False`` excludes it from the -# gradient computation DAG and increases efficiency. +# gradient computation DAG. # # The output tensor of an operation will require gradients even if only a # single input tensor has ``requires_grad=True``. @@ -231,36 +262,51 @@ z = torch.rand((5, 5), requires_grad=True) a = x + y -print("Does `a` require gradients?") -print(a.requires_grad==True) +print(f"Does `a` require gradients? : {a.requires_grad}") b = x + z -print("Does `b` require gradients?") -print(b.requires_grad==True) +print(f"Does `b` require gradients?: {b.requires_grad}") ###################################################################### -# This is especially useful when you want to freeze part of your model -# (for instance, when you’re fine-tuning a pretrained model), or you know -# in advance that you’re not going to use gradients w.r.t. some -# parameters. +# In a NN, parameters that don't compute gradients are usually called **frozen parameters**. +# It is useful to "freeze" part of your model if you know in advance that you won't need the gradients of those parameters +# (this offers some performance benefits by reducing autograd computations). # +# Another common usecase where exclusion from the DAG is important is for +# `finetuning a pretrained network `__ +# +# In finetuning, we freeze most of the model and typically only modify the classifier layers to make predictions on new labels. +# Let's walk through a small example to demonstrate this. As before, we load a pretrained resnet18 model, and freeze all the parameters. from torch import nn, optim model = torchvision.models.resnet18(pretrained=True) +# Freeze all the parameters in the network for param in model.parameters(): param.requires_grad = False -# Replace the last fully-connected layer -# Parameters of nn.Module instances have requires_grad=True by default -model.fc = nn.Linear(512, 100) -# Optimize only the classifier -optimizer = optim.SGD(model.fc.parameters(), lr=1e-2, momentum=0.9) +###################################################################### +# Let's say we want to finetune the model on a new dataset with 10 labels. +# In resnet, the classifier is the last linear layer ``model.fc``. +# We can simply replace it with a new linear layer (unfrozen by default) +# that acts as our classifier. +model.fc = nn.Linear(512, 10) ###################################################################### -# The same functionality is available as a context manager in +# Now all parameters in the model, except the parameters of ``model.fc``, are frozen. +# The only parameters that compute gradients are the weights and bias of ``model.fc``. + +# Optimize only the classifier +optimizer = optim.SGD(model.fc.parameters(), lr=1e-2, momentum=0.9) + +########################################################################## +# Notice although we register all the parameters in the optimizer, +# the only parameters that are computing gradients (and hence updated in gradient descent) +# are the weights and bias of the classifier. +# +# The same exclusionary functionality is available as a context manager in # `torch.no_grad() `__ # diff --git a/beginner_source/blitz/tensor_tutorial.py b/beginner_source/blitz/tensor_tutorial.py index 4fc9b13d120..a949f205d8b 100644 --- a/beginner_source/blitz/tensor_tutorial.py +++ b/beginner_source/blitz/tensor_tutorial.py @@ -7,7 +7,7 @@ outputs of a model, as well as the model’s parameters. Tensors are similar to NumPy’s ndarrays, except that tensors can run on -a GPU to accelerate computing. If you’re familiar with ndarrays, you’ll +GPUs or other specialized hardware to accelerate computing. If you’re familiar with ndarrays, you’ll be right at home with the Tensor API. If not, follow along in this quick API walkthrough. @@ -21,51 +21,48 @@ # Tensor Initialization # ~~~~~~~~~~~~~~~~~~~~~ # -# Tensors can be initialized in various ways. Take a look at the following examples +# Tensors can be initialized in various ways. Take a look at the following examples: # -# **Directly from data / NumPy arrays:** -# Tensors can be created directly by passing a Python list or sequence using the ``torch.tensor()`` constructor. The data type is automatically inferred from the data. +# **Directly from data** +# +# Tensors can be created directly from data. The data type is automatically inferred. data = [[1, 2],[3, 4]] -np_array = np.array(data) +x_data = torch.tensor(data) -tnsr_from_data = torch.tensor(data) -tnsr_from_np = torch.from_numpy(np_array) +###################################################################### +# **From a NumPy array** +# +# Tensors can be created from NumPy arrays (and vice versa - see :ref:`bridge-to-np-label`). +np_array = np.array(data) +x_np = torch.from_numpy(np_array) ############################################################### # **From another tensor:** -# The new tensor retains the properties of the arg tensor, unless explicitly overridden. - -new_ones_tnsr = torch.ones_like(tnsr_from_data) # 2 x 2 matrix of ones - -try: - new_rand_tnsr = torch.rand_like(tnsr_from_data) # 2 x 2 matrix of random numbers -except RuntimeError as e: - print(f"RuntimeError thrown: {e}") - print() - print(f"Random values in PyTorch are floating points. Datatype passed to torch.rand_like(): {tnsr_from_data.dtype}") - - -###################################################################### -# This works after we override the dtype property # +# The new tensor retains the properties (shape, datatype) of the argument tensor, unless explicitly overridden. + +x_ones = torch.ones_like(x_data) # retains the properties of x_data +print(f"Ones Tensor: \n {x_ones} \n") -new_rand_tnsr = torch.rand_like(tnsr_from_data, dtype=torch.float) -print(new_rand_tnsr) +x_rand = torch.rand_like(x_data, dtype=torch.float) # overrides the datatype of x_data +print(f"Random Tensor: \n {x_rand} \n") ###################################################################### # **With random or constant values:** # +# ``shape`` is a tuple of tensor dimensions. In the functions below, it determines the dimensionality of the output tensor. shape = (2,3,) -rand_tnsr = torch.rand(shape) -ones_tnsr = torch.ones(shape) -zeros_tnsr = torch.zeros(shape) -print(f"Random Tensor:\n{rand_tnsr} \n\n \ - Ones Tensor:\n{ones_tnsr} \n\n \ - Zeros Tensor:\n{zeros_tnsr}") +rand_tensor = torch.rand(shape) +ones_tensor = torch.ones(shape) +zeros_tensor = torch.zeros(shape) + +print(f"Random Tensor: \n {rand_tensor} \n") +print(f"Ones Tensor: \n {ones_tensor} \n") +print(f"Zeros Tensor: \n {zeros_tensor}") @@ -79,13 +76,13 @@ # Tensor Attributes # ~~~~~~~~~~~~~~~~~ # -# Tensor attributes describe their shape data type and where they live. +# Tensor attributes describe their shape, datatype, and the device on which they are stored. -tnsr = torch.rand(3,4) +tensor = torch.rand(3,4) -print(f"Shape of tensor: {tnsr.shape}") -print(f"Datatype of tensor: {tnsr.dtype}") -print(f"Device tensor lives on: {tnsr.device}") +print(f"Shape of tensor: {tensor.shape}") +print(f"Datatype of tensor: {tensor.dtype}") +print(f"Device tensor is stored on: {tensor.device}") ###################################################################### @@ -109,47 +106,56 @@ # We move our tensor to the GPU if available if torch.cuda.is_available(): - tnsr = tnsr.to('cuda') + tensor = tensor.to('cuda') ###################################################################### -# Try out some of the operations from the list. If you're familiar with the NumPy API, you'll find the Tensor API a breeze to use. +# Try out some of the operations from the list. +# If you're familiar with the NumPy API, you'll find the Tensor API a breeze to use. # ############################################################### # **Standard numpy-like indexing and slicing:** -tnsr = torch.ones(4, 4) -tnsr[:,1] = 0 -print(tnsr) +tensor = torch.ones(4, 4) +tensor[:,1] = 0 +print(tensor) ###################################################################### -# **Both of these are joining ops, but they are subtly different.** - -t1 = torch.cat([tnsr, tnsr], dim=1) -t2 = torch.stack([tnsr, tnsr], dim=1) +# **Joining tensors** You can use ``torch.cat`` to concatenate a sequence of tensors along a given dimension. +# See also `torch.stack `__, +# another tensor joining op that is subtly different from ``torch.cat``. +t1 = torch.cat([tensor, tensor, tensor], dim=1) print(t1) -print() -print(t2) ###################################################################### -# **Multiply op - multiple syntaxes** +# **Multiplying tensors** + +# This computes the element-wise product +print(f"tensor.mul(tensor) \n {tensor.mul(tensor)} \n") +# Alternative syntax: +print(f"tensor * tensor \n {tensor * tensor}") -# both ops compute element-wise product -print(tnsr * tnsr == tnsr.mul(tnsr)) +###################################################################### +# +# This computes the matrix multiplication between two tensors +print(f"tensor.matmul(tensor.T) \n {tensor.matmul(tensor.T)} \n") +# Alternative syntax: +print(f"tensor @ tensor.T \n {tensor @ tensor.T}") -# both ops compute matrix product -print(tnsr @ tnsr.T == tnsr.matmul(tnsr.T)) ###################################################################### -# .. note:: -# Operations that have a '_' suffix are in-place. For example: -# ``x.copy_(y)``, ``x.t_()``, will change ``x``. In-place operations -# don't work well with Autograd, and their use is discouraged. +# **In-place operations** +# Operations that have a ``_`` suffix are in-place. For example: ``x.copy_(y)``, ``x.t_()``, will change ``x``. -print(tnsr) -tnsr.t_() -print(tnsr) +print(tensor, "\n") +tensor.add_(5) +print(tensor) + +###################################################################### +# .. note:: +# In-place operations save some memory, but can be problematic when computing derivatives because of an immediate loss +# of history. Hence, their use is discouraged. ###################################################################### # -------------- @@ -157,6 +163,8 @@ ###################################################################### +# .. _bridge-to-np-label: +# # Bridge with NumPy # ~~~~~~~~~~~~~~~~~ # Tensors on the CPU and NumPy arrays can share their underlying memory @@ -166,24 +174,27 @@ ###################################################################### # Tensor to NumPy array # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -a = torch.ones(5) -print(f"a: {a}") -b = a.numpy() -print(f"b: {b}") +t = torch.ones(5) +print(f"t: {t}") +n = t.numpy() +print(f"n: {n}") ###################################################################### -# A change in ``a`` reflects in ``b`` +# A change in the tensor reflects in the NumPy array. -a.add_(1) -print(f"a: {a}") -print(f"b: {b}") +t.add_(1) +print(f"t: {t}") +print(f"n: {n}") ###################################################################### # NumPy array to Tensor # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -a = np.ones(5) -b = torch.from_numpy(a) -np.add(a, 1, out=a) -print(f"a: {a}") -print(f"b: {b}") +n = np.ones(5) +t = torch.from_numpy(n) + +###################################################################### +# Changes in the NumPy array reflects in the tensor. +np.add(n, 1, out=n) +print(f"t: {t}") +print(f"n: {n}") diff --git a/beginner_source/deep_learning_60min_blitz.rst b/beginner_source/deep_learning_60min_blitz.rst index 7249abb5519..4fc156c08ce 100644 --- a/beginner_source/deep_learning_60min_blitz.rst +++ b/beginner_source/deep_learning_60min_blitz.rst @@ -10,10 +10,10 @@ Deep Learning with PyTorch: A 60 Minute Blitz What is PyTorch? ~~~~~~~~~~~~~~~~~~~~~ -PyTorch is a Python-based scientific computing package for two broad purposes: +PyTorch is a Python-based scientific computing package serving two broad purposes: -- A replacement for NumPy to use the power of GPUs. -- A deep learning research platform that provides maximum flexibility and speed +- A replacement for NumPy to use the power of GPUs and other accelerators. +- An automatic differentiation library that is useful to implement neural networks. Goal of this tutorial: ~~~~~~~~~~~~~~~~~~~~~~~~