fdsa

Author: mcarilli

recipes_source/recipes/amp_recipe.py

@@ -169,7 +169,7 @@ def make_model(in_size, out_size, num_layers):
         opt.zero_grad()
 
 ##########################################################
-# All together ("Automatic Mixed Precision")
+# All together: "Automatic Mixed Precision"
 # ------------------------------------------
 # (The following also demonstrates ``enabled``, an optional convenience argument to ``autocast`` and ``GradScaler``.
 # If False, ``autocast`` and ``GradScaler``\ 's calls become no-ops.
@@ -270,6 +270,9 @@ def make_model(in_size, out_size, num_layers):
 # * Multiple GPUs (``torch.nn.DataParallel`` or ``torch.nn.parallel.DistributedDataParallel``)
 # * Custom autograd functions (subclasses of ``torch.autograd.Function``)
 #
+# If you perform multiple convergence runs in the same script, each run should use
+# a dedicated fresh GradScaler instance.  GradScaler instances are lightweight.
+#
 # If you're registering a custom C++ op with the dispatcher, see the
 # `autocast section <https://pytorch.org/tutorials/advanced/dispatcher.html#autocast>`_
 # of the dispatcher tutorial.