Merge branch 'master' into move_attention_out

Author: brianjo

intermediate_source/ddp_tutorial.rst

@@ -2,6 +2,8 @@ Getting Started with Distributed Data Parallel
 =================================================
 **Author**: `Shen Li <https://mrshenli.github.io/>`_
 
+**Edited by**: `Joe Zhu <https://github.com/gunandrose4u>`_
+
 Prerequisites:
 
 -  `PyTorch Distributed Overview <../beginner/dist_overview.html>`__
@@ -68,6 +70,7 @@ be found in
 .. code:: python
 
     import os
+    import sys
     import tempfile
     import torch
     import torch.distributed as dist
@@ -79,12 +82,26 @@ be found in
 
 
     def setup(rank, world_size):
-        os.environ['MASTER_ADDR'] = 'localhost'
-        os.environ['MASTER_PORT'] = '12355'
-
-        # initialize the process group
-        dist.init_process_group("gloo", rank=rank, world_size=world_size)
+        if sys.platform == 'win32':
+            # Distributed package only covers collective communications with Gloo
+            # backend and FileStore on Windows platform. Set init_method parameter
+            # in init_process_group to a local file.
+            # Example init_method="file:///f:/libtmp/some_file"
+            init_method="file:///{your local file path}"
+
+            # initialize the process group
+            dist.init_process_group(
+                "gloo",
+                init_method=init_method,
+                rank=rank,
+                world_size=world_size
+            )
+        else:
+            os.environ['MASTER_ADDR'] = 'localhost'
+            os.environ['MASTER_PORT'] = '12355'
 
+            # initialize the process group
+            dist.init_process_group("gloo", rank=rank, world_size=world_size)
 
     def cleanup():
         dist.destroy_process_group()