Merge pull request #1039 from Lyken17/master

Fix allreduce implmentation in tutorial

Author: Jessica Lin

intermediate_source/dist_tuto.rst

@@ -394,29 +394,28 @@ using point-to-point collectives.
 
     """ Implementation of a ring-reduce with addition. """
     def allreduce(send, recv):
-        rank = dist.get_rank()
-        size = dist.get_world_size()
-        send_buff = th.zeros(send.size())
-        recv_buff = th.zeros(send.size())
-        accum = th.zeros(send.size())
-        accum[:] = send[:]
-
-        left = ((rank - 1) + size) % size
-        right = (rank + 1) % size
-
-        for i in range(size - 1):
-            if i % 2 == 0:
-                # Send send_buff
-                send_req = dist.isend(send_buff, right)
-                dist.recv(recv_buff, left)
-                accum[:] += recv[:]
-            else:
-                # Send recv_buff
-                send_req = dist.isend(recv_buff, right)
-                dist.recv(send_buff, left)
-                accum[:] += send[:]
-            send_req.wait()
-        recv[:] = accum[:]
+       rank = dist.get_rank()
+       size = dist.get_world_size()
+       send_buff = send.clone()
+       recv_buff = send.clone()
+       accum = send.clone()
+
+       left = ((rank - 1) + size) % size
+       right = (rank + 1) % size
+
+       for i in range(size - 1):
+           if i % 2 == 0:
+               # Send send_buff
+               send_req = dist.isend(send_buff, right)
+               dist.recv(recv_buff, left)
+               accum[:] += recv_buff[:]
+           else:
+               # Send recv_buff
+               send_req = dist.isend(recv_buff, right)
+               dist.recv(send_buff, left)
+               accum[:] += send_buff[:]
+           send_req.wait()
+       recv[:] = accum[:]
 
 In the above script, the ``allreduce(send, recv)`` function has a
 slightly different signature than the ones in PyTorch. It takes a