Merge pull request #10676 from markalle/alltoall_large_mesg_fix

fixing an int overflow in the basic collectives alltoall

Author: markalle

ompi/mca/coll/base/coll_base_alltoall.c

@@ -14,7 +14,7 @@
  *                         reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation. All rights reserved.
  * Copyright (c) 2021      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
  * $COPYRIGHT$
@@ -104,19 +104,19 @@ mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
         ompi_proc_t *right_proc = ompi_comm_peer_lookup(comm, right);
         opal_convertor_clone(right_proc->super.proc_convertor, &convertor, 0);
         opal_convertor_prepare_for_send(&convertor, &rdtype->super, rcount,
-                                        (char *) rbuf + right * rcount * extent);
+                                        (char *) rbuf + (MPI_Aint) right * rcount * extent);
         packed_size = max_size;
         err = opal_convertor_pack(&convertor, &iov, &iov_count, &packed_size);
         if (1 != err) { goto error_hndl; }
 
         /* Receive data from the right */
-        err = MCA_PML_CALL(irecv ((char *) rbuf + right * rcount * extent, rcount, rdtype,
+        err = MCA_PML_CALL(irecv ((char *) rbuf + (MPI_Aint) right * rcount * extent, rcount, rdtype,
                                   right, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
         if (MPI_SUCCESS != err) { goto error_hndl; }
 
         if( left != right ) {
             /* Send data to the left */
-            err = MCA_PML_CALL(send ((char *) rbuf + left * rcount * extent, rcount, rdtype,
+            err = MCA_PML_CALL(send ((char *) rbuf + (MPI_Aint) left * rcount * extent, rcount, rdtype,
                                      left, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
                                      comm));
             if (MPI_SUCCESS != err) { goto error_hndl; }
@@ -125,7 +125,7 @@ mca_coll_base_alltoall_intra_basic_inplace(const void *rbuf, int rcount,
             if (MPI_SUCCESS != err) { goto error_hndl; }
 
             /* Receive data from the left */
-            err = MCA_PML_CALL(irecv ((char *) rbuf + left * rcount * extent, rcount, rdtype,
+            err = MCA_PML_CALL(irecv ((char *) rbuf + (MPI_Aint) left * rcount * extent, rcount, rdtype,
                                       left, MCA_COLL_BASE_TAG_ALLTOALL, comm, &req));
             if (MPI_SUCCESS != err) { goto error_hndl; }
         }