Skip to content
This repository was archived by the owner on Sep 30, 2022. It is now read-only.

Commit e8ae71b

Browse files
jsquyresjsquyres
committed
Merge pull request #627 from bureddy/v1.10
HCOLL: Add alltoallv interface
2 parents 4834a14 + da0b1b6 commit e8ae71b

File tree

3 files changed

+83
-9
lines changed

3 files changed

+83
-9
lines changed

‎ompi/mca/coll/hcoll/coll_hcoll.h

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,10 @@ struct mca_coll_hcoll_module_t {
134134
mca_coll_base_module_t *previous_iallreduce_module;
135135
mca_coll_base_module_igatherv_fn_t previous_igatherv;
136136
mca_coll_base_module_t *previous_igatherv_module;
137+
mca_coll_base_module_ialltoall_fn_t previous_ialltoall;
138+
mca_coll_base_module_t *previous_ialltoall_module;
139+
mca_coll_base_module_ialltoallv_fn_t previous_ialltoallv;
140+
mca_coll_base_module_t *previous_ialltoallv_module;
137141
};
138142
typedef struct mca_coll_hcoll_module_t mca_coll_hcoll_module_t;
139143

@@ -186,6 +190,15 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
186190
struct ompi_communicator_t *comm,
187191
mca_coll_base_module_t *module);
188192

193+
int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts,
194+
int *sdisps,
195+
struct ompi_datatype_t *sdtype,
196+
void *rbuf, int *rcounts,
197+
int *rdisps,
198+
struct ompi_datatype_t *rdtype,
199+
struct ompi_communicator_t *comm,
200+
mca_coll_base_module_t *module);
201+
189202
int mca_coll_hcoll_gatherv(void* sbuf, int scount,
190203
struct ompi_datatype_t *sdtype,
191204
void* rbuf, int *rcounts, int *displs,
@@ -199,10 +212,10 @@ int mca_coll_hcoll_ibarrier(struct ompi_communicator_t *comm,
199212
mca_coll_base_module_t *module);
200213

201214
int mca_coll_hcoll_ibcast(void *buff, int count,
202-
struct ompi_datatype_t *datatype, int root,
203-
struct ompi_communicator_t *comm,
204-
ompi_request_t** request,
205-
mca_coll_base_module_t *module);
215+
struct ompi_datatype_t *datatype, int root,
216+
struct ompi_communicator_t *comm,
217+
ompi_request_t** request,
218+
mca_coll_base_module_t *module);
206219

207220
int mca_coll_hcoll_iallgather(void *sbuf, int scount,
208221
struct ompi_datatype_t *sdtype,
@@ -219,6 +232,24 @@ int mca_coll_hcoll_iallreduce(void *sbuf, void *rbuf, int count,
219232
ompi_request_t** request,
220233
mca_coll_base_module_t *module);
221234

235+
int mca_coll_hcoll_ialltoall(void *sbuf, int scount,
236+
struct ompi_datatype_t *sdtype,
237+
void* rbuf, int rcount,
238+
struct ompi_datatype_t *rdtype,
239+
struct ompi_communicator_t *comm,
240+
ompi_request_t **req,
241+
mca_coll_base_module_t *module);
242+
243+
int mca_coll_hcoll_ialltoallv(void *sbuf, int *scounts,
244+
int *sdisps,
245+
struct ompi_datatype_t *sdtype,
246+
void *rbuf, int *rcounts,
247+
int *rdisps,
248+
struct ompi_datatype_t *rdtype,
249+
struct ompi_communicator_t *comm,
250+
ompi_request_t **req,
251+
mca_coll_base_module_t *module);
252+
222253
int mca_coll_hcoll_igatherv(void* sbuf, int scount,
223254
struct ompi_datatype_t *sdtype,
224255
void* rbuf, int *rcounts, int *displs,

‎ompi/mca/coll/hcoll/coll_hcoll_module.c

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,19 +81,21 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
8181
OBJ_RELEASE(hcoll_module->previous_allreduce_module);
8282
OBJ_RELEASE(hcoll_module->previous_allgather_module);
8383
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
84+
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
85+
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
8486

8587
OBJ_RELEASE(hcoll_module->previous_ibarrier_module);
8688
OBJ_RELEASE(hcoll_module->previous_ibcast_module);
8789
OBJ_RELEASE(hcoll_module->previous_iallreduce_module);
8890
OBJ_RELEASE(hcoll_module->previous_iallgather_module);
8991
OBJ_RELEASE(hcoll_module->previous_igatherv_module);
92+
OBJ_RELEASE(hcoll_module->previous_ialltoall_module);
93+
OBJ_RELEASE(hcoll_module->previous_ialltoallv_module);
9094

9195
/*
9296
OBJ_RELEASE(hcoll_module->previous_allgatherv_module);
9397
OBJ_RELEASE(hcoll_module->previous_gather_module);
9498
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
95-
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
96-
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
9799
OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
98100
OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
99101
OBJ_RELEASE(hcoll_module->previous_reduce_module);
@@ -127,12 +129,16 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
127129
HCOL_SAVE_PREV_COLL_API(allreduce);
128130
HCOL_SAVE_PREV_COLL_API(allgather);
129131
HCOL_SAVE_PREV_COLL_API(gatherv);
132+
HCOL_SAVE_PREV_COLL_API(alltoall);
133+
HCOL_SAVE_PREV_COLL_API(alltoallv);
130134

131135
HCOL_SAVE_PREV_COLL_API(ibarrier);
132136
HCOL_SAVE_PREV_COLL_API(ibcast);
133137
HCOL_SAVE_PREV_COLL_API(iallreduce);
134138
HCOL_SAVE_PREV_COLL_API(iallgather);
135139
HCOL_SAVE_PREV_COLL_API(igatherv);
140+
HCOL_SAVE_PREV_COLL_API(ialltoall);
141+
HCOL_SAVE_PREV_COLL_API(ialltoallv);
136142

137143
/*
138144
These collectives are not yet part of hcoll, so
@@ -141,9 +147,6 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
141147
HCOL_SAVE_PREV_COLL_API(gather);
142148
HCOL_SAVE_PREV_COLL_API(reduce);
143149
HCOL_SAVE_PREV_COLL_API(allgatherv);
144-
HCOL_SAVE_PREV_COLL_API(gatherv);
145-
HCOL_SAVE_PREV_COLL_API(alltoall);
146-
HCOL_SAVE_PREV_COLL_API(alltoallv);
147150
HCOL_SAVE_PREV_COLL_API(alltoallw);
148151
*/
149152
return OMPI_SUCCESS;
@@ -311,13 +314,16 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
311314
hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
312315
hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
313316
hcoll_module->super.coll_alltoall = /*hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : */ NULL;
317+
hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
314318
hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
315319
hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
316320
hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
317321
hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
318322
hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
319323
hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
320324
hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
325+
hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
326+
hcoll_module->super.coll_ialltoallv = /*hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : */ NULL;
321327

322328
*priority = cm->hcoll_priority;
323329
module = &hcoll_module->super;

‎ompi/mca/coll/hcoll/coll_hcoll_ops.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,43 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
226226
return rc;
227227
}
228228

229+
int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts, int *sdisps,
230+
struct ompi_datatype_t *sdtype,
231+
void *rbuf, int *rcounts, int *rdisps,
232+
struct ompi_datatype_t *rdtype,
233+
struct ompi_communicator_t *comm,
234+
mca_coll_base_module_t *module)
235+
{
236+
dte_data_representation_t stype;
237+
dte_data_representation_t rtype;
238+
int rc;
239+
HCOL_VERBOSE(20,"RUNNING HCOL ALLTOALLV");
240+
mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
241+
stype = ompi_dtype_2_dte_dtype(sdtype);
242+
rtype = ompi_dtype_2_dte_dtype(rdtype);
243+
if (OPAL_UNLIKELY((HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
244+
|| HCOL_DTE_IS_COMPLEX(stype) || HCOL_DTE_IS_COMPLEX(rtype)))
245+
&& mca_coll_hcoll_component.hcoll_datatype_fallback){
246+
HCOL_VERBOSE(20,"Ompi_datatype is not supported: sdtype = %s, rdtype = %s; calling fallback alltoallv;",
247+
sdtype->super.name,
248+
rdtype->super.name);
249+
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
250+
rbuf, rcounts, rdisps, rdtype,
251+
comm, hcoll_module->previous_alltoallv_module);
252+
return rc;
253+
}
254+
rc = hcoll_collectives.coll_alltoallv(sbuf, scounts, sdisps, stype,
255+
rbuf, rcounts, rdisps, rtype,
256+
hcoll_module->hcoll_context);
257+
if (HCOLL_SUCCESS != rc){
258+
HCOL_VERBOSE(20,"RUNNING FALLBACK ALLTOALLV");
259+
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
260+
rbuf, rcounts, rdisps, rdtype,
261+
comm, hcoll_module->previous_alltoallv_module);
262+
}
263+
return rc;
264+
}
265+
229266
int mca_coll_hcoll_gatherv(void* sbuf, int scount,
230267
struct ompi_datatype_t *sdtype,
231268
void* rbuf, int *rcounts, int *displs,

0 commit comments

Comments
 (0)