Merge pull request #10478 from hoopoepg/topic/oshmem-dynamic-segment-allocation

MEMHEAP/BASE: removed hard segments limit

Author: yosefe

oshmem/mca/memheap/base/base.h

@@ -40,8 +40,8 @@ OSHMEM_DECLSPEC int mca_memheap_base_select(void);
 
 extern int mca_memheap_base_already_opened;
 extern int mca_memheap_base_key_exchange;
+extern int mca_memheap_num_segments_warn;
 
-#define MCA_MEMHEAP_MAX_SEGMENTS    32
 #define HEAP_SEG_INDEX              0
 #define MCA_MEMHEAP_SEG_COUNT       2
 
@@ -54,8 +54,9 @@ typedef struct mca_memheap_base_config {
 
 
 typedef struct mca_memheap_map {
-    map_segment_t   mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
+    map_segment_t  *mem_segs;
     int             n_segments;
+    int             capacity;
     int             num_transports;
 } mca_memheap_map_t;
 
@@ -70,6 +71,7 @@ int mca_memheap_base_reg(mca_memheap_map_t *);
 int mca_memheap_base_dereg(mca_memheap_map_t *);
 int memheap_oob_init(mca_memheap_map_t *);
 void memheap_oob_destruct(void);
+map_segment_t *mca_memheap_base_allocate_segment(mca_memheap_map_t *map);
 
 OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(const void* va);
 OSHMEM_DECLSPEC sshmem_mkey_t *mca_memheap_base_get_mkey(void* va,

oshmem/mca/memheap/base/memheap_base_alloc.c

@@ -18,6 +18,7 @@
 #include "oshmem/mca/memheap/memheap.h"
 #include "oshmem/mca/memheap/base/base.h"
 #include "ompi/util/timings.h"
+#include "opal/util/minmax.h"
 
 
 int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint,
@@ -35,7 +36,12 @@ int mca_memheap_base_alloc_init(mca_memheap_map_t *map, size_t size, long hint,
         assert(HEAP_SEG_INDEX < map->n_segments);
     }
 
-    map_segment_t *s = &map->mem_segs[map->n_segments];
+    map_segment_t *s = mca_memheap_base_allocate_segment(map);
+    if (NULL == s) {
+        MEMHEAP_ERROR("failed to allocate segment");
+        return OSHMEM_ERR_OUT_OF_RESOURCE;
+    }
+
     seg_filename = oshmem_get_unique_file_name(oshmem_my_proc_id());
 
     OPAL_TIMING_ENV_NEXT(timing, "oshmem_get_unique_file_name()");
@@ -72,6 +78,11 @@ void mca_memheap_base_alloc_exit(mca_memheap_map_t *map)
             mca_sshmem_unlink(s);
         }
     }
+
+    free(map->mem_segs);
+    map->n_segments = 0;
+    map->capacity = 0;
+    map->mem_segs = NULL;
 }
 
 int mca_memheap_alloc_with_hint(size_t size, long hint, void** ptr)
@@ -90,3 +101,33 @@ int mca_memheap_alloc_with_hint(size_t size, long hint, void** ptr)
 
     return MCA_MEMHEAP_CALL(alloc(size, ptr));
 }
+
+map_segment_t *mca_memheap_base_allocate_segment(mca_memheap_map_t *map)
+{
+    static int warned = 0;
+    map_segment_t *segments;
+    int capacity;
+
+    assert(map->n_segments <= map->capacity);
+
+    if (!warned && (map->n_segments > mca_memheap_num_segments_warn)) {
+        MEMHEAP_WARN("too many segments are registered: %d. This may cause "
+                     "performance degradation. Pls try adding --mca "
+                     "memheap_base_max_segments <NUMBER> to mpirun/oshrun "
+                     "command line to suppress this message", map->n_segments);
+        warned = 1;
+    }
+
+    if (map->n_segments == map->capacity) {
+        capacity = opal_max(map->capacity * 2, 4);
+        segments = realloc(map->mem_segs, capacity * sizeof(*map->mem_segs));
+        if (segments == NULL) {
+            return NULL;
+        }
+
+        map->capacity = capacity;
+        map->mem_segs = segments;
+    }
+
+    return &map->mem_segs[map->n_segments];
+}

oshmem/mca/memheap/base/memheap_base_frame.c

@@ -36,6 +36,7 @@ int mca_memheap_base_key_exchange = 1;
 opal_list_t mca_memheap_base_components_opened = {{0}};
 int mca_memheap_base_already_opened = 0;
 mca_memheap_map_t mca_memheap_base_map = {{{{0}}}};
+int mca_memheap_num_segments_warn = 32;
 
 static int mca_memheap_base_register(mca_base_register_flag_t flags)
 {
@@ -59,6 +60,14 @@ static int mca_memheap_base_register(mca_base_register_flag_t flags)
                           MCA_BASE_VAR_SCOPE_LOCAL,
                           &mca_memheap_base_config.device_nic_mem_seg_size);
 
+    mca_base_var_register("oshmem", "memheap", "base", "max_segments",
+                          "Display a warning if the number of segments of the "
+                          "shared memheap exceeds this value",
+                          MCA_BASE_VAR_TYPE_INT, NULL, 0,
+                          MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
+                          MCA_BASE_VAR_SCOPE_LOCAL,
+                          &mca_memheap_num_segments_warn);
+
     return OSHMEM_SUCCESS;
 }
 

oshmem/mca/memheap/base/memheap_base_mkey.c

@@ -766,10 +766,6 @@ void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno)
 {
     map_segment_t *s;
 
-    if (segno >= MCA_MEMHEAP_MAX_SEGMENTS) {
-        return;
-    }
-
     s = memheap_find_seg(segno);
     assert(NULL != s);
     seg->super.va_base = s->super.va_base;