Skip to content

Commit fce4c0a

Browse files
jdoerfertjdoerfert
authored
[OpenMP] Start organizing PluginManager, PluginAdaptors (#73875)
1 parent 1b02f59 commit fce4c0a

File tree

11 files changed

+264
-229
lines changed

11 files changed

+264
-229
lines changed

‎openmp/libomptarget/include/PluginManager.h

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
//===-- PluginManager.h - Plugin loading and communication API --*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// Declarations for managing devices that are handled by RTL plugins.
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef OMPTARGET_PLUGIN_MANAGER_H
14+
#define OMPTARGET_PLUGIN_MANAGER_H
15+
16+
#include "Shared/APITypes.h"
17+
18+
#include "device.h"
19+
20+
#include "llvm/ADT/DenseSet.h"
21+
#include "llvm/ADT/SmallVector.h"
22+
#include "llvm/Support/DynamicLibrary.h"
23+
24+
#include <list>
25+
#include <mutex>
26+
27+
struct PluginAdaptorTy {
28+
typedef int32_t(init_plugin_ty)();
29+
typedef int32_t(is_valid_binary_ty)(void *);
30+
typedef int32_t(is_valid_binary_info_ty)(void *, void *);
31+
typedef int32_t(is_data_exchangable_ty)(int32_t, int32_t);
32+
typedef int32_t(number_of_devices_ty)();
33+
typedef int32_t(init_device_ty)(int32_t);
34+
typedef __tgt_target_table *(load_binary_ty)(int32_t, void *);
35+
typedef void *(data_alloc_ty)(int32_t, int64_t, void *, int32_t);
36+
typedef int32_t(data_submit_ty)(int32_t, void *, void *, int64_t);
37+
typedef int32_t(data_submit_async_ty)(int32_t, void *, void *, int64_t,
38+
__tgt_async_info *);
39+
typedef int32_t(data_retrieve_ty)(int32_t, void *, void *, int64_t);
40+
typedef int32_t(data_retrieve_async_ty)(int32_t, void *, void *, int64_t,
41+
__tgt_async_info *);
42+
typedef int32_t(data_exchange_ty)(int32_t, void *, int32_t, void *, int64_t);
43+
typedef int32_t(data_exchange_async_ty)(int32_t, void *, int32_t, void *,
44+
int64_t, __tgt_async_info *);
45+
typedef int32_t(data_delete_ty)(int32_t, void *, int32_t);
46+
typedef int32_t(launch_kernel_ty)(int32_t, void *, void **, ptrdiff_t *,
47+
const KernelArgsTy *, __tgt_async_info *);
48+
typedef int64_t(init_requires_ty)(int64_t);
49+
typedef int32_t(synchronize_ty)(int32_t, __tgt_async_info *);
50+
typedef int32_t(query_async_ty)(int32_t, __tgt_async_info *);
51+
typedef int32_t(supports_empty_images_ty)();
52+
typedef void(print_device_info_ty)(int32_t);
53+
typedef void(set_info_flag_ty)(uint32_t);
54+
typedef int32_t(create_event_ty)(int32_t, void **);
55+
typedef int32_t(record_event_ty)(int32_t, void *, __tgt_async_info *);
56+
typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *);
57+
typedef int32_t(sync_event_ty)(int32_t, void *);
58+
typedef int32_t(destroy_event_ty)(int32_t, void *);
59+
typedef int32_t(release_async_info_ty)(int32_t, __tgt_async_info *);
60+
typedef int32_t(init_async_info_ty)(int32_t, __tgt_async_info **);
61+
typedef int64_t(init_device_into_ty)(int64_t, __tgt_device_info *,
62+
const char **);
63+
typedef int32_t(data_lock_ty)(int32_t, void *, int64_t, void **);
64+
typedef int32_t(data_unlock_ty)(int32_t, void *);
65+
typedef int32_t(data_notify_mapped_ty)(int32_t, void *, int64_t);
66+
typedef int32_t(data_notify_unmapped_ty)(int32_t, void *);
67+
typedef int32_t(set_device_offset_ty)(int32_t);
68+
typedef int32_t(activate_record_replay_ty)(int32_t, uint64_t, void *, bool,
69+
bool, uint64_t &);
70+
71+
int32_t Idx = -1; // RTL index, index is the number of devices
72+
// of other RTLs that were registered before,
73+
// i.e. the OpenMP index of the first device
74+
// to be registered with this RTL.
75+
int32_t NumberOfDevices = -1; // Number of devices this RTL deals with.
76+
77+
std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler;
78+
79+
#ifdef OMPTARGET_DEBUG
80+
std::string RTLName;
81+
#endif
82+
83+
// Functions implemented in the RTL.
84+
init_plugin_ty *init_plugin = nullptr;
85+
is_valid_binary_ty *is_valid_binary = nullptr;
86+
is_valid_binary_info_ty *is_valid_binary_info = nullptr;
87+
is_data_exchangable_ty *is_data_exchangable = nullptr;
88+
number_of_devices_ty *number_of_devices = nullptr;
89+
init_device_ty *init_device = nullptr;
90+
load_binary_ty *load_binary = nullptr;
91+
data_alloc_ty *data_alloc = nullptr;
92+
data_submit_ty *data_submit = nullptr;
93+
data_submit_async_ty *data_submit_async = nullptr;
94+
data_retrieve_ty *data_retrieve = nullptr;
95+
data_retrieve_async_ty *data_retrieve_async = nullptr;
96+
data_exchange_ty *data_exchange = nullptr;
97+
data_exchange_async_ty *data_exchange_async = nullptr;
98+
data_delete_ty *data_delete = nullptr;
99+
launch_kernel_ty *launch_kernel = nullptr;
100+
init_requires_ty *init_requires = nullptr;
101+
synchronize_ty *synchronize = nullptr;
102+
query_async_ty *query_async = nullptr;
103+
supports_empty_images_ty *supports_empty_images = nullptr;
104+
set_info_flag_ty *set_info_flag = nullptr;
105+
print_device_info_ty *print_device_info = nullptr;
106+
create_event_ty *create_event = nullptr;
107+
record_event_ty *record_event = nullptr;
108+
wait_event_ty *wait_event = nullptr;
109+
sync_event_ty *sync_event = nullptr;
110+
destroy_event_ty *destroy_event = nullptr;
111+
init_async_info_ty *init_async_info = nullptr;
112+
init_device_into_ty *init_device_info = nullptr;
113+
release_async_info_ty *release_async_info = nullptr;
114+
data_lock_ty *data_lock = nullptr;
115+
data_unlock_ty *data_unlock = nullptr;
116+
data_notify_mapped_ty *data_notify_mapped = nullptr;
117+
data_notify_unmapped_ty *data_notify_unmapped = nullptr;
118+
set_device_offset_ty *set_device_offset = nullptr;
119+
activate_record_replay_ty *activate_record_replay = nullptr;
120+
121+
// Are there images associated with this RTL.
122+
bool IsUsed = false;
123+
124+
llvm::DenseSet<const __tgt_device_image *> UsedImages;
125+
126+
// Mutex for thread-safety when calling RTL interface functions.
127+
// It is easier to enforce thread-safety at the libomptarget level,
128+
// so that developers of new RTLs do not have to worry about it.
129+
std::mutex Mtx;
130+
};
131+
132+
/// RTLs identified in the system.
133+
struct PluginAdaptorManagerTy {
134+
// List of the detected runtime libraries.
135+
std::list<PluginAdaptorTy> AllRTLs;
136+
137+
// Array of pointers to the detected runtime libraries that have compatible
138+
// binaries.
139+
llvm::SmallVector<PluginAdaptorTy *> UsedRTLs;
140+
141+
int64_t RequiresFlags = OMP_REQ_UNDEFINED;
142+
143+
explicit PluginAdaptorManagerTy() = default;
144+
145+
// Register the clauses of the requires directive.
146+
void registerRequires(int64_t Flags);
147+
148+
// Initialize RTL if it has not been initialized
149+
void initRTLonce(PluginAdaptorTy &RTL);
150+
151+
// Initialize all RTLs
152+
void initAllRTLs();
153+
154+
// Register a shared library with all (compatible) RTLs.
155+
void registerLib(__tgt_bin_desc *Desc);
156+
157+
// Unregister a shared library from all RTLs.
158+
void unregisterLib(__tgt_bin_desc *Desc);
159+
160+
// not thread-safe, called from global constructor (i.e. once)
161+
void loadRTLs();
162+
163+
private:
164+
static bool attemptLoadRTL(const std::string &RTLName, PluginAdaptorTy &RTL);
165+
};
166+
167+
/// Struct for the data required to handle plugins
168+
struct PluginManager {
169+
PluginManager(bool UseEventsForAtomicTransfers)
170+
: UseEventsForAtomicTransfers(UseEventsForAtomicTransfers) {}
171+
172+
/// RTLs identified on the host
173+
PluginAdaptorManagerTy RTLs;
174+
175+
/// Executable images and information extracted from the input images passed
176+
/// to the runtime.
177+
std::list<std::pair<__tgt_device_image, __tgt_image_info>> Images;
178+
179+
/// Devices associated with RTLs
180+
llvm::SmallVector<std::unique_ptr<DeviceTy>> Devices;
181+
std::mutex RTLsMtx; ///< For RTLs and Devices
182+
183+
/// Translation table retreived from the binary
184+
HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable;
185+
std::mutex TrlTblMtx; ///< For Translation Table
186+
/// Host offload entries in order of image registration
187+
llvm::SmallVector<__tgt_offload_entry *> HostEntriesBeginRegistrationOrder;
188+
189+
/// Map from ptrs on the host to an entry in the Translation Table
190+
HostPtrToTableMapTy HostPtrToTableMap;
191+
std::mutex TblMapMtx; ///< For HostPtrToTableMap
192+
193+
// Store target policy (disabled, mandatory, default)
194+
kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
195+
std::mutex TargetOffloadMtx; ///< For TargetOffloadPolicy
196+
197+
/// Flag to indicate if we use events to ensure the atomicity of
198+
/// map clauses or not. Can be modified with an environment variable.
199+
const bool UseEventsForAtomicTransfers;
200+
201+
// Work around for plugins that call dlopen on shared libraries that call
202+
// tgt_register_lib during their initialisation. Stash the pointers in a
203+
// vector until the plugins are all initialised and then register them.
204+
bool delayRegisterLib(__tgt_bin_desc *Desc) {
205+
if (RTLsLoaded)
206+
return false;
207+
DelayedBinDesc.push_back(Desc);
208+
return true;
209+
}
210+
211+
void registerDelayedLibraries() {
212+
// Only called by libomptarget constructor
213+
RTLsLoaded = true;
214+
for (auto *Desc : DelayedBinDesc)
215+
__tgt_register_lib(Desc);
216+
DelayedBinDesc.clear();
217+
}
218+
219+
private:
220+
bool RTLsLoaded = false;
221+
llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
222+
};
223+
224+
extern PluginManager *PM;
225+
226+
#endif // OMPTARGET_PLUGIN_MANAGER_H

‎openmp/libomptarget/include/device.h

Lines changed: 3 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,8 @@
2828

2929
#include "OpenMP/Mapping.h"
3030

31-
#include "llvm/ADT/SmallVector.h"
32-
3331
// Forward declarations.
34-
struct RTLInfoTy;
32+
struct PluginAdaptorTy;
3533
struct __tgt_bin_desc;
3634
struct __tgt_target_table;
3735

@@ -53,7 +51,7 @@ typedef std::map<__tgt_bin_desc *, PendingCtorDtorListsTy>
5351

5452
struct DeviceTy {
5553
int32_t DeviceID;
56-
RTLInfoTy *RTL;
54+
PluginAdaptorTy *RTL;
5755
int32_t RTLDeviceID;
5856

5957
bool IsInit;
@@ -77,7 +75,7 @@ struct DeviceTy {
7775

7876
std::mutex PendingGlobalsMtx;
7977

80-
DeviceTy(RTLInfoTy *RTL);
78+
DeviceTy(PluginAdaptorTy *RTL);
8179
// DeviceTy is not copyable
8280
DeviceTy(const DeviceTy &D) = delete;
8381
DeviceTy &operator=(const DeviceTy &D) = delete;
@@ -243,63 +241,4 @@ struct DeviceTy {
243241

244242
extern bool deviceIsReady(int DeviceNum);
245243

246-
/// Struct for the data required to handle plugins
247-
struct PluginManager {
248-
PluginManager(bool UseEventsForAtomicTransfers)
249-
: UseEventsForAtomicTransfers(UseEventsForAtomicTransfers) {}
250-
251-
/// RTLs identified on the host
252-
RTLsTy RTLs;
253-
254-
/// Executable images and information extracted from the input images passed
255-
/// to the runtime.
256-
std::list<std::pair<__tgt_device_image, __tgt_image_info>> Images;
257-
258-
/// Devices associated with RTLs
259-
llvm::SmallVector<std::unique_ptr<DeviceTy>> Devices;
260-
std::mutex RTLsMtx; ///< For RTLs and Devices
261-
262-
/// Translation table retreived from the binary
263-
HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable;
264-
std::mutex TrlTblMtx; ///< For Translation Table
265-
/// Host offload entries in order of image registration
266-
llvm::SmallVector<__tgt_offload_entry *> HostEntriesBeginRegistrationOrder;
267-
268-
/// Map from ptrs on the host to an entry in the Translation Table
269-
HostPtrToTableMapTy HostPtrToTableMap;
270-
std::mutex TblMapMtx; ///< For HostPtrToTableMap
271-
272-
// Store target policy (disabled, mandatory, default)
273-
kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
274-
std::mutex TargetOffloadMtx; ///< For TargetOffloadPolicy
275-
276-
/// Flag to indicate if we use events to ensure the atomicity of
277-
/// map clauses or not. Can be modified with an environment variable.
278-
const bool UseEventsForAtomicTransfers;
279-
280-
// Work around for plugins that call dlopen on shared libraries that call
281-
// tgt_register_lib during their initialisation. Stash the pointers in a
282-
// vector until the plugins are all initialised and then register them.
283-
bool delayRegisterLib(__tgt_bin_desc *Desc) {
284-
if (RTLsLoaded)
285-
return false;
286-
DelayedBinDesc.push_back(Desc);
287-
return true;
288-
}
289-
290-
void registerDelayedLibraries() {
291-
// Only called by libomptarget constructor
292-
RTLsLoaded = true;
293-
for (auto *Desc : DelayedBinDesc)
294-
__tgt_register_lib(Desc);
295-
DelayedBinDesc.clear();
296-
}
297-
298-
private:
299-
bool RTLsLoaded = false;
300-
llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
301-
};
302-
303-
extern PluginManager *PM;
304-
305244
#endif

0 commit comments

Comments
 (0)