LLVM OpenMP* Runtime Library
kmp_gsupport.cpp
1/*
2 * kmp_gsupport.cpp
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#include "kmp.h"
14#include "kmp_atomic.h"
15
16#if OMPT_SUPPORT
17#include "ompt-specific.h"
18#endif
19
20enum {
21 KMP_GOMP_TASK_UNTIED_FLAG = 1,
22 KMP_GOMP_TASK_FINAL_FLAG = 2,
23 KMP_GOMP_TASK_DEPENDS_FLAG = 8
24};
25
26enum {
27 KMP_GOMP_DEPOBJ_IN = 1,
28 KMP_GOMP_DEPOBJ_OUT = 2,
29 KMP_GOMP_DEPOBJ_INOUT = 3,
30 KMP_GOMP_DEPOBJ_MTXINOUTSET = 4
31};
32
33// This class helps convert gomp dependency info into
34// kmp_depend_info_t structures
35class kmp_gomp_depends_info_t {
36 void **depend;
37 kmp_int32 num_deps;
38 size_t num_out, num_mutexinout, num_in, num_depobj;
39 size_t offset;
40
41public:
42 kmp_gomp_depends_info_t(void **depend) : depend(depend) {
43 size_t ndeps = (kmp_intptr_t)depend[0];
44 // GOMP taskdep structure:
45 // if depend[0] != 0:
46 // depend = [ ndeps | nout | &out | ... | &out | &in | ... | &in ]
47 //
48 // if depend[0] == 0:
49 // depend = [ 0 | ndeps | nout | nmtx | nin | &out | ... | &out | &mtx |
50 // ... | &mtx | &in | ... | &in | &depobj | ... | &depobj ]
51 if (ndeps) {
52 num_out = (kmp_intptr_t)depend[1];
53 num_in = ndeps - num_out;
54 num_mutexinout = num_depobj = 0;
55 offset = 2;
56 } else {
57 ndeps = (kmp_intptr_t)depend[1];
58 num_out = (kmp_intptr_t)depend[2];
59 num_mutexinout = (kmp_intptr_t)depend[3];
60 num_in = (kmp_intptr_t)depend[4];
61 num_depobj = ndeps - num_out - num_mutexinout - num_in;
62 KMP_ASSERT(num_depobj <= ndeps);
63 offset = 5;
64 }
65 num_deps = static_cast<kmp_int32>(ndeps);
66 }
67 kmp_int32 get_num_deps() const { return num_deps; }
68 kmp_depend_info_t get_kmp_depend(size_t index) const {
69 kmp_depend_info_t retval;
70 memset(&retval, '\0', sizeof(retval));
71 KMP_ASSERT(index < (size_t)num_deps);
72 retval.len = 0;
73 // Because inout and out are logically equivalent,
74 // use inout and in dependency flags. GOMP does not provide a
75 // way to distinguish if user specified out vs. inout.
76 if (index < num_out) {
77 retval.flags.in = 1;
78 retval.flags.out = 1;
79 retval.base_addr = (kmp_intptr_t)depend[offset + index];
80 } else if (index >= num_out && index < (num_out + num_mutexinout)) {
81 retval.flags.mtx = 1;
82 retval.base_addr = (kmp_intptr_t)depend[offset + index];
83 } else if (index >= (num_out + num_mutexinout) &&
84 index < (num_out + num_mutexinout + num_in)) {
85 retval.flags.in = 1;
86 retval.base_addr = (kmp_intptr_t)depend[offset + index];
87 } else {
88 // depobj is a two element array (size of elements are size of pointer)
89 // depobj[0] = base_addr
90 // depobj[1] = type (in, out, inout, mutexinoutset, etc.)
91 kmp_intptr_t *depobj = (kmp_intptr_t *)depend[offset + index];
92 retval.base_addr = depobj[0];
93 switch (depobj[1]) {
94 case KMP_GOMP_DEPOBJ_IN:
95 retval.flags.in = 1;
96 break;
97 case KMP_GOMP_DEPOBJ_OUT:
98 retval.flags.out = 1;
99 break;
100 case KMP_GOMP_DEPOBJ_INOUT:
101 retval.flags.in = 1;
102 retval.flags.out = 1;
103 break;
104 case KMP_GOMP_DEPOBJ_MTXINOUTSET:
105 retval.flags.mtx = 1;
106 break;
107 default:
108 KMP_FATAL(GompFeatureNotSupported, "Unknown depobj type");
109 }
110 }
111 return retval;
112 }
113};
114
115#ifdef __cplusplus
116extern "C" {
117#endif // __cplusplus
118
119#define MKLOC(loc, routine) \
120 static ident_t loc = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"};
121
122#include "kmp_ftn_os.h"
123
124void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) {
125 int gtid = __kmp_entry_gtid();
126 MKLOC(loc, "GOMP_barrier");
127 KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid));
128#if OMPT_SUPPORT && OMPT_OPTIONAL
129 ompt_frame_t *ompt_frame;
130 if (ompt_enabled.enabled) {
131 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
132 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
133 }
134 OMPT_STORE_RETURN_ADDRESS(gtid);
135#endif
136 __kmpc_barrier(&loc, gtid);
137#if OMPT_SUPPORT && OMPT_OPTIONAL
138 if (ompt_enabled.enabled) {
139 ompt_frame->enter_frame = ompt_data_none;
140 }
141#endif
142}
143
144// Mutual exclusion
145
146// The symbol that icc/ifort generates for unnamed for unnamed critical sections
147// - .gomp_critical_user_ - is defined using .comm in any objects reference it.
148// We can't reference it directly here in C code, as the symbol contains a ".".
149//
150// The RTL contains an assembly language definition of .gomp_critical_user_
151// with another symbol __kmp_unnamed_critical_addr initialized with it's
152// address.
153extern kmp_critical_name *__kmp_unnamed_critical_addr;
154
155void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_START)(void) {
156 int gtid = __kmp_entry_gtid();
157 MKLOC(loc, "GOMP_critical_start");
158 KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid));
159#if OMPT_SUPPORT && OMPT_OPTIONAL
160 OMPT_STORE_RETURN_ADDRESS(gtid);
161#endif
162 __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr);
163}
164
165void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_END)(void) {
166 int gtid = __kmp_get_gtid();
167 MKLOC(loc, "GOMP_critical_end");
168 KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid));
169#if OMPT_SUPPORT && OMPT_OPTIONAL
170 OMPT_STORE_RETURN_ADDRESS(gtid);
171#endif
172 __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr);
173}
174
175void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr) {
176 int gtid = __kmp_entry_gtid();
177 MKLOC(loc, "GOMP_critical_name_start");
178 KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid));
179 __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr);
180}
181
182void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr) {
183 int gtid = __kmp_get_gtid();
184 MKLOC(loc, "GOMP_critical_name_end");
185 KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid));
186 __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr);
187}
188
189// The Gnu codegen tries to use locked operations to perform atomic updates
190// inline. If it can't, then it calls GOMP_atomic_start() before performing
191// the update and GOMP_atomic_end() afterward, regardless of the data type.
192void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_START)(void) {
193 int gtid = __kmp_entry_gtid();
194 KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid));
195
196#if OMPT_SUPPORT
197 __ompt_thread_assign_wait_id(0);
198#endif
199
200 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
201}
202
203void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_END)(void) {
204 int gtid = __kmp_get_gtid();
205 KA_TRACE(20, ("GOMP_atomic_end: T#%d\n", gtid));
206 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
207}
208
209int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) {
210 int gtid = __kmp_entry_gtid();
211 MKLOC(loc, "GOMP_single_start");
212 KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid));
213
214 if (!TCR_4(__kmp_init_parallel))
215 __kmp_parallel_initialize();
216 __kmp_resume_if_soft_paused();
217
218 // 3rd parameter == FALSE prevents kmp_enter_single from pushing a
219 // workshare when USE_CHECKS is defined. We need to avoid the push,
220 // as there is no corresponding GOMP_single_end() call.
221 kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE);
222
223#if OMPT_SUPPORT && OMPT_OPTIONAL
224 kmp_info_t *this_thr = __kmp_threads[gtid];
225 kmp_team_t *team = this_thr->th.th_team;
226 int tid = __kmp_tid_from_gtid(gtid);
227
228 if (ompt_enabled.enabled) {
229 if (rc) {
230 if (ompt_enabled.ompt_callback_work) {
231 ompt_callbacks.ompt_callback(ompt_callback_work)(
232 ompt_work_single_executor, ompt_scope_begin,
233 &(team->t.ompt_team_info.parallel_data),
234 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
235 1, OMPT_GET_RETURN_ADDRESS(0));
236 }
237 } else {
238 if (ompt_enabled.ompt_callback_work) {
239 ompt_callbacks.ompt_callback(ompt_callback_work)(
240 ompt_work_single_other, ompt_scope_begin,
241 &(team->t.ompt_team_info.parallel_data),
242 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
243 1, OMPT_GET_RETURN_ADDRESS(0));
244 ompt_callbacks.ompt_callback(ompt_callback_work)(
245 ompt_work_single_other, ompt_scope_end,
246 &(team->t.ompt_team_info.parallel_data),