kmp_runtime.cpp

/*
 * kmp_runtime.cpp -- KPTS runtime support library
 */
 
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "kmp.h"
#include "kmp_affinity.h"
#include "kmp_atomic.h"
#include "kmp_environment.h"
#include "kmp_error.h"
#include "kmp_i18n.h"
#include "kmp_io.h"
#include "kmp_itt.h"
#include "kmp_settings.h"
#include "kmp_stats.h"
#include "kmp_str.h"
#include "kmp_wait_release.h"
#include "kmp_wrapper_getpid.h"
#include "kmp_dispatch.h"
#if KMP_USE_HIER_SCHED
#include "kmp_dispatch_hier.h"
#endif
 
#if OMPT_SUPPORT
#include "ompt-specific.h"
#endif
#if OMPD_SUPPORT
#include "ompd-specific.h"
#endif
 
#if OMP_PROFILING_SUPPORT
#include "llvm/Support/TimeProfiler.h"
static char *ProfileTraceFile = nullptr;
#endif
 
/* these are temporary issues to be dealt with */
#define KMP_USE_PRCTL 0
 
#if KMP_OS_WINDOWS
#include <process.h>
#endif
 
#if KMP_OS_WINDOWS
// windows does not need include files as it doesn't use shared memory
#else
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#define SHM_SIZE 1024
#endif
 
#if defined(KMP_GOMP_COMPAT)
char const __kmp_version_alt_comp[] =
    KMP_VERSION_PREFIX "alternative compiler support: yes";
#endif /* defined(KMP_GOMP_COMPAT) */
 
char const __kmp_version_omp_api[] =
    KMP_VERSION_PREFIX "API version: 5.0 (201611)";
 
#ifdef KMP_DEBUG
char const __kmp_version_lock[] =
    KMP_VERSION_PREFIX "lock type: run time selectable";
#endif /* KMP_DEBUG */
 
#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
 
/* ------------------------------------------------------------------------ */
 
#if KMP_USE_MONITOR
kmp_info_t __kmp_monitor;
#endif
 
/* Forward declarations */
 
void __kmp_cleanup(void);
 
static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
                                  int gtid);
static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
                                  kmp_internal_control_t *new_icvs,
                                  ident_t *loc);
#if KMP_AFFINITY_SUPPORTED
static void __kmp_partition_places(kmp_team_t *team,
                                   int update_master_only = 0);
#endif
static void __kmp_do_serial_initialize(void);
void __kmp_fork_barrier(int gtid, int tid);
void __kmp_join_barrier(int gtid);
void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
                          kmp_internal_control_t *new_icvs, ident_t *loc);
 
#ifdef USE_LOAD_BALANCE
static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
#endif
 
static int __kmp_expand_threads(int nNeed);
#if KMP_OS_WINDOWS
static int __kmp_unregister_root_other_thread(int gtid);
#endif
static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
 
void __kmp_resize_dist_barrier(kmp_team_t *team, int old_nthreads,
                               int new_nthreads);
void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads);
 
/* Calculate the identifier of the current thread */
/* fast (and somewhat portable) way to get unique identifier of executing
   thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
int __kmp_get_global_thread_id() {
  int i;
  kmp_info_t **other_threads;
  size_t stack_data;
  char *stack_addr;
  size_t stack_size;
  char *stack_base;
 
  KA_TRACE(
      1000,
      ("*** __kmp_get_global_thread_id: entering, nproc=%d  all_nproc=%d\n",
       __kmp_nth, __kmp_all_nth));
 
  /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
     a parallel region, made it return KMP_GTID_DNE to force serial_initialize
     by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
     __kmp_init_gtid for this to work. */
 
  if (!TCR_4(__kmp_init_gtid))
    return KMP_GTID_DNE;
 
#ifdef KMP_TDATA_GTID
  if (TCR_4(__kmp_gtid_mode) >= 3) {
    KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"));
    return __kmp_gtid;
  }
#endif
  if (TCR_4(__kmp_gtid_mode) >= 2) {
    KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"));
    return __kmp_gtid_get_specific();
  }
  KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"));
 
  stack_addr = (char *)&stack_data;
  other_threads = __kmp_threads;
 
  /* ATT: The code below is a source of potential bugs due to unsynchronized
     access to __kmp_threads array. For example:
     1. Current thread loads other_threads[i] to thr and checks it, it is
        non-NULL.
     2. Current thread is suspended by OS.
     3. Another thread unregisters and finishes (debug versions of free()
        may fill memory with something like 0xEF).
     4. Current thread is resumed.
     5. Current thread reads junk from *thr.
     TODO: Fix it.  --ln  */
 
  for (i = 0; i < __kmp_threads_capacity; i++) {
 
    kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
    if (!thr)
      continue;
 
    stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
    stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
 
    /* stack grows down -- search through all of the active threads */
 
    if (stack_addr <= stack_base) {
      size_t stack_diff = stack_base - stack_addr;
 
      if (stack_diff <= stack_size) {
        /* The only way we can be closer than the allocated */
        /* stack size is if we are running on this thread. */
        KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
        return i;
      }
    }
  }
 
  /* get specific to try and determine our gtid */
  KA_TRACE(1000,
           ("*** __kmp_get_global_thread_id: internal alg. failed to find "
            "thread, using TLS\n"));
  i = __kmp_gtid_get_specific();
 
  /*fprintf( stderr, "=== %d\n", i );  */ /* GROO */
 
  /* if we havn't been assigned a gtid, then return code */
  if (i < 0)
    return i;
 
  /* dynamically updated stack window for uber threads to avoid get_specific
     call */
  if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
    KMP_FATAL(StackOverflow, i);
  }
 
  stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
  if (stack_addr > stack_base) {
    TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
    TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
            other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
                stack_base);
  } else {
    TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
            stack_base - stack_addr);
  }
 
  /* Reprint stack bounds for ubermaster since they have been refined */
  if (__kmp_storage_map) {
    char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
    char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
    __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
                                 other_threads[i]->th.th_info.ds.ds_stacksize,
                                 "th_%d stack (refinement)", i);
  }
  return i;
}
 
int __kmp_get_global_thread_id_reg() {
  int gtid;
 
  if (!__kmp_init_serial) {
    gtid = KMP_GTID_DNE;
  } else
#ifdef KMP_TDATA_GTID
      if (TCR_4(__kmp_gtid_mode) >= 3) {
    KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"));
    gtid = __kmp_gtid;
  } else
#endif
      if (TCR_4(__kmp_gtid_mode) >= 2) {
    KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
    gtid = __kmp_gtid_get_specific();
  } else {
    KA_TRACE(1000,
             ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
    gtid = __kmp_get_global_thread_id();
  }
 
  /* we must be a new uber master sibling thread */
  if (gtid == KMP_GTID_DNE) {
    KA_TRACE(10,
             ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
              "Registering a new gtid.\n"));
    __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
    if (!__kmp_init_serial) {
      __kmp_do_serial_initialize();
      gtid = __kmp_gtid_get_specific();
    } else {
      gtid = __kmp_register_root(FALSE);
    }
    __kmp_release_bootstrap_lock(&__kmp_initz_lock);
    /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
  }
 
  KMP_DEBUG_ASSERT(gtid >= 0);
 
  return gtid;
}
 
/* caller must hold forkjoin_lock */
void __kmp_check_stack_overlap(kmp_info_t *th) {
  int f;
  char *stack_beg = NULL;
  char *stack_end = NULL;
  int gtid;
 
  KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"));
  if (__kmp_storage_map) {
    stack_end = (char *)th->th.th_info.ds.ds_stackbase;
    stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
 
    gtid = __kmp_gtid_from_thread(th);
 
    if (gtid == KMP_GTID_MONITOR) {
      __kmp_print_storage_map_gtid(
          gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
          "th_%s stack (%s)", "mon",
          (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
    } else {
      __kmp_print_storage_map_gtid(
          gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
          "th_%d stack (%s)", gtid,
          (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
    }
  }
 
  /* No point in checking ubermaster threads since they use refinement and
   * cannot overlap */
  gtid = __kmp_gtid_from_thread(th);
  if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
    KA_TRACE(10,
             ("__kmp_check_stack_overlap: performing extensive checking\n"));
    if (stack_beg == NULL) {
      stack_end = (char *)th->th.th_info.ds.ds_stackbase;
      stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
    }
 
    for (f = 0; f < __kmp_threads_capacity; f++) {
      kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
 
      if (f_th && f_th != th) {
        char *other_stack_end =
            (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
        char *other_stack_beg =
            other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
        if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
            (stack_end > other_stack_beg && stack_end < other_stack_end)) {
 
          /* Print the other stack values before the abort */
          if (__kmp_storage_map)
            __kmp_print_storage_map_gtid(
                -1, other_stack_beg, other_stack_end,
                (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
                "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
 
          __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
                      __kmp_msg_null);
        }
      }
    }
  }
  KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"));
}
 
/* ------------------------------------------------------------------------ */
 
void __kmp_infinite_loop(void) {
  static int done = FALSE;
 
  while (!done) {
    KMP_YIELD(TRUE);
  }
}
 
#define MAX_MESSAGE 512
 
void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
                                  char const *format, ...) {
  char buffer[MAX_MESSAGE];
  va_list ap;
 
  va_start(ap, format);
  KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
               p2, (unsigned long)size, format);
  __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
  __kmp_vprintf(kmp_err, buffer, ap);
#if KMP_PRINT_DATA_PLACEMENT
  int node;
  if (gtid >= 0) {
    if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
      if (__kmp_storage_map_verbose) {
        node = __kmp_get_host_node(p1);
        if (node < 0) /* doesn't work, so don't try this next time */
          __kmp_storage_map_verbose = FALSE;
        else {
          char *last;
          int lastNode;
          int localProc = __kmp_get_cpu_from_gtid(gtid);
 
          const int page_size = KMP_GET_PAGE_SIZE();
 
          p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
          p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
          if (localProc >= 0)
            __kmp_printf_no_lock("  GTID %d localNode %d\n", gtid,
                                 localProc >> 1);
          else
            __kmp_printf_no_lock("  GTID %d\n", gtid);
#if KMP_USE_PRCTL
          /* The more elaborate format is disabled for now because of the prctl
           * hanging bug. */
          do {
            last = p1;
            lastNode = node;
            /* This loop collates adjacent pages with the same host node. */
            do {
              (char *)p1 += page_size;
            } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
            __kmp_printf_no_lock("    %p-%p memNode %d\n", last, (char *)p1 - 1,
                                 lastNode);
          } while (p1 <= p2);
#else
          __kmp_printf_no_lock("    %p-%p memNode %d\n", p1,
                               (char *)p1 + (page_size - 1),
                               __kmp_get_host_node(p1));
          if (p1 < p2) {
            __kmp_printf_no_lock("    %p-%p memNode %d\n", p2,
                                 (char *)p2 + (page_size - 1),
                                 __kmp_get_host_node(p2));
          }
#endif
        }
      }
    } else
      __kmp_printf_no_lock("  %s\n", KMP_I18N_STR(StorageMapWarning));
  }
#endif /* KMP_PRINT_DATA_PLACEMENT */
  __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
}
 
void __kmp_warn(char const *format, ...) {
  char buffer[MAX_MESSAGE];
  va_list ap;
 
  if (__kmp_generate_warnings == kmp_warnings_off) {
    return;
  }
 
  va_start(ap, format);
 
  KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format);
  __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
  __kmp_vprintf(kmp_err, buffer, ap);
  __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
 
  va_end(ap);
}
 
void __kmp_abort_process() {
  // Later threads may stall here, but that's ok because abort() will kill them.
  __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
 
  if (__kmp_debug_buf) {
    __kmp_dump_debug_buffer();
  }
 
  if (KMP_OS_WINDOWS) {
    // Let other threads know of abnormal termination and prevent deadlock
    // if abort happened during library initialization or shutdown
    __kmp_global.g.g_abort = SIGABRT;
 
    /* On Windows* OS by default abort() causes pop-up error box, which stalls
       nightly testing. Unfortunately, we cannot reliably suppress pop-up error
       boxes. _set_abort_behavior() works well, but this function is not
       available in VS7 (this is not problem for DLL, but it is a problem for
       static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
       help, at least in some versions of MS C RTL.
 
       It seems following sequence is the only way to simulate abort() and
       avoid pop-up error box. */
    raise(SIGABRT);
    _exit(3); // Just in case, if signal ignored, exit anyway.
  } else {
    __kmp_unregister_library();
    abort();
  }
 
  __kmp_infinite_loop();
  __kmp_release_bootstrap_lock(&__kmp_exit_lock);
 
} // __kmp_abort_process
 
void __kmp_abort_thread(void) {
  // TODO: Eliminate g_abort global variable and this function.
  // In case of abort just call abort(), it will kill all the threads.
  __kmp_infinite_loop();
} // __kmp_abort_thread
 
/* Print out the storage map for the major kmp_info_t thread data structures
   that are allocated together. */
 
static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
  __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
                               gtid);
 
  __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
                               sizeof(kmp_desc_t), "th_%d.th_info", gtid);
 
  __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
                               sizeof(kmp_local_t), "th_%d.th_local", gtid);
 
  __kmp_print_storage_map_gtid(
      gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
      sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
 
  __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
                               &thr->th.th_bar[bs_plain_barrier + 1],
                               sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
                               gtid);
 
  __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
                               &thr->th.th_bar[bs_forkjoin_barrier + 1],
                               sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
                               gtid);
 
#if KMP_FAST_REDUCTION_BARRIER
  __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
                               &thr->th.th_bar[bs_reduction_barrier + 1],
                               sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
                               gtid);
#endif // KMP_FAST_REDUCTION_BARRIER
}
 
/* Print out the storage map for the major kmp_team_t team data structures
   that are allocated together. */
 
static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
                                         int team_id, int num_thr) {
  int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
  __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
                               header, team_id);
 
  __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
                               &team->t.t_bar[bs_last_barrier],
                               sizeof(kmp_balign_team_t) * bs_last_barrier,
                               "%s_%d.t_bar", header, team_id);
 
  __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
                               &team->t.t_bar[bs_plain_barrier + 1],
                               sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
                               header, team_id);
 
  __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
                               &team->t.t_bar[bs_forkjoin_barrier + 1],
                               sizeof(kmp_balign_team_t),
                               "%s_%d.t_bar[forkjoin]", header, team_id);
 
#if KMP_FAST_REDUCTION_BARRIER
  __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
                               &team->t.t_bar[bs_reduction_barrier + 1],
                               sizeof(kmp_balign_team_t),
                               "%s_%d.t_bar[reduction]", header, team_id);
#endif // KMP_FAST_REDUCTION_BARRIER
 
  __kmp_print_storage_map_gtid(
      -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
      sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
 
  __kmp_print_storage_map_gtid(
      -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
      sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
 
  __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
                               &team->t.t_disp_buffer[num_disp_buff],
                               sizeof(dispatch_shared_info_t) * num_disp_buff,
                               "%s_%d.t_disp_buffer", header, team_id);
}
 
static void __kmp_init_allocator() {
  __kmp_init_memkind();
  __kmp_init_target_mem();
}
static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
 
/* ------------------------------------------------------------------------ */
 
#if KMP_DYNAMIC_LIB
#if KMP_OS_WINDOWS
 
BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
  //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
 
  switch (fdwReason) {
 
  case DLL_PROCESS_ATTACH:
    KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"));
 
    return TRUE;
 
  case DLL_PROCESS_DETACH:
    KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
 
    // According to Windows* documentation for DllMain entry point:
    // for DLL_PROCESS_DETACH, lpReserved is used for telling the difference:
    //   lpReserved == NULL when FreeLibrary() is called,
    //   lpReserved != NULL when the process is terminated.
    // When FreeLibrary() is called, worker threads remain alive. So the
    // runtime's state is consistent and executing proper shutdown is OK.
    // When the process is terminated, worker threads have exited or been
    // forcefully terminated by the OS and only the shutdown thread remains.
    // This can leave the runtime in an inconsistent state.
    // Hence, only attempt proper cleanup when FreeLibrary() is called.
    // Otherwise, rely on OS to reclaim resources.
    if (lpReserved == NULL)
      __kmp_internal_end_library(__kmp_gtid_get_specific());
 
    return TRUE;
 
  case DLL_THREAD_ATTACH:
    KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"));
 
    /* if we want to register new siblings all the time here call
     * __kmp_get_gtid(); */
    return TRUE;
 
  case DLL_THREAD_DETACH:
    KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
 
    __kmp_internal_end_thread(__kmp_gtid_get_specific());
    return TRUE;
  }
 
  return TRUE;
}
 
#endif /* KMP_OS_WINDOWS */
#endif /* KMP_DYNAMIC_LIB */
 
/* __kmp_parallel_deo -- Wait until it's our turn. */
void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
  int gtid = *gtid_ref;
#ifdef BUILD_PARALLEL_ORDERED
  kmp_team_t *team = __kmp_team_from_gtid(gtid);
#endif /* BUILD_PARALLEL_ORDERED */
 
  if (__kmp_env_consistency_check) {
    if (__kmp_threads[gtid]->th.th_root->r.r_active)
#if KMP_USE_DYNAMIC_LOCK
      __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
#else
      __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
#endif
  }
#ifdef BUILD_PARALLEL_ORDERED
  if (!team->t.t_serialized) {
    KMP_MB();
    KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
             NULL);
    KMP_MB();
  }
#endif /* BUILD_PARALLEL_ORDERED */
}
 
/* __kmp_parallel_dxo -- Signal the next task. */
void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
  int gtid = *gtid_ref;
#ifdef BUILD_PARALLEL_ORDERED
  int tid = __kmp_tid_from_gtid(gtid);
  kmp_team_t *team = __kmp_team_from_gtid(gtid);
#endif /* BUILD_PARALLEL_ORDERED */
 
  if (__kmp_env_consistency_check) {
    if (__kmp_threads[gtid]->th.th_root->r.r_active)
      __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
  }
#ifdef BUILD_PARALLEL_ORDERED
  if (!team->t.t_serialized) {
    KMP_MB(); /* Flush all pending memory write invalidates.  */
 
    /* use the tid of the next thread in this team */
    /* TODO replace with general release procedure */
    team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
 
    KMP_MB(); /* Flush all pending memory write invalidates.  */
  }
#endif /* BUILD_PARALLEL_ORDERED */
}
 
/* ------------------------------------------------------------------------ */
/* The BARRIER for a SINGLE process section is always explicit   */
 
int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
  int status;
  kmp_info_t *th;
  kmp_team_t *team;
 
  if (!TCR_4(__kmp_init_parallel))
    __kmp_parallel_initialize();
  __kmp_resume_if_soft_paused();
 
  th = __kmp_threads[gtid];
  team = th->th.th_team;
  status = 0;
 
  th->th.th_ident = id_ref;
 
  if (team->t.t_serialized) {
    status = 1;
  } else {
    kmp_int32 old_this = th->th.th_local.this_construct;
 
    ++th->th.th_local.this_construct;
    /* try to set team count to thread count--success means thread got the
       single block */
    /* TODO: Should this be acquire or release? */
    if (team->t.t_construct == old_this) {
      status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
                                              th->th.th_local.this_construct);
    }
#if USE_ITT_BUILD
    if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
        KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&
        team->t.t_active_level == 1) {
      // Only report metadata by primary thread of active team at level 1
      __kmp_itt_metadata_single(id_ref);
    }
#endif /* USE_ITT_BUILD */
  }
 
  if (__kmp_env_consistency_check) {
    if (status && push_ws) {
      __kmp_push_workshare(gtid, ct_psingle, id_ref);
    } else {
      __kmp_check_workshare(gtid, ct_psingle, id_ref);
    }
  }
#if USE_ITT_BUILD
  if (status) {
    __kmp_itt_single_start(gtid);
  }
#endif /* USE_ITT_BUILD */
  return status;
}
 
void __kmp_exit_single(int gtid) {
#if USE_ITT_BUILD
  __kmp_itt_single_end(gtid);
#endif /* USE_ITT_BUILD */
  if (__kmp_env_consistency_check)
    __kmp_pop_workshare(gtid, ct_psingle, NULL);
}
 
/* determine if we can go parallel or must use a serialized parallel region and
 * how many threads we can use
 * set_nproc is the number of threads requested for the team
 * returns 0 if we should serialize or only use one thread,
 * otherwise the number of threads to use
 * The forkjoin lock is held by the caller. */
static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
                                 int master_tid, int set_nthreads,
                                 int enter_teams) {
  int capacity;
  int new_nthreads;
  KMP_DEBUG_ASSERT(__kmp_init_serial);
  KMP_DEBUG_ASSERT(root && parent_team);
  kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
 
  // If dyn-var is set, dynamically adjust the number of desired threads,
  // according to the method specified by dynamic_mode.
  new_nthreads = set_nthreads;
  if (!get__dynamic_2(parent_team, master_tid)) {
    ;
  }
#ifdef USE_LOAD_BALANCE
  else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
    new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
    if (new_nthreads == 1) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
                    "reservation to 1 thread\n",
                    master_tid));
      return 1;
    }
    if (new_nthreads < set_nthreads) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
                    "reservation to %d threads\n",
                    master_tid, new_nthreads));
    }
  }
#endif /* USE_LOAD_BALANCE */
  else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
    new_nthreads = __kmp_avail_proc - __kmp_nth +
                   (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
    if (new_nthreads <= 1) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
                    "reservation to 1 thread\n",
                    master_tid));
      return 1;
    }
    if (new_nthreads < set_nthreads) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
                    "reservation to %d threads\n",
                    master_tid, new_nthreads));
    } else {
      new_nthreads = set_nthreads;
    }
  } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
    if (set_nthreads > 2) {
      new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
      new_nthreads = (new_nthreads % set_nthreads) + 1;
      if (new_nthreads == 1) {
        KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
                      "reservation to 1 thread\n",
                      master_tid));
        return 1;
      }
      if (new_nthreads < set_nthreads) {
        KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
                      "reservation to %d threads\n",
                      master_tid, new_nthreads));
      }
    }
  } else {
    KMP_ASSERT(0);
  }
 
  // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
  if (__kmp_nth + new_nthreads -
          (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
      __kmp_max_nth) {
    int tl_nthreads = __kmp_max_nth - __kmp_nth +
                      (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
    if (tl_nthreads <= 0) {
      tl_nthreads = 1;
    }
 
    // If dyn-var is false, emit a 1-time warning.
    if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
      __kmp_reserve_warn = 1;
      __kmp_msg(kmp_ms_warning,
                KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
                KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
    }
    if (tl_nthreads == 1) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
                    "reduced reservation to 1 thread\n",
                    master_tid));
      return 1;
    }
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
                  "reservation to %d threads\n",
                  master_tid, tl_nthreads));
    new_nthreads = tl_nthreads;
  }
 
  // Respect OMP_THREAD_LIMIT
  int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
  int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
  if (cg_nthreads + new_nthreads -
          (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
      max_cg_threads) {
    int tl_nthreads = max_cg_threads - cg_nthreads +
                      (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
    if (tl_nthreads <= 0) {
      tl_nthreads = 1;
    }
 
    // If dyn-var is false, emit a 1-time warning.
    if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
      __kmp_reserve_warn = 1;
      __kmp_msg(kmp_ms_warning,
                KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
                KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
    }
    if (tl_nthreads == 1) {
      KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
                    "reduced reservation to 1 thread\n",
                    master_tid));
      return 1;
    }
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
                  "reservation to %d threads\n",
                  master_tid, tl_nthreads));
    new_nthreads = tl_nthreads;
  }
 
  // Check if the threads array is large enough, or needs expanding.
  // See comment in __kmp_register_root() about the adjustment if
  // __kmp_threads[0] == NULL.
  capacity = __kmp_threads_capacity;
  if (TCR_PTR(__kmp_threads[0]) == NULL) {
    --capacity;
  }
  // If it is not for initializing the hidden helper team, we need to take
  // __kmp_hidden_helper_threads_num out of the capacity because it is included
  // in __kmp_threads_capacity.
  if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
    capacity -= __kmp_hidden_helper_threads_num;
  }
  if (__kmp_nth + new_nthreads -
          (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
      capacity) {
    // Expand the threads array.
    int slotsRequired = __kmp_nth + new_nthreads -
                        (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
                        capacity;
    int slotsAdded = __kmp_expand_threads(slotsRequired);
    if (slotsAdded < slotsRequired) {
      // The threads array was not expanded enough.
      new_nthreads -= (slotsRequired - slotsAdded);
      KMP_ASSERT(new_nthreads >= 1);
 
      // If dyn-var is false, emit a 1-time warning.
      if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
        __kmp_reserve_warn = 1;
        if (__kmp_tp_cached) {
          __kmp_msg(kmp_ms_warning,
                    KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
                    KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
                    KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
        } else {
          __kmp_msg(kmp_ms_warning,
                    KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
                    KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
        }
      }
    }
  }
 
#ifdef KMP_DEBUG
  if (new_nthreads == 1) {
    KC_TRACE(10,
             ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
              "dead roots and rechecking; requested %d threads\n",
              __kmp_get_gtid(), set_nthreads));
  } else {
    KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
                  " %d threads\n",
                  __kmp_get_gtid(), new_nthreads, set_nthreads));
  }
#endif // KMP_DEBUG
  return new_nthreads;
}
 
/* Allocate threads from the thread pool and assign them to the new team. We are
   assured that there are enough threads available, because we checked on that
   earlier within critical section forkjoin */
static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
                                    kmp_info_t *master_th, int master_gtid,
                                    int fork_teams_workers) {
  int i;
  int use_hot_team;
 
  KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
  KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
  KMP_MB();
 
  /* first, let's setup the primary thread */
  master_th->th.th_info.ds.ds_tid = 0;
  master_th->th.th_team = team;
  master_th->th.th_team_nproc = team->t.t_nproc;
  master_th->th.th_team_master = master_th;
  master_th->th.th_team_serialized = FALSE;
  master_th->th.th_dispatch = &team->t.t_dispatch[0];
 
/* make sure we are not the optimized hot team */
#if KMP_NESTED_HOT_TEAMS
  use_hot_team = 0;
  kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
  if (hot_teams) { // hot teams array is not allocated if
    // KMP_HOT_TEAMS_MAX_LEVEL=0
    int level = team->t.t_active_level - 1; // index in array of hot teams
    if (master_th->th.th_teams_microtask) { // are we inside the teams?
      if (master_th->th.th_teams_size.nteams > 1) {
        ++level; // level was not increased in teams construct for
        // team_of_masters
      }
      if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
          master_th->th.th_teams_level == team->t.t_level) {
        ++level; // level was not increased in teams construct for
        // team_of_workers before the parallel
      } // team->t.t_level will be increased inside parallel
    }
    if (level < __kmp_hot_teams_max_level) {
      if (hot_teams[level].hot_team) {
        // hot team has already been allocated for given level
        KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
        use_hot_team = 1; // the team is ready to use
      } else {
        use_hot_team = 0; // AC: threads are not allocated yet
        hot_teams[level].hot_team = team; // remember new hot team
        hot_teams[level].hot_team_nth = team->t.t_nproc;
      }
    } else {
      use_hot_team = 0;
    }
  }
#else
  use_hot_team = team == root->r.r_hot_team;
#endif
  if (!use_hot_team) {
 
    /* install the primary thread */
    team->t.t_threads[0] = master_th;
    __kmp_initialize_info(master_th, team, 0, master_gtid);
 
    /* now, install the worker threads */
    for (i = 1; i < team->t.t_nproc; i++) {
 
      /* fork or reallocate a new thread and install it in team */
      kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
      team->t.t_threads[i] = thr;
      KMP_DEBUG_ASSERT(thr);
      KMP_DEBUG_ASSERT(thr->th.th_team == team);
      /* align team and thread arrived states */
      KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
                    "T#%d(%d:%d) join =%llu, plain=%llu\n",
                    __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
                    __kmp_gtid_from_tid(i, team), team->t.t_id, i,
                    team->t.t_bar[bs_forkjoin_barrier].b_arrived,
                    team->t.t_bar[bs_plain_barrier].b_arrived));
      thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
      thr->th.th_teams_level = master_th->th.th_teams_level;
      thr->th.th_teams_size = master_th->th.th_teams_size;
      { // Initialize threads' barrier data.
        int b;
        kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
        for (b = 0; b < bs_last_barrier; ++b) {
          balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
          KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
#if USE_DEBUGGER
          balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
#endif
        }
      }
    }
 
#if KMP_AFFINITY_SUPPORTED
    // Do not partition the places list for teams construct workers who
    // haven't actually been forked to do real work yet. This partitioning
    // will take place in the parallel region nested within the teams construct.
    if (!fork_teams_workers) {
      __kmp_partition_places(team);
    }
#endif
  }
 
  if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
    for (i = 0; i < team->t.t_nproc; i++) {
      kmp_info_t *thr = team->t.t_threads[i];
      if (thr->th.th_prev_num_threads != team->t.t_nproc ||
          thr->th.th_prev_level != team->t.t_level) {
        team->t.t_display_affinity = 1;
        break;
      }
    }
  }
 
  KMP_MB();
}
 
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// Propagate any changes to the floating point control registers out to the team
// We try to avoid unnecessary writes to the relevant cache line in the team
// structure, so we don't make changes unless they are needed.
inline static void propagateFPControl(kmp_team_t *team) {
  if (__kmp_inherit_fp_control) {
    kmp_int16 x87_fpu_control_word;
    kmp_uint32 mxcsr;
 
    // Get primary thread's values of FPU control flags (both X87 and vector)
    __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
    __kmp_store_mxcsr(&mxcsr);
    mxcsr &= KMP_X86_MXCSR_MASK;
 
    // There is no point looking at t_fp_control_saved here.
    // If it is TRUE, we still have to update the values if they are different
    // from those we now have. If it is FALSE we didn't save anything yet, but
    // our objective is the same. We have to ensure that the values in the team
    // are the same as those we have.
    // So, this code achieves what we need whether or not t_fp_control_saved is
    // true. By checking whether the value needs updating we avoid unnecessary
    // writes that would put the cache-line into a written state, causing all
    // threads in the team to have to read it again.
    KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
    KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
    // Although we don't use this value, other code in the runtime wants to know
    // whether it should restore them. So we must ensure it is correct.
    KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
  } else {
    // Similarly here. Don't write to this cache-line in the team structure
    // unless we have to.
    KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
  }
}
 
// Do the opposite, setting the hardware registers to the updated values from
// the team.
inline static void updateHWFPControl(kmp_team_t *team) {
  if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
    // Only reset the fp control regs if they have been changed in the team.
    // the parallel region that we are exiting.
    kmp_int16 x87_fpu_control_word;
    kmp_uint32 mxcsr;
    __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
    __kmp_store_mxcsr(&mxcsr);
    mxcsr &= KMP_X86_MXCSR_MASK;
 
    if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
      __kmp_clear_x87_fpu_status_word();
      __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
    }
 
    if (team->t.t_mxcsr != mxcsr) {
      __kmp_load_mxcsr(&team->t.t_mxcsr);
    }
  }
}
#else
#define propagateFPControl(x) ((void)0)
#define updateHWFPControl(x) ((void)0)
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
                                     int realloc); // forward declaration
 
/* Run a parallel region that has been serialized, so runs only in a team of the
   single primary thread. */
void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
  kmp_info_t *this_thr;
  kmp_team_t *serial_team;
 
  KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid));
 
  /* Skip all this code for autopar serialized loops since it results in
     unacceptable overhead */
  if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
    return;
 
  if (!TCR_4(__kmp_init_parallel))
    __kmp_parallel_initialize();
  __kmp_resume_if_soft_paused();
 
  this_thr = __kmp_threads[global_tid];
  serial_team = this_thr->th.th_serial_team;
 
  /* utilize the serialized team held by this thread */
  KMP_DEBUG_ASSERT(serial_team);
  KMP_MB();
 
  if (__kmp_tasking_mode != tskm_immediate_exec) {
    KMP_DEBUG_ASSERT(
        this_thr->th.th_task_team ==
        this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
    KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
                     NULL);
    KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
                  "team %p, new task_team = NULL\n",
                  global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
    this_thr->th.th_task_team = NULL;
  }
 
  kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
  if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
    proc_bind = proc_bind_false;
  } else if (proc_bind == proc_bind_default) {
    // No proc_bind clause was specified, so use the current value
    // of proc-bind-var for this parallel region.
    proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
  }
  // Reset for next parallel region
  this_thr->th.th_set_proc_bind = proc_bind_default;
 
#if OMPT_SUPPORT
  ompt_data_t ompt_parallel_data = ompt_data_none;
  void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
  if (ompt_enabled.enabled &&
      this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
 
    ompt_task_info_t *parent_task_info;
    parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
 
    parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
    if (ompt_enabled.ompt_callback_parallel_begin) {
      int team_size = 1;
 
      ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
          &(parent_task_info->task_data), &(parent_task_info->frame),
          &ompt_parallel_data, team_size,
          ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
    }
  }
#endif // OMPT_SUPPORT
 
  if (this_thr->th.th_team != serial_team) {
    // Nested level will be an index in the nested nthreads array
    int level = this_thr->th.th_team->t.t_level;
 
    if (serial_team->t.t_serialized) {
      /* this serial team was already used
         TODO increase performance by making this locks more specific */
      kmp_team_t *new_team;
 
      __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
 
      new_team =
          __kmp_allocate_team(this_thr->th.th_root, 1, 1,
#if OMPT_SUPPORT
                              ompt_parallel_data,
#endif
                              proc_bind, &this_thr->th.th_current_task->td_icvs,
                              0 USE_NESTED_HOT_ARG(NULL));
      __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
      KMP_ASSERT(new_team);
 
      /* setup new serialized team and install it */
      new_team->t.t_threads[0] = this_thr;
      new_team->t.t_parent = this_thr->th.th_team;
      serial_team = new_team;
      this_thr->th.th_serial_team = serial_team;
 
      KF_TRACE(
          10,
          ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
           global_tid, serial_team));
 
      /* TODO the above breaks the requirement that if we run out of resources,
         then we can still guarantee that serialized teams are ok, since we may
         need to allocate a new one */
    } else {
      KF_TRACE(
          10,
          ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
           global_tid, serial_team));
    }
 
    /* we have to initialize this serial team */
    KMP_DEBUG_ASSERT(serial_team->t.t_threads);
    KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
    KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
    serial_team->t.t_ident = loc;
    serial_team->t.t_serialized = 1;
    serial_team->t.t_nproc = 1;
    serial_team->t.t_parent = this_thr->th.th_team;
    serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
    this_thr->th.th_team = serial_team;
    serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
 
    KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d curtask=%p\n", global_tid,
                  this_thr->th.th_current_task));
    KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
    this_thr->th.th_current_task->td_flags.executing = 0;
 
    __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
 
    /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
       implicit task for each serialized task represented by
       team->t.t_serialized? */
    copy_icvs(&this_thr->th.th_current_task->td_icvs,
              &this_thr->th.th_current_task->td_parent->td_icvs);
 
    // Thread value exists in the nested nthreads array for the next nested
    // level
    if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
      this_thr->th.th_current_task->td_icvs.nproc =
          __kmp_nested_nth.nth[level + 1];
    }
 
    if (__kmp_nested_proc_bind.used &&
        (level + 1 < __kmp_nested_proc_bind.used)) {
      this_thr->th.th_current_task->td_icvs.proc_bind =
          __kmp_nested_proc_bind.bind_types[level + 1];
    }
 
#if USE_DEBUGGER
    serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
#endif
    this_thr->th.th_info.ds.ds_tid = 0;