kmp_lock.cpp

/*
 * kmp_lock.cpp -- lock-related functions
 */
 
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include <stddef.h>
#include <atomic>
 
#include "kmp.h"
#include "kmp_i18n.h"
#include "kmp_io.h"
#include "kmp_itt.h"
#include "kmp_lock.h"
#include "kmp_wait_release.h"
#include "kmp_wrapper_getpid.h"
 
#if KMP_USE_FUTEX
#include <sys/syscall.h>
#include <unistd.h>
// We should really include <futex.h>, but that causes compatibility problems on
// different Linux* OS distributions that either require that you include (or
// break when you try to include) <pci/types.h>. Since all we need is the two
// macros below (which are part of the kernel ABI, so can't change) we just
// define the constants here and don't include <futex.h>
#ifndef FUTEX_WAIT
#define FUTEX_WAIT 0
#endif
#ifndef FUTEX_WAKE
#define FUTEX_WAKE 1
#endif
#endif
 
/* Implement spin locks for internal library use.             */
/* The algorithm implemented is Lamport's bakery lock [1974]. */
 
void __kmp_validate_locks(void) {
  int i;
  kmp_uint32 x, y;
 
  /* Check to make sure unsigned arithmetic does wraps properly */
  x = ~((kmp_uint32)0) - 2;
  y = x - 2;
 
  for (i = 0; i < 8; ++i, ++x, ++y) {
    kmp_uint32 z = (x - y);
    KMP_ASSERT(z == 2);
  }
 
  KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
}
 
/* ------------------------------------------------------------------------ */
/* test and set locks */
 
// For the non-nested locks, we can only assume that the first 4 bytes were
// allocated, since gcc only allocates 4 bytes for omp_lock_t, and the Intel
// compiler only allocates a 4 byte pointer on IA-32 architecture.  On
// Windows* OS on Intel(R) 64, we can assume that all 8 bytes were allocated.
//
// gcc reserves >= 8 bytes for nested locks, so we can assume that the
// entire 8 bytes were allocated for nested locks on all 64-bit platforms.
 
static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
  return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
}
 
static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
  return lck->lk.depth_locked != -1;
}
 
__forceinline static int
__kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
  KMP_MB();
 
#ifdef USE_LOCK_PROFILE
  kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
  if ((curr != 0) && (curr != gtid + 1))
    __kmp_printf("LOCK CONTENTION: %p\n", lck);
/* else __kmp_printf( "." );*/
#endif /* USE_LOCK_PROFILE */
 
  kmp_int32 tas_free = KMP_LOCK_FREE(tas);
  kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
 
  if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
      __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
    KMP_FSYNC_ACQUIRED(lck);
    return KMP_LOCK_ACQUIRED_FIRST;
  }
 
  kmp_uint32 spins;
  kmp_uint64 time;
  KMP_FSYNC_PREPARE(lck);
  KMP_INIT_YIELD(spins);
  KMP_INIT_BACKOFF(time);
  kmp_backoff_t backoff = __kmp_spin_backoff_params;
  do {
#if !KMP_HAVE_UMWAIT
    __kmp_spin_backoff(&backoff);
#else
    if (!__kmp_tpause_enabled)
      __kmp_spin_backoff(&backoff);
#endif
    KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
  } while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
           !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
  KMP_FSYNC_ACQUIRED(lck);
  return KMP_LOCK_ACQUIRED_FIRST;
}
 
int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
  int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
  return retval;
}
 
static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
                                              kmp_int32 gtid) {
  char const *const func = "omp_set_lock";
  if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
      __kmp_is_tas_lock_nestable(lck)) {
    KMP_FATAL(LockNestableUsedAsSimple, func);
  }
  if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
    KMP_FATAL(LockIsAlreadyOwned, func);
  }
  return __kmp_acquire_tas_lock(lck, gtid);
}
 
int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
  kmp_int32 tas_free = KMP_LOCK_FREE(tas);
  kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
  if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
      __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
    KMP_FSYNC_ACQUIRED(lck);
    return TRUE;
  }
  return FALSE;
}
 
static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
                                           kmp_int32 gtid) {
  char const *const func = "omp_test_lock";
  if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
      __kmp_is_tas_lock_nestable(lck)) {
    KMP_FATAL(LockNestableUsedAsSimple, func);
  }
  return __kmp_test_tas_lock(lck, gtid);