/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1996 The Ohio State University
 *	RBD
 *
 *	$Id: barrier.c,v 6.3 1999/08/04 21:42:46 kmeyer1 Exp $
 *
 *	Function:	- barrier synchronization
 *	Accepts:	- communicator
 *	Returns:	- MPI_SUCCESS or error code
 */

#include <lam_config.h>
#include <blktype.h>
#include <mpi.h>
#include <mpisys.h>
#if LAM_WANT_IMPI
#include <impi.h>
#endif
/*
 * local functions
 */
static int		barrier_lin();
static int		barrier_log();

int
MPI_Barrier(comm)

MPI_Comm		comm;

{
	int		size;			/* group size */
/*
 * Prepare error handler if needed.
 */
	lam_initerr();
	lam_setfunc(BLKMPIBARRIER);
/*
 * Check for invalid arguments.
 */
	if (comm == MPI_COMM_NULL) {
		return(lam_errfunc(comm,
			BLKMPIBARRIER, lam_mkerr(MPI_ERR_COMM, 0)));
	}

	if (LAM_IS_INTER(comm)) {
		return(lam_errfunc(comm,
			BLKMPIBARRIER, lam_mkerr(MPI_ERR_COMM, 0)));
        }

	LAM_TRACE(lam_tr_cffstart(BLKMPIBARRIER));
/*
 * Decide which algorithm to use.
 */

#if LAM_WANT_IMPI
	if (LAM_IS_IMPI(comm))
	  return(IMPI_Barrier(comm));
	else {
	  MPI_Comm_size(comm, &size);
	  if (size <= LAM_COLLMAXLIN) {
	    return(barrier_lin(comm));
	  } else {
	    return(barrier_log(comm));
	  }
	}
	
#else
	MPI_Comm_size(comm, &size);

	if (size <= LAM_COLLMAXLIN) {
	  return(barrier_lin(comm));
	} else {
	  return(barrier_log(comm));
	}
#endif
}

/*
 *	barrier_lin
 *
 *	Function:	- barrier using O(N) algorithm
 *	Accepts:	- communicator
 *	Returns:	- MPI_SUCCESS or error code
 */
static int
barrier_lin(comm)

MPI_Comm		comm;

{
	int		size;			/* group size */
	int		rank;			/* caller rank */
	int		err;			/* error code */
	int		i;			/* favourite index */
	MPI_Status	stat;			/* receive status */

	MPI_Comm_size (comm, &size);
	MPI_Comm_rank(comm, &rank);

	lam_mkcoll(comm);
/*
 * All non-root send & receive zero-length message.
 */
	if (rank > 0) {
		err = MPI_Send((void *) 0, 0, MPI_BYTE,
					0, BLKMPIBARRIER, comm);

		if (err != MPI_SUCCESS) {
			lam_mkpt(comm);
			return(lam_errfunc(comm, BLKMPIBARRIER, err));
		}

		err = MPI_Recv((void *) 0, 0, MPI_BYTE,
					0, BLKMPIBARRIER, comm, &stat);

		if (err != MPI_SUCCESS) {
			lam_mkpt(comm);
			return(lam_errfunc(comm, BLKMPIBARRIER, err));
		}
	}
/*
 * The root collects and broadcasts the messages.
 */
	else {
		for (i = 1; i < size; ++i) {

			err = MPI_Recv((void *) 0, 0, MPI_BYTE, MPI_ANY_SOURCE,
						BLKMPIBARRIER, comm, &stat);

			if (err != MPI_SUCCESS) {
				lam_mkpt(comm);
				return(lam_errfunc(comm,
						BLKMPIBARRIER, err));
			}
		}

		for (i = 1; i < size; ++i) {

			err = MPI_Send((void *) 0, 0, MPI_BYTE,
						i, BLKMPIBARRIER, comm);

			if (err != MPI_SUCCESS) {
				lam_mkpt(comm);
				return(lam_errfunc(comm,
						BLKMPIBARRIER, err));
			}
		}
	}

	lam_mkpt(comm);

	LAM_TRACE(lam_tr_cffend(BLKMPIBARRIER, -1, comm, 0, 0));
	
	lam_resetfunc(BLKMPIBARRIER);
	return(MPI_SUCCESS);
}

/*
 *	barrier_log
 *
 *	Function:	- barrier using O(log(N)) algorithm
 *	Accepts:	- communicator
 *	Returns:	- MPI_SUCCESS or error code
 */
static int
barrier_log(comm)

MPI_Comm		comm;

{
	int		size;			/* group size */
	int		rank;			/* caller rank */
	int		peer;			/* peer rank */
	int		dim;			/* cube dimension */
	int		hibit;			/* high ON bit position */
	int		mask;			/* rank bit mask */
	int		err;			/* error code */
	int		i;			/* favourite index */
	MPI_Status	stat;			/* receive status */
/*
 * Send null-messages up and down the tree.
 * Synchronization at the root (rank 0).
 */
	MPI_Comm_rank(comm, &rank);
	MPI_Comm_size(comm, &size);
 
	lam_mkcoll(comm);

	dim = comm->c_cube_dim;
	hibit = lam_hibit(rank, dim);
	--dim;
/*
 * Receive from children.
 */
	for (i = dim, mask = 1 << i; i > hibit; --i, mask >>= 1) {

		peer = rank | mask;
		if (peer < size) {
			err = MPI_Recv((void *) 0, 0, MPI_BYTE,
					peer, BLKMPIBARRIER, comm, &stat);

			if (err != MPI_SUCCESS) {
				lam_mkpt(comm);
				return(lam_errfunc(comm,
						BLKMPIBARRIER, err));
			}
		}
	}
/*
 * Send to and receive from parent.
 */
	if (rank > 0) {

		peer = rank & ~(1 << hibit);

		err = MPI_Send((void *) 0, 0, MPI_BYTE,
						peer, BLKMPIBARRIER, comm);
		if (err != MPI_SUCCESS) {
			lam_mkpt(comm);
			return(lam_errfunc(comm, BLKMPIBARRIER, err));
		}

		err = MPI_Recv((void *) 0, 0, MPI_BYTE, peer,
						BLKMPIBARRIER, comm, &stat);
		if (err != MPI_SUCCESS) {
			lam_mkpt(comm);
			return(lam_errfunc(comm, BLKMPIBARRIER, err));
		}
	}
/*
 * Send to children.
 */
	for (i = hibit + 1, mask = 1 << i; i <= dim; ++i, mask <<= 1) {

		peer = rank | mask;
		if (peer < size) {
			err = MPI_Send((void *) 0, 0, MPI_BYTE,
						peer, BLKMPIBARRIER, comm);

			if (err != MPI_SUCCESS) {
				lam_mkpt(comm);
				return(lam_errfunc(comm,
						BLKMPIBARRIER, err));
			}
		}
	}

	lam_mkpt(comm);

	LAM_TRACE(lam_tr_cffend(BLKMPIBARRIER, -1, comm, 0, 0));

	lam_resetfunc(BLKMPIBARRIER);
	return(MPI_SUCCESS);
}
