/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	RBD/NJN
 *
 *	$Id: icmerge.c,v 6.4 1999/08/04 21:43:50 kmeyer1 Exp $
 *
 *	Function:	- merge inter-communicator into intra-communicator
 *	Accepts:	- inter-communicator
 *			- high/low flag
 *			- new intra-communicator (out)
 *	Returns:	- MPI_SUCCESS or error code
 */

#include <errno.h>
#include <stdlib.h>

#include <lam_config.h>
#include <blktype.h>
#include <mpi.h>
#include <mpisys.h>
#include <rpisys.h>


int
MPI_Intercomm_merge(comm, high, newcomm)

MPI_Comm		comm;
int			high;
MPI_Comm		*newcomm;

{
	MPI_Request	req;
	MPI_Status	stat;
	MPI_Group	newgroup;		/* new group */
	MPI_Group	g1, g2;			/* 2 groups to merge */
	int		mycid;			/* my context ID */
	int		cid;			/* global context ID */
	int		rhigh;			/* remote high value */
	int		err;			/* error code */
	int		inmsg[2];		/* input message buffer */
	int		outmsg[2];		/* output message buffer */
	int		i;

	lam_initerr_m();
	lam_setfunc_m(BLKMPIICOMMMERGE);
/*
 * Check the arguments.
 */
	if (comm == MPI_COMM_NULL) {
		return(lam_errfunc(MPI_COMM_WORLD,
			BLKMPIICOMMMERGE, lam_mkerr(MPI_ERR_COMM, 0)));
	}

	if (LAM_IS_INTRA(comm)) {
		return(lam_errfunc(comm,
			BLKMPIICOMMMERGE, lam_mkerr(MPI_ERR_COMM, 0)));
	}

	if (newcomm == 0) {
		return(lam_errfunc(comm,
			BLKMPIICOMMMERGE, lam_mkerr(MPI_ERR_ARG, 0)));
	}

	LAM_TRACE(lam_tr_cffstart(BLKMPIICOMMMERGE));

#if LAM_WANT_IMPI

	/* Remove this when IMPI collectives are implemented */

        if (LAM_IS_IMPI(comm)) {
	  return lam_err_comm(comm, MPI_ERR_COMM, 0, 
			      "Collectives not yet implemented on IMPI communicators");
	}
#endif

/*
 * Create the new context ID: rank0 processes are the leaders of their
 * local group.  Each leader finds the max context ID of all remote group
 * processes (excluding their leader) and their "high" setting.
 * The leaders then swap the information and broadcast to the remote group.
 * Note: this is a criss-cross effect, processes talk to the remote leader.
 */
	mycid = lam_getcid();
	if (mycid < 0) {
		return(lam_errfunc(comm, BLKMPIICOMMMERGE,
				lam_mkerr(MPI_ERR_INTERN, EFULL)));
	}

	if (comm->c_group->g_myrank == 0) {
		cid = mycid;
/*
 * Find max context ID of remote non-leader processes.
 */
		for (i = 1; i < comm->c_rgroup->g_nprocs; ++i) {

			err = lam_recv(inmsg, 1, MPI_INT,
					i, MPI_MERGE_TAG, comm, &stat);
			if (err != MPI_SUCCESS) {
				return(lam_errfunc(comm,
						BLKMPIICOMMMERGE, err));
			}

			if (inmsg[0] > cid) cid = inmsg[0];
		}
/*
 * Swap context ID and high value with remote leader.
 */
		outmsg[0] = cid;
		outmsg[1] = high;

		err = lam_irecv(inmsg, 2, MPI_INT,
					0, MPI_MERGE_TAG, comm, &req);

		if (err == MPI_SUCCESS) {
		    err = lam_send(outmsg, 2, MPI_INT,
			    		0, MPI_MERGE_TAG, comm, LAM_RQISEND);
		}

		if (err == MPI_SUCCESS) {
		    err = MPI_Wait(&req, &stat);
		}

		if (err != MPI_SUCCESS) {
			return(lam_errfunc(comm, BLKMPIICOMMMERGE, err));
		}

		if (inmsg[0] > cid) cid = inmsg[0];

		rhigh = inmsg[1];
/*
 * Broadcast context ID and local high to remote non-leader processes.
 */
		outmsg[0] = cid;
		outmsg[1] = high;

		for (i = 1; i < comm->c_rgroup->g_nprocs; ++i) {

			err = lam_send(outmsg, 2, MPI_INT,
					i, MPI_MERGE_TAG, comm, LAM_RQISEND);
			if (err != MPI_SUCCESS) {
				return(lam_errfunc(comm,
						BLKMPIICOMMMERGE, err));
			}
		}
	}
/*
 * non-leader
 */
	else {
		err = lam_send(&mycid, 1, MPI_INT,
				0, MPI_MERGE_TAG, comm, LAM_RQISEND);
		if (err != MPI_SUCCESS) {
			return(lam_errfunc(comm, BLKMPIICOMMMERGE, err));
		}

		err = lam_recv(inmsg, 2, MPI_INT, 0,
				MPI_MERGE_TAG, comm, &stat);
		if (err != MPI_SUCCESS) {
			return(lam_errfunc(comm, BLKMPIICOMMMERGE, err));
		}

		cid = inmsg[0];
		rhigh = inmsg[1];
	}
/*
 * All procs know the "high" for local and remote groups and the context ID.
 * Create the properly ordered union group.  In case of equal high values,
 * the group that has the leader with the "lowest process" goes first.
 */
	if (high && (!rhigh)) {
		g1 = comm->c_rgroup;
		g2 = comm->c_group;
	} else if ((!high) && rhigh) {
		g1 = comm->c_group;
		g2 = comm->c_rgroup;
	} else if (lam_proccmp(comm->c_group->g_procs[0],
				comm->c_rgroup->g_procs[0]) < 0) {
		g1 = comm->c_group;
		g2 = comm->c_rgroup;
	} else {
		g1 = comm->c_rgroup;
		g2 = comm->c_group;
	}

	err = MPI_Group_union(g1, g2, &newgroup);
	if (err != MPI_SUCCESS) {
		return(lam_errfunc(comm, BLKMPIICOMMMERGE, err));
	}
/*
 * Create the new communicator.
 */
	*newcomm = 0;
	if (lam_comm_new(cid, newgroup, MPI_GROUP_NULL, 0, newcomm)) {
		return(lam_errfunc(comm, BLKMPIICOMMMERGE,
				lam_mkerr(MPI_ERR_OTHER, errno)));
	}

	(*newcomm)->c_errhdl = comm->c_errhdl;
	comm->c_errhdl->eh_refcount++;

	if (!al_insert(lam_comms, newcomm)) {
		return(lam_errfunc(comm, BLKMPIICOMMMERGE,
				lam_mkerr(MPI_ERR_INTERN, errno)));
	}

	if (lam_tr_comm(*newcomm)) {
		return(lam_errfunc(comm, BLKMPIICOMMMERGE,
				lam_mkerr(MPI_ERR_INTERN, errno)));
	}

	lam_setcid(cid);

	LAM_TRACE(lam_tr_cffend(BLKMPIICOMMMERGE, -1, comm, 0, 0));

	lam_resetfunc(BLKMPIICOMMMERGE);
	return(MPI_SUCCESS);
}
