/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	RBD/GDB/NJN
 *
 *	$Id: cdup.c,v 6.5 1999/08/04 21:43:49 kmeyer1 Exp $
 *
 *	Function:	- duplicate a communicator
 *	Accepts:	- old communicator
 *			- new communicator (out)
 *	Returns:	- MPI_SUCCESS or error code
 */

#include <errno.h>
#include <stdlib.h>
#include <string.h>

#include <lam_config.h>
#include <all_hash.h>
#include <blktype.h>
#include <mpi.h>
#include <mpisys.h>
#include <rpisys.h>


int
MPI_Comm_dup(comm, newcomm)

MPI_Comm		comm;
MPI_Comm		*newcomm;

{
	MPI_Request	req;			
	MPI_Status	stat;
	int		mycid;			/* my context ID */
	int		cid;			/* global context ID */
	int		err;			/* error code */
	int		fl_copy;		/* attr. copy flag */
	int		size;			/* group size */
	struct _attr	elem;			/* attr. element */
	struct _attr	*p;			/* favourite pointer */
	struct _attrkey	*pk;			/* key pointer */

	lam_initerr_m();
	lam_setfunc_m(BLKMPICOMMDUP);
/*
 * Check the arguments.
 */
	if (comm == MPI_COMM_NULL) {
	    return(lam_errfunc(MPI_COMM_WORLD, BLKMPICOMMDUP,
		    		lam_mkerr(MPI_ERR_COMM, 0)));
	}

	if (newcomm == 0) {
	    return(lam_errfunc(comm, BLKMPICOMMDUP,
		    		lam_mkerr(MPI_ERR_ARG, 0)));
	}

	
#if LAM_WANT_IMPI

	/* Remove this when IMPI collectives are implemented */

        if (LAM_IS_IMPI(comm)) {
	  return lam_err_comm(comm, MPI_ERR_COMM, 0, 
			      "Collectives not yet implemented on IMPI communicators");
	}
#endif


/*
 * Set up tracing.
 */
	LAM_TRACE(lam_tr_cffstart(BLKMPICOMMDUP));
/*
 * Create the new context ID.
 */
	mycid = lam_getcid();
	if (mycid < 0) {
	    return(lam_errfunc(comm, BLKMPICOMMDUP,
		    		lam_mkerr(MPI_ERR_INTERN, EFULL)));
	}
/*
 * If intra-comm, do an all-reduce.
 */
	if (LAM_IS_INTRA(comm)) {
	    err = MPI_Allreduce(&mycid, &cid, 1, MPI_INT, MPI_MAX, comm);
	}
/*
 * If inter-comm, do a local reduce to leader (0) then the leaders
 * exchange the context ID and broadcast to their local group.
 * We cheat by flip-flopping the communicator inter/intra state.
 */
	else {
	    comm->c_flags &= ~LAM_CINTER;
	    err = MPI_Reduce(&mycid, &cid, 1, MPI_INT, MPI_MAX, 0, comm);
	    comm->c_flags |= LAM_CINTER;

	    if (err != MPI_SUCCESS) {
		LAM_TRACE(lam_tr_cffend(BLKMPICOMMDUP, -1, comm, 0, 0));
		lam_resetfunc(BLKMPICOMMDUP);
		return(lam_errfunc(comm, BLKMPICOMMDUP, err));
	    }

	    if (comm->c_group->g_myrank ==  0) {

		err = lam_irecv(&mycid, 1, MPI_INT,
					0, MPI_MERGE_TAG, comm, &req);

		if (err == MPI_SUCCESS) {
		    err = lam_send(&cid, 1, MPI_INT,
					0, MPI_MERGE_TAG, comm, LAM_RQISEND);
		}

		if (err == MPI_SUCCESS) {
		    err = MPI_Wait(&req, &stat);
		}

		if (err != MPI_SUCCESS) {
		    LAM_TRACE(lam_tr_cffend(BLKMPICOMMDUP, -1, comm, 0, 0));
		    lam_resetfunc(BLKMPICOMMDUP);
		    return(lam_errfunc(comm, BLKMPICOMMDUP, err));
		}

		if (mycid > cid) cid = mycid;
	    }

	    comm->c_flags &= ~LAM_CINTER;
	    err = MPI_Bcast(&cid, 1, MPI_INT, 0, comm);
	    comm->c_flags |= LAM_CINTER;
	}

	if (err != MPI_SUCCESS) {
	    LAM_TRACE(lam_tr_cffend(BLKMPICOMMDUP, -1, comm, 0, 0));
	    lam_resetfunc(BLKMPICOMMDUP);
	    return(lam_errfunc(comm, BLKMPICOMMDUP, err));
	}
/*
 * Create the new communicator.
 */
	*newcomm = 0;
	if (lam_comm_new(cid, comm->c_group, comm->c_rgroup,
				comm->c_flags & ~LAM_PREDEF, newcomm)) {
	    return(lam_errfunc(comm, BLKMPICOMMDUP,
				lam_mkerr(MPI_ERR_OTHER, errno)));
	}
/*
 * Copy the key attributes using the copy callback functions.
 */
	if (comm->c_keys) {

	    (*newcomm)->c_keys = ah_init((int4) ah_size(comm->c_keys),
		    (int4) sizeof(struct _attr),
		    (int4) MPI_KEYVAL_INVALID, INT4_NIL);

	    if ((*newcomm)->c_keys == 0) {
		return(lam_errfunc(comm, BLKMPICOMMDUP,
				lam_mkerr(MPI_ERR_OTHER, errno)));
	    }

	    p = (struct _attr *) ah_top(comm->c_keys);

	    while (p) {
		pk = lam_getattr(p->a_key);
		if (pk == 0) {
		    return(lam_errfunc(comm, BLKMPICOMMDUP,
			    lam_mkerr(MPI_ERR_INTERN, errno)));
		}
/*
 * Call the C or F77 callback function.
 */
		fl_copy = 0;
		if (((MPI_Copy_function *) (pk->ak_copy)) != MPI_NULL_COPY_FN) {
		    if (pk->ak_flags & LAM_LANGF77) {			
			(*((MPI_F_copy_function *) (pk->ak_copy)))
			    	(&comm->c_f77handle, &p->a_key,
				(int *) &pk->ak_extra, (int *) &p->a_value,
				(int *) &(elem.a_value), &fl_copy, &err);
		    } else {
			err = (*((MPI_Copy_function *) (pk->ak_copy)))
			    	(comm, p->a_key, pk->ak_extra,
				p->a_value, &(elem.a_value), &fl_copy);
		    }
		}

		if (err != MPI_SUCCESS) {
		    return(lam_errfunc(comm, BLKMPICOMMDUP, lam_mkerr(err, 0)));
		}	
/*
 * If attribute is copied, insert it and increment key reference count.
 */
		if (fl_copy) {
		    elem.a_key = p->a_key;

		    if (ah_insert((*newcomm)->c_keys, &elem)) {
			return(lam_errfunc(comm, BLKMPICOMMDUP,
					lam_mkerr(MPI_ERR_OTHER, errno)));
		    }

		    pk->ak_refcount++;
		}

		p = (struct _attr *) ah_next(comm->c_keys, p);
	    }
	}
/*
 * Copy the topology.
 */
	(*newcomm)->c_topo_type = comm->c_topo_type;
	(*newcomm)->c_topo_nprocs = comm->c_topo_nprocs;
	(*newcomm)->c_topo_ndims = comm->c_topo_ndims;
	(*newcomm)->c_topo_nedges = comm->c_topo_nedges;

	if (LAM_IS_CART(comm)) {
	    size = 2 * comm->c_topo_ndims * sizeof(int);

	    (*newcomm)->c_topo_dims = (int *) malloc((unsigned) size);
	    if ((*newcomm)->c_topo_dims == 0) {
		return(lam_errfunc(comm, BLKMPICOMMDUP,
				lam_mkerr(MPI_ERR_OTHER, errno)));
	    }

	    (*newcomm)->c_topo_coords =
				(*newcomm)->c_topo_dims + comm->c_topo_ndims;
	    memcpy((char *) (*newcomm)->c_topo_dims,
		    (char *) comm->c_topo_dims, size);
	}
	else if (LAM_IS_GRAPH(comm)) {
	    size = (comm->c_topo_nprocs+comm->c_topo_nedges) * sizeof(int);

	    (*newcomm)->c_topo_index = (int *) malloc((unsigned) size);
	    if ((*newcomm)->c_topo_index == 0) {
		return(lam_errfunc(comm, BLKMPICOMMDUP,
				lam_mkerr(MPI_ERR_OTHER, errno)));
	    }

	    (*newcomm)->c_topo_edges =
				(*newcomm)->c_topo_index + comm->c_topo_nprocs;
	    memcpy((char *) (*newcomm)->c_topo_index,
		    (char *) comm->c_topo_index, size);
	}
/*
 * Copy the error handler.
 */
	(*newcomm)->c_errhdl = comm->c_errhdl;

	comm->c_errhdl->eh_refcount++;
	comm->c_group->g_refcount++;
	if (LAM_IS_INTER(comm)) {
	    comm->c_rgroup->g_refcount++;
	}

	if (!al_insert(lam_comms, newcomm)) {
	    return(lam_errfunc(comm, BLKMPICOMMDUP,
				lam_mkerr(MPI_ERR_INTERN, errno)));
	}
	
	if (lam_tr_comm(*newcomm)) {
	    return(lam_errfunc(comm, BLKMPICOMMDUP,
		lam_mkerr(MPI_ERR_INTERN, errno)));
	}

	lam_setcid(cid);

	LAM_TRACE(lam_tr_cffend(BLKMPICOMMDUP, -1, comm, 0, 0));

	lam_resetfunc_m(BLKMPICOMMDUP);
	return(MPI_SUCCESS);
}
