/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	NJN/RBD
 *
 *	$Id: lamcomm.c,v 6.6 1999/08/01 05:16:05 jsquyres Exp $
 *
 *	Function:	- miscellaneous communicator functions
 */

#include <stdlib.h>
#include <string.h>

#include <lam_config.h>
#include <mpi.h>
#include <mpisys.h>
#include <typical.h>
#include <rpisys.h>
#include <debug.h>
#if LAM_WANT_IMPI
#include <impi.h>
#endif

/*
 * public functions
 */
int			lam_comm_free();
int			lam_comm_new();

/*
 * private functions
 */
#if WANT_IMPI
static int              num_remotes(MPI_Group group);
static MPI_Group        make_local(MPI_Group orig);
#endif

/*
 * private variables
 */
static int fl_debug = 0;

/*
 *	lam_comm_free
 *
 *	Function:	- really free a communicator
 *	Accepts:	- communicator
 *	Returns:	- 0 or LAMERROR
 */
int
lam_comm_free(comm)

MPI_Comm		comm;

{
	int		err;
	int		key;
	struct _attr	*p;
/*
 * Free the groups.
 */
	err = MPI_Group_free(&(comm->c_group));
	if (err != MPI_SUCCESS) return(err);

	if (LAM_IS_INTER(comm)) {
		err = MPI_Group_free(&(comm->c_rgroup));
		if (err != MPI_SUCCESS) return(err);
	}
/*
 * Free the attribute keys.
 */
	if (comm->c_keys) {
		p = (struct _attr *) ah_top(comm->c_keys);
		while (p) {
			key = p->a_key;
			err = MPI_Comm_delete_attr(comm, key);
			if (err != MPI_SUCCESS) return(err);

			err = MPI_Comm_free_keyval(&key);
			if (err != MPI_SUCCESS) return(err);

			p = (struct _attr *) ah_next(comm->c_keys, (void *) p);
		}
		ah_free(comm->c_keys);
	}
/*
 * Free the error handler.
 */
	err = MPI_Errhandler_free(&(comm->c_errhdl));
	if (err != MPI_SUCCESS) return(err);
/*
 * Free topology information.
 */
	if (LAM_IS_CART(comm)) {
		free((char *) comm->c_topo_dims);
	} else if (LAM_IS_GRAPH(comm)) {
		free((char *) comm->c_topo_index);
	}
/*
 * Free the context ID.
 */
	lam_rmcid(comm->c_contextid);
/*
 * Remove the communicator from the communicator list and free it.
 */
	al_delete(lam_comms, al_find(lam_comms, &comm));
	free((char *) comm);

	return(MPI_SUCCESS);
}


/*
 *	lam_comm_new
 *
 *	Function:	- initalize a new communicator
 *			- allocates storage for new communicator if necessary
 *	Accepts:	- context ID
 *			- local group
 *			- remote group
 *			- communicator flags
 *			- communicator (inout)
 *	Returns:	- 0 or LAMERROR
 */
int
lam_comm_new(cid, l_group, r_group, flags, comm)

int			cid;
MPI_Group		l_group;
MPI_Group		r_group;
int			flags;
MPI_Comm		*comm;

{
	if (*comm == 0) {
		if ((*comm = (MPI_Comm) malloc(sizeof(struct _comm))) == 0) {
			return(LAMERROR);
		}
	}

	(*comm)->c_flags = flags;
	(*comm)->c_f77handle = -1;
	(*comm)->c_refcount = 1;
	(*comm)->c_cube_dim = lam_cubedim(l_group->g_nprocs);
	(*comm)->c_group = l_group;
	(*comm)->c_rgroup = (flags & LAM_CINTER) ? r_group : MPI_GROUP_NULL;
	(*comm)->c_contextid = cid;
	(*comm)->c_errhdl = MPI_ERRORS_ARE_FATAL;
	(*comm)->c_keys = 0;
	(*comm)->c_topo_type = MPI_UNDEFINED;
	(*comm)->c_topo_nprocs = 0;
	(*comm)->c_topo_ndims = 0;
	(*comm)->c_topo_dims = 0;
	(*comm)->c_topo_index = 0;
	(*comm)->c_topo_nedges = 0;
	(*comm)->c_topo_edges = 0;
	(*comm)->c_window = MPI_WIN_NULL;

	memset((*comm)->c_name, 0, sizeof((*comm)->c_name));

	(*comm)->c_shadow = 0;
#if WANT_IMPI
        /* If at least one member of the local or remote group is not
	   within this LAM, we need to build a duplicate communicator
	   in c_shadow, but only with members that are local to this
	   LAM.  Shadow ranks will be kept in the same order as the
	   original communicator, but will be compressed to start with
	   zero. */
	if (_kio.ki_rtf & RTF_IMPI) {
	  DBUG("Checking for remote nodes\n");
	  if (num_remotes(l_group) > 0 || 
	      num_remotes((*comm)->c_rgroup) > 0) {
	    MPI_Group newl;
	    MPI_Group newr;
	    
	    DBUG("Found remote notes -- making new local groups\n");
	    newl = make_local(l_group);
	    newr = make_local((*comm)->c_rgroup);
	    DBUG("Making shadow communicator\n");
	    
	    /* Trickyness in lamcid.c -- cid's are incremented by 2, so
	       we know that we can grab this contextid + 1 */
	    
	    lam_comm_new(cid + 1, newl, newr, flags, &((*comm)->c_shadow));
	    (*comm)->c_shadow->c_errhdl->eh_refcount++;
	  }
	}
#endif

	DBUG("new communicator made!\n");
	return(0);
}


#if WANT_IMPI
/*
 *	num_remotes
 *
 *	Function:	- count to see how many procs in this group
 *                        are not in this LAM
 *	Accepts:	- group
 *	Returns:	- number of remote processes
 */
static int
num_remotes(MPI_Group group)
{
  int i, count = 0;

  if (group == MPI_GROUP_NULL || group == MPI_GROUP_EMPTY) {
    DBUG("No remote members of group\n");
    return 0;
  }

  DBUG("Checking for remote members of group\n");
  for (i = 0; i < group->g_nprocs; i++)
    if (LAM_GPSCMP(&(group->g_procs[i]->p_gps), &gimpid) == 0) {
      DBUG("Found remote member\n");
      count++;
    }

  DBUG("%d: Returning %d remote members of this group\n", 
       group->g_myrank, count);
  return count;
}



/*
 *	make_local
 *
 *	Function:	- make a new group that only has local members
 *	Accepts:	- group
 *	Returns:	- new group, subset of original, containing only
 *                        processes local to this LAM
 *
 */
static MPI_Group
make_local(MPI_Group orig)
{
  int i, j, num;
  int *ranks;
  MPI_Group newg;
  
  DBUG("Making local group\n");
  if (orig == MPI_GROUP_NULL || orig == MPI_GROUP_EMPTY)
    return orig;

  num = num_remotes(orig);
  ranks = malloc(sizeof(int) * num);
  for (j = i = 0; i < orig->g_nprocs; i++)
    if (LAM_GPSCMP(&(orig->g_procs[i]->p_gps), &gimpid) == 0) {
      DBUG("Found rank to exclude: %d (j = %d) pid %d\n", i, j,
	   orig->g_procs[i]->p_gps.gps_pid);
      ranks[j++] = i;
    }

  if (MPI_Group_excl(orig, num, ranks, &newg) != MPI_SUCCESS)
    DBUG("MPI GROUP EXCL BARF!!!\n");
  free(ranks);

  DBUG("Got new group size: %d\n", newg->g_nprocs);
  for (i = 0; i < newg->g_nprocs; i++)
    DBUG("PID in new group: %d\n", newg->g_procs[i]->p_gps.gps_pid);

  DBUG("All done making new group\n");
  return newg;
}

#endif
