/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	NJN
 *
 *	$Id: join.c,v 1.3 1999/08/30 21:38:58 jsquyres Exp $
 *
 *	Function:	- join MPI applications
 *	Accepts:	- socket file descriptor
 *			- intercomm between applications (out)
 *	Returns:	- MPI_SUCCESS or error code
 */

#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/time.h>				/* LINUX FD_SET etc. */
#include <sys/types.h>
#include <sys/socket.h>
#include <unistd.h>

#include <lam_config.h>

#if LAM_NEED_SYS_SELECT_H
#include <sys/select.h>
#endif

#include <app_mgmt.h>
#include <all_list.h>
#include <blktype.h>
#include <mpi.h>
#include <mpisys.h>
#include <portable.h>
#include <rpisys.h>
#include <typical.h>


/*
 * private functions
 */
static int		exchange_with_peer();
static int		readwrite();


int
MPI_Comm_join(fd, newcomm)

int			fd;
MPI_Comm		*newcomm;

{
	struct _proc	*p;			/* peer process */
	struct _gps	peergps;		/* peer GPS */
	MPI_Group	peergrp;		/* peer group */
	int		peercid;		/* peer ID */
	int		cid;			/* new context ID */

	lam_initerr_m();
	lam_setfunc_m(BLKMPICOMMJOIN);
/*
 * Check the arguments.
 */
	if (fd < 0) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_ARG, 0,
			"illegal file descriptor"));
	}

	if (newcomm == 0) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_ARG, 0,
			"null pointer"));
	}

	LAM_TRACE(lam_tr_cffstart(BLKMPICOMMJOIN));

	cid = lam_getcid();

	if (cid < 0) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_INTERN, EFULL, ""));
	}
/*
 * Exchange cid and GPS with peer.
 */
	if (exchange_with_peer(fd, cid, &peercid, &peergps)) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_OTHER, errno, ""));
	}
/*
 * The intercomm context ID is the maximium of the local and peer CIDs.
 */
	cid = LAM_max(cid, peercid);
/*
 * Create the remote (client) group.
 */
	peergrp = (MPI_Group) 
		malloc(sizeof(struct _group) + sizeof(struct _proc *));
	if (peergrp == 0) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_OTHER, errno, ""));
	}
	peergrp->g_nprocs = 1;
	peergrp->g_myrank = MPI_UNDEFINED;
	peergrp->g_refcount = 1;
	peergrp->g_f77handle = -1;
	peergrp->g_procs = (struct _proc **)
				((char *) peergrp + sizeof(struct _group));

	p = *peergrp->g_procs = lam_procadd(&peergps);
	if (p == 0) {
		free((char *) peergrp);
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_OTHER, errno, ""));
	}
	if (!(p->p_mode & LAM_PRPIINIT)) {
		p->p_mode |= LAM_PCLIENT;
	}
	p->p_refcount++;
/*
 * Create the server-client intercommunicator.
 */
	*newcomm = 0;
	if (lam_comm_new(cid, MPI_COMM_SELF->c_group,
				peergrp, LAM_CINTER, newcomm)) {

		free((char *) peergrp);
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_OTHER, errno, ""));
	}

	MPI_COMM_SELF->c_group->g_refcount++;
	(*newcomm)->c_errhdl = MPI_COMM_SELF->c_errhdl;
	MPI_COMM_SELF->c_errhdl->eh_refcount++;

	if (!al_insert(lam_comms, newcomm)) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_INTERN, errno, ""));
	}

	if (lam_tr_comm(*newcomm)) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_INTERN, errno, ""));
	}

	lam_setcid(cid);
/*
 * Setup the new processes.
 */
	if (RPI_SPLIT(_rpi_lamd_addprocs, _rpi_c2c_addprocs, ())) {
		return(lam_err_comm(MPI_COMM_SELF, MPI_ERR_OTHER, errno, ""));
	}

	LAM_TRACE(lam_tr_cffend(BLKMPICOMMJOIN, 0, 0, 0, 0));

        lam_resetfunc_m(BLKMPICOMMJOIN);
	return(MPI_SUCCESS);
}

/*
 *	exchange_with_peer
 *
 *	Function:	- exchange information with peer over a socket
 *	Accepts:	- socket file descriptor
 *			- local cid
 *			- peer cid (out)
 *			- peer GPS (out)
 *	Returns:	- 0 or LAMERROR
 */
static int
exchange_with_peer(fd, cid, peercid, peergps)

int			fd;
int			cid;
int			*peercid;
struct _gps		*peergps;

{
	struct _gps	mygps;
	int4		cid4;
	int4		peercid4;

	mygps = lam_myproc->p_gps;
	mltoti4(&mygps, sizeof(struct _gps) / sizeof(int4));
	cid4 = (int4) cid;
	mltoti4(&cid4, 1);

	if (readwrite(fd, sizeof(struct _gps),
			(char *) peergps, (char *) &mygps)) {
		return(LAMERROR);
	}

	if (readwrite(fd, sizeof(int4),
			(char *) &peercid4, (char *) &cid4)) {
		return(LAMERROR);
	}

	mttoli4(peergps, sizeof(struct _gps) / sizeof(int4));
	mttoli4(&peercid4, 1);
	*peercid = (int) peercid4;
	return(0);
}

/*
 *	readwrite
 *
 *	Function:	- exchange bytes over a socket
 *	Accepts:	- socket file descriptor
 *			- number of bytes to exchange
 *			- incoming data buffer
 *			- outgoing data buffer (filled)
 *	Returns:	- 0 or LAMERROR
 */
static int
readwrite(fd, nbytes, inbuf, outbuf)

int			fd;
int			nbytes;
char			*inbuf;
char			*outbuf;

{
	fd_set		rfd;
	fd_set		wfd;
	int		ntowrite;		/* # bytes left to write */
	int		ntoread;		/* # bytes left to read */
	int		nready;			/* # ready file descriptors */
	int		r;

	FD_ZERO(&rfd);
	FD_ZERO(&wfd);
	ntoread = nbytes;
	ntowrite = nbytes;
/*
 * Loop writing and reading until all is done.  Write first to avoid
 * deadlock.
 */
	while (ntowrite > 0 || ntoread > 0) {

		if (ntowrite > 0) {
			FD_SET(fd, &wfd);
		} else {
			FD_CLR(fd, &wfd);
		}

		if (ntoread > 0) {
			FD_SET(fd, &rfd);
		} else {
			FD_CLR(fd, &rfd);
		}

		do {
			nready = select(fd + 1, &rfd, &wfd,
					(fd_set *) 0, (struct timeval *) 0);

			if (nready < 0 && errno != EINTR) {
				return(LAMERROR);
			}
		} while (nready < 0);

		if (FD_ISSET(fd, &wfd)) {
			r = write(fd, outbuf, ntowrite);

			if (r < 0) {
				if (errno != EINTR) {
					return(LAMERROR);
				}	
			} else if (r == 0) {
				errno = EEOF;
			} else {
				ntowrite -= r;
				outbuf += r;
			}
		}

		if (FD_ISSET(fd, &rfd)) {
			r = read(fd, inbuf, ntoread);

			if (r < 0) {
				if (errno != EINTR) {
					return(LAMERROR);
				}	
			} else if (r == 0) {
				errno = EEOF;
			} else {
				ntoread -= r;
				inbuf += r;
			}
		}
	}

	return(0);
}
