/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	RBD/GDB/NJN
 *
 *	$Id: lamsend.c,v 6.20 1999/11/12 05:19:30 jsquyres Exp $
 *
 *	Function:	- shared code for send functions
 */

#include <mpi.h>
#include <mpisys.h>
#include <mpitrace.h>
#include <rpisys.h>
#include <typical.h>
#if LAM_WANT_IMPI
#include <impi.h>
#endif
#include <debug.h>
#include <blktype.h>

/*
 * public functions
 */
int			lam_send();
int			lam_isend();


/*
 * external variables
 */
extern struct _proc     *lam_impid_proc;


/*
 * private variables
 */
static int		fl_debug = 0;

/*
 *	lam_send
 *
 *	Function:	- send a blocking MPI message
 *	Accepts:	- MPI send args
 *			- request type
 *	Returns:	- MPI_SUCCESS or error code
 */
int
lam_send(buf, count, dtype, dest, tag, comm, reqtype)

void			*buf;
int			count;
MPI_Datatype		dtype;
int			dest;
int			tag;
MPI_Comm		comm;
int			reqtype;

{
	int		err;
	int		fl_trace;		/* do tracing? */
	int		req_err;		/* recoverable error */
	int		seqnum;			/* message sequence number */
	struct _req	req_storage;
	MPI_Request	req;
	double		startt = 0.0;		/* start time */
	double		finisht;		/* finish time */
#if LAM_WANT_IMPI
	int		flag;			
#endif
/*
 * Handle the trivial cases.
 */
	if (dest == MPI_ANY_SOURCE) return(lam_mkerr(MPI_ERR_RANK, 0));

	if (dest == MPI_PROC_NULL) return(MPI_SUCCESS);

	if ((fl_trace = LAM_TRACE_TOP())) {
		startt = MPI_Wtime();
		_kio.ki_blktime = 0.0;
	}

	DBUG("IN the lamsend\n");

#if LAM_WANT_IMPI
/* 
 * If IMPI_Send_lamgiappe returns flag=1, this is an IMPI proxy send.
 * Need to re-route to the impid.
 */
	err = IMPI_Send_lamgiappe(reqtype, count, dtype, dest, tag, comm, 
				  &flag);
	if (err != MPI_SUCCESS)
	  return err;
	if (flag == 1) {
	  dest = impid_comm->c_group->g_nprocs - 1;
	  tag  = IMPI_MESSAGE_TAG;
	  comm = impid_comm;
	}
#endif

#ifdef SHORTCIRCUIT
	if (reqtype == LAM_RQISEND && lam_c2c && lam_rq_nactv == 0) {
	    if (comm == MPI_COMM_NULL) {
		return(lam_mkerr(MPI_ERR_COMM, 0));
	    }

	    if (LAM_IS_INTER(comm) || dest != comm->c_group->g_myrank) {
		err = _rpi_c2c_fastsend(buf, count, dtype, dest, tag, comm);
/*
 * Generate a trace.
 */
		if (fl_trace) {
		    finisht = MPI_Wtime();

		    lam_tr_msg(TRTOUTPUT, startt,
			LAM_S2US(finisht - startt - _kio.ki_blktime),
			LAM_S2US(_kio.ki_blktime), dest, tag,
			comm, dtype, count, 0, 0, lam_seqnum-1, reqtype);
		}

		return(err);
	    }
	}
#endif
	LAM_ZERO_ME(req_storage);
	req = &req_storage;
	err = _mpi_req_build(buf, count, dtype, dest, tag, comm, reqtype, 
			     &req);
	if (err != MPI_SUCCESS) return(err);

	req->rq_marks |= (LAM_RQFBLKTYPE | LAM_RQFMAND);

	err = _mpi_req_start(req);
	if (err != MPI_SUCCESS) return(err);

	_mpi_req_add_m(req);
	_mpi_req_blkclr_m();
	_mpi_req_blkset_m(req);

	err = _mpi_req_advance();
	if (err != MPI_SUCCESS) return(err);

	req_err = req->rq_status.MPI_ERROR;
	seqnum = req->rq_seq;

	_mpi_req_rem_m(req);

	err = _mpi_req_destroy(&req);
	if (err != MPI_SUCCESS) return(err);
/*
 * Generate a trace.
 */
	if (fl_trace) {
		finisht = MPI_Wtime();

		lam_tr_msg(TRTOUTPUT, startt,
				LAM_S2US(finisht - startt - _kio.ki_blktime),
				LAM_S2US(_kio.ki_blktime), dest, tag,
				comm, dtype, count, 0, 0, seqnum, reqtype);
	}

	return(req_err);
}

/*
 *	lam_isend
 *
 *	Function:	- send a non-blocking MPI message
 *	Accepts:	- MPI isend args
 *			- request type
 *	Returns:	- MPI_SUCCESS or error code
 */
int
lam_isend(buf, count, dtype, dest, tag, comm, preq, reqtype)

void			*buf;
int			count;
MPI_Datatype		dtype;
int			dest;
int			tag;
MPI_Comm		comm;
MPI_Request		*preq;
int			reqtype;

{
	int		err;
#if LAM_WANT_IMPI
	MPI_Request     lreq;
#endif

	if (dest == MPI_ANY_SOURCE) return(lam_mkerr(MPI_ERR_RANK, 0));

#if LAM_WANT_IMPI
	/* If IMPI_Isend_lamgiappe fills in a request, this is an IMPI
           proxy send.  Need to re-route to the impid.  */

	IMPI_Isend_lamgiappe(LAM_RQISEND, count, dtype, dest, tag, 
			     comm, &lreq);
	if (lreq != MPI_REQUEST_NULL) {
	  dest = impid_comm->c_group->g_nprocs - 1;
	  tag  = IMPI_MESSAGE_TAG;
	  comm = impid_comm;
	}
#endif

	*preq = MPI_REQUEST_NULL;
	err = _mpi_req_build(buf, count, dtype, dest, tag, comm,
			     reqtype, preq);
	if (err != MPI_SUCCESS) return(err);

	(*preq)->rq_marks |= LAM_RQFMAND;

	err = _mpi_req_start(*preq);
	if (err != MPI_SUCCESS) return(err);

#if LAM_WANT_IMPI
/*
 * Important to add the shadow *after* it has been started (so that
 * the shadow doesn't get restarted)
 */
	(*preq)->rq_shadow = lreq;
#endif

	_mpi_req_add_m(*preq);
	_mpi_req_blkclr_m();

	err = _mpi_req_advance();

	return(err);
}
