/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	GDB
 *
 *	$Id: kernelio.c,v 6.10 1999/08/30 21:38:17 jsquyres Exp $
 * 
 *	Function:	- receives kernel requests and send replies
 *			- moves messages between clients
 */

#include <lam_config.h>
#include <sfh.h>

#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <unistd.h>

#if LAM_NEED_SYS_SELECT_H
#include <sys/select.h>
#endif

#include <dl_inet.h>
#include <kreq.h>
#include <preq.h>
#include <terror.h>
#include <typical.h>

#define KIMAX		(4 + 2 * PMAX)
#define localaddr_t	struct sockaddr *

/*
 * global functions
 */
struct kreq		*kio_req();		/* request from client */
void			kio_cleanup();		/* free resources */
void			kio_close();		/* close connection */
void			kio_init();		/* initialize */
void			kio_intr();		/* register interrupt */
void			kio_intr_clr();		/* clear interrupt */
void			kio_reply();		/* reply to client */
void			kio_send();		/* send from internal proc */
void			kio_transfer();		/* move msg, ext to ext */
int			kio_fd_ready();		/* get fd_ready */
int			kio_recv();		/* recv to internal proc */
int			kio_to();		/* register timeout */

/*
 * external functions
 */
extern struct kproc	*kpfindport();		/* find client by port */
extern void		kkillall();		/* kill all clients */
extern void		knuke();		/* remove client */
extern void		(*_lam_signal())();	/* portable signal() */
extern char		*killname();		/* get kill filename */
extern char		*sockname();		/* get socket filename */
extern int		kkillopen();		/* open kill file */
extern int		mread();
extern int		mreadv();
extern int		mwrite();
extern int		mwritev();

/*
 * external variables
 */
extern struct kproc	*prun;			/* current running client */

/*
 * local variables
 */
static struct kreq	request;		/* kernel request */
static struct timeval	to_actual;		/* timeout delay */
static void		(*tofunc)();		/* timeout function */
static fd_set		allfds;			/* current input lines */
static fd_set		exceptfds;		/* error lines */
static fd_set		readfds;		/* active read lines */
static fd_set		clientfds;		/* client input lines */
static int		sd_kernel;		/* kernel I/O socket */
static int		fd_ready;		/* current ready line */
static int		fd_max;			/* max fd value */
static int		nfd_ready;		/* # currently ready lines */
static int		to_count;		/* timeout estimater */

static char		fwdbuf[KPKTLEN];	/* forwarding buffer */

static struct sockaddr_un
			kernel_un;		/* kernel address */

static struct {
	void		(*kn_func)();		/* interrupt function */
	int		kn_fd;			/* interrupt file desc. */
	struct kproc	*kn_client;		/* associated kernel client */
	int		kn_bclear;		/* clear after trigger? */
} kintrs[KIMAX];

/*
 * local functions
 */
static void		kio_shutdown();		/* cleanup and exit */

/*
 *	kio_init
 *
 *	Function:	- initializes communication structures for
 *			  sending and receiving kernel requests
 */
void
kio_init()

{
	char		*f_kill;		/* kill filename */
	char		*f_sock;		/* socket filename */
	int		i;
/*
 * Initialize the interrupt fd->func table.
 */
	for (i = 0; i < KIMAX; ++i) {
		kintrs[i].kn_fd = -1;
		kintrs[i].kn_func = 0;
		kintrs[i].kn_client = 0;
	}
/*
 * Disable timeouts.
 */
	tofunc = 0;
	to_count = 0;
	to_actual.tv_sec = 0;
	to_actual.tv_usec = 0;
/*
 * I want to return an error on a broken connection.
 */
	if (_lam_signal(SIGPIPE, SIG_IGN) == SIG_ERR)
			lampanic("kernel (_lam_signal)");
/*
 * Set up local address.
 */
	kernel_un.sun_family = AF_UNIX;
	f_sock = sockname();
	memcpy((char *) kernel_un.sun_path, f_sock, strlen(f_sock));
/*
 * Create the socket.
 */
	if ((sd_kernel = socket(AF_UNIX, SOCK_STREAM, 0)) < 0)
			lampanic("kernel (socket)");

	fd_max = sd_kernel;
/*
 * Bind the kernel's address to the socket.
 */
	if (bind(sd_kernel, (localaddr_t) &kernel_un, sizeof(kernel_un) -
				sizeof(kernel_un.sun_path) + strlen(f_sock)))
		lampanic("kernel (bind)");
	free(f_sock);
/*
 * Establish backlog queue.
 */
	if (listen(sd_kernel, 5)) lampanic("kernel (listen)");
/*
 * Open the kill record file.
 */
	if ((f_kill = killname()) == 0) lampanic("kernel (killname)");

	if (kkillopen(f_kill)) {
		fprintf(stderr, "kernel (kkillopen): %s: ", f_kill);
		lampanic("");
	}
	free(f_kill);
/*
 * Catch SIGTERM and SIGINT and kill all attached processes.
 */
	if (_lam_signal(SIGTERM, kio_shutdown) == SIG_ERR)
			lampanic("kernel (_lam_signal)");

	if (_lam_signal(SIGINT, kio_shutdown) == SIG_ERR)
			lampanic("kernel (_lam_signal)");

	FD_ZERO(&allfds);
	FD_ZERO(&clientfds);
	nfd_ready = 0;
}

/*
 *	kio_req
 *
 *	Function:	- receives the next kernel request
 *	Returns:	- kernel request or 0 if interrupt/timeout
 */
struct kreq *
kio_req()

{
	int		i;
	LAM_SOCK_OPTLEN_T length;
	int		sd_new;		/* new client socket */
	struct timeval	*pto;		/* timeout on this loop */
	struct sockaddr_un
			client_un;	/* client address */

	for (;;) {
	    fd_ready++;
/*
 * Select on all current clients, external interrupt sources,
 * kernel server socket and timeout.
 */
	    while (nfd_ready == 0) {
		fd_ready = 0;
		memcpy((char *) &readfds, (char *) &allfds, sizeof(fd_set));
		FD_SET(sd_kernel, &readfds);
		memcpy((char *) &exceptfds, (char *) &readfds, sizeof(fd_set));

		if (tofunc == 0) {
		    pto = 0;
		} else {
		    pto = &to_actual;
/*
 * Declare a timeout based on a wild estimate.
 */
		    if (to_count > TO_DLO_ESTIMATE) {
			to_count = 0;
			(*tofunc)();
			return(0);
		    }
		}

		while (((nfd_ready = select(fd_max + 1, &readfds,
			(fd_set *) 0, (fd_set *) &exceptfds, pto)) < 0) &&
			(errno == EINTR));
		if (nfd_ready < 0) lampanic("kernel (select)");
/*
 * The actual timeout expired.
 */
		if (nfd_ready == 0) {
		    to_count = 0;
		    (*tofunc)();
		    return(0);
		}
/*
 * Advance the estimated timeout.
 */
		if (tofunc != 0) {
		    to_count++;
		}
/*
 * Check for new clients.  No known client actions can cause an
 * error on accept.  Therefore any error condition is fatal.
 */
		if (FD_ISSET(sd_kernel, &readfds) ||
			FD_ISSET(sd_kernel, &exceptfds)) {
		    length = sizeof(client_un.sun_path);
		    sd_new = accept(sd_kernel, (localaddr_t) &client_un,
			    &length);
		    if (sd_new < 0) lampanic("kernel (accept)");
		    nfd_ready--;
		    FD_SET(sd_new, &allfds);
		    FD_SET(sd_new, &clientfds);
		    FD_CLR(sd_kernel, &readfds);

		    if (sd_new > fd_max) {
			fd_max = sd_new;
		    }
/*
 * Set send and receive buffer sizes.
 */
		    if (sfh_sock_set_buf_size(sd_new, SFH_UNIX, SO_SNDBUF,
			    	KPKTLEN + sizeof(struct nmsg))) {
			lampanic("kernel (sfh_sock_set_buf_size)");
		    }

		    if (sfh_sock_set_buf_size(sd_new, SFH_UNIX, SO_RCVBUF,
			    	KPKTLEN + sizeof(struct nmsg))) {
			lampanic("kernel (sfh_sock_set_buf_size)");
		    }
		}
	    }
/*
 * Find next ready descriptor.
 */
	    while ((! FD_ISSET(fd_ready, &readfds)) &&
		    (! FD_ISSET(fd_ready, &exceptfds))) {
		fd_ready++;
	    }

	    nfd_ready--;
/*
 * Clean up errnoneous descriptor.
 */
	    if (FD_ISSET(fd_ready, &exceptfds)) {
		knuke(kpfindport(fd_ready));
		kio_close(fd_ready);
	    }
/*
 * Is this a client?
 */
	    else if (FD_ISSET(fd_ready, &clientfds)) {

		if (mread(fd_ready, (char *) &request, sizeof(struct kreq)) <
			sizeof(struct kreq)) {
		    knuke(kpfindport(fd_ready));
		    kio_close(fd_ready);
		} else {
		    return(&request);
		}
	    }
/*
 * This must be an interrupt.
 */
	    else {

		for (i = 0; (i < KIMAX) && (kintrs[i].kn_fd != fd_ready); ++i);

		if (i >= KIMAX) {
		    errno = EIMPOSSIBLE;
		    lampanic("kernel (select)");
		}

		prun = kintrs[i].kn_client;
		(*kintrs[i].kn_func)(kintrs[i].kn_fd);

		if (kintrs[i].kn_bclear) {
		    FD_CLR(kintrs[i].kn_fd, &allfds);
		    kintrs[i].kn_fd = -1;
		}

		return(0);
	    }
	}
}

/*
 *	kio_reply
 *
 *	Function:	- sends reply back to the client
 *			- message contains result of kernel request
 *	Accepts:	- client socket
 */
void
kio_reply(reply, fd_client)

struct kreply		*reply;
int4			fd_client;

{
	mwrite(fd_client, (char *) reply, sizeof(struct kreply));
}

/*
 *	kio_intr
 *
 *	Function:	- associates an input fd with a function
 *	Accepts:	- interrupt fd
 *			- interrupt function
 *			- clear interrupt after trigger?
 */
void
kio_intr(fd, func, bclear)

int			fd;
void			(*func)();
int			bclear;

{
	int		i;

	for (i = 0; (i < KIMAX) && (kintrs[i].kn_fd >= 0) &&
			(kintrs[i].kn_fd != fd); ++i);

	if (i >= KIMAX) {
		errno = EFULL;
		lampanic("kernel (kio_intr)");
	}

	kintrs[i].kn_fd = fd;
	kintrs[i].kn_func = func;
	kintrs[i].kn_client = prun;
	kintrs[i].kn_bclear = bclear;

	FD_SET(fd, &allfds);

	if (fd > fd_max) {
		fd_max = fd;
	}
}

/*
 *	kio_intr_clr
 *
 *	Function:	- clears interrupt associated with an input fd
 *	Accepts:	- interrupt fd
 */
void
kio_intr_clr(fd)

int			fd;

{
	int		i;

	for (i = 0; (i < KIMAX) && (kintrs[i].kn_fd != fd); ++i);

	if (i < KIMAX) {
		kintrs[i].kn_fd = -1;
		FD_CLR(fd, &allfds);
	}
}

/*
 *	kio_to
 *
 *	Function:	- associates a timeout delay with a function
 *			- overwrites existing registration
 *	Accepts:	- delay
 *			- function
 *	Returns:	- 0
 */
int
kio_to(delay, func)

struct timeval		*delay;
void			(*func)();

{
	tofunc = func;

	if (delay) {
		to_actual.tv_sec = delay->tv_sec;
		to_actual.tv_usec = delay->tv_usec;
	}

	return(0);
}

/*
 *	kio_recv
 *
 *	Function:	- receives a message from an external process
 *			  to an internal process
 *	Accepts:	- receive message descriptor
 *			- minimum message length
 *			- send client socket
 */
int
kio_recv(recvkmsg, minlen, fd_client)

struct kmsg		*recvkmsg;
int4			minlen;
int			fd_client;

{
	struct iovec	iov[2];			/* scatter vectors */
	char		*sv_msg;

	if (recvkmsg->k_flags & KNMSG) {
		sv_msg = ((struct nmsg *) (recvkmsg->k_msg))->nh_msg;
		iov[0].iov_base = recvkmsg->k_msg;
		iov[0].iov_len = sizeof(struct nmsg);
		iov[1].iov_base = ((struct nmsg *) (recvkmsg->k_msg))->nh_msg;
		iov[1].iov_len = minlen;

		if (mreadv(fd_client, iov, 2) < (iov[0].iov_len +
				iov[1].iov_len)) return(LAMERROR);
		((struct nmsg *) (recvkmsg->k_msg))->nh_msg = sv_msg;
	} else {
		if (mread(fd_client, recvkmsg->k_msg, minlen) < minlen)
				return(LAMERROR);
	}

	return(0);
}

/*
 *	kio_send
 *
 *	Function:	- sends a message from an internal process
 *			  to an external process
 *	Accepts:	- send message descriptor
 *			- minimum message length
 *			- receive client socket
 */
void
kio_send(sendkmsg, minlen, fd_client)

struct kmsg		*sendkmsg;
int4			minlen;
int			fd_client;

{
	struct iovec	iov[2];			/* scatter vectors */

	if (sendkmsg->k_flags & KNMSG) {
		iov[0].iov_base = sendkmsg->k_msg;
		iov[0].iov_len = sizeof(struct nmsg);
		iov[1].iov_base = ((struct nmsg *) (sendkmsg->k_msg))->nh_msg;
		iov[1].iov_len = minlen;

		mwritev(fd_client, iov, 2);
	} else {
		mwrite(fd_client, sendkmsg->k_msg, minlen);
	}
}

/*
 *	kio_transfer
 *
 *	Function:	- moves message between two external processes
 *	Accepts:	- send message descriptor
 *			- send client socket
 *			- receive client socket
 *			- minimum message length
 */
void
kio_transfer(sendkmsg, fd_send, fd_recv, minlen)

struct kmsg		*sendkmsg;
int			fd_send;
int			fd_recv;
int4			minlen;

{
	int4		remain;			/* remaining length */
	int4		len;			/* packet length */

	remain = minlen;

	if (sendkmsg->k_flags & KNMSG) {
		remain += sizeof(struct nmsg);
	}
/*
 * Transfer the message in blocks.
 */
	while (remain > 0) {
		len = (remain > KPKTLEN) ? KPKTLEN : remain;
		mread(fd_send, fwdbuf, len);
		mwrite(fd_recv, fwdbuf, len);
		remain -= len;
	}
}

/*
 *	kio_close
 *
 *	Function:	- closes a client socket
 *	Accepts:	- client socket
 */
void
kio_close(fd_client)

int			fd_client;

{
	shutdown(fd_client, 2);
	close(fd_client);
	FD_CLR(fd_client, &allfds);
	FD_CLR(fd_client, &clientfds);
}

/*
 *	kio_fd_ready
 *
 *	Returns:	- fd_ready
 */
int
kio_fd_ready()

{
	return(fd_ready);
}

/*
 *	kio_shutdown
 *
 *	Function:	- cleanup and exit
 */
static void
kio_shutdown()

{
	kkillall();
	shutdown(sd_kernel, 2);
	kio_cleanup();
	exit(0);
}

/*
 *	kio_cleanup
 *
 *	Function:	- cleanup for fork() purposes
 */
void
kio_cleanup()

{
	close(sd_kernel);
}
