/*
 *
 * Copyright 1998-1999, University of Notre Dame.
 * Authors: Jeffrey M. Squyres, Kinis L. Meyer with M. D. McNally 
 *          and Andrew Lumsdaine
 *
 * This file is part of the Notre Dame LAM implementation of MPI.
 *
 * You should have received a copy of the License Agreement for the
 * Notre Dame LAM implementation of MPI along with the software; see
 * the file LICENSE.  If not, contact Office of Research, University
 * of Notre Dame, Notre Dame, IN 46556.
 *
 * Permission to modify the code and to distribute modified code is
 * granted, provided the text of this NOTICE is retained, a notice that
 * the code was modified is included with the above COPYRIGHT NOTICE and
 * with the COPYRIGHT NOTICE in the LICENSE file, and that the LICENSE
 * file is distributed with the modified code.
 *
 * LICENSOR MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.
 * By way of example, but not limitation, Licensor MAKES NO
 * REPRESENTATIONS OR WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY
 * PARTICULAR PURPOSE OR THAT THE USE OF THE LICENSED SOFTWARE COMPONENTS
 * OR DOCUMENTATION WILL NOT INFRINGE ANY PATENTS, COPYRIGHTS, TRADEMARKS
 * OR OTHER RIGHTS.  
 *
 * Additional copyrights may follow.
 *
 *	Ohio Trollius
 *	Copyright 1997 The Ohio State University
 *	GDB/NJN
 *
 *	$Id: pqcreate.c,v 6.7 1999/09/11 18:18:19 jsquyres Exp $
 *
 *	Function:	- OTB kenyad
 *			- creates a process
 */

#include "lam_config.h"
#include "sfh.h"

#include <fcntl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include "args.h"
#include "kio.h"
#include "kreq.h"
#include "laminternal.h"
#include "net.h"
#include "portable.h"
#include "preq.h"
#include "t_types.h"
#include "terror.h"
#include "typical.h"

/*
 * private functions
 */
static int		pcreate();		/* create process */
static char		*argsread();		/* read argv from file */
static void		get_argv_env();
static void		exit_protocol();
static void		close_stdio_fds();
static void		check_dir(char *);
static int		recv_stdio_fds();

/*
 * external variables
 */
extern struct pprocess	ptable[];		/* kenya process table */
extern struct preq	pq;			/* kenya request */
extern struct preply	pr;			/* kenya reply */
extern char		**pathv;		/* argv of search paths */

/*
 * external functions
 */
extern void		psendr();
extern int		pfind();
extern char		*rforget();		/* tag -> filename */

/*
 * local variables
 */
static char		fdbuf[32];		/* TROLLIUSFD env space */
static char		rtfbuf[32];		/* TROLLIUSRTF env space */
static char		jobbuf[32];		/* LAMJOB env space */
static char		pidbuf[32];		/* LAMKENYAPID env space */
static char		parentbuf[32];		/* LAMPARENT env space */
static char		worldbuf[32];		/* LAMWORLD env space */
static char		rankbuf[32];		/* LAMRANK env space */

void
pqcreate(void)
{
    char		**argv = 0;		/* new process args */
    char		**envv = 0;		/* new process environment */
    char		*loadname;		/* load module filename */
    char		*cwd;			/* current working dir */
    int			argc = 0;		/* # args */

    pr.pr_reply = 0;
    loadname = 0;
/*
 * Get argv and environment from tag.
 */
    if (pq.pq_argv) {
	get_argv_env(&argv, &envv);
	argc = argvcount(argv);
    }
/*
 * If flag set to try parents working dir, check if it exists.
 */
    if (pq.pq_rtf & RTF_TRYCWD) {
	check_dir(pq.pq_wrkdir);
    }
/*
 * Find the filename of the load module.
 */
    if (pq.pq_rtf & RTF_FLAT) {
	loadname = rforget(LOCAL, pq.pq_loadpt);
    } else {
	if (!LAM_isNullStr(pq.pq_wrkdir)) {
	    loadname = sfh_path_env_findv(pq.pq_name, R_OK | X_OK,
				envv, pq.pq_wrkdir);
	} else {
	    cwd = getworkdir();
	    loadname = sfh_path_env_findv(pq.pq_name, R_OK | X_OK,
				envv, cwd);
	    if (cwd) free(cwd);
	}
    }

    if (loadname == 0) {
	pr.pr_reply = ltot(errno);
    }
/*
 * Construct an argv from the executable name.
 */
    if (!pq.pq_argv) {
	if (loadname) {
	    argc = 1;
	    argv = argvbreak(loadname, 0xa);
	    if (argv == 0) lampanic("kenyad (argvbreak)");
	} else {
	    argv = 0;
	}
    }
/*
 * Clean up.
 */
    if (pr.pr_reply) {

	if (loadname) {

	    if (pq.pq_rtf & RTF_FLAT) {
		unlink(loadname);
	    }

	    free(loadname);
	}

	if (argv) {
	    free((char *) argv);
	}
/*
 * The file descriptor passer (filed client) is blocked on an accept.  We must
 * exit and let it continue.  This is done by opening and immediately closing
 * a connection to it.  
 */
	if (pq.pq_rtf & RTF_PFDIO) {
	    exit_protocol();
	}
    }
/*
 * Create the process.
 */
    else {
	if (pcreate(loadname, argc, argv, envv)) {
	    pr.pr_reply = ltot(errno);
	}
    }
/*
 * Reply to client.
 */
    psendr((char *) &pr, sizeof(pr), 0);
}

/*
 *	pcreate
 *
 *	Function:	- creates a new process for a given load module
 *	Accepts:	- load module filename
 *			- command line args
 *	Returns:	- 0 or ERROR
 */
static int
pcreate(loadname, argc, argv, envv)

char			*loadname;
int4			argc;
char			**argv;
char			**envv;

{
	struct pprocess	*pnew;			/* newborn process pointer */
	int		inew;			/* index of newborn process */
	sigset_t	newset;			/* new signal set */
	sigset_t	oldset;			/* old signal set */
	int		fds[3];			/* stdio file descriptors */
	char		*wrkdir = 0;		/* working directory */
	char		*p;
/*
 * Find a slot in the process table.
 */
	inew = pfind((int4) NOTPROCESS);

	if (inew < 0) {
		if (pq.pq_rtf & RTF_PFDIO) {
			exit_protocol();
		}

		errno = ENOPDESCRIPTORS;
		return(LAMERROR);
	}

	pnew = ptable + inew;
	pnew->p_loadpt = loadname;
	pnew->p_rtf = pq.pq_rtf | RTF_KENYA;
	pnew->p_argc = argc;
	pnew->p_argv = argv;
	pnew->p_jobid.jid_node = pq.pq_jobid.jid_node;
	pnew->p_jobid.jid_pid = pq.pq_jobid.jid_pid;
	pnew->p_nodeid = pq.pq_src_node;
	pnew->p_event = pq.pq_src_event;
/*
 * Receive stdio file descriptors. The process about to be created
 * inherits them.
 */
	if (pq.pq_rtf & RTF_PFDIO) {
		if (recv_stdio_fds(fds)) {
			return(LAMERROR);
		}
	}
/*
 * Pass Trillium runtime variables through environment.
 */
	sprintf(fdbuf, "TROLLIUSFD=%d:%d:%d:%d", pq.pq_jobid.jid_node,
			pq.pq_stdin, pq.pq_stdout, pq.pq_stderr);

	if (putenv(fdbuf)) lampanic("kenyad (putenv)");

	sprintf(rtfbuf, "TROLLIUSRTF=%d", pnew->p_rtf);
	if (putenv(rtfbuf)) lampanic("kenyad (putenv)");

	sprintf(jobbuf, "LAMJOBID=%d:%d", pq.pq_jobid.jid_node, 
		pq.pq_jobid.jid_pid);
	if (putenv(jobbuf)) lampanic("kenyad (putenv)");

	sprintf(pidbuf, "LAMKENYAPID=%d", (int) getpid());
	if (putenv(pidbuf)) lampanic("kenyad (putenv)");

	sprintf(worldbuf, "LAMWORLD=%d", pq.pq_world);
	if (putenv(worldbuf)) lampanic("kenyad (putenv)");

	sprintf(parentbuf, "LAMPARENT=%d", pq.pq_parent);
	if (putenv(parentbuf)) lampanic("kenyad (putenv)");

	sprintf(rankbuf, "LAMRANK=%d", pq.pq_rank);
	if (putenv(rankbuf)) lampanic("kenyad (putenv)");
/*
 * Set up working dir.
 */
	if (pq.pq_rtf & RTF_APPWD) {
	    wrkdir = strdup(loadname);
	    if (wrkdir == 0) {
		return(LAMERROR);
	    }
	    p = strrchr(wrkdir+1, STRDIR);
	    if (!p)
		p = wrkdir+1;
	    *p = 0;
	} 
	else if (!LAM_isNullStr(pq.pq_wrkdir)) {
	    wrkdir = pq.pq_wrkdir;
	}
/*
 * Create the process.
 */
	sigemptyset(&newset);
	sigaddset(&newset, SIGCHLD);
	sigprocmask(SIG_BLOCK, &newset, &oldset);

	pnew->p_pid = kcreatev(loadname, argv, envv,
	    			wrkdir, (pq.pq_rtf & RTF_PFDIO) ? fds : 0);

	sigprocmask(SIG_SETMASK, &oldset, (sigset_t *) 0);
/*
 * Close the passed file descriptors.  Kenyad does not use them.
 */
	if (pq.pq_rtf & RTF_PFDIO) {
		close_stdio_fds(fds);
	}

	if (pq.pq_rtf & RTF_APPWD) {
	    free(wrkdir);
	}

	if (pnew->p_pid < 0) {
		return(LAMERROR);
	}

	pr.pr_pid = ltot(pnew->p_pid);
	pr.pr_index = ltot(inew + 1);
	return(0);
}

/*
 *	argsread
 *
 *	Function:	- reads in glued argv structure from a file
 *	Accepts:	- filename
 *	Returns:	- argv ptr or null ptr
 */
static char *
argsread(argvname)

char			*argvname;

{
	char		*args;			/* glued argv buffer */
	int		avfd;			/* argv file descriptor */
	struct stat	avstat;			/* argv file status */

	if (stat(argvname, &avstat)) return(0);

	args = malloc((unsigned int) avstat.st_size);
	if (args == 0) lampanic("kenyad (malloc)");

	avfd = open(argvname, O_RDONLY, 0);

	if (avfd < 0) {
		free(args);
		return(0);
	}

	if (read(avfd, args, (int) avstat.st_size) < 0) {
		free(args);
		return(0);
	}

	close(avfd);
	return(args);
}

/*
 *	get_argv_env
 *
 *	Function:	- reconstruct argv and environment from a file
 *	Accepts:	- argv (out)
 *			- environment (out)
 */
static void
get_argv_env(char ***argv, char ***envv)
{
	char		*args;			/* glued argv and environment */
	char		*argvname;		/* argv filename */
	int		len;

	*argv = 0;
	*envv = 0;
/*
 * Convert argument tag to filename.
 */
	argvname = rforget(LOCAL, pq.pq_argv);

	if (argvname == 0) {
		pr.pr_reply = ltot(errno);
		return;
	}
/*
 * Read in argv structure.
 */
	args = argsread(argvname);

	if (args == 0) {
		pr.pr_reply = ltot(errno);
		unlink(argvname);
		free(argvname);
		return;
	}
/*
 * Reconstruct the argv and environment structure.
 */
	if (pq.pq_envsize) {
		len = strlen(args);
		*envv = argvbreak(args + len - pq.pq_envsize, 0xa);
		if (*envv == 0) lampanic("kenyad (argvbreak)");

		if (pq.pq_envsize < len) {
			args[len - pq.pq_envsize] = 0;
			*argv = argvbreak(args, 0xa);
			if (*argv == 0) lampanic("kenyad (argvbreak)");
		}
	} else {
		*argv = argvbreak(args, 0xa);
		if (*argv == 0) lampanic("kenyad (argvbreak)");
	}

	free(args);
	unlink(argvname);
	free(argvname);
}

/*
 *	recv_stdio_fds
 *
 *	Function:	- read file descriptors from server (filed client)
 *	Accepts:	- file descriptor array (out)
 *	Returns:	- 0 or LAMERROR
 */
static int
recv_stdio_fds(fds)

int			*fds;

{
	int		pass_fd;		/* stream to receive fds over */
	char		server[LAM_PATH_MAX];	/* fd server socket name */
/*
 * Open connection to server.
 */
	if (lam_mktmpid((int) -pq.pq_src_event, server, sizeof(server))) {
		return(LAMERROR);
	}
	
        if ((pass_fd = sfh_sock_open_clt_unix_stm(server)) < 0) {
		return(LAMERROR);
	}
/*
 * Receive the file descriptors.
 */
	if ((fds[0] = sfh_recv_fd(pass_fd)) < 0) {
		close(pass_fd);
		return(LAMERROR);
	}
	
	if ((fds[1] = sfh_recv_fd(pass_fd)) < 0) {
		close(fds[0]);
		close(pass_fd);
		return(LAMERROR);
	}

	if ((fds[2] = sfh_recv_fd(pass_fd)) < 0) {
		close(fds[0]);
		close(fds[1]);
		close(pass_fd);
		return(LAMERROR);
	}
/*
 * Close connection to server.
 */
	close(pass_fd);
	return(0);
}

/*
 *	exit_protocol
 *
 *	Function:	- exit the file descriptor passing protocol
 *			  unblocking the server side
 *			- connects to server and immediately closes
 *			  the connection
 */
static void
exit_protocol()

{
	int		fd;
	char		server[LAM_PATH_MAX];	/* fd server socket name */

	if (lam_mktmpid((int) -pq.pq_src_event, server, sizeof(server)) == 0) {
		if ((fd = sfh_sock_open_clt_unix_stm(server)) >= 0) {
			close(fd);
		}
	}
}

/*
 *	close_stdio_fds
 *
 *	Function:	- close passed stdio file descriptors
 *	Accepts:	- file descriptor array 
 */
static void
close_stdio_fds(fds)

int			*fds;

{
	close(fds[0]);
	close(fds[1]);
	close(fds[2]);
}

/*
 *	check_dir
 *
 *	Function:	- check if directory exists
 *			- if not nulls the directory name
 *	Accepts:	- directory name
 */
static void
check_dir(char *dir)
{
    struct stat		fileinfo;

    if (stat(dir, &fileinfo) || !S_ISDIR(fileinfo.st_mode)) {
	strcpy(dir, "");
    }
}
