/*
 * © Copyright 1996-2012 ECMWF.
 * 
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 
 * In applying this licence, ECMWF does not waive the privileges and immunities 
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"

#define PAD(l,n)     ( ( ( (l)+(n)-1 ) / (n) * (n) ) - (l) )

static marslist *targets = NULL;

#define MAXKEY 20

typedef struct remotefile {
	struct remotefile* next;
	char* local;
	char* remote;
	char* host;
	boolean  ecfs;
} remotefile;

static remotefile* remote_files = NULL;

typedef struct targetfile {

	struct targetfile *next;
	FILE              *file;
	char              *name;
	char*             key;
	off_t	      start;

	char              *buffer;

	int               pipe;

} targetfile;

static timer *target_timer = NULL;

typedef struct targetdata {

	gribfile  *file;
	int	   	   padding;
	int	   	   bufr_padding;
	int        count;
	int        expect;    /* initialised with count_fields() */
	fieldset   *fs;
	hypercube  *cube;     /* Hypercube to speed up field_order(). Call cube_order */
	variable   *v;
	boolean    order;
	request    *r;
	boolean    split;
	boolean       dispnew;
	char       *target;

	targetfile* current;
	targetfile* files;

	int    keycount;
	char*  key[MAXKEY];
	int    api[MAXKEY];

	boolean    has_error;

	int               bufsize;
	boolean           multitarget_format;

	boolean inited;

}targetdata;

static void target_init(void);
static err  target_open(void *data,request*,request*,int);
static err  target_close(void *data);
static err  target_read(void *data,request *r,void *buffer,long *length);
static err  target_write(void *data,request *r,void *buffer,long *length);
static err  target_cntl(void *data,int code,void *param,int size);


static option opts[] = {
	{"padding","MARS_PADDING","-padding","120",t_int,sizeof(int),
		OFFSET(targetdata,padding),},

		{"bufr_padding","MARS_BUFR_PADDING","-bufr_padding","8",t_int,sizeof(int),
			OFFSET(targetdata,bufr_padding),},

			{"bufsize","MARS_WRITE_BUFFER","-bufsize","0",t_int,sizeof(int),
				OFFSET(targetdata,bufsize),},

				{"multitarget_format","MARS_MULTITARGET_STRICT_FORMAT","-multitarget_format","0",t_boolean,sizeof(boolean),
					OFFSET(targetdata,multitarget_format),},
};

static base_class _targetbase = {

	NULL,                       /* parent class */
	"targetbase",                 /* name         */

	false,                      /* inited       */

	sizeof(targetdata),           /* private size */
	NUMBER(opts),               /* option count */
	opts,                       /* options      */


	target_init,                  /* init         */

	target_open,                  /* open         */
	target_close,                 /* close        */

	target_read,                  /* read         */
	target_write,                 /* write        */

	target_cntl,                  /* cntl        */

};


base_class *targetbase = &_targetbase;


static void target_init(void)
{
	target_timer = get_timer("Writing to target file","writetarget",false);
}


static err close_all(targetdata *g)
{
	targetfile *f = g->files;
	err e = 0;
	f = g->files;
	while(f) 
	{
		targetfile *n = f->next;
		if(f->pipe)
		{
			if(pclose(f->file))
			{
				marslog(LOG_EROR|LOG_PERR,"pclose(%s)",f->name);
				e = -2;
			}
		}

		else
		{
			if(timed_fclose(f->file,target_timer))
			{
				marslog(LOG_EROR|LOG_PERR,"fclose(%s)",f->name);
				e = -2;
			}
		}

		strfree(f->name);
		if(f->buffer) free(f->buffer);
		strfree(f->key);

		FREE(f);
		f = n;
	}

	g->files = g->current = NULL;

	return e;
}

static targetfile* open_file(targetdata *g,const char *name,int* error)
{
	targetfile *f = NEW_CLEAR(targetfile);
	char   *cmode = target_open_mode(name);
	*error = 0;

	if(*name == '|')
	{
		f->file = popen(name+1,cmode);
		f->pipe = 1;
	}
	else
	{
		if(g->dispnew) cmode = "w+";
		f->file  = fopen(name,cmode);
		*error = errno;
	}
	marslog(LOG_DBUG,"target_open, mode: %s",cmode);

	if(f->file == NULL)
	{
		marslog(LOG_EROR|LOG_PERR,"cannot open %s",name);
		FREE(f);
		return NULL;
	}

	f->name = strcache(name);

	if(g->bufsize)
	{
		f->buffer = valloc(g->bufsize);
		if(!f->buffer) 
			marslog(LOG_WARN,"Could not valloc buffer %d", g->bufsize);
		else
			if(setvbuf(f->file,f->buffer,_IOFBF,g->bufsize))
				marslog(LOG_WARN|LOG_PERR,"setvbuf failed");
	}


	if(!f->pipe)
	{
		check_nfs_target(name);

		fseek(f->file,(off_t)0,SEEK_END);
		f->start = ftell(f->file);
	}

	f->next  = g->files;
	g->files = f;

	return f;
}

#define IS_ECFS(x) ((strcmp(x,"ec") == 0) || (strcmp(x,"ectmp") == 0))

static char *make_target(targetdata* g,request* r,void* message,long length)
{
	static char  target[1024];
	char  word[1024];
	char  name[1024];
	char  host[1024];
	char val[1024];
	size_t k       = 0;
	int mode       = 0;
	boolean remote  = false;
	const char *value;
	const char *p = g->target;
	int api = 0;

	target[0] = 0;
	host[0]   = 0;

	while(*p)
	{
		switch(*p)
		{

			case ':':

				if(!remote && k != 0 && strncmp(g->target,word,k) == 0)
				{
					word[k] = 0;

					/* check if it is a valide host name */

					if(is_hostname(word))
					{
						k      = 0;
						strcpy(host,word);
						remote = true;
					} else if(IS_ECFS(word))
					{
						k      = 0;
						strcpy(host,"ectmp");
						remote = true;
					} else word[k++] = *p;
				}
				else word[k++] = *p;

				break;

			case '{':
			case '[':
				if(mode == 0)
				{
					word[k] = 0;
					strcat(target,word);
					mode = *p;
					k    = 0;
					api  = *p == '{';
				}
				else
					word[k++] = *p;
				break;

			case ']':
			case '}':
				if( (*p == ']' && mode != '[') || (*p == '}' && mode != '{') )
					word[k++] = *p;
				else
				{
					char tmp[1024];
					size_t size = 0;

					word[k] = 0;

					if(mode != 0)
					{
						if(api)
						{
							size_t size = sizeof(val);
							strcpy(name,word);
							if(message != NULL)
							{
								grib_handle *h = grib_handle_new_from_message(0,message,length);
								if(h) {
									if(grib_get_string(h,word,val,&size) == 0)
										value = val;
									grib_handle_delete(h);
								}
							}
							else
							{
								value = NULL;
								strcat(target,"{");
								strcat(target,word);
								strcat(target,"}");
							}
						}
						else
						{
							value = get_value(r,upcase(word),0);
							strcpy(name,upcase(word));
							if(value == NULL)
							{
								strcat(target,"[");
								strcat(target,word);
								strcat(target,"]");
							}
						}
						size = 0;
					}
					else size = atol(word);
					if(value)
					{
						static boolean already_logged = false;
						/* New style, more strict to MARS output values, 
						   eg, time=0000 instead of 0 */
						if(g->multitarget_format)
						{
							sprintf(tmp,"%0*s",size,value);
							if(!already_logged)
								log_statistics("multitarget","new");

						}
						else
						{
							if(!already_logged)
							{
								marslog(LOG_WARN,"In order to get filenames according to MARS values,");
								marslog(LOG_WARN,"please, set env. var. MARS_MULTITARGET_STRICT_FORMAT=1");
								marslog(LOG_WARN,"before executing your MARS request");
								log_statistics("multitarget","old");
							}
							if(is_number(value))
								sprintf(tmp,"%0*d",size,atol(value));
							else
								sprintf(tmp,"%0*s",size,value);
						}
						already_logged=true;

						if(!g->inited) {
							if(g->keycount >= MAXKEY)
							{
								marslog(LOG_EROR,"Target name: maximum number of variables is %d",MAXKEY);
								return 0;
							}
							g->api[g->keycount  ] = api;
							g->key[g->keycount++] = strcache(name);
						}

						strcat(target,tmp);
					}
					mode = 0;
					k = 0;
				}
				break;

			case ',':
				if(mode == 1)
				{
					mode    = 2;
					word[k] = 0;
					value   = get_value(r,word,0);
					strcpy(name,word);
					if(value == NULL)
					{
						strcat(target,"[");
						strcat(target,word);
						strcat(target,",");
						mode = 0;
					}
					k = 0;
				}
				else word[k++] = *p;
				break;

			default:
				word[k++] = *p;
				break;
		}
		p++;
	}

	word[k] = 0;
	strcat(target,word);

	if(!g->inited && g->keycount != 0)
	{
		int i;
		marslog(LOG_INFO,"Enabling automatic target name generation using:");
		for(i = 0 ; i < g->keycount; i++)
			marslog(LOG_INFO,"%s",g->key[i]);
		g->split = true;
		g->inited = true;
	}

	if(g->keycount) marslog(LOG_INFO,"Creating target name: %s",target);

	if(remote) {

		remotefile* f = remote_files;
		while(f)
		{
			if(strcmp(host,f->host) == 0 && strcmp(target,f->remote) == 0)
				return f->local;
			f = f->next;
		}

		f = NEW_CLEAR(remotefile);
		f->host   = strcache(host);
		f->remote = strcache(target);
		f->next   = remote_files;
		f->local  = strcache(marstmp());
		f->ecfs   = IS_ECFS(f->host);
		remote_files  = f;

		if(f->ecfs)
			log_statistics("target","ecfs");
		else
			log_statistics("target","rcp:%s",f->host);

		strcpy(target,f->local);

	}

	return target;
}

static err select_file(targetdata *g,request *r,int retry, void* message,long length)
{

	int error = 0;
	int i;
	char buffer[1024];
	char val[1024];
	char *k = NULL;
	char *t = NULL;
	targetfile *f = 0;

	if(!g->inited)
	{
		/* First time in */
		t = make_target(g,r,message,length);
		g->inited = true;
		if(!t) return -1;
	}
	else if(!g->split) 
		return 0;

	buffer[0] = 0;

	for(i = 0; i < g->keycount; i++)
	{
		const char* value   = NULL;

		if(g->api[i])
		{
			size_t size = sizeof(val);
			if(message != NULL)
			{
				grib_handle *h = grib_handle_new_from_message(0,message,length);
				if(h) {
					if(grib_get_string(h,g->key[i],val,&size) == 0)
						value = val;
					grib_handle_delete(h);
				}
			}
		}
		else
			value = get_value(r,g->key[i],0);

		if(value == NULL)
		{
			marslog(LOG_EROR,"Target split: no value for %s",g->key[i]);
			return -1;
		}

		strcat(buffer,value);
		strcat(buffer,"+");
	}

	k = strcache(buffer);

	if(g->current && g->current->key == k)
		return 0;

	f = g->files;
	while(f) 
	{
		if(f->key == k)
		{
			g->current = f;
			return 0;
		}
		f = f->next;
	}


	t = t?t:make_target(g,r,message,length);
	if(!t) return -1;

	g->current = open_file(g,t,&error);
	if(!g->current) {

		if(error == EMFILE && retry)
		{
			err e;
			marslog(LOG_WARN,"Closing all target files");
			
			e  = close_all(g);
			if(e) return e;
			return select_file(g,r,0,message,length);
		}
	
		return -1;
	}
	else
	{
		if(!in_list(targets,t,NULL))
			add_to_list(&targets,t,NULL);
	}

	g->current->key = k;

	return 0;
}

static err target_open(void *data,request *r,request *e,int mode)
{
	targetdata *g      = (targetdata*)data;
	const char *fs     = get_value(r,"FIELDSET",0);
	const char *target = no_quotes(get_value(r,"TARGET",0));
	boolean isfield    = (!observation(r) && !image(r) && !simulated_image(r) && !fetch(r) && !feedback(r) && !bias(r) && !track(r));
	const char *disp   = get_value(r,"DISP",0);
	const char *padding= get_value(r,"PADDING",0);

	const char *type   = get_value(r,"TYPE",0);
	boolean isfg       = type && EQ(type,"FG");

    if( (target || (!target && !fs))  && mars.webmars_target) {
        target = mars.webmars_target;
    }


	if(isfg)
		g->r = un_first_guess(r);
	else
		g->r = clone_all_requests(r);

	g->dispnew = (disp && EQ(disp,"NEW"));

	if(mode == WRITE_MODE)
	{
		if(observation(r))
			g->padding = g->bufr_padding;

		if(fetch(r))
			g->padding = 0;

		if(is_netcdf(r))
			g->padding = 0;

		if(is_odb(r))
			g->padding = 0;

		if(target == NULL && fs == NULL)
		{
            marslog(LOG_EROR,"TARGET or FIELDSET value is missing");
            return -1;
		}

		/* Use provided padding if any */
		if(padding)
			if(isdigit(*padding))
				g->padding = atoi(padding);

		/* No fieldset allowed for image or */

		if(target == NULL && !isfield)
		{
			marslog(LOG_EROR,"FIELDSET not valid for non-field requests");
			return -2;
		}

		if((target == NULL || fs == NULL) && (EQ(r->name,"WRITE") || EQ(r->name,"COMPUTE")))
		{
			marslog(LOG_EROR,"TARGET and FIELDSET value are missing");
			return -2;
		}

		if(fs && (!(EQ(r->name,"WRITE") || EQ(r->name,"COMPUTE"))) && isfield)
		{
			int count = count_fields(r);

			g->expect = count;

			g->fs = new_fieldset(count);
			if(g->fs == NULL) return -1;
			g->v = new_variable(fs,g->fs,0);

			if((count == 0) && EQ(r->name,"RETRIEVE"))
				g->order = false;
			else 
				g->order = true;

			g->file = new_gribfile(NULL);
			target = g->file->fname;

			g->cube = new_hypercube_from_mars_request(g->r);
		}

		g->target = strcache(target);

	}
	else
	{
		marslog(LOG_EXIT,"target_open read not imp");
	}

	return 0;
}

char* target_open_mode(const char* target)
{
	if(*target== '|')
		return "w";

	if(in_list(targets,target,NULL))
		return "a+";
	else
	{
		add_to_list(&targets,target,NULL);
		return "w+";
	}
}

static err  target_close(void *data)
{
	targetdata *g = (targetdata*)data;
	targetfile *f = NULL;
	err e = 0;
	int i;

	marslog(LOG_DBUG,"target_close");

	free_all_requests(g->r);

	e = close_all(g);

	strfree(g->target);

	for(i = 0; i < MAXKEY; i++)
		strfree(g->key[i]);

	if(g->fs && !g->has_error)
	{
		free_hypercube(g->cube);
		/* if EXPECT=ANY, g->expect is 0, so don't check */
		if(g->expect && check_fieldset(g->fs,g->expect))
		{
			free_variable(g->v);
			e =  -2;
		}
	}

	return e;
}

static err  target_read(void *data,request *r,void *buffer,long *length)
{
	return -1;
}

static err  target_write(void *data,request *r,void *buffer,long *length)
{
	long len = *length;
	targetdata *g = (targetdata*)data;
	off_t pos;

	if(select_file(g,r,1,buffer,*length) != 0)
		return -2;

	if(g->fs) pos = ftell(g->current->file);

	/* marslog(LOG_DBUG,"target_write"); */

	if((len = timed_fwrite(buffer,1,*length,g->current->file,target_timer)) != *length)
	{
		marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
		*length = len;
		return EOF;
	}

	if(*length == 0)
		return 0;

	if(g->padding)
	{
		static int first = true;
		size_t pad = PAD(len,g->padding);
		static char c[10240];      /* 10Kb of padding should be enough */

		if(first)
		{
			memset(c,0,sizeof(c));
			first=false;
		}

		timer_start(target_timer);
		if(fwrite(c,1,pad,g->current->file) != pad)
		{
			marslog(LOG_EROR|LOG_PERR,"Error while writing to disk");
			timer_stop(target_timer,0);
			*length = len;
			return EOF;
		}
		timer_stop(target_timer,(long64)pad);
	}

	if(g->fs)
	{
		int n = g->order? cube_order(g->cube,r): g->count++;
		if(n < 0)
		{
			if(mars.debug)
			{
				request *w = empty_request("GRIB");
				marslog(LOG_DBUG,"Got order %d for following field",n);
				grib_to_request(w,buffer,*length);
				print_one_request(w);
				free_all_requests(w);
			}
		}

		return set_field(g->fs,read_field(g->file,pos,*length),n);
	}

	return 0;
}

static err  target_cntl(void *data,int code,void *param,int size)
{
	targetdata *g = (targetdata*)data;
	targetfile *f = 0;
	switch(code)
	{
		case CNTL_REWIND:
			g->count = 0;
			f = g->files;
			while(f) 
			{
				if(f->pipe)
				{
					marslog(LOG_EROR,"Cannot rewind a pipe");
					return -2;
				}
				fseek(f->file,f->start,SEEK_SET);
				ftruncate(fileno(f->file),f->start);
				f = f->next;
			}
			marslog(LOG_DBUG,"target_cntl");
			return NOERR;
			/*NOTREACHED*/
			break;

		case CNTL_ERROR:
			g->has_error = true;
			marslog(LOG_DBUG,"target_cntl");
			return NOERR;
			/*NOTREACHED*/
			break;

		default:
			return -1;
	}
}

err send_remote_targets()
{

	remotefile *f = remote_files;
	char buf[80];
	char buffer[2048];
	err e = 0;
	char *cmd = 0;

	if(remote_files == NULL)
		return 0;

	start_timer();

	while(f != NULL && e == 0)
	{
		marslog(LOG_INFO,"Sending %s:%s",f->host,f->remote);

		if(f->ecfs)
			cmd = "$ECFS_SYS_PATH/ecp.p -o";
		else
			cmd = "scp";

		sprintf(buffer,"%s %s %s:%s 2>&1",cmd,f->local,f->host,f->remote);

		if(system(buffer) != 0)
		{
			marslog(LOG_EROR|LOG_PERR,"Command \"%s\" failed",buffer);
			e = -2;
		}

		f = f->next;
	}


	f = remote_files;
	while(f)
	{
		remotefile *n = f->next;
		strfree(f->host);
		strfree(f->local);
		strfree(f->remote);
		FREE(f);
		f = n;
	}

	stop_timer(buf);
	if(*buf) marslog(LOG_INFO,"Sending time %s",buf);

	return e;
}
