/***************************** LICENSE START ***********************************

 Copyright 2014 ECMWF and INPE. This software is distributed under the terms
 of the Apache License version 2.0. In applying this license, ECMWF does not
 waive the privileges and immunities granted to it by virtue of its status as
 an Intergovernmental Organization or submit itself to any jurisdiction.

 ***************************** LICENSE END *************************************/

#include <iostream>
#include <sstream>
#include <fstream>
#include <cstdio>
#include <stdio.h>
#include <algorithm>

#include "grib_api.h"

#include "GribMetaData.h"

#include "MvKeyProfile.h"
#include "MvMessageMetaData.h"

#include "LogHandler.h"

static string GRIB_DEFINITION_PATH_ENV;
static string GRIB_DUMP_EXE;
static string GRIB_COUNT_EXE;

map<int,string> GribMvDump::keyMap_;


static void shellCommand(string &command, stringstream& out,stringstream& err, string &ftmp)
{
	FILE *in;
	char cbuf[512];

	string cmd;
	if(!GRIB_DEFINITION_PATH_ENV.empty())
	{
		cmd+="export GRIB_DEFINITION_PATH=" + GRIB_DEFINITION_PATH_ENV + ";";
	}
	
	cmd+=command + " 2>" + ftmp;
		
	if (!(in = popen(cmd.c_str() ,"r")) )
	{
		return;
	}

	while(fgets(cbuf, sizeof(cbuf), in) != NULL)
	{
		out << cbuf;		
	}	
	
	pclose(in);


	if (!(in = fopen(ftmp.c_str() ,"r")) )
	{
		return;
	}

	while(fgets(cbuf, sizeof(cbuf), in) != NULL)
	{
		err << cbuf;		
	}	
	
	fclose(in);

}




GribSection::~GribSection()
{
	for(vector<GribItem*>::iterator it=item_.begin(); it != item_.end(); it++)
	{
		delete *it;
	}
}


//===================================================
//
//  GribWmoDump
//
//===================================================

GribWmoDump::~GribWmoDump()
{
	clear();
}

void  GribWmoDump::clear()
{
	for(vector<GribSection*>::iterator it=section_.begin(); it != section_.end(); it++)
	{
		delete *it;
	}
	section_.clear();

	text_.clear();
}

void GribWmoDump::read(const string& fgrib, int count)
{
 	//Get grib dump for the given field
	//int count=1;

	stringstream in,err, cnt_s;

	cnt_s << count;
	string buf=GRIB_DUMP_EXE + " -O -w count=" + cnt_s.str() + " " + "\"" +fgrib +"\"";

	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	stringstream sst;
	sst << "Generating WMO-style dump for message: " << count;
	log->description(sst.str());
	log->command(buf);
	
	//Run dump command
	shellCommand(buf,in,err,tmpfilename_);
	
	//Get stdout of command
	text_=in.str();

	//Get stderr of command
	log->error(err.str());
		
	//Put stdout into a proper data structure	

        string stmp, sp1, sp2;
    	char c[1024];
	
	GribSection *asec=0;

    	while(in.getline(c,1024))
	{
		//cout << c << endl ;

		string::size_type pos, pos1;

		string buf; // Have a buffer string
    		stmp=c;
		stringstream ss(stmp); // Insert the string into a stream

		if( stmp.find("MESSAGE") != string::npos)
		{
		
		}
		else if((pos=stmp.find("SECTION_")) != string::npos || 
			(pos=stmp.find("SECTION")) != string::npos)
		{
			vector<string> items;
			GribSection *sec = new GribSection;
			sec->name("Section " + stmp.substr(pos+8,1));	
			asec=sec;
			section_.push_back(sec);
		}

		else if(asec != 0)
		{
			vector<string> sv;
    			int i=0;
			
			GribItem *item = new GribItem;
			while (ss >> buf)
			{
				if(i==0)
				{
					item->pos(buf);
				}
				else if(i==1)
				{
					item->name(buf);
				}
				else if(i== 2 && buf == "=")
				{					
					pos=stmp.find("=");
					pos1=stmp.rfind("{");
					if(pos1 != string::npos)
					{
						item->value(stmp.substr(pos,pos1-pos+1));
						
						in.getline(c,256);
						stmp=c;	
						while(stmp.find("}") == string::npos)
						{
							if(stmp.find("}") == string::npos)
							{
								if(stmp.find("...") == string::npos)
								{
									stringstream ssdata(stmp);
									while(ssdata >> buf)
									{
										item->addArrayData(buf);
									}
								}
								else
								{	
									item->addArrayData(stmp);
								}
							}

							if(! in.getline(c,256))
							{
								log->error("GribWmoDump::read() ---> Fatal formatting error in parsing WMO style dump!\n");
								return;
							}
							stmp=c;
						}
						if(stmp.find("}") != string::npos)
						{
							item->addArrayData(stmp);
						}
						//stmp now contains the "}" character
					}	

					else //if(buf.find("[") == string::npos || buf.find("]") == string::npos )
					{	
						item->value(stmp.substr(pos+1));
						//item->value(buf);	
					}

	
					break;
				}
				i++;	
			}
			asec->addItem(item);
		}
		//GribFieldMetaData* field = new GribFieldMetaData;		
	}
}

//===================================================
//
//  GribStdDump
//
//===================================================

GribStdDump::~GribStdDump()
{
	clear();
}

void GribStdDump::clear()
{
	for(vector<GribItem*>::iterator it=item_.begin(); it != item_.end(); it++)
	{
		delete *it;
	}
	item_.clear();

	text_.clear();
}

void GribStdDump::read(const string& fgrib, int count)
{
 	//Get grib dump for the given field
	
	stringstream in, err, cnt_s;

	cnt_s << count;
	string buf=GRIB_DUMP_EXE + " -w count=" + cnt_s.str() + " " + "\"" + fgrib + "\"" ; //+ "  > " + tmpFile_;

	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	stringstream sst;
	sst << "Generating default dump for message: " << count;
	log->description(sst.str());
	log->command(buf);

	shellCommand(buf,in,err,tmpfilename_);

	//Get stdout of command
	text_=in.str();
	
	//Get stderr of command	
	log->error(err.str());	
	
        string stmp, sp1, sp2, descText;   
    	char c[512];

	while(in.getline(c,512))
	{
		stmp=c;
		if( stmp.find("GRIB {") != string::npos)
		{
			break;
		}
	}

    	while(in.getline(c,512))
	{
		//cout << c << endl ;

		string::size_type pos;

		string buf; // Have a buffer string
    		stmp=c;
		stringstream ss(stmp); // Insert the string into a stream

		if(stmp.find("#") != string::npos && stmp.find("#-READ ONLY-") == string::npos)
		{
			descText= stmp;	
		}	
		else
		{		
			//vector<string> sv;
    			int i=0;
			
			GribItem *item = new GribItem;
			item_.push_back(item);

			while (ss >> buf)
			{
				if(buf.find("#-READ") != string::npos || 
				   buf.find("ONLY-") != string::npos)
				{
					continue;
				}

				if(stmp.find("=") == string::npos)
				{
					return;
				}

				if(i==0)
				{
					item->name(buf);
				}
				else if(i==2 && buf != "{")
				{
					item->value(buf.substr(0,buf.size()-1));

					if(descText.size() > 0)
					{
						item->description(descText);
						descText.clear();
					}

				}

				else if(i == 2)
				{			
					pos=stmp.rfind("{");
					
					//item->value(stmp.substr(pos,pos1-pos));
						
					in.getline(c,512);
					stmp=c;	
					while(stmp.find("}") == string::npos)
					{
						if(stmp.find("}") == string::npos)
						{
							if(stmp.find("...") == string::npos)
							{
								stringstream ssdata(stmp);
								while(ssdata >> buf)
								{
									item->addArrayData(buf);
								}
							}
							else
							{	
								item->addArrayData(stmp);
							}
						}

						if(! in.getline(c,512))
						{
							log->error("GribStdDump::read() ---> Fatal formatting error in parsing default style dump!\n");
							return;
						}
						stmp=c;
					}
					if(stmp.find("}") != string::npos)
					{
						item->addArrayData(stmp);
					}
						//stmp now contains the "}" character
				}	

			i++;	
			}
		}		
	}
}

//===================================================
//
//  GribMvDump
//
//===================================================

GribMvDump::GribMvDump(string &tmpfilename)
{
    tmpfilename_ = tmpfilename;

	if(keyMap_.empty())
	{
	  	keyMap_[GRIB_TYPE_STRING]="string";
		keyMap_[GRIB_TYPE_LONG]="long";
		keyMap_[GRIB_TYPE_DOUBLE]="double";
	}
}	
	
GribMvDump::~GribMvDump()
{
	clear();
}

void GribMvDump::clear()
{
	for(vector<GribItem*>::iterator it=item_.begin(); it != item_.end(); it++)
	{
		delete *it;
	}
	item_.clear();
}

void GribMvDump::read(const string& fgrib, int count)
{
 	//Get grib dump for the given field
	
	stringstream in, cnt_s;

	vector<std::string> name_space;
	name_space.push_back("Default");
	name_space.push_back("geography");
	name_space.push_back("ls");
	name_space.push_back("mars");
	name_space.push_back("parameter");
	name_space.push_back("statistics");
	name_space.push_back("time");
	name_space.push_back("vertical");

	// list of blacklisted keys - these can be very expensive to compute; we can
	// always put them back in if someone really really needs them...
	// to be considered: minimum, maximum
	vector<std::string> key_blacklist;
	key_blacklist.push_back("distinctLatitudes");
	key_blacklist.push_back("distinctLongitudes");
	key_blacklist.push_back("distinctLatitudes");

	// keys which are redundant or not properly defined
	key_blacklist.push_back("GRIBEditionNumber");


	//Other keys which define constant values and are not really useful for users
	key_blacklist.push_back("7777");
	key_blacklist.push_back("x");

  	FILE* fp;
  	grib_handle* gh=NULL;
  	grib_keys_iterator* kiter=NULL;
 	//const int MAX_KEY_LEN=255;
	const int MAX_VAL_LEN=1024;
	unsigned long key_iterator_filter_flags=GRIB_KEYS_ITERATOR_ALL_KEYS;

 	int err=0;
 	int grib_count=0;	

 	int keyType;
	//long longValue;
	string svalue;
 	char value[MAX_VAL_LEN];
  	size_t vlen=MAX_VAL_LEN;
	string cbuff;
	
	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	stringstream sst;
	sst << "Generating namespace dump for message: " << count;
	log->description(sst.str());
	log->method("GRIB API C interface");

	//open grib file for reading
  	fp = fopen(fgrib.c_str(),"r");
  	if(!fp) 
	{
		log->error("GribMetaData::getKeyList() ---> Cannot open grib file: \n        " + fgrib + "\n");
    		return;
  	}

	//Get messages form the file
  	while((gh = grib_handle_new_from_file(0,fp,&err)) != NULL || err != GRIB_SUCCESS)
	 {
		grib_count++;

		//if(!gh) 
		//{
      		//	log->error("GribMvDump::read() ---> Unable to create grib handle\n");
      		//	return;
    		//}

		if(grib_count == count)
		{
   			//cout << "-- GRIB N. " << grib_count << "---/n";
    			
			if(!gh) 
			{
      				stringstream err_s;
				err_s << "GribMvDump::read() ---> Unable to create grib handle for message no: " <<
                                                 grib_count << "\n";
				log->error(err_s.str());	
      				return;
			}
				
			for(unsigned int ns = 0 ; ns  < name_space.size(); ns++)
			{
   				//Get key iterator
				if (name_space[ns]!="Default")
					kiter=grib_keys_iterator_new(gh,key_iterator_filter_flags,const_cast<char *>(name_space[ns].c_str()));
				else
					kiter=grib_keys_iterator_new(gh,key_iterator_filter_flags,NULL);

				if (!kiter)
				{
					stringstream err_s;
					err_s << "GribMvDump::read() ---> Unable to create keys iterator for message no: " <<
                                                 grib_count << "\n";
					log->error(err_s.str());					
					return;
				}

				//Loop for the keys in the current message until all the keys are found
   				while(grib_keys_iterator_next(kiter))
   				{
					const char* name = grib_keys_iterator_get_name(kiter);

					if(!name)
					{
						continue;
					}	
					  
					string cbuff=name; 

					// if this key is in our blacklist, then do not use it
					vector<string>::const_iterator i = find(key_blacklist.begin(), key_blacklist.end(), name);

					if (i != key_blacklist.end())
						continue;


					//Add namespace prefix to the key name (e.g. mars.param)
					if(name_space[ns] != "Default")
					{
						cbuff = name_space[ns] + "." + cbuff;				
					}
					
					//Get string value
					vlen=MAX_VAL_LEN;
					size_t  len;
					
					if(grib_get_native_type(gh,name,&keyType) == GRIB_SUCCESS &&				
					   grib_get_size(gh,name,&len) == GRIB_SUCCESS && 
					   (keyType == GRIB_TYPE_STRING || keyType == GRIB_TYPE_LONG ||
					    keyType == GRIB_TYPE_DOUBLE ))
					{					  
						GribItem *item = new GribItem;
						item_.push_back(item);
						item->name(cbuff);
					
						if(keyType == GRIB_TYPE_STRING || len == 1)
						{												  
							if(grib_get_string(gh,name,value,&vlen) == GRIB_SUCCESS)
      							{
								//cout << "   " << cbuff << " =  " << value << endl;
								//Create the grib item

								item->value(value);
							}
						}
						else if(strcmp(name,"paramId") == 0 && keyType == GRIB_TYPE_LONG)
						{
							if(grib_get_string(gh,name,value,&vlen) == GRIB_SUCCESS)
							{
								item->value(value);
							}						
						}  			
						else
						{
							stringstream s;
							s << len;
							svalue="Array (" + s.str() + ")";
							item->value(svalue);	
						}
						
						map<int,string>::const_iterator  it=keyMap_.find(keyType);
						if(it  != keyMap_.end())
						{
							item->type(it->second);
						}	
					}
					
				}	

				grib_keys_iterator_delete(kiter);
			}    

			fclose(fp);
			break;			
		}

		if(gh)
			grib_handle_delete(gh);
  	}
}

//===================================================
//
//  GribValueDump
//
//===================================================

GribValueDump::GribValueDump()
{
	latitude_=0;
	longitude_=0;
	value_=0;
	num_=0;	
	decimalPlaces_=0;
}

GribValueDump::~GribValueDump()
{
	clear();
}

void GribValueDump::clear()
{
	if(latitude_)
		delete [] latitude_;
		
	if(longitude_)
		delete [] longitude_;
		
	if(value_)
		delete [] value_;
	
	latitude_=0;
	longitude_=0;
	value_=0;
	num_=0;	
	gridType_.clear();
				
}

void GribValueDump::readFailed(FILE* fp,grib_handle* gh)
{
	grib_handle_delete(gh);
	fclose(fp);
	clear();
}	

void GribValueDump::read(const string& fgrib, int count)
{ 
 	//Get grib dump for the given field
	
	stringstream in, cnt_s;

  	FILE* fp;
  	grib_handle* gh=NULL;
  	
 	int err=0;
 	int grib_count=0;	

	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	stringstream sst;
	sst << "Generating value dump for message: " << count;
	log->description(sst.str());
	log->method("GRIB API C interface");

	//Clear the currently allocated data
	clear();

	//open grib file for reading
  	fp = fopen(fgrib.c_str(),"r");
  	if(!fp) 
	{
		log->error("GribValueDump::read() ---> Cannot open grib file: \n        " + fgrib + "\n");
    		return;
  	}

	//Get messages form the file
  	while(((gh = grib_handle_new_from_file(0,fp,&err)) != NULL || err != GRIB_SUCCESS) && grib_count < count)
	 {
		grib_count++;

		//if(!gh) 
		//{
      		//	log->error("GribValueDump::read() ---> Unable to create grib handle\n");
      		//	return;
    		//}

		if(grib_count == count)
		{

		  	if(!gh) 
			{
      				stringstream err_s;
				err_s << "GribValueDump::read() ---> Unable to create grib handle for message no: " <<
                                                 grib_count << "\n";
				log->error(err_s.str());
				return;
    			}		  
		  
		  	size_t latNum, lonNum, valNum;
			string latName, lonName;
			
			//cout << "-- GRIB N. " << grib_count << "---/n";

			const int MAX_VAL_LEN=1024;				
			char cval[MAX_VAL_LEN];					
			size_t vlen=MAX_VAL_LEN;
			
			//Grid type
			if(grib_get_string(gh,"gridType",cval,&vlen) == GRIB_SUCCESS)
			{
				gridType_=string(cval);
			}
			else
			{
				log->error("GribValueDump::read() ---> No gridType data found\n");
				readFailed(fp,gh);
				return;	
			}		
			
			if(gridType_ == "sh")
			{
				latName="";
				lonName="";
				log->error("GribValueDump::read() ---> gridType \"sh\" is not supported\n");
				readFailed(fp,gh);
				return;	
			}
			else
			{
				latName="latitudes";
				lonName="longitudes";
			}			


			//Latitude
			grib_get_size(gh,latName.c_str(),&latNum);
			if(latNum >0)
			{
				latitude_ = new double[latNum]; 
			}
			else
			{	
				log->error("GribValueDump::read() ---> No latitude data found\n");
				readFailed(fp,gh);
				return;	
			}	
				
			if(grib_get_double_array(gh,latName.c_str(),latitude_,&latNum) != 0)
			{
				log->error("GribValueDump::read() ---> Failed to read latitude data\n");
				readFailed(fp,gh);
				return;
	 		}
			
			//Longitude
			grib_get_size(gh,lonName.c_str(),&lonNum);
			if(lonNum != latNum)
			{
				log->error("GribValueDump::read() ---> No longitude data found\n");
				readFailed(fp,gh);
				return;
			}	
			else
			{
				longitude_ = new double[lonNum]; 
			}	
			
	
			if(grib_get_double_array(gh,lonName.c_str(),longitude_,&lonNum) != 0)
			{
				log->error("GribValueDump::read() ---> Failed to read longitude data\n");
				readFailed(fp,gh);
				return;
	 		}
			//Value
			grib_get_size(gh,"values",&valNum);
			if(valNum != latNum)
			{
				log->error("GribValueDump::read() ---> No value data found\n");
				readFailed(fp,gh);
				return;
			}	
			else
			{
				value_ = new double[valNum]; 
			}
			
			if(grib_get_double_array(gh,"values",value_,&valNum) != 0)
			{
				log->error("GribValueDump::read() ---> Failed to read value data\n");
				readFailed(fp,gh);
				return;
	 		}
			
			double max=-999999999.;
			double min=999999999.;
			for(unsigned int i=0; i < valNum; i++)
			{
			  	if(value_[i] > max)
				  	max=value_[i];
				if(value_[i] < min)
				  	min=value_[i];
			}
			
			double maxAbs=(fabs(max) > fabs(min))?fabs(max):fabs(min);
			decimalPlaces_=0;
			for(int i=0; i >-10; i--)
			{
			  	if(maxAbs > pow(10,static_cast<double>(i)))
				{
				  	decimalPlaces_=-i;
					break;
				}
			}	
			
					
			
			//Statistics
			/*double dval;
			if(grib_get_double(gh,"average",&dval) == GRIB_SUCCESS)
				average_=dval;
			if(grib_get_double(gh,"standardDeviation",&dval) == GRIB_SUCCESS)
				stdev_=dval;				
			if(grib_get_double(gh,"skewness",&dval) == GRIB_SUCCESS)
				skewness_=dval;
			if(grib_get_double(gh,"kurtosis",&dval) == GRIB_SUCCESS)
				kurtosis_=dval;*/
			num_=latNum;
				
		}
		
		if(gh)
			grib_handle_delete(gh);
  	}
	
	fclose(fp);
}



//=========================================
//
//  GribMetaData
//
//=========================================

GribMetaData::GribMetaData()
{		
        string cmd;
	
	GRIB_DEFINITION_PATH_ENV="";
	const char* grb_def_path=getenv("GRIB_DEFINITION_PATH");
	if(grb_def_path)
	{
		GRIB_DEFINITION_PATH_ENV=string(grb_def_path);
	}

        //Define the grib dump program name
	char *gdexe=getenv("METVIEW_GRIB_DUMP");

 	if (gdexe == 0)  
	{	
		GRIB_DUMP_EXE="grib_dump";
		GRIB_COUNT_EXE="grib_count";
	}
	else
	{
		GRIB_DUMP_EXE=gdexe;
		GRIB_COUNT_EXE="grib_count";
	}

	observer_=0;
	hasMultiMessage_=false;
	
	grib_multi_support_on(0);
}

GribMetaData::~GribMetaData()
{
}

void GribMetaData::setFilter(vector<off_t> offset, vector<int> len)
{
	filterOffset_=offset;
	filterLen_=len;
	filterCnt_.clear();
	for(unsigned int i=0; i < filterOffset_.size(); i++)
	{
		filterCnt_.push_back(-1);
	}

	if(offset.size() != 0 && 
	   offset.size() == len.size())
	{
		filterEnabled_=true;
	}
}

void GribMetaData::setFileName(string fname)
{
	fileName_=fname;
	messageNum_=0;
	totalMessageNum_=computeTotalMessageNum();

}

int GribMetaData::computeTotalMessageNum()
{	
  	FILE* fp;
	fp = fopen(fileName_.c_str(),"r");
  	if(!fp) 
	{
    		//log->error("GribMetaData::readMessages() ---> Cannot open grib file: \n        " + fileName_ + "\n");
    		return 0;
  	}
  	
  	int res=0;
  	if(grib_count_in_file(0,fp,&res) != GRIB_SUCCESS)
		res=0;
	
	fclose(fp);
	return res;
	  	
	/*stringstream in, err;
	
	string buf=GRIB_COUNT_EXE + " " + "\"" + fileName_ + "\"" ; 
	shellCommand(buf,in,err,tmpfilename_);

	if(!err.str().empty())
		return -1;

	int res;
	std::istringstream iss(in.str());
  	iss >> res;
	
	return res;*/
}

int GribMetaData::getEstimatedMessageNum()
{
	if(filterEnabled_ == true)
	{
		return filterOffset_.size();
	}

	return totalMessageNum_;
}


void GribMetaData::loadKeyProfile(MvKeyProfile *prof)
{
	prof->clearKeyData();
	readMessages(prof);
}

void GribMetaData::readMessages(MvKeyProfile *prof)
{
	/*vector<char*> name_space;
	name_space.push_back(0);
	name_space.push_back("geography");
	name_space.push_back("ls");== GRIB_SUCCESS
	name_space.push_back("mars");
	name_space.push_back("parameter");
	name_space.push_back("time");
	name_space.push_back("vertical");*/

  	FILE* fp;
  	grib_handle* gh=NULL;
 	int err=0;
 	int grib_count=0;
	long longValue;
	string cbuff;

	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	log->description("Generating grib key list for all the messages");
	log->method("GRIB API C interface");

	messageNum_=0;

	//open grib file for reading
  	fp = fopen(fileName_.c_str(),"r");
  	if(!fp) 
	{
    		log->error("GribMetaData::readMessages() ---> Cannot open grib file: \n        " + fileName_ + "\n");
    		return;
  	}

	//For filtered datasets
	if(filterEnabled_==true)
	{					
		//At the first scan we need to find the message cnt for each filtered message.
		//The goal is to fill up the filterCnt_ vector.
		if(firstScan_)
		{
			//long currentOffset=0;
			int msgCnt=0;
			while((gh = grib_handle_new_from_file(0,fp,&err)) != NULL || err != GRIB_SUCCESS)
	 		{
    				if(!gh) 
				{
      					stringstream sst;
					sst <<  "GribMetaData::readMessages() --->  Unable to create grib handle for message count: " << grib_count+1;
					log->error(sst.str());
    				}
				
				msgCnt++;

				if(gh)
				{  
					if(grib_get_long(gh,"offset",&longValue) == GRIB_SUCCESS)
					{
						for(unsigned int i=0; i < filterOffset_.size(); i++)
						{
							off_t offset_diff = longValue - filterOffset_[i];
							if(ABS(offset_diff) < 120)
							{
								filterCnt_[i] = msgCnt;
								break;
							}
						}
					}
					else
					{
						stringstream sst;
						sst <<  "GribMetaData::readMessages() --->  Cannot get offset for message count: " << grib_count+1;
						log->error(sst.str());
      						grib_handle_delete(gh);
						return;
					}

					//currentOffset+=longValue;
					grib_handle_delete(gh);
				}	
			}
		}
		
		//Read the filtered messages
		for(unsigned int i=0; i < filterOffset_.size(); i++)
		{
			grib_count++;

			fseek(fp,filterOffset_[i],SEEK_SET);
	
			int err;
			gh = grib_handle_new_from_file(0,fp,&err);
	
			if(!gh) 
			{
				stringstream sst;
				sst <<  "GribMetaData::readMessages() --->  Unable to create grib handle for offset: " << filterOffset_[i];
				log->error(sst.str());
				//return;
			}
				
			readMessage(prof,gh);
	
			observer_->messageScanStepChanged(grib_count);
						
			if(gh)
				grib_handle_delete(gh);
		}	
	}	
	else
	{	  
		//Get messages form the file
  		while((gh = grib_handle_new_from_file(0,fp,&err)) != NULL || err != GRIB_SUCCESS)
	 	{
    			if(!gh) 
			{
      				stringstream sst;
				sst <<  "GribMetaData::readMessages() --->  Unable to create grib handle for message count: " << grib_count+1 << endl;
				log->error(sst.str());
      				//return;
				//continue;
    			}
		
			grib_count++;

			readMessage(prof,gh);

			observer_->messageScanStepChanged(grib_count);
			
			if(gh) 
				grib_handle_delete(gh);
		}

		if(firstScan_)
		{
			if(totalMessageNum_ != grib_count)
			{
				if(totalMessageNum_ != 0)
				  	hasMultiMessage_=true;
			  	
				totalMessageNum_=grib_count;			  	
			}
		}
	}
	
	fclose(fp);
	
	messageNum_=grib_count;
	firstScan_=false;


	//------------------------------
	// Format key "count"
	//------------------------------

	char c[20];
	int n,c_format_id;
	const char c_format[4][5]={"%02d","%03d","%04d","%05d"};

	int num=messageNum_;
	if(num < 100)
	{
		c_format_id=0;
	}
	else if(num < 1000)
	{
		c_format_id=1;		
	}
	else if(num < 10000)
	{
		c_format_id=2;	
	}	
	else
	{
		c_format_id=3;	
	}

	MvKey* key=prof->key("count");
	if(key)
	{
		for(unsigned int i=0; i< key->value().size(); i++)
		{
			string s=key->value()[i];
			if(s != "")
			{ 
				stringstream sst(s);
				sst >> n;
				sprintf(c,c_format[c_format_id],n);
				s=c;
				key->setValue(i,s);
			}
		}
	
	}

	key=prof->key("MV_Index");
	if(key)
	{
		for(unsigned int i=0; i< key->value().size(); i++)
		{
			sprintf(c,c_format[c_format_id],i+1);
			string s(c);
			key->setValue(i,s);
		}
	
	}
}

void GribMetaData::getKeyList(int count,string nameSpace,MvKeyProfile *prof)
{
	list<string> keys;
	getKeyList(count,nameSpace,keys);
	for(list<string>::iterator it=keys.begin(); it != keys.end(); it++)
	{
		prof->addKey(new MvKey((*it),(*it)));
	}
}


void GribMetaData::getKeyList(int count,string nameSpace,list<string> &keys)
{
	vector<char*> nameSpaceL;
	
	if(nameSpace=="Default")
	{
		nameSpaceL.push_back(0);
	}
	else
	{
		char* nsC = new char[nameSpace.size() + 1];
		std::copy(nameSpace.begin(),nameSpace.end(),nsC);
		nsC[nameSpace.size()] = '\0';
		nameSpaceL.push_back(nsC);
	}

	// list of blacklisted keys - these can be very expensive to compute; we can
	// always put them back in if someone really really needs them...
	// to be considered: minimum, maximum
	vector<string> key_blacklist;
	key_blacklist.push_back("distinctLatitudes");
	key_blacklist.push_back("distinctLongitudes");
	key_blacklist.push_back("distinctLatitudes");

	//Other keys which define constant values and are not really useful for users
	key_blacklist.push_back("7777");
	key_blacklist.push_back("x");


  	FILE* fp;
  	grib_handle* gh=NULL;
  	grib_keys_iterator* kiter=NULL;
 	//const int MAX_KEY_LEN=255;
	//const int MAX_VAL_LEN=1024;
	unsigned long key_iterator_filter_flags=GRIB_KEYS_ITERATOR_ALL_KEYS;

 	int err=0;
 	int grib_count=0;
	int keyType;

 	//char value[MAX_VAL_LEN];
  	//size_t vlen=MAX_VAL_LEN;
	string cbuff;
	
	LogItem *log=new LogItem;
	LogHandler::instance()->add(log);

	stringstream sst;
	sst << "Generating grib key list for message: " << count;
	log->description(sst.str());
	log->method("GRIB API C interface");

	//open grib file for reading
  	fp = fopen(fileName_.c_str(),"r");
  	if(!fp) 
	{
		log->error("GribMetaData::getKeyList() ---> Cannot open grib file: \n        " + fileName_ + "\n");
    		return;
  	}

	//Get messages form the file
  	while((gh = grib_handle_new_from_file(0,fp,&err)) != NULL || err != GRIB_SUCCESS)
	 {
		grib_count++;

		/*if(!gh) 
		{
      			log->error("GribMetaData::getKeyList() ---> Unable to create grib handle\n");
      			return;
    		}*/

		if(grib_count == count)
		{
   			//cout << "-- GRIB N. " << grib_count << "---/n";
    			
			if(!gh) 
			{
      				stringstream err_s;
				err_s << "GribMetaData::getKeyList() ---> Unable to create grib handle for message no: " <<
                                                 grib_count << "\n";
				log->error(err_s.str());		 
      				
				for(vector<char*>::iterator it=nameSpaceL.begin(); it != nameSpaceL.end(); it++)
				{
					if((*it) != 0)
						delete [] *it;
				}
				
				return;
    			}
			
			for(unsigned int ns = 0 ; ns  < nameSpaceL.size(); ns++)
			{
   				//Get key iterator
				kiter=grib_keys_iterator_new(gh,key_iterator_filter_flags,nameSpaceL[ns]);
    				if (!kiter)
				{
      					stringstream err_s;
					err_s << "GribMetaData::getKeyList() ---> Unable to create keys iterator for message no: " <<
                                                 grib_count << "\n";
					log->error(err_s.str());					
      					
					for(vector<char*>::iterator it=nameSpaceL.begin(); it != nameSpaceL.end(); it++)
					{
						if((*it) != 0)
							delete [] *it;
					}
					
					return;
    				}

				//Loop for the keys in the current message until all the keys are found
   				while(grib_keys_iterator_next(kiter))
   				{
					const char* name = grib_keys_iterator_get_name(kiter);
				        if(!name)
					{
						continue;
					}	
					
					string cbuff=name; 
				
					// if this key is in our blacklist, then do not use it
					vector<string>::const_iterator i = find(key_blacklist.begin(), key_blacklist.end(), name);

					if (i != key_blacklist.end())
						continue;

					//Add namespace prefix to the key name (e.g. mars.param)
					if(nameSpaceL[ns] != 0)
					{
						cbuff = string(nameSpaceL[ns]) + "." + cbuff;				
					}	
						
					if(grib_get_native_type(gh,cbuff.c_str(),&keyType) == GRIB_SUCCESS &&  
					   (keyType == GRIB_TYPE_STRING || keyType == GRIB_TYPE_LONG ||
		    		           keyType == GRIB_TYPE_DOUBLE))
					 {						
						keys.push_back(cbuff);	
					 }	
				}	

				grib_keys_iterator_delete(kiter);
    			}    

			grib_handle_delete(gh);
			break;			
		}

		grib_handle_delete(gh);
  	}

	keys.sort();

	fclose(fp);

	//This has to be called before calling return in this method!!
	for(vector<char*>::iterator it=nameSpaceL.begin(); it != nameSpaceL.end(); it++)
	{
		if((*it) != 0)
			delete [] *it;
	}
 

}

long GribMetaData::gribapiVersionNumber()
{
	return grib_get_api_version();
}			

void GribMetaData::readMessage(MvKeyProfile* prof,grib_handle *gh)
{
	
  	if(!gh)
	{
		for(unsigned int i=0; i< prof->size() ; i++ )
		{	
			prof->at(i)->addValue("N/A");
		}
	  	return;
	}
  
  	const int MAX_VAL_LEN=1024;	
	long longValue;
	char value[MAX_VAL_LEN];
	string svalue;
	size_t vlen=MAX_VAL_LEN;

	for(unsigned int i=0; i< prof->size() ; i++ )
	{
		size_t len;
		int keyType;
		const char* name=prof->at(i)->name().c_str();
		bool foundValue=false;

		if(grib_get_native_type(gh,name,&keyType) == GRIB_SUCCESS &&
		   grib_get_size(gh,name,&len) == GRIB_SUCCESS &&
		   (keyType == GRIB_TYPE_STRING || keyType == GRIB_TYPE_LONG ||
		    keyType == GRIB_TYPE_DOUBLE))
		{
			if(keyType == GRIB_TYPE_STRING || len == 1)
			{						
				if(prof->at(i)->readIntAsString() == false && 
				   keyType ==GRIB_TYPE_LONG)			
				{					
				   	if(grib_get_long(gh,name,&longValue) == GRIB_SUCCESS)
					{
						stringstream s;
						s << longValue;
						prof->at(i)->addValue(s.str().c_str());
						foundValue=true;
					}
				}
				else
				{
					vlen=MAX_VAL_LEN;
					if(grib_get_string(gh,name,value,&vlen) == GRIB_SUCCESS)
					{						
						prof->at(i)->addValue(value);
						foundValue=true;
					}
				}
			}
			else if((strcmp(name,"paramId") == 0 || strcmp(name,"parameter.paramId") == 0) &&
			        keyType == GRIB_TYPE_LONG)
			{
				if(prof->at(i)->readIntAsString() == false)
				{  				  
					if(grib_get_long(gh,name,&longValue) == GRIB_SUCCESS)
					{
						stringstream s;
						s << longValue;
						prof->at(i)->addValue(s.str().c_str());
						foundValue=true;
					}						
				}  
				else
				{
					vlen=MAX_VAL_LEN;
					if(grib_get_string(gh,name,value,&vlen) == GRIB_SUCCESS)
					{						
						prof->at(i)->addValue(value);
						foundValue=true;
					}
				}					
			}  			
			else
			{		
				stringstream s;
				s << len;
				svalue="Array (" + s.str() + ")";	
				prof->at(i)->addValue(svalue);
				foundValue=true;
			}
		}


		if(!foundValue)
		{
			prof->at(i)->addValue("N/A");	
		}		
	}
}	
