//============================================================================
// Name        :
// Author      : Avi
// Revision    : $Revision: #66 $ 
//
// Copyright 2009-2020 ECMWF.
// This software is licensed under the terms of the Apache Licence version 2.0 
// which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 
// In applying this licence, ECMWF does not waive the privileges and immunities 
// granted to it by virtue of its status as an intergovernmental organisation 
// nor does it submit to any jurisdiction. 
//
// Description :
//============================================================================
#include <stdexcept>
#include <sstream>
#include <sys/stat.h>
#include <cerrno>
#include <memory>
#include <sys/wait.h> // for waitpid

#include "boost/filesystem/operations.hpp"
#include <boost/algorithm/string/trim.hpp>

#include "EcfFile.hpp"
#include "Log.hpp"
#include "Str.hpp"
#include "Ecf.hpp"
#include "File.hpp"
#include "Submittable.hpp"
#include "JobsParam.hpp"

//#define DEBUG_ECF_ 1
//#define DEBUG_PRE_PROCESS 1
//#define DEBUG_PRE_PROCESS_INCLUDES 1    // be careful with this as it will skew  the diffs in test code
//#define DEBUG_MIGRATE 1                 // for TEST ONLY

//#define DEBUG_PRE_PROCESS_OUTPUT 1
//#define DEBUG_VAR_SUB_OUTPUT 1
//#define DEBUG_MAN_FILE 1

#ifdef DEBUG_ECF_
#include <iostream>
#endif

namespace fs = boost::filesystem;
using namespace std;
using namespace ecf;
using namespace boost;
static const char* T_NOOP        = "nopp";
static const char* T_COMMENT     = "comment";
static const char* T_MANUAL      = "manual";
static const char* T_END         = "end";
static const char* T_ECFMICRO    = "ecfmicro";
static const char* T_INCLUDE     = "include ";
static const char* T_INCLUDENOPP = "includenopp ";
static const char* T_INCLUDEONCE = "includeonce ";

static void vector_to_string(const std::vector<std::string>& vec, std::string& str)
{
   // Determine size of string, to avoid reallocation
   size_t the_string_size = 0;
   size_t theSize = vec.size();
   for(size_t i = 0; i < theSize; i++) { the_string_size += vec[i].size() + 1; } // +1 is for "\n";
   str.reserve( str.size() + the_string_size);

   // populate string using the vector
   for(size_t i = 0; i < theSize; i++) {
      str += vec[i];
      str += "\n";
   }
}

EcfFile::EcfFile()= default;

EcfFile& EcfFile::operator=(const EcfFile& rhs)
{
   /// This preserves the caches, used to avoid opening/stat of include file more than once.
   // assign in order or declaration
   node_ = rhs.node_;
   ecfMicroCache_ = rhs.ecfMicroCache_;
   script_path_or_cmd_ = rhs.script_path_or_cmd_;
   jobLines_.resize(0);  // the most expensive
   job_size_.clear();
   script_origin_ = rhs.script_origin_;
   ecf_file_search_algorithm_ = rhs.ecf_file_search_algorithm_;
   return *this;
}

// ==================================================================================
// Avoid making any data model/defs state changes. Changes like:
//       node_->flag().set(ecf::Flag::NO_SCRIPT);
// Which will cause a state change.
// If the script fails then it is up to the calling Task to set this flag
// This class is also used to extract the scripts/manual, etc form
// a read only command. hence this class can not make state changes
// ===================================================================================
EcfFile::EcfFile( Node* t,
                  const std::string& pathToEcfFileOrCommand,
                  EcfFile::Origin  script_origin,
                  EcfFile::EcfFileSearchAlgorithm search_algo
)
: node_( t ),
  script_path_or_cmd_( pathToEcfFileOrCommand ),
  script_origin_( script_origin ),
  ecf_file_search_algorithm_( search_algo )
{
   node_->findParentUserVariableValue(Str::ECF_MICRO(),ecfMicroCache_);
   if ( ecfMicroCache_.empty() || ecfMicroCache_.size() != 1) {
      std::stringstream ss;
      ss << "EcfFile::EcfFile: Node " << t->absNodePath() << " is referencing a invalid ECF_MICRO variable(' " << ecfMicroCache_ << "). ECF_MICRO when overridden, must be a single character.";
      throw std::runtime_error(ss.str());
   }

#ifdef DEBUG_ECF_
   cout << "   EcfFile::EcfFile pathToEcfFileOrCommand = " << script_path_or_cmd_ << " script_type = " << script_type << "\n";
#endif
}

void EcfFile::manual(std::string& theManual)
{
   /// Pre-process the file accessible from the server
   std::vector<std::string> lines;
   std::string error_msg;
   EcfFile::Type file_type = (node_->isSubmittable()) ? EcfFile::SCRIPT : EcfFile::MANUAL ;
   if (!open_script_file(script_path_or_cmd_, file_type, lines,  error_msg)) {
      std::stringstream ss; ss << "EcfFile::manual: For node " << node_->debugNodePath() << ", failed to open file " << script_path_or_cmd_ << " : " << error_msg;
      throw std::runtime_error(ss.str());
   }

   // expand all %includes this will expand %includenopp by enclosing in %nopp %end, will populate jobLines_
   PreProcessor data(this,"EcfFile::manual:");
   data.preProcess(lines);

   // perform variable sub's but don't error if failure
   try {
      JobsParam dummy; // create jobs = false, spawn jobs =  false
      variableSubstitution(dummy);
   }
   catch (...) {}

   vector<string> theManualLines;
   if (!extractManual(jobLines_,theManualLines,error_msg)) {
      std::stringstream ss;
      ss << "EcfFile::manual: extraction failed for task " << node_->absNodePath() << " " << error_msg;
      throw std::runtime_error(ss.str());
   }

   if (theManualLines.empty()) {
      // There is no %manual -> %end in the file. However this may be .man file for Suites/Family
      // For this case just include the pre-processed contents as is, ie since the whole file
      // is the manual
      if (node_->isNodeContainer()) {
         vector_to_string(jobLines_,theManual);
         return;
      }
   }

   vector_to_string(theManualLines,theManual);
}

std::string EcfFile::origin_str(EcfFile::Origin origin)
{
   string ret;
   switch (origin) {
      case ECF_FILES:     ret = "ECF_FILES"; break;
      case ECF_HOME:      ret = "ECF_HOME"; break;
      case ECF_SCRIPT:    ret = "ECF_SCRIPT"; break;
      case ECF_FETCH_CMD: ret = "ECF_FETCH"; break;
      case ECF_SCRIPT_CMD:ret = "ECF_SCRIPT_CMD"; break;
   }
   return ret;
}

std::string EcfFile::search_algorithm_str(EcfFile::EcfFileSearchAlgorithm sa)
{
   string ret;
   switch (sa) {
      case PRUNE_ROOT:ret = "PRUNE_ROOT"; break;
      case PRUNE_LEAF:ret = "PRUNE_LEAF"; break;
   }
   return ret;
}

std::string EcfFile::ecf_file_origin_dump() const
{
   string origin = "# ecf_script_origin :";
   switch (script_origin_) {
      case ECF_FILES:{    origin += " ECF_FILES(";
                          if (ecf_file_search_algorithm_ == EcfFile::PRUNE_ROOT) origin += "PRUNE_ROOT) : ";
                          else                                                   origin += "PRUNE_LEAF) : ";
                          break;}
      case ECF_HOME:{     origin += " ECF_HOME(";
                          if (ecf_file_search_algorithm_ == EcfFile::PRUNE_ROOT) origin += "PRUNE_ROOT) : ";
                          else                                                   origin += "PRUNE_LEAF) : ";
                          break;}
      case ECF_SCRIPT:    origin += " ECF_SCRIPT : "; break;
      case ECF_FETCH_CMD: origin += " ECF_FETCH : "; break;
      case ECF_SCRIPT_CMD:origin += " ECF_SCRIPT_CMD : "; break;
   }
   origin += script_path_or_cmd_;
   return origin;
}

void EcfFile::script(std::string& theScript) const
{
   if ( script_origin_ == EcfFile::ECF_SCRIPT) {
      if (!File::open(script_path_or_cmd_,theScript)) {
         std::stringstream ss;
         ss << "EcfFile::script: Could not open script for task/alias " << node_->absNodePath() << " at path " << script_path_or_cmd_ << " (" << strerror(errno) << ")";
         throw std::runtime_error(ss.str());
      }
      return;
   }

   // record script origin, as a comment on the first line
   std::vector<std::string> lines; lines.push_back(ecf_file_origin_dump());
   std::string error_msg;
   if (!open_script_file(script_path_or_cmd_, EcfFile::SCRIPT, lines,  error_msg)) {
      std::stringstream ss;
      ss << "EcfFile::script: Could not open script for task/alias " << node_->absNodePath() << " using command " << script_path_or_cmd_;
      throw std::runtime_error(ss.str());
   }
   vector_to_string(lines,theScript);
}

void EcfFile::pre_process_user_file(std::vector<std::string>& user_edit_file, std::string& pre_processed_file)
{
   // expand all %includes this will expand %includenopp by enclosing in %nopp %end, will populate jobLines_
   PreProcessor data(this,"EcfFile::pre_process_user_file");
   data.preProcess(user_edit_file);

   remove_comment_manual_and_noop_tokens();

   JobsParam dummy;
   variableSubstitution(dummy);

   vector_to_string(jobLines_,pre_processed_file);
}

void EcfFile::pre_process(std::string& pre_processed_file)
{
   /// Pre-process the ECF file accessible from the server
   std::vector<std::string> lines;
   std::string error_msg;
   if (!open_script_file(script_path_or_cmd_, EcfFile::SCRIPT, lines,  error_msg)) {
      std::stringstream ss;
      ss << "EcfFile::pre_process: Failed to open file " << script_path_or_cmd_ << " : " << error_msg;
      throw std::runtime_error(ss.str());
   }

   // expand all %includes this will expand %includenopp by enclosing in %nopp %end, will populate jobLines_
   PreProcessor data(this,"EcfFile::pre_process");
   data.preProcess(lines);

   /// Find Used variables, *after* all %includes expanded, can throw std::runtime_error
   get_used_variables(pre_processed_file);

   /// Add pre-processed content
   vector_to_string(jobLines_,pre_processed_file);
}

void EcfFile::edit_used_variables(std::string& return_script_with_used_variables)
{
   std::string errorMsg;
   std::vector<std::string> lines;
   if (!open_script_file(script_path_or_cmd_, EcfFile::SCRIPT, lines,  errorMsg)) {
      throw std::runtime_error( "EcfFile::edit_used_variables: Open script failed : " + errorMsg ) ;
   }

   // Copy the script file, *BEFORE* expanding the includes
   std::string script;
   vector_to_string(lines,script);

   // expand all %includes
   PreProcessor data(this,"EcfFile::edit_used_variables");
   data.preProcess(lines);

   /// Find Used variables, *after* all %includes expanded, Can throw std::runtime_error
   get_used_variables(return_script_with_used_variables);

   /// Return Used variables and SCRIPT before pre-processing
   return_script_with_used_variables += script;
}


const std::string& EcfFile::create_job( JobsParam& jobsParam)
{
#ifdef DEBUG_ECF_
   cout << "EcfFile::createJob task " << node_->absNodePath() << " script_path_or_cmd_ = " << script_path_or_cmd_ << "\n";
#endif

   // NOTE: When editing pure python jobs, we may have *NO* variable specified, but only user_edit_file
   //       hence whenever we have user_edit_file, we should follow the else part below
   std::string error_msg;
   { // add scope to limit lifetime of lines variable
      std::vector<std::string> lines;
      if (jobsParam.user_edit_variables().empty() && jobsParam.user_edit_file().empty()) {
         /// The typical *NORMAL* path
         if (!open_script_file(script_path_or_cmd_, EcfFile::SCRIPT, lines,  error_msg)) {
            throw std::runtime_error("EcfFile::create_job: failed " + error_msg );
         }
      }
      else {
         // *USER* edit, two kinds
         if (jobsParam.user_edit_file().empty()) {
            // *USE* user variables, but ECF file accessible from the server
            if (!open_script_file(script_path_or_cmd_, EcfFile::SCRIPT, lines,  jobsParam.errorMsg())) {
               throw std::runtime_error("EcfFile::create_job: User variables, Could not open script: " + error_msg );
            }
         }
         else {
            // *USE* the user supplied ECF file *AND* user variables
            lines = jobsParam.user_edit_file();
         }
      }

      // expand all %includes this will expand %includenopp by enclosing in %nopp %end

      PreProcessor data(this,"EcfFile::create_job");
      data.preProcess(lines);
   }

#ifdef DEBUG_PRE_PROCESS_OUTPUT
   std::string err;
   File::create("preProcess" + get_extn(),jobLines_,err);
#endif

   // _IF_ ECF_CLIENT is specified provide Special support for migration.
   // The variable ECF_CLIENT is used to specify the path to client exe.
   // This is then used to replace smsinit,smscomplete, smsevent,smsmeter.smslabel,smsabort
   std::string clientPath;
   if (node_->findParentUserVariableValue("ECF_CLIENT", clientPath)) {
      if (!replaceSmsChildCmdsWithEcf(clientPath,error_msg) ) {
         throw std::runtime_error("EcfFile::create_job: ECF_CLIENT replacement failed " + error_msg );
      }
#ifdef DEBUG_MIGRATE
      std::string err;
      File::create("migrate" + get_extn(),jobLines_,err);
#endif
   }

   /// Will use *USER* supplied edit variables in preference to node tree variable *IF* supplied
   /// expand %VAR% or %VAR:sub% & replace %% with %
   // Allow variable substitution in comment and manual blocks. But if it fails, don't report as an error
   variableSubstitution(jobsParam);

#ifdef DEBUG_VAR_SUB_OUTPUT
   std::string err1;
   File::create("variableSub" + get_extn(),jobLines_,err1);
#endif

#ifdef DEBUG_MAN_FILE
   if (!doCreateManFile(error_msg))  {
      throw std::runtime_error("EcfFile::create_job: manual file creation failed " + error_msg );
   }
#endif

   /// Create the user file for tasks, when submitting without aliases,  before removing comments
   if (node_->isTask() && !jobsParam.user_edit_variables().empty()) {
      doCreateUsrFile();
   }

   remove_comment_manual_and_noop_tokens();

   return doCreateJobFile(jobsParam/* this is only past in for profiling */); // create job on disk
}

void EcfFile::extract_used_variables(NameValueMap& used_variables_as_map,const std::vector<std::string> &script_lines)
{
   // we only process the contents of the FIRST %comment  %end
   bool comment = false;
   size_t theSize = script_lines.size();
   for(size_t i=0; i < theSize; ++i) {

      // std::cout << "EcfFile::extract_used_variables found:'" << script_lines[i] << "\n";
      if (script_lines[i].empty()) continue;

      // take into account micro char during variable substitution
      string::size_type ecfmicro_pos = script_lines[i].find(Ecf::MICRO());
      if ( ecfmicro_pos == 0) {

         // We can not do variable substitution between %nopp/%end
         if (script_lines[i].find(T_COMMENT) == 1)  { comment = true; continue;}
         if (script_lines[i].find(T_NOOP)    == 1)  { return;}
         if (script_lines[i].find(T_MANUAL)  == 1)  { return;}
         if (script_lines[i].find(T_END)     == 1)  { return; }
      }

      if (comment) {

         // expect  name =  value
         string::size_type equal_pos = script_lines[i].find("=");
         if ( equal_pos == string::npos) continue;
         string name = script_lines[i].substr(0,equal_pos);
         string value = script_lines[i].substr(equal_pos+1);
         boost::algorithm::trim(name);
         boost::algorithm::trim(value);

         //std::cout << "   extracted as '" << name << "' = '" << value << "'\n";
         used_variables_as_map.insert( std::make_pair(name, value) );
      }
   }
}

bool EcfFile::open_script_file(
         const std::string& file_or_cmd,
         EcfFile::Type type,
         std::vector<std::string>& lines,
         std::string& errormsg) const
{
#ifdef DEBUG_ECF_
   std::cout << "EcfFile::open_script_file file(" << file_or_cmd << ") type(" << fileType(type) << ")\n";
#endif
   if (file_or_cmd.empty()) {
      std::stringstream ss;
      ss << "EcfFile::open_script_file: Could not open ecf " << fileType(type) << " file. Input File/cmd string is empty.";
      errormsg += ss.str();
      return false;
   }

   switch (script_origin_) {
      case ECF_FILES:
      case ECF_HOME:
      case ECF_SCRIPT: {
         if (type == EcfFile::INCLUDE) {
            return open_include_file(file_or_cmd,lines,errormsg);
         }
         if ( ! File::splitFileIntoLines(file_or_cmd, lines) ) {
            std::stringstream ss; ss << "Could not open " << fileType(type) << " file:" << file_or_cmd << " (" << strerror(errno) << ")";
            errormsg += ss.str();
            return false;
         }
         break;
      }

      case ECF_FETCH_CMD: {
         // Not tested.
         string theFile ;
         string theCommand = file_or_cmd; // variables have already been substituted
         switch (type) {
            case EcfFile::SCRIPT:  { theCommand += " -s "; theFile = node_->name() + get_extn(); break;}
            case EcfFile::INCLUDE:   theCommand += " -i "; break;
            case EcfFile::MANUAL:  { theCommand += " -m "; theFile = node_->name() + get_extn(); break;}
            case EcfFile::COMMENT: { theCommand += " -c "; theFile = node_->name() + get_extn(); break;}
         }
         theCommand += theFile;
         if (!do_popen(theCommand,type,lines,errormsg)) return false;
         break;
      }

      case ECF_SCRIPT_CMD: {
         switch (type) {
            case EcfFile::SCRIPT:  {
               if (!do_popen(file_or_cmd,type,lines,errormsg)) return false;
               break;
            }
            case EcfFile::INCLUDE: return open_include_file(file_or_cmd,lines,errormsg); break;
            case EcfFile::MANUAL:
            case EcfFile::COMMENT:
               if ( ! File::splitFileIntoLines(file_or_cmd, lines) ) {
                   std::stringstream ss; ss  << "Could not open " <<  fileType(type) << " file:" << file_or_cmd << " (" << strerror(errno) << ")";
                   errormsg += ss.str();
                   return false;
               }
            break;
         }
         break;
      }
   }
   return true;
}

#define USE_INCLUDE_CACHE  1
//  ulimit -Hn            # hard limit, of number of open files allowed
//  ulimit -Sn            # soft limit
//  ulimit -n <new-limit> # takes affect on current shell only
//
//  Check limit of running process:
//     ps aux | grep process-name
//     cat /proc/XXX/limits
//  max open files allowed is:
//     VM:       cat /proc/sys/fs/file-max:  188086
//     Desk top: cat /proc/sys/fs/file-max: 3270058
//  Desktop:
//     Without cache: real:10.15  user: 5.58  sys: 1.62                                     # opensuse131
//     With cache:    real: 4.46  user: 3.72  sys: 0.74  Only open/close include file once. # opensuse131
//     With cache:    real: 3.82  user: 3.22  sys: 0.59  Only open/close include file once. # leap42
//
bool EcfFile::open_include_file(const std::string& file,std::vector<std::string>& lines,std::string& errormsg) const
{
#ifdef USE_INCLUDE_CACHE
   // SEARCH THE CACHE
   size_t include_file_cache_size = include_file_cache_.size();
   for(size_t i = 0; i < include_file_cache_size; i++) {
      if (include_file_cache_[i]->path() == file) {
         //cout << "found " << file << " in cache, cache size = " << include_file_cache_.size() << "\n";
         if (!include_file_cache_[i]->lines(lines)) {
            std::stringstream ss; ss  << "Could not open include file: " << file << " (" << strerror(errno) << ") : include file cache size:" << include_file_cache_.size();
            errormsg += ss.str();
            return false;
         }
         return true;
      }
   }

   //cout << "NOT found " << file << " in cache, cache size = " << include_file_cache_.size() << "\n";

   if (include_file_cache_size > 1000) {
      // avoid hitting limit for open file descriptors ~1024, valgrind(takes 10 fd). clear cache
      include_file_cache_.clear();
   }

   // ADD to cache
   std::shared_ptr<IncludeFileCache> ptr = std::make_shared<IncludeFileCache>(file);
   include_file_cache_.push_back( ptr );

   if (!ptr->lines(lines)) {
      if ( errno == EMFILE/*Too many open files*/) {

         log(Log::WAR,"EcfFile::open_include_file: Too many files open(errno=EMFILE), Clearing cache, and trying again. Check limits with ulimit -Sn");
         include_file_cache_.clear();

         std::shared_ptr<IncludeFileCache> a_ptr = std::make_shared<IncludeFileCache>(file);
         include_file_cache_.push_back( a_ptr );
         if (!a_ptr->lines(lines)) {
            std::stringstream ss;
            ss << "Could not open include file: " << file << " (" << strerror(errno) << ") include file cache size:" << include_file_cache_.size() ;
            errormsg += ss.str();
            return false;
         }
      }
      else {
         std::stringstream ss;
         ss << "Could not open include file: " << file << " (" << strerror(errno) << ") include file cache size:" << include_file_cache_.size() ;
         errormsg += ss.str();
         return false;
      }
   }
#else
   if ( ! File::splitFileIntoLines(file, lines) ) {
       std::stringstream ss; ss  << "Could not open include file:" << file << " (" << strerror(errno) << ")";
       errormsg += ss.str();
       return false;
    }
#endif
   return true;
}

bool EcfFile::do_popen(const std::string& the_cmd, EcfFile::Type type, std::vector<std::string>& lines, std::string& errormsg) const
{
   FILE *fp = popen(the_cmd.c_str(),"r");
   if (!fp) {
      std::stringstream ss;
      ss  << "EcfFile::do_popen:: Could not open " << fileType(type) << " via cmd " << the_cmd << " for task " << node_->absNodePath() << " (" << strerror(errno) << ") ";
      errormsg += ss.str();
      return false;
   }
   char  line[LINE_MAX];
   while( fgets(line,LINE_MAX,fp) ) {
      lines.emplace_back(line);
      // remove any trailing new lines
      std::string& the_line = lines.back();
      if (!the_line.empty() && the_line[the_line.size()-1] == '\n') {
         the_line.erase(the_line.begin() + the_line.size()-1);
      }
   }
   int status = pclose(fp);
   if (status == -1) {
      std::stringstream ss;
      ss  << "EcfFile::do_popen: error on pclose for " << fileType(type) << " via cmd " << the_cmd << " for task " << node_->absNodePath() << " (" << strerror(errno) << ") ";
      errormsg += ss.str();
      return false;
   }

   if (WIFEXITED(status)) {
      // *Normal* termination via exit
      if ( WEXITSTATUS( status )) {
         std::stringstream ss;
         ss  << "EcfFile::do_popen: non-zero exit : " << fileType(type) << " via cmd " << the_cmd << " for task " << node_->absNodePath() << " (" << strerror(errno) << ") ";
         errormsg += ss.str();
         return false;
      }
      else {
         // exit(0) child terminated normally
         return true;
      }
   }
   if ( WIFSIGNALED( status )) {
      std::stringstream ss;
      ss  << "EcfFile::do_popen: child process terminated by a signal  : " << fileType(type) << " via cmd " << the_cmd << " for task " << node_->absNodePath() << " (" << strerror(errno) << ") ";
      errormsg += ss.str();
      return false;
   }

   return true;
}

std::string EcfFile::fileType(EcfFile::Type t)
{
   switch (t) {
      case EcfFile::SCRIPT:  return "script";  break;
      case EcfFile::INCLUDE: return "include"; break;
      case EcfFile::MANUAL:  return "manual";  break;
      case EcfFile::COMMENT: return "comment"; break;
   }
   assert(false);
   return string();
}

static void replace(  string::size_type commentPos,
         std::string& jobLine,
         const std::string& smsChildCmd,
         const std::string& ecfEquiv,
         const std::string& clientPath)
{
   string::size_type childPos = jobLine.find(smsChildCmd);
   if ( childPos != std::string::npos ) {
      if ( commentPos == std::string::npos) {
         std::string replace1 = clientPath;
         replace1 +=  ecfEquiv;
         Str::replace(jobLine,smsChildCmd,replace1);
      }
      else if ( childPos < commentPos) {
         std::string replace2 = clientPath;
         replace2 +=  ecfEquiv;
         Str::replace(jobLine,smsChildCmd,replace2);
      }
   }
}

bool EcfFile::replaceSmsChildCmdsWithEcf(const std::string& clientPath, std::string& errormsg)
{
   //   smsinit $$          	   ---> ECF_CLIENT(value) --init=$$
   //   smscomplete         	   ---> ECF_CLIENT(value)
   //   smsevent eventname       ---> ECF_CLIENT(value) --event=eventname
   //   smsmeter metername value ---> ECF_CLIENT(value) --meter=metername value
   //   smslabel value           ---> ECF_CLIENT(value) --label=value
   //   smswait expr             ---> ECF_CLIENT(value) --wait=expr
   //   smsabort                 ---> ECF_CLIENT(value) --abort
   size_t jobLines_size = jobLines_.size();
   for(size_t i=0; i < jobLines_size; ++i) {

      // ONLY do the replacement if there is no leading comment
      string::size_type commentPos = jobLines_[i].find("#");
      replace(commentPos, jobLines_[i], "smsinit",    " --init ",     clientPath);
      replace(commentPos, jobLines_[i], "smscomplete"," --complete ", clientPath);
      replace(commentPos, jobLines_[i], "smsabort",   " --abort ",    clientPath);
      replace(commentPos, jobLines_[i], "smsevent",   " --event ",    clientPath);
      replace(commentPos, jobLines_[i], "smsmeter",   " --meter ",    clientPath);
      replace(commentPos, jobLines_[i], "smslabel",   " --label ",    clientPath);
      replace(commentPos, jobLines_[i], "smswait",    " --wait ",     clientPath);
   }
   return true;
}

bool EcfFile::extract_ecfmicro(const std::string& line, std::string& ecfmicro, std::string& error_msg) const
{
   if (!Str::get_token(line,1,ecfmicro)) {
      std::stringstream ss; ss << "ecfmicro does not have a replacement character, in " << script_path_or_cmd_;
      error_msg += ss.str();
      return false;
   }
   // This is typically a single character, however $/£ will be multi-character i.e size 2
   if (ecfmicro.size() > 2) {
      std::stringstream ss; ss << "Expected ecfmicro replacement to be a single character, but found '" << ecfmicro << "' " <<  ecfmicro.size() << " in file : " << script_path_or_cmd_;
      error_msg += ss.str();
      return false;
   }
   return true;
}

void EcfFile::variableSubstitution(const JobsParam& jobsParam)
{
   // Allow variable substitution in comment and manual blocks.
   // But if it fails, don't report as an error

   // get the cached ECF_MICRO variable, typically its one char.
   string ecfMicro = ecfMicroCache_;
   char microChar = ecfMicro[0];

   // We need a stack to properly implement nopp. This is required since we need to pair
   // the %end, with nopp. i.e need to handle
   // %nopp
   // %comment
   // %end      // this is paired with comment
   // %end      // This is paired with nopp
   const int NOPP = 0;
   const int COMMENT = 1;
   const int MANUAL = 2;
   std::vector<int> pp_stack;

   bool nopp =  false;
   size_t jobLines_size = jobLines_.size();
   for(size_t i=0; i < jobLines_size; ++i) {

      // take into account micro char during variable substitution
      string::size_type ecfmicro_pos = jobLines_[i].find(ecfMicro);
      if (ecfmicro_pos == 0) {

         // We can not do variable substitution between %nopp/%end
         if (jobLines_[i].find(T_MANUAL)  == 1) { pp_stack.push_back(MANUAL); continue;  }
         if (jobLines_[i].find(T_COMMENT) == 1) { pp_stack.push_back(COMMENT); continue; }
         if (jobLines_[i].find(T_NOOP)    == 1) { pp_stack.push_back(NOPP); nopp = true; continue; }
         if (jobLines_[i].find(T_END) == 1) {
            if (pp_stack.empty()) throw std::runtime_error("EcfFile::variableSubstitution: failed unpaired %end");
            int last_directive = pp_stack.back(); pp_stack.pop_back();
            if (last_directive == NOPP) nopp = false;
            continue;
         }

         if (jobLines_[i].find(T_ECFMICRO) == 1) {   // %ecfmicro #
            // override ecfMicro char
            std::string error_msg;
            if (!extract_ecfmicro(jobLines_[i],ecfMicro,error_msg)) {
               throw std::runtime_error("EcfFile::variableSubstitution: failed : " + error_msg);
            }
            microChar = ecfMicro[0];
            continue;                                                      // no point in doing variable subs on %ecfmicro ^
         }
      }
      if ( nopp ) continue;


      /// For variable substitution % can occur anywhere on the line
      if (ecfmicro_pos != string::npos) {

         /// In the *NORMAL* flow jobsParam.user_edit_variables() will be EMPTY
         if ( !node_->variable_substitution( jobLines_[i], jobsParam.user_edit_variables(), microChar ) ) {

            // Allow variable substitution in comment and manual blocks.
            // But if it fails, don't report as an error
            int last_directive = -1;
            if (!pp_stack.empty()) last_directive = pp_stack.back();
            if ( last_directive == COMMENT || last_directive == MANUAL) continue;

            std::stringstream ss;  ss << "EcfFile::variableSubstitution: failed : '" << jobLines_[i] << "'";
            dump_expanded_script_file( jobLines_ );
            throw std::runtime_error(ss.str());
         }
      }
   }
}


void EcfFile::get_used_variables(std::string& used_variables) const
{
   /// Find Used variables, *after* all %includes expanded
   NameValueMap used_variables_map;
   std::string errorMsg;
   if (!get_used_variables(used_variables_map, errorMsg) ) {
      throw std::runtime_error( "EcfFile::get_used_variables: Extract used variables failed : " + errorMsg ) ;
   }

   if (!used_variables_map.empty()) {

      // add %comment - edit user variable, %end - ecf user variable
      used_variables = ecfMicroCache_;
      used_variables += "comment - ecf user variables\n";

      // ***************************************************************************************
      // Custom handling of dynamic variables, i.e ECF_TRYNO, ECF_PASS and
      // any variable that embeds a try number, i.e. ECF_JOB, ECF_JOBOUT
      // This is required since the try number is *always* incremented *before* job submission,
      // hence the value extracted from the job file will *not* be accurate, hence we exclude it.
      // This way at job submission we use the latest/correct value, which is in-sync with JOB OUTPUT
      // Note: Otherwise the job output will not be in sync
      //
      // Custom handling of ECF_PORT,ECF_HOST,ECF_NAME do not show these variables, these variables
      // including ECF_PASS appear in the script. If the user accidentally edits them,
      // Child communication with the server will be broken. Hence not shown
      //
      // All the above are examples of generated variables, which should not really be edited
      // The used variables are typically *user* variable *in* the scripts, that user may need
      // to modify. Hence we have also excluded generated variables SUITE, FAMILY, TASK
      // ****************************************************************************************
      for(std::pair<std::string, std::string> item : used_variables_map) {
         if ( item.first.find(Str::ECF_TRYNO())   != std::string::npos) continue;
         if ( item.first.find(Str::ECF_JOB())     != std::string::npos) continue;
         if ( item.first.find(Str::ECF_JOBOUT())  != std::string::npos) continue;
         if ( item.first.find(Str::ECF_PASS())    != std::string::npos) continue;
         if ( item.first.find(Str::ECF_PORT())    != std::string::npos) continue;
         if ( item.first.find(Str::ECF_HOST())    != std::string::npos) continue;
         if ( item.first.find(Str::ECF_NAME())    != std::string::npos) continue;

         // We must use exact match, to avoid user variables like ESUITE,EFAMILY,ETASK
         if ( item.first == Str::TASK())  continue;
         if ( item.first == Str::FAMILY()) continue;
         if ( item.first == "FAMILY1")     continue;
         if ( item.first == Str::SUITE())  continue;
         used_variables += item.first;
         used_variables += " = ";
         used_variables += item.second;
         used_variables += "\n";
      }

      used_variables += ecfMicroCache_;
      used_variables += "end - ecf user variables\n";
   }
}

bool EcfFile::get_used_variables(NameValueMap& used_variables, std::string& errormsg) const
{
   // get the cached ECF_MICRO variable, typically its one char.
   string ecfMicro = ecfMicroCache_;

   char microChar = ecfMicro[0];

   // We need a stack to properly implement nopp. This is required since we need to pair
   // the %end, with nopp. i.e need to handle
   // %nopp
   // %comment
   // %end      // this is paired with comment
   // %end      // This is paired with nopp
   const int NOPP = 0;
   const int COMMENT = 1;
   const int MANUAL = 2;
   std::vector<int> pp_stack;

   bool nopp =  false;
   std::stringstream ss;

   size_t job_lines_size = jobLines_.size();
   for(size_t i=0; i < job_lines_size; ++i) {

      if (jobLines_[i].empty()) continue;

      // take into account micro char during variable substitution
      string::size_type ecfmicro_pos = jobLines_[i].find(ecfMicro);
      if (ecfmicro_pos == 0) {

         // We can not do variable substitution between %nopp/%end
         if (jobLines_[i].find(T_MANUAL)  == 1) { pp_stack.push_back(MANUAL); continue;  }
         if (jobLines_[i].find(T_COMMENT) == 1) { pp_stack.push_back(COMMENT); continue; }
         if (jobLines_[i].find(T_NOOP)    == 1) { pp_stack.push_back(NOPP); nopp = true; continue; }
         if (jobLines_[i].find(T_END) == 1) {
            if (pp_stack.empty()) throw std::runtime_error("EcfFile::get_used_variables: failed  unpaired %end");
            int last_directive = pp_stack.back(); pp_stack.pop_back();
            if (last_directive == NOPP) nopp = false;
            continue;
         }

         if (!nopp && jobLines_[i].find(T_ECFMICRO) == 1) {  // %ecfmicro #
            // override ecfMicro char
            std::string error_msg;
            if (!extract_ecfmicro(jobLines_[i],ecfMicro,error_msg)) {
               throw std::runtime_error("EcfFile::get_used_variables: failed : " + error_msg);
            }
            microChar = ecfMicro[0];
            continue;
         }
      }
      if ( nopp ) continue;


      if (ecfmicro_pos != string::npos) {

         /// *Note:* currently this modifies jobLines_[i]
         std::string line_copy = jobLines_[i];  // avoid modifying the jobs Lines, end up doing  variable substitution
         if ( !node_->find_all_used_variables( line_copy, used_variables, microChar ) ) {

            // Allow variable substitution in comment and manual blocks.
            // But if it fails, dont report as an error
            int last_directive = -1;
            if (!pp_stack.empty()) last_directive = pp_stack.back();
            if ( last_directive == COMMENT || last_directive == MANUAL) continue;

            ss << "Variable find failed for '" << jobLines_[i] << "'  microChar='" << microChar << "' ";
            dump_expanded_script_file( jobLines_ );
         }
      }
   }

   // Append to error message if any
   errormsg += ss.str();

   return errormsg.empty();
}


const std::string& EcfFile::doCreateJobFile(JobsParam& jobsParam) const
{
   if ( !jobLines_.empty() ) {
      // Guard against ecf file that exist's but is empty,
      // no point in creating empty job files for them

      // ECF_JOB is used with ECF_JOB_CMD when submitting a job.
      // First look for a user variable of name ECF_JOB, otherwise look for
      // the generated variable, hence this should never fail.
      // *This* assumes that:
      //   a/ if the user has overridden ECF_JOB then it has been specified at the task level
      //      Otherwise findParentVariableValue will find the generated ECF_JOB on the task
      //   b/ The value of the user variable has a valid directory paths and job file name
      //   c/ The user will lose the try number.
      std::string ecf_job;
      if (!node_->findParentVariableValue(Str::ECF_JOB(), ecf_job)) {
         LOG_ASSERT( !ecf_job.empty() ,"EcfFile::doCreateJobFile: ECF_JOB should have been generated, program error");
      }

      // *** The location of the ECF_ file may not always be the same as the location
      // *** of the job file. Job file location is specified by ECF_JOB
      //cout << "EcfFile::createJob ecf " << script_path_or_cmd_ << " ECF_JOB(" << ecf_job << ")\n";

      if (!File::createMissingDirectories(ecf_job)) {
         std::stringstream ss;
         ss << "EcfFile::doCreateJobFile: Could not create missing directories for ECF_JOB " << ecf_job << " (" << strerror(errno) << ")";
         throw std::runtime_error(ss.str());
      }

      // Create the jobs file.
      std::string error_msg;
      if (!File::create(ecf_job, jobLines_,error_msg)) {
         std::stringstream ss;
         if ( errno == EMFILE/*Too many open files*/) {
            // CLEAR cache and try again. Can test with ulimit -n 60, (Base/bin/gcc-5.3.0/release/perf_job_gen ./metabuilder.def)
            LogToCout log_to_cout;
            ss << "EcfFile::doCreateJobFile: Too many files open(errno=EMFILE), include file cache size(" << include_file_cache_.size() << ") Clearing cache. Check limits with ulimit -Sn";
            log(Log::WAR,ss.str());

            include_file_cache_.clear();

            error_msg.clear();
            if (!File::create(ecf_job, jobLines_,error_msg)) {
               ss << "EcfFile::doCreateJobFile: Could not create job file, even after clearing include cache: " << error_msg; // error_msg includes strerror(errno)
               throw std::runtime_error(ss.str());
            }
         }
         else {
            ss << "EcfFile::doCreateJobFile: Could not create job file : " << error_msg; // error_msg includes strerror(errno)
            throw std::runtime_error(ss.str());
         }
      }

      // make the job file executable
      if ( chmod( ecf_job.c_str(), 0755 ) != 0 ) {
         std::stringstream ss;
         ss << "EcfFile::doCreateJobFile: Could not make job file " << ecf_job << "  executable by using chmod (" << strerror(errno) << ")";
         throw std::runtime_error(ss.str());
      }

      // record job size, for placement into log files
      size_t job_output_size = 0;
      size_t jobLines_size = jobLines_.size();
      //cout << " jobLines_.size() " << jobLines_size << " jobLines_.capacity() " << jobLines_.capacity() << "\n";
      for(size_t i = 0; i < jobLines_size; ++i)  job_output_size += jobLines_[i].size();
      job_output_size += jobLines_size; // take into account new lines for each line of output
      job_size_ = "job_size:";
      job_size_ += boost::lexical_cast<std::string>(job_output_size);
      return job_size_;
   }

   std::stringstream ss;
   ss << "EcfFile::doCreateJobFile: The ecf file '" << script_path_or_cmd_ << "' that is associated with task '" << node_->absNodePath() << "' is empty";
   throw std::runtime_error(ss.str());
}

boost::filesystem::path EcfFile::file_creation_path() const
{
   return fs::path(script_or_job_path());
}

std::string EcfFile::script_or_job_path() const
{
   if (script_origin_ == ECF_SCRIPT) return script_path_or_cmd_;

   // ECF_FETCH or ECF_SCRIPT_CMD
   std::string ecf_job;
   (void)node_->findParentVariableValue(Str::ECF_JOB(), ecf_job);
   return ecf_job;
}

bool EcfFile::doCreateManFile( std::string& errormsg)
{
   vector<string> manFile;
   if (!extractManual(jobLines_,manFile,errormsg)) {
      return false;
   }
   if ( !manFile.empty() ) {

      // find the directory associated with job file and place Man file there.
      fs::path script_file_path = file_creation_path();
      fs::path parent_path = script_file_path.parent_path();
      if ( fs::is_directory( parent_path ) ) {

         fs::path theManFilePath( parent_path.string() + '/' + node_->name() + File::MAN_EXTN() );

         // cout << "EcfFile::doCreateManFile job " << manFile.string() << "\n";
         if (!File::create(theManFilePath.string(),manFile,errormsg)) return false;
      }
      else {
         std::stringstream ss;
         ss << "man file creation failed. The path '" << script_file_path.parent_path() << "' is not a directory";
         errormsg += ss.str();
         return false;
      }
   }
   return true;
}


void EcfFile::doCreateUsrFile() const
{
   // find the directory associated with ecf file and place .usr file there.
   fs::path script_file_path = file_creation_path();
   fs::path parent_path = script_file_path.parent_path();
   if ( fs::is_directory( parent_path ) ) {

      fs::path theUsrFilePath( parent_path.string() + '/' + node_->name() + File::USR_EXTN() );

      // cout << "EcfFile::doCreateUsrFile job " << theUsrFilePath.string() << "\n";
      std::string error_msg;
      if (!File::create(theUsrFilePath.string(),jobLines_,error_msg)) {
         throw std::runtime_error("EcfFile::doCreateUsrFile: file creation failed : " + error_msg);
      }
   }
   else {
      std::stringstream ss;
      ss << "EcfFile::doCreateUsrFile: file creation failed. The path '" << script_file_path.parent_path() << "' is not a directory";
      throw std::runtime_error(ss.str());
   }
}


bool EcfFile::extractManual(const std::vector< std::string >& lines,
         std::vector< std::string >& theManualLines,
         std::string& errormsg) const
{
   // Note: we have already done pre-processing, ie since the manual is obtained after
   // all the includes have been pre-procssed, hence most errors should have been caught
   // get the cached ECF_MICRO variable, typically its one char.
   string ecfMicro = ecfMicroCache_;

   bool add = false;
   for (const auto & line : lines){
      if ( line.find(ecfMicro) == 0) {
         if ( line.find( T_MANUAL ) == 1 )     { add = true;  continue; }
         if ( add && line.find( T_END ) == 1 ) { add = false; continue; }

         if (line.find(T_ECFMICRO) == 1) {  // %ecfmicro #

            if (!extract_ecfmicro(line,ecfMicro,errormsg )) {
               return false;
            }

            continue;
         }
      }
      if (add) { theManualLines.push_back(line); }
   }
   if (add) {
      std::stringstream ss; ss << "Unterminated manual. Matching 'end' is missing, for " << script_path_or_cmd_;
      errormsg += ss.str();
      dump_expanded_script_file(lines);
      return false;
   }
   return true;
}

void EcfFile::remove_comment_manual_and_noop_tokens()
{
	// preserve *all* lines between %noop and %end
    // remove all line between %comment and %end | %manual and %end, *provided* they are not embedded in %noop/%end
	// remove tokens %noop,%end,%ecfmicro and only remove %comment,%manual if they not embedded in %noop/%end

	// get the cached ECF_MICRO variable, typically its one char.
	string ecfMicro = ecfMicroCache_;

	// We need a stack to properly implement nopp. This is required since we need to pair
	// the %end, with nopp. i.e need to handle
	// %nopp       // delete this line
	// %comment    // preserve
	// -- comment  // preserve
	// %end        // preserve , this is paired with comment
	// blah %blah% // preserve, i.e. don't preprocess
	// %end        // delete this line, This is paired with nopp
	//
	// For the following
	// %comment    // delete this line
	// -- comment  // delete this line
	// %end        // delete this line

	const int NOPP = 0;
	const int COMMENT = 1;
	const int MANUAL = 2;
	std::vector<int> pp_stack;
	bool nopp = false;
	bool manual_erase = false;
	bool comment_erase = false;

	for(auto i=jobLines_.begin(); i!=jobLines_.end(); ++i) {

		string::size_type ecfmicro_pos = (*i).find(ecfMicro);
		if ( ecfmicro_pos == 0) {
			if ((*i).find(T_MANUAL) == 1) {
				if (manual_erase) {
					std::stringstream ss; ss << "EcfFile::remove_comment_manual_and_noop_tokens: Embedded manuals are not allowed in " << script_path_or_cmd_;
					throw std::runtime_error( ss.str() );
				}

				pp_stack.push_back(MANUAL);
	            if (nopp) continue;      // preserve
				jobLines_.erase( i-- );  // remove  %manual
				manual_erase = true;
				continue;
			}
			if ((*i).find(T_COMMENT) == 1) {
				if (comment_erase) {
					std::stringstream ss; ss << "EcfFile::remove_comment_manual_and_noop_tokens: Embedded comments are not allowed in " << script_path_or_cmd_;
					throw std::runtime_error( ss.str() );
				}

				pp_stack.push_back(COMMENT);
	            if (nopp) continue;     // preserve
				jobLines_.erase( i-- ); // remove %comment
				comment_erase = true;
				continue;
			}
			if ((*i).find(T_NOOP) == 1) {
				if (nopp) {
					std::stringstream ss; ss << "Embedded nopp are not allowed " << script_path_or_cmd_;
					throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed " + ss.str());
				}

				pp_stack.push_back(NOPP); nopp = true;
				jobLines_.erase( i-- );      // remove %nopp
				continue;
			}
			if ((*i).find(T_END) == 1) {
				if (pp_stack.empty()) throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed unpaired %end");
				int last_directive = pp_stack.back(); pp_stack.pop_back();
				if (last_directive == NOPP) {
					nopp = false;
					jobLines_.erase( i-- );        // remove %end associated with %nopp
					continue;
				}
				else if (last_directive == MANUAL) {
					manual_erase = false;
		            if (nopp) continue;     // preserve
					jobLines_.erase( i-- ); // remove %end associated with %manual
					continue;
				}
				else if (last_directive == COMMENT) {
					comment_erase = false;
		            if (nopp) continue;      // preserve
					jobLines_.erase( i-- );  // remove %end associated with %comment
					continue;
				}
				throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed unpaired %end does not match noop,comment or manual");
			}
			if (!nopp && (*i).find(T_ECFMICRO) == 1) {  // %ecfmicro #

				std::string error_msg;
				if (!extract_ecfmicro((*i),ecfMicro,error_msg)) {
					throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed : " + error_msg);
				}
				jobLines_.erase( i-- );                // remove %ecfmicro &
				continue;
			}
		}

		// *** For noop we only remove the tokens(%noop,%end) *NOT* the lines between. noop means no pre-processing, keep the lines as is.
		if (nopp) continue;

		if (manual_erase || comment_erase) {
		    // remove all line between %comment and %end | %manual and %end
			jobLines_.erase( i-- );
		}
	}

	if (nopp) {
		std::stringstream ss;
		ss << "Unterminated nopp. Matching 'end' is missing, in " << script_path_or_cmd_;
		throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed " + ss.str());
	}
	if (manual_erase) {
		std::stringstream ss;
		ss << "Unterminated manual. Matching 'end' is missing, in " << script_path_or_cmd_;
		throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed " + ss.str());
	}
	if (comment_erase) {
		std::stringstream ss;
		ss << "Unterminated comment. Matching 'end' is missing, in " << script_path_or_cmd_;
		throw std::runtime_error("EcfFile::remove_comment_manual_and_noop_tokens: failed " + ss.str());
	}
}


int EcfFile::countEcfMicro(const std::string& line, const std::string& ecfMicro)
{
   // ignore ecfmicro character in comments
   // However pay special attenstion to:
   // var="%VAR: # fred%  # %fred
   if (ecfMicro.empty()) {
      return 0;
   }

   int count = 0;
   const char theChar = ecfMicro[0];
   size_t end = line.size();
   size_t comment_pos = 0;
   for(size_t i = 0; i < end; ++i) {
      if (line[i] == theChar) {
         count++;
      }
      if (line[i] == '#') {
         if (i == 0) return 0; // very first character is comment
         comment_pos = i;      // remember *last* comment pos
      }
   }

   if (count % 2 != 0 && comment_pos != 0) {
      // could have:
      // var="%VAR: # fred%  # %fred
      // discount trailing #
      count = 0;
      for(size_t i = 0; i < comment_pos; ++i) {
         if (line[i] == theChar) {
            count++;
         }
      }
   }
   //	cerr << "line " << line << " count = " << count << "\n";
   return count;
}

void EcfFile::dump_expanded_script_file(const std::vector<std::string>& lines)
{
#ifdef DEBUG_PRE_PROCESS
   std::string err;
   if (!File::create("tmp" + get_extn(),lines,err))  std::cout << "Could not create file tmp.ecf\n";
#endif
}

/// returns the extension, i.e for task->.ecf for alias->.usr
const std::string& EcfFile::get_extn() const
{
   Submittable* task_or_alias = node_->isSubmittable();
   if (task_or_alias) return task_or_alias->script_extension();
   else {
      std::stringstream ss; ss << "EcfFile::get_extn(): Can only return extension for task/alias but found " << node_->debugNodePath();
      throw std::runtime_error(ss.str());
   }
   return Str::EMPTY();
}

// =======================================================================================

PreProcessor::PreProcessor(EcfFile* ecfile,const char* error_context)
: ecfile_(ecfile),
  error_context_(error_context),
  ecf_micro_(ecfile->ecfMicroCache_),
  jobLines_(ecfile->jobLines_)
{
   pp_nopp_ = ecf_micro_;    pp_nopp_    += T_NOOP;
   pp_comment_ = ecf_micro_; pp_comment_ += T_COMMENT;
   pp_manual_ = ecf_micro_;  pp_manual_  += T_MANUAL;
   pp_end_ = ecf_micro_;     pp_end_     += T_END;

   /// Clear existing jobLines, pre-processing will populate jobLines_
   jobLines_.clear();
   jobLines_.reserve(512); // estimate for includes
}

PreProcessor::~PreProcessor()= default;

void PreProcessor::preProcess(std::vector<std::string>& script_lines )
{
	bool ignore_end_check = false;
	if (manual_ || comment_) ignore_end_check = true; // %include inside a %manual || %comment, ignore end check in preProcess

   // preProcess is called recursively, i.e for processing includes
   // Uses a Depth first traversal
   for(auto& line: script_lines) {
      jobLines_.emplace_back(std::move(line));    
      preProcess_line();
   }

   if (nopp_) throw std::runtime_error( error_context() + "Unterminated nopp, matching 'end' is missing");
   if (comment_ && !ignore_end_check) throw std::runtime_error( error_context() + "Unterminated comment, matching 'end' is missing");
   if (manual_ && !ignore_end_check)  throw std::runtime_error( error_context() + "Unterminated manual, matching 'end' is missing");
}

void PreProcessor::preProcess_line()
{
	const std::string& script_line = jobLines_.back();

   // For variable substitution % can occur anywhere on the line, for pre -processing of
   // %ecfmicro,%manual,%comment,%end,%include,%includenopp it must be the very *first* character
   string::size_type ecfmicro_pos = script_line.find(ecf_micro_);
   if (ecfmicro_pos == string::npos) return;

   // Check for Mismatched micro i.e %FRED or %FRED%%
   if (ecfmicro_pos != 0 && !nopp_ && !comment_ && !manual_) {
	   // For variable substitution '%' can occur anywhere on the line.
	   int ecfMicroCount = EcfFile::countEcfMicro( script_line, ecf_micro_ );
	   if (ecfMicroCount % 2 != 0 ) {
		   ecfile_->dump_expanded_script_file(jobLines_);
		   std::stringstream ss;
		   ss << "Mismatched ecfmicro(" << ecf_micro_ << ") count(" << ecfMicroCount << ")  at : " << script_line ;
		   throw std::runtime_error( error_context() + ss.str());
	   }
   }

   // %ecfmicro,%manual,%comment,%end,%include,%includenopp,%includeonce it must be the very *first* character
   if (ecfmicro_pos != 0) return; //handle 'garbage%include'

#ifdef DEBUG_PRE_PROCESS
   std::cout << i << ": " << script_line << "\n";
#endif
   if (script_line.find(pp_manual_) == 0) {
      if (comment_ || manual_) {
         ecfile_->dump_expanded_script_file(jobLines_);
         std::stringstream ss; ss << "Embedded comments/manuals not supported : '" << script_line << "'";
		 throw std::runtime_error( error_context() + ss.str());
      }
      manual_ = true ; return;
   }
   if (script_line.find(pp_comment_) == 0) {
      if (comment_ || manual_) {
         ecfile_->dump_expanded_script_file(jobLines_);
         std::stringstream ss; ss << "Embedded comments/manuals not supported : '" << script_line << "'";
		 throw std::runtime_error( error_context() + ss.str());
      }
      comment_ = true ; return;
   }
   if (script_line.find(pp_nopp_) == 0) {
      if (nopp_) {
         ecfile_->dump_expanded_script_file(jobLines_);
         std::stringstream ss; ss << "Embedded nopp not supported : '" << script_line << "'";
		 throw std::runtime_error( error_context() + ss.str());
      }
      nopp_ = true ; return;
   }
   if (script_line.find(pp_end_) == 0) {
      if (comment_) { comment_ = false; return;}
      if (manual_)  { manual_ = false;  return;}
      if (nopp_)    { nopp_ = false;    return;}

      ecfile_->dump_expanded_script_file(jobLines_);
      std::stringstream ss;
      ss << pp_end_ << " found with no matching %comment | %manual | %nopp  : '" << script_line << "'" ;
	  throw std::runtime_error( error_context() + ss.str());
   }
   if (nopp_) return;


   // =================================================================================
   // Handle ecfmicro replacement
   // =================================================================================
   if (script_line.find(T_ECFMICRO) == 1) {    // %ecfmicro #

      // keep %ecfmicro in jobs file later processing, i.e for comments/manuals
	  std::string err;
      if (!ecfile_->extract_ecfmicro(script_line,ecf_micro_,err)) {
    	 throw std::runtime_error( error_context() + err);
      }

      pp_nopp_ = ecf_micro_;    pp_nopp_    += T_NOOP;
      pp_comment_ = ecf_micro_; pp_comment_ += T_COMMENT;
      pp_manual_ = ecf_micro_;  pp_manual_  += T_MANUAL;
      pp_end_ = ecf_micro_;     pp_end_     += T_END;

      return;
   }
   else {
      if (script_line.find("ecf_micro") == 1) {    //  Mistyped ecfmicro
    	 throw std::runtime_error( error_context() + "Replace with 'ecf_micro' with 'ecfmicro' at line: '" + script_line + "'");
      }
   }

   // If there's no second token, when what kind of directive is it ?
   // allow   : %FRED:val%
   // disallow: %FRED
   std::string the_include_token;
   if (!Str::get_token(script_line,1,the_include_token)) {
      int ecfMicroCount = EcfFile::countEcfMicro( script_line, ecf_micro_ );
      if (ecfMicroCount % 2 != 0 ) {
     	 throw std::runtime_error( error_context() + " unrecognised pre-processing directive at: '" + script_line + "'");
      }
      return;
   }

   // we only end up here if we have includes
   preProcess_includes();
}

void PreProcessor::preProcess_includes()
{
   std::string includedFile;
   bool fnd_includenopp= false;
   {
	   // Take note: when we call jobLines_.pop_back() below, script_line will be invalidated
	   const std::string& script_line = jobLines_.back();

	   // =================================================================================
	   // order is *IMPORTANT*, hence search for includenopp, includeonce, include
	   // Otherwise string::find() of include will match includenopp and includeonce
	   // =================================================================================
       bool fnd_includeonce = false;
	   bool fnd_include = false;
	   fnd_includenopp = (script_line.find(T_INCLUDENOPP) == 1);
	   if (!fnd_includenopp) {
		   fnd_includeonce = (script_line.find(T_INCLUDEONCE) == 1);
		   if (!fnd_includeonce) fnd_include = (script_line.find(T_INCLUDE) == 1);
	   }
	   if (!fnd_include && !fnd_includenopp && !fnd_includeonce) {
		   if (script_line.find("include") == 1) {
		       throw std::runtime_error( error_context() + ", unrecognised or miss-spelled include at: '" + script_line + "'" );
		   }
		   return;
	   }

	   std::string the_include_token;
	   if (!Str::get_token(script_line,1,the_include_token)) {
	       throw std::runtime_error( error_context() + "Could not extract include token at : " + script_line );
	   }

#ifdef DEBUG_PRE_PROCESS_INCLUDES
	   // Output the includes for debug purposes. Will appear in preProcess.ecf
	   // Note: Will interfere with diff
	   jobLines_.push_back("========== include of " + the_include_token  + " ===========================");
#endif

	   includedFile = getIncludedFilePath(the_include_token, script_line);


	   // remove %include from the job lines, since were going to expand or ignore it.
	   // **** input script_line life_time is tied, jobLines_.back(), hence jobLines_.pop_back() will invalidate script_lines
	   // **** Hence don't use script_line after this point.
	   jobLines_.pop_back();


	   // handle %includeonce
	   if (fnd_includeonce) {
		   if (std::find(include_once_set_.begin(),include_once_set_.end(),includedFile) != include_once_set_.end() ) {
			   return; // Already processed once ignore
		   }
		   include_once_set_.push_back(includedFile);
	   }
   }

#ifdef DEBUG_PRE_PROCESS
   cout << "EcfFile::preProcess processing " << includedFile  << "\n";
#endif

   // Check for recursive includes. some includes like %include <endt.h>
   // are included many times, but the include is not recursive.
   // To get round this will use a simple count.
   // replace map with vector
   bool fnd = false;
   for(auto& p:  globalIncludedFileSet_) {
      if (p.first == includedFile) {
         fnd = true;
         if ( p.second > 100) {
            std::stringstream ss;
            ss << " Recursive include of file " << includedFile << " for " << ecfile_->script_path_or_cmd_;
 	        throw std::runtime_error( error_context() + ss.str() );
         }
         p.second++;
         break;
      }
   }
   if (!fnd) {
      globalIncludedFileSet_.emplace_back(includedFile,0 );
   }


   std::vector<std::string> include_lines;
   if (fnd_includenopp) include_lines.push_back(ecf_micro_ + T_NOOP);
   std::string err;
   if (!ecfile_->open_script_file(includedFile, EcfFile::INCLUDE, include_lines, err)) {
       throw std::runtime_error( error_context() + err );
   }
   if (fnd_includenopp) include_lines.push_back(ecf_micro_ + T_END);


   preProcess(include_lines);
}

std::string PreProcessor::getIncludedFilePath(const std::string& includedFile1,const std::string& line)
{
   // Include can have following format: [ %include | %includeonce | %includenopp ]
   //   %include /tmp/file.name   -> /tmp/filename
   //   %include file.name        -> file.name
   //   %include "../file.name"   -> script_file_location/../file.name
   //   %include "./file.name"    -> script_file_location/./file.name
   //   %include "file.name"      -> %ECF_HOME%/%SUITE%/%FAMILY%/filename
   //   %include <file.name>      -> %ECF_INCLUDE%/filename || %ECF_HOME%/filename # note: ECF_HOME is always a single path
   //
   //   %include <%file%.name>        -> %ECF_INCLUDE%/filename || %ECF_HOME%/filename
   //   %include "%file%.name"        -> %ECF_HOME%/%SUITE%/%FAMILY%/filename
   //   %include "./%file%.name"      -> script_file_location/./file.name
   //   %include "/%tmp%/%file%.name" -> /%tmp%/%file%.name
   //   %include %INCLUDE%            -> any of the above
   //
   // When ECF_INCLUDE           -> path1:path2:path3
   //   %include <filename>      -> path1/filename || path2/filename || path3/filename || ECF_HOME/filename

   // the included file could have variables,(ECFLOW-765), check for miss-matched ecf_micro
   std::string includedFile = includedFile1;
   if ( includedFile.find(ecf_micro_) != std::string::npos) {
      int ecfMicroCount = EcfFile::countEcfMicro( includedFile, ecf_micro_ );
      if (ecfMicroCount % 2 != 0 ) {
         std::stringstream ss;
         ss << "Mismatched ecfmicro(" << ecf_micro_ << ") count(" << ecfMicroCount << ") at : " << line ;
         throw std::runtime_error( error_context() + ss.str() );
      }
      NameValueMap user_edit_variables;
      ecfile_->node_->variable_substitution(includedFile,user_edit_variables,ecf_micro_[0]);
   }

   std::string the_include_file = includedFile.substr( 1, includedFile.size() - 2 );
   if ( includedFile.size() >=2 && includedFile[1] == '/') {
      // filename starts with '/' no interpretation return as in
      // %include </home/ecf/fred.ecf>
      // %include "/home/ecf/fred.ecf"
      return the_include_file;
   }

   Node* node = ecfile_->node_;
   if ( includedFile[0] == '<' ) {
      // %include <filename> can be one of:
      //    o When ECF_INCLUDE is a single path -> path1/filename
      //    o When ECF_INCLUDE is a multi  path -> path1:path2:path3  -> ECFLOW-261
      //                                        -> path1/filename || path2/filename || path3/filename
      //    o ECF_HOME/filename
      std::string ecf_include;
      if (node->findParentUserVariableValue( Str::ECF_INCLUDE() , ecf_include ) && !ecf_include.empty() ) {

         // if ECF_INCLUDE is a set a paths, search in order. i.e like $PATH
         if (ecf_include.find(':') != std::string::npos) {
            std::vector<std::string> include_paths;
            Str::split(ecf_include,include_paths,":");
            for(const auto & include_path : include_paths) {
               ecf_include.clear();
               ecf_include = include_path;
               ecf_include += '/';
               ecf_include += the_include_file;

               // Don't rely on hard coded paths. Added for testing, but could be generally useful
               // since in test scenario ECF_INCLUDE is defined relative to $ECF_HOME
               node->variable_dollar_subsitution(ecf_include);

               if (ecfile_->file_exists(ecf_include)) return ecf_include;
            }
         }
         else {
            ecf_include += '/';
            ecf_include += the_include_file;
            node->variable_dollar_subsitution(ecf_include);
            if (ecfile_->file_exists(ecf_include)) return ecf_include;
         }

         // ECF_INCLUDE is specified *BUT* the file does *NOT* exist, Look in ECF_HOME
      }

      // WE get HERE *if* ECF_INCLUDE not specified, or if specified but file *not found*
      ecf_include.clear();
      node->findParentVariableValue( Str::ECF_HOME() , ecf_include );
      if (ecf_include.empty()) {
    	 std::stringstream ss;
         ss << "ECF_INCLUDE/ECF_HOME not specified, at : " << line;
         throw std::runtime_error( error_context() + ss.str() );
      }

      ecf_include += '/';
      ecf_include += the_include_file;

      return ecf_include;
   }
   else if ( includedFile[0] == '"' ) {

      // we have two forms: "head.h" & "../head.h"

      //  ECFLOW-274 need to allow:
      //     - %include "../fred.h"     # this is one directory up
      //     - %include "./fred.h"      # this is at the same level as script
      std::string path;
      if ( includedFile.find("./") == 1 || includedFile.find("../") == 1) {
         // remove the leading and trailing '"'
         std::string the_included_file = includedFile;
         Str::removeQuotes(the_included_file);

         // Get the root path, i.e. script_or_job_path() is of the form:
         //    "/user/home/ma/mao/course/t1.ecf || /user/home/ma/mao/course/t1.job"
         // we need "/user/home/ma/mao/course/"
         std::string the_script_or_job_path = ecfile_->script_or_job_path();
         std::string::size_type last_slash = the_script_or_job_path.rfind("/");
         if (last_slash != std::string::npos) {
            path = the_script_or_job_path.substr( 0, last_slash + 1 );
            path += the_included_file;
            // std::cout << "path == " << path << "\n";
            return path;
         }
      }

      // include contents of %ECF_HOME%/%SUITE%/%FAMILY%/filename
      node->findParentUserVariableValue( Str::ECF_HOME() , path);
      if ( path.empty() ) {
    	 std::stringstream ss;
         ss << "ECF_HOME not specified, at : " << line;
         throw std::runtime_error( error_context() + ss.str() );
      }
      path += '/';
      std::string suite;
      node->findParentVariableValue( "SUITE" , suite);   // SUITE is a generated variable
      if ( suite.empty() ) {
     	 std::stringstream ss;
         ss << "SUITE not specified, at : " << line;
         throw std::runtime_error( error_context() + ss.str() );
      }
      path += suite;
      path += '/';
      std::string family;
      node->findParentVariableValue( "FAMILY" , family); // FAMILY is a generated variable
      if ( family.empty() ) {
      	 std::stringstream ss;
         ss << "FAMILY not specified, at : " << line;
         throw std::runtime_error( error_context() + ss.str() );
      }
      path += family;
      path += '/';
      path += the_include_file ; // "filename"
      return path;
   }

   // File either has an absolute pathname or is in the current working dir.
   // include file name as is, from current working directory
   return includedFile;
}

std::string PreProcessor::error_context() const
{
	std::string ret(error_context_);
	ret += ": Failed preprocessing : ";
	ret += ecfile_->node_->debugNodePath();
	ret += " : path/cmd(" ;
	ret += ecfile_->script_path_or_cmd_;
	ret += ")\n: ";
    return ret;
}


bool EcfFile::file_exists(const std::string& ecf_include) const
{
//   return fs::exists(ecf_include);
   size_t file_stat_cache_size = file_stat_cache_.size();
   for(size_t i = 0 ; i < file_stat_cache_size; i++) {
      if ( file_stat_cache_[i].first == ecf_include) {
         //cout << "EcfFile::file_exists " << ecf_include << " found in cache " << file_stat_cache_[i].second << "\n";
         return file_stat_cache_[i].second;
      }
   }
   if (fs::exists(ecf_include)) {
      //cout << "EcfFile::file_exists " << ecf_include << " exists add to cache\n";
      file_stat_cache_.emplace_back(ecf_include,true);
      return true;
   }
   //cout << "EcfFile::file_exists " << ecf_include << " does NOT exist add to cache\n";
   file_stat_cache_.emplace_back(ecf_include,false);
   return false;
}

// **********************************************************************************

IncludeFileCache::IncludeFileCache(const std::string& path) : path_(path), fp_(path.c_str(), std::ios_base::in),no_of_lines_(0) {}

IncludeFileCache::~IncludeFileCache() { fp_.close(); }

bool IncludeFileCache::lines(std::vector<std::string>& lns) {
   if ( !fp_ ) return false;

   if (no_of_lines_ != 0 ) {
      lns.reserve(no_of_lines_);
      fp_.seekg(0);               // Using cache, reset pos back to 0
   }

   // Note if we use: while( getline( theEcfFile, line)), then we will miss the *last* *empty* line
   string line;
   while ( std::getline(fp_,line) ) { lns.push_back(line); }  // c++11
   fp_.clear();               // eol fp_ will be in bad state reset. So we can re-use
   no_of_lines_ = lns.size(); // cache for next time

   return true;
}
