/* $Id: visir_spc_obs_ech.c,v 1.98 2012-09-27 15:45:07 jtaylor Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003,2012 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jtaylor $
 * $Date: 2012-09-27 15:45:07 $
 * $Revision: 1.98 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "visir_recipe.h"
#include "visir_spectro.h"
#include "visir_spc_distortion.h"
#include "visir_spc_photom.h"


/*-----------------------------------------------------------------------------
                            Defines
 -----------------------------------------------------------------------------*/

#define RECIPE_STRING   "visir_old_spc_obs_ech"

/* FITS keys to be loaded for all raw files */
#define RECIPE_KEYS_REGEXP_ALL            \
        VISIR_PFITS_REGEXP_IMG_RECOMBINE  \
    "|" VISIR_PFITS_REGEXP_SPC_GET_RES_WL

/* FITS keys to be loaded for first raw file */
#define RECIPE_KEYS_REGEXP               \
        RECIPE_KEYS_REGEXP_ALL           \
    "|" VISIR_PFITS_REGEXP_CAPA          \
    "|" VISIR_PFITS_REGEXP_SPC_WCAL_PAF

/* FITS keys to be loaded for first raw file, in case WCS is used */
#define RECIPE_KEYS_REGEXP_WCS \
        RECIPE_KEYS_REGEXP \
    "|" IRPLIB_PFITS_WCS_REGEXP

#define RAW_INPUT 0
#define PREPROCESSED_INPUT 1

/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_ech_save(cpl_frameset *,
                                         const cpl_parameterlist *,
                                         const cpl_propertylist *,
                                         const cpl_propertylist *,
                                         const cpl_image *,
                                         const cpl_image *,
                                         const cpl_table *,
                                         const char *,
                                         const char *,
                                         const char *);

VISIR_RECIPE_DEFINE(visir_old_spc_obs_ech,
                    VISIR_PARAM_ZERODIST | VISIR_PARAM_ORDEROFF |
                    VISIR_PARAM_EMIS_TOL |
                    VISIR_PARAM_REFINE | VISIR_PARAM_XCORR |
                    VISIR_PARAM_OFFSETS | VISIR_PARAM_OBJECTS |
                    VISIR_PARAM_NODPOS | VISIR_PARAM_AUTOBPM |
                    VISIR_PARAM_GLITCH | VISIR_PARAM_PURGE |
                    VISIR_PARAM_UNION  | VISIR_PARAM_REJECT |
                    VISIR_PARAM_STRIPITE | VISIR_PARAM_STRIPMOR |
                    VISIR_PARAM_PLOT   | VISIR_PARAM_SLITSKEW |
                    VISIR_PARAM_SPECSKEW | VISIR_PARAM_VERTARC |
                    VISIR_PARAM_REJLEFT | VISIR_PARAM_REJRIGHT |
                    VISIR_PARAM_HORIARC | VISIR_PARAM_FIXCOMBI |
                    VISIR_PARAM_GAIN | VISIR_PARAM_RONOISE |
                    VISIR_PARAM_OXSIGMA | VISIR_PARAM_OXNITER |
                    VISIR_PARAM_OXSMOOTH | VISIR_PARAM_OXKERNEL |
                    VISIR_PARAM_BKG_CORRECT,
                    "Old DRS detector: Spectroscopic Observation recipe in Echelle mode",
                    "This recipe estimates the dispersion relation using the "
                    "atmospheric spectrum\n"
                    "in a grism spectroscopy half-cycle frame.\n"
                    "It also extracts the spectrum of an observed object using "
                    "a combined frame.\n"
                    "The files listed in the Set Of Frames (sof-file) "
                    "must be tagged:\n"
                    "VISIR-Echelle-Spectroscopy-file.fits "
                    VISIR_SPC_OBS_ECH_RAW "\n"
                    "VISIR-Quantum-Efficiency-Calibration-file.fits "
                    VISIR_CALIB_QEFF_SPC "\n"
                    "VISIR-Atmospheric-Emission-Lines-Calibration-file.fits "
                    VISIR_CALIB_LINES_SPC
                    "\n"
                    MAN_VISIR_CALIB_BPM_SPC);

/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_spc_obs_ech   Spectroscopic Observation in Echelle mode
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/

/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_old_spc_obs_ech(cpl_frameset            * framelist,
                             const cpl_parameterlist * parlist)
{
    irplib_framelist * allframes = NULL;
    irplib_framelist * rawframes = NULL;
    irplib_framelist * skyframes = NULL;
    cpl_propertylist * qclist    = cpl_propertylist_new();
    cpl_propertylist * paflist   = cpl_propertylist_new();
    const char      *   badpix;
    const char      *   star_cat;
    const char      *   spc_cal_qeff;
    const char      *   spc_cal_lines;
    const char      *   flat;
    cpl_image       *   combined = NULL;
    cpl_image       *   flipped = NULL;
    cpl_image       *   comnarrow = NULL;
    cpl_imagelist   *   hcycle = NULL;
    cpl_table       *   spc_table = NULL;
    cpl_image       *   weight2d  = NULL;
    cpl_image       *   imhcycle = NULL;
    cpl_image       *   imhcyclenarrow = NULL;
    cpl_image      **   combinedpair;
    double              wlen, slitw, temp, fwhm;
    visir_spc_resol     resol;
    cpl_boolean        drop_wcs;
    int                input_mode;
    cpl_errorstate     cleanstate = cpl_errorstate_get();
    visir_spc_config   spc_config;
    const char       * dit_key = VISIR_PFITS_DOUBLE_DIT;
    cpl_boolean        has_phot = CPL_FALSE;

    spc_config.recipename = RECIPE_STRING;
    spc_config.parlist = parlist;
    spc_config.orderoffset = 0;
    spc_config.phot_emis_tol = 1.0;

    /* Retrieve input parameters */
    spc_config.plot = visir_parameterlist_get_int(
            parlist, RECIPE_STRING, VISIR_PARAM_PLOT);
    spc_config.phi = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_SLITSKEW);
    spc_config.ksi = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_SPECSKEW);
    spc_config.eps = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_VERTARC);
    spc_config.delta = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_HORIARC);
    spc_config.gain = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_GAIN);
    spc_config.ron = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_RONOISE);
    spc_config.ox_sigma = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_OXSIGMA);
    spc_config.ox_niters = visir_parameterlist_get_int(
            parlist, RECIPE_STRING, VISIR_PARAM_OXNITER);
    spc_config.ox_smooth = visir_parameterlist_get_int(
            parlist, RECIPE_STRING, VISIR_PARAM_OXSMOOTH);
    spc_config.ox_kernel = visir_parameterlist_get_int(
            parlist, RECIPE_STRING, VISIR_PARAM_OXKERNEL);
    spc_config.orderoffset = visir_parameterlist_get_int(
            parlist, RECIPE_STRING, VISIR_PARAM_ORDEROFF);
    spc_config.do_fixcombi = visir_parameterlist_get_bool(
            parlist, RECIPE_STRING, VISIR_PARAM_FIXCOMBI);
    spc_config.phot_emis_tol = visir_parameterlist_get_double(
            parlist, RECIPE_STRING, VISIR_PARAM_EMIS_TOL);
    spc_config.bkgcorrect = visir_parameterlist_get_bool(
            parlist, RECIPE_STRING, VISIR_PARAM_BKG_CORRECT);

    skip_if (0);

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));

    /* Objects observation */
    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);
    rawframes = irplib_framelist_extract(allframes, VISIR_SPC_OBS_ECH_RAW);
    if (rawframes == NULL) {
        cpl_errorstate_set(cleanstate);
        rawframes = irplib_framelist_extract(allframes, VISIR_SPC_OBS_ECH_PP);
        skip_if (rawframes == NULL);
        skyframes = irplib_framelist_extract(allframes, VISIR_SPC_OBS_SKYFRAME);
        skip_if (skyframes == NULL);
        input_mode = PREPROCESSED_INPUT;
        cpl_msg_info(cpl_func, "Input: preprocessed data");
    } else {
        cpl_msg_info(cpl_func, "Input: raw data");
        input_mode = RAW_INPUT;
    }
    skip_if (rawframes == NULL);
    const cpl_size nframes = irplib_framelist_get_size(rawframes);

    irplib_framelist_empty(allframes);

    skip_if(irplib_framelist_load_propertylist_all(rawframes, 0, "^("
                                                   VISIR_PFITS_REGEXP_SPC_SENSIT
                                                   "|" RECIPE_KEYS_REGEXP_ALL
                                                   "|" RECIPE_KEYS_REGEXP_WCS
                                                   "|" VISIR_PFITS_REGEXP_DIT
                                                   "|" "ESO DRS .*"
                                                   "|" "ESO PRO DATANCOM"
                                                   "|" "ESO QC EXPTIME"
                                                   "|" "ESO QC BACKGD MEAN"
                                                   ")$", CPL_FALSE));

    skip_if(visir_dfs_check_framelist_tag(rawframes));
    
    if (cpl_propertylist_has(irplib_framelist_get_propertylist(rawframes, 0),
                             VISIR_PFITS_DOUBLE_SEQ1_DIT))
        dit_key = VISIR_PFITS_DOUBLE_SEQ1_DIT;
    skip_if(0);

    visir_data_type data_type;
    const cpl_frame * frm = irplib_framelist_get_const(rawframes, 0);
    skip_if(visir_get_data_type(frm,
           irplib_framelist_get_propertylist(rawframes, 0), &data_type, NULL));

    /* Standard star catalog */
    star_cat = irplib_frameset_find_file(framelist, VISIR_CALIB_STDSTAR_SPC);

    /* Quantum efficiency file */
    spc_cal_qeff = irplib_frameset_find_file(framelist, VISIR_CALIB_QEFF_SPC);

    /* Spectral lines calibration file */
    spc_cal_lines = irplib_frameset_find_file(framelist, VISIR_CALIB_LINES_SPC);

    /* Bad pixels calibration file */
    badpix = irplib_frameset_find_file(framelist, VISIR_CALIB_BPM);

    /* Flatfield calibration file */
    flat = irplib_frameset_find_file(framelist, VISIR_CALIB_FLAT);

    /* Get Resolution and Central Wavelength */
    resol = visir_spc_get_res_wl(rawframes, &wlen, &slitw, &temp, &fwhm,
                                 visir_data_is_aqu(data_type));

    skip_if (0);

    if (resol != VISIR_SPC_R_GHR) {
        if (spc_config.orderoffset == 0) {
            cpl_msg_warning(cpl_func,"Reducing non-HR Grism data as main order");
        } else {
            cpl_msg_error(cpl_func, "This recipe cannot reduce non-HR Grism "
                          "data with an order-offset of %d",
                          spc_config.orderoffset);
            visir_error_set(CPL_ERROR_TYPE_MISMATCH);
            skip_if(1);
        }
    }

    if (input_mode == PREPROCESSED_INPUT) {
        cpl_imagelist * imagelist = cpl_imagelist_new();
        const cpl_propertylist ** plists =
            cpl_malloc(nframes * sizeof(cpl_propertylist *));
        double bg_sum = 0;

        for (cpl_size i = 0; i < nframes; i++) {
            cpl_frame * frame = irplib_framelist_get(rawframes, i);
            const char * fname = cpl_frame_get_filename(frame);
            const cpl_size next = cpl_fits_count_extensions(fname);
            cpl_imagelist * tmplist = cpl_imagelist_new();
            cpl_errorstate prestate = cpl_errorstate_get();
            cpl_image * tmp;

            plists[i] = irplib_framelist_get_propertylist_const(rawframes, i);
            if (cpl_propertylist_has(plists[i], "ESO QC BACKGD MEAN"))
                bg_sum += cpl_propertylist_get_double(plists[i],
                                                      "ESO QC BACKGD MEAN");
            skip_if(0);

            for (cpl_size j = 0; j < 1 + next; j++) {
                cpl_image * img =
                    cpl_image_load(fname, CPL_TYPE_UNSPECIFIED, 0, j);
                if (img == NULL) {
                    cpl_errorstate_set(prestate);
                    cpl_msg_info(cpl_func, "No image in extension %d", (int)j);
                    continue;
                }
                cpl_imagelist_set(tmplist, img, cpl_imagelist_get_size(tmplist));
                skip_if(0);
            }
            tmp = cpl_imagelist_collapse_create(tmplist);

            skip_if (visir_spc_det_fix(&tmp, 1, CPL_TRUE, wlen, resol,
                                       spc_config.phi, spc_config.ksi,
                                       spc_config.eps, spc_config.delta,
                                       spc_config.plot));
            cpl_imagelist_set(imagelist, tmp,
                              cpl_imagelist_get_size(imagelist));
            cpl_imagelist_delete(tmplist);
        }
        combinedpair =
            visir_img_recombine_list(RECIPE_STRING, parlist, imagelist,
                                     plists, CPL_GEOM_FIRST, &drop_wcs);


        cpl_propertylist_append_double(qclist, "ESO QC BACKGD MEAN",
                                       bg_sum / nframes);
        cpl_imagelist_delete(imagelist);
        cpl_free(plists);
        bug_if(0);

        /* FIXME: make use of multiple skyframes */
        {
            cpl_frame * frame = irplib_framelist_get(skyframes, 0);
            const char * fname = cpl_frame_get_filename(frame);
            imhcycle  = cpl_image_load(fname, CPL_TYPE_UNSPECIFIED, 0, 0);//1);
            skip_if(imhcycle == NULL);
        }

        skip_if (visir_spc_det_fix(&imhcycle, 1, CPL_TRUE, wlen, resol,
                                   spc_config.phi, spc_config.ksi,
                                   spc_config.eps, spc_config.delta,
                                   spc_config.plot));

    } else if (input_mode == RAW_INPUT) {
        combinedpair = visir_img_recombine(RECIPE_STRING, parlist, rawframes, badpix,
                                           flat, CPL_GEOM_FIRST, &drop_wcs,
                                           !spc_config.do_fixcombi, wlen, resol);

        /* Get the hcycle image from the reference file */
        hcycle = visir_load_hcycle(rawframes, 0);
        skip_if (0);

        imhcycle = cpl_imagelist_unset(hcycle, 0);

        skip_if (visir_spc_det_fix(&imhcycle, 1, CPL_FALSE, wlen, resol,
                                   spc_config.phi, spc_config.ksi,
                                   spc_config.eps, spc_config.delta,
                                   spc_config.plot));

        skip_if(visir_qc_append_background(qclist, rawframes, 0, 0));
    }
    else
        bug_if(1);

    if (combinedpair == NULL) {
        cpl_msg_error(cpl_func, "Could not combine the input frames");
        skip_if (1);
    }

    cpl_image_delete(combinedpair[1]);
    combined = cpl_image_cast(combinedpair[0], CPL_TYPE_DOUBLE);
    cpl_image_delete(combinedpair[0]);
    cpl_free(combinedpair);

    /* Convert the combined image
     * recipe only supported with drs data */
    bool rev;
    flipped = visir_spc_flip(combined, wlen, resol, VISIR_DATA_CUBE2, &rev);
    skip_if (0);
    cpl_image_delete(combined);
    combined = flipped;
    flipped = NULL;

    /* Convert the half cycle image
     * recipe only supported with drs data */
    flipped = visir_spc_flip(imhcycle, wlen, resol, VISIR_DATA_CUBE2, NULL);
    skip_if (0);
    cpl_image_delete(imhcycle);
    imhcycle = flipped;
    flipped = NULL;

    skip_if (0);

    skip_if (spc_config.do_fixcombi && input_mode == RAW_INPUT &&
             visir_spc_det_fix(&combined, 1, CPL_TRUE, wlen, resol,
                               spc_config.phi, spc_config.ksi,
                               spc_config.eps, spc_config.delta,
                               spc_config.plot));

    const int ncomb = visir_get_ncombine(rawframes);
    skip_if (!ncomb);

    int lcol = -1, rcol = -1;
    skip_if (visir_spc_extract_order(&imhcyclenarrow, &comnarrow, &lcol, &rcol,
                                     combined, imhcycle, wlen, &spc_config,
                                     CPL_TRUE, visir_data_is_aqu(data_type)));

    int const ident = -1;  // neg ident: use old extract alg
    visir_apdefs * const aps = visir_apdefs_new(1, ident, 'O', 0);
    aps->limits[0] = (visir_aplimits){ lcol, rcol };

    skip_if (visir_spc_extract_wcal(comnarrow, imhcyclenarrow, lcol, rcol, wlen,
                                    slitw, temp, fwhm, resol, &spc_config,
                                    spc_cal_lines, spc_cal_qeff,
                                    visir_data_is_aqu(data_type), aps, ncomb,
                                    rev, &spc_table, &weight2d, qclist));
                                    


    bug_if (visir_spectro_qc(qclist, paflist, drop_wcs, rawframes, NULL,
                             "^(" VISIR_PFITS_REGEXP_SPC_WCAL_PAF ")$"));

    if (star_cat) {
        cpl_errorstate prestate = cpl_errorstate_get();
        const cpl_propertylist * plist =
            irplib_framelist_get_propertylist_const(rawframes, 0);
        skip_if(0);
        spc_config.phu = qclist;
        if (visir_spc_phot_sensit(rawframes, &spc_config, plist, star_cat,
                                  &weight2d, qclist, spc_table, resol,
                                  dit_key)) {
            irplib_error_recover(prestate, "Could not perform photometric "
                                 "calibration");
        } else {
            has_phot = CPL_TRUE;
        }
    } else {
        /* This column is not part of the product */
        skip_if (cpl_table_erase_column(spc_table, "SPC_EMISSIVITY"));
    }
   
    irplib_framelist_empty(rawframes);

    /* Save the spectrum */
    cpl_msg_info(cpl_func, "Saving the produced spectrum");

    /* PRO.CATG */
    bug_if (cpl_propertylist_append_string(paflist, CPL_DFS_PRO_CATG,
                                           VISIR_SPC_OBS_ECH_COMBINED_PROCATG));

    if (has_phot)
        skip_if (visir_spc_ech_save(framelist, parlist, qclist, paflist,
                                     combined, weight2d, spc_table,
                                     VISIR_SPC_PHOT_ECH_TAB_PROCATG,
                                     VISIR_SPC_PHOT_ECH_COMBINED_PROCATG,
                                     VISIR_SPC_PHOT_ECH_WEIGHT_PROCATG));
    else
        skip_if (visir_spc_ech_save(framelist, parlist, qclist, paflist,
                                    combined, weight2d, spc_table,
                                     VISIR_SPC_OBS_ECH_TAB_PROCATG,
                                     VISIR_SPC_OBS_ECH_COMBINED_PROCATG,
                                     VISIR_SPC_OBS_ECH_WEIGHT_PROCATG));

    end_skip;

    cpl_propertylist_delete(qclist);
    cpl_propertylist_delete(paflist);
    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    irplib_framelist_delete(skyframes);
    cpl_image_delete(flipped);
    cpl_image_delete(combined);
    cpl_image_delete(comnarrow);
    cpl_image_delete(imhcyclenarrow);
    cpl_table_delete(spc_table);
    cpl_image_delete(weight2d);
    cpl_image_delete(imhcycle);
    cpl_imagelist_delete(hcycle);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Save the visir_spc_ech recipe products on disk
  @param    set         The input frame set
  @param    parlist     The input list of parameters
  @param    qclist      List of QC parameters
  @param    paflist     List of PAF parameters
  @param    combined    The combined image produced
  @param    weight2d    2D weights image
  @param    table       The spectral fields
  @param    tab_procatg Table procatg
  @param    com_procatg Combined procatg
  @param    wgt_procatg Weight procatg
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_ech_save(cpl_frameset            * set,
                                         const cpl_parameterlist * parlist,
                                         const cpl_propertylist  * qclist,
                                         const cpl_propertylist  * paflist,
                                         const cpl_image         * combined,
                                         const cpl_image         * weight2d,
                                         const cpl_table         * table,
                                         const char              * tab_procatg,
                                         const char              * com_procatg,
                                         const char              * wgt_procatg)
{

    bug_if (0);

    /* THE TABLE */
    skip_if (irplib_dfs_save_table(set, parlist, set, table, NULL, RECIPE_STRING,
                               tab_procatg,
                               qclist, NULL, visir_pipe_id,
                               RECIPE_STRING "_tab" CPL_DFS_FITS));

    /* THE COMBINED IMAGE */
    skip_if (irplib_dfs_save_image(set, parlist, set, combined, CPL_BPP_IEEE_FLOAT,
                               RECIPE_STRING, com_procatg,
                               qclist, NULL, visir_pipe_id,
                               RECIPE_STRING CPL_DFS_FITS));

    /* THE WEIGHT IMAGE */
    skip_if (irplib_dfs_save_image(set, parlist, set, weight2d, CPL_BPP_IEEE_FLOAT,
                               RECIPE_STRING, wgt_procatg,
                               qclist, NULL, visir_pipe_id,
                               RECIPE_STRING "_weight" CPL_DFS_FITS));

#ifdef VISIR_SAVE_PAF
    /* THE PAF FILE FOR QC PARAMETERS */

    skip_if (cpl_dfs_save_paf("VISIR", RECIPE_STRING, paflist,
                             RECIPE_STRING CPL_DFS_PAF));
#else
    bug_if(paflist == NULL);
#endif

    end_skip;

    return cpl_error_get_code();

}
