/* $Id: visir_spc_obs.c,v 1.111 2013-05-13 16:04:49 jtaylor Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003,2012 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jtaylor $
 * $Date: 2013-05-13 16:04:49 $
 * $Revision: 1.111 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/
#include "cxlist.h"
#define SBRM_UNDERSCORE_MACRO  // enable the _() macro
#include "sbrm.h"

#include "visir_inputs.h"
#include "visir_recipe.h"
#include "visir_spectro.h"
#include "visir_spc_distortion.h"
#include "visir_spc_photom.h"


/*-----------------------------------------------------------------------------
                            Defines
 -----------------------------------------------------------------------------*/

// aliases for SBRM features
#define INIT        SBRM_INIT
#define SET         SBRM_SET
#define CSET        SBRM_CSET
#define RESET       SBRM_RESET
#define FREE        SBRM_FREE
#define CLEANUP     SBRM_CLEANUP
#define YANK        SBRM_YANK
#define ERR         SBRM_ABORT
#define ERR_TO_WARN SBRM_WARN
#define ERR_TO_INFO SBRM_INFO
#define VLIST       SBRM_VLIST
#define VALLOC      SBRM_VALLOC
#define NALLOC      SBRM_NALLOC

#define RECIPE_STRING  "visir_old_spc_obs"
/* still used by chain recipe internally, so keep name for output */
#define RECIPE_SAVE_STRING "visir_spc_obs"

/* FITS keys to be loaded for all raw files */
#define RECIPE_KEYS_REGEXP_ALL            \
        VISIR_PFITS_REGEXP_IMG_RECOMBINE  \
    "|" VISIR_PFITS_REGEXP_SPC_GET_RES_WL

/* FITS keys to be loaded for first raw file */
#define RECIPE_KEYS_REGEXP               \
        RECIPE_KEYS_REGEXP_ALL           \
    "|" VISIR_PFITS_REGEXP_CAPA          \
    "|" VISIR_PFITS_REGEXP_SPC_WCAL_PAF

/* FITS keys to be loaded for first raw file, in case WCS is used */
#define RECIPE_KEYS_REGEXP_WCS \
        RECIPE_KEYS_REGEXP \
    "|" IRPLIB_PFITS_WCS_REGEXP

#define IS_PREPROCESSED 1
#define IS_PHOT 2
#define IS_ECH 4
#define RAW_INPUT 0
#define PREPROCESSED_INPUT IS_PREPROCESSED
#define RAW_PHOT_INPUT IS_PHOT
#define PREPROCESSED_PHOT_INPUT (IS_PHOT | IS_PREPROCESSED)

/* the number of orders we try to extract */
#define MAX_ORDERS 9

/* a flag value used to indicate an unused order in the data structures */
#define UNUSED_ORDER 99

#define MSG_WARN(...) cpl_msg_warning(cpl_func, __VA_ARGS__)
#define MSG_INFO(...) cpl_msg_info(cpl_func, __VA_ARGS__)
#define MSG_ERR(...) cpl_msg_error(cpl_func, __VA_ARGS__)
#define MSG_DBG(...) cpl_msg_debug(cpl_func, __VA_ARGS__)

/*-----------------------------------------------------------------------------
                                  Typedefs
 -----------------------------------------------------------------------------*/
/* possibly more than 1 of these per order */
typedef struct {
    int extract;                   /* The extraction status (UNUSED_ORDER =
                                    * failed, anything else = succeeded) */
    cpl_image * weight2d;          /* The 2D weight images */
    cpl_table * spc_tbl;           /* The extracted spectrum */
    cpl_propertylist * qclist;     /* The QC properties */
    visir_apdefs * aps;            /* The matching aperture definition */
} match;
static void match_delete(match m);

typedef VLIST(match) matchlist;
#define matchlist_new(size) VALLOC(matchlist, match, size)
static void matchlist_delete(matchlist * ml);

typedef struct {
    int cutout;             /* The order offset, which is +/-4 from the
                             * central order, doubling as cutout status
                             * (UNUSED_ORDER = failed, anything else =
                             * succeeded) */
    cpl_image * comnarrow;  /* The saved image cutout, per order */
    matchlist * matches;
    visir_aplist * aplist;
} order;
static void order_delete(order o);

typedef VLIST(order) orderlist;
#define orderlist_new(size, ...) NALLOC(orderlist, order, size, __VA_ARGS__)
static void orderlist_delete(orderlist * ol);

/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_obs_save(cpl_frameset *,
                                         const cpl_parameterlist *,
                                         cpl_propertylist *,
                                         cpl_propertylist *,
                                         const cpl_propertylist *,
                                         const cpl_image *,
                                         const cpl_table *,
                                         const cpl_image *,
                                         const char *,
                                         const int, const int, const size_t);

static cpl_error_code visir_spc_obs_extend(cpl_propertylist *,
                                          const cpl_image *,
                                          const cpl_table *,
                                          const cpl_image *,
                                          const int, const int, const size_t);

static cpl_error_code visir_spc_obs_extend_combined(const cpl_image *);

static cpl_error_code visir_spc_qc_summarise(cpl_propertylist *, orderlist *,
                                             const char *);

#ifdef VISIR_CHAIN
#define cpl_plugin_get_info visir_old_spc_obs_get_info
#endif
VISIR_RECIPE_DEFINE(visir_old_spc_obs,
                    VISIR_PARAM_EMIS_TOL |
                    VISIR_PARAM_REFINE | VISIR_PARAM_XCORR |
                    VISIR_PARAM_OFFSETS | VISIR_PARAM_OBJECTS |
                    VISIR_PARAM_NODPOS | VISIR_PARAM_AUTOBPM |
                    VISIR_PARAM_GLITCH | VISIR_PARAM_PURGE |
                    VISIR_PARAM_UNION  | VISIR_PARAM_REJECT |
                    VISIR_PARAM_STRIPITE | VISIR_PARAM_STRIPMOR |
                    VISIR_PARAM_PLOT   | VISIR_PARAM_SLITSKEW |
                    VISIR_PARAM_SPECSKEW | VISIR_PARAM_VERTARC |
                    VISIR_PARAM_REJLEFT | VISIR_PARAM_REJRIGHT |
                    VISIR_PARAM_HORIARC | VISIR_PARAM_FIXCOMBI |
                    VISIR_PARAM_BKG_CORRECT | VISIR_PARAM_APERT_FILE |
                    VISIR_PARAM_GAIN | VISIR_PARAM_RONOISE |
                    VISIR_PARAM_OXSIGMA | VISIR_PARAM_OXNITER |
                    VISIR_PARAM_OXSMOOTH | VISIR_PARAM_OXKERNEL |
                    VISIR_PARAM_RESPCAL,
                    "Old DRS detector: Spectroscopic Observation recipe",
                    "This recipe estimates the dispersion relation using the "
                    "atmospheric spectrum\n"
                    "in a long-slit spectroscopy half-cycle frame.\n"
                    "It also extracts the spectrum of an observed object using "
                    "a combined frame.\n"
                    "The files listed in the Set Of Frames (sof-file) "
                    "must be tagged:\n"
                    "VISIR-Long-Slit-Spectroscopy-file.fits "
                    VISIR_SPC_OBS_RAW "\n"
                    "VISIR-Quantum-Efficiency-Calibration-file.fits "
                    VISIR_CALIB_QEFF_SPC "\n"
                    "VISIR-Atmospheric-Emission-Lines-Calibration-file.fits "
                    VISIR_CALIB_LINES_SPC
                    "\n"
                    MAN_VISIR_CALIB_BPM_SPC);

/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_spc_obs   Spectroscopic Observation
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/

static void biimage_delete(cpl_image ** bi)
{
    if (!bi) return;
    cpl_image_delete(bi[0]);
    cpl_image_delete(bi[1]);
    cpl_free(bi);
}

static void match_delete(match m)
{
    cpl_propertylist_delete(m.qclist);
    // original algorithm apertures (with negative ident) are the only ones
    // owned, hence are the only ones that need to be freed: the rest are just
    // references into other owning structs
    if (m.aps->ident < 0) visir_apdefs_delete(m.aps);
    cpl_table_delete(m.spc_tbl);
    cpl_image_delete(m.weight2d);
}

static void matchlist_delete(matchlist * ml)
{
    if (!ml) return;
    for (size_t i = 0; i < ml->sz; ++i) match_delete(ml->data[i]);
    cpl_free(ml);
}

static void order_delete(order o)
{
    cpl_image_delete(o.comnarrow);
    visir_aplist_delete(o.aplist);  // free list only (elems are just refs)
    matchlist_delete(o.matches);
}

static void orderlist_delete(orderlist * ol)
{
    if (!ol) return;
    for (size_t i = 0; i < ol->sz; ++i) order_delete(ol->data[i]);
    cpl_free(ol);
}

/*----------------------------------------------------------------------------*/
/**
  @brief    convert an integral order offset into its string representation:
            n=0 => "0", n<0 => "-n", n>0 => "+n"
  @param    oo          the order offset as an integer
  @return   the string representation of the order offset
 */
/*----------------------------------------------------------------------------*/
static const char * pn(const int oo)
{
    static char buf[80];  // FIXME: don't use static vars
    const char * sign = oo ? (oo > 0 ? "+" : "-") : "";
    snprintf(buf, sizeof(buf), "%s%d", sign, abs(oo));
    return buf;
}

/*----------------------------------------------------------------------------*/
/**
  @brief    The actual recipe data reduction implementation
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   NULL, always (with the appropriate CPL error being set)
 */
/*----------------------------------------------------------------------------*/
static void * visir_old_spc_obs_(cpl_frameset * framelist,
                                 const cpl_parameterlist * parlist)
{
    INIT(20);

    visir_spc_config spc_config;
    spc_config.recipename = RECIPE_STRING;
    spc_config.parlist = parlist;
    spc_config.phot_emis_tol = 1.0;

    /* Initialise the QC property lists */
    SET(phu, cpl_propertylist) = cpl_propertylist_new();
    spc_config.phu = phu->o;

    /* Retrieve input parameters */
    spc_config.plot = visir_parameterlist_get_int _(
            parlist, RECIPE_STRING, VISIR_PARAM_PLOT);
    spc_config.phi = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_SLITSKEW);
    spc_config.ksi = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_SPECSKEW);
    spc_config.eps = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_VERTARC);
    spc_config.delta = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_HORIARC);
    spc_config.gain = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_GAIN);
    spc_config.ron = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_RONOISE);
    spc_config.ox_sigma = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_OXSIGMA);
    spc_config.ox_niters = visir_parameterlist_get_int _(
            parlist, RECIPE_STRING, VISIR_PARAM_OXNITER);
    spc_config.ox_smooth = visir_parameterlist_get_int _(
            parlist, RECIPE_STRING, VISIR_PARAM_OXSMOOTH);
    spc_config.ox_kernel = visir_parameterlist_get_int _(
            parlist, RECIPE_STRING, VISIR_PARAM_OXKERNEL);
    spc_config.do_fixcombi = visir_parameterlist_get_bool _(
            parlist, RECIPE_STRING, VISIR_PARAM_FIXCOMBI);
    spc_config.phot_emis_tol = visir_parameterlist_get_double _(
            parlist, RECIPE_STRING, VISIR_PARAM_EMIS_TOL);
    spc_config.bkgcorrect = visir_parameterlist_get_bool _(
            parlist, RECIPE_STRING, VISIR_PARAM_BKG_CORRECT);
    spc_config.respcal = visir_parameterlist_get_string _(
            parlist, RECIPE_STRING, VISIR_PARAM_RESPCAL);

    char const * const apfile = visir_parameterlist_get_string _(
            parlist, RECIPE_STRING, VISIR_PARAM_APERT_FILE);
    // re-enable this if it is an error to extract without sky-subtraction
    //if (!visir_str_par_is_empty(apfile) && !spc_config.bkgcorrect)
    //    return ERR(CPL_ERROR_INCOMPATIBLE_INPUT,
    //               "--bkgcorrect must be true when using --apfile=...");

    // parse apfile into a list of apertures
    SET(aplist, visir_aplist, v, visir_aplist_destroy) = NULL;
    if (visir_str_par_is_empty(apfile))
        RESET(aplist) = visir_aplist_new(); // no_
    else {
        RESET(aplist) = visir_aplist_new_from_file _(apfile);
    }

    //// write aps parsed from apfile into PHU (this is now done in the
    //// extraction routine, but the advantage of doing it here is that *all*
    //// lines from the apfile are put in the header, even processing
    //// fails before getting to the extraction routine)
    //FOR_EACH_T(visir_apdefs const * const aps, aplist->o) {
    //    SET(line, char, v, cpl_free) = visir_apdefs_dump _(aps);
    //    SET(key, char, v, cpl_free) = cpl_sprintf _("ESO DRS APDEF%d", aps->ident);
    //    cpl_propertylist_append_string _(phu->o, key->o, line->o);
    //}

    /* Identify the RAW and CALIB frames in the input frameset */
    visir_dfs_set_groups _(framelist);

    /* Objects observation */
    SET(all, irplib_framelist) = irplib_framelist_cast _(framelist);
    char const * const inraw = "Input: raw %s frame (%s)";
    char const * const inprep = "Input: preprocessed %s frame (%s)";

    int input_mode;
    const char * rawtag;
    SET(raws, irplib_framelist) = NULL;
    if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_PP)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_PP);
        rawtag = VISIR_SPC_OBS_PP;
        MSG_INFO(inprep, "observation", VISIR_SPC_OBS_PP);
        input_mode = PREPROCESSED_INPUT;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_PP)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_PP);
        rawtag = VISIR_SPC_PHOT_PP;
        MSG_INFO(inprep, "photometric", VISIR_SPC_PHOT_PP);
        input_mode = PREPROCESSED_PHOT_INPUT;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_ECH_PP)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_ECH_PP);
        rawtag = VISIR_SPC_OBS_ECH_PP;
        MSG_INFO(inprep, "observation echelle", VISIR_SPC_OBS_ECH_PP);
        input_mode = PREPROCESSED_INPUT | IS_ECH;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_ECH_PP)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_ECH_PP);
        rawtag = VISIR_SPC_PHOT_ECH_PP;
        MSG_INFO(inprep, "photometric echelle", VISIR_SPC_PHOT_ECH_PP);
        input_mode = PREPROCESSED_PHOT_INPUT | IS_ECH;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_RAW)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_RAW);
        rawtag = VISIR_SPC_OBS_RAW;
        MSG_INFO(inraw, "observation", VISIR_SPC_OBS_RAW);
        input_mode = RAW_INPUT;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_RAW)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_RAW);
        rawtag = VISIR_SPC_PHOT_RAW;
        MSG_INFO(inraw, "photometric", VISIR_SPC_PHOT_RAW);
        input_mode = RAW_PHOT_INPUT;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_OBS_ECH_RAW)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_OBS_ECH_RAW);
        rawtag = VISIR_SPC_OBS_ECH_RAW;
        MSG_INFO(inraw, "observation echelle", VISIR_SPC_OBS_ECH_RAW);
        input_mode = RAW_INPUT | IS_ECH;

    } else if (irplib_frameset_find_file(framelist, VISIR_SPC_PHOT_ECH_RAW)) {
        RESET(raws) = irplib_framelist_extract _(all->o, VISIR_SPC_PHOT_ECH_RAW);
        rawtag = VISIR_SPC_PHOT_ECH_RAW;
        MSG_INFO(inraw, "photometric echelle", VISIR_SPC_PHOT_ECH_RAW);
        input_mode = RAW_PHOT_INPUT | IS_ECH;

    } else {
        return ERR(CPL_ERROR_ILLEGAL_INPUT, "Input frame set must contain "
                   "frame(s) tagged as one of: " VISIR_SPC_OBS_RAW ", "
                   VISIR_SPC_PHOT_RAW ", " VISIR_SPC_OBS_PP ","
                   VISIR_SPC_PHOT_PP "," VISIR_SPC_OBS_ECH_RAW ","
                   VISIR_SPC_OBS_ECH_PP "," VISIR_SPC_PHOT_ECH_RAW ","
                   VISIR_SPC_PHOT_ECH_PP);
    }

    cpl_size const nframes = irplib_framelist_get_size _(raws->o);

    irplib_framelist_load_propertylist_all _(raws->o, 0, "^("
                                             VISIR_PFITS_REGEXP_SPC_SENSIT
                                             "|" RECIPE_KEYS_REGEXP_ALL
                                             "|" RECIPE_KEYS_REGEXP_WCS
                                             "|" VISIR_PFITS_REGEXP_DIT
                                             "|" VISIR_PFITS_DOUBLE_WLEN
                                             "|" VISIR_PFITS_DOUBLE_PWLEN
                                             "|" "ESO DRS .*"
                                             "|" "ESO PRO DATANCOM"
                                             "|" "ESO QC EXPTIME"
                                             "|" "ESO QC EXECTIME"
                                             "|" "ESO QC BACKGD MEAN"
                                             ")$", CPL_FALSE);

    visir_dfs_check_framelist_tag _(raws->o);
    
    cpl_propertylist const * const pl =
            irplib_framelist_get_propertylist_const _(raws->o, 0);
    int const has_seq = cpl_propertylist_has _(pl, VISIR_PFITS_DOUBLE_SEQ1_DIT);
    char const * const dit_key = has_seq ? VISIR_PFITS_DOUBLE_SEQ1_DIT
                                         : VISIR_PFITS_DOUBLE_DIT;

    /* Standard star catalog */
    char const * const star_cat =
            irplib_frameset_find_file _(framelist, VISIR_CALIB_STDSTAR_SPC);
    if ((input_mode & IS_PHOT) && star_cat == NULL) return ERR(
            CPL_ERROR_DATA_NOT_FOUND, "Frame set with a photometric frame (%s) "
            "must include a standard star catalogue (" VISIR_CALIB_STDSTAR_SPC
            ")", rawtag);

    visir_data_type data_type;
    cpl_frame const * const frm = irplib_framelist_get_const _(raws->o, 0);
    visir_get_data_type _(frm, pl, &data_type, NULL);

    if (visir_data_is_drs(data_type)) {
        /* if parameters are (aqu) defaults and drs data use old parameters */
        double tmp;
        cpl_parameter const * par = cpl_parameterlist_find_const _(
                parlist, PACKAGE "." RECIPE_STRING ".ksi");
        tmp = cpl_parameter_get_default_double _(par);
        if (spc_config.ksi == tmp * CPL_MATH_RAD_DEG)
            spc_config.ksi = VISIR_DRS_DIST_KSI;

        par = cpl_parameterlist_find_const _(
                parlist, PACKAGE "." RECIPE_STRING ".eps");
        tmp = cpl_parameter_get_default_double _(par);
        if (spc_config.eps == tmp) spc_config.eps = VISIR_DRS_DIST_EPS;

        par = cpl_parameterlist_find_const _(
                parlist, PACKAGE "." RECIPE_STRING ".delta");
        tmp = cpl_parameter_get_default_double _(par);
        if (spc_config.delta == tmp) spc_config.delta = VISIR_DRS_DIST_DELTA;

        par = cpl_parameterlist_find_const _(
                parlist, PACKAGE "." RECIPE_STRING ".phi");
        tmp = cpl_parameter_get_default_double _(par);
        if (spc_config.phi == tmp * CPL_MATH_RAD_DEG)
            spc_config.phi = VISIR_DRS_DIST_PHI;
    }

    /* Quantum efficiency file */
    char const * const spc_cal_qeff =
            irplib_frameset_find_file _(framelist, VISIR_CALIB_QEFF_SPC);

    /* Spectral lines calibration file */
    char const * const spc_cal_lines =
            irplib_frameset_find_file _(framelist, VISIR_CALIB_LINES_SPC);

    /* Bad pixels calibration file */
    char const * const badpix =
            irplib_frameset_find_file _(framelist, VISIR_CALIB_BPM);

    /* Flatfield calibration file */
    char const * const flat =
            irplib_frameset_find_file _(framelist, VISIR_CALIB_FLAT);

    /* Get Resolution and Central Wavelength */
    double wlen, slitw, temp, fwhm;
    visir_spc_resol resol = visir_spc_get_res_wl _(raws->o, &wlen, &slitw,
            &temp, &fwhm, visir_data_is_aqu(data_type));

    /* The number of orders currently in use */
    int n_orders = 1;
    if (resol == VISIR_SPC_R_GHR) {
        if (!(input_mode & IS_ECH) && !(input_mode & IS_PREPROCESSED))
            return ERR(CPL_ERROR_TYPE_MISMATCH, "Will not reduce echelle data "
                       "tagged as long slit data");
        n_orders = MAX_ORDERS;  // try to extract all orders
    } else {
        MSG_WARN("Reducing non-HR Grism data as main order");
    }

    cpl_boolean drop_wcs;
    SET(imhcycle, cpl_image) = NULL;
    SET(combinedpair, cpl_image*, v, biimage_delete) = NULL;
    if (input_mode & IS_PREPROCESSED) {
        SET(imagelist, cpl_imagelist) = cpl_imagelist_new();
        CSET(plists, cpl_propertylist*, v, cpl_free) = cpl_malloc(
                nframes * sizeof(cpl_propertylist *));
        double bg_sum = 0;

        for (cpl_size i = 0; i < nframes; i++) {
            cpl_frame const * const frame = irplib_framelist_get _(raws->o, i);
            char const * const fname = cpl_frame_get_filename _(frame);
            cpl_size const next = cpl_fits_count_extensions _(fname);
            SET(tmplist, cpl_imagelist) = cpl_imagelist_new();

            plists->o[i] = irplib_framelist_get_propertylist_const _(raws->o,i);
            if (cpl_propertylist_has(plists->o[i], "ESO QC BACKGD MEAN")) {
                bg_sum += cpl_propertylist_get_double _(plists->o[i],
                                                        "ESO QC BACKGD MEAN");
            }

            for (cpl_size j = 0; j < 1 + next; j++) {
                SET(img, cpl_image) = cpl_image_load(  // no_
                        fname, CPL_TYPE_UNSPECIFIED, 0, j);
                if (cpl_error_get_code()) {
                    ERR_TO_INFO("No image in extension %d", (int)j);
                    continue;
                }
                cpl_image_reject_value _(img->o, CPL_VALUE_NAN);
                cpl_size const lsz = cpl_imagelist_get_size _(tmplist->o);
                cpl_imagelist_set _(tmplist->o, img->o, lsz);
                YANK(img);
            }
            SET(tmp, cpl_image) = cpl_imagelist_collapse_create _(tmplist->o);
            cpl_size const lsz = cpl_imagelist_get_size _(imagelist->o);
            cpl_imagelist_set _(imagelist->o, tmp->o, lsz);
            YANK(tmp);
        }

        RESET(combinedpair) = visir_img_recombine_list _(RECIPE_STRING, parlist,
            imagelist->o, plists->o, CPL_GEOM_FIRST, &drop_wcs);

        cpl_propertylist_append_double _(phu->o, "ESO QC BACKGD MEAN",
                                         bg_sum / nframes);
        FREE(imagelist);
        FREE(plists);

        {
            SET(skys, irplib_framelist) = irplib_framelist_extract(  // no_
                all->o, VISIR_SPC_OBS_SKYFRAME);
            if (cpl_error_get_code()) return ERR(CPL_ERROR_ILLEGAL_INPUT,
                    "Frame set with a preprocessed frame (%s) must include a "
                    "sky frame (" VISIR_SPC_OBS_SKYFRAME ")", rawtag);

            /* FIXME: make use of multiple skyframes */
            cpl_frame const * const frame = irplib_framelist_get _(skys->o, 0);
            char const * const fname = cpl_frame_get_filename _(frame);
            RESET(imhcycle) = cpl_image_load _(fname, CPL_TYPE_UNSPECIFIED,0,0);
            cpl_image_reject_value _(imhcycle->o, CPL_VALUE_NAN);
        }


    } else {
        RESET(combinedpair) = visir_img_recombine _(RECIPE_STRING, parlist,
                raws->o, badpix, flat, CPL_GEOM_FIRST, &drop_wcs,
                !spc_config.do_fixcombi, wlen, resol);

        /* Get the hcycle image from the reference file */
        SET(hcycle, cpl_imagelist) = visir_load_hcycle _(raws->o, 0);
        RESET(imhcycle) = cpl_imagelist_unset _(hcycle->o, 0);

        visir_spc_det_fix _(&imhcycle->o, 1, CPL_FALSE, wlen, resol,
                            spc_config.phi, spc_config.ksi,
                            spc_config.eps, spc_config.delta,
                            spc_config.plot);

        visir_qc_append_background _(phu->o, raws->o, 0, 0);
    }

    SET(combined, cpl_image) = cpl_image_cast _(combinedpair->o[0],
                                                CPL_TYPE_DOUBLE);
    FREE(combinedpair);

    bool rev = false;
    if (!(input_mode & IS_PREPROCESSED)) {
        /* Convert the combined image
         * unpreprocessed mode only works with drs data */
        SET(tmp, cpl_image) = visir_spc_flip _(combined->o, wlen, resol,
                                               VISIR_DATA_CUBE2, &rev);
        RESET(combined) = YANK(tmp);

        /* Convert the half cycle image */
        RESET(tmp) = visir_spc_flip _(imhcycle->o, wlen, resol,
                                      VISIR_DATA_CUBE2, NULL);
        RESET(imhcycle) = YANK(tmp);
    }

    int const ncomb = visir_get_ncombine _(raws->o);

    if (spc_config.do_fixcombi && !(input_mode & IS_PREPROCESSED)) {
        visir_spc_det_fix _(&combined->o, 1, CPL_TRUE, wlen, resol,
                            spc_config.phi, spc_config.ksi,
                            spc_config.eps, spc_config.delta,
                            spc_config.plot);
    }

    visir_optmod ins_settings;
    visir_spc_optmod_init _(resol, wlen, &ins_settings,
                            visir_data_is_aqu(data_type));
    
    MSG_DBG("resol = %d, input_mode = %d", resol, input_mode);
    SET(paflist, cpl_propertylist) = cpl_propertylist_new();

    // there must be at least n_orders args after n_orders for list init'n
    SET(orders, orderlist) = orderlist_new(n_orders, {0, 0, 0, 0},
            {1, 0, 0, 0}, {-1, 0, 0, 0}, {2, 0, 0, 0}, {-2, 0, 0, 0},
            {3, 0, 0, 0}, {-3, 0, 0, 0}, {4, 0, 0, 0}, {-4, 0, 0, 0});

    // for each order; this loop sets 'cutout' to 99 upon failure, over-writing
    // the order offset (0, -1, +1, -2, etc...)
    for (size_t i = 0; i < orders->o->sz; ++i) {
        order * const oi = &orders->o->data[i];
        char pfx[80] = "";
        if (input_mode & IS_ECH) snprintf(pfx, sizeof(pfx), "Echelle order %2d: "
                "offset %s: ", oi->cutout + visir_spc_optmod_get_echelle_order(
                &ins_settings), pn(oi->cutout));
        MSG_INFO("%sattempting extraction", pfx);

        int lcol = -1, rcol = -1;
        spc_config.orderoffset = oi->cutout;
        SET(imhcyclenarrow, cpl_image) = NULL;
        visir_spc_extract_order(&imhcyclenarrow->o, &oi->comnarrow, &lcol, //no_
                                &rcol, combined->o, imhcycle->o, wlen,
                                &spc_config, (input_mode & IS_ECH) == IS_ECH,
                                visir_data_is_aqu(data_type));
        if (cpl_error_get_code()) {
            ERR_TO_WARN("%scutout failed", pfx);
            oi->cutout = UNUSED_ORDER;
            continue;
        }

        MSG_INFO("%scutout success", pfx);
        MSG_DBG("extents mapping:\t[raw] =>\t[relative (to cutout)]");
        MSG_DBG("=======================================================");
        MSG_DBG(" cutout:\t\t[%d;%d] =>\t[%d;%d]", lcol, rcol,
                  lcol - lcol + 1, rcol - lcol + 1);

        /* find matching apfile lines for this order */
        oi->aplist = visir_aplist_new();
        int const ncol = rcol - lcol + 1;
        FOR_EACH_T(visir_apdefs const * const aps, aplist->o) {
            bool matched = false;
            int const l = visir_norm_coord(rev, -0.0, lcol, rcol, aps);
            int const r = visir_norm_coord(rev, +0.0, lcol, rcol, aps);
            if (1 <= l && l <= ncol && 1 <= r && r <= ncol) {
                matched = true;
                visir_aplist_push_back(oi->aplist, aps);  // shallow copy
            }
            MSG_DBG("%capfile entry %d:\t[%d;%d] =>\t[%d;%d]",
                matched ? '*' : ' ', aps->ident, aps->limits[0].l,
                aps->limits[0].r, l, r);
        }

        /* form the list of extraction/match combos to iterate over later */
        cxsize const nmatches = visir_aplist_size(oi->aplist);
        if (nmatches > 0) {
            size_t j = 0;
            oi->matches = matchlist_new(nmatches);
            FOR_EACH_T(visir_apdefs * const aps, oi->aplist) {
                cpl_propertylist * const pl = cpl_propertylist_new();
                oi->matches->data[j++] = (match){ oi->cutout, 0, 0, pl, aps };
            }
        } else {
            /* extraction proceeds using defaults even if no apfile line
             * matches, so still need space for at least one result */
            int const ident = -(i+1);  // neg ident: use old extract alg
            visir_apdefs * const aps = visir_apdefs_new(1, ident, 'O', 0);
            aps->limits[0] = (visir_aplimits){ lcol, rcol };
            cpl_propertylist * const pl = cpl_propertylist_new();
            oi->matches = matchlist_new(1);
            oi->matches->data[0] = (match){ oi->cutout, 0, 0, pl, aps };
        }

        // for each extraction requested (potentially >1 if multiple apfile
        // lines matched); this loop sets 'extract' to 99 upon failure, over-
        // writing the order offset (0, -1, +1, -2, etc...)
        for (size_t j = 0; j < oi->matches->sz; ++j) {
            match * const mj = &oi->matches->data[j];  // alias

            cpl_propertylist_update_int _(mj->qclist, "ESO DRS APDEF",
                                          mj->aps->ident);

            if (visir_aplist_size(aplist->o) && nmatches < 1) {
                MSG_WARN("%sextraction %ld: no apdefs matched", pfx, j);
                mj->extract = UNUSED_ORDER;
                /* skip this order if no aperture defintions provided */
                continue;
            }

            spc_config.extract = j;
            visir_spc_extract_wcal(oi->comnarrow, imhcyclenarrow->o,  // no_
                lcol, rcol, wlen, slitw, temp, fwhm, resol, &spc_config,
                spc_cal_lines, spc_cal_qeff, visir_data_is_aqu(data_type),
                mj->aps, ncomb, rev, &mj->spc_tbl, &mj->weight2d, mj->qclist);
            if (cpl_error_get_code()) {
                ERR_TO_WARN("%sextraction %ld: wlen calib failed", pfx, j);
                // spot #1 where EMISSIVITY (not part of the product) is removed
                // use the X() macro in the line below to reset any resulting
                // CPL error since visir_spc_extract_wcal might've failed before
                // adding the column
                cpl_table_erase_column X(mj->spc_tbl, "SPC_EMISSIVITY");
                mj->extract = UNUSED_ORDER;
                continue;
            }
            MSG_INFO("%sextraction %ld: success", pfx, j);

            visir_spectro_qc _(mj->qclist, paflist->o, drop_wcs, raws->o, NULL,
                               "^(" VISIR_PFITS_REGEXP_SPC_WCAL_PAF ")$");

            if (star_cat) {
                if (!(input_mode & IS_PHOT))
                    MSG_INFO("The frame set contains observation frame(s) "
                             "(%s) together with a standard star catalogue ("
                             VISIR_CALIB_STDSTAR_SPC "), attempting to "
                             "perform a photometric calibration", rawtag);

                // spot #2 where EMISSIVITY (not part of the product) is removed
                visir_spc_phot_sensit(raws->o, &spc_config, pl, star_cat, // no_
                                      &mj->weight2d, mj->qclist, mj->spc_tbl,
                                      resol, dit_key);
                if (cpl_error_get_code()) {
                    // photometry failed
                    if (input_mode & IS_PHOT) return ERR();
                    else ERR_TO_WARN(
                            "The frame set contains observation frame(s) (%s) "
                            "together with a standard star catalogue ("
                            VISIR_CALIB_STDSTAR_SPC ") but the photometric "
                            "calibration failed", rawtag);
                } else {
                    input_mode |= IS_PHOT;
                }
            } else {
                // spot #3 where EMISSIVITY (not part of the product) is removed
                cpl_table_erase_column _(mj->spc_tbl, "SPC_EMISSIVITY");
            }
        }
    }
   
    /* Summarise the relevant QC headers from multiple orders
     * into the primary header */
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC BACKGD SIGMA");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT FWHM");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT FWHM_ERR");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT PEAK");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC GAUSSFIT PEAK_ERR");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC XFWHM");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS MEDIAN");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS MEAN");
    visir_spc_qc_summarise _(phu->o, orders->o, "ESO QC SENS STDEV");

    /* PRO.CATG */
    cpl_propertylist_append_string _(paflist->o, CPL_DFS_PRO_CATG,
                                     VISIR_SPC_OBS_COMBINED_PROCATG);

    char const * const procat = input_mode & IS_PHOT ? (input_mode & IS_ECH ?
        VISIR_SPC_PHOT_ECH_TAB_PROCATG : VISIR_SPC_PHOT_TAB_PROCATG) : (
        input_mode & IS_ECH ? VISIR_SPC_OBS_ECH_TAB_PROCATG :
        VISIR_SPC_OBS_TAB_PROCATG);

    bool first = true;
    // loop only executes once for non-echelle
    for (size_t i = 0; i < orders->o->sz; ++i) {
        order const * const oi = &orders->o->data[i];  // alias
        if (oi->cutout == UNUSED_ORDER) continue;  // skip cutout failures

        char pfx[80] = "";
        if (input_mode & IS_ECH) snprintf(pfx, sizeof(pfx), "Echelle order %2d: "
                "offset %s: ", oi->cutout + visir_spc_optmod_get_echelle_order(
                &ins_settings), pn(oi->cutout));
        for (size_t j = 0; j < oi->matches->sz; ++j) {
            match const * const mj = &oi->matches->data[j];  // alias
            if (mj->extract == UNUSED_ORDER) continue;  // skip extract failures

            MSG_INFO("%sextraction %ld: saving", pfx, j);
            if (first) {
                first = false;
                visir_spc_obs_save _(framelist, parlist, phu->o, mj->qclist,
                                     paflist->o, mj->weight2d, mj->spc_tbl,
                                     oi->comnarrow, procat, input_mode,
                                     oi->cutout, j);
            } else {
                visir_spc_obs_extend _(mj->qclist, mj->weight2d, mj->spc_tbl,
                                       oi->comnarrow, input_mode, oi->cutout,
                                       j);
            }
        }
    }

    if (!first) {
        visir_spc_obs_extend_combined _(combined->o);
    }

    return CLEANUP();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    The stub required by the the cpl_plugin infrastructure
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_old_spc_obs(cpl_frameset * framelist,
                             const cpl_parameterlist * parlist)
{
   visir_old_spc_obs_(framelist, parlist);
   return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Save the visir_spc_obs recipe products on disk
  @param    set         The input frame set
  @param    parlist     The input list of parameters
  @param    qclist      List of QC parameters
  @param    paflist     List of PAF parameters
  @param    combined    The combined image produced
  @param    weight2d    2D weights image
  @param    table       The spectral fields
  @param    tab_procatg Table procatg
  @param    com_procatg Combined procatg
  @param    wgt_procatg Weight procatg
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_obs_save(cpl_frameset      * set,
                                         const cpl_parameterlist * parlist,
                                         cpl_propertylist  * phu_qclist,
                                         cpl_propertylist  * qclist,
                                         const cpl_propertylist  * paflist,
                                         const cpl_image   * weight2d,
                                         const cpl_table   * table,
                                         const cpl_image   * comnarrow,
                                         const char        * tab_procatg,
                                         const int input_mode,
                                         const int ord, const size_t extract)
{
    char offset[8] = "";
    char msgbuf[32];
    bug_if (0);

    if (input_mode & IS_ECH) snprintf(offset, sizeof(offset), "%s:", pn(ord));

    /* THE TABLE */
    cpl_propertylist * plist = cpl_propertylist_new();
    snprintf(msgbuf, sizeof(msgbuf), "TAB_SPECTRUM_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    cpl_propertylist_append(plist, qclist);
    // Put current qclist into plist instead, and then have the primary qclist
    // as the new qclist.
    skip_if (irplib_dfs_save_table(set, parlist, set, table, plist,
                                   RECIPE_SAVE_STRING, tab_procatg,
                                   phu_qclist, NULL, visir_pipe_id,
                                   RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS));
    cpl_propertylist_delete(plist);

    plist = cpl_propertylist_new();
    /* save in single file with extensions for consistency with imaging */
    snprintf(msgbuf, sizeof(msgbuf), "IMG_WEIGHT_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    skip_if(cpl_image_save(weight2d, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
                           CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
    snprintf(msgbuf, sizeof(msgbuf), "IMG_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    skip_if(cpl_image_save(comnarrow, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
                           CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
    cpl_propertylist_delete(plist);

#ifdef VISIR_SAVE_PAF
    /* THE PAF FILE FOR QC PARAMETERS */

    skip_if (cpl_dfs_save_paf("VISIR", RECIPE_SAVE_STRING, paflist,
                             RECIPE_SAVE_STRING CPL_DFS_PAF));
#else
    bug_if(paflist == NULL);
#endif

    end_skip;

    return cpl_error_get_code();

}

/*----------------------------------------------------------------------------*/
/**
  @brief    Save additional visir_spc_obs recipe products on disk
  @param    qclist      List of QC parameters
  @param    combined    The combined image produced
  @param    weight2d    2D weights image
  @param    table       The spectral fields
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_obs_extend(cpl_propertylist      * qclist,
                                         const cpl_image         * weight2d,
                                         const cpl_table         * table,
                                         const cpl_image         * comnarrow,
                                         const int input_mode,
                                         const int ord, const size_t extract)
{
    char offset[8] = "";
    char msgbuf[32];
    bug_if (0);

    if (input_mode & IS_ECH) snprintf(offset, sizeof(offset), "%s:", pn(ord));

    /* THE TABLE */
    cpl_propertylist * plist = cpl_propertylist_duplicate(qclist);
    snprintf(msgbuf, sizeof(msgbuf), "TAB_SPECTRUM_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    skip_if (cpl_table_save(table, NULL, plist, RECIPE_SAVE_STRING "_tab"
                            CPL_DFS_FITS, CPL_IO_EXTEND));
    cpl_propertylist_delete(plist);

    plist = cpl_propertylist_new();
    snprintf(msgbuf, sizeof(msgbuf), "IMG_WEIGHT_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    skip_if(cpl_image_save(weight2d, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
                           CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
    snprintf(msgbuf, sizeof(msgbuf), "IMG_%s%ld", offset, extract);
    cpl_propertylist_update_string(plist, "EXTNAME", msgbuf);
    skip_if(cpl_image_save(comnarrow, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
                           CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
    cpl_propertylist_delete(plist);

    end_skip;

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Save additional visir_spc_obs combined image product on disk
  @param    combined    The combined image produced
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_obs_extend_combined(const cpl_image * combined)
{
    bug_if (0);

    cpl_propertylist * plist = cpl_propertylist_new();
    cpl_propertylist_update_string(plist, "EXTNAME", "IMG_COMBINED");
    skip_if(cpl_image_save(combined, RECIPE_SAVE_STRING "_tab" CPL_DFS_FITS,
                           CPL_TYPE_FLOAT, plist, CPL_IO_EXTEND));
    cpl_propertylist_delete(plist);

    end_skip;

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Summarise a list of per-extension QC parameters by averaging into
            the primary list of QC parameters
  @param    phu_qclist  The primary list of QC parameters
  @param    qclist      Array of per-extension QC parameters
  @param    size        The size of the qclist array
  @param    key         The QC parameter to average
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_spc_qc_summarise(cpl_propertylist * phu_qclist,
                                             orderlist * orders,
                                             const char * key)
{
    double sum = 0.0;
    int count = 0;
    for (size_t i = 0; i < orders->sz; ++i) {
        order const * const oi = &orders->data[i];  // alias
        if (oi->cutout == UNUSED_ORDER) continue;  // skip cutout failures

        for (size_t j = 0; oi->matches && j < oi->matches->sz; ++j) {
            match const * const mj = &oi->matches->data[j];  // alias
            if (mj->extract == UNUSED_ORDER) continue;  // skip extract failures
            
            double const val = cpl_propertylist_get_double(mj->qclist, key);
            if (cpl_error_get_code()) cpl_error_reset();
            else {
                count++;
                sum += val;
                break;  // only need one from this order (they're all the same)
            }
        }
    }
    if (count > 0) {
        cpl_propertylist_append_double(phu_qclist, key, sum / count);
        return CPL_ERROR_NONE;
    }
    return CPL_ERROR_DATA_NOT_FOUND;
}

