/* $Id: visir_util_convert_weight.c,v 1.16 2012-05-09 11:34:11 jtaylor Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2011 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jtaylor $
 * $Date: 2012-05-09 11:34:11 $
 * $Revision: 1.16 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "visir_recipe.h"

/* Verify self-consistency of header files by including system files last */
#include <string.h>

/*-----------------------------------------------------------------------------
                                Defines
 -----------------------------------------------------------------------------*/

#define RECIPE_STRING   "visir_util_convert_weight"

#define VISIR_FRAME_WEIGHT   0
#define VISIR_FRAME_ERROR    1
#define VISIR_FRAME_VARIANCE 2


/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

static cpl_error_code visir_util_convert_weight_one(cpl_frameset *,
                                                  irplib_framelist *,
                                                  const cpl_frame *, int,
                                                  const cpl_parameterlist *);

cpl_recipe_define(visir_util_convert_weight, VISIR_BINARY_VERSION,
                  "Lars Lundin", PACKAGE_BUGREPORT, "2011",
                  "Conversion between different weight maps",
                  "The files listed in the Set Of Frames (sof-file) "
                  "must be tagged:\n"
                  "VISIR-weight-map.fits " VISIR_UTIL_WEIGHT_MAP " or\n"
                  "VISIR-error-map.fits "  VISIR_UTIL_ERROR_MAP " or\n"
                  "VISIR-variance-map.fits " VISIR_UTIL_VARIANCE_MAP "\n"
                  "VISIR-bpm-file.fits " VISIR_CALIB_BPM " (optional)\n"
                  "\nThe relation between an error e, a weight w and a "
                  "variance v is:\n"
                  "e = sqrt(v) = 1/sqrt(w).\n"
                  "\n"
                  "Depending on the setting of the boolean recipe parameters "
                  "each input file will cause the following product(s) to be "
                  "created (each with a FITS card 'HIERARCH " CPL_DFS_PRO_CATG
                  "' with the specified value)\n"
                  VISIR_UTIL_ERROR_MAP_PROCATG
                  ": An error map, requires input v >= 0, w > 0\n"
                  VISIR_UTIL_WEIGHT_MAP_PROCATG
                  ": A weight map, requires input e > 0, v > 0\n"
                  VISIR_UTIL_VARIANCE_MAP_PROCATG
                  ": A variance map, requires input e >= 0, w > 0\n"
                  "If for a given input the identical output is specified, "
                  "the data unit(s) may still change according to the provided "
                  "bad pixel map\n"
                  "In a weight map product any bad pixel is set to 0.\n"
                  "In a variance or error map product any bad pixel is set to "
                  "the value specified by the relevant recipe option.");

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_util_convert_weight   Conversion of weights
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    Fill the recipe parameterlist
  @param    self  The parameterlist
  @return   0 iff everything is ok

 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
visir_util_convert_weight_fill_parameterlist(cpl_parameterlist * self)
{
    const char * context = PACKAGE "." RECIPE_STRING;
    cpl_error_code err;

    cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);

    /* Fill the parameters list */

    /* --error */
    err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "error",
                                        CPL_TRUE, NULL, context, "Convert to "
                                        "error");
    cpl_ensure_code(!err, err);


    /* --weight */
    err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "weight",
                                        CPL_FALSE, NULL, context, "Convert to "
                                        "weight. Bad pixels are seto to 0");
    cpl_ensure_code(!err, err);


    /* --variance */
    err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "variance",
                                        CPL_FALSE, NULL, context, "Convert to "
                                        "variance. Bad pixels are seto to 0");
    cpl_ensure_code(!err, err);


    /* --bad_value */
    err = irplib_parameterlist_set_double(self, PACKAGE, RECIPE_STRING,
                                          "bad_value", FLT_MAX, NULL, context,
                                          "The value to use for any bad pixel "
                                          "in an error or variance map");
    cpl_ensure_code(!err, err);

    /* --bad_flag */
    err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
                                        "bad_flag", CPL_TRUE, NULL, context,
                                        "The value used to flag a bad pixel "
                                        "in an input bad pixel map. True: 1. "
                                        "False: 0.");
    cpl_ensure_code(!err, err);

    return CPL_ERROR_NONE;
}



/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    framelist   the frames list
  @param    parlist     the parameters list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_util_convert_weight(cpl_frameset            * framelist,
                                   const cpl_parameterlist * parlist)
{
    cpl_error_code     didfail   = CPL_ERROR_NONE;
    irplib_framelist * allframes = NULL;
    irplib_framelist * rawframes = NULL;
    const cpl_frame  * bpmframe  = NULL;
    int                i, n;
    

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));
    cpl_fits_set_mode(CPL_FITS_START_CACHING);

    /* Objects observation */
    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);
    rawframes = irplib_framelist_extract_regexp(allframes, "^("
                                                VISIR_UTIL_WEIGHT_MAP   "|"
                                                VISIR_UTIL_VARIANCE_MAP "|"
                                                VISIR_UTIL_ERROR_MAP    ")$",
                                                CPL_FALSE);
    skip_if (rawframes == NULL);
    bpmframe = cpl_frameset_find_const(framelist, VISIR_CALIB_BPM);

    any_if("Propagating error");
    
    n = irplib_framelist_get_size(rawframes);
    for (i = 0; i < n; i++) {

        skip_if (visir_util_convert_weight_one(framelist, rawframes, bpmframe,
                                               i, parlist));
        cpl_fits_set_mode(CPL_FITS_RESTART_CACHING);
    }

    error_if(didfail, didfail, "Failed to convert data in %d frame(s)", n);

    end_skip;

    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Process one frame
  @param    framelist   The frameset to append products to
  @param    rawframes   The frames list (will load the propertylist)
  @param    bpmframe    NULL or a bpm frame
  @param    i           The frame to process (0 for first)
  @param    parlist     The parameters list
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.
 */
/*----------------------------------------------------------------------------*/
static
cpl_error_code visir_util_convert_weight_one(cpl_frameset * framelist,
                                             irplib_framelist * rawframes,
                                             const cpl_frame * bpmframe, int i,
                                             const cpl_parameterlist * parlist)
{

    const int          n = irplib_framelist_get_size(rawframes);
    const cpl_boolean  do_w = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                            RECIPE_STRING,
                                                            "weight");
    const cpl_boolean  do_e = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                            RECIPE_STRING,
                                                            "error");
    const cpl_boolean  do_v = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                            RECIPE_STRING,
                                                            "variance");
    const double bad_val = irplib_parameterlist_get_double(parlist, PACKAGE,
                                                           RECIPE_STRING,
                                                           "bad_value");

    const cpl_boolean bad_flag = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                               RECIPE_STRING,
                                                               "bad_flag");

    cpl_errorstate     prestate   = cpl_errorstate_get();
    cpl_frameset     * products   = cpl_frameset_new();
    cpl_frameset     * usedframes = cpl_frameset_new();
    const cpl_frame  * frame      = irplib_framelist_get_const(rawframes, i);
    const char       * tag        = cpl_frame_get_tag(frame);
    const int          frametype  =
        tag != NULL && !strcmp(tag, VISIR_UTIL_WEIGHT_MAP)
        ? VISIR_FRAME_WEIGHT : (tag != NULL && !strcmp(tag, VISIR_UTIL_ERROR_MAP)
                                ? VISIR_FRAME_ERROR : VISIR_FRAME_VARIANCE);
    const char       * filename   = cpl_frame_get_filename(frame);
    const int          next       = cpl_fits_count_extensions(filename);
    cpl_imagelist    * map        = NULL;
    cpl_imagelist    * emap       = NULL;
    cpl_imagelist    * wmap       = NULL;
    cpl_imagelist    * vmap       = NULL;
    const char       * bpmname    = bpmframe ? cpl_frame_get_filename(bpmframe)
        : NULL;
    const int          mext       = bpmname ? cpl_fits_count_extensions(bpmname)
        : -1;
    cpl_image        * ibpm       = NULL;
    cpl_image        * isqr       = NULL;
    cpl_image        * ione       = NULL;
    cpl_image        * idiv       = NULL;
    cpl_mask         * bpm        = NULL;
    cpl_mask         * bpmi       = NULL;
    cpl_propertylist * plist      = NULL;

    char             * eproname   = do_e == CPL_FALSE ? NULL
        : cpl_sprintf(RECIPE_STRING "_%03d_error" CPL_DFS_FITS, 1+i);
    char             * wproname   = do_w == CPL_FALSE ? NULL
        : cpl_sprintf(RECIPE_STRING "_%03d_weight" CPL_DFS_FITS, 1+i);
    char             * vproname   = do_v == CPL_FALSE ? NULL
        : cpl_sprintf(RECIPE_STRING "_%03d_variance" CPL_DFS_FITS, 1+i);
    int                iext;

    cpl_msg_info(cpl_func, "Converting %d Data unit(s) in frame %d/%d",
                 1+next, 1+i, n);

    any_if("Propagating error");

    if (bpmname != NULL) {
        error_if(mext != 1 && mext != next, CPL_ERROR_INCOMPATIBLE_INPUT,
                 "Raw file %s has %d extension(s) <=> %d extension(s) of bpm-"
                 "file %s", filename, next, mext, bpmname);
    }

    for (iext = 0; iext <= next; iext++) {

        /* Do not load the PRO.CATG */
        cpl_propertylist_delete(plist);
        plist = cpl_propertylist_load_regexp(filename, iext, CPL_DFS_PRO_CATG,
                                             CPL_TRUE);
        /* Update the headers before saving */
        visir_dfs_update_header(plist);

        skip_if(plist == NULL);

        if (emap != map) cpl_imagelist_delete(emap);
        emap = NULL;
        if (wmap != map) cpl_imagelist_delete(wmap);
        wmap = NULL;
        if (vmap != map) cpl_imagelist_delete(vmap);
        vmap = NULL;

        cpl_imagelist_delete(map);
        map = cpl_imagelist_load(filename, CPL_TYPE_FLOAT, iext);
        if (!cpl_errorstate_is_equal(prestate)) {
            cpl_errorstate_set(prestate);
        } else {
            /* FIXME: Use cpl_imagelist_power(map, -0.5);
               once DFS10528 is fixed */
            int j;
            const int m = cpl_imagelist_get_size(map);

            if (iext <= mext) {
                /* Get new bad pixel map, if any */
                cpl_image_delete(ibpm);
                ibpm = cpl_image_load(bpmname, CPL_TYPE_UNSPECIFIED, 0, iext);
                cpl_mask_delete(bpm);
                bpm = cpl_mask_threshold_image_create(ibpm, 0.5, DBL_MAX);
                if (!bad_flag) cpl_mask_not(bpm);
                skip_if(bpm == NULL);
            }

            for (j = 0; j < m; j++) {
                cpl_image * imap = cpl_imagelist_get(map, j);

                /* Reject anything non-positive */
                cpl_mask_delete(bpmi);
                bpmi = cpl_mask_threshold_image_create(imap, 0.0, DBL_MAX);
                bug_if(cpl_mask_not(bpmi));

                /* Reject also any a-priori bad pixels */
                if (bpm)
                    skip_if(cpl_mask_or(bpmi, bpm));

                bug_if(cpl_image_reject_from_mask(imap, bpmi));

                /* FIXME: Protect division/power from any bad input pixels */
                bug_if(cpl_image_fill_rejected(imap, 1.0));

                if (frametype == VISIR_FRAME_WEIGHT) {

                    if (do_w) {
                        if (wmap == NULL) wmap = map;
                    }

                    if (do_v || do_e) {
                        /* Create variance map from weight map */

                        if (ione == NULL) {
                            ione = cpl_image_new(cpl_image_get_size_x(imap),
                                                 cpl_image_get_size_y(imap),
                                                 CPL_TYPE_FLOAT);
                            cpl_image_add_scalar(ione, 1.0);

                        }

                        idiv = cpl_image_divide_create(ione, imap);

                        /* Division should not create bad pixels,
                           but just be sure */
                        if (cpl_image_get_bpm_const(idiv))
                            bug_if(cpl_mask_or(bpmi,
                                               cpl_image_get_bpm_const(idiv)));

                        bug_if(cpl_image_reject_from_mask(idiv, bpmi));
                    }

                    if (do_v) {
                        if (vmap == NULL)
                            vmap = do_w ? cpl_imagelist_new() : map;

                        /* Set/replace image in list */
                        bug_if(cpl_imagelist_set(vmap, idiv, j));
                        idiv = NULL;
                    }

                    if (do_e) {
                        if (emap == NULL)
                            emap = do_w || do_v ? cpl_imagelist_new() : map;

                        if (do_v) {
                            idiv = cpl_image_power_create(cpl_imagelist_get_const
                                                          (vmap, j), 0.5);
                        } else {
                            bug_if(cpl_image_power(idiv, 0.5));
                        }

                        /* Set/replace image in list */
                        bug_if(cpl_imagelist_set(emap, idiv, j));
                        idiv = NULL;
                    }
                } else if (frametype == VISIR_FRAME_ERROR) {

                    if (do_e) {
                        if (emap == NULL) emap = map;
                    }

                    if (do_v) {
                        if (vmap == NULL)
                            vmap = do_e ? cpl_imagelist_new() : map;

                        if (do_e) {
                            isqr = cpl_image_power_create(imap, 2.0);

                            /* Set image in list */
                            bug_if(cpl_imagelist_set(vmap, isqr, j));
                            isqr = NULL;
                        } else {
                            bug_if(cpl_image_power(imap, 2.0));
                        }
                    }

                    if (do_w) {
                        if (wmap == NULL)
                            wmap = do_e || do_v ? cpl_imagelist_new() : map;

                        if (ione == NULL) {
                            ione = cpl_image_new(cpl_image_get_size_x(imap),
                                                 cpl_image_get_size_y(imap),
                                                 CPL_TYPE_FLOAT);
                            cpl_image_add_scalar(ione, 1.0);

                        }

                        if (do_v) {
                            idiv = cpl_image_divide_create
                                (ione, cpl_imagelist_get(vmap, j));
                        } else if (do_e) {
                            isqr = cpl_image_power_create(imap, 2.0);
                            idiv = cpl_image_divide_create(ione, isqr);
                        } else {
                            bug_if(cpl_image_power(imap, 2.0));
                            idiv = cpl_image_divide_create(ione, imap);
                        }
                        cpl_image_delete(isqr);
                        isqr = NULL;

                        /* Set image in list */
                        bug_if(cpl_imagelist_set(wmap, idiv, j));
                        idiv = NULL;
                    }

                } else if (frametype == VISIR_FRAME_VARIANCE) {
                    if (do_v) {
                        if (vmap == NULL) vmap = map;
                    }

                    if (do_w) {
                        if (wmap == NULL)
                            wmap = do_v ? cpl_imagelist_new() : map;

                        if (ione == NULL) {
                            ione = cpl_image_new(cpl_image_get_size_x(imap),
                                                 cpl_image_get_size_y(imap),
                                                 CPL_TYPE_FLOAT);
                            cpl_image_add_scalar(ione, 1.0);

                        }

                        idiv = cpl_image_divide_create(ione, imap);

                        /* Division should not create bad pixels,
                           but just be sure */
                        if (cpl_image_get_bpm_const(idiv))
                            bug_if(cpl_mask_or(bpmi,
                                               cpl_image_get_bpm_const(idiv)));

                        bug_if(cpl_image_reject_from_mask(idiv, bpmi));

                        /* Set image in list */
                        bug_if(cpl_imagelist_set(wmap, idiv, j));
                        idiv = NULL;
                    }

                    if (do_e) {
                        if (emap == NULL)
                            emap = do_v || do_w ? cpl_imagelist_new() : map;

                        if (do_v || do_w) {
                            isqr = cpl_image_power_create(imap, 0.5);
                            /* Set/replace image in list */
                            bug_if(cpl_imagelist_set(emap, isqr, j));
                            isqr = NULL;

                        } else {
                            bug_if(cpl_image_power(imap, 0.5));
                        }
                    }
                } else {
                    bug_if(1);
                }

                /* Set/Zero bad pixels */
                if (do_w) {
                    bug_if(cpl_image_fill_rejected(cpl_imagelist_get
                                                   (wmap, j), 0.0));
                }
                if (do_e) {
                    bug_if(cpl_image_fill_rejected(cpl_imagelist_get
                                                   (emap, j), bad_val));
                }
                if (do_v) {
                    /* Set bad pixels  */
                    bug_if(cpl_image_fill_rejected(cpl_imagelist_get
                                                   (vmap, j), bad_val));
                }
            }

        }

        if (iext == 0) {

            bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate
                                       (frame)));
            if (bpmframe != NULL)
                bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate
                                           (bpmframe)));

            if (do_w) {
                if (wmap == NULL) {
                    skip_if(irplib_dfs_save_propertylist(products, parlist,
                                                         usedframes,
                                                         RECIPE_STRING,
                                                         VISIR_UTIL_WEIGHT_MAP_PROCATG,
                                                         plist, NULL,
                                                         visir_pipe_id,
                                                         wproname));
                } else {
                    skip_if(irplib_dfs_save_imagelist(products, parlist,
                                                      usedframes, wmap,
                                                      CPL_BPP_IEEE_FLOAT,
                                                      RECIPE_STRING,
                                                      VISIR_UTIL_WEIGHT_MAP_PROCATG,
                                                      plist, NULL,
                                                      visir_pipe_id, wproname));
                }
            }
            if (do_e) {
                if (emap == NULL) {
                    skip_if(irplib_dfs_save_propertylist(products, parlist,
                                                         usedframes,
                                                         RECIPE_STRING,
                                                         VISIR_UTIL_ERROR_MAP_PROCATG,
                                                         plist, NULL,
                                                         visir_pipe_id,
                                                         eproname));
                } else {
                    skip_if(irplib_dfs_save_imagelist(products, parlist,
                                                      usedframes, emap,
                                                      CPL_BPP_IEEE_FLOAT,
                                                      RECIPE_STRING,
                                                      VISIR_UTIL_ERROR_MAP_PROCATG,
                                                      plist, NULL,
                                                      visir_pipe_id, eproname));
                }
            }
            if (do_v) {
                if (vmap == NULL) {
                    skip_if(irplib_dfs_save_propertylist(products, parlist,
                                                         usedframes,
                                                         RECIPE_STRING,
                                                         VISIR_UTIL_VARIANCE_MAP_PROCATG,
                                                         plist, NULL,
                                                         visir_pipe_id,
                                                         vproname));
                } else {
                    skip_if(irplib_dfs_save_imagelist(products, parlist,
                                                      usedframes, vmap,
                                                      CPL_BPP_IEEE_FLOAT,
                                                      RECIPE_STRING,
                                                      VISIR_UTIL_VARIANCE_MAP_PROCATG,
                                                      plist, NULL,
                                                      visir_pipe_id, vproname));
                }
            }
        } else {
            if (do_w) {
                if (wmap == NULL) {
                    skip_if(cpl_propertylist_save(plist, wproname,
                                                  CPL_IO_EXTEND));
                } else {
                    skip_if(cpl_imagelist_save(wmap, wproname,
                                               CPL_BPP_IEEE_FLOAT, plist,
                                               CPL_IO_EXTEND));
                }
            }
            if (do_e) {
                if (emap == NULL) {
                    skip_if(cpl_propertylist_save(plist, eproname,
                                                  CPL_IO_EXTEND));
                } else {
                    skip_if(cpl_imagelist_save(emap, eproname,
                                               CPL_BPP_IEEE_FLOAT, plist,
                                               CPL_IO_EXTEND));
                }
            }
            if (do_v) {
                if (vmap == NULL) {
                    skip_if(cpl_propertylist_save(plist, vproname,
                                                  CPL_IO_EXTEND));
                } else {
                    skip_if(cpl_imagelist_save(vmap, vproname,
                                               CPL_BPP_IEEE_FLOAT, plist,
                                               CPL_IO_EXTEND));
                }
            }
        }
    }

    FOR_EACH_FRAMESET_C(frm, products) {
        cpl_frame * copy = cpl_frame_duplicate(frm);
        cpl_error_code error = cpl_frameset_insert(framelist, copy);

        if (error) break;
    }
    bug_if(frame != NULL);

    end_skip;

    if (emap != map) cpl_imagelist_delete(emap);
    if (wmap != map) cpl_imagelist_delete(wmap);
    if (vmap != map) cpl_imagelist_delete(vmap);
    cpl_imagelist_delete(map);
    cpl_free(eproname);
    cpl_free(wproname);
    cpl_free(vproname);
    cpl_image_delete(ione);
    cpl_image_delete(idiv);
    cpl_image_delete(isqr);
    cpl_image_delete(ibpm);
    cpl_mask_delete(bpm);
    cpl_mask_delete(bpmi);
    cpl_propertylist_delete(plist);
    cpl_frameset_delete(usedframes);
    cpl_frameset_delete(products);

    return cpl_error_get_code();

}


