/* $Id: visir_img_dark.c,v 1.28 2011/12/21 14:42:52 llundin Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: llundin $
 * $Date: 2011/12/21 14:42:52 $
 * $Revision: 1.28 $
 * $Name: visir-3_5_1 $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "visir_recipe.h"

/*-----------------------------------------------------------------------------
                            Defines
 -----------------------------------------------------------------------------*/

#define RECIPE_STRING "visir_img_dark"


/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

static cpl_error_code visir_img_dark_reduce(cpl_propertylist *,
                                           const irplib_framelist *,
                                           cpl_image **, cpl_mask **,
                                           cpl_mask **, cpl_mask **);

static cpl_error_code visir_img_dark_save(cpl_frameset *,
                                          const cpl_parameterlist *, 
                                          const cpl_propertylist *,
                                          const cpl_image *, const cpl_mask *,
                                          const cpl_mask *, const cpl_mask *,
                                          int, const irplib_framelist *);

static char * visir_img_dark_make_tag(const cpl_frame*,
                                     const cpl_propertylist *, int);

VISIR_RECIPE_DEFINE(visir_img_dark,
                    VISIR_PARAM_REJBORD |                        
                    VISIR_PARAM_HOT_LIM |                        
                    VISIR_PARAM_COLD_LIM |                       
                    VISIR_PARAM_DEV_LIM |                        
                    VISIR_PARAM_NSAMPLES |                       
                    VISIR_PARAM_HALFSIZE,
                    "Dark recipe",
                    "This recipe computes the dark.\n"
                    "The files listed in the Set Of Frames (sof-file) must be "
                    "tagged either\n"
                    "VISIR-dark-image-raw-file.fits " VISIR_IMG_DARK_RAW " or\n"
                    "VISIR-dark-spectro-raw-file.fits " VISIR_SPC_DARK_RAW "\n"
                    "\n"
                    "The corresponding four products will each have a FITS "
                    "card\n'HIERARCH ESO PRO CATG' with values (for imaging)\n"
                    VISIR_IMG_DARK_AVG_PROCATG   "\n"
                    VISIR_IMG_DARK_HOT_PROCATG   "\n"
                    VISIR_IMG_DARK_COLD_PROCATG  "\n"
                    VISIR_IMG_DARK_DEV_PROCATG   "\n"
                    "  or (for spectroscopy)\n"
                    VISIR_SPC_DARK_AVG_PROCATG   "\n"
                    VISIR_SPC_DARK_HOT_PROCATG   "\n"
                    VISIR_SPC_DARK_COLD_PROCATG  "\n"
                    VISIR_SPC_DARK_DEV_PROCATG   "\n");

/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/

enum _visir_dark_mode_ {
    visir_dark_none = 0,
    visir_dark_img,
    visir_dark_spc
};

typedef enum _visir_dark_mode_ visir_dark_mode;

static struct {
    /* Inputs */
    visir_dark_mode mode;
    int         rej_left;
    int         rej_right;
    int         rej_bottom;
    int         rej_top;
    double      hot_thresh;
    double      cold_thresh;
    double      dev_thresh;
    int         hsize;
    int         nsamples;
} visir_img_dark_config;

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_img_dark   Dark
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/


/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_img_dark(cpl_frameset            * framelist,
                          const cpl_parameterlist * parlist)
{
    irplib_framelist* allframes = NULL;
    irplib_framelist* rawframes = NULL;
    const char     ** taglist   = NULL;
    const char      * rej_bord;
    const char      * orgtag = NULL; /* Avoid (false) uninit warning */
    irplib_framelist* f_one = NULL;
    cpl_imagelist   * i_one = NULL;
    cpl_image       * avg = NULL;
    cpl_mask        * hot = NULL;
    cpl_mask        * cold = NULL;
    cpl_mask        * dev = NULL;
    cpl_propertylist * qclist = cpl_propertylist_new();
    int               nsets;
    int               i;
    int               nb_good = 0;


    /* Retrieve input parameters */
    rej_bord = visir_parameterlist_get_string(parlist, RECIPE_STRING,
                                              VISIR_PARAM_REJBORD);
    skip_if (0);
    skip_if (sscanf(rej_bord, "%d %d %d %d",
                    &visir_img_dark_config.rej_left,
                    &visir_img_dark_config.rej_right,
                    &visir_img_dark_config.rej_bottom,
                    &visir_img_dark_config.rej_top) != 4);

    visir_img_dark_config.hot_thresh =
        visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                       VISIR_PARAM_HOT_LIM);
    visir_img_dark_config.dev_thresh =
        visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                       VISIR_PARAM_DEV_LIM);
    visir_img_dark_config.cold_thresh =
        visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                       VISIR_PARAM_COLD_LIM);
    visir_img_dark_config.hsize =
        visir_parameterlist_get_int(parlist, RECIPE_STRING, VISIR_PARAM_HALFSIZE);
    visir_img_dark_config.nsamples =
        visir_parameterlist_get_int(parlist, RECIPE_STRING, VISIR_PARAM_NSAMPLES);

    skip_if (0);

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));

    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);

    rawframes = irplib_framelist_extract_regexp(allframes,
                                                "^(" VISIR_IMG_DARK_RAW
                                                "|" VISIR_SPC_DARK_RAW  ")$",
                                                CPL_FALSE);
    skip_if(rawframes == NULL);

    /* Verify uniqueness of frame type */
    visir_img_dark_config.mode = visir_dark_none;
    if (cpl_frameset_find(framelist, VISIR_IMG_DARK_RAW)) {
        visir_img_dark_config.mode = visir_dark_img;
        orgtag = VISIR_IMG_DARK_RAW;
    }
    if (cpl_frameset_find(framelist, VISIR_SPC_DARK_RAW)) {
        skip_if (visir_img_dark_config.mode);
        visir_img_dark_config.mode = visir_dark_spc;
        orgtag = VISIR_SPC_DARK_RAW;
    }

    bug_if(visir_img_dark_config.mode == visir_dark_none);

    skip_if(irplib_framelist_load_propertylist_all(rawframes, 0, "^(" 
                                                   VISIR_PFITS_REGEXP_DARK "|"
                                                   VISIR_PFITS_REGEXP_DARK_PAF
                                                   ")$", CPL_FALSE));

    skip_if(visir_dfs_check_framelist_tag(rawframes));

    taglist = visir_framelist_set_tag(rawframes, visir_img_dark_make_tag, &nsets);
    skip_if(taglist == NULL);

    cpl_msg_info(cpl_func, "Identified %d setting(s) in %d frames",
                 nsets, irplib_framelist_get_size(rawframes));

    /* Extract settings and reduce each of them */
    for (i=0 ; i < nsets ; i++) {

        /* Reduce data set nb i */
        f_one = irplib_framelist_extract(rawframes, taglist[i]);

        /* Reset the tag */
        skip_if(irplib_framelist_set_tag_all(f_one, orgtag));

        cpl_msg_info(cpl_func, "Reducing frame set %d of %d (size=%d) with "
                     "setting: %s", i+1, nsets,
                     irplib_framelist_get_size(f_one), taglist[i]);

        skip_if (f_one == NULL);
        
        /* At least 2 frames required */
        if (irplib_framelist_get_size(f_one) < 2) {
            cpl_msg_warning(cpl_func, "Setting %d skipped (Need at least 2 "
                            "frames)", i+1);
            irplib_framelist_delete(f_one);
            f_one = NULL;
            continue;
        }

        skip_if(visir_img_dark_reduce(qclist, f_one, &avg, &hot, &cold, &dev));

        /* Save the products */
        skip_if (visir_img_dark_save(framelist, parlist, qclist, avg, hot,
                                     cold, dev, i+1, f_one));

        nb_good++;

        cpl_image_delete(avg);
        cpl_mask_delete(hot);
        cpl_mask_delete(cold);
        cpl_mask_delete(dev);
        irplib_framelist_delete(f_one);
        cpl_propertylist_empty(qclist);
        avg   = NULL;
        cold  = NULL;
        hot   = NULL;
        dev   = NULL;
        f_one = NULL;
    }

    skip_if (nb_good == 0);

    end_skip;

    cpl_imagelist_delete(i_one);
    cpl_free(taglist);
    cpl_image_delete(avg);
    cpl_mask_delete(hot);
    cpl_mask_delete(cold);
    cpl_mask_delete(dev);
    irplib_framelist_delete(f_one);
    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    cpl_propertylist_delete(qclist);

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction of frames with one setting
  @param    qclist    List of QC parameters
  @param    f_one     the list of frames with one setting
  @param    pavg       the averaged image
  @param    phot       the hot pixel mask
  @param    pcol       the cold pixel mask
  @param    pdev       the deviant pixel mask
  @return   0 iff everything is ok
  @note On error the modifiable images/masks may have been allocated.

 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_img_dark_reduce(cpl_propertylist * qclist,
                                           const irplib_framelist * f_one,
                                           cpl_image ** pavg, cpl_mask ** phot,
                                           cpl_mask ** pcold, cpl_mask ** pdev)
{

    cpl_image * dark = NULL;
    cpl_image * diff  = NULL;
    char      * ron_key = NULL;
    double      rms;
    double      lower, upper;
    double      dark_med;
    double      mean;
    int         ndevpix;
    cpl_size    zone[4];
    int         coldpix_nb;
    int         hotpix_nb;
    int nfiles;
    int i;

    skip_if (f_one == NULL);

    nfiles = irplib_framelist_get_size(f_one);

    skip_if (nfiles < 2);

    skip_if (irplib_framelist_contains(f_one, "NAXIS1",
                                       CPL_TYPE_INT, CPL_TRUE, 0.0));
   
    skip_if (irplib_framelist_contains(f_one, "NAXIS2",
                                       CPL_TYPE_INT, CPL_TRUE, 0.0));
 
    for (i=0 ; i < nfiles ; i++) {
        const cpl_frame        * frame = irplib_framelist_get_const(f_one, i);
        const char             * name  = cpl_frame_get_filename(frame);

        cpl_image_delete(diff);
        diff = dark;
        irplib_check(dark = cpl_image_load(name, CPL_TYPE_FLOAT, 0, 0),
                     "Could not load FITS-image from %s", name);

        if (i == 0) {
            const int nx = cpl_image_get_size_x(dark);
            const int ny = cpl_image_get_size_y(dark);

            zone[0] = visir_img_dark_config.rej_left+1;
            zone[1] = nx - visir_img_dark_config.rej_right;
            zone[2] = visir_img_dark_config.rej_bottom+1;
            zone[3] = ny - visir_img_dark_config.rej_top;

            *pavg = cpl_image_duplicate(dark);
            skip_if(*pavg == NULL);
        } else {
            const cpl_propertylist * plist
                = irplib_framelist_get_propertylist_const(f_one, i-1);
            const int    ndit = visir_pfits_get_ndit(plist);
            const char   ron_format[] = "ESO QC RON%d";
            double       ron;

            skip_if(0);

            irplib_ensure(ndit > 0, CPL_ERROR_ILLEGAL_INPUT,
                          VISIR_PFITS_INT_NDIT " must be positive, not %d",
                           ndit);

            skip_if(cpl_image_subtract(diff, dark));

            /* Compute the read-out noise */
            irplib_check(cpl_flux_get_noise_window(diff, zone,
                                                   visir_img_dark_config.hsize,
                                                   visir_img_dark_config.nsamples,
                                                   &rms, NULL),
                         "Cannot compute the RON for difference between images "
                          "%d and %d", i, i+1);

            /* Normalise the RON with NDIT */
            ron = rms * sqrt(ndit/2.0);

            /* Add QC parameters */
            cpl_free(ron_key);
            ron_key = cpl_sprintf(ron_format, i);

            bug_if(ron_key == NULL);

            skip_if(cpl_propertylist_append_double(qclist, ron_key, ron));

            /* Sum up the darks */
            skip_if(cpl_image_add(*pavg, dark));

        }

    }
    cpl_image_delete(dark);
    dark = NULL;

    mean = cpl_image_get_mean(diff);

    /* Use the rms of the difference of the last two images to
       create the deviant pixel map */
    lower = mean - rms * visir_img_dark_config.dev_thresh;
    upper = mean + rms * visir_img_dark_config.dev_thresh;
    cpl_mask_delete(*pdev);
    irplib_check(*pdev = cpl_mask_threshold_image_create(diff, lower, upper),
                 "Cannot compute the deviant pixel map");
    cpl_image_delete(diff);
    diff = NULL;
    
    skip_if (cpl_mask_not(*pdev));
    ndevpix = cpl_mask_count(*pdev);
    skip_if (0);
    
    /* Average it to the master dark */
    skip_if(cpl_image_divide_scalar(*pavg, (double)nfiles));

    /* Compute median-rms of the central part of the dark  */
    dark_med = cpl_image_get_median_window(*pavg, zone[0], zone[2], zone[1],
                                           zone[3]);

    irplib_check (cpl_flux_get_noise_window(*pavg, zone,
                                            visir_img_dark_config.hsize,
                                            visir_img_dark_config.nsamples,
                                            &rms, NULL),
                  "Cannot compute the RON of the master dark");

    lower = dark_med - rms * visir_img_dark_config.cold_thresh;
    upper = dark_med + rms * visir_img_dark_config.hot_thresh;

    /* Create the cold pixel map */
    cpl_mask_delete(*pcold);
    irplib_check(*pcold = cpl_mask_threshold_image_create(*pavg, -FLT_MAX,
                                                          lower),
                 "Cannot compute the cold pixel map");
    coldpix_nb = cpl_mask_count(*pcold);
    skip_if (0);

    /* Create the hot pixel map */
    cpl_mask_delete(*phot);
    irplib_check(*phot = cpl_mask_threshold_image_create(*pavg, upper, DBL_MAX),
                 "Cannot compute the hot pixel map");
    hotpix_nb = cpl_mask_count(*phot);
    skip_if (0);

    /* Add QC parameters */

    skip_if(cpl_propertylist_append_double(qclist, "ESO QC DARKMED", dark_med));
    skip_if(cpl_propertylist_append_int(qclist, "ESO QC NBCOLPIX", coldpix_nb));
    skip_if(cpl_propertylist_append_int(qclist, "ESO QC NBHOTPIX", hotpix_nb));
    skip_if(cpl_propertylist_append_int(qclist, "ESO QC NBDEVPIX", ndevpix));

    end_skip;

    cpl_image_delete(dark);
    cpl_image_delete(diff);
    cpl_free(ron_key);

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Save the dark recipe products on disk
  @param    set_tot the total input frame set to register the products
  @param    parlist     the input list of parameters
  @param    qclist  List of QC parameters
  @param    avg     the average image
  @param    hot     the hot pixels map
  @param    cold    the cold pixels map
  @param    dev     the deviant pixels map (can be NULL)
  @param    set_nb  the current setting number
  @param    f_one   the frame set for the current setting
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_img_dark_save(cpl_frameset            * set_tot,
                                          const cpl_parameterlist * parlist,
                                          const cpl_propertylist  * qclist,
                                          const cpl_image         * avg,
                                          const cpl_mask          * hot,
                                          const cpl_mask          * cold,
                                          const cpl_mask          * dev,
                                          int                       set_nb,
                                          const irplib_framelist  * f_one)
{
    cpl_frameset           * set   = irplib_frameset_cast(f_one);
    const cpl_propertylist * plist
        = irplib_framelist_get_propertylist_const(f_one, 0);
    cpl_propertylist       * paflist = cpl_propertylist_new();
    cpl_image              * image = NULL;
    const char               pafcopy[]  = "^(" VISIR_PFITS_REGEXP_DARK_PAF ")$";
    char                   * filename = NULL;
    const char             * procatg_avg;
    const char             * procatg_dev;
    const char             * procatg_hot;
    const char             * procatg_cold;


    /* This will catch f_one == NULL or plist == NULL */
    skip_if (0);


    switch (visir_img_dark_config.mode) {
    case visir_dark_img:
        procatg_avg  = VISIR_IMG_DARK_AVG_PROCATG;
        procatg_dev  = VISIR_IMG_DARK_DEV_PROCATG;
        procatg_hot  = VISIR_IMG_DARK_HOT_PROCATG;
        procatg_cold = VISIR_IMG_DARK_COLD_PROCATG;
        break;
    case visir_dark_spc:
        procatg_avg  = VISIR_SPC_DARK_AVG_PROCATG;
        procatg_dev  = VISIR_SPC_DARK_DEV_PROCATG;
        procatg_hot  = VISIR_SPC_DARK_HOT_PROCATG;
        procatg_cold = VISIR_SPC_DARK_COLD_PROCATG;
        break;
    default:
        bug_if(1);
    }


    /* Write the average image */
    filename = cpl_sprintf(RECIPE_STRING "_set%02d_avg" CPL_DFS_FITS,
                              set_nb);
    skip_if (irplib_dfs_save_image(set_tot, parlist, set, avg, CPL_BPP_IEEE_FLOAT,
                               RECIPE_STRING, procatg_avg, qclist, NULL,
                               visir_pipe_id, filename));

    /* Write the hotpixel map */
    image = cpl_image_new_from_mask(hot);
    skip_if(0);

    cpl_free(filename);
    filename = cpl_sprintf(RECIPE_STRING "_set%02d_hotpix" CPL_DFS_FITS,
                              set_nb);
    skip_if (irplib_dfs_save_image(set_tot, parlist, set, image, CPL_BPP_32_SIGNED,
                               RECIPE_STRING, procatg_hot, qclist, NULL,
                               visir_pipe_id, filename));
    cpl_image_delete(image);
    image = NULL;

    /* Write the coldpixel map */
    image = cpl_image_new_from_mask(cold);
    skip_if(0);

    cpl_free(filename);
    filename = cpl_sprintf(RECIPE_STRING "_set%02d_coldpix" CPL_DFS_FITS,
                              set_nb);
    skip_if (irplib_dfs_save_image(set_tot, parlist, set, image, CPL_BPP_32_SIGNED,
                               RECIPE_STRING, procatg_cold, qclist, NULL,
                               visir_pipe_id, filename));
    cpl_image_delete(image);
    image = NULL;

    /* Write the deviant pixel map */
    image = cpl_image_new_from_mask(dev);
    skip_if(0);

    cpl_free(filename);
    filename = cpl_sprintf(RECIPE_STRING "_set%02d_devpix" CPL_DFS_FITS,
                              set_nb);
    skip_if (irplib_dfs_save_image(set_tot, parlist, set, image,
                               CPL_BPP_32_SIGNED, RECIPE_STRING,
                               procatg_dev, qclist, NULL, visir_pipe_id,
                               filename));
    cpl_image_delete(image);
    image = NULL;

#ifdef VISIR_SAVE_PAF
    /* THE PAF FILE FOR QC PARAMETERS */

    skip_if (cpl_propertylist_copy_property_regexp(paflist, plist, pafcopy, 0));
    skip_if (cpl_propertylist_append(paflist, qclist));

    /* PRO.CATG */
    bug_if (cpl_propertylist_append_string(paflist, CPL_DFS_PRO_CATG,
                                           procatg_avg));

    cpl_free(filename);
    filename = cpl_sprintf(RECIPE_STRING "_set%02d" CPL_DFS_PAF, set_nb);
    skip_if (cpl_dfs_save_paf("VISIR", RECIPE_STRING, paflist, filename));
#else
    bug_if(paflist == NULL);
    bug_if(plist == NULL);
    bug_if(pafcopy == NULL);
#endif

    end_skip;

    cpl_image_delete(image);
    cpl_frameset_delete(set);
    cpl_propertylist_delete(paflist);
    cpl_free(filename);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Create a string suitable for frame comparison
  @param    self   Frame to create the new tag from
  @param    plist  The propertylist of the frame
  @param    dummy  A non-negative number (required in the API, but unused here)
  @return   Comparison string or NULL on error
  @note The comparison string must be deallocated with cpl_free().

 */
/*----------------------------------------------------------------------------*/
static char * visir_img_dark_make_tag(const cpl_frame* self,
                                      const cpl_propertylist * plist, int dummy)
{

    char       * tag = NULL;
    double       etime;


    skip_if (0);

    skip_if(self  == NULL);
    skip_if(plist == NULL);
    skip_if(dummy < 0); /* Avoid warning of unused variable */

    /* exposure time */
    etime = visir_pfits_get_exptime(plist);
    skip_if(0);

    tag = cpl_sprintf("%.5f", etime);
    bug_if(tag == NULL);

    end_skip;

    if (cpl_error_get_code()) {
        cpl_free(tag);
        tag = NULL;
    }

    return tag;

}

