/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*---------------------------------------------------------------------------*
 *                             Includes                                      *
 *---------------------------------------------------------------------------*/
#include <string.h>
#include <math.h>

#include <muse.h>
#include "muse_create_sky_z.h"

/*---------------------------------------------------------------------------*
 *                             Functions code                                *
 *---------------------------------------------------------------------------*/
static muse_pixtable *
muse_create_sky_load_pixtable(muse_processing *aProcessing,
                              muse_create_sky_params_t *aParams)
{
   muse_pixtable *pixtable = NULL;

  /* sort input pixel tables into different exposures */
  cpl_table *exposures = muse_processing_sort_exposures(aProcessing);
  if (!exposures) {
    cpl_msg_error(__func__, "no science exposures found in input");
    return NULL;
  }
  int nexposures = cpl_table_get_nrow(exposures);
  if (nexposures != 1) {
    cpl_msg_error(__func__, "More than one exposure (%i) in sky creation",
                  nexposures);
  }

  /* now process all the pixel tables, do it separately for each exposure */
  int i;
  for (i = 0; i < nexposures; i++) {
    cpl_table *thisexp = cpl_table_extract(exposures, i, 1);
    muse_pixtable *p = muse_pixtable_load_merge_channels(thisexp,
                                                         aParams->lambdamin,
                                                         aParams->lambdamax);
    cpl_table_delete(thisexp);
    /* erase pre-existing QC parameters */
    cpl_propertylist_erase_regexp(p->header, "ESO QC ", 0);
    if (pixtable == NULL) {
      pixtable = p;
    } else {
      cpl_table_insert(pixtable->table, p->table,
                       cpl_table_get_nrow(pixtable->table));
      muse_pixtable_delete(p);
    }
  }
  cpl_table_delete(exposures);

  if ((pixtable != NULL) && (muse_pixtable_is_skysub(pixtable) == CPL_TRUE)) {
    cpl_msg_error(__func__, "Pixel table already sky subtracted");
    muse_pixtable_delete(pixtable);
    pixtable = NULL;
  }

  cpl_table *response = muse_table_load(aProcessing, MUSE_TAG_STD_RESPONSE, 0);
  cpl_table *extinction = muse_table_load(aProcessing, MUSE_TAG_EXTINCT_TABLE, 0);
  cpl_table *telluric = muse_table_load(aProcessing, MUSE_TAG_STD_TELLURIC, 0);

  if ((pixtable != NULL) && (response != NULL)) {
    if (muse_pixtable_is_fluxcal(pixtable) == CPL_TRUE) {
      cpl_msg_error(__func__,
                    "Pixel table already flux calibrated. Dont specify %s, %s, %s",
                    MUSE_TAG_STD_RESPONSE, MUSE_TAG_EXTINCT_TABLE,
                    MUSE_TAG_STD_TELLURIC);
      muse_pixtable_delete(pixtable);
      pixtable = NULL;
    } else {
      cpl_error_code rc = muse_flux_calibrate(pixtable, response, extinction, telluric);
      if (rc != CPL_ERROR_NONE) {
        cpl_msg_error(__func__, "while muse_flux_calibrate");
        muse_pixtable_delete(pixtable);
        pixtable = NULL;
      }
    }
  }

  cpl_table_delete(response);
  cpl_table_delete(extinction);
  cpl_table_delete(telluric);

  if ((pixtable != NULL) && (muse_pixtable_is_fluxcal(pixtable) != CPL_TRUE)) {
    cpl_msg_error(__func__, "Pixel table not flux calibrated, cannot create "
                  "sky!");
    muse_pixtable_delete(pixtable);
    pixtable = NULL;
  }

  if (pixtable != NULL) {
    cpl_table_and_selected_int(pixtable->table, MUSE_PIXTABLE_DQ,
                               CPL_NOT_EQUAL_TO, EURO3D_GOODPIXEL);
    cpl_table_erase_selected(pixtable->table);
  }

  /* do DAR correction for WFM data */
  if (muse_pfits_get_mode(pixtable->header) <= MUSE_MODE_WFM_AO_N) {
    cpl_msg_debug(__func__, "WFM detected: starting DAR correction");
    cpl_error_code rc = muse_dar_correct(pixtable, aParams->lambdaref);
    cpl_msg_debug(__func__, "DAR correction returned rc=%d: %s", rc,
                  rc != CPL_ERROR_NONE ? cpl_error_get_message() : "");
  } /* if WFM */

  return pixtable;
}

static muse_sky_master *
muse_create_sky_compute_master(cpl_table *spectrum, muse_processing *aProcessing)
{
  cpl_array *lambda = muse_cpltable_extract_column(spectrum, "lambda");
  double lambda_low = cpl_array_get_min(lambda);
  double lambda_high = cpl_array_get_max(lambda);
  cpl_table *sky_lines = muse_sky_lines_load(aProcessing);
  if (sky_lines != NULL) {
    muse_sky_lines_set_range(sky_lines, lambda_low-5, lambda_high+5);
  }

  cpl_array *data = muse_cpltable_extract_column(spectrum, "data");
  cpl_array *stat = muse_cpltable_extract_column(spectrum, "stat");
  // do the fit, ignoring possible errors
  cpl_errorstate prestate = cpl_errorstate_get();
  muse_sky_master *res = muse_sky_master_fit(lambda, data, stat, sky_lines);
  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_errorstate_dump(prestate, CPL_FALSE, NULL);
    cpl_errorstate_set(prestate);
  }

  cpl_table_delete(sky_lines);
  cpl_array_unwrap(lambda);
  cpl_array_unwrap(data);
  cpl_array_unwrap(stat);
  cpl_table_delete(spectrum);

  return res;
}

static void
muse_create_sky_save_master(cpl_propertylist *aHeader,
                            const muse_sky_master *aMaster,
                            muse_processing *aProcessing)
{
  cpl_frame *frame
    = muse_processing_new_frame(aProcessing, -1, aHeader,
                                "SKY_LINES", CPL_FRAME_TYPE_TABLE);
  if (frame != NULL) {
    const char *filename = cpl_frame_get_filename(frame);
    int r;
    r = cpl_propertylist_save(aHeader, filename, CPL_IO_CREATE);
    r = muse_cpltable_append_file(aMaster->lines, filename,
                                  "LINES", muse_sky_lines_lines_def);
    if (r == CPL_ERROR_NONE) {
      cpl_frameset_insert(aProcessing->outframes, frame);
    } else {
      cpl_frame_delete(frame);
    }
  }
  frame = muse_processing_new_frame(aProcessing, -1, aHeader,
                                    "SKY_CONTINUUM", CPL_FRAME_TYPE_TABLE);
  if (frame != NULL) {
    const char *filename = cpl_frame_get_filename(frame);
    int r;
    r = cpl_propertylist_save(aHeader, filename, CPL_IO_CREATE);
    r = muse_cpltable_append_file(aMaster->continuum, filename,
                                  "CONTINUUM", muse_fluxspectrum_def);
    if (r == CPL_ERROR_NONE) {
      cpl_frameset_insert(aProcessing->outframes, frame);
    } else {
      cpl_frame_delete(frame);
    }
  }
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Create a header with the QC parameters for the sky.
  @param    aMaster Computed sky parameters
  @return   Header with the QC parameters for the sky.
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
muse_create_sky_qc_sky(cpl_propertylist *aHeader,
                       const muse_sky_master *aMaster)
{
  cpl_ensure_code(aHeader && aMaster, CPL_ERROR_NULL_INPUT);

  cpl_ensure_code(aMaster->lines && cpl_table_get_nrow(aMaster->lines) > 0,
                  CPL_ERROR_DATA_NOT_FOUND);
  char keyword[KEYWORD_LENGTH];
  int i, ngroups = cpl_table_get_column_max(aMaster->lines, "group") + 1;
  for (i = 0; i < ngroups; i++) {
    cpl_table_unselect_all(aMaster->lines);
    cpl_table_or_selected_int(aMaster->lines, "group", CPL_EQUAL_TO, i);
    cpl_table *gtable = cpl_table_extract_selected(aMaster->lines);
    cpl_size row;
    cpl_table_get_column_maxpos(gtable, "flux", &row);
    const char *name = cpl_table_get_string(gtable, "name", row);
    double wavelength = cpl_table_get_double(gtable, "lambda", row, NULL),
           flux = cpl_table_get_double(gtable, "flux", row, NULL);
    snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY LINE%i NAME", i+1);
    cpl_propertylist_append_string(aHeader, keyword, name);
    snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY LINE%i AWAV", i+1);
    cpl_propertylist_append_double(aHeader, keyword, wavelength);
    snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY LINE%i FLUX", i+1);
    if (!isfinite(flux)) {
      /* add obviously wrong value to the header,  *
       * since CFITSIO chokes on NANs and the like */
      cpl_propertylist_append_double(aHeader, keyword, -9999.999);
      cpl_msg_error(__func__, "Sky-line fit failed for group %d, computed "
                    "flux is infinite!", i+1);
    } else {
      cpl_propertylist_append_double(aHeader, keyword, flux);
    }
    cpl_table_delete(gtable);
    double offset = aMaster->lsf[0]->offset
                  + (aMaster->lsf[0]->refraction - 1) * wavelength;
    snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY LINE%i OFFSET", i+1);
    cpl_propertylist_append_double(aHeader, keyword, offset);
  } /* for ngroups */
  cpl_table_unselect_all(aMaster->lines);

  cpl_ensure_code(aMaster->continuum && cpl_table_get_nrow(aMaster->continuum) > 0,
                  CPL_ERROR_DATA_NOT_FOUND);
  cpl_size row, nrows = cpl_table_get_nrow(aMaster->continuum);
  double flux = 0.0;
  for (row = 0; row < nrows; row++) {
    flux += cpl_table_get_double(aMaster->continuum, "flux", row, NULL);
  }
  snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY CONT FLUX");
  if (!isfinite(flux)) {
    /* again, workaround for weird CFITSIO error */
    cpl_propertylist_append_double(aHeader, keyword, -9999.999);
    cpl_msg_error(__func__, "Sky-continuum contains infinite values, fit may "
                  "have failed!");
  } else {
    cpl_propertylist_append_double(aHeader, keyword, flux);
  }
  double maxdev = 0.0,
         prev = cpl_table_get_double(aMaster->continuum, "flux", 0, NULL),
         l_prev = cpl_table_get_double(aMaster->continuum, "lambda", 0, NULL);
  for (row = 1; row < nrows; row++) {
    double cur = cpl_table_get_double(aMaster->continuum, "flux", row, NULL),
           l_cur = cpl_table_get_double(aMaster->continuum, "lambda", row, NULL),
           dev = fabs((cur - prev)/ (l_cur - l_prev));
    if (maxdev < dev) {
      maxdev = dev;
    }
    prev = cur;
    l_prev = l_cur;
  } /* for row (continuum table rows) */
  snprintf(keyword, KEYWORD_LENGTH, "ESO QC SKY CONT MAXDEV");
  cpl_propertylist_append_double(aHeader, keyword, maxdev);

  return CPL_ERROR_NONE;
} /* muse_create_sky_qc_sky() */

/*----------------------------------------------------------------------------*/
/**
  @brief    Create a white-light image of the field of view.
  @param    aPixtable   Pixel table with the exposure.
  @param    aCr         Detect cosmic rays, flag them in the pixel table, and
                        ignore them in the whitelight image.
  @return   A muse_image * on success or NULL on error.
 */
/*----------------------------------------------------------------------------*/
static muse_image *
muse_create_sky_whitelight_image(muse_pixtable *aPixtable, int aCr)
{
  cpl_boolean usegrid = getenv("MUSE_COLLAPSE_PIXTABLE")
                      && atoi(getenv("MUSE_COLLAPSE_PIXTABLE")) > 0;
  muse_resampling_type type = usegrid ? MUSE_RESAMPLE_NONE
                                      : MUSE_RESAMPLE_WEIGHTED_DRIZZLE;
  muse_resampling_params *params = muse_resampling_params_new(type);
  if (aCr) {
    params->crtype = MUSE_RESAMPLING_CRSTATS_MEDIAN;
    params->crsigma = 15.;
  }
  muse_pixgrid *grid = NULL;
  muse_datacube *cube = muse_resampling_cube(aPixtable, params,
                                             usegrid ? &grid : NULL);
  if (cube == NULL) {
    cpl_msg_error(__func__, "Could not create cube for whitelight image");
    muse_resampling_params_delete(params);
    muse_pixgrid_delete(grid);
    return NULL;
  }
  muse_image *image = NULL;
  cpl_table *fwhite = muse_table_load_filter(NULL, "white");
  if (usegrid) {
    params->method = MUSE_RESAMPLE_WEIGHTED_DRIZZLE;
    image = muse_resampling_collapse_pixgrid(aPixtable, grid,
                                             cube, fwhite, params);
  } else {
    image = muse_datacube_collapse(cube, fwhite);
  }
  muse_resampling_params_delete(params);
  muse_pixgrid_delete(grid);
  muse_datacube_delete(cube);
  cpl_table_delete(fwhite);
  return image;
} /* muse_create_sky_whitelight_image() */

/*----------------------------------------------------------------------------*/
/**
   @private
   @brief Create a continuum spectrum.
   @param aPixtable Pixel table to take lambda values from.
   @param aMaster Master sky parameters.
   @param aLsf Array with slice specific LSF parameters.

   Replace the continuum spectrum in aMaster with a continuum that is
   calculated from the sky pixel table data (selected rows in the pixel table)
   and the LSF parameters.
 */
/*----------------------------------------------------------------------------*/
static void
muse_create_sky_create_continuum(muse_pixtable *aPixtable,
                                 muse_sky_master *aMaster,
                                 muse_lsf_params **aLsf,
                                 double binwidth) {
  cpl_table_delete(aMaster->continuum);
  aMaster->continuum = NULL;
  muse_sky_subtract_pixtable(aPixtable, aMaster, aLsf);
  aMaster->continuum = muse_resampling_spectrum(aPixtable, binwidth);
  cpl_table_erase_column(aMaster->continuum, "stat");
  cpl_table_erase_column(aMaster->continuum, "dq");
  cpl_table_name_column(aMaster->continuum, "data", "flux");
}

/*----------------------------------------------------------------------------*/
/**
  @brief    Interpret the command line options and execute the data processing
  @param    aProcessing   the processing structure
  @param    aParams       the parameters list
  @return   0 if everything is ok, -1 something went wrong
 */
/*----------------------------------------------------------------------------*/
int
muse_create_sky_compute(muse_processing *aProcessing,
                        muse_create_sky_params_t *aParams)
{
  muse_pixtable *pixtable = muse_create_sky_load_pixtable(aProcessing, aParams);
  if (pixtable == NULL) {
    cpl_msg_error(__func__, "Could not load pixel table");
    return -1;
  }

  /*
     First step: find the regions containing sky:
     - create a whitelight image
     - create the mask from it
     - select pixtable rows containing sky
     - create spectrum from selected rows
  */
  muse_mask *smask = muse_processing_mask_load(aProcessing, MUSE_TAG_SKY_MASK);
  if (smask) { /* apply existing mask on input pixel table */
    cpl_table_select_all(pixtable->table);
    muse_pixtable_and_selected_mask(pixtable, smask);
    muse_mask_delete(smask);
  }

  int cube_cr = (!strncmp(aParams->cr_s, "cube", 5));
  muse_image *whitelight = muse_create_sky_whitelight_image(pixtable, cube_cr);
  if (whitelight == NULL) {
    cpl_msg_error(__func__, "Could not create whitelight image");
    return -1;
  }
  muse_processing_save_image(aProcessing, -1, whitelight, MUSE_TAG_IMAGE_FOV);

  muse_mask *sky_mask = muse_sky_create_skymask(whitelight, aParams->fraction,
                                                "ESO QC SKY");
  muse_processing_save_mask(aProcessing, -1, sky_mask, MUSE_TAG_SKY_MASK);

  cpl_table_select_all(pixtable->table);
  muse_pixtable_and_selected_mask(pixtable, sky_mask);
  muse_image_delete(whitelight);
  muse_mask_delete(sky_mask);

  cpl_table *spectrum = muse_resampling_spectrum(pixtable, aParams->sampling);

  if (!strncmp(aParams->cr_s, "spectrum", 9)) {
    muse_sky_mark_cosmic(spectrum, pixtable);
    cpl_table_delete(spectrum);
    spectrum = muse_resampling_spectrum(pixtable, aParams->sampling);
  }

  if (spectrum == NULL) {
    muse_pixtable_delete(pixtable);
    return -1;
  }
  muse_processing_save_table(aProcessing, -1, spectrum, pixtable->header,
                             MUSE_TAG_SKY_SPECTRUM, MUSE_TABLE_TYPE_CPL);

  /*
    Second step: create master sky, containing line list and
    continuum.  If a continuum was given as calibration frame, we
    ignore the computed one and replace it by the external one.
   */
  cpl_msg_info(__func__, "Creating master sky spectrum using fits to lines "
               "(fluxes) and residual continuum");
  muse_sky_master *master
    = muse_create_sky_compute_master(spectrum, aProcessing);
  if (master == NULL) {
    muse_pixtable_delete(pixtable);
    return -1;
  }

  cpl_errorstate prestate = cpl_errorstate_get();

  cpl_table *continuum = muse_sky_continuum_load(aProcessing);
  muse_lsf_params **lsfParam =
    muse_processing_lsf_params_load(aProcessing, 0);
  if (continuum != NULL) {
    cpl_table_delete(master->continuum);
    master->continuum = continuum;
  } else if (lsfParam != NULL) {
    muse_create_sky_create_continuum(pixtable, master,
                                     lsfParam, aParams->csampling);
  }
  muse_lsf_params_delete(lsfParam);

  cpl_propertylist *qc_header = cpl_propertylist_new();
  muse_create_sky_qc_sky(qc_header, master);

  muse_create_sky_save_master(qc_header, master, aProcessing);

  /* Clean up the local objects. */
  muse_sky_master_delete(master);
  cpl_propertylist_delete(qc_header);
  muse_pixtable_delete(pixtable);
  return cpl_errorstate_is_equal(prestate) ? 0 : -1;
}
