/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#define STORE_SLIT_WIDTH
#define STORE_BIN_WIDTH

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#include <math.h>
#include <string.h>
#include <fenv.h>
#include <cpl.h>

#include "muse_lsf.h"

#include "muse_instrument.h"
#include "muse_optimize.h"
#include "muse_pfits.h"
#include "muse_pixtable.h"
#include "muse_quality.h"
#include "muse_resampling.h"
#include "muse_tracing.h"
#include "muse_utils.h"
#include "muse_wavecalib.h"

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_lsf          Line Spread Function related functions
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/**
   @brief Check the pixels for one arc line
   @param aPixtable The pixel table with pixels around the line
   @param aLambda   The arc line reference wavelength

   The basic check is done with a gaussian fit, which also gives the
   total flux for this line. Lines with a low flux, and lines with an
   implausible Gaussian width are completely ignored (the function
   returns 0). Otherwise, pixels that deviate too much from the
   gaussian are deleted from the pixel table.

   Finally, new columns <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt> are added to the pixel table and set with
   the constant values of the aLambda parameter, the calculated total
   line flux, and the calculated continuum background.
 */
static cpl_size
muse_lsf_check_arc_line(cpl_table *aPixtable, double aLambda) {
  cpl_size nrows = cpl_table_get_nrow(aPixtable);
  /* If we have too few data pointe, remove the line */
  if (nrows <= 100) {
    return 0;
  }
  cpl_errorstate prestate = cpl_errorstate_get();

  /* wrap table columns into vectors for the Gaussian fit */
  cpl_vector *pos = cpl_vector_new(nrows);
  cpl_vector *val = cpl_vector_new(nrows);
  cpl_vector *err = cpl_vector_new(nrows);
  double *d_pos = cpl_vector_get_data(pos);
  double *d_val = cpl_vector_get_data(val);
  double *d_err = cpl_vector_get_data(err);
  float *d_lambda = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_LAMBDA);
  float *d_data = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_DATA);
  float *d_stat = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_STAT);
  cpl_size i, s = 1;
  double bglevel = 0;
  for (i = 0; i < nrows; i++) {
    if (fabs(d_lambda[i] - aLambda) > 5.5) {
      bglevel += d_data[i];
      s += 1;
    }
    d_pos[i] = d_lambda[i];
    d_val[i] = d_data[i];
    d_err[i] = sqrt(d_stat[i]);
  }
  bglevel /= s;
  double meansigma = sqrt(cpl_table_get_column_mean(aPixtable, MUSE_PIXTABLE_STAT));
  double sigma, area, mse;
  double xc = aLambda;
  cpl_fit_mode fit_pars = CPL_FIT_STDEV | CPL_FIT_AREA | CPL_FIT_CENTROID;
  cpl_vector_fit_gaussian(pos, NULL, val, NULL, fit_pars, &xc, &sigma,
                          &area, &bglevel, &mse, NULL, NULL);

  /* Subtract the fit */
  for (i = 0; i < nrows; i++) {
    double e = (d_pos[i] - xc)/sigma;
    d_val[i] -= area / (CPL_MATH_SQRT2PI * sigma) * exp(-e*e/2) - bglevel;
  }

  /* Remove all pixels that are too far from the gaussian estimate */
  double maxerror = 0.5 * sqrt(area) + 0.15 * area;
  cpl_size inv = 0;
  cpl_table_unselect_all(aPixtable);
  for (i = 0; i < nrows; i++) {
    if ((d_val[i] < -maxerror) || (d_val[i] > maxerror)) {
      inv++;
      cpl_table_select_row(aPixtable, i);
    }
  }
  cpl_table_erase_selected(aPixtable);
  nrows = cpl_table_get_nrow(aPixtable);
  cpl_vector_delete(pos);
  cpl_vector_delete(val);
  cpl_vector_delete(err);

  cpl_table_fill_column_window_float(aPixtable, "line_lambda",
                                     0, nrows, xc);
  cpl_table_fill_column_window_float(aPixtable, "line_flux",
                                     0, nrows, area);
  cpl_table_fill_column_window_float(aPixtable, "line_background",
                                     0, nrows, bglevel);

  /* If the fit failed, just ignore the errors, and remove the line */
  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_errorstate_set(prestate);
    return 0;
  }
  /* If the flux (compared to noise) is too low, silently remove the line */
  if (area/meansigma < 50) {
    return 0;
  }
  /* If the estimated line width is too far away from our expectation,
     remove the line */
  if ((sigma > 3) || (sigma < 0.7)) {
    uint32_t origin = (uint32_t)cpl_table_get_int(aPixtable,
                                                  MUSE_PIXTABLE_ORIGIN,
                                                  0, NULL);
    int i_ifu = muse_pixtable_origin_get_ifu(origin);
    int i_slice = muse_pixtable_origin_get_slice(origin);
    cpl_msg_debug(__func__, "Slice %2i.%02i: "
                  "Ignoring line %.1f with implausible width %f (flux=%.0f)",
                  i_ifu, i_slice, aLambda, sigma, area);
    return 0;
  }
  return nrows;
}

/**
   @brief Select the pixels for one arc line
   @param aPixtable The pixel table for one slice
   @param aLambda   The arc line reference wavelength
   @param aWindow     Wavelength window around each line
   @return The pixel table with pixels selected by this arc line, or
           NULL if the line could not be found.

   From the pixel table, the pixels around the specified wavelength
   are selected. Then, a simple gaussian fit is made and the pixels
   that are too far from the gaussian are removed as a simple
   rejection of cosmics. Also, the reference wavelength and the
   estimated flux (from the gaussian fit) are added as additional
   columns <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt>

   If no line is found, NULL is returned.

 */
static cpl_table *
select_arc_line(muse_pixtable *aPixtable, double aLambda, double aWindow) {
  cpl_size low = muse_cpltable_find_sorted(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                                           aLambda - aWindow);
  cpl_size high = muse_cpltable_find_sorted(aPixtable->table, MUSE_PIXTABLE_LAMBDA,
                                            aLambda + aWindow);
  cpl_table *sel = cpl_table_extract(aPixtable->table, low + 1, high - low);
  cpl_table_select_all(sel);
  cpl_table_and_selected_int(sel, "dq", CPL_NOT_EQUAL_TO, 0);
  cpl_table_erase_selected(sel);

  cpl_table_new_column(sel, "line_lambda", CPL_TYPE_FLOAT);
  cpl_table_new_column(sel, "line_flux", CPL_TYPE_FLOAT);
  cpl_table_new_column(sel, "line_background", CPL_TYPE_FLOAT);
  
  cpl_size nrows = muse_lsf_check_arc_line(sel, aLambda);

  if (nrows == 0) {
    cpl_table_delete(sel);
    return NULL;
  }
  return sel;
}


/**
   @brief Read images and combine the arc lines into one pixtable.
   @param aImages   List of arc images
   @param aTrace    Table containing the trace solution
   @param aWave     Table containing the wavelength calibration solution
   @param aArcLines ARC reference line table
   @param aQuality  Minimal line quality to select
   @param aWindow   Wavelength window around each line
   @retval Combined pixel table or NULL on error
   @cpl_ensure{aImages, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aTrace, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aWave, CPL_ERROR_NULL_INPUT, NULL}
   @cpl_ensure{aArcLines, CPL_ERROR_NULL_INPUT, NULL}

   Create pixel tables from all images in the aImages list, select the pixels
   around each arc line, and store the resulting pixels in a new pixtable.

   The output pixel table will have additional columns
   <tt>line_lambda</tt>, <tt>line_flux</tt>, and
   <tt>line_background</tt>, containing the reference wavelength, the
   estimated flux, and the background level of the corresponding line.
 */
muse_pixtable *
muse_lsf_create_arcpixtable(muse_imagelist *aImages,
                            cpl_table *aTrace, cpl_table *aWave,
                            cpl_table *aArcLines, int aQuality, double aWindow)
{
  cpl_ensure(aImages, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aTrace, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aWave, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aArcLines, CPL_ERROR_NULL_INPUT, NULL);

  /* create the output pixel table with two extra columns */
  muse_pixtable *pt = cpl_calloc(1, sizeof(muse_pixtable));
  pt->table = muse_cpltable_new(muse_pixtable_def, 0);
  cpl_table_new_column(pt->table, "line_lambda", CPL_TYPE_FLOAT);
  cpl_table_new_column(pt->table, "line_flux", CPL_TYPE_FLOAT);
  cpl_table_new_column(pt->table, "line_background", CPL_TYPE_FLOAT);

  cpl_size i_image, n_images = muse_imagelist_get_size(aImages);
#if 0 /* no extra levels of parallelization in basic processing recipes */
  #pragma omp parallel for default(none)                 /* as req. by Ralf */ \
          shared(i_image, n_images, aImages, aTrace, aWave, aArcLines,         \
                 aQuality, aWindow, pt)
#endif
  for (i_image = 0; i_image < n_images; i_image++) {
    muse_image *image = muse_imagelist_get(aImages, i_image);
    muse_pixtable *pixtable = muse_pixtable_create(image, aTrace, aWave, NULL);
    if (pixtable == NULL) {
      continue;
    }
    /* create header of output pixel table, if it's not there yet */
    if (!pt->header) {
      pt->header = cpl_propertylist_duplicate(pixtable->header);
    }
    char *l = muse_utils_header_get_lamp_names(pixtable->header, '|');
    if (strlen(l) == 0) {
      cpl_msg_warning(__func__,
                      "Ignoring frame without arc lamp switched on");
      cpl_free(l);
      cpl_free(pixtable);
      continue;
    }
    char *lamp = cpl_malloc(strlen(l)+3);
    sprintf(lamp, "|%s|", l);
    cpl_free(l);
    cpl_size i_line;
    cpl_size n_entries = 0;
    muse_pixtable **slice_pixtable = muse_pixtable_extracted_get_slices(pixtable);
    int n_slices = muse_pixtable_extracted_get_size(slice_pixtable);
    for (i_line = 0; i_line < cpl_table_get_nrow(aArcLines); i_line++) {
      if (cpl_table_get_int(aArcLines, "quality", i_line, NULL) < aQuality) {
        continue;
      }
      const char *n = muse_wave_lines_get_lampname(aArcLines, i_line);
      char *name = cpl_malloc(strlen(n) + 3);
      sprintf(name, "|%s|", n);
      if (strstr(name, lamp)) {
        double lambda = cpl_table_get_float(aArcLines, "lambda", i_line, NULL);
        int i_slice;
#if 0 /* no extra levels of parallelization in basic processing recipes */
        #pragma omp parallel for default(none)           /* as req. by Ralf */ \
                shared(i_slice, n_slices, n_entries, lambda, slice_pixtable,   \
                       aWindow, pt)
#endif
        for (i_slice = 0; i_slice < n_slices; i_slice++) {
          cpl_table *sel = select_arc_line(slice_pixtable[i_slice],
                                           lambda, aWindow);
          if (sel != NULL) {
#if 0 /* no parallelization here, not necessary any more */
            #pragma omp critical(construct_pixtable)
#endif
            cpl_table_insert(pt->table, sel, cpl_table_get_nrow(pt->table));
#if 0 /* no parallelization here, not necessary any more */
            #pragma omp atomic
#endif
            n_entries += cpl_table_get_nrow(sel);
            cpl_table_delete(sel);
          }
        }
      }
      cpl_free(name);
    } /* for i_line (arc lines) */
    muse_pixtable_extracted_delete(slice_pixtable);
    muse_pixtable_delete(pixtable);
    cpl_msg_info(__func__, "Using %"CPL_SIZE_FORMAT" entries with lamp %s",
                 n_entries, lamp);
    cpl_free(lamp);
  } /* for i_image (all images in list) */
  return pt;
} /* muse_lsf_create_arcpixtable() */

/**
   @brief Do a polynomial fit on an arc pixtable
   @param aPixtable      special pixel table with arc lines
   @param aOrderLsf      polynomial order in x (LSF) direction
   @param aOrderLambda   polynomial order in y (arc line wavelength) direction
   @return Fitted polynomial, or NULL or errors
   @cpl_ensure{aPixtable["d_lambda"] != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_lambda"] != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["data"] != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_flux"] != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL}
   @cpl_ensure{aPixtable["line_background"] != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL}

   The fit is done in two dimensions: x direction is the pixel wavelength,
   relative to the arc line (column <tt>d_lambda</tt>),
   and y direction is the arc line wavelength (column <tt>line_lambda</tt>).

   The data are normalized by dividing by the according line flux
   (column <tt>line_flux</tt>) and line background (column
   <tt>line_background</tt>)

   If there are not enough data points, or if the fit fails for other
   reasons, the funtion returns NULL, without setting an error.
 */
static cpl_polynomial *
muse_lsf_fit_polynomial(cpl_table *aPixtable, int aOrderLsf, int aOrderLambda)
{
  cpl_errorstate prestate = cpl_errorstate_get();
  cpl_size nrows = cpl_table_get_nrow(aPixtable);
  if (nrows <= (aOrderLsf + 1) * (aOrderLambda)) {
    return NULL;
  }
  double *p_dlambda = cpl_table_get_data_double(aPixtable, "d_lambda");
  cpl_ensure(p_dlambda != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_line = cpl_table_get_data_float(aPixtable, "line_lambda");
  cpl_ensure(p_line != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_data = cpl_table_get_data_float(aPixtable, MUSE_PIXTABLE_DATA);
  cpl_ensure(p_data != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_flux = cpl_table_get_data_float(aPixtable, "line_flux");
  cpl_ensure(p_flux != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);
  float *p_background = cpl_table_get_data_float(aPixtable, "line_background");
  cpl_ensure(p_background != NULL, CPL_ERROR_ILLEGAL_INPUT, NULL);

  cpl_matrix *coord = cpl_matrix_new(2, nrows);
  cpl_vector *data = cpl_vector_new(nrows);
  double *coord_ptr = cpl_matrix_get_data(coord);
  double *data_ptr = cpl_vector_get_data(data);
  cpl_size i_row;

  for (i_row = 0; i_row < nrows; i_row++) {
    coord_ptr[i_row] = p_dlambda[i_row];
    coord_ptr[i_row + nrows] = p_line[i_row];
    data_ptr[i_row] = (p_data[i_row] - p_background[i_row]) / p_flux[i_row];
  }

  cpl_polynomial *p = cpl_polynomial_new(2);
  const cpl_size maxdeg2d[] = { aOrderLsf, aOrderLambda };
  cpl_polynomial_fit(p, coord, NULL, data, NULL, CPL_TRUE, NULL, maxdeg2d);
  cpl_matrix_delete(coord);
  cpl_vector_delete(data);

  if (!cpl_errorstate_is_equal(prestate)) {
    cpl_errorstate_set(prestate);
    return NULL;
  }

  return p;
}

/**
   @brief Compute the LSF for all wavelengths of one slice.
   @param aPixtable              Special pixel table with arc lines.
   @param aLsfImage              Image to store the LSF.
   @param aWCS                   The WCS of the image.
   @param aLsfRegressionWindow   The Regression window in LSF direction.
   @retval CPL_ERROR_NONE Everything went OK
   @cpl_ensure_code{aPixtable, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aLsfImage, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aWCS, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aLsfRegressionWindow > 0, CPL_ERROR_ILLEGAL_INPUT}

   The pixel table is assumed to be restricted to one slice.

   The LSF is computed as a twodimensional regression fit, with the
   wavelength of the pixel in the x (LSF) direction, and the arc line
   reference wavelength in y direction. The regression window in x
   (LSF) direction is taken as parameter; in y (line wavelength)
   direction always the full range is used.

   As fit, a polynomial fit with order 2 in x (LSF) and order 3 in y
   (line wavelength) direction is used. One fit per x value is done.

   After the fit, the according pixels in the image are set to the
   LSF. If a fit did not succeed, the pixels in the according image
   column are set invalid.
 */
cpl_error_code
muse_lsf_fit_slice(const muse_pixtable *aPixtable, cpl_image *aLsfImage,
                   muse_wcs *aWCS, double aLsfRegressionWindow)
{
  cpl_ensure_code(aPixtable && aPixtable->table, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aLsfImage, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aLsfRegressionWindow > 0, CPL_ERROR_ILLEGAL_INPUT);

  cpl_size nrows = muse_pixtable_get_nrow(aPixtable);
  if (nrows == 0) {
    return CPL_ERROR_NONE;
  }
  /* work on a locally duplicated table (since we mess with the columns) */
  cpl_table *pixtable = cpl_table_duplicate(aPixtable->table);
  uint32_t origin = (uint32_t)cpl_table_get_int(pixtable,
                                                MUSE_PIXTABLE_ORIGIN, 0, NULL);
  int i_ifu = muse_pixtable_origin_get_ifu(origin);
  int i_slice = muse_pixtable_origin_get_slice(origin);
  cpl_msg_info(__func__, "processing slice %2i.%02i"
               " with %"CPL_SIZE_FORMAT" entries",
               i_ifu, i_slice, nrows);

  /* Create temporary column with (lmbda - line_lambda) and sort pixtable
     according to this column. Sorting is important since we use it later
     to extract the values according to the regession window using
     find_sorted(). */
  cpl_table_cast_column(pixtable, MUSE_PIXTABLE_LAMBDA, "d_lambda", CPL_TYPE_DOUBLE);
  cpl_table_subtract_columns(pixtable, "d_lambda", "line_lambda");
  cpl_propertylist *order = cpl_propertylist_new();
  cpl_propertylist_append_bool(order, "d_lambda", CPL_FALSE);
  cpl_table_sort(pixtable, order);
  cpl_propertylist_delete(order);

  /* Loop over all x (LSF) data points and create a polynomial for each
     covering the whole wavelength range */
  cpl_size i_lsf;
  cpl_size n_lsf = cpl_image_get_size_x(aLsfImage);
  cpl_size n_lambda = cpl_image_get_size_y(aLsfImage);
  for (i_lsf = 1; i_lsf <= n_lsf; i_lsf++) {
    double x = aWCS->crval1 + (i_lsf - aWCS->crpix1) * aWCS->cd11;
    double x_ref = CPL_MAX(x, cpl_table_get_column_min(pixtable, "d_lambda")
                           + aLsfRegressionWindow);
    x_ref = CPL_MIN(x, cpl_table_get_column_max(pixtable, "d_lambda")
                    - aLsfRegressionWindow);
    cpl_size low = muse_cpltable_find_sorted(pixtable, "d_lambda",
                                             x_ref - aLsfRegressionWindow);
    cpl_size high = muse_cpltable_find_sorted(pixtable, "d_lambda",
                                              x_ref + aLsfRegressionWindow);
    cpl_table *selected = cpl_table_extract(pixtable, low+1, high-low);
    cpl_polynomial *p = muse_lsf_fit_polynomial(selected, 2, 3);
    if (p != NULL) {
      cpl_vector *c = cpl_vector_new(2);
      cpl_vector_set(c, 0, x);
      cpl_size i_lambda;
      /* Fill the image column with the values calculated from the polynomial */
      for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
        double y = aWCS->crval2 + (i_lambda - aWCS->crpix2) * aWCS->cd22;
        cpl_vector_set(c, 1, y);
        double flux = cpl_polynomial_eval(p, c);
        cpl_image_set(aLsfImage, i_lsf, i_lambda, flux);
      }
      cpl_vector_delete(c);
    } else {
      cpl_msg_warning(__func__,
                      "Failed polynomial fit %2i.%02i %.2f;"
                      " %"CPL_SIZE_FORMAT" entries",
                      i_ifu, i_slice, x, cpl_table_get_nrow(selected));
      cpl_size i_lambda;
      for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
        cpl_image_reject(aLsfImage, i_lsf, i_lambda);
      }
    }
    cpl_polynomial_delete(p);
    cpl_table_delete(selected);
  }
  cpl_table_delete(pixtable);
  /* Re-normalize total area of each LSF, since normalization may
     not kept during the polynomial fit */
  cpl_size i_lambda;
  for (i_lambda = 1; i_lambda <= n_lambda; i_lambda++) {
    double line_flux = cpl_image_get_flux_window(aLsfImage, 1, i_lambda,
                                                 n_lsf, i_lambda) * aWCS->cd11;
    for (i_lsf = 1; i_lsf <= n_lsf; i_lsf++) {
      int invalid;
      double flux = cpl_image_get(aLsfImage, i_lsf, i_lambda, &invalid);
      if (!invalid) {
        cpl_image_set(aLsfImage, i_lsf, i_lambda, flux / line_flux);
      }
    }
  }
  return CPL_ERROR_NONE;
}

/**
   @brief Create a new LSF datacube.
   @param aLsfHalfRange Half width of the range in LSF direction
   @param aNLsf         Number of data points in LSF direction
   @param aNLambda      Number of data points in line wavelength direction
   @param aHeader       Header to derive the LSF cube header from (can be NULL)
   @return The newly created data cube.
 */
muse_lsf_cube *
muse_lsf_cube_new(double aLsfHalfRange, cpl_size aNLsf, cpl_size aNLambda,
                  const cpl_propertylist *aHeader)
{
  muse_lsf_cube *lsf = cpl_calloc(1, sizeof(muse_lsf_cube));
  lsf->header = cpl_propertylist_new();
  if (aHeader) {
    /* copy all keywords, except keys that don't belong here */
    const char *regex = MUSE_HDR_OVSC_REGEXP"|"MUSE_WCS_KEYS"|"MUSE_HDR_PT_REGEXP;
    cpl_propertylist_copy_property_regexp(lsf->header, aHeader, regex, 1);
  }
  lsf->img = cpl_imagelist_new();
  cpl_size i;
  for (i = 0; i < kMuseSlicesPerCCD; i++) {
    cpl_imagelist_set(lsf->img,
                      cpl_image_new(aNLsf, aNLambda, CPL_TYPE_FLOAT), i);
  }
  double lsf_step = 2*aLsfHalfRange / (aNLsf - 1),
         lambda_step = (kMuseNominalLambdaMax - kMuseNominalLambdaMin)
                     / (aNLambda - 1);
  lsf->wcs = cpl_calloc(1, sizeof(muse_wcs));
  lsf->wcs->cd11 = lsf_step;
  lsf->wcs->cd12 = 0.;
  lsf->wcs->cd21 = 0.;
  lsf->wcs->cd22 = lambda_step;
  lsf->wcs->crval1 = -aLsfHalfRange;
  lsf->wcs->crval2 = kMuseNominalLambdaMin;
  lsf->wcs->crpix1 = 1.;
  lsf->wcs->crpix2 = 1.;
  return lsf;
} /* muse_lsf_cube_new() */

/**
   @brief Deallocate the memory for the LSF cube.
   @param aLsfCube The LSF cube to deallocate
*/
void
muse_lsf_cube_delete(muse_lsf_cube *aLsfCube)
{
  if (!aLsfCube) {
    return;
  }
  cpl_propertylist_delete(aLsfCube->header);
  cpl_imagelist_delete(aLsfCube->img);
  cpl_free(aLsfCube->wcs);
  cpl_free(aLsfCube);
}

/**
   @brief Save the LSF cube to disk.
   @param aLsfCube  The LSF cube
   @param aFileName FITS file name
 */
cpl_error_code
muse_lsf_cube_save(muse_lsf_cube *aLsfCube, const char *aFileName)
{
  cpl_ensure_code(aLsfCube, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = cpl_propertylist_save(aLsfCube->header, aFileName,
                                            CPL_IO_CREATE);
  if (rc != CPL_ERROR_NONE) {
    return rc;
  }
  /* create the extension header */
  cpl_propertylist *header = cpl_propertylist_new();
  cpl_propertylist_append_string(header, "EXTNAME", "LSF_PROFILE");
  cpl_propertylist_append_int(header, "WCSAXES", 2);
  cpl_propertylist_append_double(header, "CD1_1", aLsfCube->wcs->cd11);
  cpl_propertylist_append_double(header, "CD1_2", aLsfCube->wcs->cd12);
  cpl_propertylist_append_double(header, "CD2_1", aLsfCube->wcs->cd21);
  cpl_propertylist_append_double(header, "CD2_2", aLsfCube->wcs->cd22);
  cpl_propertylist_append_double(header, "CRPIX1", aLsfCube->wcs->crpix1);
  cpl_propertylist_append_double(header, "CRPIX2", aLsfCube->wcs->crpix2);
  cpl_propertylist_append_double(header, "CRVAL1", aLsfCube->wcs->crval1);
  cpl_propertylist_append_double(header, "CRVAL2", aLsfCube->wcs->crval2);
  cpl_propertylist_append_string(header, "CTYPE1", "PARAM");
  cpl_propertylist_append_string(header, "CTYPE2", "AWAV");
  cpl_propertylist_append_string(header, "CUNIT1", "Angstrom");
  cpl_propertylist_append_string(header, "CUNIT2", "Angstrom");

  rc = cpl_imagelist_save(aLsfCube->img, aFileName,
                          CPL_TYPE_FLOAT, header, CPL_IO_EXTEND);
  cpl_propertylist_delete(header);
  return rc;
} /* muse_lsf_cube_save() */

/*----------------------------------------------------------------------------*/
/**
   @brief Load all LSF cubes for all IFUs into an array.
   @param aProcessing   the processing structure
   @return array of up to kMuseNumIFUs LSF cubes or NULL on error

   This function output a warning message, if one of the files from the input
   frameset cannot be loaded or if the LSFs for one of the IFUs cannot be
   found.

   @cpl_ensure{aProcessing, CPL_ERROR_NULL_INPUT, NULL}
 */
/*----------------------------------------------------------------------------*/
muse_lsf_cube **
muse_lsf_cube_load_all(muse_processing *aProcessing)
{
  cpl_ensure(aProcessing, CPL_ERROR_NULL_INPUT, NULL);
  muse_lsf_cube **lsf = cpl_calloc(kMuseNumIFUs, sizeof(muse_lsf_cube *));
  unsigned char ifu, nlsfs = 0; /* count number of LSF cubes loaded */
  for (ifu = 1; ifu <= kMuseNumIFUs; ifu++) {
    cpl_frameset *frames = muse_frameset_find(aProcessing->inframes,
                                              MUSE_TAG_LSF_PROFILE, ifu, CPL_FALSE);
    cpl_errorstate es = cpl_errorstate_get();
    cpl_frame *frame = cpl_frameset_get_position(frames, 0);
    if (frame == NULL) {
      cpl_msg_warning(__func__, "No %s (cube format) specified for IFU %2hhu!",
                      MUSE_TAG_LSF_PROFILE, ifu);
      cpl_errorstate_set(es); /* swallow the illegal input state */
      cpl_frameset_delete(frames);
      continue;
    }
    const char *fn = cpl_frame_get_filename(frame);
    lsf[ifu-1] = muse_lsf_cube_load(fn, ifu);
    if (lsf[ifu-1] == NULL) {
      cpl_msg_warning(__func__, "Could not load LSF (cube format) for IFU %2hhu "
                      "from \"%s\"!", ifu, fn);
      cpl_frameset_delete(frames);
      muse_lsf_cube_delete_all(lsf);
      return NULL;
    }
    muse_processing_append_used(aProcessing, frame, CPL_FRAME_GROUP_CALIB, 1);
    cpl_frameset_delete(frames);
    nlsfs++;
  } /* for ifu */
  if (!nlsfs) {
    cpl_msg_error(__func__, "Did not load any %ss (cube format)!",
                  MUSE_TAG_LSF_PROFILE);
    muse_lsf_cube_delete_all(lsf);
    return NULL;
  } /* if none loaded */
  cpl_msg_info(__func__, "Successfully loaded %s%hhu %ss (cube format).",
               nlsfs == kMuseNumIFUs ? "all ": "", nlsfs, MUSE_TAG_LSF_PROFILE);
  return lsf;
} /* muse_lsf_cube_load_all() */

/** @brief Delete all LSF cubes.
    @param aLsfCube Array of 48 data cubes
 */
void
muse_lsf_cube_delete_all(muse_lsf_cube **aLsfCube) {
  if (aLsfCube != NULL) {
    int ifu;
    for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
      muse_lsf_cube_delete(aLsfCube[ifu]);
    }
    cpl_free(aLsfCube);
  }
}

/**
   @brief Load a LSF cube for one single IFU from disk.
   @param aFileName FITS file name
   @param aIFU      the IFU number
   @retval The LSF cube.
   @cpl_ensure{aFileName, CPL_ERROR_NULL_INPUT, NULL}
 */
muse_lsf_cube *
muse_lsf_cube_load(const char *aFileName, unsigned char aIFU)
{
  cpl_ensure(aFileName, CPL_ERROR_NULL_INPUT, NULL);

  /* start looking for the LSF_PROFILE extension */
  int ext = cpl_fits_find_extension(aFileName, "LSF_PROFILE");
  char *extname = NULL;
  if (ext <= 0) { /* if not in an extension then look for merged name */
    extname = cpl_sprintf("CHAN%02hhu.LSF_PROFILE", aIFU);
    ext = cpl_fits_find_extension(aFileName, extname);
    if (ext <= 0) { /* no extension for given IFU found */
      cpl_free(extname);
      cpl_error_set(__func__, CPL_ERROR_DATA_NOT_FOUND);
      return NULL;
    }
  } /* if no LSF_PROFILE extension */
  cpl_free(extname);

  muse_lsf_cube *lsf = cpl_calloc(1, sizeof(muse_lsf_cube));
  lsf->header = cpl_propertylist_load(aFileName, 0); /* primary header */
  lsf->img = cpl_imagelist_load(aFileName, CPL_TYPE_DOUBLE, ext);
  lsf->wcs = cpl_calloc(1, sizeof(muse_wcs));
  if (lsf->img == NULL) {
    muse_lsf_cube_delete(lsf);
    return NULL;
  }
  /* now load the WCS from the extension header */
  cpl_propertylist *header = cpl_propertylist_load(aFileName, ext);
  if (header == NULL) {
    muse_lsf_cube_delete(lsf);
    return NULL;
  }
  lsf->wcs->cd11 = muse_pfits_get_cd(header, 1, 1);
  lsf->wcs->cd12 = muse_pfits_get_cd(header, 1, 2);
  lsf->wcs->cd21 = muse_pfits_get_cd(header, 2, 1);
  lsf->wcs->cd22 = muse_pfits_get_cd(header, 2, 2);
  lsf->wcs->crpix1 = muse_pfits_get_crpix(header, 1);
  lsf->wcs->crpix2 = muse_pfits_get_crpix(header, 2);
  lsf->wcs->crval1 = muse_pfits_get_crval(header, 1);
  lsf->wcs->crval2 = muse_pfits_get_crval(header, 2);
  /* XXX check types and units of the WCS */
  // muse_pfits_get_ctype(header, 1) "PARAM"
  // muse_pfits_get_ctype(header, 2) "AWAV"
  // muse_pfits_get_cunit(header, 1) "Angstrom"
  // muse_pfits_get_cunit(header, 2) "Angstrom"
  cpl_propertylist_delete(header);

  return lsf;
} /* muse_lsf_cube_load() */

/**
   @brief Create an average image from all LSF cubes
   @param aLsfCube Array of all LSF cubes
   @param aPixtable Pixtable for the weighting, or NULL
   @return Average LSF image

   All images from all cubes are squeezed into one image. If a
   pixtable is given, each IFU/slice is weighted with the number of
   entries in the pixtable for this IFU/slice.
 */
cpl_image *
muse_lsf_average_cube_all(muse_lsf_cube **aLsfCube, muse_pixtable *aPixtable)
{
  cpl_ensure(aLsfCube, CPL_ERROR_NULL_INPUT, NULL);
  cpl_image *img = NULL;
  cpl_size wsum = 0;
  cpl_size w[kMuseNumIFUs][kMuseSlicesPerCCD];
  cpl_size ifu;
  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    cpl_size i_slice;
    for (i_slice = 0; i_slice < kMuseSlicesPerCCD; i_slice++) {
      w[ifu][i_slice] = (aPixtable == NULL)?1.0:0.0;
    }
  }

  if (aPixtable != NULL) {
    cpl_size n_rows = muse_pixtable_get_nrow(aPixtable);
    cpl_size i_row;
    uint32_t *origin = (uint32_t*)
      cpl_table_get_data_int(aPixtable->table, MUSE_PIXTABLE_ORIGIN);
    for (i_row = 0; i_row < n_rows; i_row++) {
      int i_ifu = muse_pixtable_origin_get_ifu(origin[i_row]);
      int i_slice = muse_pixtable_origin_get_slice(origin[i_row]);
      w[i_ifu-1][i_slice-1]++;
    }
  }

  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    if (aLsfCube[ifu] == NULL) {
      continue;
    }
    cpl_size i_slice;
    cpl_size n_slices = cpl_imagelist_get_size(aLsfCube[ifu]->img);
    for (i_slice = 0; i_slice < n_slices; i_slice++) {
      if (w[ifu][i_slice] > 0) {
        cpl_image *m = cpl_imagelist_get(aLsfCube[ifu]->img, i_slice);
        m = cpl_image_duplicate(m);
        cpl_image_multiply_scalar(m, w[ifu][i_slice]);
        wsum += w[ifu][i_slice];
        if (img == NULL) {
          img = m;
        } else {
          cpl_errorstate pre = cpl_errorstate_get();
          cpl_error_code rc = cpl_image_add(img, m);
          cpl_image_delete(m);
          if (rc != CPL_ERROR_NONE) {
            cpl_msg_warning(__func__, "Could not add cube of IFU %"
                            CPL_SIZE_FORMAT": %s",
                            ifu+1, cpl_error_get_message());
            cpl_errorstate_set(pre);
            break;
          }
        }
      }
    }
  }
  if ((img != NULL) && (wsum > 0)) {
    cpl_image_divide_scalar(img, wsum);
    return img;
  } else {
    cpl_image_delete(img);
    return NULL;
  }
}

// XXX this should be documented and actually check that all WCSs are identical!
muse_wcs *
muse_lsf_cube_get_wcs_all(muse_lsf_cube **aLsfCube) {
  cpl_size ifu;
  for (ifu = 0; ifu < kMuseNumIFUs; ifu++) {
    if (aLsfCube[ifu] != NULL) {
      return aLsfCube[ifu]->wcs;
    }
  }
  return NULL;
}

/*----------------------------------------------------------------------------*/
/**
   @brief Filter an LSF image with a rectangle to model spectrum binning.
   @param aLsfImage   The LSF image. Filtering is in-place.
   @param aWCS        The WCS of the image.
   @param aBinWidth   The width of the rectangle.
   @return CPL_ERROR_NONE on success another CPL error code on failure.

   @error{return CPL_ERROR_NULL_INPUT, aLsfImage and/or aWcs are NULL}
   @error{return CPL_ERROR_ILLEGAL_INPUT, aBinWidth is not positive}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_fold_rectangle(cpl_image *aLsfImage, const muse_wcs *aWCS,
                        double aBinWidth)
{
  cpl_ensure_code(aLsfImage && aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aBinWidth > 0., CPL_ERROR_ILLEGAL_INPUT);

  cpl_size ncol_kernel = aBinWidth / aWCS->cd11;
  ncol_kernel = ((ncol_kernel + 1) / 2) * 2 + 1; /* odd number */
  double r = ncol_kernel - aBinWidth / aWCS->cd11;
  cpl_matrix *kernel = cpl_matrix_new(1, ncol_kernel);
  cpl_matrix_fill(kernel, 1.0);
  cpl_matrix_set(kernel, 0, 0, 1 - r/2);
  cpl_matrix_set(kernel, 0, ncol_kernel-1, 1 - r/2);
  cpl_image *lsfImage0 = cpl_image_duplicate(aLsfImage);
  cpl_image_filter(aLsfImage, lsfImage0, kernel, CPL_FILTER_LINEAR,
                   CPL_BORDER_FILTER);
  cpl_matrix_delete(kernel);
  cpl_image_delete(lsfImage0);

  return CPL_ERROR_NONE;
} /* muse_lsf_fold_rectangle() */

/*----------------------------------------------------------------------------*/
/**
   @brief Apply the LSF to a number of data points of one slice
   @param aLsfImage The slice specific image from the LSF cube
   @param aWCS      The WCS of the image.
   @param aVal     Array with wavelengths [A]. Will be replaced by the
                   normalized LSF data points
   @param aLambda  Line wavelength reference [A].
   @retval CPL_ERROR_NONE Everything went OK
   @cpl_ensure_code{aLsfImage, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aWCS, CPL_ERROR_NULL_INPUT}
   @cpl_ensure_code{aVal, CPL_ERROR_NULL_INPUT}
 */
/*----------------------------------------------------------------------------*/
cpl_error_code
muse_lsf_apply(const cpl_image *aLsfImage, const muse_wcs *aWCS,
               cpl_array *aVal, double aLambda)
{
  cpl_ensure_code(aLsfImage, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aWCS, CPL_ERROR_NULL_INPUT);
  cpl_ensure_code(aVal, CPL_ERROR_NULL_INPUT);
  cpl_size n_lsf = cpl_image_get_size_x(aLsfImage);
  cpl_size n_lambda = cpl_image_get_size_y(aLsfImage);

  /* Find y position in image corresponding to line wavelength */
  double lambda_pix = (aLambda - aWCS->crval2) / aWCS->cd22 + aWCS->crpix2;
  if (lambda_pix < 1) {
    lambda_pix = 1;
  }
  if (lambda_pix > n_lambda) {
    lambda_pix = n_lambda;
  }
  cpl_size i_lambda = floor(lambda_pix);
  lambda_pix -= i_lambda; /* lambda_pix has fraction of y position */

  /* Create vector containing the LSF by linear interpolation.
     For speed reasons, this is a plain C array. */
  cpl_array *lsf_lambda = cpl_array_new(n_lsf + 4, CPL_TYPE_DOUBLE);
  cpl_array *lsf_values = cpl_array_new(n_lsf + 4, CPL_TYPE_DOUBLE);
  cpl_size i;
  for (i = 1; i <= n_lsf; i++) {
    int res;
    double z = cpl_image_get(aLsfImage, i, i_lambda, &res);
    if (lambda_pix > 0) {
      z = z * (1 - lambda_pix)
        + cpl_image_get(aLsfImage, i, i_lambda+1, &res) * lambda_pix;
    }
    cpl_array_set(lsf_values, i+1, z);
    cpl_array_set(lsf_lambda, i+1, aWCS->crval1 + (i - aWCS->crpix1) * aWCS->cd11);
  }
  // Set outer limits of LSF to zero to avoid invalid values in the fit
  // +- 10000 A is used since the pixtable range will never be in this range
  cpl_array_set(lsf_lambda, 0, -10000);
  cpl_array_set(lsf_values, 0, 0);
  cpl_array_set(lsf_lambda, 1, aWCS->crval1 - aWCS->crpix1 * aWCS->cd11);
  cpl_array_set(lsf_values, 1, 0);
  cpl_array_set(lsf_lambda, n_lsf+2,  aWCS->crval1 + (n_lsf+1 - aWCS->crpix1) * aWCS->cd11);
  cpl_array_set(lsf_values, n_lsf+2, 0);
  cpl_array_set(lsf_lambda, n_lsf+3, 10000);
  cpl_array_set(lsf_values, n_lsf+3, 0);

  // Correct the central position of the LSF function
  cpl_array *v = cpl_array_duplicate(lsf_values);
  cpl_array_multiply(v, lsf_lambda);
  double offset = cpl_array_get_mean(v)/cpl_array_get_mean(lsf_values);
  cpl_array_delete(v);
  cpl_array_subtract_scalar (lsf_lambda, offset);

  // Re-normalize, just to be safe
  double norm = cpl_array_get_mean(lsf_values) * (n_lsf + 4) * aWCS->cd11;
  cpl_array_divide_scalar(lsf_values, norm);

  cpl_array *values = muse_cplarray_interpolate_linear(aVal, lsf_lambda,
                                                       lsf_values);

  // Copy the values back to the function argument
  memcpy(cpl_array_get_data_double(aVal), cpl_array_get_data_double(values),
         cpl_array_get_size(aVal) * sizeof(double));

  cpl_array_delete(values);
  cpl_array_delete(lsf_lambda);
  cpl_array_delete(lsf_values);
  return CPL_ERROR_NONE;
}

/**@}*/
