/* $Id: visir_util_repack.c,v 1.72 2012/01/13 12:02:03 jtaylor Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jtaylor $
 * $Date: 2012/01/13 12:02:03 $
 * $Revision: 1.72 $
 * $Name: visir-3_5_1 $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "visir_recipe.h"

/* Verify self-sufficiency of CPL header files by including system files last */
#include <string.h>

/*-----------------------------------------------------------------------------
                                Defines
 -----------------------------------------------------------------------------*/

#ifndef VISIR_UTIL_REPACK_CONAD
#define VISIR_UTIL_REPACK_CONAD 1.1
#endif

#define RECIPE_STRING   "visir_util_repack"

#define VISIR_DRS_CUMOFFSETX "ESO DRS CUMOFFSETX"
#define VISIR_DRS_CUMOFFSETY "ESO DRS CUMOFFSETY"
/* FIXME: Change to ESO DRS OUT1 CONAD when that is supported by mime */
#define VISIR_DRS_CONAD      "ESO DET OUT1 CONAD"

/*
  FIXME: Manual replacement of the keys needed by the Austrian mime code:
    perl -pi -e 's/\bESO DRS CUMOFFSET\B/ESO SEQ CUMOFFSET/g'
             -e 's/\bESO DRS OUT1 CONAD\b/ESO DET OUT1 CONAD/g'
            visir_util_repack_*.fits
*/


/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

static cpl_error_code visir_util_repack_one(cpl_frameset *,
                                            irplib_framelist *, int,
                                            const cpl_parameterlist *,
                                            const cpl_image *);

static cpl_error_code visir_util_repack_one_(cpl_frameset *,
                                             irplib_framelist *, int,
                                             const cpl_parameterlist *,
                                             const char * tag,
                                             const cpl_image *);

static cpl_error_code visir_util_repack_check(const cpl_image *,
                                              const cpl_imagelist *,
                                              const cpl_imagelist *);

cpl_recipe_define(visir_util_repack, VISIR_BINARY_VERSION,
                  "Lars Lundin", PACKAGE_BUGREPORT, "2011", 
                  "Conversion of raw CUBE2 or BURST images to on- and off-cubes",
                  "The files listed in the Set Of Frames (sof-file) "
                  "must be tagged:\n"
                  "VISIR-CUBE2-raw-file.fits " VISIR_UTIL_REPACK_RAW
                  "\nor\n"
                  "VISIR-BURST-raw-file.fits " VISIR_IMG_BURST"\n"
                  "VISIR-BURST-bpm-file.fits " VISIR_CALIB_STATIC_MASK
                  "\nFor BURST data it will remove planes where the chopper "
                  "switched from on <-> off based on the "
                  "timestamps in the header."
                  "\nThe product(s) will have a FITS card\n"
                  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
                  VISIR_UTIL_REPACK_A_ON_PROCATG  " (NodPos: A, ChopPos: on)\n"
                  VISIR_UTIL_REPACK_A_OFF_PROCATG " (NodPos: A, ChopPos: off)\n"
                  VISIR_UTIL_REPACK_B_ON_PROCATG  " (NodPos: B, ChopPos: on)\n"
                  VISIR_UTIL_REPACK_B_OFF_PROCATG " (NodPos: B, ChopPos: off)\n"
                  /* FIXME: reimplement
                  "Additionally, the recipe collapses the on- and off-cubes, "
                  "these product(s) will have a FITS card\n"
                  "'HIERARCH ESO PRO CATG' with a value of one of:\n"
                  VISIR_UTIL_REPACK_MEAN_A_ON_PROCATG
                  " (NodPos: A, ChopPos: on)\n"
                  VISIR_UTIL_REPACK_MEAN_A_OFF_PROCATG
                  " (NodPos: A, ChopPos: off)\n"
                  VISIR_UTIL_REPACK_MEAN_B_ON_PROCATG
                  " (NodPos: B, ChopPos: on)\n"
                  VISIR_UTIL_REPACK_MEAN_B_OFF_PROCATG
                  " (NodPos: B, ChopPos: off)\n" */
                  "For CUBE2, the recipe will produce a static "
                  "bad-pixel map, it will have a FITS card\n"
                  "'HIERARCH ESO PRO CATG' with a value of:\n"
                  VISIR_CALIB_STATIC_MASK"\n"
                  );

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_util_repack   Conversion of raw images to nodded images
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    pop one image from end of a list
  @param    list       list to pop from
  @return   popped image or null when empty
 */
/*----------------------------------------------------------------------------*/
static cpl_image *
cpl_imagelist_pop(cpl_imagelist * list)
{
    if (cpl_imagelist_get_size(list) == 0)
        return NULL;
    return cpl_imagelist_unset(list, cpl_imagelist_get_size(list) - 1);
}


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    Fill the recipe parameterlist
  @param    self  The parameterlist
  @return   0 iff everything is ok

 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
visir_util_repack_fill_parameterlist(cpl_parameterlist * self)
{
    const char * context = PACKAGE "." RECIPE_STRING;
    cpl_error_code err;

    cpl_ensure_code(self, CPL_ERROR_NULL_INPUT);

    /* Fill the parameters list */

    /* --planestart */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "planestart",
                                       0, NULL, context, "Plane range startpoint.");
    cpl_ensure_code(!err, err);

    /* --ncycles */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "ncycles",
                                       -1, NULL, context, "Number of full "
                                       "on-off cycles to repack. <= 0 for all.");
    /* --trimlow */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimlow",
                                       0, NULL, context, "Burst data only. "
                                       "Number of additional planes to cut "
                                       "from before each plane with chopper "
                                       "movement.");
    cpl_ensure_code(!err, err);

    /* --trimhigh */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING, "trimhigh",
                                       0, NULL, context, "Burst data only. "
                                       "Number of additional planes to cut "
                                       "from after each plane with chopper "
                                       "movement.\n A value of -1 does not "
                                       "skip the plane of the movement.");
    cpl_ensure_code(!err, err);

    /* --bkgcorrect */
    err =
        irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "bkgcorrect",
                                      CPL_FALSE, NULL, context, "Output "
                                      " background corrected planes by "
                                      " subtracting chop planes instead of "
                                      " outputting each plane");
    cpl_ensure_code(!err, err);

    /* --bkgcorrect */
    err =
        irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING, "error",
                                      CPL_FALSE, NULL, context, "Output error map"
                                      " by assuming  poisson distributed photon "
                                      "counts. Error will be sqrt(2*|on - off|)."
                                      " Implies --bkgcorrect.");
    cpl_ensure_code(!err, err);

    return CPL_ERROR_NONE;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    framelist   the frames list
  @param    parlist     the parameters list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_util_repack(cpl_frameset            * framelist,
                             const cpl_parameterlist * parlist)
{
    cpl_errorstate     cleanstate = cpl_errorstate_get();
    cpl_error_code     errori = CPL_ERROR_NONE;
    irplib_framelist * allframes = NULL;
    irplib_framelist * rawframes = NULL;
    int                i, n;
    cpl_image *        bpm = NULL;

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));

    /* Objects observation */
    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);
    rawframes = irplib_framelist_extract_regexp(allframes, "^(" VISIR_IMG_BURST
                                                "|" VISIR_UTIL_REPACK_RAW
                                                "|" VISIR_CALIB_STATIC_MASK ")$",
                                                CPL_FALSE);
    skip_if (rawframes == NULL);
    
    n = irplib_framelist_get_size(rawframes);

    for (i = 0; i < n; i++) {
        if (visir_util_repack_one(framelist, rawframes, i, parlist, bpm)) {
            errori = cpl_error_set_where(cpl_func);
            if (errori != CPL_ERROR_NONE)
                break;
        }
    }
    cpl_errorstate_dump(cleanstate, CPL_FALSE, NULL);
    cpl_errorstate_set(cleanstate);

    error_if(errori, errori, "Failed to repack cubes in %d frame(s)", n);

    end_skip;

    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    cpl_image_delete(bpm);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Process one frame
  @param    framelist   The frameset to append products to
  @param    rawframes   The frames list (will load the propertylist)
  @param    i           The frame to process (0 for first)
  @param    parlist     The parameters list
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_util_repack_one(cpl_frameset * framelist,
                                            irplib_framelist * rawframes, int i,
                                            const cpl_parameterlist * parlist,
                                            const cpl_image * bpm)
{

    const cpl_frame * rawframe = irplib_framelist_get_const(rawframes, i);
    const char * tag = cpl_frame_get_tag(rawframe);

    if (strcmp(tag, VISIR_CALIB_STATIC_MASK) == 0)
        return cpl_error_get_code();

    skip_if(irplib_framelist_load_propertylist(rawframes, i, 0, "^(NAXIS3|BITPIX|"
                                               VISIR_PFITS_DOUBLE_DIT "|"
                                               VISIR_PFITS_INT_NDITSKIP "|"
                                               VISIR_PFITS_INT_NDIT "|"
                                               IRPLIB_PFITS_WCS_REGEXP "|"
                                               VISIR_PFITS_INT_CHOP_NCYCLES "|"
                                               VISIR_PFITS_STRING_CHOP_START "|"
                                               VISIR_PFITS_DOUBLE_CHOP_FREQ "|"
                                               VISIR_PFITS_STRING_CHOP_START "|"
                                               VISIR_PFITS_STRING_OBS_START "|"
                                               VISIR_PFITS_DOUBLE_CHOP_THROW "|"
                                               VISIR_PFITS_STRING_PIXSCALE "|"
                                               VISIR_PFITS_STRING_FRAME_TYPE "|"
                                               VISIR_PFITS_DOUBLE_CUMOFFSETX "|"
                                               VISIR_PFITS_DOUBLE_CUMOFFSETY "|"
                                               VISIR_PFITS_STRING_NODPOS ")$",
                                               CPL_FALSE));

    bug_if(tag == NULL); /* Already extracted CUBE2 or BURST */

    skip_if(visir_util_repack_one_(framelist, rawframes, i, parlist, tag, bpm));

    end_skip;

    return cpl_error_get_code();
}


typedef enum {
    VISIR_DATA_CUBE2,
    VISIR_DATA_BURST
} visir_data_type;

/* convinience data structure to hold common data needed
 * to process one frame */
typedef struct {
    int planestart;
    int planeend;
    int trimlow;
    int trimhigh;
    cpl_boolean bkgcorrect;
    cpl_boolean esterror;
    double conad;
    int nframes;
    const irplib_framelist * rawframes;
    int iframe;
    const cpl_propertylist * plist;
    visir_data_type datatype;
    cpl_type_bpp save_type;

    char * onname;
    char * offname;
    const char * remregexp;

    double pthrow;
    const char * comoffx;
    const char * comoffy;
    double offsetx;
    double offsety;

    const cpl_image * bpm;
    double bpmthresh;

    int naxis3;
    const char * procatgon;
    const char * procatgoff;
} repack_frame_state;


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    creates the repack frame state structure
  @param    rawframes   list of rawframes
  @param    iframe      index of the frame looked at
  @param    parlist     list of recipe parameters
  @param    tag         tag of the frame
  @return   structure with holding the state

  create a convinience structure with all the data needed for repacking.
 */
/*----------------------------------------------------------------------------*/
static repack_frame_state *
init_frame_state(const irplib_framelist * rawframes, int iframe,
                 const cpl_parameterlist * parlist, const char * tag)
{
    repack_frame_state * state = cpl_malloc(sizeof(repack_frame_state));
    int ndit, ncycles, planes_per_cycle;

    state->rawframes = rawframes;

    state->iframe = iframe;

    state->plist = irplib_framelist_get_propertylist_const(rawframes, iframe);

    state->naxis3     = visir_pfits_get_naxis3(state->plist);

    state->datatype =
        !strcmp(tag, "CUBE2") ? VISIR_DATA_CUBE2 : VISIR_DATA_BURST;

    state->bkgcorrect = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                      RECIPE_STRING,
                                                      "bkgcorrect");

    state->esterror = irplib_parameterlist_get_bool(parlist, PACKAGE,
                                                    RECIPE_STRING,
                                                    "error");
    state->bkgcorrect |= state->esterror;

    state->planestart = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                     RECIPE_STRING,
                                                     "planestart");

    ndit = visir_pfits_get_ndit(state->plist);
    if (state->datatype == VISIR_DATA_CUBE2)
        planes_per_cycle = 2;
    else if (state->datatype == VISIR_DATA_BURST)
        planes_per_cycle = ndit * 2;
    else
        bug_if(1);

    ncycles = irplib_parameterlist_get_int(parlist, PACKAGE,
                                              RECIPE_STRING, "ncycles");

    state->planeend = state->planestart + ncycles * planes_per_cycle;
    /* ensure we always repack a multiple of a full cycle to get equal number
     * of on and off planes */
    if (state->planeend >= state->naxis3 || state->planeend <= 0) {
        if (state->planestart % planes_per_cycle == 0)
            state->planeend = state->naxis3;
        else
            state->planeend = state->naxis3 -
                (planes_per_cycle - (state->planestart % planes_per_cycle));
    }

#ifdef CPL_BPP_IEEE_FLOAT
/* FIXME: Use just CPL_TYPE_UNSPECIFIED once CPL 5.3.1 is no longer supported */
    state->save_type = CPL_TYPE_UNSPECIFIED;
#else
    state->save_type = cpl_propertylist_get_int(state->plist, "BITPIX");
#endif


    state->trimlow = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                  RECIPE_STRING,
                                                  "trimlow");
    state->trimhigh = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                   RECIPE_STRING,
                                                   "trimhigh");
    state->conad = VISIR_UTIL_REPACK_CONAD;

    state->nframes = irplib_framelist_get_size(rawframes);
    state->remregexp  = "^(" VISIR_PFITS_DOUBLE_CUMOFFSETX
                        "|" VISIR_PFITS_DOUBLE_CUMOFFSETY ")$";

    /* Get the chopping throw in pixels */
    {
        const char * sscale       = visir_pfits_get_pixscale(state->plist);
        double pscale       = sscale ? atof(sscale) : 0.0;
        state->pthrow       = pscale > 0.0 ?
                              visir_pfits_get_chop_throw(state->plist) / pscale : 0.0;
    }
    /* Copy comments from CUMOFFSET[XY] */
    state->comoffx      = cpl_propertylist_get_comment
        (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETX);
    state->comoffy      = cpl_propertylist_get_comment
        (state->plist, VISIR_PFITS_DOUBLE_CUMOFFSETY);
    state->offsetx      = visir_pfits_get_cumoffsetx(state->plist);
    state->offsety      = visir_pfits_get_cumoffsety(state->plist);

    state->bpm = NULL; /* set and freed elsewhere */
    state->bpmthresh = VISIR_HCYCLE_BPM_THRESHOLD - VISIR_HCYCLE_OFFSET;

    if (state->bkgcorrect == CPL_FALSE) {
        const char * nodpos     = visir_pfits_get_nodpos(state->plist);
        const cpl_boolean is_a     = nodpos != NULL &&
            (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
        state->procatgon = is_a ? VISIR_UTIL_REPACK_A_ON_PROCATG
            : VISIR_UTIL_REPACK_B_ON_PROCATG;
        state->procatgoff = is_a ? VISIR_UTIL_REPACK_A_OFF_PROCATG
            : VISIR_UTIL_REPACK_B_OFF_PROCATG;
        state->onname     = cpl_sprintf(RECIPE_STRING "_%s_on_%d"
                                        CPL_DFS_FITS, nodpos, 1+iframe);
        state->offname    = cpl_sprintf(RECIPE_STRING "_%s_off_%d"
                                        CPL_DFS_FITS, nodpos, 1+iframe);

        if (!is_a) {
            /* Verify that the NODPOS is either A or B */
            skip_if (nodpos == NULL);
            skip_if (strstr(nodpos, "B") == NULL && strstr(nodpos, "b") == NULL);
        }
    } else  {
        const char * nodpos     = visir_pfits_get_nodpos(state->plist);
        const cpl_boolean is_a     = nodpos != NULL &&
            (strstr(nodpos, "A") || strstr(nodpos, "a")) ? CPL_TRUE : CPL_FALSE;
        state->procatgon = is_a ? "BKG_CORRECTED_A" : "BKG_CORRECTED_B";
        state->onname    = cpl_sprintf(RECIPE_STRING "_%s_bkgcor_%d"
                                       CPL_DFS_FITS, nodpos, 1+iframe);
        if (state->esterror) {
            state->procatgoff = is_a ?
                "BKG_CORRECTED_ERROR_A" : "BKG_CORRECTED_ERROR_B";
            state->offname    = cpl_sprintf(RECIPE_STRING "_%s_bkgcor_err_%d"
                                            CPL_DFS_FITS, nodpos, 1+iframe);
        } else {
            state->procatgoff = NULL;
            state->offname = NULL;
        }

    }

    if (state->offsety != 0.0)
        cpl_msg_warning(cpl_func, "Frame %d/%d has '%s' = %g != 0.0",
                        1+iframe, state->nframes,
                        VISIR_PFITS_DOUBLE_CUMOFFSETY, state->offsety);

    end_skip;

    return state;
}


static cpl_error_code
subtract_images(const repack_frame_state * state,
                cpl_imagelist * on, cpl_imagelist * off,
                const cpl_propertylist * plist)
{
    const int non = cpl_imagelist_get_size(on);
    cpl_image * errmap = NULL;
    cpl_image * imgabs = NULL;
    bug_if(cpl_imagelist_get_size(off) != non);

    for (int j = 0; j < non; j++) {
        cpl_image * onimg  = cpl_imagelist_get(on,  j);
        cpl_image * offimg  = cpl_imagelist_get(off,  j);
        cpl_image * minuend = onimg;
        cpl_image * subtrahend = offimg;
        cpl_image * errmap;

        /* burst data is not preprocessed so it negative */
        if (state->datatype == VISIR_DATA_BURST) {
            minuend = offimg;
            subtrahend = onimg;
        }
        cpl_image_subtract(minuend, subtrahend);
        /* FIXME: gain */
        if (state->esterror) {
            cpl_image_delete(errmap);
            cpl_image_delete(imgabs);
            imgabs = cpl_image_cast(minuend, CPL_TYPE_FLOAT);
            cpl_image_abs(imgabs);
            errmap = cpl_image_power_create(imgabs, 0.5);
            cpl_image_multiply_scalar(errmap, sqrt(2));
            skip_if(cpl_image_save(errmap, state->offname, CPL_BPP_IEEE_FLOAT,
                                   plist, CPL_IO_EXTEND));
        }


        /* Pending closure of DFS 10468:
           Duplicate on/off lists in all extension headers
           */
        skip_if(cpl_image_save(minuend, state->onname, state->save_type,
                               plist, CPL_IO_EXTEND));
    }

    end_skip;

    cpl_image_delete(errmap);
    cpl_image_delete(imgabs);
    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    append images to a file
  @param    state       repack state structure
  @param    images      imagelist to save
  @param    plist       propertylist to save with the image
  @param    on          whether its an on or off image
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.

  Will append images to an existing file.
  In case of burst data it will flip the sign.
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
append_images(const repack_frame_state * state,
              cpl_imagelist * images,
              const cpl_propertylist * plist, const cpl_boolean on)
{
    const char * name = on ? state->onname : state->offname;
    const int n = cpl_imagelist_get_size(images);

    for (int j = 0; j < n; j++) {
        cpl_image * img  = cpl_imagelist_get(images,  j);
        /* burst data is not preprocessed so it needs flipping */
        if (state->datatype == VISIR_DATA_BURST) {
            cpl_image_multiply_scalar(img, -1);
            /* we want to save it in 16 bit, so decrease the offset by one
             * we don't really care about the offset as the images are
             * subtracted later anyway */
            cpl_image_add_scalar(img, -1);
        }

        /* Pending closure of DFS 10468:
           Duplicate on/off lists in all extension headers
           */
        skip_if(cpl_image_save(img,  name, state->save_type,
                               plist, CPL_IO_EXTEND));
    }

    end_skip;

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    split input data into on and off frames and save
  @param    state       repack state structure
  @param    products    product frameset
  @param    usedframes  list of raw/calibration frames used for this product
  @param    onlist      propertylist to save with the on image
  @param    offlist     propertylist to save with the off image
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.

   Will process the data in chunks to reduce memory usage.
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
process_data(repack_frame_state * state, cpl_frameset * products,
             cpl_frameset * usedframes, const cpl_parameterlist * parlist,
             cpl_propertylist * onlist, cpl_propertylist * offlist)
{
    int to_off = 0, halfcycle = 0;
    int nton = 0, ntoff = 0;
    int chunksize = 200;
    cpl_imagelist * on  = NULL;
    cpl_imagelist * off = NULL;

    /* chunksize must be a multiple of the period to avoid having a
     * #on-#off difference larger than chunksize */
    if (state->datatype == VISIR_DATA_BURST) {
        skip_if(visir_img_burst_find_delta_chop(state->plist,
                                                &to_off, &halfcycle));
        chunksize = 2 * halfcycle;
        /* parameter sanity check, ensures equal length A and B lists */
        error_if(state->planeend - state->planestart < 2 * halfcycle,
                 CPL_ERROR_ILLEGAL_INPUT,
                 "Number of planes to be repacked must be larger than "
                 "a full cycle of %d planes.", halfcycle * 2);
    }

    for (int c = state->planestart;
         c < state->planeend + chunksize - 1;
         c += chunksize) {
        const int pend = c + chunksize > state->planeend ? state->planeend :
            c + chunksize;
        if (c >= pend)
            break;

        cpl_imagelist_delete(on);
        cpl_imagelist_delete(off);
        on  = cpl_imagelist_new();
        off = cpl_imagelist_new();

        /* sort a chunk of data */
        if (state->datatype == VISIR_DATA_CUBE2) {
            skip_if(visir_load_cube2_split(on, off, state->rawframes, state->iframe, c, pend));

            /* extract bpm from preprocessed burst data */
            if (state->iframe == 0 && c == state->planestart) {
                cpl_image * bpm;
                error_if(state->bpm, CPL_ERROR_ILLEGAL_INPUT, "CUBE2 data has "
                         "the badpixel map encoded. No extra STATIC_MASK "
                         "allowed");
                bpm = cpl_image_duplicate(cpl_imagelist_get(off, 0));
                bug_if(cpl_image_threshold(bpm, state->bpmthresh,
                                           state->bpmthresh, 0.0, 1.0));
                state->bpm = bpm;
                /* mime_estimate_error (DFS 10470) not needed
                 * as we create error maps in this recipe directly so only create
                 * one bpm */
                skip_if(irplib_dfs_save_image(products, parlist,
                                              usedframes, state->bpm,
                                              CPL_BPP_8_UNSIGNED, RECIPE_STRING,
                                              VISIR_CALIB_STATIC_MASK, NULL,
                                              state->remregexp, visir_pipe_id,
                                              RECIPE_STRING "_bpm"
                                              CPL_DFS_FITS));
            }
        }
        else if (state->datatype == VISIR_DATA_BURST)
            skip_if(visir_load_burst(on, off,
                                     irplib_framelist_get_const(state->rawframes, state->iframe),
                                     state->plist, to_off, halfcycle, c, pend,
                                     state->trimlow, state->trimhigh));
        else
            error_if(1, CPL_ERROR_ILLEGAL_INPUT, "invalid data tag");

        nton += cpl_imagelist_get_size(on);
        ntoff += cpl_imagelist_get_size(off);

        /* if on last chunk, try to equalize on and off lists
         * as chunksize is a multiple of the period this should work
         * within a single chunk*/
        if (c + chunksize >= state->planeend) {
            int diff = nton - ntoff;
            if (diff != 0)
                cpl_msg_warning(cpl_func, "Unequal number of planes in on "
                                "and off list: #on %d, #off %d. Skipping %d"
                                " planes.", nton, ntoff, abs(diff));
            if (diff > 0) {
                for (int i = 0; i < diff; i++)
                    cpl_image_delete(cpl_imagelist_pop(on));
                nton -= diff;
            }
            else {
                for (int i = 0; i < -diff; i++)
                    cpl_image_delete(cpl_imagelist_pop(off));
                ntoff += diff;
            }
        }

        if (state->bkgcorrect)
            skip_if(subtract_images(state, on, off, onlist));
        else {
            skip_if(append_images(state, on, onlist, CPL_TRUE));
            skip_if(append_images(state, off, offlist, CPL_FALSE));
        }
    }
    /* sanity check */
    error_if(nton != ntoff, CPL_ERROR_ILLEGAL_OUTPUT, "Unequal number of "
             "planes in on and offlist: #on %d, #off %d. Correct the"
             "recipe parameters to the period of the data.", nton, ntoff);

    end_skip;

    cpl_imagelist_delete(on);
    cpl_imagelist_delete(off);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Process one frame
  @param    framelist   The frameset to append products to
  @param    rawframes   The frames list (will load the propertylist)
  @param    iframe      The frame to process (0 for first)
  @param    parlist     The parameters list
  @param    tag         The frame tag
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.
 */
/*----------------------------------------------------------------------------*/
static
cpl_error_code visir_util_repack_one_(cpl_frameset * framelist,
                                      irplib_framelist * rawframes, int iframe,
                                      const cpl_parameterlist * parlist,
                                      const char * tag,
                                      const cpl_image * bpm)
{
    cpl_frameset  * products   = cpl_frameset_new();
    cpl_frameset  * usedframes = cpl_frameset_new();
    cpl_propertylist * onlist  = cpl_propertylist_new();
    cpl_propertylist * offlist = cpl_propertylist_new();

    repack_frame_state * state = init_frame_state(rawframes, iframe,
                                                  parlist, tag);
    state->bpm = bpm;

    cpl_msg_info(cpl_func, "Repacking frame %d/%d ('%s'=%g)",
                 1 + iframe, state->nframes, VISIR_DRS_CUMOFFSETY, state->pthrow);

    /* Need a copy of the WCS cards for subsequent beam stacking */
    bug_if(cpl_propertylist_copy_property_regexp(onlist, state->plist, "^("
                                                 IRPLIB_PFITS_WCS_REGEXP ")$",
                                                 CPL_FALSE));

    bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CONAD, state->conad));
    bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CONAD, "Default "
                                        "single frame value: " IRPLIB_STRINGIFY
                                        (VISIR_UTIL_REPACK_CONAD)));

    bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETX,
                                          state->offsetx));
    bug_if(cpl_propertylist_append_double(onlist, VISIR_DRS_CUMOFFSETY,
                                          state->offsety));
    bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETX,
                                        state->comoffx));
    bug_if(cpl_propertylist_set_comment(onlist, VISIR_DRS_CUMOFFSETY,
                                        state->comoffy));
    bug_if(cpl_propertylist_append(offlist, onlist));
    bug_if(cpl_propertylist_update_double(offlist, VISIR_DRS_CUMOFFSETY,
                                          state->offsety + state->pthrow));

    bug_if(cpl_frameset_insert(usedframes, cpl_frame_duplicate
                               (irplib_framelist_get_const(rawframes, iframe))));

#ifdef VISIR_MIME_MULTI_MASK_SUPPORT
#error VISIR_MIME_MULTI_MASK_SUPPORT not implemented
#endif

    if (state->bkgcorrect) {
        skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
                                             RECIPE_STRING, state->procatgon,
                                             offlist, state->remregexp,
                                             visir_pipe_id,
                                             state->onname));
        if (state->esterror)
            skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
                                                 RECIPE_STRING, state->procatgoff,
                                                 offlist, state->remregexp,
                                                 visir_pipe_id,
                                                 state->offname));
    } else {
        /* Save each image in a separate extension */
        /* Do not save data in the primary data unit (DFS10475) */
        skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
                                             RECIPE_STRING, state->procatgon,
                                             onlist, state->remregexp,
                                             visir_pipe_id, state->onname));
        skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
                                             RECIPE_STRING, state->procatgoff,
                                             offlist, state->remregexp,
                                             visir_pipe_id, state->offname));
    }

    /* process the data */
    skip_if(process_data(state, products, usedframes, parlist, onlist, offlist));

    for (const cpl_frame * frame = cpl_frameset_get_first_const(products);
         frame != NULL;
         frame = cpl_frameset_get_next_const(products)) {
        cpl_frame * copy = cpl_frame_duplicate(frame);
        cpl_error_code error;

        error = cpl_frameset_insert(framelist, copy);

        if (error) break;
    }

    end_skip;
    cpl_propertylist_delete(onlist);
    cpl_propertylist_delete(offlist);
    cpl_free(state->onname);
    cpl_free(state->offname);
    cpl_frameset_delete(usedframes);
    cpl_frameset_delete(products);
    cpl_free(state);
    return 0;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Check the consistency of the repacked data
  @param    self The average image
  @param    on   The on-imagelist
  @param    off  The off-imagelist
  @return   CPL_ERROR_NONE iff OK, otherwise the relevant CPL error code.
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_util_repack_check(const cpl_image     * self,
                                              const cpl_imagelist * on,
                                              const cpl_imagelist * off)
{
    /* UNUSED due to chunking */
    cpl_image          * meanon  = cpl_imagelist_collapse_create(on);
    cpl_image          * meanoff = cpl_imagelist_collapse_create(off);
    const cpl_error_code err1  = cpl_image_subtract(meanon, meanoff);
    const cpl_error_code err2  = cpl_image_subtract(meanon, self);
    const unsigned bitmask = CPL_STATS_MIN | CPL_STATS_MAX | CPL_STATS_MEAN
        | CPL_STATS_MEDIAN | CPL_STATS_MEDIAN_DEV | CPL_STATS_STDEV;

    cpl_stats * stats = cpl_stats_new_from_image(meanon, bitmask);

    bug_if(err1 + err2);

    bug_if(cpl_stats_dump(stats, bitmask, stderr));

    end_skip;

    cpl_image_delete(meanon);
    cpl_image_delete(meanoff);
    cpl_stats_delete(stats);

    return cpl_error_get_code();

}
