/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2015 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                              Includes                                      *
 *----------------------------------------------------------------------------*/
#include <cpl.h>
#include <math.h>
#include <string.h>

#include "muse_datacube.h"

#include "muse_flux.h"
#include "muse_pfits.h"
#include "muse_quality.h"
#include "muse_utils.h"
#include "muse_wcs.h"

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_datacube     Datacube handling
 *
 * This contains all functions concerned with handling datacube in the FITS
 * NAXIS=3 and Euro3D formats.
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/*---------------------------------------------------------------------------*/
/**
  @private
  @brief   Interpolate filter curve onto the wavelength grid of the data.
  @param   aFilter   the filter response curve
  @param   aCRVAL    WCS starting value
  @param   aCRPIX    WCS starting pixel
  @param   aCDELT    WCS step width
  @param   aIsLog    set to true if the wavelength is sampling in log space
  @param   aL1       the out-parameter to hold the filter start element
  @param   aL2       the out-parameter to hold the filter end element;
                     this also has to be prefilled with maximum possible
                     filter length!
  @return  A double * buffer on success or NULL on error.

  The returned pointer has to be deallocated using cpl_free() after use.
 */
/*---------------------------------------------------------------------------*/
static double *
muse_datacube_collapse_filter_buffer(cpl_table *aFilter,
                                     double aCRVAL, double aCRPIX, double aCDELT,
                                     cpl_boolean aIsLog, int *aL1, int *aL2)
{
  if (!aFilter || !aL1 || !aL2) {
    return NULL;
  }
  double *fdata = (double *)cpl_calloc(*aL2, sizeof(double));

  int l;
  for (l = 0; l < *aL2; l++) {
    double lambda = (l + 1. - aCRPIX) * aCDELT + aCRVAL;
    if (aIsLog) {
      lambda = aCRVAL * exp((l + 1. - aCRPIX) * aCDELT / aCRVAL);
    }
    fdata[l] = muse_flux_response_interpolate(aFilter, lambda, NULL,
                                              MUSE_FLUX_RESP_FILTER);
  } /* for l (z / wavelengths) */

  /* try to find filter edges */
  l = 0;
  while (l < *aL2 && fabs(fdata[l]) < DBL_EPSILON) {
    *aL1 = l++;
  }
  l = *aL2 - 1;
  while (l > *aL1 && fabs(fdata[l]) < DBL_EPSILON) {
    *aL2 = l--;
  }
  cpl_msg_debug(__func__, "filter wavelength range %.1f..%.1fA (planes %d..%d)",
                (*aL1 + 1. - aCRPIX) * aCDELT + aCRVAL,
                (*aL2 + 1. - aCRPIX) * aCDELT + aCRVAL,
                *aL1, *aL2);

  return fdata;
} /* muse_datacube_collapse_filter_buffer() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Integrate a Euro3D datacube along the wavelength direction.
  @param   aEuro3D     the Euro3D cube to integrate
  @param   aFilter     the filter response curve
  @return  A MUSE image of the field of view (with empty stat component) or
           NULL on error.

  Loop through all wavelengths of the input Euro3D cube and add them up
  into the output image, weighting with the filter response curve.
  Optionally, if the environment variable MUSE_COLLAPSE_USE_VARIANCE is set to
  something positive, the variance information in the input Euro3D table will
  be used to weight the data according to their S/N.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, the input Euro3D table is NULL}
  @error{integrate over the whole wavelength range, filter table is NULL}
  @error{output file will only contain the minimal set of FITS headers,
         header information NULL}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         the aEuro3D table does not contain units for the XPOS and/or YPOS column(s)}
 */
/*---------------------------------------------------------------------------*/
muse_image *
muse_euro3dcube_collapse(muse_euro3dcube *aEuro3D, cpl_table *aFilter)
{
  cpl_ensure(aEuro3D && aEuro3D->dtable && aEuro3D->hdata, CPL_ERROR_NULL_INPUT,
             NULL);

  /* guess dimensions of the output image from the Euro3D data */
  muse_wcs *wcs = muse_wcs_new(aEuro3D->header);
  /* similar test as in muse_pixtable_wcs_check() */
  wcs->iscelsph = CPL_FALSE;
  const char *unitx = cpl_table_get_column_unit(aEuro3D->dtable, "XPOS"),
             *unity = cpl_table_get_column_unit(aEuro3D->dtable, "YPOS");
  cpl_ensure(unitx && unity, CPL_ERROR_DATA_NOT_FOUND, NULL);
  /* should be equal in the first three characters */
  if (!strncmp(unitx, unity, 4) && !strncmp(unitx, "deg", 4)) {
    wcs->iscelsph = CPL_TRUE;
  }
  /* extreme coordinates in pixel or celestial coordinates */
  double xmin = cpl_table_get_column_min(aEuro3D->dtable, "XPOS"),
         xmax = cpl_table_get_column_max(aEuro3D->dtable, "XPOS"),
         ymin = cpl_table_get_column_min(aEuro3D->dtable, "YPOS"),
         ymax = cpl_table_get_column_max(aEuro3D->dtable, "YPOS");
  double x1, x2, y1, y2; /* pixel coordinates */
  if (wcs->iscelsph) {
    wcs->crval1 /= CPL_MATH_DEG_RAD; /* convert to radians */
    wcs->crval2 /= CPL_MATH_DEG_RAD;
    muse_wcs_pixel_from_celestial_fast(wcs, xmin / CPL_MATH_DEG_RAD,
                                       ymin / CPL_MATH_DEG_RAD, &x1, &y1);
    muse_wcs_pixel_from_celestial_fast(wcs, xmax / CPL_MATH_DEG_RAD,
                                       ymax / CPL_MATH_DEG_RAD, &x2, &y2);
  } else {
    x1 = xmin;
    x2 = xmax;
    y1 = ymin;
    y2 = ymax;
  }
  int zmin = cpl_table_get_column_min(aEuro3D->dtable, "SPEC_STA"),
      zmax = cpl_table_get_column_max(aEuro3D->dtable, "SPEC_STA"),
      nx = lround(fabs(x2 - x1)) + 1,
      ny = lround(fabs(y2 - y1)) + 1,
      nz = zmax - zmin + 1;

  /* count how many valid elements really are in the most shifted spectrum */
  cpl_size zmaxpos = -1;
  cpl_table_get_column_maxpos(aEuro3D->dtable, "SPEC_STA", &zmaxpos);
  const cpl_array *amax = cpl_table_get_array(aEuro3D->dtable, "DATA_SPE",
                                              zmaxpos);
  int l, nsize = cpl_array_get_size(amax);
  for (l = nsize - 1; l > 0; l--) {
    /* find last valid element */
    if (cpl_array_is_valid(amax, l) == 1) {
      break;
    }
  }
  nz += l + 1; /* add the number of valid elements in spectral direction */
  int nspec = cpl_table_get_nrow(aEuro3D->dtable);
  cpl_msg_debug(__func__, "Euro3D dimensions: %dx%dx%d (z = %d - %d, valid %d),"
                " %d spectra", nx, ny, nz, zmax, zmin, l + 1, nspec);

  /* resample the filter curve on the wavelength grid */
  double crvals = cpl_propertylist_get_double(aEuro3D->hdata, "CRVALS"),
         cdelts = cpl_propertylist_get_double(aEuro3D->hdata, "CDELTS");
  const char *ctypes = cpl_propertylist_get_string(aEuro3D->hdata, "CTYPES");
  cpl_boolean loglambda = ctypes && (!strncmp(ctypes, "AWAV-LOG", 9) ||
                                     !strncmp(ctypes, "WAVE-LOG", 9));
  cpl_msg_debug(__func__, "spectral WCS: %f / %f %s", crvals, cdelts,
                loglambda ? "log" : "linear");
  int l1 = 0, l2 = nz; /* loop boundaries for filter operation */
  double *fdata = muse_datacube_collapse_filter_buffer(aFilter, crvals, zmin,
                                                       cdelts, loglambda, &l1, &l2);

  /* create output muse_image, but only with two image extensions */
  muse_image *image = muse_image_new();
  image->header = cpl_propertylist_duplicate(aEuro3D->header);
  /* the axis3 WCS was already erased in muse_resampling_euro3d() */
  image->data = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
  float *outdata = cpl_image_get_data_float(image->data);
  image->dq = cpl_image_new(nx, ny, CPL_TYPE_INT);
  /* pre-fill with Euro3D status for missing data */
  cpl_image_add_scalar(image->dq, EURO3D_MISSDATA);
  int *outdq = cpl_image_get_data_int(image->dq);
  /* image->stat remains NULL, variance information is lost by collapsing */

  /* check if we need to use the variance for weighting */
  cpl_boolean usevariance = CPL_FALSE;
  if (getenv("MUSE_COLLAPSE_USE_VARIANCE") &&
      atoi(getenv("MUSE_COLLAPSE_USE_VARIANCE")) > 0) {
    usevariance = CPL_TRUE;
  }

  /* loop through all spaxels and over all wavelengths */
  int k, noutside = 0;
  #pragma omp parallel for default(none) private(l)      /* as req. by Ralf */ \
          shared(aEuro3D, fdata, l1, l2, noutside, nspec, nx, ny, outdata,     \
                 outdq, usevariance, wcs)
  for (k = 0; k < nspec; k++) {
    int err;
    double xpos = cpl_table_get(aEuro3D->dtable, "XPOS", k, &err),
           ypos = cpl_table_get(aEuro3D->dtable, "YPOS", k, &err);
    if (err) {
      cpl_msg_warning(__func__, "spectrum %d in Euro3D table does not have "
                      "position information!", k + 1);
      continue;
    }
    /* coordinate in the output image (indices starting at 0) */
    double xpx, ypx;
    if (wcs->iscelsph) {
      muse_wcs_pixel_from_celestial_fast(wcs, xpos * CPL_MATH_RAD_DEG,
                                         ypos * CPL_MATH_RAD_DEG, &xpx, &ypx);
    } else {
      muse_wcs_pixel_from_projplane_fast(wcs, xpos, ypos, &xpx, &ypx);
    }
    int i = lround(xpx) - 1,
        j = lround(ypx) - 1;
    if (i >= nx || j >= ny) {
      /* should not happen, but skip this spectrum to avoid segfault */
      noutside++;
      continue;
    }

    int nstart = cpl_table_get_int(aEuro3D->dtable, "SPEC_STA", k, &err);
    const cpl_array *adata = cpl_table_get_array(aEuro3D->dtable, "DATA_SPE", k),
                    *adq = cpl_table_get_array(aEuro3D->dtable, "QUAL_SPE", k),
                    *astat = NULL;
    if (usevariance) {
      astat = cpl_table_get_array(aEuro3D->dtable, "STAT_SPE", k);
    }

    double sumdata = 0., sumweight = 0.;
    for (l = l1; l < l2; l++) {
      /* array index for this wavelength */
      int idx = l - nstart + 1;

      /* filter weight with fallback for operation without filter */
      double fweight = fdata ? fdata[l] : 1.;
      cpl_errorstate prestate = cpl_errorstate_get();
      int dq = cpl_array_get_int(adq, idx, &err);
      if (dq || err) { /* if pixel marked bad or inaccessible */
        cpl_errorstate_set(prestate);
        continue; /* ignore bad pixels of any kind */
      }
      /* make the variance information optional */
      double variance = 1.;
      if (usevariance) {
        variance = astat ? cpl_array_get(astat, idx, &err) : 1.;
        /* use ^2 as for Euro3D we store errors not variances directly */
        variance *= variance;
        if (err > 0) {
          variance = DBL_MAX;
        }
        if (err || !isnormal(variance)) {
          continue;
        }
      }
      double data = cpl_array_get(adata, idx, &err);
      /* weight by inverse variance */
      sumdata += data * fweight / variance;
      sumweight += fweight / variance;
    } /* for l (z / wavelengths) */
    if (isnormal(sumweight)) {
      outdata[i + j*nx] = sumdata / sumweight;
      outdq[i + j*nx] = EURO3D_GOODPIXEL;
    } else { /* leave bad/missing pixels zero, but mark them bad */
      outdq[i + j*nx] = EURO3D_MISSDATA;
    }
  } /* for k (all spaxels) */
  cpl_free(wcs);
  cpl_free(fdata); /* won't crash on NULL */
  if (noutside > 0) {
    cpl_msg_warning(__func__, "Skipped %d spaxels, due to their location "
                    "outside the recostructed image!", noutside);
  }

  return image;
} /* muse_euro3dcube_collapse() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Integrate a FITS NAXIS=3 datacube along the wavelength direction.
  @param   aCube       the datacube to integrate
  @param   aFilter     the filter response curve
  @return  A MUSE image of the field of view (with empty stat component) or
           NULL on error.

  Loop through all wavelength planes of the input cube and add them up into
  the output image, weighting with the filter response curve at every
  wavelength.
  Optionally, if the environment variable MUSE_COLLAPSE_USE_VARIANCE is set to
  something positive, the variance information in the input cube will be used to
  weight the data according to their S/N.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL,
         the input cube\, its data\, or its header is NULL}
  @error{set CPL_ERROR_INCOMPATIBLE_INPUT\, return NULL,
         the input cube components\, if they exist\, are not of the expected type (float and int)}
  @error{integrate over the whole wavelength range, filter table is NULL}
 */
/*---------------------------------------------------------------------------*/
muse_image *
muse_datacube_collapse(muse_datacube *aCube, cpl_table *aFilter)
{
  cpl_ensure(aCube && aCube->data && aCube->header, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(cpl_image_get_type(cpl_imagelist_get(aCube->data, 0)) == CPL_TYPE_FLOAT,
             CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  if (aCube->dq) {
    cpl_ensure(cpl_image_get_type(cpl_imagelist_get(aCube->dq, 0)) == CPL_TYPE_INT,
               CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  }
  if (aCube->stat) {
    cpl_ensure(cpl_image_get_type(cpl_imagelist_get(aCube->stat, 0)) == CPL_TYPE_FLOAT,
               CPL_ERROR_INCOMPATIBLE_INPUT, NULL);
  }

  /* find in/output dimensions */
  int nx = cpl_image_get_size_x(cpl_imagelist_get(aCube->data, 0)),
      ny = cpl_image_get_size_y(cpl_imagelist_get(aCube->data, 0)),
      nz = cpl_imagelist_get_size(aCube->data);
  /* resample the filter curve on the wavelength grid */
  double crpix3 = muse_pfits_get_crpix(aCube->header, 3),
         crval3 = muse_pfits_get_crval(aCube->header, 3),
         cd33 = muse_pfits_get_cd(aCube->header, 3, 3);
  const char *ctype3 = muse_pfits_get_ctype(aCube->header, 3);
  cpl_boolean loglambda = ctype3 && (!strncmp(ctype3, "AWAV-LOG", 9) ||
                                     !strncmp(ctype3, "WAVE-LOG", 9));
  int l1 = 0, l2 = nz; /* loop boundaries for filter operation */
  double *fdata = muse_datacube_collapse_filter_buffer(aFilter, crval3, crpix3,
                                                       cd33, loglambda, &l1, &l2);

  /* create output muse_image, but only with two image extensions */
  muse_image *image = muse_image_new();
  image->header = cpl_propertylist_duplicate(aCube->header);
  cpl_propertylist_erase_regexp(image->header, "^C...*3$|^CD3_.$", 0);
  image->data = cpl_image_new(nx, ny, CPL_TYPE_FLOAT);
  float *outdata = cpl_image_get_data_float(image->data);
  image->dq = cpl_image_new(nx, ny, CPL_TYPE_INT);
  int *outdq = cpl_image_get_data_int(image->dq);
  /* image->stat remains NULL, variance information is lost by collapsing */

  /* check if we need to use the variance for weighting */
  cpl_boolean usevariance = CPL_FALSE;
  if (getenv("MUSE_COLLAPSE_USE_VARIANCE") &&
      atoi(getenv("MUSE_COLLAPSE_USE_VARIANCE")) > 0) {
    usevariance = CPL_TRUE;
  }

  /* loop through all pixels in all wavelength planes */
  int i;
  #pragma omp parallel for default(none)                 /* as req. by Ralf */ \
          shared(aCube, nx, ny, l1, l2, fdata, outdata, outdq, usevariance)
  for (i = 0; i < nx; i++) {
    int j;
    for (j = 0; j < ny; j++) {
      double sumdata = 0., sumweight = 0.;
      int l;
      for (l = l1; l < l2; l++) {
        /* filter weight with fallback for operation without filter */
        double fweight = fdata ? fdata[l] : 1.;
        float *pdata = cpl_image_get_data_float(cpl_imagelist_get(aCube->data, l)),
              *pstat = NULL;
        if (!isfinite(pdata[i + j*nx])) {
          continue;
        }
        if (aCube->dq) {
          int *pdq = cpl_image_get_data_int(cpl_imagelist_get(aCube->dq, l));
          if (pdq && pdq[i + j*nx]) {
            continue; /* ignore bad pixels of any kind */
          }
        }
        /* make the variance information optional */
        double variance = 1.;
        if (usevariance) {
          pstat = cpl_image_get_data_float(cpl_imagelist_get(aCube->stat, l));
          variance = pstat ? pstat[i + j*nx] : 1.;
          if (!isnormal(variance)) {
            continue;
          }
        }
        /* weight by inverse variance */
        sumdata += pdata[i + j*nx] * fweight / variance;
        sumweight += fweight / variance;
      } /* for l (z / wavelengths) */
      if (isnormal(sumweight)) {
        outdata[i + j*nx] = sumdata / sumweight;
        outdq[i + j*nx] = EURO3D_GOODPIXEL;
      } else { /* leave bad/missing pixels zero, but mark them bad */
        outdq[i + j*nx] = EURO3D_MISSDATA;
      }
    } /* for j (y pixels) */
  } /* for i (x pixels) */

  cpl_free(fdata); /* won't crash on NULL */

  return image;
} /* muse_datacube_collapse() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Save reconstructed images of a cube in extra extensions.
  @param    aFilename   name of the output file
  @param    aImages     image list to save as separate extensions
  @param    aNames      names of all filters used to create the images
  @return   CPL_ERROR_NONE or the relevant cpl_error_code on error

  Saving of the BUNIT keywords is modeled after @ref muse_image_save(), i.e.
  the data extension uses the incoming BUNIT entry (if present), the variance
  extension gets the squared unit, and the data quality does not get any BUNIT.

  @error{return CPL_ERROR_NULL_INPUT, aFilename is NULL}
  @error{immediately return CPL_ERROR_NONE, aImages or aNames are NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_datacube_save_recimages(const char *aFilename, muse_imagelist *aImages,
                             cpl_array *aNames)
{
  cpl_ensure_code(aFilename, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = CPL_ERROR_NONE;
  if (!aImages || !aNames) {
    return rc; /* this is a valid case, return without error */
  }
  unsigned int i, nimages = muse_imagelist_get_size(aImages);
  for (i = 0; i < nimages; i++) {
    muse_image *image = muse_imagelist_get(aImages, i);

    /* create small header with EXTNAME=filtername and WCS for images */
    cpl_propertylist *header = cpl_propertylist_new();
    cpl_errorstate es = cpl_errorstate_get();
    const char *unit = muse_pfits_get_bunit(image->header),
               *ucomment = cpl_propertylist_get_comment(image->header, "BUNIT");
    if (!cpl_errorstate_is_equal(es) && !unit) {
      cpl_errorstate_set(es); /* swallow errors, if there is no unit to propagate */
    }
    char dataext[KEYWORD_LENGTH], obj[KEYWORD_LENGTH],
         *dqext = NULL, *statext = NULL;
    snprintf(dataext, KEYWORD_LENGTH, "%s", cpl_array_get_string(aNames, i));
    if (image->dq) {
      dqext = cpl_sprintf("%s_%s", cpl_array_get_string(aNames, i), EXTNAME_DQ);
    }
    if (image->stat) {
      statext = cpl_sprintf("%s_%s", cpl_array_get_string(aNames, i), EXTNAME_STAT);
    }
    snprintf(obj, KEYWORD_LENGTH, "%s", cpl_array_get_string(aNames, i));
    cpl_propertylist_append_string(header, "EXTNAME", dataext);
    cpl_propertylist_set_comment(header, "EXTNAME", "reconstructed image (data values)");
    /* propagate the unit in the same way as in muse_image_save() */
    if (unit) {
      cpl_propertylist_append_string(header, "BUNIT", unit);
      cpl_propertylist_set_comment(header, "BUNIT", ucomment);
    }
    /* update the OBJECT header with the filter name */
    muse_utils_copy_modified_header(image->header, header, "OBJECT", obj);
    cpl_propertylist_update_string(header, MUSE_HDR_FILTER,
                                   cpl_array_get_string(aNames, i));
    cpl_propertylist_copy_property_regexp(header, image->header,
                                          MUSE_WCS_KEYS, 0);
    muse_utils_set_hduclass(header, "DATA", dataext, dqext, statext);
    rc = cpl_image_save(image->data, aFilename, CPL_TYPE_UNSPECIFIED, header,
                        CPL_IO_EXTEND);

    if (image->dq) {
      cpl_propertylist_update_string(header, "EXTNAME", dqext);
      cpl_propertylist_set_comment(header, "EXTNAME", "reconstructed image (bad pixel status values)");
      cpl_propertylist_erase(header, "BUNIT"); /* no unit for data quality */
      snprintf(obj, KEYWORD_LENGTH, "%s, %s", cpl_array_get_string(aNames, i),
               EXTNAME_DQ);
      muse_utils_copy_modified_header(image->header, header, "OBJECT", obj);
      muse_utils_set_hduclass(header, "QUALITY", dataext, dqext, statext);
      rc = cpl_image_save(image->dq, aFilename, CPL_TYPE_INT, header,
                          CPL_IO_EXTEND);
    }
    if (image->stat) {
      cpl_propertylist_update_string(header, "EXTNAME", statext);
      cpl_propertylist_set_comment(header, "EXTNAME", "reconstructed image (variance)");
      if (unit) {
        char *ustat = cpl_sprintf("(%s)**2", unit); /* variance in squared units */
        cpl_propertylist_append_string(header, "BUNIT", ustat);
        cpl_free(ustat);
      }
      snprintf(obj, KEYWORD_LENGTH, "%s, %s", cpl_array_get_string(aNames, i),
               EXTNAME_STAT);
      muse_utils_copy_modified_header(image->header, header, "OBJECT", obj);
      muse_utils_set_hduclass(header, "ERROR", dataext, dqext, statext);
      rc = cpl_image_save(image->stat, aFilename, CPL_TYPE_UNSPECIFIED, header,
                          CPL_IO_EXTEND);
    }

    cpl_propertylist_delete(header);
    cpl_free(dqext);
    cpl_free(statext);
  } /* for i (all reconstructed images) */

  return rc;
} /* muse_datacube_save_recimages */

/*---------------------------------------------------------------------------*/
/**
  @brief    Save a Euro3D cube object to a file.
  @param    aEuro3D     input Euro3D cube
  @param    aFilename   name of the output file
  @return   CPL_ERROR_NONE or the relevant cpl_error_code on error

  Just calls @c cpl_table_save() for the two table components of a
  <tt><b>muse_euro3dcube</b></tt>, using the header components to construct
  primary and both extension headers. It appends further image extensions for
  each reconstructed image in the Euro3D structure.

  @error{return CPL_ERROR_NULL_INPUT, aEuro3D or aFilename are NULL}
  @error{propagate CPL error code, failure to save one of the tables}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_euro3dcube_save(muse_euro3dcube *aEuro3D, const char *aFilename)
{
  cpl_ensure_code(aEuro3D && aFilename, CPL_ERROR_NULL_INPUT);

  /* save primary header and the data table in the first extension */
  cpl_error_code rc = cpl_table_save(aEuro3D->dtable, aEuro3D->header,
                                     aEuro3D->hdata, aFilename, CPL_IO_CREATE);
  if (rc != CPL_ERROR_NONE) {
    cpl_msg_warning(__func__, "failed to save data part of the Euro3D table: "
                    "%s", cpl_error_get_message());
  }
  /* save group header and data in the second extension */
  rc = cpl_table_save(aEuro3D->gtable, NULL, aEuro3D->hgroup, aFilename,
                      CPL_IO_EXTEND);
  if (rc != CPL_ERROR_NONE) {
    cpl_msg_warning(__func__, "failed to save group part of the Euro3D table: "
                    "%s", cpl_error_get_message());
  }

  rc = muse_datacube_save_recimages(aFilename, aEuro3D->recimages,
                                    aEuro3D->recnames);
  return rc;
} /* muse_euro3dcube_save() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Deallocate memory associated to a muse_euro3dcube object.
  @param    aEuro3D   input Euro3D cube

  Just calls @c cpl_table_delete() and @c cpl_propertylist_delete() for the
  five components of a <tt><b>muse_euro3dcube</b></tt>, and frees memory for the
  aEuro3D pointer. As a safeguard, it checks if a valid pointer was passed,
  so that crashes cannot occur.
 */
/*---------------------------------------------------------------------------*/
void
muse_euro3dcube_delete(muse_euro3dcube *aEuro3D)
{
  /* if the euro3d object doesn't exists at all, we don't need to do anything */
  if (!aEuro3D) {
    return;
  }

  /* checks for the existence of the sub-images *
   * are done in the CPL functions              */
  cpl_table_delete(aEuro3D->dtable);
  cpl_table_delete(aEuro3D->gtable);
  cpl_propertylist_delete(aEuro3D->header);
  cpl_propertylist_delete(aEuro3D->hdata);
  cpl_propertylist_delete(aEuro3D->hgroup);
  muse_imagelist_delete(aEuro3D->recimages);
  cpl_array_delete(aEuro3D->recnames);
  cpl_free(aEuro3D);
} /* muse_euro3dcube_delete() */

/*---------------------------------------------------------------------------*/
/**
  @private
  @brief    Convert the DQ extension of all reconstructed images to NANs in DATA
            (and STAT).
  @param    aCube       input MUSE datacube
  @return   CPL_ERROR_NONE or the relevant cpl_error_code on error

  @error{return CPL_ERROR_NULL_INPUT, aCube is NULL}
 */
/*---------------------------------------------------------------------------*/
static cpl_error_code
muse_datacube_convert_dq_recimages(muse_datacube *aCube)
{
  cpl_ensure_code(aCube, CPL_ERROR_NULL_INPUT);
  if (!aCube->recimages) {
    return CPL_ERROR_NONE; /* valid case, return without error */
  }
  unsigned int k, nimages = muse_imagelist_get_size(aCube->recimages);
  for (k = 0; k < nimages; k++) {
    muse_image *image = muse_imagelist_get(aCube->recimages, k);
    if (!image->dq) { /* no point trying to convert... */
      continue;
    }
    muse_image_dq_to_nan(image);
  } /* for k (all reconstructed images) */

  return CPL_ERROR_NONE;
} /* muse_datacube_convert_dq_recimages() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Convert the DQ extension of a datacube to NANs in DATA and STAT.
  @param    aCube       input MUSE datacube
  @return   CPL_ERROR_NONE or the relevant cpl_error_code on error

  This also converts the DQ extension of all reconstructed images that are
  part of the cube in the same way.
  All DQ components are deallocated and their pointers are set to NULL.

  @error{return CPL_ERROR_NULL_INPUT,
         aCube or one of its data\, stat\, dq extensions are NULL}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_datacube_convert_dq(muse_datacube *aCube)
{
  cpl_ensure_code(aCube && aCube->data && aCube->stat && aCube->dq,
                  CPL_ERROR_NULL_INPUT);

  /* loop through all wavelength planes and then all pixels per plane */
  int l, nx = cpl_image_get_size_x(cpl_imagelist_get(aCube->data, 0)),
      ny = cpl_image_get_size_y(cpl_imagelist_get(aCube->data, 0)),
      nz = cpl_imagelist_get_size(aCube->data);
  #pragma omp parallel for default(none)                 /* as req. by Ralf */ \
          shared(aCube, nx, ny, nz)
  for (l = 0; l < nz; l++) {
    int i;
    for (i = 0; i < nx; i++) {
      int j;
      for (j = 0; j < ny; j++) {
        float *pdata = cpl_image_get_data_float(cpl_imagelist_get(aCube->data, l)),
              *pstat = cpl_image_get_data_float(cpl_imagelist_get(aCube->stat, l));
        int *pdq = cpl_image_get_data_int(cpl_imagelist_get(aCube->dq, l));
        if (pdq[i + j*nx] == EURO3D_GOODPIXEL) {
          continue; /* nothing to do for good pixels */
        }
        /* set bad pixels of any type to not-a-number in both extensions */
        pdata[i + j*nx] = NAN; /* supposed to be quiet NaN */
        pstat[i + j*nx] = NAN;
      } /* for j (y direction) */
    } /* for i (x direction) */
  } /* for l (wavelength planes) */

  /* deallocate DQ and set it to NULL to be able to check for it */
  cpl_imagelist_delete(aCube->dq);
  aCube->dq = NULL;

  /* do the same for the reconstructed images, if there are any */
  muse_datacube_convert_dq_recimages(aCube);

  return CPL_ERROR_NONE;
} /* muse_datacube_convert_dq() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Save the three cube extensions and the FITS headers of a MUSE
            datacube to a file.
  @param    aCube       input MUSE datacube
  @param    aFilename   name of the output file
  @return   CPL_ERROR_NONE or the relevant cpl_error_code on error
  @remark   The primary header of the output file is constructed from the
            header member of the <tt><b>muse_datacube</b></tt> structure,
            but without WCS keys.
  @remark   The extension headers will only contain the minimal keywords
            (including WCS) plus EXTNAMEs that advertise their function (DATA,
            DQ, STAT) and a comment explaining their purpose.
  @remark   Both dq and stat components of aCube are optional, the saved result
            may not contain them.

  Just calls @c cpl_imagelist_save() for the three components of a
  <tt><b>muse_datacube</b></tt>, using the keywords in the header element of the
  <tt><b>muse_datacube</b></tt> structure for construction of the primary header.
  It appends further image extensions for each reconstructed image in the cube
  structure.

  This function uses @ref muse_utils_set_hduclass() to add the special FITS
  headers to support the ESO format.

  @error{return CPL_ERROR_NULL_INPUT,
         aCube\, its header component\, or aFilename are NULL}
  @error{propagate CPL error code, failure to save any of the three components}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_datacube_save(muse_datacube *aCube, const char *aFilename)
{
  cpl_ensure_code(aCube && aCube->header && aFilename, CPL_ERROR_NULL_INPUT);

  /* save headers into primary area */
  cpl_propertylist *header = cpl_propertylist_new();
  /* copy all headers, except the main WCS keys */
  cpl_propertylist_copy_property_regexp(header, aCube->header,
                                        MUSE_WCS_KEYS, 1);
  cpl_error_code rc = cpl_propertylist_save(header, aFilename, CPL_IO_CREATE);
  cpl_propertylist_delete(header);

  /* save data */
  header = cpl_propertylist_new();
  cpl_propertylist_append_string(header, "EXTNAME", EXTNAME_DATA);
  cpl_propertylist_set_comment(header, "EXTNAME", EXTNAME_DATA_COMMENT);
  muse_utils_copy_modified_header(aCube->header, header, "OBJECT",
                                  EXTNAME_DATA);
  cpl_propertylist_copy_property_regexp(header, aCube->header,
                                        MUSE_WCS_KEYS"|^BUNIT", 0);
  muse_utils_set_hduclass(header, "DATA", "DATA",
                          aCube->dq ? "DQ" : NULL, aCube->stat ? "STAT" : NULL);
  rc = cpl_imagelist_save(aCube->data, aFilename, CPL_TYPE_FLOAT, header,
                          CPL_IO_EXTEND);
  cpl_propertylist_delete(header);
  /* save bad pixels, if available */
  if (rc == CPL_ERROR_NONE && aCube->dq) {
    header = cpl_propertylist_new();
    cpl_propertylist_append_string(header, "EXTNAME", EXTNAME_DQ);
    cpl_propertylist_set_comment(header, "EXTNAME", EXTNAME_DQ_COMMENT);
    muse_utils_copy_modified_header(aCube->header, header, "OBJECT",
                                    EXTNAME_DQ);
    cpl_propertylist_copy_property_regexp(header, aCube->header,
                                          MUSE_WCS_KEYS, 0);
    muse_utils_set_hduclass(header, "QUALITY", "DATA", "DQ",
                            aCube->stat ? "STAT" : NULL);
    rc = cpl_imagelist_save(aCube->dq, aFilename, CPL_TYPE_INT, header,
                            CPL_IO_EXTEND);
    cpl_propertylist_delete(header);
  }
  /* save variance, if available */
  if (rc == CPL_ERROR_NONE && aCube->stat) {
    header = cpl_propertylist_new();
    cpl_propertylist_append_string(header, "EXTNAME", EXTNAME_STAT);
    cpl_propertylist_set_comment(header, "EXTNAME", EXTNAME_STAT_COMMENT);
    /* add the correct BUNIT, depending on the data units */
    if (!strncmp(muse_pfits_get_bunit(aCube->header),
                 kMuseFluxUnitString, strlen(kMuseFluxUnitString) + 1)) {
      /* flux calibrated data */
      cpl_propertylist_append_string(header, "BUNIT", kMuseFluxStatString);
    }
    muse_utils_copy_modified_header(aCube->header, header, "OBJECT",
                                    EXTNAME_STAT);
    cpl_propertylist_copy_property_regexp(header, aCube->header,
                                          MUSE_WCS_KEYS, 0);
    muse_utils_set_hduclass(header, "ERROR", "DATA", aCube->dq ? "DQ" : NULL,
                            "STAT");
    rc = cpl_imagelist_save(aCube->stat, aFilename, CPL_TYPE_FLOAT, header,
                            CPL_IO_EXTEND);
    cpl_propertylist_delete(header);
  }

  rc = muse_datacube_save_recimages(aFilename, aCube->recimages, aCube->recnames);
  return rc;
} /* muse_datacube_save() */

/*---------------------------------------------------------------------------*/
/*
  @private
  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aFilename is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL, aFilename does not exist}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         aFilename does not contain a DATA extension}
 */
/*---------------------------------------------------------------------------*/
static cpl_propertylist *
muse_datacube_load_header(const char *aFilename)
{
  cpl_ensure(aFilename, CPL_ERROR_NULL_INPUT, NULL);
  int extdata = cpl_fits_find_extension(aFilename, "DATA");
  cpl_ensure(extdata >= 0, CPL_ERROR_ILLEGAL_INPUT, NULL);
  cpl_ensure(extdata > 0, CPL_ERROR_DATA_NOT_FOUND, NULL);

  cpl_propertylist *header = cpl_propertylist_load(aFilename, 0);
  cpl_propertylist *hdata = cpl_propertylist_load(aFilename, extdata);
  cpl_propertylist_copy_property_regexp(header, hdata,
                                        MUSE_WCS_KEYS"|BUNIT", 0);
  cpl_propertylist_delete(hdata);
  return header;
} /* muse_datacube_load_header() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Load header, DATA and optionally STAT and DQ extensions as well as
            the reconstructed images of a MUSE cube from disk.
  @param    aFilename   name of the file on disk
  @return   a muse_datacube * or NULL on error
  @remark   The primary header of the returned cube is constructed from the
            primary HDU of the file and the WCS keywords as well as BUNIT of
            the DATA extension.
  @remark   Both DQ and STAT components of the file are optional.
  @remark   The types of the three primary extensions (DATA, DQ, and STAT) are
            are casted to float, int, and float, respectively, during loading.
            The types of the reconstructed images are taken from the file.
  @warning  This function ignores the ESO-format headers like HDUCLASS.
  @note     When loading extra reconstructed images from the cube file, this
            function assumes that DQ and STAT extensions belonging to a
            reconstructed image are in the extensions directly after the
            main image extension for that reconstructed image. I.e., if an
            image extension "filt1" exists, then the corresponding "filt1_DQ"
            and/or "filt1_STAT" extensions have to follow right after it.
            Since this is what @ref muse_datacube_save() saves to disk, it
            should be true in most cases.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aFilename is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL, aFilename does not exist}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         aFilename does not contain a DATA extension}
  @error{output error message\, return NULL,
         header could not be loaded/merged from primary and DATA extensions}
 */
/*---------------------------------------------------------------------------*/
muse_datacube *
muse_datacube_load(const char *aFilename)
{
  cpl_ensure(aFilename, CPL_ERROR_NULL_INPUT, NULL);
  muse_datacube *cube = cpl_calloc(1, sizeof(muse_datacube));
  /* load primary header and merge in relevant entries from the DATA header */
  cpl_errorstate state = cpl_errorstate_get();
  cube->header = muse_datacube_load_header(aFilename);
  if (!cpl_errorstate_is_equal(state) || !cube->header) {
    cpl_msg_error(__func__, "Loading cube-like headers from \"%s\" failed!",
                  aFilename);
    cpl_free(cube);
    return NULL;
  }
  int ext = cpl_fits_find_extension(aFilename, "DATA");
  cube->data = cpl_imagelist_load(aFilename, CPL_TYPE_FLOAT, ext);
  /* DQ is usually not written to disk, but see if it's there and load it, if so */
  ext = cpl_fits_find_extension(aFilename, "DQ");
  if (ext > 0) {
    cube->stat = cpl_imagelist_load(aFilename, CPL_TYPE_INT, ext);
  }
  ext = cpl_fits_find_extension(aFilename, "STAT");
  if (ext > 0) {
    cube->stat = cpl_imagelist_load(aFilename, CPL_TYPE_FLOAT, ext);
  }
  int next = cpl_fits_count_extensions(aFilename);
  while (++ext <= next) {
    /* there is (one more) collapsed image, load it */
    muse_image *image = muse_image_new();
    image->header = cpl_propertylist_load(aFilename, ext);
    image->data = cpl_image_load(aFilename, CPL_TYPE_UNSPECIFIED, 0, ext);
    /* search for an load DQ and STAT extensions of this image, if present */
    const char *extname = muse_pfits_get_extname(image->header);
    char *extname2 = cpl_sprintf("%s_DQ", extname);
    int ext2 = cpl_fits_find_extension(aFilename, extname2);
    cpl_free(extname2);
    if (ext2 > 0) {
      image->dq = cpl_image_load(aFilename, CPL_TYPE_INT, 0, ext2);
      ext = ext2; /* skip this as a normal reconstructed image */
    }
    extname2 = cpl_sprintf("%s_STAT", extname);
    ext2 = cpl_fits_find_extension(aFilename, extname2);
    cpl_free(extname2);
    if (ext2 > 0) {
      image->stat = cpl_image_load(aFilename, CPL_TYPE_UNSPECIFIED, 0, ext2);
      ext = ext2; /* skip this as a normal reconstructed image */
    }

    if (!cube->recnames) {
      cube->recnames = cpl_array_new(1, CPL_TYPE_STRING);
    } else {
      cpl_array_set_size(cube->recnames, cpl_array_get_size(cube->recnames) + 1);
    }
    /* append name of this reconstructed image at the end of the names array */
    cpl_array_set_string(cube->recnames, cpl_array_get_size(cube->recnames) - 1,
                         extname);
    if (!cube->recimages) {
      cube->recimages = muse_imagelist_new();
    }
    /* append the image to the end of the reconstructed image list */
    muse_imagelist_set(cube->recimages, image,
                       muse_imagelist_get_size(cube->recimages));
  }
  return cube;
} /* muse_datacube_load() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Concatenate one datacube at the end of another one.
  @param    aCube     the cube to append to
  @param    aAppend   the cube to append
  @return   CPL_ERROR_NONE on success, another CPL error code on failure

  @note The reconstructed images present in aCube are most likely meaningless
        after concatenating. Hence, they are removed by this function!
  @note Both cubes must be of the same spectral type (AWAV or WAVE).
  @note A logarithmic dispersion axis is currently not supported.

  @error{return and set CPL_ERROR_NULL_INPUT, aCube and/or aAppend are NULL}
  @error{return and set CPL_ERROR_ILLEGAL_INPUT,
         image planes of the cubes have different sizes}
  @error{return and set CPL_ERROR_ILLEGAL_INPUT,
         sampling in dispersion direction is different and/or not adjacent}
  @error{return and set CPL_ERROR_ILLEGAL_INPUT,
         cpl_imagelist sizes in one of the input cubes do not match between data and stat components}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_datacube_concat(muse_datacube *aCube, const muse_datacube *aAppend)
{
  cpl_ensure_code(aCube && aAppend, CPL_ERROR_NULL_INPUT);

  cpl_size ndata = cpl_imagelist_get_size(aCube->data),
           nstat = cpl_imagelist_get_size(aCube->stat);
  cpl_ensure_code(ndata == nstat, CPL_ERROR_ILLEGAL_INPUT);
  cpl_size i, n = cpl_imagelist_get_size(aAppend->data);
  cpl_ensure_code(n == cpl_imagelist_get_size(aAppend->stat),
                  CPL_ERROR_ILLEGAL_INPUT);

  /* check sizes of the image planes */
  cpl_size nx1 = cpl_image_get_size_x(cpl_imagelist_get(aCube->data, ndata - 1)),
           ny1 = cpl_image_get_size_y(cpl_imagelist_get(aCube->data, ndata - 1)),
           nx2 = cpl_image_get_size_x(cpl_imagelist_get(aAppend->data, 0)),
           ny2 = cpl_image_get_size_y(cpl_imagelist_get(aAppend->data, 0));
  cpl_ensure_code(nx1 == nx2 && ny1 == ny2, CPL_ERROR_ILLEGAL_INPUT);
  const char *ctype1 = muse_pfits_get_ctype(aCube->header, 3),
             *ctype2 = muse_pfits_get_ctype(aCube->header, 3);
  cpl_ensure_code(ctype1 && ctype2, CPL_ERROR_UNSUPPORTED_MODE);
  cpl_ensure_code((!strncmp(ctype1, "AWAV", 5) && !strncmp(ctype2, "AWAV", 5)) ||
                  (!strncmp(ctype1, "WAVE", 5) && !strncmp(ctype2, "WAVE", 5)),
                  CPL_ERROR_UNSUPPORTED_MODE);
  /* compute wavelengths of last plane of first cube and first plane *
   * of the cube to append, to check that they are adjacent          */
  double pix1 = muse_pfits_get_crpix(aCube->header, 3),
         val1 = muse_pfits_get_crval(aCube->header, 3),
         cd1 = muse_pfits_get_cd(aCube->header, 3, 3),
         pix2 = muse_pfits_get_crpix(aAppend->header, 3),
         val2 = muse_pfits_get_crval(aAppend->header, 3),
         cd2 = muse_pfits_get_cd(aAppend->header, 3, 3),
         lbda1 = ((ndata - 1) + 1. - pix1) * cd1 + val1,
         lbda2 = (0 + 1. - pix2) * cd2 + val2;
  /* AWAV-LOG and WAVE-LOG axes need more complex handling, so exclude them above */
  cpl_msg_debug(__func__, "lambdas: %f %f (%f %f)", lbda1, lbda2, cd1, cd2);
  cpl_ensure_code(fabs(cd1 - cd2) < DBL_EPSILON &&
                  fabs(lbda2 - cd2 - lbda1) < DBL_EPSILON,
                  CPL_ERROR_ILLEGAL_INPUT);

  /* reconstructed images are now obsolete, remove them */
  cpl_array_delete(aCube->recnames);
  aCube->recnames = NULL;
  muse_imagelist_delete(aCube->recimages);
  aCube->recimages = NULL;

  /* Check if there is a usable DQ component in both cubes. If so, *
   * propagate them as the other two components, otherwise delete  *
   * the incoming DQ of the first (= output) cube.                 */
  cpl_boolean usedq = CPL_FALSE;
  if (aCube->dq && cpl_imagelist_get_size(aCube->dq) == ndata &&
      aAppend->dq && cpl_imagelist_get_size(aAppend->dq) == n) {
    usedq = CPL_TRUE;
  } else {
    cpl_imagelist_delete(aCube->dq);
    aCube->dq = NULL;
  }

  /* append the CPL imagelists */
  for (i = 0; i < n; i++) {
    cpl_imagelist_set(aCube->data,
                      cpl_image_duplicate(cpl_imagelist_get(aAppend->data, i)),
                      cpl_imagelist_get_size(aCube->data));
    if (usedq) {
      cpl_imagelist_set(aCube->dq,
                        cpl_image_duplicate(cpl_imagelist_get(aAppend->dq, i)),
                        cpl_imagelist_get_size(aCube->dq));
    }
    cpl_imagelist_set(aCube->stat,
                      cpl_image_duplicate(cpl_imagelist_get(aAppend->stat, i)),
                      cpl_imagelist_get_size(aCube->stat));
  }
  return CPL_ERROR_NONE;
} /* muse_datacube_concat() */

/*---------------------------------------------------------------------------*/
/**
  @brief    Deallocate memory associated to a muse_datacube object.
  @param    aCube   input MUSE datacube

  Just calls @c cpl_imagelist_delete() and @c cpl_propertylist_delete() for the
  four components of a <tt><b>muse_datacube</b></tt>, and frees memory for the
  aCube pointer. As a safeguard, it checks if a valid pointer was passed,
  so that crashes cannot occur.
 */
/*---------------------------------------------------------------------------*/
void
muse_datacube_delete(muse_datacube *aCube)
{
  /* if the cube does not exists at all, we don't need to do anything */
  if (!aCube) {
    return;
  }

  /* checks for the existence of the sub-images *
   * are done in the CPL functions              */
  cpl_imagelist_delete(aCube->data);
  aCube->data = NULL;
  cpl_imagelist_delete(aCube->dq);
  aCube->dq = NULL;
  cpl_imagelist_delete(aCube->stat);
  aCube->stat = NULL;

  /* delete the FITS header, too */
  cpl_propertylist_delete(aCube->header);
  aCube->header = NULL;

  /* remove reconstructed image data, if present */
  muse_imagelist_delete(aCube->recimages);
  cpl_array_delete(aCube->recnames);
  cpl_free(aCube);
} /* muse_datacube_delete() */

/**@}*/
