/*****************************************************************************
 *   GATB : Genome Assembly Tool Box
 *   Copyright (C) 2014  INRIA
 *   Authors: R.Chikhi, G.Rizk, E.Drezen
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/

/** \file IteratorFile.hpp
 *  \date 01/03/2013
 *  \author edrezen
 *  \brief Iterator implementation for file
 */

#ifndef _GATB_CORE_TOOLS_COLLECTIONS_IMPL_ITERATOR_FILE_HPP_
#define _GATB_CORE_TOOLS_COLLECTIONS_IMPL_ITERATOR_FILE_HPP_

/********************************************************************************/

#include <gatb/tools/designpattern/api/Iterator.hpp>
#include <gatb/tools/collections/api/Iterable.hpp>
#include <gatb/system/impl/System.hpp>

#include <string>
#include <vector>
#include <zlib.h>

/********************************************************************************/
namespace gatb          {
namespace core          {
namespace tools         {
namespace collections   {
/** Implementation of API from collections */
namespace impl          {
/********************************************************************************/

#define BUFFER_SIZE (128*1024)

/** \brief Iterator implementation for file
 */
template <class Item> class IteratorFile : public dp::Iterator<Item>
{
public:

    /** Constructor. */
    IteratorFile () : _file(0), _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(0), _isDone(true) {}

    IteratorFile (const IteratorFile& it):
        _filename(it._filename), _file(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(it._cacheItemsNb), _isDone(true)
    {
        _file    = system::impl::System::file().newFile (_filename, "rb");
        _buffer  = (Item*) malloc (sizeof(Item) * _cacheItemsNb);
    }

    /** Constructor. */
    IteratorFile (const std::string& filename, size_t cacheItemsNb=10000) :
        _filename(filename), _file(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(cacheItemsNb), _isDone(true)

    {
        _file    = system::impl::System::file().newFile (filename, "rb");
        _buffer  = (Item*) malloc (sizeof(Item) * _cacheItemsNb);
    }

    /** Destructor. */
    ~IteratorFile ()
    {
        if (_file)  { delete _file;  }
        if (_buffer) { free (_buffer); }
    }

    /** Affectation. */
    IteratorFile& operator= (const IteratorFile& it)
    {
        if (this != &it)
        {
            if (_file)    { delete _file; }
            if (_buffer)  { free(_buffer); }

            _filename     = it._filename;
            _cpt_buffer   = it._cpt_buffer;
            _idx          = it._idx;
            _cacheItemsNb = it._cacheItemsNb;
            _isDone       = it._isDone;

            _file    = system::impl::System::file().newFile (it._filename, "rb");
            _buffer  = (Item*) malloc (sizeof(Item) * it._cacheItemsNb);
        }
        return *this;
    }

    /** \copydoc dp::Iterator::first */
    void first()
    {
        _file->seeko (0, SEEK_SET);
        _cpt_buffer = 0;
        _idx        = 0;
        _isDone     = false;
        next ();
    }

    /** \copydoc dp::Iterator::next */
    void next()
    {
        if (_cpt_buffer==0)
        {
            _idx = 0;
            _cpt_buffer = _file->fread (_buffer, sizeof(Item), _cacheItemsNb);
            if (_cpt_buffer==0)  { _isDone = true;  return; }
        }

        *(this->_item) =  _buffer[_idx];
        _cpt_buffer --;
        _idx ++;
    }

    /** \copydoc dp::Iterator::isDone */
    bool isDone()  { return _isDone; }

    /** \copydoc dp::Iterator::item */
    Item& item ()  { return *(this->_item); }

    /** */
    size_t fill (std::vector<Item>& vec, size_t len=0)
    {
        if (len==0)  { len = vec.size(); }
        return _file->fread (vec.data(), sizeof(Item), len);
    }

private:
    std::string     _filename;
    system::IFile*  _file;
    Item*           _buffer;
    int             _cpt_buffer;
    int             _idx;
    size_t          _cacheItemsNb;
    bool            _isDone;
};

    
    
    
/** \brief Iterator implementation for file
 */
template <class Item> class IteratorGzFile : public dp::Iterator<Item>
{
public:
    
    /** Constructor. */
    IteratorGzFile () : _gzfile(0), _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(0), _isDone(true) {}
    
    IteratorGzFile (const IteratorGzFile& it):
    _filename(it._filename), _gzfile(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(it._cacheItemsNb), _isDone(true)
    {
        _gzfile =   gzopen(_filename.c_str(),"rb");
        _buffer  = (Item*) malloc (sizeof(Item) * _cacheItemsNb);
    }
    
    /** Constructor. */
    IteratorGzFile (const std::string& filename, size_t cacheItemsNb=10000) :
    _filename(filename), _gzfile(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(cacheItemsNb), _isDone(true)
    
    {
        _gzfile =   gzopen(_filename.c_str(),"rb");
        _buffer  = (Item*) malloc (sizeof(Item) * _cacheItemsNb);
    }
    
    /** Destructor. */
    ~IteratorGzFile ()
    {
        if (_gzfile)  { gzclose(_gzfile);   }
        if (_buffer) { free (_buffer); }
    }
    
    /** Affectation. */
    IteratorGzFile& operator= (const IteratorGzFile& it)
    {
        if (this != &it)
        {
            if (_gzfile)    {  gzclose(_gzfile);  }
            if (_buffer)  { free(_buffer); }
            
            _filename     = it._filename;
            _cpt_buffer   = it._cpt_buffer;
            _idx          = it._idx;
            _cacheItemsNb = it._cacheItemsNb;
            _isDone       = it._isDone;
            
            _gzfile =   gzopen(_filename.c_str(),"r");
            _buffer  = (Item*) malloc (sizeof(Item) * it._cacheItemsNb);
        }
        return *this;
    }
    
    /** \copydoc dp::Iterator::first */
    void first()
    {
        gzseek(_gzfile,0,SEEK_SET);
        _cpt_buffer = 0;
        _idx        = 0;
        _isDone     = false;
        next ();
    }
    
    /** \copydoc dp::Iterator::next */
    void next()
    {
        if (_cpt_buffer==0)
        {
            _idx = 0;
            _cpt_buffer = (gzread(_gzfile, _buffer, sizeof(Item)*_cacheItemsNb)  /  sizeof(Item))  ;  // gzread returns number of bytes uncompressed returned in _buffer
            //printf("refreshing buffer from gzread(), cptbuffer now: %d\n",_cpt_buffer);
            if (_cpt_buffer < 0)
            { // On other errors, gzread() shall return a value less than 0 and and applications may examine the cause using gzerror().
                // FIXME: more tests are needed but on my system (R: linux64), gzread returns the fixed number of items, then a lower number of items (as it reaches the end), then it returns -1 instead of 0 and prints a "data error" but at this point it's fine to just return
                int err;
                fprintf(stderr, "gzread error: %s\n", gzerror(_gzfile, &err));
            }
            if (_cpt_buffer<=0)  { _isDone = true;  return; }
        }
        *(this->_item) =  _buffer[_idx];
        _cpt_buffer --;
        _idx ++;
    }
    
    /** \copydoc dp::Iterator::isDone */
    bool isDone()  { return _isDone; }
    
    /** \copydoc dp::Iterator::item */
    Item& item ()  { return *(this->_item); }
    
    /** */
    size_t fill (std::vector<Item>& vec, size_t len=0)
    {
        if (len==0)  { len = vec.size(); }
        
        return (gzread(_gzfile,vec.data(), sizeof(Item)*len)  /  sizeof(Item));
 
    }
    
private:
    std::string     _filename;
    gzFile  _gzfile;
    Item*           _buffer;
    int             _cpt_buffer;
    int             _idx;
    size_t          _cacheItemsNb;
    bool            _isDone;
};


/** \brief Iterator implementation for file
 */
    //reading from a  sorted compressed file, read is buffered
template <class Item> class IteratorCountCompressedFile : public dp::Iterator<Item>
{
public:
    
    /** Constructor. */
    IteratorCountCompressedFile () : _file(0), _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(0), _isDone(true),_abundance(0) {}
    
    IteratorCountCompressedFile (const IteratorCountCompressedFile& it):
    _filename(it._filename), _file(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(it._cacheItemsNb), _isDone(true),_abundance(0)
    {
        _file    = system::impl::System::file().newFile (_filename, "rb");
        _buffer  = (u_int8_t*) malloc (sizeof(u_int8_t) * _cacheItemsNb);
    }
    
    /** Constructor. */
    IteratorCountCompressedFile (const std::string& filename, size_t cacheItemsNb=10000) :
    _filename(filename), _file(0),  _buffer(0), _cpt_buffer(0), _idx(0), _cacheItemsNb(cacheItemsNb), _isDone(true),_abundance(0)
    
    {
        _file    = system::impl::System::file().newFile (filename, "rb");
        _buffer  = (u_int8_t*) malloc (sizeof(u_int8_t) * _cacheItemsNb);
    }
    
    /** Destructor. */
    ~IteratorCountCompressedFile ()
    {
        if (_file)  { delete _file;  }
        if (_buffer) { free (_buffer); }
    }
    
    /** Affectation. */
    IteratorCountCompressedFile& operator= (const IteratorCountCompressedFile& it)
    {
        if (this != &it)
        {
            if (_file)    { delete _file; }
            if (_buffer)  { free(_buffer); }
            
            _filename     = it._filename;
            _cpt_buffer   = it._cpt_buffer;
            _idx          = it._idx;
            _cacheItemsNb = it._cacheItemsNb;
            _isDone       = it._isDone;
            _abundance    = it._abundance;
            _file    = system::impl::System::file().newFile (it._filename, "rb");
            _buffer  = (u_int8_t*) malloc (sizeof(u_int8_t) * it._cacheItemsNb);
        }
        return *this;
    }
    
    /** \copydoc dp::Iterator::first */
    void first()
    {
        _file->seeko (0, SEEK_SET);
        _cpt_buffer = 0;
        _idx        = 0;
        _abundance  = 0;
        _isDone     = false;
        next ();
    }
    
    /** \copydoc dp::Iterator::next */
    void next()
    {

        
        if(_abundance)
        {
           // printf("rem abond %i \n",_abundance);

            *(this->_item) =  _previous;
            _abundance--;
        }
        else
        
        {

            
            if (!readChunkIfNeeded (1)) return;

            //read a couple (byte, Item)
            _abundance =  _buffer[_idx];
            _cpt_buffer --;  _idx ++;
          //  printf(" read new couple  %i  \n",_abundance);

            if (!readChunkIfNeeded (sizeof(Item))) return; // this one should succeed (ie, in the file always a couple  (byte, elem))
            _previous =  *((Item *) (_buffer + _idx ));
            _cpt_buffer -= sizeof(Item); _idx += sizeof(Item);
            

            *(this->_item) =  _previous;
            _abundance--;
            

        }
        
    }
    
    /** \copydoc dp::Iterator::isDone */
    bool isDone()  { return _isDone; }
    
    /** \copydoc dp::Iterator::item */
    Item& item ()  { return *(this->_item); }
    
    /** */
    size_t fill (std::vector<Item>& vec, size_t len=0)
    {
        printf("Not yet implemented \n");
//        if (len==0)  { len = vec.size(); }
//        return _file->fread (vec.data(), sizeof(Item), len);
    }
    
private:
    std::string     _filename;
    system::IFile*  _file;
    u_int8_t*           _buffer;
    int             _cpt_buffer; // how many unread bytes are remaining in the buffer
    int             _idx; // where we should read the next elem in the buffer
    size_t          _cacheItemsNb; //in bytes for this  compressed type file
    bool            _isDone;
    u_int8_t  _abundance ;
    Item          _previous;

    bool readChunkIfNeeded (size_t needNBytes)
    {


        if (_cpt_buffer < needNBytes)
        {
         //   printf("Read new chunk  _cacheItemsNb %zu B nedd %zu have %i  \n",_cacheItemsNb,needNBytes,_cpt_buffer);

        //_idx = 0;
         //   printf(" A new pos in file %p  %llu  \n",_file, _file->tell());

            //deplacer ce quil reste au debut avant
            memcpy (_buffer,_buffer + _idx,_cpt_buffer ); _idx = 0;
            int remaining = _cpt_buffer;
            _cpt_buffer += _file->fread (_buffer+ remaining , sizeof(u_int8_t), _cacheItemsNb - remaining  );
          //  printf(" B new pos in file %p  %llu  \n",_file, _file->tell());

            if (_cpt_buffer==0)  {
                _isDone = true;
                //printf("should end iterating file \n");
            }
         //   printf("end read have %i \n",_cpt_buffer);

        }

        return !_isDone;
    }

};



    
/********************************************************************************/

template <class Item> class IterableFile : public tools::collections::Iterable<Item>, public virtual system::SmartPointer
{
public:

    /** */
    IterableFile (const std::string& filename, size_t cacheItemsNb=10000)
        :   _filename(filename), _cacheItemsNb (cacheItemsNb)  {}

    /** */
    ~IterableFile () {}

    /** */
    dp::Iterator<Item>* iterator ()  { return new IteratorFile<Item> (_filename, _cacheItemsNb); }

    /** */
    int64_t getNbItems ()   {  return system::impl::System::file().getSize(_filename) / sizeof(Item); }

    /** */
    int64_t estimateNbItems ()   {  return getNbItems(); }

private:
    std::string     _filename;
    size_t          _cacheItemsNb;
};
    
    
template <class Item> class IterableGzFile : public tools::collections::Iterable<Item>, public virtual system::SmartPointer
{
public:
    
    /** */
    IterableGzFile (const std::string& filename, size_t cacheItemsNb=10000)
    :   _filename(filename), _cacheItemsNb (cacheItemsNb)  {}
    
    /** */
    ~IterableGzFile () {}
    
    /** */
    dp::Iterator<Item>* iterator ()  { return new IteratorGzFile<Item> (_filename, _cacheItemsNb); }
    
    /** */
    int64_t getNbItems ()   {  return -1; } // does not know value
    
    int64_t estimateNbItems ()   {  return 3* (system::impl::System::file().getSize(_filename) / sizeof(Item)); }

private:
    std::string     _filename;
    size_t          _cacheItemsNb;
};

template <class Item> class IterableCountCompressedFile : public tools::collections::Iterable<Item>, public virtual system::SmartPointer
{
public:
    
    /** */
    IterableCountCompressedFile (const std::string& filename, size_t cacheItemsNb=10000)
    :   _filename(filename), _cacheItemsNb (cacheItemsNb)  {}
    
    /** */
    ~IterableCountCompressedFile () {}
    
    /** */
    dp::Iterator<Item>* iterator ()  { return new IteratorCountCompressedFile<Item> (_filename, _cacheItemsNb); }
    
    /** */
    int64_t getNbItems ()   {  return -1; } // does not know value
    
    int64_t estimateNbItems ()   {  return 2* (system::impl::System::file().getSize(_filename) / sizeof(Item)); }
    
private:
    std::string     _filename;
    size_t          _cacheItemsNb;
};

    
/********************************************************************************/
} } } } } /* end of namespaces. */
/********************************************************************************/

#endif /* _GATB_CORE_TOOLS_COLLECTIONS_IMPL_ITERATOR_FILE_HPP_ */
