/**
 * Copyright INRIA and ENS, contributors Peterlongo and Chikhi
 * pierre.peterlongo@inria.fr
 * rayan.chikhi@irisa.fr
 *
 * This software is a computer program whose purpose is to detect the
 * presence of a starter in a set of NGS reads, and to provide the neighbor
 * in case of success..
 *
 * This software is governed by the CeCILL license under French law and
 * abiding by the rules of distribution of free software.  You can  use,
 * modify and/ or redistribute the software under the terms of the CeCILL
 * license as circulated by CEA, CNRS and INRIA at the following URL
 * "http://www.cecill.info".

 * As a counterpart to the access to the source code and  rights to copy,
 * modify and redistribute granted by the license, users are provided only
 * with a limited warranty  and the software's author,  the holder of the
 * economic rights,  and the successive licensors  have only  limited
 * liability.

 * In this respect, the user's attention is drawn to the risks associated
 * with loading,  using,  modifying and/or developing or reproducing the
 * software by the user in light of its specific status of free software,
 * that may mean  that it is complicated to manipulate,  and  that  also
 * therefore means  that it is reserved for developers  and  experienced
 * professionals having in-depth computer knowledge. Users are therefore
 * encouraged to load and test the software's suitability as regards their
 * requirements in conditions enabling the security of their systems and/or
 * data to be ensured and,  more generally, to use and operate it in the
 * same conditions as regards security.
 *
 * The fact that you are presently reading this means that you have had
 * knowledge of the CeCILL license and that you accept its terms.
 */

/*
 * read_coherence.c
 *
 *  Created on: 22 oct. 2010
 *      Author: rayan chikhi
 */


#ifdef __cplusplus
extern "C"
{
#endif
#define _GNU_SOURCE // needed for strndup
#include <assert.h>
#include <string.h>

#include "couple.h"
#include "../commons.h"
#include "hash.h"
#include "list.h"
#include "Read_coherence_mapsembler.h"
//#include <fragment_is_covered.h>
#include "consensus_common.h"
#include "read_groups.h"

//#define MAX(X,Y) ((X) < (Y) ? (Y) : (X))

#define debug_read_coherence_everywhere
//#define debug_groups
//#define debug_votes

#define SPACE (MAX_VOTES/2) // don't increase this too much, has to fit in MAX_VOTES
// macro to initialize some variables useful for the hashmap iterator
#define MAPPING_POSITIONS_ITERATOR_DECLARATION() \
		char *key; \
		char *current_read=NULL;\
		int position_fragment;\
		list *mapping_positions;\
		int current_read_len=0;\
		hash_iter iterator;

// Go through the hashmap Mapped and for each fragment
// get the reads that have seeds that match somewhere on this fragment
// returns:
// - current_read
// - current_read_len
// - position_fragment
#define MAPPING_POSITIONS_ITERATOR_START(map) \
		iterator=hash_iterator_init(map); /* get the first element of the Mapped hashmap*/ \
		if((int)iterator!=-1){ \
			/*printf("Fragment %s covered by \n", fragment);*/ \
		while(!hash_iterator_is_end(map, iterator)){     /* go through all the mapped reads*/ \
		/* get key(read) and corresponding mapping positions (pwi) */ \
		hash_iterator_return_entry(map, iterator, &key, (void**)&mapping_positions); \
		current_read_len=strlen(key); /* needed to compute votes*/ \
		current_read=key; \
		if(mapping_positions){ \
			cell *c=mapping_positions->first; /* == list_start_iterator(mapping_positions) */ \
		while(c!=NULL){ /* == list_is_end_iterator(mapping_positions) */ \
		p_couple value= (p_couple) c->val; /* == list_get_current_element(mapping_positions) */ \
		position_fragment=value->a;

// end iterator
#define MAPPING_POSITIONS_ITERATOR_END(map) \
		c=c->prox; /* == list_next(mapping_positions) */ \
		} \
		} \
		iterator=hash_iterator_next(map, iterator); \
		} \
		}

#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_DECLARATION() \
		list *sorted_reads; \
		cell *read; \
		int current_read_weight;

/*
 * return a flat list of reads/positions, sorted by increasing positions
 * inspired by print_fragment_cover
 */
void* hash_list_sorted_by_positions(hash_t map)
{
	char *key=NULL;
	int position_fragment;
	list *mapping_positions;
	list *sorted_list_of_reads_and_positions=list_create();
	hash_iter iterator;
	iterator=hash_iterator_init(map); // get the first element of the Mapped hashmap

	// Go through the hashmap Mapped and for each fragment
	// get the reads that have seeds that match somewhere on this fragment
	if((int)iterator!=-1){
		while(!hash_iterator_is_end(map, iterator)){     // go through all the mapped reads
			hash_iterator_return_entry(map, iterator, &key, (void**)&mapping_positions); // get key(read) and corresponding mapping positions (pwi)
			if(mapping_positions){
				cell *c=mapping_positions->first;
				while(c!=NULL){
					position_fragment=((*(int *) (c->val)));

					list_add(sorted_list_of_reads_and_positions,create_charint_couple(key,position_fragment));
					c=c->prox;
				}
			}
			iterator=hash_iterator_next(map, iterator);
		}
	}



	list_quicksort_by_charint_couple(&sorted_list_of_reads_and_positions);

	return (void *)sorted_list_of_reads_and_positions;
}

#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_START(map) \
		sorted_reads=hash_list_sorted_by_positions(map);\
		read=sorted_reads->first;\
		while (read != NULL) \
		{ \
			/* discard duplicated reads by testing if next read == current read*/ \
			current_read_weight=1;\
			while (read->prox!=NULL && strcmp((char *)((p_charint_couple)(read->val))->a,(char *)((p_charint_couple)(read->prox->val))->a)==0) \
			{\
				read = read->prox;\
				current_read_weight++;\
			}\
			char *current_read=(char *)((p_charint_couple)(read->val))->a; \
			assert( current_read != NULL);\
			int position_fragment=((p_charint_couple)(read->val))->b; \
			int current_read_len=((p_charint_couple)(read->val))->a_len;


#define SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_END() \
			read=read->prox; \
		}\
		list_specific_free(sorted_reads, (void *) free_charint_couple);

#define GROUP_ITERATOR_DECLARATION() \
		cell *read;

#define GROUP_ITERATOR_START(reads) \
		read=reads->first;\
		while (read != NULL) \
		{ \
			char *current_read=(char *)((p_charint_couple)(read->val))->a; \
			assert( current_read != NULL);\
			int position_fragment=((p_charint_couple)(read->val))->b; \
			int current_read_len=((p_charint_couple)(read->val))->a_len;

#define GROUP_ITERATOR_END() \
		read=read->prox; \
		}

// macro to print votes
#define PRINT_VOTES() {int vx,vy,start_vx;char ACTG[4]={'A','C','T','G'};printf("votes for %s\n",fragment);start_vx=0;while (votes[start_vx][1]==0 &&votes[start_vx][0]==0&&votes[start_vx][2]==0&&votes[start_vx][3]==0) start_vx++;for (vx=start_vx;vx<MAX_VOTES;vx++){int cont=0;for (vy=0;vy<4;vy++)	if (votes[vx][vy]!=0) cont=1;if (cont==0)break;printf(" vote[%d] ",vx);for (vy=0;vy<4;vy++)printf("%c:%d ",ACTG[vy],votes[vx][vy]);printf("\n");	}}

// macro to print nucleotide-based consensus (certainly not read-coherent)
#define PRINT_CONSENSUS() {int vx,vy,start_vx;char ACTG[4]={'A','C','T','G'};printf("nucleotide consensus for\n%s\n",fragment);start_vx=0;while (votes[start_vx][1]==0 &&votes[start_vx][0]==0&&votes[start_vx][2]==0&&votes[start_vx][3]==0) start_vx++;int nb_subst=0;for (vx=start_vx;vx<MAX_VOTES;vx++){int cont=0;for (vy=0;vy<4;vy++)	if (votes[vx][vy]!=0) cont=1;if (cont==0)break;int max=0, max_vy=0; for (vy=0;vy<4;vy++)if (votes[vx][vy]>max){max=votes[vx][vy]; max_vy=vy;} printf("%c",ACTG[max_vy]); if (ACTG[max_vy]!=fragment[vx])nb_subst++;}printf(" (%d substitutions)\n",nb_subst);}


/*
 * function called in read_coherent_homologs to free local variables
 */
char ** stop_and_clear_everything(list *read_groups, list *sorted_reads) {
	/*
	 * free sorted_reads
	 */
	list_specific_free(sorted_reads, (void *) free_charint_couple); // need to free the charint couple, but doesnt free the char inside

	/*
	 * free read_groups
	 */

	free_read_groups(read_groups, 1);

	/*
	 * return NULL, as it indicates that no consensus was found
	 */
	return NULL;
}

int is_empty_read(char *current_read) {
	int i = 0;
	while (current_read[i] != '\0') {
		if (current_read[i] != 'U')
			return 0;
		i++;
	}
	return 1;
}

/*
 * before attempting actual fork, make sure we're not creating a consensus that's a substring of an existing group
 */
int check_before_forking(list *read_groups, group_info* current_group_info,
		int group_fork_up_to_position, char *current_read,
		int position_fragment, const char *fragment, int len_fragment,
		int max_substitutions_allowed, hash_t groups_consensuses) {
	int brand_new_consensus = 1;
	// construct potential consensus
	char *potential_consensus = strdup(current_group_info->consensus);
	int current_read_len = strlen(current_read);
	int l;
	for (l = MAX(0,position_fragment); l
			<MIN(len_fragment,position_fragment+current_read_len); l++) {
		potential_consensus[l] = current_read[l - position_fragment];
	}

	/*int *dummy; // WIP-HASH
	if (hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0)
	{*/
	// check that it's not already a substring of a previous consensus
	cell *other_group_cell = read_groups->first;
	while (other_group_cell != NULL) {
		group_info *other_read_group_info =
				(group_info *) ((p_pointers_couple) (other_group_cell->val))->b;

		if (strcmp(other_read_group_info->consensus, potential_consensus)==0) {
			brand_new_consensus = 0;
			break;
		}
		other_group_cell = other_group_cell->prox;
	}
	/*} // WIP-HASH
	if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0) && brand_new_consensus==0)
			printf("confirmed by hash\n");
	if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)==0) && brand_new_consensus==1)
			printf("confirmed by hash\n");
	if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)!=0) && brand_new_consensus==1)
			printf("missed by hash\n");
	if ((hash_entry_by_key(groups_consensuses,potential_consensus,&dummy)==0) && brand_new_consensus==0)
			printf("missed by hash\n");*/


	// count substitutions with fragment
	int len_potential_consensus = strlen(potential_consensus);
	int number_substitutions = 0;
	for (l = 0; l < len_potential_consensus; l++)
		if (fragment[l] != potential_consensus[l])
			number_substitutions++;
	free(potential_consensus);
	return brand_new_consensus && (number_substitutions
			<= max_substitutions_allowed);
}

int check_fragment_coverage(group_t current_read_group, group_info *current_read_group_info, int min_coverage, int up_to)
{
	int fragment_fully_covered = 1;
	int l;
	for (l = 0; l < up_to; l++)
	{
		if (current_read_group_info->fragment_coverage[l] < min_coverage) {
			if (debug_read_coherence)
				printf("group %X pos %d below coverage (%d<%d)\n\n",
				       (int) current_read_group, l, current_read_group_info->fragment_coverage[l] ,
				       min_coverage);

			fragment_fully_covered = 0;
			break;
		}
	}
	return fragment_fully_covered;
}

void find_farthest_perfect_overlapping_read(char *current_read, int current_read_len, int position_fragment, char*ri, int ri_length, int ri_position_fragment, char **farthest_read,int *farthest_position, int *farthest_read_position_fragment, int*farthest_read_len, int*so_far_good_overlaps)
{
		// we only need to test those which have a chance to perfect_overlap
		// ----------------------------------------------------- fragment
		//                            ri ---------------
		//                 				             excluded ----------------
		//   excluded ----------------
		//        good current_read  -----------------
		if ((!(position_fragment
				> ri_position_fragment
						+ ri_length))
				&& (!(position_fragment
						+ current_read_len
						< ri_position_fragment))) {
			// don't test reads before an already valid far overlap
			if (position_fragment
					+ current_read_len
					> *farthest_position) {
				// finally, make sure there's a perfect overlap
				if (*so_far_good_overlaps
						&& overlap_perfect(
								current_read,
								position_fragment,
								current_read_len,
								ri,
								ri_position_fragment)) {
					*farthest_read
							= current_read;
					*farthest_read_position_fragment
							= position_fragment;
					*farthest_read_len
							= current_read_len;
					*farthest_position
							= position_fragment
									+ current_read_len;
				} else {
					// should have overlapped but did not: this read cannot fork that group
					*so_far_good_overlaps = 0;
				}
			}
		}
}

/**
 * given a fragment and the set of reads that map on it (cf "print_fragment_cover" in "fragment is covered.c" for an overview on how the map is used"). Basically for each fragment, the map gives its relative position with respect to the fragment (may be negative).
 * Note that each read is given in the correct order: no need to check reverse complements.
 * Since errors in reads are permitted, the fragment has to be covered at a depth == error_threshold
 *
 *  We have to find a subset of reads that
 *  a) fully cover the fragment with at most max_subsitutions_allowed substitutions
 *  b) are coherent together (no error between them)
 *
 */
char ** read_coherent_homologs(const hash_t map, const char * fragment,
		const int max_substitutions_allowed, const int min_coverage,
		const int error_threshold, const int size_seed, int *nb_consensuses) {
	char ** substarters = (char **) malloc((MAX_GROUPS + 1) * sizeof(char *));
	test_alloc(substarters);
	int nb_groups = 0;
	int len_fragment;
	int votes[MAX_VOTES][4];
	MAPPING_POSITIONS_ITERATOR_DECLARATION()

	assert( map != NULL);
	assert( fragment != NULL);

	/* if (export_mapped_reads) */
	/* 	export_reads_mapped_on_fragment(file_reads_mapped, map, */
	/* 			(char *) fragment); */

	/*

	 * dynamic tests are made during the construction of groups to make sure that each group indeed has a chance to be read-coherent
	 * they are controlled with the following switches:
	 */
	int group_coverage_aware = 1; // change this to include DYNAMIC TEST that groups are fully covering the fragment
	int group_substitutions_aware = 1; // change this to include DYNAMIC TEST that groups have less than max_substitutions_allowed with the fragment
	/*
	 * but anyway, keep in mind we also do a final post-processing test at consensus construction..
	 */

	/*
	 * a group is a hash table that contains reads and positions.. quite similar to the "map" parameter really
	 * here is the definition of the list of groups, each element of the list is a couple (group,group_info)
	 */
	list *read_groups = list_create();

	*nb_consensuses = 0;

	// create a votes array that spans the whole multiple alignement (reads aligned to the fragment)
	init_votes(votes);

	len_fragment = strlen(fragment);

	/*
	 * populate votes for all the reads that are aligned to this fragment
	 */
	int nb_reads = 0;
	init_votes(votes);
	MAPPING_POSITIONS_ITERATOR_START(map)
	{
		int offset_read, offset_fragment;
		if (position_fragment < 0) {
			offset_read = abs(position_fragment);
			offset_fragment = 0;
		} else {
			offset_read = 0;
			offset_fragment = position_fragment;
		}
		populate_votes(current_read, current_read_len,
				offset_read, offset_fragment, votes);
		nb_reads++;

	}
	MAPPING_POSITIONS_ITERATOR_END(map)

	/*
	 * give up if too many reads were mapped
	 */
	if (nb_reads > MAX_MAPPED_READS) {
		fprintf(
				stderr,
				"%d reads mapped to this starter, i'm not wasting time computing its homologs\n",
				nb_reads);
		return NULL;
	}

	/*
	 * only consider votes spanning the fragment itself
	 */
	trim_votes(votes, len_fragment);

	/*
	 * warn the user if error_threshold don't match votes
	 */
	analyze_votes(votes, error_threshold);

#ifdef  debug_votes
	PRINT_VOTES()
	PRINT_CONSENSUS()
#endif

#ifdef debug_read_coherence_everywhere
	debug_read_coherence = 1;
#endif

	/*
	 * error-correct all these reads
	 *
	 */
	if (debug_read_coherence)
		printf("correcting reads\n");
	MAPPING_POSITIONS_ITERATOR_START(map)
			//printf("correcting read:            %s, threshold %d\n",current_read,error_threshold);
	rudimentary_error_correction(current_read, votes,
			position_fragment, error_threshold,
			current_read);

			//printf("error_corrected: %s\n",current_read);
	MAPPING_POSITIONS_ITERATOR_END(map)
	/*
	 * at this point, reads in ((map)) can be altered (cutted, corrected, even empty)
	 *
	 TODO: maybe take into account U's inside reads in read_coherent_holomogs such that theyre not considered as substitutions
	 */

	// (optimization) create a hash of all consensus for faster execution of check_before_forking
	// (it's not yet in production)
	hash_t groups_consensuses = hash_create(MAX_GROUPS); // WIP-HASH

	/*
	 * perform the multiple grouping procedure
	 * here, we loop over all reads mapped to the fragment, in sorted order
	 * this creates groups.
	 */
	if (debug_read_coherence)
		printf("performing multiple grouping procedure\n");
	//	int last_checked_coverage_position=-len_fragment;
	SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_DECLARATION()
	SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_START(map)
	{
		int read_was_added_to_one_group = 0;
		int read_forked_one_group = 0;
		assert( current_read != NULL);

		// discard un-informative (UUUU) reads
		if (is_empty_read(current_read)) {
			read = read->prox;
			continue;
		}

		/*
		 * FIXME: what to do with reads smaller than size_seed (and also long ones like UUUUU..UUUA) ?
		 * right now im allowing them.. except the UUUUU one
		 * and even modified overlap_perfect to consider them (since useful info is below size_seed), is that good?
		 */

#ifdef debug_groups
		{
			printf("* considering read %s (pos=%d); printing current groups:\n",current_read,position_fragment);
			cell *group_cell=read_groups->first;
			while (group_cell != NULL)
			{
				group_t current_read_group=(group_t)((p_pointers_couple)(group_cell->val))->a;
				group_info *current_read_group_info = (group_info *) ((p_pointers_couple) (group_cell->val))->b;
				printf("group %lX:\n",(unsigned long)current_read_group);
				print_group_cover(current_read_group, current_read_group_info, (char *)fragment,-1);
				group_cell=group_cell->prox;
			}
		}
#endif
		/*
		 * loop over groups to test for perfect overlap and fork if necessary
		 */
		list *forked_read_groups = list_create();
		int nb_forked_groups = 0;
		cell *group_cell = read_groups->first;

		while (group_cell != NULL) {
			group_t
					current_read_group =
							(group_t) ((p_pointers_couple) (group_cell->val))->a;
			group_info
					*current_read_group_info =
							(group_info *) ((p_pointers_couple) (group_cell->val))->b;
			assert( current_read_group != NULL);
			assert( current_read_group_info != NULL);

			/*  1. if overlap_perfect( farthest_read_in_the_group, current_read ) then add it to the group */

			/*
			 * 1.a finding farthest read from the current group
			 */
			int farthest_position = -len_fragment;
			int farthest_read_position_fragment = 0;
			int farthest_read_len = 0;
			char* farthest_read = NULL;

			{
				GROUP_ITERATOR_DECLARATION()
				GROUP_ITERATOR_START(current_read_group)
					{
						if (position_fragment + current_read_len
								> farthest_position) {
							farthest_read = current_read;
							farthest_read_position_fragment
									= position_fragment;
							farthest_read_len = current_read_len;
							farthest_position = position_fragment
									+ current_read_len;
						}
					}
				GROUP_ITERATOR_END()
			}
			assert( farthest_read != NULL);

			/*
			 * 1.b.(i) if the farthest position is before the current read position, it means the group can't be
			 * 	   extended anymore, so lets remove that group
			 */

			if (farthest_position < position_fragment) {
				if (debug_read_coherence)
					printf(
							"that group (%X) cannot be extended anymore, removing it\n",
							(int) current_read_group);
				cell *next = group_cell->prox;
				remove_group_from_groups(&nb_groups, read_groups,
						group_cell);
				group_cell = next;
				continue;
			}

			/*
			 * 1.b.(ii) before the current position_fragment, test the group coverage and if it has a position below min_coverage, lets remove that group
			 */

			if (!check_fragment_coverage(current_read_group, current_read_group_info,  min_coverage, position_fragment)) {
				if (debug_read_coherence)
					printf(
							"that group (%X) has insufficient coverage so far (positions < %d)\n",
							(int) current_read_group, position_fragment );
				cell *next = group_cell->prox;
				remove_group_from_groups(&nb_groups, read_groups,
						group_cell);
				group_cell = next;
				continue;
			}

			/*
			 * 1.c testing for perfect overlap and insert if match
			 */
			// uncomment the next line for extreme group debugging
			//printf("group id %X, testing perfect_overlap(%s,\n                        %s) = %d\n",(int)current_read_group,farthest_read,current_read,overlap_perfect(farthest_read, farthest_read_position_fragment, farthest_read_len, current_read, position_fragment));
			if (overlap_perfect(farthest_read,
					farthest_read_position_fragment, farthest_read_len,
					current_read, position_fragment)) {
					// reads which are strictly contained in the current group are uninformative, we insert them in a separate structure (used only for coverage info)
					if (farthest_position<position_fragment+current_read_len)
					{
						if (append_read_to_group(current_read_group,
							current_read_group_info, current_read, current_read_len,
							position_fragment, fragment, len_fragment,
							current_read_weight, group_coverage_aware,
							group_substitutions_aware,
							max_substitutions_allowed,groups_consensuses)) {
							if (debug_read_coherence)
								printf(
									"inserted read %s (pos=%d) into group %X\n",
									current_read, position_fragment,
									(int) current_read_group);
						}
					}
					else
					{
						append_contained_read_to_group(current_read_group,
													current_read_group_info, current_read, current_read_len,
													position_fragment, fragment, len_fragment, current_read_weight,
													group_coverage_aware,
													group_substitutions_aware,
													max_substitutions_allowed,groups_consensuses);
					}
					read_was_added_to_one_group = 1;
			} else {
				/* 2. if we could not perfect_overlap this read to the group,
				 * then fork it if it includes a read r such that o_p(r, current_read)*/

				group_t
						current_read_group =
								(group_t) ((p_pointers_couple) (group_cell->val))->a;
				group_info
						*current_read_group_info =
								(group_info *) ((p_pointers_couple) (group_cell->val))->b;
				assert( current_read_group != NULL);
				assert( current_read_group_info != NULL);

				int ri_position_fragment = position_fragment;
				int ri_length = current_read_len;
				char *ri = current_read;
				int ri_weight = current_read_weight;
				int group_fork_up_to_position = 0;
				int found_a_perfect_overlap = 0;
				{
					int farthest_position = -len_fragment;
					int farthest_read_position_fragment = 0;
					int farthest_read_len = 0;
					char* farthest_read = NULL;

					/* we need to take the farthest such r to make sure the future read will be group-coherent*/
					int so_far_good_overlaps = 1;
					GROUP_ITERATOR_DECLARATION()
					GROUP_ITERATOR_START(current_read_group)
						{
							find_farthest_perfect_overlapping_read(current_read,current_read_len,position_fragment, ri,ri_length,ri_position_fragment,&farthest_read,&farthest_position, &farthest_read_position_fragment, &farthest_read_len, &so_far_good_overlaps);
						}
					GROUP_ITERATOR_END()

					/*
					 * if the non-contained reads cannot fork, try with the contained reads
					 */
					if (farthest_read == NULL) {
						so_far_good_overlaps = 1;
						GROUP_ITERATOR_DECLARATION()
						GROUP_ITERATOR_START(current_read_group_info->contained_reads)
						{
							find_farthest_perfect_overlapping_read(current_read,current_read_len,position_fragment, ri,ri_length,ri_position_fragment,&farthest_read,&farthest_position, &farthest_read_position_fragment, &farthest_read_len, &so_far_good_overlaps);
						}
						GROUP_ITERATOR_END()
					}

					// if it found a valid fork, record some infos. check_before_forking will be called later
					if (farthest_read != NULL) {
						found_a_perfect_overlap = 1;
						group_fork_up_to_position
								= farthest_position;
					}

				}

				if (found_a_perfect_overlap && check_before_forking(
						read_groups, current_read_group_info,
						group_fork_up_to_position, ri,
						ri_position_fragment, fragment, len_fragment,
						max_substitutions_allowed,groups_consensuses)) {

					/* 2.b fork the group! */
					group_t new_group;
					group_info *new_group_info;

					create_new_group(&new_group, &new_group_info,
							fragment);

					if (debug_read_coherence) {//print_fragment_cover(current_read_group, (char *)fragment,-1);
						printf(
								"forking group id %X with read %s (pos=%d) (new group id: %X) because perfect overlap found up to pos=%d\n",
								(int) current_read_group, current_read,
								position_fragment, (int) new_group,
								group_fork_up_to_position);
					}


					/*
					 * cloning the group with every read that aligns before position group_fork_up_to_position
					 */
					GROUP_ITERATOR_DECLARATION()
					GROUP_ITERATOR_START(current_read_group)
						{
							if (position_fragment+current_read_len <= group_fork_up_to_position)
								append_read_to_group(new_group,
										new_group_info,
										current_read, current_read_len,
										position_fragment,
										fragment, len_fragment, 1,
										group_coverage_aware,
										group_substitutions_aware,
										max_substitutions_allowed,groups_consensuses);

						}
					GROUP_ITERATOR_END()

					// add contained reads too
					GROUP_ITERATOR_START(current_read_group_info->contained_reads)
						{
							if (position_fragment+current_read_len <= group_fork_up_to_position)
								append_contained_read_to_group(new_group,
										new_group_info,
										current_read, current_read_len,
										position_fragment,
										fragment, len_fragment, 1,
										group_coverage_aware,
										group_substitutions_aware,
										max_substitutions_allowed,groups_consensuses);
						}
					GROUP_ITERATOR_END()

					/*
					 * append the current read to the newly forked group
					 */
					if (!append_read_to_group(new_group,
							new_group_info, ri, ri_length,
							ri_position_fragment, fragment,
							len_fragment, ri_weight, group_coverage_aware,
							group_substitutions_aware,
							max_substitutions_allowed,groups_consensuses)) {
						/*
						 * if it turns out that adding the read would make the group not read-coherent, remove it.. all that for nothing
						 */
						free_group(new_group, new_group_info);

					} else {
						/*
						 * else add the group to the list of newly forked read groups, as well as the list of all groups
						 */
						read_forked_one_group = 1;
						if (append_group_to_groups(&nb_forked_groups,
								forked_read_groups, new_group,
								new_group_info))
							return stop_and_clear_everything(
									read_groups, sorted_reads);
					}
				}
			}
			group_cell = group_cell->prox;
		}
		// end of group loop for overlapping/forking

		/*
		 * remove newly forked groups that are strictly included in others
		 * add the others to the groups list
		 */
		group_cell = forked_read_groups->first;
		list *removed_forked_groups = list_create();
		int nb_removed_forked_groups = 0;
		while (group_cell != NULL) {

			group_t
					current_read_group =
							(group_t) ((p_pointers_couple) (group_cell->val))->a;
			group_info
					*current_read_group_info =
							(group_info *) ((p_pointers_couple) (group_cell->val))->b;

			int is_not_included = 1;
			int brand_new_consensus = 1;

			// now compare that group with of newly forked groups
			cell *other_group_cell = forked_read_groups->first;
			while (other_group_cell != NULL) {
				if (other_group_cell != group_cell) {
					group_t
							other_read_group =
									(group_t) ((p_pointers_couple) (other_group_cell->val))->a;
					group_info
							*other_read_group_info =
									(group_info *) ((p_pointers_couple) (other_group_cell->val))->b;

					// first filter: consensus should be a substring of other consensus
					if (strstr(other_read_group_info->consensus,
							current_read_group_info->consensus)) {

						// don't compare to already erased groups
						if (find_group(removed_forked_groups,
								other_read_group) == 0) {

							// actual comparison: group list of reads is included in other group list of reads
							if (list_included(current_read_group,
									other_read_group,
									charint_couple_exact_compare)) {

								if (debug_read_coherence)
									printf(
											"removing newly forked group %X (%d reads) because it's strictly included in one of the newly forked groups %X (%d reads)\n",
											(int) current_read_group,
											current_read_group_info->nb_reads,
											(int) other_read_group,
											other_read_group_info->nb_reads);
								is_not_included = 0;

								break;
							}

						}
					}
				}
				other_group_cell = other_group_cell->prox;
			}

			if (is_not_included) {
				// compare that group with of previous groups
				other_group_cell = read_groups->first;
				while (other_group_cell != NULL) {
					if (other_group_cell != group_cell) {
						group_t
								other_read_group =
										(group_t) ((p_pointers_couple) (other_group_cell->val))->a;
						group_info
								*other_read_group_info =
										(group_info *) ((p_pointers_couple) (other_group_cell->val))->b;

						if (strstr(other_read_group_info->consensus,
								current_read_group_info->consensus)) {
							brand_new_consensus = 0;

							if (list_included(current_read_group,
									other_read_group,
									charint_couple_exact_compare)) {

								if (debug_read_coherence)
									printf(
											"removing newly forked group %X (%d reads) because it's strictly included in one of the previous groups %X (%d reads) %s\n",
											(int) current_read_group,
											current_read_group_info->nb_reads,
											(int) other_read_group,
											other_read_group_info->nb_reads,
											brand_new_consensus ? "was new consensus"
													: "");
								is_not_included = 0;
								break;
							}
						}

					}
					other_group_cell = other_group_cell->prox;
				}
			}

			if (is_not_included) {
				// we verified that the newly forked group not included in others, let's add it to the main groups list
				if (debug_read_coherence)
					printf("valid fork (group %lX, %d reads) %s\n",
							(long int)current_read_group,
							current_read_group_info->nb_reads,
							brand_new_consensus ? "brand new consensus"
									: "");
				if (append_group_to_groups(&nb_groups, read_groups,
						current_read_group, current_read_group_info))
					return stop_and_clear_everything(read_groups,
							sorted_reads);
			} else {
				append_group_to_groups(&nb_removed_forked_groups,
						removed_forked_groups, current_read_group,
						current_read_group_info);
			}
			group_cell = group_cell->prox;
		}
		free_read_groups(removed_forked_groups, 1);
		free_read_groups(forked_read_groups, 0);

		/*
		 * if the read was not added to a group, nor forked any group
		 * create a new group with the read (also initializes the very first group)
		 */
		if ((!read_was_added_to_one_group) && (!read_forked_one_group)) {
			if (group_coverage_aware && position_fragment > 0) {
				if (debug_read_coherence)
					printf(
							"this fragment %s is certainly not read-coherent as the first reads aligns to it at %d\n",
							current_read, position_fragment);
				if (nb_groups == 0) /* if it is the first group and the read is already not covering it,
				 * then nothing else can be done in this function */
				{
					if (debug_read_coherence)
						printf(
								"Will not create any new fragment. We give up!\n");
					return stop_and_clear_everything(read_groups,
							sorted_reads);
				}
			} else {
				group_t new_group;
				group_info *new_group_info;
				create_new_group(&new_group, &new_group_info, fragment);
				if (debug_read_coherence)
					printf(
							"creating new group %X (group number %d) for read %s (pos=%d) because it could not contribute to other groups\n",
							(int) new_group, nb_groups + 1,
							current_read, position_fragment);
				if (!append_read_to_group(new_group, new_group_info,
						current_read, 0, position_fragment, fragment,
						len_fragment, current_read_weight, group_coverage_aware,
						group_substitutions_aware,
						max_substitutions_allowed,groups_consensuses)) {
					/* can't create a new group with this read, oh well */
					free_group(new_group, new_group_info);
				} else {
					if (append_group_to_groups(&nb_groups, read_groups,
							new_group, new_group_info))
					{
						if (debug_read_coherence)
							printf("can't append group %x to groups\n",(unsigned long)new_group);
						return stop_and_clear_everything(read_groups,
								sorted_reads);
					}
				}
			}
		}
	} // end of the big read loop
	SORTED_WEIGHTED_MAPPING_POSITIONS_ITERATOR_END()

	hash_free(groups_consensuses);

	if (debug_read_coherence) {
		printf(
				"%d %s %s groups constructed from the reads for the fragment %s.\n",
				nb_groups, group_coverage_aware ? "fragment-coverage coherent"
						: "",
				group_substitutions_aware ? "fragment-substitutions coherent"
						: "", fragment);
#ifdef debug_groups
		cell *group_cell = read_groups->first;
		while (group_cell != NULL) {
			group_t current_read_group =
			(group_t) ((p_pointers_couple) (group_cell->val))->a;
			group_info *current_read_group_info =
					(group_info *) ((p_pointers_couple) (group_cell->val))->b;
			printf("group %X:\n",(int)current_read_group);
			print_group_cover(current_read_group, current_read_group_info, (char *)fragment, -1);
			group_cell = group_cell->prox;
		}
#endif
		printf("\n");
	}

	/*
	 * construct consensuses from groups, removing those which lack coverage or have too many substitutions
	 */
	cell *group_cell = read_groups->first;
	while (group_cell != NULL) {
		group_t current_read_group =
				(group_t) ((p_pointers_couple) (group_cell->val))->a;
		group_info *current_read_group_info =
				(group_info *) ((p_pointers_couple) (group_cell->val))->b;
		int last_written_position = 0;
		int l;
		int number_substitutions = 0;

		substarters[*nb_consensuses] = strdup(fragment);

		if (debug_read_coherence)
			printf(
					"construct consensus from group %X\n (fragment=%s) (dynamic consensus=%s) with reads:\n",
					(int)current_read_group, fragment, current_read_group_info->consensus);
		/*
		 * create the new consensus  also compute the number of substitutions
		 * for coverage, we can only trust what we computed dynamically (because contained reads and reads multiplicities were discarded)
		 */
		GROUP_ITERATOR_DECLARATION()
		GROUP_ITERATOR_START(current_read_group)
				{
					if (debug_read_coherence)
						printf("\t %s pos %d\n",(char *)((charint_couple*)(read->val))->a, ((charint_couple*)(read->val))->b);
					for (l = MAX(last_written_position,position_fragment); l
							< MIN(position_fragment+current_read_len,len_fragment); l++) {
						substarters[*nb_consensuses][l] = current_read[l
								- position_fragment];
						last_written_position
								=MAX(last_written_position+1,l+1);
						if (fragment[l] != substarters[*nb_consensuses][l]) {
							if (debug_read_coherence)
								printf(
										"subst %c %c between %s and %s position %d\n",
										fragment[l],
										substarters[*nb_consensuses][l],
										fragment, substarters[*nb_consensuses],
										l);
							number_substitutions++;
						}
					}
				}
				GROUP_ITERATOR_END()

		/*
		 * redundant check for read coherence.. very helpful!
		 */
		if (number_substitutions
				!= current_read_group_info->number_substitutions) {
			fprintf(
					stderr,
					"error group %X: recomputed number of substitutions (%d) not matching dynamic number of substitutions (%d) (so i'm glad i coded this test)\n fragment: %s\nconsensus:%s\n",
					(int) current_read_group, number_substitutions,
					current_read_group_info->number_substitutions, fragment,
					substarters[*nb_consensuses]);
			exit(1);
		}

		/*
		 * redundant check for consensus..
		 */
		if (strcmp(substarters[*nb_consensuses],
				current_read_group_info->consensus) != 0) {
			fprintf(
					stderr,
					"error group %X: recomputed consensus (%s) not matching dynamic consensus (%s) (so i'm glad i coded this test)\n fragment: %s\n",
					(int) current_read_group, substarters[*nb_consensuses],
					current_read_group_info->consensus, fragment);
			exit(1);
		}

		/*
		 * test if the fragment is fully covered by reads with per-position depth > min_coverage
		 */
		int fragment_fully_covered = check_fragment_coverage(current_read_group, current_read_group_info,  min_coverage, len_fragment);


		/*
		 * the new consensus is validated if the fragment is fully covered and with less than max_substitutions_allowed
		 */
		if ((!fragment_fully_covered) || (number_substitutions
				> max_substitutions_allowed)) {
			if (debug_read_coherence)
				printf(
						"group %X was not consensus'ed because %s, nb_substitutions: %d/%d \nfragment: %s\nconsensus:%s\n",
						(int) current_read_group,
						(!fragment_fully_covered) ? "not fully covered" : "",
						number_substitutions, max_substitutions_allowed,
						fragment, substarters[*nb_consensuses]);
			free(substarters[*nb_consensuses]);
		} else {
			if (*nb_consensuses < MAX_GROUPS_TO_RETURN)
				(*nb_consensuses)++;
			else {
				fprintf(
						stderr,
						" Warning, too many sub-groups created for fragment %s, we keep only %d of them\n",
						fragment, *nb_consensuses);
				break;
			}
		}

		group_cell = group_cell->prox;
	}

	/*
	 * free the read groups
	 */

	free_read_groups(read_groups, 1);

	if (debug_read_coherence) {
		printf(
				"%d consensuses constructed from read-coherent groups related to the fragment %s.\n",
				*nb_consensuses, fragment);
		int ci;
		for (ci = 0; ci < *nb_consensuses; ci++) {
			printf("consensus %d: %s\n", ci, substarters[ci]);
		}
		printf("\n");
	}

	// reallocating substarters because they were MAX_GROUPS+1 of them before
	substarters = realloc(substarters, (*nb_consensuses+1) * sizeof(char *));
	test_alloc(substarters);
	return substarters;
}

#ifdef __cplusplus
}
#endif
