
/***************************************************************************
            mysql2bin.c - Convert data from MySQL to binary format
                             -------------------
    begin                : Mon May 26 2008
    copyright            : (C) 2008 by Akarsh Simha
    email                : akarshsimha@gmail.com
***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

// Program to convert star data stored in a MySQL database into 
// the binary file format used by KStars


#define DB_TBL "tycho2"
#define DB_DB "stardb"
#define VERBOSE 0
#define LONG_NAME_LIMIT 32
#define BAYER_LIMIT 8
#define HTM_LEVEL 3
#define NTRIXELS 512                         // TODO: Change if HTM Level Changes
#define INDEX_ENTRY_SIZE 12
#define GLOBAL_MAG_LIMIT 8.00
#define MYSQL_STARS_PER_QUERY 1000000

#include "binfile.h"
#include <mysql/mysql.h>
#include <sys/types.h>
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <string.h>

/*
 * struct to store star data, to be written in this format, into the binary file.
 */

typedef struct starData {
    int32_t RA;
    int32_t Dec;
    int32_t dRA;
    int32_t dDec;
    int32_t parallax;
    int32_t HD;
    int16_t mag;
    int16_t bv_index;
    char spec_type[2];
    char flags;
    char unused;
} starData;

/*
 * struct to store star name data, to be written in this format, into the star name binary file.
 */

typedef struct starName {
    char bayerName[BAYER_LIMIT];
    char longName[LONG_NAME_LIMIT];
} starName;


/*
 * Dump the data file header.
 *
 * WARNING: Must edit every time the definition of the starData structures changes
 *
 * f : Data file handle
 */

int writeDataFileHeader(FILE *f) {

    char ASCII_text[124];
    u_int16_t nfields;
    u_int32_t nindexes;
    int16_t endian_id=0x4B53;
    u_int8_t version_no = 1;

    if(f == NULL)
        return 0;

    nfields = 11;

    str2charv(ASCII_text, "KStars Star Data v1.0. To be read using the 32-bit starData structure only.", 124);
    fwrite(ASCII_text, 124, 1, f);
    fwrite(&endian_id, 2, 1, f);
    fwrite(&version_no, 1, 1, f);
    fwrite(&nfields, sizeof(u_int16_t), 1, f);

    writeDataElementDescription(f, "RA", 4, DT_INT32, 1000000);
    writeDataElementDescription(f, "Dec", 4, DT_INT32, 100000);
    writeDataElementDescription(f, "dRA", 4, DT_INT32, 10);
    writeDataElementDescription(f, "dDec", 4, DT_INT32, 10);
    writeDataElementDescription(f, "parallax", 4, DT_INT32, 10);
    writeDataElementDescription(f, "HD", 4, DT_INT32, 10);
    writeDataElementDescription(f, "mag", 2, DT_INT16, 100);
    writeDataElementDescription(f, "bv_index", 2, DT_INT16, 100);
    writeDataElementDescription(f, "spec_type", 2, DT_CHARV, 0);
    writeDataElementDescription(f, "flags", 1, DT_CHAR, 0);
    writeDataElementDescription(f, "unused", 1, DT_CHAR, 100);

    nindexes = NTRIXELS;
    fwrite(&nindexes, sizeof(u_int32_t), 1, f);

    return 1;
}

/*
 * Dump the name file header.
 *
 * WARNING: Must edit every time the definition of the starName structures changes
 *
 * nf : Name file handle
 */

int writeNameFileHeader(FILE *nf) {

    char ASCII_text[124];
    u_int16_t nfields;
    u_int32_t nindexes;
    int16_t endian_id=0x4B53;
    u_int32_t data_offset = 0;
    u_int8_t version_no = 1;

    if(nf == NULL)
        return 0;

    nfields = 2;

    str2charv(ASCII_text, "KStars Star Name Data v1.0. Refer associated documentation for format description.", 124);
    fwrite(ASCII_text, 124, 1, nf);
    fwrite(&endian_id, 2, 1, nf);
    fwrite(&version_no, 1, 1, nf);
    fwrite(&nfields, sizeof(u_int16_t), 1, nf);

    writeDataElementDescription(nf, "bayerName", BAYER_LIMIT, DT_STR, 0);
    writeDataElementDescription(nf, "longName", LONG_NAME_LIMIT, DT_STR, 0);

    nindexes = 1;
    fwrite(&nindexes, sizeof(u_int32_t), 1, nf);

    return 1;
}

int main(int argc, char *argv[]) {

    /* === Declarations === */

    int ret, i, exitflag;
    long int lim;
    char named, shallow;

    unsigned long ns_header_offset, us_header_offset, ds_header_offset;
    unsigned long nsf_trix_begin, usf_trix_begin, dsf_trix_begin;
    unsigned long nsf_trix_count, usf_trix_count, dsf_trix_count;
    unsigned long ntrixels;
    unsigned long nf_header_offset;
    unsigned int names_count;
    int16_t maglim;
    u_int8_t htm_level;
    u_int16_t MSpT_named, MSpT_unnamed, MSpT_deep;

    char query[512];
    u_int32_t current_trixel, new_trixel;

    /* File streams */
    FILE *f;                 /* Pointer to "current" file */
    FILE *nsf, *usf, *dsf;    /* Handles of named and unnamed star data files */
    FILE *nsfhead, *usfhead, *dsfhead; /* Handles of named and unnamed star header files */
    FILE *namefile;          /* Pointer to the name file */
    FILE *hdindex;           /* Pointer to the HD Index file */

    /* starData and starName structures */
    starData data;
    starName name;

    /* MySQL structures */
    MYSQL link;
    MYSQL_RES *result;
    MYSQL_ROW row;

    /* Check the number of arguments */
    if(argc <= 9) {
        fprintf(stderr, 
                "USAGE %s DBUserName DBPassword DBName UnnamedStarDataFile UnnamedStarHeaderFile DeepStarDataFile DeepStarHeaderFile ShallowStarDataFile ShallowStarHeaderFile StarNameFile HDIndex [TableName]\n", 
                argv[0]);
        fprintf(stderr, "The magnitude limit for a Shallow Star is set in the program source using GLOBAL_MAG_LIMIT\n");
        fprintf(stderr, "The database used is a MySQL DB on localhost. The default table name is `allstars`\n");
        fprintf(stderr, "HDIndex must contain 360000 blank records of 32 bits each. You can use dd if=/dev/zero of=... ... to create it\n");
    }

    /* == Open all file streams required == */
    /* Unnamed Star Handling */
    usf = fopen(argv[4], "wb");
    if(usf == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Deep Star Data File] for binary write.\n", argv[4]);
        return 1;
    }

    usfhead = fopen(argv[5], "wb");
    if(usfhead == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[5]);
        fcloseall();
        return 1;
    }

    dsf = fopen(argv[6], "wb");
    if(usfhead == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[6]);
        fcloseall();
        return 1;
    }

    dsfhead = fopen(argv[7], "wb");
    if(usfhead == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Deep Star Header File] for binary write.\n", argv[7]);
        fcloseall();
        return 1;
    }

    /* Bright / Named Star Handling */
    nsf = fopen(argv[8], "wb");
    if(nsf == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Shallow Star Data File] for binary write.\n", argv[8]);
        fcloseall();
        return 1;
    }

    nsfhead = fopen(argv[9], "wb");
    if(nsfhead == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Shallow Star Header File] for binary write.\n", argv[9]);
        fcloseall();
        return 1;
    }

    namefile = fopen(argv[10], "wb");
    if(namefile == NULL) {
        fprintf(stderr, "ERROR: Could not open %s [Star Name File] for binary write.\n", argv[7]);
        fcloseall();
        return 1;
    }

    hdindex = fopen( argv[11], "wb" );
    if( hdindex == NULL ) {
        fprintf(stderr, "ERROR: Could not open %s [HD Index] for binary read/write.\n", argv[9] );
        fcloseall();
        return 1;
    }
    u_int32_t foo = 0;
    for( i = 0; i < 360000; ++i )
        fwrite( &foo, sizeof( u_int32_t ), 1, hdindex );

    /* MySQL connect */
    if(mysql_init(&link) == NULL) {
        fprintf(stderr, "ERROR: Failed to initialize MySQL connection!\n");
        return 1;
    }

    ret = mysql_real_connect(&link, "localhost", argv[1], argv[2], argv[3], 0, NULL, 0);

    if(!ret) {
        fprintf(stderr, "ERROR: MySQL connect failed for the following reason: %s\n", mysql_error(&link));
        fcloseall();
        return 1;
    }

    if(mysql_select_db(&link, argv[3])) {
        fprintf(stderr, "ERROR: Could not select MySQL database %s. MySQL said: %s", argv[3], mysql_error(&link));
        fcloseall();
        mysql_close(&link);
        return 1;
    }

    /* Write file headers */
    writeNameFileHeader(namefile);
    writeDataFileHeader(nsfhead);
    writeDataFileHeader(usfhead);
    writeDataFileHeader(dsfhead);
    ns_header_offset = ftell(nsfhead); 
    us_header_offset = ftell(usfhead);
    nf_header_offset = ftell(namefile);
    ds_header_offset = ftell(dsfhead);

    /* Write a bogus index entry into the namefile, to be filled with correct data later */
    writeIndexEntry(namefile, 0, ftell( namefile ) + INDEX_ENTRY_SIZE, 0);

    /* Leave space for / write a deep magnitude limit specification in the data files */
    maglim = GLOBAL_MAG_LIMIT * 100;
    fwrite(&maglim, 2, 1, nsf); // This is also a bogus entry, because it will be overwritten later
    maglim = GLOBAL_MAG_LIMIT * 100;
    fwrite(&maglim, 2, 1, dsf); // This is also a bogus entry, because it will be overwritten later
    maglim = (int)(-5.0 * 100);
    fwrite(&maglim, 2, 1, usf); // Bogus entry

    /* Write a HTM level specification in the data file */
    htm_level = HTM_LEVEL;
    fwrite(&htm_level, 1, 1, nsf);
    fwrite(&htm_level, 1, 1, usf);
    fwrite(&htm_level, 1, 1, dsf);

    /* Leave space for a specification of MSpT (Maximum Stars per Trixel) in the data files */
    MSpT_named = MSpT_unnamed = 0;
    fwrite(&MSpT_named, 2, 1, nsf); // Bogus entry
    fwrite(&MSpT_deep, 2, 1, dsf); // Bogus entry
    fwrite(&MSpT_unnamed, 2, 1, usf); // Bogus entry


    /* Initialize some variables */
    lim = 0;
    exitflag = 0;
    current_trixel = 0;
    nsf_trix_count = usf_trix_count = dsf_trix_count = 0;
    nsf_trix_begin = usf_trix_begin = dsf_trix_begin = 2 + 1 + 2; // The 2+1+2 is to leave space for deep magnitude limit, HTM Level and MSpT specification. TODO: Change this if things change.
    ntrixels = 0;
    names_count = 0;

    /* Recurse over every MYSQL_STARS_PER_QUERY DB entries */
    while(!exitflag) {

        /* Build MySQL query for next MYSQL_STARS_PER_QUERY stars */
        sprintf(query, 
                "SELECT `Trixel`, `RA`, `Dec`, `dRA`, `dDec`, `Parallax`, `Mag`, `BV_Index`, `Spec_Type`, `Mult`, `Var`, `HD`, `UID`, `Name`, `GName` FROM `%s` ORDER BY `trixel`, `mag` ASC LIMIT %ld, %d", 
                (argc >= 13) ? argv[12] : DB_TBL, lim, MYSQL_STARS_PER_QUERY);

        if(VERBOSE) { fprintf(stderr, "SQL Query: %s\n", query); }
    
        /* Execute MySQL query and get the result */
        mysql_real_query(&link, query, (unsigned int)strlen(query)); 
        result = mysql_use_result(&link);
    
        if(!result) {
            fprintf(stderr, "MySQL returned NULL result: %s\n", mysql_error(&link));
            fcloseall();
            mysql_close(&link);
            return 1;
        }

        exitflag = -1;

        /* Process the result row by row */
        while(row = mysql_fetch_row(result)) {
      
            /* Very verbose details */
            if(VERBOSE > 1) {
                fprintf(stderr, "UID = %s\n", row[12]);
                for(i = 0; i <= 14; ++i)
                    fprintf(stderr, "\tField #%d = %s\n", i, row[i]);
            }

            if(exitflag == -1)
                exitflag = 0;

            /* Write index entries if we've changed trixel */
            new_trixel = atoi( row[ 0 ] );
            if( new_trixel != current_trixel ) { 
                if(VERBOSE) { fprintf(stderr, "Trixel Changed from %d to %d!\n", current_trixel, new_trixel ); }
                while(new_trixel != current_trixel) {
                    writeIndexEntry(nsfhead, current_trixel, ns_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + nsf_trix_begin, nsf_trix_count);
                    writeIndexEntry(dsfhead, current_trixel, ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + dsf_trix_begin, dsf_trix_count);
                    writeIndexEntry(usfhead, current_trixel, us_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + usf_trix_begin, usf_trix_count);
                    nsf_trix_begin = ftell(nsf);
                    dsf_trix_begin = ftell(dsf);
                    usf_trix_begin = ftell(usf);
                    if( nsf_trix_count > MSpT_named )
                        MSpT_named = nsf_trix_count;
                    if( dsf_trix_count > MSpT_deep )
                        MSpT_deep = dsf_trix_count;
                    if( usf_trix_count > MSpT_unnamed )
                        MSpT_unnamed = usf_trix_count;
                    nsf_trix_count = dsf_trix_count = usf_trix_count = 0;
                    current_trixel++;
                    ntrixels++;
                }
            }

            /* ==== Set up the starData structure ==== */

            /* Initializations */
            data.flags = 0;

            /* This data is required early! */
            str2int16(&data.mag, row[6], 2);

            /* Are we looking at a named star */
            named = 0;
            if(!isblank(row[13]) || !isblank(row[14])) {
                named = 1;

                /* Print out messages */
                if(VERBOSE) 
                    fprintf(stderr, "Named Star!\n");
                if(VERBOSE > 1)
                    fprintf(stderr, "Bayer Name = %s, Long Name = %s\n", row[14], row[13]);

                /* Check for overflows */
                if(strlen(row[13]) > LONG_NAME_LIMIT)
                    fprintf(stderr,
                            "ERROR: Long Name %s with length %d exceeds LONG_NAME_LIMIT = %d\n", 
                            row[13], strlen(row[13]), LONG_NAME_LIMIT);
                if(strlen(row[14]) > BAYER_LIMIT)
                    fprintf(stderr, 
                            "ERROR: Bayer designation %s with length %d exceeds BAYER_LIMIT = %d\n",
                            row[14], strlen(row[14]), BAYER_LIMIT);
        
                /* Set up the starName structure */
                str2charv(name.bayerName, row[14], BAYER_LIMIT);
                str2charv(name.longName, row[13], LONG_NAME_LIMIT);
                data.flags = data.flags | 0x01; /* Switch on the 'named' bit */
            }
      
            /* Are we looking at a 'global' star [always in memory] or dynamically loaded star? */
            if( named ) {
                f = nsf;
                nsf_trix_count++;
                shallow = 1;
            }
            else if( (data.mag/100.0) <= GLOBAL_MAG_LIMIT ) {
                f = usf;
                usf_trix_count++;
                shallow = 1;
            }
            else {
                dsf_trix_count++;
                f = dsf;
                if( maglim < data.mag )
                    maglim = data.mag;
                shallow = 0;
            }

            /* Convert various fields and make entries into the starData structure */
            str2int32(&data.RA, row[1], 6);
            str2int32(&data.Dec, row[2], 5);
            str2int32(&data.dRA, row[3], 1);
            str2int32(&data.dDec, row[4], 1);
            str2int32(&data.parallax, row[5], 1);
            str2int32(&data.HD, row[11], 0);                  
            str2int16(&data.bv_index, row[7], 2);
            if(str2charv(data.spec_type, row[8], 2) < 0)
                fprintf(stderr, "Spectral type entry %s in DB is possibly invalid for UID = %s\n", row[8], row[12]); 
            if(row[9][0] != '0' && row[9][0] != '\0')
                data.flags = data.flags | 0x02;
            if(row[10][0] != '0' && row[10][0] != '\0')
                data.flags = data.flags | 0x04;
      
            if( data.HD != 0 && !shallow ) {
                // Use this information to generate the HD index we want to have.
                int32_t off;
                if( fseek( hdindex, ( data.HD - 1 ) * sizeof( int32_t ), SEEK_SET ) ) {
                    fprintf( stderr, "Invalid seek ( to %X ) on HD Index file. Index will be corrupt.\n", ( data.HD - 1 ) * sizeof( u_int32_t ) );
                }
                off = ftell( f ) + ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE;
                fprintf( stderr, "DEBUG: HD %d at %X. Writing at %X\n", data.HD, off, ftell( hdindex ) );
                if( !fwrite( &off, sizeof( int32_t ), 1, hdindex ) )
                    fprintf( stderr, "fwrite() on HD Index file failed. Index will be empty or incomplete.\n" );
            }

            /* Write the data into the appropriate data file and any names into the name file */
            if(VERBOSE) 
                fprintf(stderr, "Writing UID = %s...", row[12]);
            fwrite(&data, sizeof(starData), 1, f);
            if(named) {
                fwrite(&name.bayerName, BAYER_LIMIT, 1, namefile);
                fwrite(&name.longName, LONG_NAME_LIMIT, 1, namefile);
                names_count++;
                if(VERBOSE) fprintf(stderr, "Named star count = %ul", names_count);
            }
            if(VERBOSE) 
                fprintf(stderr, "Done.\n");

        }
        mysql_free_result(result);
        lim += MYSQL_STARS_PER_QUERY;
    }

    do {
        writeIndexEntry(nsfhead, current_trixel, ns_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + nsf_trix_begin, nsf_trix_count);
        writeIndexEntry(dsfhead, current_trixel, ds_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + dsf_trix_begin, dsf_trix_count);
        writeIndexEntry(usfhead, current_trixel, us_header_offset + NTRIXELS * INDEX_ENTRY_SIZE + usf_trix_begin, usf_trix_count);
        nsf_trix_begin = ftell(nsf);
        dsf_trix_begin = ftell(dsf);
        usf_trix_begin = ftell(usf);
        nsf_trix_count = usf_trix_count = dsf_trix_count = 0;
        current_trixel++;
        ntrixels++;
    }while(current_trixel != NTRIXELS);

    fseek(namefile, nf_header_offset, SEEK_SET);
    writeIndexEntry(namefile, 0, nf_header_offset + INDEX_ENTRY_SIZE, names_count);

    if(ntrixels != NTRIXELS) {
        fprintf(stderr, "ERROR: Expected %u trixels, but found %u instead. Please redefine NTRIXELS in this program, or check the source database for bogus trixels\n", NTRIXELS, ntrixels);
    }

    rewind(usf);
    rewind(dsf);
    rewind(nsf);
    fwrite(&maglim, 2, 1, dsf);
    maglim = GLOBAL_MAG_LIMIT * 100;
    fwrite(&maglim, 2, 1, nsf);
    fwrite(&maglim, 2, 1, usf);
    fwrite(&htm_level, 1, 1, usf);
    fwrite(&htm_level, 1, 1, nsf);
    fwrite(&htm_level, 1, 1, dsf);
    fwrite(&MSpT_unnamed, 2, 1, usf);
    fwrite(&MSpT_deep, 2, 1, dsf);
    fwrite(&MSpT_named, 2, 1, nsf);

    fcloseall();
    mysql_close(&link);
    
    return 0;
}
