BinaryDataLoader Class Reference

A class to load a whole population from a binary data file. More...

#include <binarydataloader.h>

Collaboration diagram for BinaryDataLoader:

[legend]

List of all members.

Public Member Functions
	BinaryDataLoader ()
	~BinaryDataLoader ()
int	extractOffsetTable (int FD)
Metapop *	extractPop (string &filename, unsigned int generation, SimBuilder *sim, Metapop *popPtr)
Metapop *	getPop () const
unsigned int	getGeneration () const
void	setGeneration (unsigned int gen)
const BinaryStorageBuffer *	getBuffer () const
void	clear ()
Private Attributes
SimBuilder *	_current_sim
BinaryStorageBuffer	_buff
Metapop *	_in_pop
SimBuilder *	_new_sim
unsigned int	_gen
std::string	_filename
std::map< unsigned int, unsigned int >	_offset_table
BinaryFileParser	_pExtractor

---

Detailed Description

A class to load a whole population from a binary data file.

Definition at line 40 of file binarydataloader.h.

---

Constructor & Destructor Documentation

BinaryDataLoader::BinaryDataLoader

(

)

[inline]

Definition at line 60 of file binarydataloader.h.

                     : _current_sim(0),_buff(),_in_pop(0),
                       _new_sim(0),_gen(0),_filename(),_pExtractor(0)
    { }

BinaryDataLoader::~BinaryDataLoader

(

)

Definition at line 44 of file binarydataloader.cc.

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
}

---

Member Function Documentation

void BinaryDataLoader::clear

(

)

Definition at line 49 of file binarydataloader.cc.

{
  if(_in_pop != NULL) delete _in_pop;
  if(_new_sim != NULL) delete _new_sim;
  _in_pop = NULL;
  _new_sim = NULL;
  _buff.clear();
  
}

int BinaryDataLoader::extractOffsetTable

(

int

FD

)

Definition at line 58 of file binarydataloader.cc.

Referenced by extractPop().

{
  int current_pos = 0, rout, nb_recgen = 0, off_table = 0;
  lseek(FD,0,SEEK_END);
  current_pos = lseek(FD,0,SEEK_CUR);
//#ifdef _DEBUG_ 
//  message("eof at: %i\n",current_pos);
//  message("position at the end of the file, at the position to read table info: ");
//#endif
  if((current_pos = lseek(FD,current_pos -2*sizeof(int),SEEK_SET)) == -1) 
    fatal("BinaryDataLoader::extractOffsetTable::lseek %s\n",strerror(errno));
  
//#ifdef _DEBUG_ 
//  message("now at: %i\n",current_pos);
//  message("get the number of generations stored in the file: "); 
//#endif
  unsigned int tot=0;
  do{
  if((rout = read(FD,&nb_recgen,sizeof(int))) == -1)// || rout != sizeof(int)) 
    fatal("BinaryDataLoader::extractOffsetTable::read %s\n",strerror(errno));
    tot+=rout;
    }while(tot < sizeof(int) && rout != 0);
//#ifdef _DEBUG_ 
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%i, bytes read:%i, current offset %i\n",nb_recgen, rout, current_pos);
//  message("get offset of the beginning of the offset table: "); 
//#endif
  if((rout = read(FD,&off_table,sizeof(int))) == -1)// || rout != sizeof(int)) 
    fatal("BinaryDataLoader::extractOffsetTable::read %s\n",strerror(errno));
//#ifdef _DEBUG_ 
//  current_pos = lseek(FD,0,SEEK_CUR);
//  message("%i, bytes read:%i, current offset %i\n",off_table, rout, current_pos);
//  message("position at the beginning of the offset table:\n"); 
//#endif
  if((current_pos = lseek(FD,off_table,SEEK_SET)) == -1) 
    fatal("BinaryDataLoader::extractOffsetTable::lseek %s",strerror(errno));
 
//#ifdef _DEBUG_ 
//  message("now at: %i\n",current_pos);
//  message("check if we are correctly positionned, get the separator: "); 
//#endif
  unsigned char separator[3];
  if((rout = read(FD, &separator, 3)) == -1)// || rout != 3) 
    fatal("BinaryDataLoader::extractOffsetTable::read %s\n",strerror(errno));
  
  if( separator[0] != '@' || separator[1] != 'O' || separator[2] != 'T')
    fatal("BinaryDataLoader::extractOffsetTable:: wrong separator\n");
//#ifdef _DEBUG_ 
//  message(" ok\n read the table elements:\n"); 
//#endif
  int table_elt[2];
  _offset_table.clear();
  for(int i = 0; i < nb_recgen; ++i) {
    
    if((rout = read(FD,&table_elt,2*sizeof(int))) == -1 || rout != 2*sizeof(int)) 
      fatal("BinaryDataLoader::extractOffsetTable::read %s\n",strerror(errno));
    
    _offset_table[table_elt[0]] = table_elt[1];
//#ifdef _DEBUG_ 
//    message("gen %i at %i\n",table_elt[0], table_elt[1]); 
//#endif
  }
//#ifdef _DEBUG_ 
//    message("returning %i\n",off_table); 
//#endif
  return off_table;
}

Metapop * BinaryDataLoader::extractPop	(	string &	filename,
		unsigned int	generation,
		SimBuilder *	sim,
		Metapop *	popPtr
	)

Definition at line 126 of file binarydataloader.cc.

Referenced by Metapop::loadPopFromBinarySource().

{
  int FD, gen_length, gen_offset=0, otab_offset;
  string cmd;
  bool do_compress = 0;
  _filename = file;
  _gen = generation;
  _pExtractor.setName(file.c_str());
  
  if(_in_pop != NULL) delete _in_pop;
  _in_pop = new Metapop();
  _in_pop->setMPImanager( popPtr->_mpimgr );
  //create new sim builder from copy of existing one, will copy templates
  if(_current_sim != sim || _new_sim == 0) {
    if(_new_sim != 0) delete _new_sim;
    _new_sim = new SimBuilder(*sim);
    _current_sim = sim;
  } else
    _new_sim->build_allParams(); //reset the list of params, get rid of the old population paramset.
  //add the current metapop paramSet:
  _new_sim->add_paramset(_in_pop->get_paramset());

  if((FD = open(_filename.c_str(),O_RDONLY)) == -1){
    close(FD);
    //check if we have to uncompress it:
    string alt_name = _filename + ".bz2";
    int status;
    if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
      close(FD);
      //check if gziped:
      alt_name = _filename + ".gz";
      if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
        close(FD);
        fatal("BinaryDataLoader::extractPop::open %s: %s\n",_filename.c_str(),strerror(errno));
      } else {
        //try to ungzip it:
        cmd = "gunzip " + alt_name;
        status = system(cmd.c_str());
        if(status == 1 || status < 0)
          fatal("BinaryDataLoader::extractPop::gunzip failed on %s\n", alt_name.c_str());
        else {
          do_compress = 1;
          cmd = "gzip " + _filename;
        }
      }
    } else {
      //try to bunzip2:
      cmd = "bunzip2 " + alt_name;
      status = system(cmd.c_str());
      if(status != 0)
        fatal("BinaryDataLoader::extractPop::bunzip2 failed on %s\n", alt_name.c_str());
      else {
        do_compress = 1;
        cmd = "bzip2 " + _filename;
      }
    }
    
    if((FD = open(_filename.c_str(),O_RDONLY)) == -1) {//this should work now...
      close(FD);
      fatal("BinaryDataLoader::extractPop::open %s failed: %s\n", _filename.c_str(), strerror(errno));
    }
  }
  
  //1. read the offset table:
  otab_offset = extractOffsetTable(FD);

  //2. read the params and build the prototypes
  //extract params from binary file and set the sim params
  if( !(_new_sim->build_currentParams(_pExtractor.getParameters(NULL))) ){
    error("BinaryDataLoader::extractPop::could not set params\n");
    return NULL;
  }
  //set Metapop params
  //build the list of the selected trait templates, will init() the prototypes
  //and load them into the pop, build the individual prototype
//  if( !(_in_pop->init(_new_sim->build_currentTraits(),_new_sim->build_currentLifeCycle())) )
//    return NULL;
  if(!_in_pop->setParameters()) return NULL;  
  _in_pop->buildPatchArray();
  _in_pop->makePrototype(_new_sim->build_currentTraits());
  
  //3. check prototypes coherence with existing pop?
  /* check prototypes coherence with existing pop, might have to add or remove traits? */
  
  //4. load the pop
  //compute the number of bytes to read:
  std::map<unsigned int, unsigned int>::iterator OT = _offset_table.end(); 
  //starting point:
  if(generation == 0) {
    OT--;
    gen_offset = OT->second; //load last generation recorded
    _gen = OT->first;
  } else {
    if( (OT = _offset_table.find(generation)) == _offset_table.end() )
      fatal("BinaryDataLoader::extractPop::generation %i not found in file \"%s\"\n",generation,_filename.c_str());
    else
       gen_offset = OT->second;
  }
//#ifdef _DEBUG_ 
//  message("\ngeneration offset is: %i\notab offset is: %i\n",gen_offset,otab_offset); 
//#endif
  //length
  if(_offset_table.size() == 1 || (++OT) == _offset_table.end())
    gen_length = otab_offset - gen_offset;
  else
    gen_length = OT->second - gen_offset;
  
  assert(gen_length > 0);

  //+ 2 bytes to get the next separator:
  gen_length += 2;

//#ifdef _DEBUG_ 
//  message("nb bytes to mmap are: %i\n",gen_length); 
//#endif
  int current_pos = 0;
  
  if((current_pos = lseek(FD,gen_offset,SEEK_SET)) == -1) 
    fatal("BinaryDataLoader::extractPop::lseek %s",strerror(errno));  
  
  int rout = -1, rcount = 0, rest = gen_length;
  char *data = new char [gen_length];
  
  while(rest != 0 && rout != 0) {
    if( (rout = read(FD,&data[rcount],rest)) == -1) fatal("BinaryDataLoader::extractPop::read data %s\n",strerror(errno));
    rcount += rout;
    rest -= rout;
#ifdef _DEBUG_ 
    message("BinaryDataLoader::extractPop:read %i bytes from file\n",rout);
#endif
  }
  
  _buff.set_buff(data, gen_length);
  
  delete [] data;
  
  unsigned char separator[2];
  
  _buff.BSBread(&separator, 2 * sizeof(unsigned char));
  
  if( separator[0] != '@' || separator[1] != 'G')
    fatal("BinaryDataLoader::extractPop:: wrong generation separator\n");
  
  _buff.BSBread(&generation, sizeof(unsigned int));
  if(generation != _gen)
    fatal("BinaryDataLoader::extractPop:: wrong generation in file\n");
  
  bool status = true;
  //retrieve data from the buffer:
  status = _in_pop->retrieve_data(&_buff);
  
  close(FD);
  
  if(do_compress){
    if( system(cmd.c_str()) == 1)
      warning("BinaryDataLoader::extractPop:: compress source file failed on %s\n", _filename.c_str());
  }
  
  if( !status ) return NULL;

  else return _in_pop;

}

const BinaryStorageBuffer* BinaryDataLoader::getBuffer

(

)

const [inline]

Definition at line 78 of file binarydataloader.h.

{return &_buff;}

unsigned int BinaryDataLoader::getGeneration

(

)

const [inline]

Definition at line 74 of file binarydataloader.h.

{return _gen;}

Metapop* BinaryDataLoader::getPop

(

)

const [inline]

Definition at line 72 of file binarydataloader.h.

{return _in_pop;}

void BinaryDataLoader::setGeneration

(

unsigned int

gen

)

[inline]

Definition at line 76 of file binarydataloader.h.

{_gen = gen;}

---

Member Data Documentation

BinaryStorageBuffer BinaryDataLoader::_buff [private]

Definition at line 44 of file binarydataloader.h.

Referenced by clear(), extractPop(), and getBuffer().

SimBuilder* BinaryDataLoader::_current_sim [private]

Definition at line 42 of file binarydataloader.h.

Referenced by extractPop().

std::string BinaryDataLoader::_filename [private]

Definition at line 52 of file binarydataloader.h.

Referenced by extractPop().

unsigned int BinaryDataLoader::_gen [private]

Definition at line 50 of file binarydataloader.h.

Referenced by extractPop(), getGeneration(), and setGeneration().

Metapop* BinaryDataLoader::_in_pop [private]

Definition at line 46 of file binarydataloader.h.

Referenced by clear(), extractPop(), getPop(), and ~BinaryDataLoader().

SimBuilder* BinaryDataLoader::_new_sim [private]

Definition at line 48 of file binarydataloader.h.

Referenced by clear(), extractPop(), and ~BinaryDataLoader().

std::map<unsigned int, unsigned int> BinaryDataLoader::_offset_table [private]

Definition at line 54 of file binarydataloader.h.

Referenced by extractOffsetTable(), and extractPop().

BinaryFileParser BinaryDataLoader::_pExtractor [private]

Definition at line 56 of file binarydataloader.h.

Referenced by extractPop().

---

The documentation for this class was generated from the following files:

• binarydataloader.h
• binarydataloader.cc