Nemo  2.3.46
BinaryDataLoader Class Reference

A class to load a whole population from a binary data file. More...

#include <binarydataloader.h>

+ Collaboration diagram for BinaryDataLoader:

Public Member Functions

 BinaryDataLoader ()
 
 ~BinaryDataLoader ()
 
int extractOffsetTable (int FD)
 
MetapopextractPop (string &filename, unsigned int generation, SimBuilder *sim, Metapop *popPtr)
 
MetapopgetPop () const
 
unsigned int getGeneration () const
 
void setGeneration (unsigned int gen)
 
const BinaryStorageBuffergetBuffer () const
 
void clear ()
 

Private Attributes

SimBuilder_current_sim
 
BinaryStorageBuffer _buff
 
Metapop_in_pop
 
SimBuilder_new_sim
 
unsigned int _gen
 
std::string _filename
 
std::map< unsigned int,
unsigned int
_offset_table
 
BinaryFileParser _pExtractor
 

Detailed Description

A class to load a whole population from a binary data file.

Constructor & Destructor Documentation

BinaryDataLoader::BinaryDataLoader ( )
inline
61  : _current_sim(0),_buff(),_in_pop(0),
63  { }
SimBuilder * _new_sim
Definition: binarydataloader.h:49
std::string _filename
Definition: binarydataloader.h:53
BinaryFileParser _pExtractor
Definition: binarydataloader.h:57
unsigned int _gen
Definition: binarydataloader.h:51
BinaryStorageBuffer _buff
Definition: binarydataloader.h:45
Metapop * _in_pop
Definition: binarydataloader.h:47
SimBuilder * _current_sim
Definition: binarydataloader.h:43
BinaryDataLoader::~BinaryDataLoader ( )

References _in_pop, and _new_sim.

47 {
48  if(_in_pop != NULL) delete _in_pop;
49  if(_new_sim != NULL) delete _new_sim;
50 }
SimBuilder * _new_sim
Definition: binarydataloader.h:49
Metapop * _in_pop
Definition: binarydataloader.h:47

Member Function Documentation

void BinaryDataLoader::clear ( )

References _buff, _in_pop, _new_sim, and BinaryStorageBuffer::clear().

52 {
53  if(_in_pop != NULL) delete _in_pop;
54  if(_new_sim != NULL) delete _new_sim;
55  _in_pop = NULL;
56  _new_sim = NULL;
57  _buff.clear();
58 
59 }
SimBuilder * _new_sim
Definition: binarydataloader.h:49
void clear()
Definition: binarystoragebuffer.h:54
BinaryStorageBuffer _buff
Definition: binarydataloader.h:45
Metapop * _in_pop
Definition: binarydataloader.h:47
int BinaryDataLoader::extractOffsetTable ( int  FD)

References _offset_table, and fatal().

Referenced by extractPop().

61 {
62  int rout, nb_recgen = 0, off_table = 0;
63  lseek(FD,0,SEEK_END);
64  off_t current_pos = 0L;
65 
66  current_pos = lseek(FD,0,SEEK_CUR);
67 //#ifdef _DEBUG_
68 // message("eof at: %i\n",current_pos);
69 // message("position at the end of the file, at the position to read table info: ");
70 //#endif
71  if((current_pos = lseek(FD,current_pos -2*sizeof(int),SEEK_SET)) == -1)
72  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::lseek failed on %s\n",strerror(errno));
73 
74 //#ifdef _DEBUG_
75 // message("now at: %i\n",current_pos);
76 // message("get the number of generations stored in the file: ");
77 //#endif
78  unsigned int tot=0;
79  do{
80  if((rout = read(FD,&nb_recgen,sizeof(int))) == -1)// || rout != sizeof(int))
81  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
82  tot+=rout;
83  }while(tot < sizeof(int) && rout != 0);
84 //#ifdef _DEBUG_
85 // current_pos = lseek(FD,0,SEEK_CUR);
86 // message("%i, bytes read:%i, current offset %i\n",nb_recgen, rout, current_pos);
87 // message("get offset of the beginning of the offset table: ");
88 //#endif
89  if((rout = read(FD,&off_table,sizeof(int))) == -1)// || rout != sizeof(int))
90  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
91 //#ifdef _DEBUG_
92 // current_pos = lseek(FD,0,SEEK_CUR);
93 // message("%i, bytes read:%i, current offset %i\n",off_table, rout, current_pos);
94 // message("position at the beginning of the offset table:\n");
95 //#endif
96  if((current_pos = lseek(FD,off_table,SEEK_SET)) == -1)
97  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::lseek failed on %s",strerror(errno));
98 
99 //#ifdef _DEBUG_
100 // message("now at: %i\n",current_pos);
101 // message("check if we are correctly positionned, get the separator: ");
102 //#endif
103  unsigned char separator[3];
104  if((rout = read(FD, &separator, 3)) == -1)// || rout != 3)
105  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
106 
107  if( separator[0] != '@' || separator[1] != 'O' || separator[2] != 'T')
108  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable:: wrong separator\n");
109 //#ifdef _DEBUG_
110 // message(" ok\n read the table elements:\n");
111 //#endif
112  int table_elt[2];
113  _offset_table.clear();
114  for(int i = 0; i < nb_recgen; ++i) {
115 
116  if((rout = read(FD,&table_elt,2*sizeof(int))) == -1 || rout != 2*sizeof(int))
117  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractOffsetTable::read failed on %s\n",strerror(errno));
118 
119  _offset_table[table_elt[0]] = table_elt[1];
120 //#ifdef _DEBUG_
121 // message("gen %i at %i\n",table_elt[0], table_elt[1]);
122 //#endif
123  }
124 //#ifdef _DEBUG_
125 // message("returning %i\n",off_table);
126 //#endif
127  return off_table;
128 }
void fatal(const char *str,...)
Definition: output.cc:90
std::map< unsigned int, unsigned int > _offset_table
Definition: binarydataloader.h:55
Metapop * BinaryDataLoader::extractPop ( string &  filename,
unsigned int  generation,
SimBuilder sim,
Metapop popPtr 
)

References _buff, _current_sim, _filename, _gen, _in_pop, Metapop::_mpimgr, _new_sim, _offset_table, _pExtractor, ParamManager::add_paramset(), BinaryStorageBuffer::BSBread(), SimBuilder::build_currentParams(), SimBuilder::build_currentTraits(), Metapop::buildPatchArray(), error(), extractOffsetTable(), fatal(), SimComponent::get_paramset(), ParamsParser::getParameters(), tstring::int2str(), IndFactory::makePrototype(), message(), Metapop::retrieve_data(), BinaryStorageBuffer::set_buff(), Metapop::setMPImanager(), ParamsParser::setName(), Metapop::setParameters(), RAND::Uniform(), and warning().

Referenced by Metapop::loadPopFromBinarySource().

131 {
132  int FD, gen_length, gen_offset=0, otab_offset;
133  string cmd, magic_name, old_name;
134  bool do_compress = 0;
135  _filename = file;
136  _gen = generation;
137  _pExtractor.setName(file.c_str());
138 
139  if(_in_pop != NULL) delete _in_pop;
140  _in_pop = new Metapop();
141  _in_pop->setMPImanager( popPtr->_mpimgr );
142 
143  //create new sim builder from copy of existing one, will copy templates
144  if(_new_sim != 0) delete _new_sim;
145  _new_sim = new SimBuilder(*sim);
146  _current_sim = sim;
147 
148  //add the current metapop paramSet:
150 
151  if((FD = open(_filename.c_str(),O_RDONLY)) == -1){
152  close(FD);
153  //check if we have to uncompress it:
154  string alt_name = _filename + ".bz2";
155  magic_name = _filename + tstring::int2str(RAND::Uniform(5477871));
156  int status;
157  if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
158  close(FD);
159  //check if gziped:
160  alt_name = _filename + ".gz";
161  if((FD = open(alt_name.c_str(),O_RDONLY)) == -1){
162  close(FD);
163  fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n",_filename.c_str(),strerror(errno));
164  } else {
165  //try to ungzip it:
166  cmd = "gzip -cd " + alt_name + " > " + magic_name;
167 
168  }
169  }
170  close(FD);
171  //try to bunzip2:
172  cmd = "bunzip2 -ck " + alt_name + " > " + magic_name;
173 
174  status = system(cmd.c_str());
175  if(status != 0)
176  fatal("BinaryDataLoader::extractPop::bunzip2 failed on %s\n", alt_name.c_str());
177 
178  if((FD = open(magic_name.c_str(),O_RDONLY)) == -1) {//this should work now...
179  close(FD);
180  fatal("BinaryDataLoader::extractPop::open failed on %s: %s\n", _filename.c_str(), strerror(errno));
181  }
182  old_name = _filename;
183  _filename = magic_name;
184  _pExtractor.setName(magic_name.c_str());
185  do_compress = 1; //to remove the duplicated source file
186  }
187 
188  //1. read the offset table:
189  otab_offset = extractOffsetTable(FD);
190 
191  //2. read the params and build the prototypes
192  //extract params from binary file and set the sim params
193  // !!! the param values in init part might not reflect pop state, esp if temporal arguments used !!!
195  error("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::could not set parameters from binary file\n");
196  return NULL;
197  }
198  //set Metapop params
199  //build the list of the selected trait templates, will init() the prototypes
200  //and load them into the pop, build the individual prototype
201 // if( !(_in_pop->init(_new_sim->build_currentTraits(),_new_sim->build_currentLifeCycle())) )
202 // return NULL;
203  if(!_in_pop->setParameters()) return NULL;
206 
207  //3. check prototypes coherence with existing pop?
208  /* check prototypes coherence with existing pop, might have to add or remove traits? */
209 
210  //4. load the pop
211  //compute the number of bytes to read:
212  std::map<unsigned int, unsigned int>::iterator OT = _offset_table.end();
213  //starting point:
214  if(generation == 0) {
215  OT--;
216  gen_offset = OT->second; //load last generation recorded
217  _gen = OT->first;
218  } else {
219  if( (OT = _offset_table.find(generation)) == _offset_table.end() )
220  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::generation %i not found in file \"%s\"\n",generation,_filename.c_str());
221  else
222  gen_offset = OT->second;
223  }
224 //#ifdef _DEBUG_
225 // message("\ngeneration offset is: %i\notab offset is: %i\n",gen_offset,otab_offset);
226 //#endif
227  //length
228  if(_offset_table.size() == 1 || (++OT) == _offset_table.end())
229  gen_length = otab_offset - gen_offset;
230  else
231  gen_length = OT->second - gen_offset;
232 
233  assert(gen_length > 0);
234 
235  //+ 2 bytes to get the next separator:
236  gen_length += 2;
237 
238 //#ifdef _DEBUG_
239 // message("nb bytes to mmap are: %i\n",gen_length);
240 //#endif
241  off_t current_pos = lseek(FD,gen_offset,SEEK_SET);
242 
243  if(current_pos == -1)
244  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::lseek failed on %s\n",strerror(errno));
245 
246  int rout = -1, rcount = 0, rest = gen_length;
247  char *data = new char [gen_length];
248 
249  while(rest != 0 && rout != 0) {
250  if( (rout = read(FD,&data[rcount],rest)) == -1) fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop::read data %s\n",strerror(errno));
251  rcount += rout;
252  rest -= rout;
253 #ifdef _DEBUG_
254  message("BinaryDataLoader::extractPop:read %i bytes from file\n",rout);
255 #endif
256  }
257 
258  close(FD);
259 
260  _buff.set_buff(data, gen_length);
261 
262  delete [] data;
263 
264  unsigned char separator[2];
265 
266  _buff.BSBread(&separator, 2 * sizeof(unsigned char));
267 
268  if( separator[0] != '@' || separator[1] != 'G')
269  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation separator\n");
270 
271  _buff.BSBread(&generation, sizeof(unsigned int));
272  if(generation != _gen)
273  fatal("Binary file appears corrupted:\n >>>> BinaryDataLoader::extractPop:: wrong generation in file\n");
274 
275  bool status = true;
276  //retrieve data from the buffer:
277  status = _in_pop->retrieve_data(&_buff);
278 
279 
280  if(do_compress){
281  cmd = "rm -f " + magic_name;
282  if( system(cmd.c_str()) == 1)
283  warning("BinaryDataLoader::extractPop:: deleting duplicated source file failed on %s\n", _filename.c_str());
284  }
285 
286  delete _new_sim;
287  _new_sim = NULL;
288 
289  if( !status ) return NULL;
290 
291  else return _in_pop;
292 
293 }
Provides methods to build the user's selected set of life cycle events and traits from the parameters...
Definition: basicsimulation.h:162
MPImanager * _mpimgr
Definition: metapop.h:84
void message(const char *message,...)
Definition: output.cc:40
SimBuilder * _new_sim
Definition: binarydataloader.h:49
std::string _filename
Definition: binarydataloader.h:53
virtual bool setParameters()
Definition: metapop.cc:114
map< string, string > & getParameters(const char *stream_name)
Definition: paramsparser.cc:46
map< trait_t, TraitPrototype * > & build_currentTraits()
Selects the trait prototypes that have their parameters set.
Definition: basicsimulation.cc:647
void makePrototype(map< trait_t, TraitPrototype * > TTlist)
Creates the individuals prototype from the selected trait prototypes.
Definition: indfactory.cc:50
Top class of the metapopulation structure, contains the patches.
Definition: metapop.h:79
void fatal(const char *str,...)
Definition: output.cc:90
void buildPatchArray()
Builds the new population from parameter values.
Definition: metapop.cc:282
int error(const char *str,...)
Definition: output.cc:73
bool build_currentParams(map< string, string > &simparams)
Builds the list of parameters from user's defined input parameters.
Definition: basicsimulation.cc:621
BinaryFileParser _pExtractor
Definition: binarydataloader.h:57
void warning(const char *str,...)
Definition: output.cc:56
unsigned int _gen
Definition: binarydataloader.h:51
BinaryStorageBuffer _buff
Definition: binarydataloader.h:45
static string int2str(const int i)
Writes an integer value into a string.
Definition: tstring.h:95
static double Uniform()
Generates a random number from [0.0, 1.0[ uniformly distributed.
Definition: Uniform.h:101
virtual bool retrieve_data(BinaryStorageBuffer *reader)
Definition: metapop.cc:529
int extractOffsetTable(int FD)
Definition: binarydataloader.cc:60
void setMPImanager(MPImanager *mgr)
Definition: metapop.h:170
virtual ParamSet * get_paramset()
ParamSet accessor.
Definition: simcomponent.h:108
Metapop * _in_pop
Definition: binarydataloader.h:47
void BSBread(void *out, unsigned int nb_bytes)
Definition: binarystoragebuffer.h:164
void setName(const char *name)
Definition: paramsparser.h:47
std::map< unsigned int, unsigned int > _offset_table
Definition: binarydataloader.h:55
SimBuilder * _current_sim
Definition: binarydataloader.h:43
void set_buff()
Definition: binarystoragebuffer.h:63
void add_paramset(ParamSet *paramset)
Adds a ParamSet to the list of the parameter sets of the simulation.
Definition: basicsimulation.h:95
const BinaryStorageBuffer* BinaryDataLoader::getBuffer ( ) const
inline

References _buff.

77 {return &_buff;}
BinaryStorageBuffer _buff
Definition: binarydataloader.h:45
unsigned int BinaryDataLoader::getGeneration ( ) const
inline

References _gen.

73 {return _gen;}
unsigned int _gen
Definition: binarydataloader.h:51
Metapop* BinaryDataLoader::getPop ( ) const
inline

References _in_pop.

71 {return _in_pop;}
Metapop * _in_pop
Definition: binarydataloader.h:47
void BinaryDataLoader::setGeneration ( unsigned int  gen)
inline
75 {_gen = gen;}
unsigned int _gen
Definition: binarydataloader.h:51

Member Data Documentation

BinaryStorageBuffer BinaryDataLoader::_buff
private

Referenced by clear(), extractPop(), and getBuffer().

SimBuilder* BinaryDataLoader::_current_sim
private

Referenced by extractPop().

std::string BinaryDataLoader::_filename
private

Referenced by extractPop().

unsigned int BinaryDataLoader::_gen
private

Referenced by extractPop(), and getGeneration().

Metapop* BinaryDataLoader::_in_pop
private
SimBuilder* BinaryDataLoader::_new_sim
private
std::map<unsigned int, unsigned int> BinaryDataLoader::_offset_table
private

Referenced by extractOffsetTable(), and extractPop().

BinaryFileParser BinaryDataLoader::_pExtractor
private

Referenced by extractPop().


The documentation for this class was generated from the following files:

Generated for Nemo v2.3.0 by  doxygen 1.8.8 --
Catalogued on GSR