Commit 045f0e51 authored by Anthony Larcher's avatar Anthony Larcher
Browse files

+ statserver to_hdf5

parent 8ce08538
......@@ -423,62 +423,120 @@ class StatServer:
stop[numpy.isnan(self.stop.astype('float'))] = -1
stop = stop.astype('int8', copy=False)
if isinstnace(output_file_name, h5py._hl.files.File):
f = output_file_name
else;
f = h5py.File(output_file_name, mode)
with h5py.File(output_file_name, mode) as f:
# If the file doesn't exist before, create it
if mode == "w" or not file_already_exist:
if mode == "w" or not file_already_exist:
f.create_dataset(prefix+"modelset", data=self.modelset.astype('S'),
f.create_dataset(prefix+"modelset", data=self.modelset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"segset", data=self.segset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat0", data=self.stat0.astype(numpy.float32),
maxshape=(None, self.stat0.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat1", data=self.stat1.astype(numpy.float32),
maxshape=(None, self.stat1.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"start", data=start,
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stop", data=stop,
maxshape=(None,),
compression="gzip",
fletcher32=True)
# If the file already exist, we extend all datasets and add the new data
else:
previous_size = f[prefix+"modelset"].shape[0]
# Extend the size of each dataset
f[prefix+"modelset"].resize((previous_size + self.modelset.shape[0],))
f[prefix+"segset"].resize((previous_size + self.segset.shape[0],))
f[prefix+"start"].resize((previous_size + start.shape[0],))
f[prefix+"stop"].resize((previous_size + stop.shape[0],))
f[prefix+"stat0"].resize((previous_size + self.stat0.shape[0], self.stat0.shape[1]))
f[prefix+"stat1"].resize((previous_size + self.stat1.shape[0], self.stat1.shape[1]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f[prefix+"modelset"][previous_size:] = self.modelset.astype('S')
f[prefix+"segset"][previous_size:] = self.segset.astype('S')
f[prefix+"start"][previous_size:] = start
f[prefix+"stop"][previous_size:] = stop
f[prefix+"stat0"][previous_size:, :] = self.stat0.astype(STAT_TYPE)
f[prefix+"stat1"][previous_size:, :] = self.stat1.astype(STAT_TYPE)
def to_hdf5(self, h5f, prefix='', mode='w'):
"""Write the StatServer to disk in hdf5 format.
:param output_file_name: name of the file to write in.
:param prefix:
"""
assert self.validate(), "Error: wrong StatServer format"
start = copy.deepcopy(self.start)
start[numpy.isnan(self.start.astype('float'))] = -1
start = start.astype('int8', copy=False)
stop = copy.deepcopy(self.stop)
stop[numpy.isnan(self.stop.astype('float'))] = -1
stop = stop.astype('int8', copy=False)
# If the file doesn't exist before, create it
if mode == "w":
h5f.create_dataset(prefix + "modelset", data=self.modelset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"segset", data=self.segset.astype('S'),
h5f.create_dataset(prefix + "segset", data=self.segset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat0", data=self.stat0.astype(numpy.float32),
h5f.create_dataset(prefix + "stat0", data=self.stat0.astype(numpy.float32),
maxshape=(None, self.stat0.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat1", data=self.stat1.astype(numpy.float32),
h5f.create_dataset(prefix + "stat1", data=self.stat1.astype(numpy.float32),
maxshape=(None, self.stat1.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"start", data=start,
h5f.create_dataset(prefix + "start", data=start,
maxshape=(None,),
compression="gzip",
fletcher32=True)
h5f.create_dataset(prefix + "stop", data=stop,
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stop", data=stop,
maxshape=(None,),
compression="gzip",
fletcher32=True)
# If the file already exist, we extend all datasets and add the new data
else:
previous_size = f[prefix+"modelset"].shape[0]
previous_size = f[prefix + "modelset"].shape[0]
# Extend the size of each dataset
f[prefix+"modelset"].resize((previous_size + self.modelset.shape[0],))
f[prefix+"segset"].resize((previous_size + self.segset.shape[0],))
f[prefix+"start"].resize((previous_size + start.shape[0],))
f[prefix+"stop"].resize((previous_size + stop.shape[0],))
f[prefix+"stat0"].resize((previous_size + self.stat0.shape[0], self.stat0.shape[1]))
f[prefix+"stat1"].resize((previous_size + self.stat1.shape[0], self.stat1.shape[1]))
h5f[prefix + "modelset"].resize((previous_size + self.modelset.shape[0],))
h5f[prefix + "segset"].resize((previous_size + self.segset.shape[0],))
h5f[prefix + "start"].resize((previous_size + start.shape[0],))
h5f[prefix + "stop"].resize((previous_size + stop.shape[0],))
h5f[prefix + "stat0"].resize((previous_size + self.stat0.shape[0], self.stat0.shape[1]))
h5f[prefix + "stat1"].resize((previous_size + self.stat1.shape[0], self.stat1.shape[1]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f[prefix+"modelset"][previous_size:] = self.modelset.astype('S')
f[prefix+"segset"][previous_size:] = self.segset.astype('S')
f[prefix+"start"][previous_size:] = start
f[prefix+"stop"][previous_size:] = stop
f[prefix+"stat0"][previous_size:, :] = self.stat0.astype(STAT_TYPE)
f[prefix+"stat1"][previous_size:, :] = self.stat1.astype(STAT_TYPE)
if not isinstnace(output_file_name, h5py._hl.files.File):
f.close()
h5f[prefix + "modelset"][previous_size:] = self.modelset.astype('S')
h5f[prefix + "segset"][previous_size:] = self.segset.astype('S')
h5f[prefix + "start"][previous_size:] = start
h5f[prefix + "stop"][previous_size:] = stop
h5f[prefix + "stat0"][previous_size:, :] = self.stat0.astype(STAT_TYPE)
h5f[prefix + "stat1"][previous_size:, :] = self.stat1.astype(STAT_TYPE)
def get_model_stat0(self, mod_id):
"""Return zero-order statistics of a given model
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment