Commit 8ce08538 authored by Anthony Larcher's avatar Anthony Larcher
Browse files

write statserver without closing h5py.File

parent 6be57928
......@@ -423,57 +423,62 @@ class StatServer:
stop[numpy.isnan(self.stop.astype('float'))] = -1
stop = stop.astype('int8', copy=False)
with h5py.File(output_file_name, mode) as f:
if isinstnace(output_file_name, h5py._hl.files.File):
f = output_file_name
else;
f = h5py.File(output_file_name, mode)
# If the file doesn't exist before, create it
if mode == "w" or not file_already_exist:
f.create_dataset(prefix+"modelset", data=self.modelset.astype('S'),
if mode == "w" or not file_already_exist:
f.create_dataset(prefix+"modelset", data=self.modelset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"segset", data=self.segset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat0", data=self.stat0.astype(numpy.float32),
maxshape=(None, self.stat0.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat1", data=self.stat1.astype(numpy.float32),
maxshape=(None, self.stat1.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"start", data=start,
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stop", data=stop,
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"segset", data=self.segset.astype('S'),
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat0", data=self.stat0.astype(numpy.float32),
maxshape=(None, self.stat0.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stat1", data=self.stat1.astype(numpy.float32),
maxshape=(None, self.stat1.shape[1]),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"start", data=start,
maxshape=(None,),
compression="gzip",
fletcher32=True)
f.create_dataset(prefix+"stop", data=stop,
maxshape=(None,),
compression="gzip",
fletcher32=True)
# If the file already exist, we extend all datasets and add the new data
else:
# If the file already exist, we extend all datasets and add the new data
else:
previous_size = f[prefix+"modelset"].shape[0]
# Extend the size of each dataset
f[prefix+"modelset"].resize((previous_size + self.modelset.shape[0],))
f[prefix+"segset"].resize((previous_size + self.segset.shape[0],))
f[prefix+"start"].resize((previous_size + start.shape[0],))
f[prefix+"stop"].resize((previous_size + stop.shape[0],))
f[prefix+"stat0"].resize((previous_size + self.stat0.shape[0], self.stat0.shape[1]))
f[prefix+"stat1"].resize((previous_size + self.stat1.shape[0], self.stat1.shape[1]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f[prefix+"modelset"][previous_size:] = self.modelset.astype('S')
f[prefix+"segset"][previous_size:] = self.segset.astype('S')
f[prefix+"start"][previous_size:] = start
f[prefix+"stop"][previous_size:] = stop
f[prefix+"stat0"][previous_size:, :] = self.stat0.astype(STAT_TYPE)
f[prefix+"stat1"][previous_size:, :] = self.stat1.astype(STAT_TYPE)
previous_size = f[prefix+"modelset"].shape[0]
# Extend the size of each dataset
f[prefix+"modelset"].resize((previous_size + self.modelset.shape[0],))
f[prefix+"segset"].resize((previous_size + self.segset.shape[0],))
f[prefix+"start"].resize((previous_size + start.shape[0],))
f[prefix+"stop"].resize((previous_size + stop.shape[0],))
f[prefix+"stat0"].resize((previous_size + self.stat0.shape[0], self.stat0.shape[1]))
f[prefix+"stat1"].resize((previous_size + self.stat1.shape[0], self.stat1.shape[1]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f[prefix+"modelset"][previous_size:] = self.modelset.astype('S')
f[prefix+"segset"][previous_size:] = self.segset.astype('S')
f[prefix+"start"][previous_size:] = start
f[prefix+"stop"][previous_size:] = stop
f[prefix+"stat0"][previous_size:, :] = self.stat0.astype(STAT_TYPE)
f[prefix+"stat1"][previous_size:, :] = self.stat1.astype(STAT_TYPE)
if not isinstnace(output_file_name, h5py._hl.files.File):
f.close()
def get_model_stat0(self, mod_id):
"""Return zero-order statistics of a given model
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment