Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Ambuj Mehrish
sidekit
Commits
8ce08538
Commit
8ce08538
authored
Mar 15, 2019
by
Anthony Larcher
Browse files
write statserver without closing h5py.File
parent
6be57928
Changes
1
Hide whitespace changes
Inline
Side-by-side
statserver.py
View file @
8ce08538
...
...
@@ -423,57 +423,62 @@ class StatServer:
stop
[
numpy
.
isnan
(
self
.
stop
.
astype
(
'float'
))]
=
-
1
stop
=
stop
.
astype
(
'int8'
,
copy
=
False
)
with
h5py
.
File
(
output_file_name
,
mode
)
as
f
:
if
isinstnace
(
output_file_name
,
h5py
.
_hl
.
files
.
File
):
f
=
output_file_name
else
;
f
=
h5py
.
File
(
output_file_name
,
mode
)
# If the file doesn't exist before, create it
if
mode
==
"w"
or
not
file_already_exist
:
f
.
create_dataset
(
prefix
+
"modelset"
,
data
=
self
.
modelset
.
astype
(
'S'
),
if
mode
==
"w"
or
not
file_already_exist
:
f
.
create_dataset
(
prefix
+
"modelset"
,
data
=
self
.
modelset
.
astype
(
'S'
),
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"segset"
,
data
=
self
.
segset
.
astype
(
'S'
),
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stat0"
,
data
=
self
.
stat0
.
astype
(
numpy
.
float32
),
maxshape
=
(
None
,
self
.
stat0
.
shape
[
1
]),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stat1"
,
data
=
self
.
stat1
.
astype
(
numpy
.
float32
),
maxshape
=
(
None
,
self
.
stat1
.
shape
[
1
]),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"start"
,
data
=
start
,
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stop"
,
data
=
stop
,
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"segset"
,
data
=
self
.
segset
.
astype
(
'S'
),
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stat0"
,
data
=
self
.
stat0
.
astype
(
numpy
.
float32
),
maxshape
=
(
None
,
self
.
stat0
.
shape
[
1
]),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stat1"
,
data
=
self
.
stat1
.
astype
(
numpy
.
float32
),
maxshape
=
(
None
,
self
.
stat1
.
shape
[
1
]),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"start"
,
data
=
start
,
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
f
.
create_dataset
(
prefix
+
"stop"
,
data
=
stop
,
maxshape
=
(
None
,),
compression
=
"gzip"
,
fletcher32
=
True
)
# If the file already exist, we extend all datasets and add the new data
else
:
# If the file already exist, we extend all datasets and add the new data
else
:
previous_size
=
f
[
prefix
+
"modelset"
].
shape
[
0
]
# Extend the size of each dataset
f
[
prefix
+
"modelset"
].
resize
((
previous_size
+
self
.
modelset
.
shape
[
0
],))
f
[
prefix
+
"segset"
].
resize
((
previous_size
+
self
.
segset
.
shape
[
0
],))
f
[
prefix
+
"start"
].
resize
((
previous_size
+
start
.
shape
[
0
],))
f
[
prefix
+
"stop"
].
resize
((
previous_size
+
stop
.
shape
[
0
],))
f
[
prefix
+
"stat0"
].
resize
((
previous_size
+
self
.
stat0
.
shape
[
0
],
self
.
stat0
.
shape
[
1
]))
f
[
prefix
+
"stat1"
].
resize
((
previous_size
+
self
.
stat1
.
shape
[
0
],
self
.
stat1
.
shape
[
1
]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f
[
prefix
+
"modelset"
][
previous_size
:]
=
self
.
modelset
.
astype
(
'S'
)
f
[
prefix
+
"segset"
][
previous_size
:]
=
self
.
segset
.
astype
(
'S'
)
f
[
prefix
+
"start"
][
previous_size
:]
=
start
f
[
prefix
+
"stop"
][
previous_size
:]
=
stop
f
[
prefix
+
"stat0"
][
previous_size
:,
:]
=
self
.
stat0
.
astype
(
STAT_TYPE
)
f
[
prefix
+
"stat1"
][
previous_size
:,
:]
=
self
.
stat1
.
astype
(
STAT_TYPE
)
previous_size
=
f
[
prefix
+
"modelset"
].
shape
[
0
]
# Extend the size of each dataset
f
[
prefix
+
"modelset"
].
resize
((
previous_size
+
self
.
modelset
.
shape
[
0
],))
f
[
prefix
+
"segset"
].
resize
((
previous_size
+
self
.
segset
.
shape
[
0
],))
f
[
prefix
+
"start"
].
resize
((
previous_size
+
start
.
shape
[
0
],))
f
[
prefix
+
"stop"
].
resize
((
previous_size
+
stop
.
shape
[
0
],))
f
[
prefix
+
"stat0"
].
resize
((
previous_size
+
self
.
stat0
.
shape
[
0
],
self
.
stat0
.
shape
[
1
]))
f
[
prefix
+
"stat1"
].
resize
((
previous_size
+
self
.
stat1
.
shape
[
0
],
self
.
stat1
.
shape
[
1
]))
# add the new data; WARNING: no check is done on the new data, beware of duplicated entries
f
[
prefix
+
"modelset"
][
previous_size
:]
=
self
.
modelset
.
astype
(
'S'
)
f
[
prefix
+
"segset"
][
previous_size
:]
=
self
.
segset
.
astype
(
'S'
)
f
[
prefix
+
"start"
][
previous_size
:]
=
start
f
[
prefix
+
"stop"
][
previous_size
:]
=
stop
f
[
prefix
+
"stat0"
][
previous_size
:,
:]
=
self
.
stat0
.
astype
(
STAT_TYPE
)
f
[
prefix
+
"stat1"
][
previous_size
:,
:]
=
self
.
stat1
.
astype
(
STAT_TYPE
)
if
not
isinstnace
(
output_file_name
,
h5py
.
_hl
.
files
.
File
):
f
.
close
()
def
get_model_stat0
(
self
,
mod_id
):
"""Return zero-order statistics of a given model
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment