Skip to content
GitLab
Commit 3859800a authored by Anthony Larcher's avatar Anthony Larcher
Browse files

new release

parent 8f28cac0
Show whitespace changes
Inline Side-by-side
docs/build/html/_modules/features_server.html
View file @ 3859800a
......@@ -87,6 +87,7 @@
<span class="kn">import</span> <span class="nn">h5py</span>
<span class="kn">from</span> <span class="nn">sidekit.frontend.features</span> <span class="k">import</span> <span class="n">pca_dct</span><span class="p">,</span> <span class="n">shifted_delta_cepstral</span><span class="p">,</span> <span class="n">compute_delta</span><span class="p">,</span> <span class="n">framing</span><span class="p">,</span> <span class="n">dct_basis</span>
<span class="kn">from</span> <span class="nn">sidekit.frontend.io</span> <span class="k">import</span> <span class="n">read_hdf5_segment</span>
<span class="kn">from</span> <span class="nn">sidekit.frontend.vad</span> <span class="k">import</span> <span class="n">label_fusion</span>
<span class="kn">from</span> <span class="nn">sidekit.frontend.normfeat</span> <span class="k">import</span> <span class="n">cms</span><span class="p">,</span> <span class="n">cmvn</span><span class="p">,</span> <span class="n">stg</span><span class="p">,</span> <span class="n">cep_sliding_norm</span><span class="p">,</span> <span class="n">rasta_filt</span>
<span class="kn">from</span> <span class="nn">sidekit.sv_utils</span> <span class="k">import</span> <span class="n">parse_mask</span>
......@@ -375,7 +376,7 @@
<div class="viewcode-block" id="FeaturesServer.get_context"><a class="viewcode-back" href="../api/featuresserver.html#features_server.FeaturesServer.get_context">[docs]</a> <span class="k">def</span> <span class="nf">get_context</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">feat</span><span class="p">,</span> <span class="n">start</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">stop</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Add a left annd right context to each frame.</span>
<span class="sd"> Add a left and right context to each frame.</span>
<span class="sd"> First and last frames are duplicated to provide context at the begining and at the end</span>
<span class="sd"> :param feat: sequence of feature frames (one fame per line)</span>
......@@ -529,62 +530,19 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">h5f</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">features_extractor</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">show</span><span class="p">,</span> <span class="n">channel</span><span class="p">,</span> <span class="n">input_audio_filename</span><span class="o">=</span><span class="n">input_feature_filename</span><span class="p">)</span>
<span class="c1"># Get the selected segment</span>
<span class="n">dataset_length</span> <span class="o">=</span> <span class="n">h5f</span><span class="p">[</span><span class="n">show</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="nb">next</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="n">show</span><span class="p">]</span><span class="o">.</span><span class="fm">__iter__</span><span class="p">())]</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="c1"># Deal with the case where start &lt; 0 or stop &gt; feat.shape[0]</span>
<span class="k">if</span> <span class="n">start</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">start</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">pad_begining</span> <span class="o">=</span> <span class="o">-</span><span class="n">start</span> <span class="k">if</span> <span class="n">start</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="k">else</span> <span class="mi">0</span>
<span class="n">start</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">start</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="n">stop</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">stop</span> <span class="o">=</span> <span class="n">dataset_length</span>
<span class="n">pad_end</span> <span class="o">=</span> <span class="n">stop</span> <span class="o">-</span> <span class="n">dataset_length</span> <span class="k">if</span> <span class="n">stop</span> <span class="o">&gt;</span> <span class="n">dataset_length</span> <span class="k">else</span> <span class="mi">0</span>
<span class="n">stop</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">stop</span><span class="p">,</span> <span class="n">dataset_length</span><span class="p">)</span>
<span class="n">global_cmvn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">global_cmvn</span> <span class="ow">and</span> <span class="ow">not</span> <span class="p">(</span><span class="n">start</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">stop</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span>
<span class="c1"># Get the data between start and stop</span>
<span class="c1"># Concatenate all required datasets</span>
<span class="n">feat</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">global_mean</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">global_std</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;energy&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_list</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">newaxis</span><span class="p">])</span>
<span class="n">global_mean</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy_mean&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">global_std</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy_std&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;cep&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_list</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep&quot;</span><span class="p">))][</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">,</span> <span class="p">:])</span>
<span class="n">global_mean</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep_mean&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">global_std</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep_std&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;fb&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_list</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb&quot;</span><span class="p">))][</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">,</span> <span class="p">:])</span>
<span class="n">global_mean</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb_mean&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">global_std</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb_std&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;bnf&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset_list</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;bnf&quot;</span><span class="p">))][</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">,</span> <span class="p">:])</span>
<span class="n">global_mean</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;bnf_mean&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">global_std</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;bnf_std&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">feat</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">feat</span><span class="p">)</span>
<span class="n">global_mean</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">global_mean</span><span class="p">)</span>
<span class="n">global_std</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">global_std</span><span class="p">)</span>
<span class="k">if</span> <span class="n">label</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;vad&quot;</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">h5f</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;vad&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">value</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;bool&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()[</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="n">feat</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">&#39;bool&#39;</span><span class="p">)</span>
<span class="c1"># Pad the segment if needed</span>
<span class="n">feat</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">feat</span><span class="p">,</span> <span class="p">((</span><span class="n">pad_begining</span><span class="p">,</span> <span class="n">pad_end</span><span class="p">),</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;edge&#39;</span><span class="p">)</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="p">(</span><span class="n">pad_begining</span><span class="p">,</span> <span class="n">pad_end</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;edge&#39;</span><span class="p">)</span>
<span class="n">stop</span> <span class="o">+=</span> <span class="n">pad_begining</span> <span class="o">+</span> <span class="n">pad_end</span>
<span class="n">h5f</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="n">feat</span><span class="p">,</span> <span class="n">label</span><span class="p">,</span> <span class="n">global_mean</span><span class="p">,</span> <span class="n">global_std</span><span class="p">,</span> <span class="n">global_cmvn</span> <span class="o">=</span> <span class="n">read_hdf5_segment</span><span class="p">(</span><span class="n">h5f</span><span class="p">,</span>
<span class="n">show</span><span class="p">,</span>
<span class="n">dataset_list</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset_list</span><span class="p">,</span>
<span class="n">label</span><span class="o">=</span><span class="n">label</span><span class="p">,</span>
<span class="n">start</span><span class="o">=</span><span class="n">start</span><span class="p">,</span> <span class="n">stop</span><span class="o">=</span><span class="n">stop</span><span class="p">,</span>
<span class="n">global_cmvn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">global_cmvn</span><span class="p">)</span>
<span class="c1"># Post-process the features and return the features and vad label</span>
<span class="k">if</span> <span class="n">global_cmvn</span><span class="p">:</span>
<span class="n">feat</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">post_processing</span><span class="p">(</span><span class="n">feat</span><span class="p">,</span> <span class="n">label</span><span class="p">,</span> <span class="n">global_mean</span><span class="p">,</span> <span class="n">global_std</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feat</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">post_processing</span><span class="p">(</span><span class="n">feat</span><span class="p">,</span> <span class="n">label</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mask</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">feat</span> <span class="o">=</span> <span class="n">feat</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">mask</span><span class="p">]</span>
<span class="k">return</span> <span class="n">feat</span><span class="p">,</span> <span class="n">label</span></div>
......
docs/build/html/_modules/frontend/io.html
View file @ 3859800a
......@@ -596,7 +596,12 @@
<span class="k">return</span> <span class="n">features</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">numpy</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span></div>
<div class="viewcode-block" id="read_hdf5_segment"><a class="viewcode-back" href="../../api/frontend/io.html#frontend.io.read_hdf5_segment">[docs]</a><span class="k">def</span> <span class="nf">read_hdf5_segment</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="n">dataset</span><span class="p">,</span> <span class="n">mask</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">):</span>
<div class="viewcode-block" id="read_hdf5_segment"><a class="viewcode-back" href="../../api/frontend/io.html#frontend.io.read_hdf5_segment">[docs]</a><span class="k">def</span> <span class="nf">read_hdf5_segment</span><span class="p">(</span><span class="n">file_handler</span><span class="p">,</span>
<span class="n">show</span><span class="p">,</span>
<span class="n">dataset_list</span><span class="p">,</span>
<span class="n">label</span><span class="p">,</span>
<span class="n">start</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">stop</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">global_cmvn</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Read a segment from a stream in HDF5 format. Return the features in the</span>
<span class="sd"> range start:end</span>
<span class="sd"> In case the start and end cannot be reached, the first or last feature are copied</span>
......@@ -610,24 +615,74 @@
<span class="sd"> :return:read_hdf5_segment</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">with</span> <span class="n">h5py</span><span class="o">.</span><span class="n">File</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fh</span><span class="p">:</span>
<span class="n">n_frames</span><span class="p">,</span> <span class="n">feat_size</span> <span class="o">=</span> <span class="n">fh</span><span class="p">[</span><span class="n">dataset</span><span class="p">]</span><span class="o">.</span><span class="n">shape</span>
<span class="n">h5f</span> <span class="o">=</span> <span class="n">file_handler</span>
<span class="n">compressed</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">dataset</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="o">+</span> <span class="s2">&quot;/comp&quot;</span> <span class="ow">in</span> <span class="n">fh</span><span class="p">:</span>
<span class="n">compressed</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">compression_type</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">&#39;none&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">&#39;htk&#39;</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">&#39;percentile&#39;</span><span class="p">}</span>
<span class="k">if</span> <span class="s2">&quot;compression&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="n">compression</span> <span class="o">=</span> <span class="s1">&#39;none&#39;</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Warning, default feature storage mode is now using compression&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">compression</span> <span class="o">=</span> <span class="n">compression_type</span><span class="p">[</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;compression&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">value</span><span class="p">]</span>
<span class="c1"># Check that the segment is within the range of the file</span>
<span class="n">s</span><span class="p">,</span> <span class="n">e</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">start</span><span class="p">),</span> <span class="nb">min</span><span class="p">(</span><span class="n">n_frames</span><span class="p">,</span> <span class="n">end</span><span class="p">)</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="p">(</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">)</span> <span class="o">=</span> <span class="n">fh</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">dataset</span> <span class="o">+</span> <span class="s2">&quot;_comp&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span>
<span class="n">features</span> <span class="o">=</span> <span class="p">(</span><span class="n">fh</span><span class="p">[</span><span class="n">dataset</span><span class="p">][</span><span class="n">s</span><span class="p">:</span><span class="n">e</span><span class="p">,</span> <span class="n">mask</span><span class="p">]</span><span class="o">-</span><span class="n">B</span><span class="p">)</span><span class="o">/</span><span class="n">A</span>
<span class="k">if</span> <span class="n">show</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;show </span><span class="si">{}</span><span class="s1"> is not in the HDF5 file&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">show</span><span class="p">))</span>
<span class="c1"># Get the selected segment</span>
<span class="n">dataset_length</span> <span class="o">=</span> <span class="n">h5f</span><span class="p">[</span><span class="n">show</span> <span class="o">+</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="nb">next</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="n">show</span><span class="p">]</span><span class="o">.</span><span class="fm">__iter__</span><span class="p">())]</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="c1"># Deal with the case where start &lt; 0 or stop &gt; feat.shape[0]</span>
<span class="k">if</span> <span class="n">start</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">start</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">pad_begining</span> <span class="o">=</span> <span class="o">-</span><span class="n">start</span> <span class="k">if</span> <span class="n">start</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="k">else</span> <span class="mi">0</span>
<span class="n">start</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">start</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="n">stop</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">stop</span> <span class="o">=</span> <span class="n">dataset_length</span>
<span class="n">pad_end</span> <span class="o">=</span> <span class="n">stop</span> <span class="o">-</span> <span class="n">dataset_length</span> <span class="k">if</span> <span class="n">stop</span> <span class="o">&gt;</span> <span class="n">dataset_length</span> <span class="k">else</span> <span class="mi">0</span>
<span class="n">stop</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">stop</span><span class="p">,</span> <span class="n">dataset_length</span><span class="p">)</span>
<span class="n">global_cmvn</span> <span class="o">=</span> <span class="n">global_cmvn</span> <span class="ow">and</span> <span class="ow">not</span> <span class="p">(</span><span class="n">start</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">stop</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span>
<span class="c1"># Get the data between start and stop</span>
<span class="c1"># Concatenate all required datasets</span>
<span class="n">feat</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">global_mean</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">global_std</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">feat</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">data_id</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;energy&#39;</span><span class="p">,</span> <span class="s1">&#39;cep&#39;</span><span class="p">,</span> <span class="s1">&#39;fb&#39;</span><span class="p">,</span> <span class="s1">&#39;bnf&#39;</span><span class="p">]:</span>
<span class="k">if</span> <span class="n">data_id</span> <span class="ow">in</span> <span class="n">dataset_list</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="n">data_id</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="n">dataset_id</span> <span class="o">=</span> <span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">compression</span> <span class="o">==</span> <span class="s1">&#39;none&#39;</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">_read_segment</span><span class="p">(</span><span class="n">h5f</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">stop</span><span class="p">)</span>
<span class="k">if</span> <span class="n">data</span><span class="o">.</span><span class="n">ndim</span> <span class="o">==</span><span class="mi">1</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="p">[:,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">compression</span> <span class="o">==</span> <span class="s1">&#39;htk&#39;</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_read_segment_htk</span><span class="p">(</span><span class="n">h5f</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">stop</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">features</span> <span class="o">=</span> <span class="n">fh</span><span class="p">[</span><span class="n">dataset</span><span class="p">][</span><span class="n">s</span><span class="p">:</span><span class="n">e</span><span class="p">,</span> <span class="n">mask</span><span class="p">]</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">end</span> <span class="o">&gt;</span> <span class="n">n_frames</span><span class="p">:</span> <span class="c1"># repeat first or/and last frame as required</span>
<span class="n">features</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">r_</span><span class="p">[</span><span class="n">numpy</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">features</span><span class="p">[[</span><span class="mi">0</span><span class="p">]],</span> <span class="n">s</span><span class="o">-</span><span class="n">start</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span>
<span class="n">features</span><span class="p">,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">features</span><span class="p">[[</span><span class="o">-</span><span class="mi">1</span><span class="p">]],</span> <span class="n">end</span><span class="o">-</span><span class="n">e</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)]</span>
<span class="k">return</span> <span class="n">features</span></div>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_read_segment_percentile</span><span class="p">(</span><span class="n">h5f</span><span class="p">,</span> <span class="n">dataset_id</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">stop</span><span class="p">))</span>
<span class="n">global_mean</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">_mean&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data_id</span><span class="p">)))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="n">global_std</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">_std&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data_id</span><span class="p">)))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> is not in the HDF5 file&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">data_id</span><span class="p">))</span>
<span class="n">feat</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">feat</span><span class="p">)</span>
<span class="n">global_mean</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">global_mean</span><span class="p">)</span>
<span class="n">global_std</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">global_std</span><span class="p">)</span>
<span class="k">if</span> <span class="n">label</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;vad&quot;</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">h5f</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;vad&quot;</span><span class="p">)))</span><span class="o">.</span><span class="n">value</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;bool&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()[</span><span class="n">start</span><span class="p">:</span><span class="n">stop</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="n">feat</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">&#39;bool&#39;</span><span class="p">)</span>
<span class="c1"># Pad the segment if needed</span>
<span class="n">feat</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">feat</span><span class="p">,</span> <span class="p">((</span><span class="n">pad_begining</span><span class="p">,</span> <span class="n">pad_end</span><span class="p">),</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">)),</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;edge&#39;</span><span class="p">)</span>
<span class="n">label</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">label</span><span class="p">,</span> <span class="p">(</span><span class="n">pad_begining</span><span class="p">,</span> <span class="n">pad_end</span><span class="p">),</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;edge&#39;</span><span class="p">)</span>
<span class="c1">#stop += pad_begining + pad_end</span>
<span class="k">return</span> <span class="n">feat</span><span class="p">,</span> <span class="n">label</span><span class="p">,</span> <span class="n">global_mean</span><span class="p">,</span> <span class="n">global_std</span><span class="p">,</span> <span class="n">global_cmvn</span></div>
<div class="viewcode-block" id="read_spro4_segment"><a class="viewcode-back" href="../../api/frontend/io.html#frontend.io.read_spro4_segment">[docs]</a><span class="k">def</span> <span class="nf">read_spro4_segment</span><span class="p">(</span><span class="n">input_file_name</span><span class="p">,</span> <span class="n">start</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">end</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
......@@ -765,212 +820,6 @@
<span class="n">features</span> <span class="o">=</span> <span class="n">features</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;&gt;f&#39;</span><span class="p">)</span>
<span class="n">features</span><span class="o">.</span><span class="n">tofile</span><span class="p">(</span><span class="n">fh</span><span class="p">)</span>
<div class="viewcode-block" id="write_hdf5"><a class="viewcode-back" href="../../api/frontend/io.html#frontend.io.write_hdf5">[docs]</a><span class="k">def</span> <span class="nf">write_hdf5</span><span class="p">(</span><span class="n">show</span><span class="p">,</span>
<span class="n">fh</span><span class="p">,</span>
<span class="n">cep</span><span class="p">,</span> <span class="n">cep_mean</span><span class="p">,</span> <span class="n">cep_std</span><span class="p">,</span>
<span class="n">energy</span><span class="p">,</span> <span class="n">energy_mean</span><span class="p">,</span> <span class="n">energy_std</span><span class="p">,</span>
<span class="n">fb</span><span class="p">,</span> <span class="n">fb_mean</span><span class="p">,</span> <span class="n">fb_std</span><span class="p">,</span>
<span class="n">bnf</span><span class="p">,</span> <span class="n">bnf_mean</span><span class="p">,</span> <span class="n">bnf_std</span><span class="p">,</span>
<span class="n">label</span><span class="p">,</span>
<span class="n">compressed</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> :param show: identifier of the show to write</span>
<span class="sd"> :param fh: HDF5 file handler</span>
<span class="sd"> :param cep: cepstral coefficients to store</span>
<span class="sd"> :param cep_mean: pre-computed mean of the cepstral coefficient</span>
<span class="sd"> :param cep_std: pre-computed standard deviation of the cepstral coefficient</span>
<span class="sd"> :param energy: energy coefficients to store</span>
<span class="sd"> :param energy_mean: pre-computed mean of the energy</span>
<span class="sd"> :param energy_std: pre-computed standard deviation of the energy</span>
<span class="sd"> :param fb: filter-banks coefficients to store</span>
<span class="sd"> :param fb_mean: pre-computed mean of the filter bank coefficient</span>
<span class="sd"> :param fb_std: pre-computed standard deviation of the filter bank coefficient</span>
<span class="sd"> :param bnf: bottle-neck features to store</span>
<span class="sd"> :param bnf_mean: pre-computed mean of the bottleneck features</span>
<span class="sd"> :param bnf_std: pre-computed standard deviation of the bottleneck features</span>
<span class="sd"> :param label: vad labels to store</span>
<span class="sd"> :param compressed: boolean, default is False</span>
<span class="sd"> :return:</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/comp&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cep</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="n">A_cep</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">cep</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">cep</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">B_cep</span> <span class="o">=</span> <span class="p">(</span><span class="n">cep</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">+</span> <span class="n">cep</span><span class="o">.</span><span class="n">min</span><span class="p">())</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">cep</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">cep</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/cep_comp&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">A_cep</span><span class="p">,</span> <span class="n">B_cep</span><span class="p">])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/cep&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">(</span><span class="n">A_cep</span><span class="o">*</span><span class="n">cep</span> <span class="o">-</span> <span class="n">B_cep</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;short&quot;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/cep&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">cep</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cep_mean</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/cep_mean&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">cep_mean</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">cep_std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/cep_std&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">cep_std</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">energy</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="n">A_energy</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">energy</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">energy</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">B_energy</span> <span class="o">=</span> <span class="p">(</span><span class="n">energy</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">+</span> <span class="n">energy</span><span class="o">.</span><span class="n">min</span><span class="p">())</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">energy</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">energy</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/energy_comp&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">A_energy</span><span class="p">,</span> <span class="n">B_energy</span><span class="p">])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/energy&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">(</span><span class="n">A_energy</span> <span class="o">*</span> <span class="n">energy</span> <span class="o">-</span> <span class="n">B_energy</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;short&quot;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/energy&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">energy</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">energy_mean</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/energy_mean&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">energy_mean</span><span class="p">)</span>
<span class="k">if</span> <span class="n">energy_std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/energy_std&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">energy_std</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fb</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="n">A_fb</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">fb</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">fb</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">B_fb</span> <span class="o">=</span> <span class="p">(</span><span class="n">fb</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">+</span> <span class="n">fb</span><span class="o">.</span><span class="n">min</span><span class="p">())</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">fb</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">fb</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/fb_comp&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">A_fb</span><span class="p">,</span> <span class="n">B_fb</span><span class="p">])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/fb&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">(</span><span class="n">A_fb</span> <span class="o">*</span> <span class="n">fb</span> <span class="o">-</span> <span class="n">B_fb</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;short&quot;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/fb&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">fb</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fb_mean</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/fb_mean&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">fb_mean</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">fb_std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/fb_std&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">fb_std</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">bnf</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="n">A_bnf</span> <span class="o">=</span> <span class="mi">2</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">bnf</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">bnf</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">B_bnf</span> <span class="o">=</span> <span class="p">(</span><span class="n">bnf</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">+</span> <span class="n">bnf</span><span class="o">.</span><span class="n">min</span><span class="p">())</span> <span class="o">*</span> <span class="mf">32767.</span> <span class="o">/</span> <span class="p">(</span><span class="n">bnf</span><span class="o">.</span><span class="n">max</span><span class="p">()</span> <span class="o">-</span> <span class="n">bnf</span><span class="o">.</span><span class="n">min</span><span class="p">())</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/bnf_comp&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">numpy</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">A_bnf</span><span class="p">,</span> <span class="n">B_bnf</span><span class="p">])</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/bnf&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">(</span><span class="n">A_bnf</span> <span class="o">*</span> <span class="n">bnf</span> <span class="o">-</span> <span class="n">B_bnf</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;short&quot;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/bnf&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">bnf</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">bnf_mean</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/bnf_mean&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">bnf_mean</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">bnf_std</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/bnf_std&#39;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">bnf_std</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;float32&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="n">label</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">show</span> <span class="o">+</span> <span class="s2">&quot;/vad&quot;</span> <span class="ow">in</span> <span class="n">fh</span><span class="p">:</span>
<span class="n">fh</span><span class="o">.</span><span class="n">create_dataset</span><span class="p">(</span><span class="n">show</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span> <span class="o">+</span> <span class="s2">&quot;vad&quot;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">label</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;int8&#39;</span><span class="p">),</span>
<span class="n">maxshape</span><span class="o">=</span><span class="p">(</span><span class="kc">None</span><span class="p">,),</span>
<span class="n">compression</span><span class="o">=</span><span class="s2">&quot;gzip&quot;</span><span class="p">,</span>
<span class="n">fletcher32</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span></div>
<div class="viewcode-block" id="read_hdf5"><a class="viewcode-back" href="../../api/frontend/io.html#frontend.io.read_hdf5">[docs]</a><span class="k">def</span> <span class="nf">read_hdf5</span><span class="p">(</span><span class="n">h5f</span><span class="p">,</span> <span class="n">show</span><span class="p">,</span> <span class="n">dataset_list</span><span class="o">=</span><span class="p">(</span><span class="s2">&quot;cep&quot;</span><span class="p">,</span> <span class="s2">&quot;fb&quot;</span><span class="p">,</span> <span class="s2">&quot;energy&quot;</span><span class="p">,</span> <span class="s2">&quot;vad&quot;</span><span class="p">,</span> <span class="s2">&quot;bnf&quot;</span><span class="p">)):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> :param h5f: HDF5 file handler to read from</span>
<span class="sd"> :param show: identifier of the show to read</span>
<span class="sd"> :param dataset_list: list of datasets to read and concatenate</span>
<span class="sd"> :return:</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">show</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;show </span><span class="si">{}</span><span class="s1"> is not in the HDF5 file&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">show</span><span class="p">))</span>
<span class="n">compressed</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">show</span> <span class="o">+</span> <span class="s2">&quot;/comp&quot;</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="n">compressed</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">feat</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="s2">&quot;energy&quot;</span> <span class="ow">in</span> <span class="n">dataset_list</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy&quot;</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="p">(</span><span class="n">A_energy</span><span class="p">,</span> <span class="n">B_energy</span><span class="p">)</span> <span class="o">=</span> <span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy_comp&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span> <span class="o">+</span> <span class="n">B_energy</span><span class="p">)</span> <span class="o">/</span> <span class="n">A_energy</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;energy&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">[:,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">newaxis</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;energy is not in the HDF5 file&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;cep&quot;</span> <span class="ow">in</span> <span class="n">dataset_list</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep&quot;</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="p">(</span><span class="n">A_cep</span><span class="p">,</span> <span class="n">B_cep</span><span class="p">)</span> <span class="o">=</span> <span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep_comp&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span> <span class="o">+</span> <span class="n">B_cep</span><span class="p">)</span><span class="o">/</span><span class="n">A_cep</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;cep&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;cep is not in the HDF5 file&#39;</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;fb&quot;</span> <span class="ow">in</span> <span class="n">dataset_list</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb&quot;</span><span class="p">))</span> <span class="ow">in</span> <span class="n">h5f</span><span class="p">:</span>
<span class="k">if</span> <span class="n">compressed</span><span class="p">:</span>
<span class="p">(</span><span class="n">A_fb</span><span class="p">,</span> <span class="n">B_fb</span><span class="p">)</span> <span class="o">=</span> <span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb_comp&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span> <span class="o">+</span> <span class="n">B_fb</span><span class="p">)</span> <span class="o">/</span> <span class="n">A_fb</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">feat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">h5f</span><span class="p">[</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="n">show</span><span class="p">,</span> <span class="s2">&quot;fb&quot;</span><span class="p">))]</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s1">&#39;fb is not in the HDF5 file&#39;</span><span class=<