model_iv.py 9.22 KB
Newer Older
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
1
2
# -*- coding: utf-8 -*-
#
Anthony Larcher's avatar
Anthony Larcher committed
3
# This file is part of s4d.
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
4
#
Anthony Larcher's avatar
Anthony Larcher committed
5
6
# s4d is a python package for speaker diarization.
# Home page: http://www-lium.univ-lemans.fr/s4d/
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
7
#
Anthony Larcher's avatar
Anthony Larcher committed
8
# s4d is free software: you can redistribute it and/or modify
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
9
10
11
12
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
Anthony Larcher's avatar
Anthony Larcher committed
13
# s4d is distributed in the hope that it will be useful,
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
14
15
16
17
18
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
Anthony Larcher's avatar
Anthony Larcher committed
19
20
21
22
23
24
# along with s4d.  If not, see <http://www.gnu.org/licenses/>.


"""
Copyright 2014-2020 Sylvain Meignier and Anthony Larcher
"""
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
25

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
26
27
import copy
import numpy as np
Sylvain Meignier's avatar
Sylvain Meignier committed
28

Anthony Larcher's avatar
Anthony Larcher committed
29
30
31
32
from sidekit import Mixture, StatServer, FactorAnalyser, Scores, Ndx, PLDA_scoring, cosine_scoring, mahalanobis_scoring, two_covariance_scoring
from sidekit.sidekit_io import *


Sylvain Meignier's avatar
Sylvain Meignier committed
33
class ModelIV:
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
34
    def __init__(self, model_filename=None, nb_thread=1):
Anthony Larcher's avatar
Anthony Larcher committed
35
36
37
38
39
        """

        :param model_filename:
        :param nb_thread:
        """
Sylvain Meignier's avatar
Sylvain Meignier committed
40
41
42
43
44
45
46
47
48
49
50
        self.ubm = None
        self.tv = None
        self.tv_mean = None
        self.tv_sigma = None
        self.sn_mean = None
        self.sn_cov = None
        self.plda_mean = None
        self.plda_f = None
        self.plda_g = None
        self.plda_sigma = None
        self.ivectors = None
Sylvain Meignier's avatar
??    
Sylvain Meignier committed
51
        self.scores = None
Sylvain Meignier's avatar
Sylvain Meignier committed
52

Sylvain Meignier's avatar
new    
Sylvain Meignier committed
53
        self.nb_thread = nb_thread
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
54
55
56
        self.model_filename = model_filename
        if model_filename is not None:
            self._load_model()
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
57

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
58
    def _load_model(self):
Anthony Larcher's avatar
Anthony Larcher committed
59
60
61
62
        """

        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
63
64
65
66
67
        print('load: ', self.model_filename)
        self.ubm = Mixture()
        self.ubm.read(self.model_filename, prefix='ubm/')
        self.tv, self.tv_mean, self.tv_sigma = read_tv_hdf5(self.model_filename)
        self.norm_mean, self.norm_cov = read_norm_hdf5(self.model_filename)
Anthony Larcher's avatar
Anthony Larcher committed
68
        self.plda_mean, self.plda_f, self.plda_g, self.plda_sigma = read_plda_hdf5(self.model_filename)
Sylvain Meignier's avatar
Sylvain Meignier committed
69

Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
70
    def debug_model(self):
Anthony Larcher's avatar
Anthony Larcher committed
71
72
73
74
        """

        :return:
        """
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
75
        print('ubm_mu: ', self.ubm.mu.shape)
Sylvain Meignier's avatar
Sylvain Meignier committed
76
77
78
        print('tv: ', self.tv.shape)
        print('tv mean: ', self.tv_mean.shape)
        print('tv Sigma: ', self.tv_sigma.shape)
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
79
80
        if self.plda_mean is not None:
            print('plda mean: ', self.plda_mean.shape)
Sylvain Meignier's avatar
Sylvain Meignier committed
81
82
83
            print('plda Sigma: ', self.plda_sigma.shape)
            print('plda F: ', self.plda_f.shape)
            print('plda G: ', self.plda_g.shape)
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
84
85
86
87
        if self.sn_mean is not None:
            print('sn_mean: ', self.sn_mean.shape)
            print('sn_cov: ', self.sn_cov.shape)

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
88
    def train(self, feature_server, idmap, normalization=True):
Anthony Larcher's avatar
Anthony Larcher committed
89
90
91
92
93
94
95
        """

        :param feature_server:
        :param idmap:
        :param normalization:
        :return:
        """
96
        stat = StatServer(idmap, distrib_nb=self.ubm.distrib_nb(), feature_size=self.ubm.dim()) 
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
97
98
        stat.accumulate_stat(ubm=self.ubm, feature_server=feature_server, seg_indices=range(stat.segset.shape[0]), num_thread=self.nb_thread)
        stat = stat.sum_stat_per_model()[0]
99
100
101
102
        
        fa = FactorAnalyser(mean=self.tv_mean, Sigma=self.tv_sigma, F=self.tv)
        self.ivectors = fa.extract_ivectors_single(self.ubm, stat)
        
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
103
104
105
        if normalization:
            self.ivectors.spectral_norm_stat1(self.norm_mean[:1], self.norm_cov[:1])

Sylvain Meignier's avatar
Sylvain Meignier committed
106
        return self.ivectors
Anthony Larcher's avatar
Anthony Larcher committed
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

    def train_per_segment(self, feature_server, idmap, normalization=True):
        """

        :param feature_server:
        :param idmap:
        :param normalization:
        :return:
        """
        stat = StatServer(idmap, distrib_nb=self.ubm.distrib_nb(), feature_size=self.ubm.dim())
        stat.accumulate_stat(ubm=self.ubm, feature_server=feature_server, seg_indices=range(stat.segset.shape[0]),
                             num_thread=self.nb_thread)
        fa = FactorAnalyser(mean=self.tv_mean, Sigma=self.tv_sigma, F=self.tv)
        self.ivectors = fa.extract_ivectors_single(self.ubm, stat)

        if normalization:
            self.ivectors.spectral_norm_stat1(self.norm_mean[:1], self.norm_cov[:1])

        return self.ivectors
Sylvain Meignier's avatar
Sylvain Meignier committed
126

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
127
    def score_cosine(self, use_wccn=True):
Anthony Larcher's avatar
Anthony Larcher committed
128
129
130
131
132
        """

        :param use_wccn:
        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
133
134
135
136
137
138
139
140
141
        wccn = None
        if use_wccn:
            wccn = read_key_hdf5(self.model_filename, 'wccn_choleski')
        ndx = Ndx(models=self.ivectors.modelset, testsegs=self.ivectors.modelset)
        self.scores = cosine_scoring(self.ivectors, self.ivectors, ndx, wccn=wccn, check_missing=False)

        return self.scores

    def score_mahalanobis(self, use_covariance=True):
Anthony Larcher's avatar
Anthony Larcher committed
142
143
144
145
146
        """

        :param use_covariance:
        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
147
148
149
150
151
152
153
154
155
156
        if use_covariance:
            m = read_key_hdf5(self.model_filename, 'mahalanobis_matrix')
        else:
            m = numpy.identity(self.tv.shape[2])
        ndx = Ndx(models=self.ivectors.modelset, testsegs=self.ivectors.modelset)
        self.scores = mahalanobis_scoring(self.ivectors, self.ivectors, ndx, m, check_missing=False)

        return self.scores

    def score_two_covariance(self):
Anthony Larcher's avatar
Anthony Larcher committed
157
158
159
160
        """

        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
161
162
163
164
165
166
        W = read_key_hdf5(self.model_filename, 'two_covariance/within_covariance')
        B = read_key_hdf5(self.model_filename, 'two_covariance/between_covariance')
        ndx = Ndx(models=self.ivectors.modelset, testsegs=self.ivectors.modelset)
        self.scores = two_covariance_scoring(self.ivectors, self.ivectors, ndx, W, B, check_missing=False)

        return self.scores
Sylvain Meignier's avatar
Sylvain Meignier committed
167

Sylvain Meignier's avatar
Sylvain Meignier committed
168
    def score_plda(self):
Anthony Larcher's avatar
Anthony Larcher committed
169
170
171
172
        """

        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
173
174
175
176
177
178
179
180
181
182
183
184
185
186
        self.plda_mean, self.plda_f, self.plda_g, self.plda_sigma = read_plda_hdf5(self.file_name)
        ndx = Ndx(models=self.ivectors.modelset, testsegs=self.ivectors.modelset)

        self.scores = PLDA_scoring(self.ivectors, self.ivectors, ndx, self.plda_mean, self.plda_f, self.plda_g, self.plda_sigma, p_known=0.0)

        return self.scores

#    def update(self, i, j):
#        cluster_list = self.diarization.make_index(['cluster'])
#
#        stat_server_merge(self.ivectors, i, j, 1, 1)
#        self.scores = self.score_plda()

    def score_plda_slow(self):
Anthony Larcher's avatar
Anthony Larcher committed
187
188
189
190
        """

        :return:
        """
Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
191
        self.plda_mean, self.plda_f, self.plda_g, self.plda_sigma = read_plda_hdf5(self.model_filename)
Sylvain Meignier's avatar
??    
Sylvain Meignier committed
192
193
        local_ndx = Ndx(models=self.ivectors.modelset, testsegs=self.ivectors.modelset)

Sylvain Meignier's avatar
Sylvain Meignier committed
194
195
196
197
198
199
        enroll_copy = copy.deepcopy(self.ivectors)

        # Center the i-vectors around the PLDA mean
        enroll_copy.center_stat1(self.plda_mean)

        # Compute temporary matrices
Sylvain Meignier's avatar
Sylvain Meignier committed
200
        invSigma = np.linalg.inv(self.plda_sigma)
Sylvain Meignier's avatar
Sylvain Meignier committed
201
        I_iv = np.eye(self.plda_mean.shape[0], dtype='float')
Sylvain Meignier's avatar
Sylvain Meignier committed
202
203
204
205
206
        I_ch = np.eye(self.plda_g.shape[1], dtype='float')
        I_spk = np.eye(self.plda_f.shape[1], dtype='float')
        A = np.linalg.inv(self.plda_g.T.dot(invSigma).dot(self.plda_g) + I_ch)
        B = self.plda_f.T.dot(invSigma).dot(I_iv - self.plda_g.dot(A).dot(self.plda_g.T).dot(invSigma))
        K = B.dot(self.plda_f)
Sylvain Meignier's avatar
Sylvain Meignier committed
207
208
209
210
211
212
213
214
215
216
        K1 = np.linalg.inv(K + I_spk)
        K2 = np.linalg.inv(2 * K + I_spk)

        # Compute the Gaussian distribution constant
        alpha1 = np.linalg.slogdet(K1)[1]
        alpha2 = np.linalg.slogdet(K2)[1]
        constant = alpha2 / 2.0 - alpha1

        # Compute verification scores
        l = enroll_copy.segset.shape[0]
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
217
218
219
220
        scores = Scores()
        scores.scoremat = np.zeros((l, l))
        scores.modelset = enroll_copy.modelset
        scores.segset = enroll_copy.modelset
Sylvain Meignier's avatar
??    
Sylvain Meignier committed
221
        scores.scoremask = local_ndx.trialmask
Sylvain Meignier's avatar
Sylvain Meignier committed
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

        # Project data in the space that maximizes the speaker separability
        enroll_tmp = B.dot(enroll_copy.stat1.T)

        # Compute verification scores
        # Loop on the models
        for model_idx in range(enroll_copy.modelset.shape[0]):

            s2 = enroll_tmp[:, model_idx].dot(K1).dot(enroll_tmp[:, model_idx])

            mod_plus_test_seg = enroll_tmp + np.atleast_2d(enroll_tmp[:, model_idx]).T

            tmp1 = enroll_tmp.T.dot(K1)
            tmp2 = mod_plus_test_seg.T.dot(K2)

Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
237
            for seg_idx in range(model_idx, enroll_copy.segset.shape[0]):
Sylvain Meignier's avatar
Sylvain Meignier committed
238
239
                s1 = tmp1[seg_idx, :].dot(enroll_tmp[:, seg_idx])
                s3 = tmp2[seg_idx, :].dot(mod_plus_test_seg[:, seg_idx])
Sylvain Meignier's avatar
merge    
Sylvain Meignier committed
240
241
242
243
244
                scores.scoremat[model_idx, seg_idx] = (s3 - s1 - s2)/2. + constant
                scores.scoremat[seg_idx, model_idx] = (s3 - s1 - s2)/2. + constant
        self.scores = scores
        return scores

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
245
246
#def plda_scores_from_diar(model_fn, feature_server, diar, idmap, return_model=False):
#    model_iv = ModelIV(model_filename=model_fn, feature_server=feature_server, diarization=diar, idmap=idmap)
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
247

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
248
249
250
251
252
#    model_iv.train()
#    scores = model_iv.score_plda()
#    if return_model:
#        return scores, model_iv
#    return scores
Sylvain Meignier's avatar
Sylvain Meignier committed
253

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
254
255
#def cosine_scores_from_diar(model_fn, feature_server, diar, return_model=False):
#    model_iv = ModelIV(model_filename=model_fn, feature_server=feature_server, diarization=diar)
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
256
257
258
    #if vad:
    #model_iv.vad()

Sylvain Meignier's avatar
stable    
Sylvain Meignier committed
259
260
261
262
#    model_iv.train(normalization=False)
#    scores = model_iv.score_cosine()
#    if return_model:
#        return scores, model_iv
Anthony Larcher's avatar
Anthony Larcher committed
263
#    return scores