hac_clr.py 7.94 KB
Newer Older
Anthony Larcher's avatar
Anthony Larcher committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# -*- coding: utf-8 -*-
#
# This file is part of s4d.
#
# s4d is a python package for speaker diarization.
# Home page: http://www-lium.univ-lemans.fr/s4d/
#
# s4d is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# s4d is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with s4d.  If not, see <http://www.gnu.org/licenses/>.


"""
Anthony Larcher's avatar
Anthony Larcher committed
23
Copyright 2014-2020 Sylvain Meignier
Anthony Larcher's avatar
Anthony Larcher committed
24
25
"""

Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
26
import copy
Anthony Larcher's avatar
Anthony Larcher committed
27
28
29
30
31
32
import logging
import numpy as np

from bottleneck import argpartition
from ..diar import Diar
from .hac_utils import argmin, roll
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
33
34
from sidekit import Mixture, FeaturesServer
from sidekit.statserver import StatServer
Anthony Larcher's avatar
Anthony Larcher committed
35

Sylvain Meignier's avatar
Sylvain Meignier committed
36

Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
37
38
39
40
class HAC_CLR:
    """
    CLR Hierarchical Agglomerative Clustering (HAC) with GMM trained by MAP
    """
Sylvain Meignier's avatar
Sylvain Meignier committed
41
    def __init__(self, features_server, diar, ubm, ce=False, ntop=5):
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
42
43
44
45
46
47
48
49
50
51
52
53
        assert isinstance(features_server, FeaturesServer), 'First parameter has to be a FeatureServer'
        assert isinstance(diar, Diar), '2sd parameter has to be a Diar (segmentationContener)'
        assert isinstance(ubm, Mixture), '3rd parameter has to be a Mixture'

        self.features_server = features_server
        self.diar = copy.deepcopy(diar)
        self.merge = []
        self.nb_merge = 0
        self.ubm = ubm
        self.ce = ce
        self.stat_speaker = None
        self.stat_seg = None
Sylvain Meignier's avatar
Sylvain Meignier committed
54
        self.llr = None
Sylvain Meignier's avatar
Sylvain Meignier committed
55
        self.ntop = ntop
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
56

57
    def _get_cep(self, map, cluster):
Anthony Larcher's avatar
Anthony Larcher committed
58
59
60
61
62
63
        """

        :param map:
        :param cluster:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
64
        cep_list = list()
65
66
        for show in map[cluster]:
            idx = self.diar.features_by_cluster(show)[cluster]
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
67
68
69
70
71
72
            if len(idx) > 0:
                tmp, vad = self.features_server.load(show)
                cep_list.append(tmp[0][idx])
        cep = np.concatenate(cep_list, axis=0)
        return cep

73
    def _ll(self, ubm, cep, mu=None, name='ubm', argtop = None):
Anthony Larcher's avatar
Anthony Larcher committed
74
75
76
77
78
79
80
81
82
        """

        :param ubm:
        :param cep:
        :param mu:
        :param name:
        :param argtop:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
83
84
        # ajouter le top gaussien
        lp = ubm.compute_log_posterior_probabilities(cep, mu=mu)
Sylvain Meignier's avatar
Sylvain Meignier committed
85
86

        if argtop is None:
Sylvain Meignier's avatar
Sylvain Meignier committed
87
            #logging.info('compute argtop '+speaker)
88
            argtop = argpartition(lp*-1.0 , self.ntop, axis=1)[:, :self.ntop]
Sylvain Meignier's avatar
Sylvain Meignier committed
89
            #logging.info(argtop.shape)
Sylvain Meignier's avatar
Sylvain Meignier committed
90
        if self.ntop is not None:
Sylvain Meignier's avatar
Sylvain Meignier committed
91
            #logging.info('use ntop '+speaker)
Sylvain Meignier's avatar
Sylvain Meignier committed
92
93
94
            #logging.info(argtop.shape)
            #logging.info(lp.shape)
            lp = lp[np.arange(argtop.shape[0])[:, np.newaxis], argtop]
Sylvain Meignier's avatar
Sylvain Meignier committed
95

Sylvain Meignier's avatar
new    
Sylvain Meignier committed
96
        # ppMax = numpy.max(lp, axis=1)
Sylvain Meignier's avatar
Sylvain Meignier committed
97

Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
98
        ll = np.log(np.sum(np.exp(lp), axis=1))
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
99
        # ll = ppMax + numpy.log(numpy.sum(numpy.exp((lp.transpose() - ppMax).transpose()),
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
100
101
102
        #                    axis=1))
        not_finite = np.logical_not(np.isfinite(ll))
        cpt = np.count_nonzero(not_finite)
Sylvain Meignier's avatar
new    
Sylvain Meignier committed
103
        # ll[finite] = numpy.finfo('d').min
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
104
105
106
        ll[not_finite] = 1.0e-200
        m = np.mean(ll)
        if cpt > 0:
107
            logging.info('model ' + name + '), nb trame with llk problem: %d/%d \t %f', cpt, cep.shape[0], m)
Sylvain Meignier's avatar
Sylvain Meignier committed
108
        return m, argtop
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
109
110

    def initial_models(self, nb_threads=1):
Anthony Larcher's avatar
Anthony Larcher committed
111
112
113
114
115
        """

        :param nb_threads:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
116
117
118
119
120
121
122
123
        # sort by show to minimize the reading of mfcc by the statServer
        self.diar.sort(['show'])
        # Compute statistics by segments
        self.stat_seg = StatServer(self.diar.id_map())
        self.stat_seg.accumulate_stat(self.ubm, self.features_server)
        self.stat_speaker = self.stat_seg.adapt_mean_MAP_multisession(self.ubm)

    def initial_distances(self, nb_threads=1):
Anthony Larcher's avatar
Anthony Larcher committed
124
125
126
127
128
        """

        :param nb_threads:
        :return:
        """
129
        map = self.diar.make_index(['cluster', 'show'])
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
130
131
132
133
134
135
        nb = self.stat_speaker.modelset.shape[0]

        self.llr = np.full((nb, nb), np.nan)
        self.dist = np.full((nb, nb), np.nan)
        for i, name_i in enumerate(self.stat_speaker.modelset):
            cep_i = self._get_cep(map, name_i)
Sylvain Meignier's avatar
Sylvain Meignier committed
136
137
138
139
140
            argtop = None
            ll_ubm = None
            if self.ntop is not None or self.ce == False:
                ll_ubm, argtop = self._ll(self.ubm, cep_i, argtop=argtop)

Sylvain Meignier's avatar
Sylvain Meignier committed
141
            # self.merge.append([])
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
142
143
            for j, name_j in enumerate(self.stat_speaker.modelset):
                mu = self.stat_speaker.get_model_stat1_by_index(j)
Sylvain Meignier's avatar
Sylvain Meignier committed
144
                # if i == 0:
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
145
                #    logging.debug(mu)
146
                self.llr[i, j], _ = self._ll(self.ubm, cep_i, mu=mu, name=name_j, argtop=argtop)
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
147
148
149
150
151
            if self.ce:
                self.llr[i,:] -= self.llr[i,i]
            else:
                self.llr[i,:] -= ll_ubm

Sylvain Meignier's avatar
Sylvain Meignier committed
152
        # logging.debug(self.llr)
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
153
154
155
156
        self.dist = (self.llr + self.llr.T)*-1.0
        np.fill_diagonal(self.dist, np.finfo('d').max)

    def update(self, i, j, nb_threads=1):
Anthony Larcher's avatar
Anthony Larcher committed
157
158
159
160
161
162
163
        """

        :param i:
        :param j:
        :param nb_threads:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
164
165
        name_i = self.stat_speaker.modelset[i]
        name_j = self.stat_speaker.modelset[j]
Sylvain Meignier's avatar
Sylvain Meignier committed
166
        # logging.debug('%d %d / %s %s', i, j, name_i, name_j)
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
167
168
169
170
171
172
173
174
175

        for k in range(len(self.stat_seg.modelset)):
            if self.stat_seg.modelset[k] == name_j:
                self.stat_seg.modelset[k] = name_i

        self.stat_speaker = self.stat_seg.adapt_mean_MAP_multisession(self.ubm)

        self.llr = roll(self.llr, j)

176
177
        self.diar.rename('cluster', [name_j], name_i)
        map = self.diar.make_index(['cluster', 'show'])
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
178
        cep_i = self._get_cep(map, name_i)
Sylvain Meignier's avatar
Sylvain Meignier committed
179
180
181
182
        argtop = None
        ll_ubm = None
        if self.ntop > 0 or self.ce == False:
            ll_ubm, argtop = self._ll(self.ubm, cep_i, argtop=argtop)
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
183
184
        for k, name_k in enumerate(self.stat_speaker.modelset):
            mu = self.stat_speaker.get_model_stat1_by_index(k)
185
            self.llr[i, k], _ = self._ll(self.ubm, cep_i, mu=mu, name=name_k)
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
186
187
188
189
190
191
192
193
194
        if self.ce:
            self.llr[i,:] -= self.llr[i,i]
        else:
            self.llr[i,:] -= ll_ubm

        self.dist = (self.llr + self.llr.T)*-1.0
        np.fill_diagonal(self.dist, np.finfo('d').max)

    def information(self, i, j, value):
Anthony Larcher's avatar
Anthony Larcher committed
195
196
197
198
199
200
201
        """

        :param i:
        :param j:
        :param value:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
202
203
204
205
        models = self.stat_speaker.modelset
        self.merge.append([self.nb_merge, models[i], models[j], value])

    def perform(self, thr = 0.0, to_the_end=False):
Anthony Larcher's avatar
Anthony Larcher committed
206
207
208
209
210
211
        """

        :param thr:
        :param to_the_end:
        :return:
        """
Sylvain Meignier's avatar
Origin  
Sylvain Meignier committed
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        models = self.stat_speaker.modelset
        nb = len(models)
        self.nb_merge = -1
        for i in range(nb):
            self.information(i, i, 0)

        i, j, v = argmin(self.dist, nb)
        self.nb_merge = 0
        while v < thr and nb > 1:
            self.information(i ,j, v)
            self.nb_merge += 1
            logging.debug('merge: %d c1: %s (%d) c2: %s (%d) dist: %f',
                          self.nb_merge, models[i], i, models[j], j, v)
            # update merge
            # update model and distance
            self.update(i, j)
            nb -= 1
            i, j, v = argmin(self.dist, nb)

        end_diar = copy.deepcopy(self.diar)
        if to_the_end:
            while nb > 1:
                self.information(i ,j, v)
                self.nb_merge += 1
                logging.debug('merge: %d c1: %s (%d) c2: %s (%d) dist: %f',
                              self.nb_merge, models[i], i, models[j], j, v)
                # update merge
                # update model and distance
                self.update(i, j)
                nb -= 1
                i, j, v = argmin(self.dist, nb)

        return end_diar