Commit 61a72193 authored by Anthony Larcher's avatar Anthony Larcher
Browse files

new cross show

parent 7ec3a6c8
......@@ -36,19 +36,45 @@ from .utils import rename_models
from .utils import concat_statservers
def cross_show(model, model_cfg, previous_iv, previous_diar, within_iv, within_diar, th_x, incremental=True):
def cross_show(model, model_cfg, speaker_dictionary, archive_diar, th_x, human_assisted=False):
"""
:param model:
:param model_cfg:
:param previous_iv:
:param previous_diar:
:param within_iv:
:param within_diar:
:param speaker_dictionary:
:param archive_diar:
:param th_x:
:param incremental:
:param human_assisted:
:return:
"""
"""
1) on va garder dans le dictionnaire un x-vecteur par locuteur-fichier
on crée un dictionnaire key = (speaker_ID, show) et value = x-vecteur
2) On garde aussi un objet Diar avec tous les shows précédents
3) On calcule les scores entre tous les locuteurs du show courant et les locuteurs précédents
4) Pour chaque locuteur courant on prend les tops N locuteurs :
on classe ces couples (old/new, score) du plus proche au plus loin
* Cas SANS HAL:
On filtre pour s'assurer qu'un même locuteur "old" ne peut pas être associé à un même locuteur "courant"
(on ne remet pas en cause la diarization within-show)
Pour chaque locuteur "new" si le "old" est assez proche on renomme "courant" avec le nom du "old"
* Cas AVEC HAL cross show:
"""
#TODO to update
# Make sure model names are not overlapping (change within show models if needed)
within_iv, within_diar = rename_models(within_iv, within_diar, existing_models=previous_iv.modelset)
......@@ -158,40 +184,115 @@ def cross_show(model, model_cfg, previous_iv, previous_diar, within_iv, within_d
return new_previous_iv, new_previous_diar, within_diar
def allies_cross_show_clustering(show_idx, model, model_cfg, current_diar, current_vec, th_x, adapt=False):
def allies_cross_show_clustering(show,
model,
model_cfg,
speaker_dictionary,
archive_diar,
current_diar,
current_vec_per_speaker,
th_x,
human_assisted=False):
"""
:param show_idx:
:param model:
:param model_cfg:
:param current_diar:
:param current_vec:
:param speaker_dictionary:
:param archive_diar:
:param th_x:
:param adapt:
:param human_assisted:
:return:
"""
if show_idx == 0:
model["previous_vec"] = copy.deepcopy(current_vec)
model["previous_diar"] = current_diar
"""
1) on va garder dans le dictionnaire un x-vecteur par locuteur-fichier
on crée un dictionnaire key = (speaker_ID, show) et value = x-vecteur
2) On garde aussi un objet Diar avec tous les shows précédents
3) On calcule les scores entre tous les locuteurs du show courant et les locuteurs précédents
4) Pour chaque locuteur courant on prend les tops N locuteurs :
on classe ces couples (old/new, score) du plus proche au plus loin
* Cas SANS HAL:
On filtre pour s'assurer qu'un même locuteur "old" ne peut pas être associé à un même locuteur "courant"
(on ne remet pas en cause la diarization within-show)
Pour chaque locuteur "new" si le "old" est assez proche on renomme "courant" avec le nom du "old"
* Cas AVEC HAL cross show:
"""
# Process the first file
if not speaker_dictionary:
# Initialize with the current diar
archive_diar.append_diar(current_diar)
# Add all speakers from the current file in the dictionary
for idx, spk in enumerate(current_vec_per_speaker.modelset):
speaker_dictionary[(show, spk)] = current_vec_per_speaker.stat1[idx, :]
# Process other files
else:
# previous_diar.segments += within_diar.segments
# Apply cross-show clustering
previous_vec, previous_diar, current_diar = cross_show(model=model,
model_cfg=model_cfg,
previous_iv=model["previous_vec"],
previous_diar=model["previous_diar"],
within_iv=current_vec,
within_diar=current_diar,
th_x=th_x,
incremental=True)
model["previous_vec"] = previous_vec
model["previous_diar"] = previous_diar
# TODO not implemented yet, we need to include the code from AMBUJ to adapt the X-vector extractor
#if adapt:
# norm_iv = sidekit.StatServer.read('tmp/model/training_xv.h5'.format('ester_repere'))
# model['model_iv'] = adapt_model(model['model_iv'], previous_vec, norm_iv, 0.2)
return model, current_diar
# Get the matrix of archive_speakers x-vectors to compute the scores
spk_ids = []
shows = []
archive_models = []
archive_xv = numpy.empty((len(speaker_dictionary), model_cfg["model"]["vectors"]["size"]))
for idx, (k, v) in enumerate(speaker_dictionary.items):
shows.append(k[0])
spk_ids.append(k[1])
archive_models.append(k[0] + "#" + k[1])
archive_xv[idx, :] = v
archive_models = numpy.array(archive_models)
# Compute the scores (for x-vector and cosine first)
# TODO: the same for i-vectors and PLDA
# Compute scores
ndx = sidekit.Ndx(models=archive_models, testsegs=current_vec_per_speaker.modelset)
scores = sidekit.iv_scoring.cosine_scoring(archive_xv, current_vec_per_speaker, ndx,
wccn=None,
check_missing=False)
clustering_dict = dict()
#############
# In case speakers from the past have been seen in several files, reduce the matrix to get a unique
# line per past-speaker
unique_past_speakers = numpy.array(set(spk_ids))
unique_past_speakers_scores = numpy.empty((len(unique_past_speakers), scores.scoremat.shape[1]))
for idx, spk in enumerate(unique_past_speakers):
unique_past_speakers_scores[idx, :] = scores.scoremat[numpy.array(spk_ids) == spk, :].max(axis=0)
for idx, current_spk in enumerate(current_vec_per_speaker.modelset):
if unique_past_speakers_scores[:, idx].max() > th_x and unique_past_speakers_scores[, :].argmax()[0] == idx:
# if the current speaker is the closest of its own closest archived speaker: match!
clustering_dict[current_spk] = spk_ids[unique_past_speakers_scores[:, idx].argmax()[0]]
##############
# Display the number of speaker identified to old ones
print(f"Found {len(clustering_dict)} speakers from the past")
# Modify current_diar according to the clustering
for idx, seg in enumerate(current_diar.segments):
if seg["cluster"] in clustering_dict:
current_diar.segments[idx]["cluster"] = clustering_dict[seg["cluster"]]
# Add the final current_diar to the archive
archive_diar.append_diar(current_diar)
# Add all speakers from the current file in the dictionary
for idx, spk in enumerate(current_vec_per_speaker.modelset):
if spk in clustering_dict:
# Add a speaker that has been seen in the past
speaker_dictionary[(show, clustering_dict[spk])] = current_vec_per_speaker.stat1[idx, :]
else:
# Add a new speaker
speaker_dictionary[(show, spk)] = current_vec_per_speaker.stat1[idx, :]
return speaker_dictionary, archive_diar
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment