Files modification

egs/iemocap/config/custom/Iemocap.yaml

@@ -4,7 +4,7 @@
 # General options
 data_path: /
 data_file_extension: .wav
-dataset_csv: list/iemocap_ses1-test.csv
+dataset_csv: list/iemocap_ses3-test.csv
 
 sample_rate: 16000
 
@@ -13,7 +13,7 @@ batch_size: 4
 
 # Training set
 train:
-  duration: 3.
+  duration: -1 #3.
   chunk_per_segment: -1
   overlap: 3.
 
@@ -38,7 +38,7 @@ train:
 
 # Validation set
 valid:
-  duration: 3.
+  duration: -1 #3.
 
   transformation:
     pipeline:     # no transformation

egs/iemocap/config/custom/model.py

@@ -9,15 +9,37 @@ import sidekit.nnet
 
 def build():
     #  You can also inherit nn.sidekit.nnet.Xtractor directly (change model_archi)
-    """
+    
     class Net(sidekit.nnet.Xtractor):
         def __init__(self, speaker_number, loss=None, embedding_size=256):
-            super().__init__(speaker_number, model_archi="xvector", loss=loss, embedding_size=embedding_size)
+            super().__init__(speaker_number, model_archi="halfresnet34", loss="aam", embedding_size=embedding_size)
             # add additional logic here
+            self.param_device_detection = nn.Parameter(torch.empty(0)) # Empty parameter used to detect model device location
+
 
         def forward(self, x, target=None, norm_embedding=True):
-            return super().forward(x, target, norm_embedding)
+            #return super().forward(x, target, norm_embedding)
             # add additional logic here
+            x = x["speech"].to(self.param_device_detection.device)
+            x = x.squeeze(1)
+            x = self.preprocessor(x)
+            x = self.sequence_network(x)
+            x = self.stat_pooling(x)
+            x = self.before_speaker_embedding(x)
+
+            if norm_embedding:
+                x = F.normalize(x, dim=1)
+
+                speaker_loss, s_layer = self.after_speaker_embedding(x, target=target)
+                return (speaker_loss, s_layer), x
+
+                e_layer = self.after_speaker_embedding_emotion(x)
+                emotion_loss = torch.tensor(torch.nan)
+                if "emotion" in args:
+                    emotion_loss = self.after_speaker_embedding_emotion_loss(
+                                e_layer, args["emotion"]
+                                    )
+                return (emotion_loss, e_layer), x
 
     return Net
     """
@@ -103,7 +125,7 @@ def build():
             return self.optimizer
 
         def forward(self, args, target=None, norm_embedding=True):
-            """
+            """"""
             The forward mothod MUST take 3 arguemnts
             The forward mothod MUST return 2 values:
                - a tuple of: (loss: to train the model, in testing (target==None) you should return torch.tensor(torch.nan).
@@ -111,7 +133,7 @@ def build():
                  - In this example the returned value handled by: ArcMarginProduct
                - the x-vector embedding
                i.e., (loss, cce), x_vector = model([...])
-            """
+            """"""
             x = args["speech"]
             x = x.squeeze(1)
             x = self.preprocessor(x)
@@ -227,7 +249,7 @@ def build():
         def new_epoch_hook(self, current_epoch, total_epoch):
             pass
             # example of modifying the optimizer / freezing some layers depending on the epoch
-            """
+            """"""
             self.optimizer.param_groups[0]["lr"] = self.optimizer_option["lr"]
             if current_epoch < total_epoch * 0.40:
                 self.optimizer.param_groups[0]["lr"] = self.optimizer_option["lr"] / 2
@@ -237,8 +259,7 @@ def build():
                 if name.startswith("sequence_network.conv4"):
                     switch_require_grad = True
                 param.requires_grad = switch_require_grad
-            """
-
+            """"""
         @torch.no_grad()
         def validate_model(self):
             # fmt: off
@@ -251,7 +272,7 @@ def build():
             assert x_vector.shape[1] == 256
 
     return Net
-
+    """
 
 # DATA loading hook to add your own data/target
 # used by sidekit internaly
@@ -261,6 +282,7 @@ def get_data_loading_hook(sessions):
 
     # This hook is exectued during dataloading (Done by the CPU in parallel)
     def _hook(speech, csv_line, file_ext):
+        """
 
         if speech.ndim == 1:
             speech = speech.unsqueeze(0)
@@ -274,9 +296,36 @@ def get_data_loading_hook(sessions):
         args["F0"] = torch.rand((1, speech.size(1) // 320))  # fake F0 extractor
 
         # Fake emotion anontation
-        n_emo = 5
-        indice = torch.randint(0, 5, size=(1,))[0]  # (Either 0,1,2,3,4)
+        n_emo = 4
+        indice = torch.randint(0, 4, size=(1,))[0]  # (Either 0,1,2,3)
         args["emotion"] = indice  # fake emotion anontation
-        return args
+        """
+        args = {}                                                               
+        args["speech"] = speech # transformed wav                               
+        args["F0"] = torch.rand((1, speech.size(1) // 320))  # fake F0 extractor
+        args["emotion"] = csv_line["speaker_idx"]                               
+        args["file"] = csv_line["file_id"] + file_ext                           
+        return args                                                             
 
     return _hook
+
+# Custom data collate for padding with zeroes
+# when the whole audio file is considered
+def collate_hook(batch):
+    data_speech_list, data_f0_list, data_emotion_list, target_spk_list = [], [], [], []
+
+    # Extract data from batch
+    for data, target in batch:
+        data_speech_list.append(data["speech"].squeeze(0))
+        data_f0_list.append(data["F0"].squeeze(0))
+        data_emotion_list.append(data["emotion"])
+        target_spk_list.append(target)
+
+    # Pad tensors lists if required and construct output data
+    out_speech = nn.utils.rnn.pad_sequence(data_speech_list, batch_first=True, padding_value=0.0)
+    out_f0 = nn.utils.rnn.pad_sequence(data_f0_list, batch_first=True, padding_value=0.0)
+    out_data_dict = {"speech": out_speech.unsqueeze(1), "F0": out_f0.unsqueeze(1), "emotion": torch.tensor(data_emotion_list)}
+    out_target = torch.tensor(target_spk_list)
+
+    return out_data_dict, out_target
+

egs/iemocap/config/custom/model.yaml

@@ -9,6 +9,8 @@ loss:
 
 # Warning, this hook is experimental, it is broking some other scripts (extract_xvectors.py, scoring..)
 data_loading_hook: ./config/custom/model.py
+# # Hook to use a custom collate when selected duration is -1
+collate_hook: ./config/custom/model.py
 
 # Initialize model from file, reset and freeze parts of it
 initial_model_name:

egs/iemocap/config/custom/model_original.py

+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from collections import OrderedDict
+
+import sidekit.nnet
+
+
+def build():
+    #  You can also inherit nn.sidekit.nnet.Xtractor directly (change model_archi)
+    
+    class Net(sidekit.nnet.Xtractor):
+        def __init__(self, speaker_number, loss=None, embedding_size=256):
+            super().__init__(speaker_number, model_archi="halfresnet34", loss="aam", embedding_size=embedding_size)
+            # add additional logic here
+            self.param_device_detection = nn.Parameter(torch.empty(0)) # Empty parameter used to detect model device location
+
+
+        def forward(self, x, target=None, norm_embedding=True):
+            return super().forward(x, target, norm_embedding)
+            # add additional logic here
+
+    return Net
+    """
+
+    # Define your model, you can use building blocks from sidekit.nnet
+    class Net(nn.Module):
+        def __init__(self, speaker_number, loss=None, embedding_size=256):
+            # You can change the parameters value by changing the 'config/custom/model.yaml' config
+            super().__init__()
+
+            if loss not in ["aam"]:
+                raise NotImplementedError(f"Loss not implemented")
+
+            self.preprocessor = sidekit.nnet.MfccFrontEnd()
+            feature_size = self.preprocessor.n_mfcc
+            self.loss = loss
+            self.speaker_number = speaker_number
+
+            self.sequence_network = nn.Sequential(
+                OrderedDict(
+                    [
+                        ("conv1", nn.Conv1d(feature_size, 512, 5, dilation=1)),
+                        ("activation1", nn.LeakyReLU(0.2)),
+                        ("batch_norm1", nn.BatchNorm1d(512)),
+                        ("conv2", nn.Conv1d(512, 512, 3, dilation=2)),
+                        ("activation2", nn.LeakyReLU(0.2)),
+                        ("batch_norm2", nn.BatchNorm1d(512)),
+                        ("conv3", nn.Conv1d(512, 512, 3, dilation=3)),
+                        ("activation3", nn.LeakyReLU(0.2)),
+                        ("batch_norm3", nn.BatchNorm1d(512)),
+                        ("conv4", nn.Conv1d(512, 512, 1)),
+                        ("activation4", nn.LeakyReLU(0.2)),
+                        ("batch_norm4", nn.BatchNorm1d(512)),
+                        ("conv5", nn.Conv1d(512, 1536, 1)),
+                        ("activation5", nn.LeakyReLU(0.2)),
+                        ("batch_norm5", nn.BatchNorm1d(1536)),
+                    ]
+                )
+            )
+
+            self.embedding_size = embedding_size
+
+            self.stat_pooling = sidekit.nnet.MeanStdPooling()
+            self.before_speaker_embedding = nn.Sequential(
+                OrderedDict([("linear6", nn.Linear(3072, self.embedding_size))])
+            )
+
+            # The final layer computes the loss
+            if self.loss == "aam":
+                self.after_speaker_embedding = sidekit.nnet.ArcMarginProduct(
+                    self.embedding_size,
+                    int(self.speaker_number),
+                    s=30.0,
+                    m=0.2,
+                    easy_margin=False,
+                )
+
+            self.after_speaker_embedding_emotion = nn.Linear(
+                self.embedding_size, 5
+            )  # 5 -> 5 emotions
+            self.after_speaker_embedding_emotion_loss = torch.nn.CrossEntropyLoss()
+
+        def set_lr_weight_decay_layers_for_optim(self, _optimizer, _options):
+            self._optimizer_option = _options
+            self._optimizer = _optimizer
+
+            # fmt: off
+            param_list = []
+            param_list.append({"params": self.preprocessor.parameters(), "weight_decay": 0.0002})
+            param_list.append({"params": self.sequence_network.parameters(), "weight_decay": 0.0002})
+            param_list.append({ "params": self.stat_pooling.parameters(), "weight_decay": 0})
+            param_list.append({ "params": self.before_speaker_embedding.parameters(), "weight_decay": 0.002})
+            param_list.append({ "params": self.after_speaker_embedding.parameters(), "weight_decay": 0.002})
+
+            # EMOTION:  param_list.append({ "params": self.after_speaker_embedding_emotion.parameters(), "weight_decay": 0.002})
+            # fmt: on
+
+            self.optimizer = _optimizer(param_list, **_options)
+
+            # example on applying different LR to different layers
+            #  self.optimizer.param_groups[0]["lr"] = _options["lr"] / 2
+
+            return self.optimizer
+
+        def forward(self, args, target=None, norm_embedding=True):
+            """"""
+            The forward mothod MUST take 3 arguemnts
+            The forward mothod MUST return 2 values:
+               - a tuple of: (loss: to train the model, in testing (target==None) you should return torch.tensor(torch.nan).
+                               cross-entroy prediction: raw output of the network to compute accuracy on
+                 - In this example the returned value handled by: ArcMarginProduct
+               - the x-vector embedding
+               i.e., (loss, cce), x_vector = model([...])
+            """"""
+            x = args["speech"]
+            x = x.squeeze(1)
+            x = self.preprocessor(x)
+            x = self.sequence_network(x)
+            x = self.stat_pooling(x)
+            x = self.before_speaker_embedding(x)
+
+            if norm_embedding:
+                x = F.normalize(x, dim=1)
+
+            speaker_loss, s_layer = self.after_speaker_embedding(x, target=target)
+
+            return (speaker_loss, s_layer), x
+
+            e_layer = self.after_speaker_embedding_emotion(x)
+
+            emotion_loss = torch.tensor(torch.nan)
+            if "emotion" in args:
+                emotion_loss = self.after_speaker_embedding_emotion_loss(
+                    e_layer, args["emotion"]
+                )
+            return (emotion_loss, e_layer), x
+
+            #  possible to add losses together for multitask training i.e.: emotion_loss + speaker_loss[0] * 0.2
+
+        def test(self, model_opts, dataset_opts, training_opts, device="cpu"):
+            # EER computation for the testing dataset
+            # you can tweak this to your own task (emotion reco...)
+
+            enroll_dataset = sidekit.nnet.IdMapSet(
+                idmap_name=dataset_opts["test"]["idmap"],
+                data_path=dataset_opts["test"]["data_path"],
+                file_extension=dataset_opts["data_file_extension"].replace(".", ""),
+                transform_number=0,
+                id_wavs_maps=dataset_opts["test"]["id2wav"],
+                sliding_window=False,
+                hook=get_data_loading_hook(dataset_opts),  # local usage of the hook
+            )
+
+            enroll_dataloader = torch.utils.data.DataLoader(
+                enroll_dataset,
+                batch_size=1,
+                shuffle=False,
+                drop_last=False,
+                pin_memory=True,
+                num_workers=training_opts["num_cpu"],
+            )
+
+            # reverse IdMap
+            ndx = sidekit.bosaris.Ndx(dataset_opts["test"]["ndx"])
+            key = sidekit.bosaris.Key(dataset_opts["test"]["key"])
+            test_idmap = sidekit.bosaris.IdMap()
+            test_idmap.leftids = ndx.segset
+            test_idmap.rightids = ndx.segset
+            test_idmap.start = [None] * len(ndx.segset)
+            test_idmap.stop = [None] * len(ndx.segset)
+
+            test_dataset = sidekit.nnet.IdMapSet(
+                idmap_name=test_idmap,
+                data_path=dataset_opts["test"]["data_path"],
+                file_extension=dataset_opts["data_file_extension"].replace(".", ""),
+                transform_number=0,
+                id_wavs_maps=dataset_opts["test"]["id2wav"],
+                sliding_window=False,
+                hook=get_data_loading_hook(dataset_opts),  # local usage of the hook
+            )
+
+            test_dataloader = torch.utils.data.DataLoader(
+                test_dataset,
+                batch_size=1,
+                shuffle=False,
+                drop_last=False,
+                pin_memory=True,
+                num_workers=training_opts["num_cpu"],
+            )
+
+            def _format(data):
+                data = data["speech"].to(device)
+                return data
+
+            def _pre_forward(x):
+                return {"speech": x}
+
+            enrolls_stat = sidekit.nnet.extract_embeddings_from_dataloader(
+                self,
+                enroll_dataloader,
+                device=device,
+                format=_format,
+                pre_forward=_pre_forward,
+                mixed_precision=training_opts["mixed_precision"],
+            )
+            test_stat = sidekit.nnet.extract_embeddings_from_dataloader(
+                self,
+                test_dataloader,
+                device=device,
+                format=_format,
+                pre_forward=_pre_forward,
+                mixed_precision=training_opts["mixed_precision"],
+            )
+
+            # Compute cosine similarity
+            cosine_scores = sidekit.iv_scoring.cosine_scoring(
+                enrolls_stat, test_stat, ndx, device=device
+            )
+            scores = cosine_scores.scoremat
+            scores = scores[ndx.trialmask]
+            key.tar = key.tar[ndx.trialmask]
+            key.non = key.non[ndx.trialmask]
+            pmiss, pfa = sidekit.bosaris.detplot.rocch(scores[key.tar], scores[key.non])
+            eer = sidekit.bosaris.detplot.rocch2eer(pmiss, pfa)
+            print(f"**Test metrics - Test EER = {eer * 100} %")
+
+        def new_epoch_hook(self, current_epoch, total_epoch):
+            pass
+            # example of modifying the optimizer / freezing some layers depending on the epoch
+            """"""
+            self.optimizer.param_groups[0]["lr"] = self.optimizer_option["lr"]
+            if current_epoch < total_epoch * 0.40:
+                self.optimizer.param_groups[0]["lr"] = self.optimizer_option["lr"] / 2
+
+            switch_require_grad = False
+            for name, param in self.named_parameters():
+                if name.startswith("sequence_network.conv4"):
+                    switch_require_grad = True
+                param.requires_grad = switch_require_grad
+            """"""
+        @torch.no_grad()
+        def validate_model(self):
+            # fmt: off
+            print("Model_parameters_count: {:d}".format( sum( p.numel() for p in self.sequence_network.parameters() if p.requires_grad ) + sum( p.numel() for p in self.before_speaker_embedding.parameters() if p.requires_grad ) + sum( p.numel() for p in self.stat_pooling.parameters() if p.requires_grad ) ))
+            # fmt: on
+
+            batch = torch.rand(16, 32000)
+            indices = torch.randint(0, 5, size=(16,))
+            _, x_vector = self.forward({"speech": batch, "emotion": indices})
+            assert x_vector.shape[1] == 256
+
+    return Net
+    """
+
+# DATA loading hook to add your own data/target
+# used by sidekit internaly
+# you can also call it on your own (i.e.: in the test function)
+def get_data_loading_hook(sessions):
+    #  print(sessions)
+
+    # This hook is exectued during dataloading (Done by the CPU in parallel)
+    def _hook(speech, csv_line, file_ext):
+
+        if speech.ndim == 1:
+            speech = speech.unsqueeze(0)
+        #  print(speech.shape, csv_line, file_ext)
+        # check for test dset with csv_line["dataset"] == "test"
+
+        # Here you can modify what is
+        args = {}
+        args["speech"] = speech
+
+        args["F0"] = torch.rand((1, speech.size(1) // 320))  # fake F0 extractor
+
+        # Fake emotion anontation
+        n_emo = 4
+        indice = torch.randint(0, 4, size=(1,))[0]  # (Either 0,1,2,3)
+        args["emotion"] = indice  # fake emotion anontation
+        return args
+
+    return _hook
+
+# Custom data collate for padding with zeroes
+# when the whole audio file is considered
+def collate_hook(batch):
+    data_speech_list, data_f0_list, data_emotion_list, target_spk_list = [], [], [], []
+
+    # Extract data from batch
+    for data, target in batch:
+        data_speech_list.append(data["speech"].squeeze(0))
+        data_f0_list.append(data["F0"].squeeze(0))
+        data_emotion_list.append(data["emotion"])
+        target_spk_list.append(target)
+
+    # Pad tensors lists if required and construct output data
+    out_speech = nn.utils.rnn.pad_sequence(data_speech_list, batch_first=True, padding_value=0.0)
+    out_f0 = nn.utils.rnn.pad_sequence(data_f0_list, batch_first=True, padding_value=0.0)
+    out_data_dict = {"speech": out_speech.unsqueeze(1), "F0": out_f0.unsqueeze(1), "emotion": torch.tensor(data_emotion_list)}
+    out_target = torch.tensor(target_spk_list)
+
+    return out_data_dict, out_target
+

egs/iemocap/config/custom/modification_yaml.py

@@ -31,11 +31,9 @@ with open("config/custom/model.yaml") as fp:
     data = yaml.load(fp)
 data['speaker_number'] = int(nb_cate)
 
-
 with open("config/custom/model.yaml", 'w') as fp:
     yaml.dump(data, fp)
     
-
 # training.yaml
 with open("config/custom/training.yaml") as fp:
     data = yaml.load(fp)
@@ -57,3 +55,4 @@ data["dataset_csv"] = "list/iemocap_ses{}-test.csv".format(session_test)
 with open("config/custom/Iemocap.yaml", 'w') as fp:
     yaml.dump(data, fp)
 
+print("YAML files modified")

egs/iemocap/config/custom/training.yaml

 # Training description
 
 # General options
-log_file: logs/custom.log
+log_file: logs/half_resnet34_4emo_4batch_lr-0.0001_Test-IEMOCAP3.log
 torch_seed: 42
 numpy_seed: 42
 random_seed: 42
@@ -35,6 +35,6 @@ log_interval: 50
 validation_frequency: 1
 
 # Save options
-tmp_model_name: model_custom/tmp_custom_4emo_4batch_lr-0.0001_Test-IEMOCAP1.pt
-best_model_name: model_custom/best_custom_4emo_4batch_lr-0.0001_Test-IEMOCAP1.pt
+tmp_model_name: model_custom/tmp_custom_4emo_4batch_lr-0.0001_Test-IEMOCAP3.pt
+best_model_name: model_custom/best_custom_4emo_4batch_lr-0.0001_Test-IEMOCAP3.pt
 checkpoint_frequency:

egs/iemocap/config/half_resnet34/Iemocap.yaml

@@ -4,7 +4,7 @@
 # General options
 data_path: / # path to add before each wavs of list/voxceleb2.csv
 data_file_extension: .wav
-dataset_csv: list/iemocap_ses2-test.csv
+dataset_csv: list/iemocap_ses1-test.csv
 
 sample_rate: 16000
 

egs/iemocap/config/half_resnet34/modification_yaml.py

@@ -32,11 +32,9 @@ with open("config/half_resnet34/model.yaml") as fp:
     data = yaml.load(fp)
 data['speaker_number'] = int(nb_cate)
 
-
 with open("config/half_resnet34/model.yaml", 'w') as fp:
     yaml.dump(data, fp)
-    
-
+   
 # training.yaml
 with open("config/half_resnet34/training.yaml") as fp:
     data = yaml.load(fp)
@@ -58,3 +56,4 @@ data["dataset_csv"] = "list/iemocap_ses{}-test.csv".format(session_test)
 with open("config/half_resnet34/Iemocap.yaml", 'w') as fp:
     yaml.dump(data, fp)
 
+print("YAML files modified")

egs/iemocap/config/half_resnet34/training.yaml

 # Training description
 
 # General options
-log_file: logs/half_resnet34_4emo_4batch_lr-0.001_Test-IEMOCAP2.log
+log_file: logs/half_resnet34_4emo_4batch_lr-0.1_Test-IEMOCAP1.log
 torch_seed: 42
 numpy_seed: 42
 random_seed: 42
 deterministic: false
 
 epochs: 100
-lr: 0.001
+lr: 0.1
 patience: 30
 
 multi_gpu: false
@@ -35,6 +35,6 @@ log_interval: 461
 validation_frequency: 1
 
 # Save options
-tmp_model_name: model_half_resnet34/tmp_half_resnet34_4emo_4batch_lr-0.001_Test-IEMOCAP2.pt
-best_model_name: model_half_resnet34/best_half_resnet34_4emo_4batch_lr-0.001_Test-IEMOCAP2.pt
+tmp_model_name: model_half_resnet34/tmp_half_resnet34_4emo_4batch_lr-0.1_Test-IEMOCAP1.pt
+best_model_name: model_half_resnet34/best_half_resnet34_4emo_4batch_lr-0.1_Test-IEMOCAP1.pt
 checkpoint_frequency:

egs/iemocap/config/wavlm_ecapa_circle/Iemocap.yaml

@@ -4,53 +4,53 @@
 # General options
 data_path: / # path to add before each wavs of list/voxceleb2.csv
 data_file_extension: .wav
-dataset_csv: list/voxceleb2.csv
+dataset_csv: list/iemocap_ses1-test.csv
 
 sample_rate: 16000
 
 
 validation_ratio: 0.0
-batch_size: 192
+batch_size: 4
 
 # Training set
 train:
-    duration: 3
-    chunk_per_segment: -1
-    overlap: 3
+  duration: 3
+  chunk_per_segment: -1
+  overlap: 3
 
-    sampler:
-        examples_per_speaker: 1
-        samples_per_speaker: 192
-        augmentation_replica: 1
+  sampler:
+    examples_per_speaker: 1
+    samples_per_speaker: 192
+    augmentation_replica: 1
 
-    transform_number: 1
+  transform_number: 1
 
-    transformation:
-        pipeline: add_reverb,add_noise,phone_filtering,codec
+  transformation:
+    pipeline: add_reverb,add_noise,phone_filtering,codec
 
-        add_noise:
-            noise_db_csv: list/musan.csv
-            data_path: /
+    add_noise:
+      noise_db_csv: list/musan.csv
+      data_path: /
 
-        add_reverb:
-            rir_db_csv: list/reverb.csv
-            data_path: /
+    add_reverb:
+      rir_db_csv: list/reverb.csv
+      data_path: /