debug

nnet/xsets.py

@@ -147,7 +147,7 @@ class SideSampler(torch.utils.data.Sampler):
             self.index_iterator[idx] = self.labels_to_indices[value][self.segment_cursors[value]]
             self.segment_cursors[value] += 1
 
-        self.index_iterator = torch.repeat_interleave(self.index_iterator, self.num_replicas)
+        self.index_iterator = numpy.repeat(self.index_iterator, self.num_replicas)
 
         self.index_iterator = self.index_iterator.reshape(-1, self.num_process * self.examples_per_speaker * self.num_replicas)[:, self.rank * self.examples_per_speaker * self.num_replicas:(self.rank + 1) * self.examples_per_speaker * self.num_replicas].flatten()
 

nnet/xvector.py

@@ -79,7 +79,7 @@ __status__ = "Production"
 __docformat__ = 'reS'
 
 
-def seed_worker():
+def seed_worker(seed_val):
     """
 
     :param worker_id:
@@ -869,7 +869,7 @@ def update_training_dictionary(dataset_description,
     dataset_opts["train"]["sampler"] = dict()
     dataset_opts["train"]["sampler"]["examples_per_speaker"] = 1
     dataset_opts["train"]["sampler"]["samples_per_speaker"] = 100
-    dataset_opts["train"]["sampler"]["augmentation_replicas"] = 1
+    dataset_opts["train"]["sampler"]["augmentation_replica"] = 1
     dataset_opts["train"]["transform_number"] = 2
     dataset_opts["train"]["transformation"] = dict()
     dataset_opts["train"]["transformation"]["pipeline"] = ""
@@ -1072,7 +1072,7 @@ def get_loaders(dataset_opts, training_opts, model_opts, local_rank=0):
                                    seed=training_opts['torch_seed'],
                                    rank=local_rank,
                                    num_process=torch.cuda.device_count(),
-                                   num_replicas=dataset_opts["train"]["sampler"]["augmentation_replicas"]
+                                   num_replicas=dataset_opts["train"]["sampler"]["augmentation_replica"]
                                    )
     else:
         batch_size = dataset_opts["batch_size"]
@@ -1084,11 +1084,11 @@ def get_loaders(dataset_opts, training_opts, model_opts, local_rank=0):
                                    seed=training_opts['torch_seed'],
                                    rank=0,
                                    num_process=torch.cuda.device_count(),
-                                   num_replicas=dataset_opts["train"]["sampler"]["augmentation_replicas"]
+                                   num_replicas=dataset_opts["train"]["sampler"]["augmentation_replica"]
                                    )
 
     training_loader = DataLoader(training_set,
-                                 batch_size=batch_size,
+                                 batch_size=batch_size * dataset_opts["train"]["sampler"]["augmentation_replica"],
                                  shuffle=False,
                                  drop_last=True,
                                  pin_memory=True,
@@ -1118,7 +1118,7 @@ def get_loaders(dataset_opts, training_opts, model_opts, local_rank=0):
     return training_loader, validation_loader, side_sampler, tar_indices, non_indices
 
 
-def get_optimizer(model, model_opts, train_opts):
+def get_optimizer(model, model_opts, train_opts, training_loader):
     """
 
     :param model:
@@ -1336,7 +1336,7 @@ def xtrain(dataset_description,
         monitor.logger.info(f"\t {numpy.sum(validation_non_indices)} non-target trials")
 
     # Create optimizer and scheduler
-    optimizer, scheduler = get_optimizer(model, model_opts, training_opts)
+    optimizer, scheduler = get_optimizer(model, model_opts, training_opts, training_loader)
 
     scaler = None
     if training_opts["mixed_precision"]:
@@ -1435,6 +1435,7 @@ def train_epoch(model,
     running_loss = 0.0
     for batch_idx, (data, target) in enumerate(training_loader):
         data = data.squeeze().to(device)
+
         target = target.squeeze()
         target = target.to(device)
         optimizer.zero_grad(set_to_none=True)