Commit 40f96fc8 authored by Anthony Larcher's avatar Anthony Larcher
Browse files

intregrating TensorBoard

parent 0c506bf4
......@@ -958,6 +958,8 @@ class FactorAnalyser:
self.mean = alpha * in_stat_server.get_mean_stat1() + (1-alpha) * out_stat_server.get_mean_stat1()
self.Sigma = alpha * in_stat_server.get_total_covariance_stat1_mix(self.mean) \
+ (1-alpha) * out_stat_server.get_total_covariance_stat1_mix(self.mean)
#self.mean = in_stat_server.get_mean_stat1()
#self.Sigma = in_stat_server.get_total_covariance_stat1()
# # Second version
# in_stat_server.center_stat1(in_stat_server.get_mean_stat1())
......@@ -994,18 +996,14 @@ class FactorAnalyser:
logging.info('Estimate between class covariance, it %d / %d', it + 1, nb_iter)
# E-step
# print("E_step")
print("E_step")
# Copy stats as they will be whitened with a different Sigma for each iteration
out_local_stat = copy.deepcopy(out_model_shifted_stat)
# Whiten statistics (with the new mean and Sigma)
out_local_stat.whiten_stat1(self.mean, self.Sigma)
# Copy stats as they will be whitened with a different Sigma for each iteration
in_local_stat = copy.deepcopy(in_model_shifted_stat)
# Whiten statistics (with the new mean and Sigma)
out_local_stat.whiten_stat1(self.mean, self.Sigma)
in_local_stat.whiten_stat1(self.mean, self.Sigma)
......@@ -1026,6 +1024,7 @@ class FactorAnalyser:
e_hh = numpy.zeros((out_class_nb, rank_f, rank_f))
# loop on model id's
fa_model_loop(batch_start=0,
mini_batch_indices=numpy.arange(out_class_nb),
factor_analyser=self,
......@@ -1034,17 +1033,16 @@ class FactorAnalyser:
e_h=e_h,
e_hh=e_hh,
num_thread=1)
# Accumulate for minimum divergence step
out_R = numpy.sum(e_hh, axis=0) / out_session_per_model.shape[0]
out_C = e_h.T.dot(out_local_stat.stat1).dot(scipy.linalg.inv(sqr_inv_sigma))
out_A = numpy.einsum('ijk,i->jk', e_hh, out_local_stat.stat0.squeeze())
# Replicate self.stat0
index_map = numpy.zeros(vect_size, dtype=int)
_stat0 = in_local_stat.stat0[:, index_map]
e_h = numpy.zeros((in_class_nb, rank_f))
e_hh = numpy.zeros((in_class_nb, rank_f, rank_f))
......@@ -1057,7 +1055,6 @@ class FactorAnalyser:
e_h=e_h,
e_hh=e_hh,
num_thread=1)
# Accumulate for minimum divergence step
in_R = numpy.sum(e_hh, axis=0) / in_session_per_model.shape[0]
......@@ -1067,15 +1064,23 @@ class FactorAnalyser:
in_alpha = alpha / in_session_per_model.sum()
out_alpha = (1 - alpha) / out_session_per_model.sum()
#import ipdb
#ipdb.set_trace()
_A = in_alpha * in_A + out_alpha * out_A
_C = in_alpha * in_C + out_alpha * out_C
_R = alpha * in_R + (1 - alpha) * out_R
# M-step
self.F = scipy.linalg.solve(_A, _C).T
# Update the residual covariance
self.Sigma = sigma_obs - self.F.dot(_C)
#self.Sigma = sigma_obs - self.F.dot(_C) / (in_alpha * in_session_per_model.sum() + out_alpha * out_session_per_model.sum())
"""
dans la ligne du dessus on ne divise pas par le nombre total de sessions ???
"""
# Minimum Divergence step
self.F = self.F.dot(scipy.linalg.cholesky(_R))
......@@ -1086,4 +1091,9 @@ class FactorAnalyser:
if save_partial and it < nb_iter - 1:
self.write(output_file_name + "_it-{}.h5".format(it))
elif it == nb_iter - 1:
self.write(output_file_name + ".h5")
\ No newline at end of file
self.write(output_file_name + ".h5")
......@@ -25,7 +25,11 @@
Copyright 2014-2020 Yevhenii Prokopalo, Anthony Larcher
"""
import pdb
import traceback
import logging
import matplotlib.pyplot as plt
import numpy
import pandas
import pickle
......@@ -46,6 +50,8 @@ from ..statserver import StatServer
from torch.utils.data import DataLoader
from sklearn.model_selection import train_test_split
from .sincnet import SincNet, SincConv1d
#from torch.utils.tensorboard import SummaryWriter
import tqdm
__license__ = "LGPL"
......@@ -59,6 +65,96 @@ __docformat__ = 'reS'
logging.basicConfig(format='%(asctime)s %(message)s')
# Make PyTorch Deterministic
torch.manual_seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
numpy.random.seed(0)
class GuruMeditation (torch.autograd.detect_anomaly):
def __init__(self):
super(GuruMeditation, self).__init__()
def __enter__(self):
super(GuruMeditation, self).__enter__()
return self
def __exit__(self, type, value, trace):
super(GuruMeditation, self).__exit__()
if isinstance(value, RuntimeError):
traceback.print_tb(trace)
halt(str(value))
def halt(msg):
print (msg)
pdb.set_trace()
def select_n_random(data, labels, n=100):
'''
Selects n random datapoints and their corresponding labels from a dataset
'''
assert len(data) == len(labels)
perm = torch.randperm(len(data))
return data[perm][:n], labels[perm][:n]
def matplotlib_imshow(img, one_channel=False):
if one_channel:
img = img.mean(dim=0)
img = img / 2 + 0.5 # unnormalize
npimg = img.cpu().numpy()
if one_channel:
plt.imshow(npimg, cmap="Greys")
else:
plt.imshow(np.transpose(npimg, (1, 2, 0)))
def speech_to_probs(model, speech):
'''
Generates predictions and corresponding probabilities from a trained
network and a list of images
'''
output = model(speech)
# convert output probabilities to predicted class
_, preds_tensor = torch.max(output, 1)
preds = numpy.squeeze(preds_tensor.cpu().numpy())
return preds, [torch.nn.functional.softmax(el, dim=0)[i].item() for i, el in zip(preds, output)]
def plot_classes_preds(model, speech, labels):
'''
Generates matplotlib Figure using a trained network, along with images
and labels from a batch, that shows the network's top prediction along
with its probability, alongside the actual label, coloring this
information based on whether the prediction was correct or not.
Uses the "speech_to_probs" function.
'''
preds, probs = speech_to_probs(model, speech)
# plot the images in the batch, along with predicted and true labels
fig = plt.figure(figsize=(12, 48))
for idx in numpy.arange(4):
ax = fig.add_subplot(1, 4, idx+1, xticks=[], yticks=[])
#matplotlib_imshow(speech[idx], one_channel=True)
ax.set_title("{0}, {1:.1f}%\n(label: {2})".format(
preds[idx],
probs[idx] * 100.0,
labels[idx]),
color=("green" if preds[idx]==labels[idx].item() else "red"))
return fig
def get_lr(optimizer):
"""
......@@ -375,6 +471,7 @@ class Xtractor(torch.nn.Module):
"""
if self.preprocessor is not None:
x = self.preprocessor(x)
print("go through preprocessor")
x = self.sequence_network(x)
......@@ -425,8 +522,12 @@ def xtrain(speaker_number,
:param num_thread:
:return:
"""
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Add for tensorboard
# Writer will output to ./runs/ directory by default
#writer = SummaryWriter("runs/xvectors_experiments_2")
writer = None
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Start from scratch
if model_name is None:
# Initialize a first model
......@@ -460,6 +561,7 @@ def xtrain(speaker_number,
if name.split(".")[0] in freeze_parts:
param.requires_grad = False
if torch.cuda.device_count() > 1 and multi_gpu:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = torch.nn.DataParallel(model)
......@@ -467,22 +569,6 @@ def xtrain(speaker_number,
print("Train on a single GPU")
model.to(device)
if device.type == 'cuda':
print(torch.cuda.get_device_name())
print('Memory Usage:')
print('Allocated:', round(torch.cuda.memory_allocated()/1024**3,1), 'GB')
print('Cached: ', round(torch.cuda.memory_cached()/1024**3,1), 'GB')
test = torch.tensor(numpy.ones((128, 16000 * 5), dtype=numpy.float32))
test.to(device)
if device.type == 'cuda':
print(torch.cuda.get_device_name())
print('Memory Usage:')
print('Allocated:', round(torch.cuda.memory_allocated()/1024**3,1), 'GB')
print('Cached: ', round(torch.cuda.memory_cached()/1024**3,1), 'GB')
"""
Set the dataloaders according to the dataset_yaml
......@@ -504,14 +590,22 @@ def xtrain(speaker_number,
batch_size=dataset_params["batch_size"],
shuffle=True,
drop_last=True,
pin_memory=True,
num_workers=num_thread)
validation_set = SideSet(dataset_yaml, set_type="validation", dataset_df=validation_df)
validation_loader = DataLoader(validation_set,
batch_size=dataset_params["batch_size"],
drop_last=True,
pin_memory=True,
num_workers=num_thread)
# Add for TensorBoard
#dataiter = iter(training_loader)
#data, labels = dataiter.next()
#writer.add_graph(model, data)
"""
Set the training options
"""
......@@ -581,7 +675,8 @@ def xtrain(speaker_number,
optimizer,
dataset_params["log_interval"],
device=device,
clipping=clipping)
clipping=clipping,
tb_writer=writer)
# Add the cross validation here
accuracy, val_loss = cross_validation(model, validation_loader, device=device)
......@@ -589,7 +684,6 @@ def xtrain(speaker_number,
# Decrease learning rate according to the scheduler policy
scheduler.step(val_loss)
print(f"Learning rate is {optimizer.param_groups[0]['lr']}")
# remember best accuracy and save checkpoint
is_best = accuracy > best_accuracy
......@@ -614,10 +708,11 @@ def xtrain(speaker_number,
if is_best:
best_accuracy_epoch = epoch
#writer.close()
logging.critical(f"Best accuracy {best_accuracy * 100.} obtained at epoch {best_accuracy_epoch}")
def train_epoch(model, epoch, training_loader, optimizer, log_interval, device, clipping=False):
def train_epoch(model, epoch, training_loader, optimizer, log_interval, device, clipping=False, tb_writer=None):
"""
:param model:
......@@ -633,23 +728,43 @@ def train_epoch(model, epoch, training_loader, optimizer, log_interval, device,
criterion = torch.nn.CrossEntropyLoss(reduction='mean')
accuracy = 0.0
running_loss = 0.0
for batch_idx, (data, target) in enumerate(training_loader):
target = target.squeeze()
optimizer.zero_grad()
output = model(data.to(device))
#with GuruMeditation():
loss = criterion(output, target.to(device))
loss.backward()
if clipping:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
optimizer.step()
accuracy += (torch.argmax(output.data, 1) == target.to(device)).sum()
if not torch.isnan(loss):
loss.backward()
if clipping:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.)
running_loss += loss.item()
optimizer.step()
running_loss += loss.item()
accuracy += (torch.argmax(output.data, 1) == target.to(device)).sum()
if batch_idx % log_interval == 0:
batch_size = target.shape[0]
logging.critical('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAccuracy: {:.3f}'.format(
epoch, batch_idx + 1, training_loader.__len__(),
100. * batch_idx / training_loader.__len__(), loss.item(),
100.0 * accuracy.item() / ((batch_idx + 1) * batch_size)))
if batch_idx % log_interval == 0:
batch_size = target.shape[0]
logging.critical('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\tAccuracy: {:.3f}'.format(
epoch, batch_idx + 1, training_loader.__len__(),
100. * batch_idx / training_loader.__len__(), loss.item(),
100.0 * accuracy.item() / ((batch_idx + 1) * batch_size)))
#tb_writer.add_scalar('training loss',
# running_loss / log_interval,
# epoch * len(training_loader) + batch_idx)
#tb_writer.add_scalar('training_accuracy',
# 100.0 * accuracy.item() / ((batch_idx + 1) * batch_size),
# epoch * len(training_loader) + batch_idx)
# ...log a Matplotlib Figure showing the model's predictions on a
# random mini-batch
#tb_writer.add_figure('predictions vs. actuals',
# plot_classes_preds(model, data.to(device), target.to(device)),
# global_step=epoch * len(training_loader) + batch_idx)
running_loss = 0.0
return model
......
......@@ -869,6 +869,16 @@ class StatServer:
C = self.stat1 - self.stat1.mean(axis=0)
return numpy.dot(C.transpose(), C) / self.stat1.shape[0]
def get_total_covariance_stat1_mix(self, mean):
"""Compute and return the total covariance matrix of the first-order
statistics.
:return: the total co-variance matrix of the first-order statistics
as a ndarray.
"""
C = self.stat1 - mean
return numpy.dot(C.transpose(), C) / self.stat1.shape[0]
def get_within_covariance_stat1(self):
"""Compute and return the within-class covariance matrix of the
first-order statistics.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment