Commit cb585357 authored by Anthony Larcher's avatar Anthony Larcher
Browse files

debug

parent 0a2924b9
......@@ -569,7 +569,7 @@ class SideSet(Dataset):
pin_memory=True,
num_workers=num_thread)
for batch_idx, (data, target) in enumerate(tmp_loader):
for batch_idx, (data, target) in tqdm.tqdm(enumerate(tmp_loader)):
write_batch(batch_idx, data, target, batch_fn_format)
def createSideSets(data_set_yaml,
......@@ -726,4 +726,4 @@ class FileSet(Dataset):
:return:
"""
return self.len
\ No newline at end of file
return self.len
......@@ -43,7 +43,7 @@ import yaml
from torchvision import transforms
from collections import OrderedDict
from .xsets import SideSet
from .xsets import Fileset
from .xsets import FileSet
from .xsets import IdMapSet
from .res_net import RawPreprocessor, ResBlockWFMS
from ..bosaris import IdMap
......@@ -56,6 +56,7 @@ from .loss import ArcLinear
from .loss import ArcFace
from .loss import l2_norm
import torch.autograd.profiler as profiler
from torch.nn.parallel import DistributedDataParallel as DDP
__license__ = "LGPL"
......@@ -716,6 +717,10 @@ def xtrain(speaker_number,
print("Train on a single GPU")
model.to(device)
with open(dataset_yaml, "r") as fh:
dataset_params = yaml.load(fh, Loader=yaml.FullLoader)
df = pandas.read_csv(dataset_params["dataset_description"])
if load_batches_from_disk:
train_batch_fn_format = tmp_batch_dir + "/train/train_{}_batch.h5"
val_batch_fn_format = tmp_batch_dir + "/val/val_{}_batch.h5"
......@@ -733,10 +738,6 @@ def xtrain(speaker_number,
else:
output_format = "pytorch"
with open(dataset_yaml, "r") as fh:
dataset_params = yaml.load(fh, Loader=yaml.FullLoader)
df = pandas.read_csv(dataset_params["dataset_description"])
training_df, validation_df = train_test_split(df, test_size=dataset_params["validation_ratio"])
torch.manual_seed(dataset_params['seed'])
training_set = SideSet(dataset_yaml,
......@@ -753,25 +754,33 @@ def xtrain(speaker_number,
if write_batches_to_disk:
logging.critical("Start writing batches on disk")
training_set.write_to_disk(dataset_params["batch_size"], train_batch_fn_format, num_thread)
validation_set.write_to_disk(dataset_params["batch_size"], val_batch_fn_format, num_thread)
else:
training_loader = DataLoader(training_set,
batch_size=dataset_params["batch_size"],
shuffle=True,
drop_last=True,
pin_memory=True,
num_workers=num_thread)
validation_loader = DataLoader(validation_set,
batch_size=dataset_params["batch_size"],
drop_last=True,
pin_memory=True,
num_workers=num_thread)
logging.critical("---> Done")
if load_batches_from_disk:
training_loader = Fileset(train_batch_fn_format)
validation_loader = Fileset(train_batch_fn_format)
training_set = FileSet(train_batch_fn_format)
validation_set = FileSet(train_batch_fn_format)
batch_size = 1
else:
batch_size = dataset_params["batch_size"]
print(f"Size of batches = {batch_size}")
training_loader = DataLoader(training_set,
batch_size=batch_size,
shuffle=True,
drop_last=True,
pin_memory=True,
num_workers=num_thread)
validation_loader = DataLoader(validation_set,
batch_size=batch_size,
drop_last=True,
pin_memory=True,
num_workers=num_thread)
"""
Set the training options
......@@ -786,19 +795,19 @@ def xtrain(speaker_number,
_optimizer = torch.optim.SGD
_options = {'lr': lr, 'momentum': 0.9}
params = [
{
'params': [
param for name, param in model.named_parameters() if 'bn' not in name
]
},
{
'params': [
param for name, param in model.named_parameters() if 'bn' in name
],
'weight_decay': 0
},
]
#params = [
# {
# 'params': [
# param for name, param in model.named_parameters() if 'bn' not in name
# ]
# },
# {
# 'params': [
# param for name, param in model.named_parameters() if 'bn' in name
# ],
# 'weight_decay': 0
# },
#]
param_list = []
if type(model) is Xtractor:
......@@ -912,15 +921,16 @@ def train_epoch(model, epoch, training_loader, optimizer, log_interval, device,
accuracy = 0.0
running_loss = 0.0
for batch_idx, (data, target) in enumerate(training_loader):
data = data.to(device)
data = data.squeeze().to(device)
print(f"Shape of data: {data.shape}")
target = target.squeeze()
target = target.to(device)
optimizer.zero_grad()
if loss_criteria == 'aam':
output = model(data), target=target)
output = model(data, target=target)
else:
output = model(data), target=None)
output = model(data, target=None)
#with GuruMeditation():
loss = criterion(output, target)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment