Commit f122f042 authored by Colleen Beaumard's avatar Colleen Beaumard
Browse files

Correction of scoring scripts

parent 13a20f51
......@@ -53,10 +53,11 @@ labels = list(args.emotions.split(" "))
nb_batch = str(args.batchs)
model_type = args.model
if "(" in model_type:
model_type, model_2nd = model_type.split("(")[0], "_(" + model_type.split("(")[1]
model_type, model_2nd = model_type.split("(")[0], "-" + model_type.split("(")[1]
else:
model_2nd = ""
lr = str(args.lr)
lr = str(float(args.lr))
cates = str(args.categories)
if args.freeze is not None:
freeze = "_freeze"
......@@ -85,7 +86,8 @@ index.close()
dic_index_emo = {}
# Needed when the model_type == custom
dico = {}
if model_type == "custom":
dico = {}
for elmt in index_emo:
dic_index_emo[int(elmt.split(": ")[1].replace("\n", ""))] = elmt.split(": ")[0]
......@@ -121,27 +123,24 @@ UAR = metrics.recall_score(gold_anno, predictions, average="macro")
UARPercent = round(UAR * 100, 2)
print("\nUAR:", UARPercent, "%\n")
confMatrix = metrics.confusion_matrix(gold_anno, predictions, labels=labels)
print(confMatrix)
confMatrix = metrics.confusion_matrix(gold_anno, predictions)
print(confMatrix, "\n")
gold_dic = {key: 0 for key in labels}
dico = {key: gold_dic for key in labels}
gold_dic = []
for gold, pred in zip(gold_anno, predictions):
dico[gold][pred] += 1
gold_dic[gold] += 1
for i in range(len(confMatrix)):
gold_dic.append(sum(confMatrix[i]))
[print("Total", key, ":", value) for key, value in gold_dic.items()]
[print("Total", labels[i], ":", gold_dic[i]) for i in range(len(labels))]
annot = []
em = [value for value in gold_dic.values()]
for i in range(0, len(confMatrix)): # row
annot.append([])
tot = 0
for j in range(0, len(confMatrix[i])): # column
nbr = confMatrix[i][j]
percent = round(nbr/em[i], 2)*100
percent = round(nbr/gold_dic[i], 2)*100
tot += percent
if j == len(confMatrix[i])-1:
if tot > 100:
......@@ -162,7 +161,8 @@ valid_loss = [line for line in file if "Validation Loss" in line]
# "Loss:" and "Epoch" in same line
if model_type == "custom":
loss_epoch = [line for line in file if "Epoch" in line]
loss_epoch = loss_epoch[::4] # 2 because only 1 gpu used: if more gpus are used, you have to do 2*nb_gpus
#loss_epoch = loss_epoch[::2] # 2 because only 1 gpu used: if more gpus are used, you have to do 2*nb_gpus
loss_epoch = [line for line in loss_epoch if "[1/" in line]
else:
loss_epoch = [line for line in file if "Epoch" in line]
......@@ -186,9 +186,10 @@ for e, t, v in zip(aepoch, tloss, vloss):
### Plotting of confusion matrix and losses ###
if model_type == "custom":
path = "model_{}/Sess{}_test/{}/{}emo_{}batch_lr-{}{}".format(model_type, ses_nb, model_2nd[2:-2] ,cates, nb_batch, lr, freeze)
path = "model_{}/Sess{}_test/{}/{}emo_{}batch_lr-{}{}".format(model_type, ses_nb, model_2nd[1:] ,cates, nb_batch, lr, freeze)
else:
path = "model_{}/Sess{}_test/{}emo_{}batch_lr-{}{}".format(model_type, ses_nb, cates, nb_batch, lr, freeze)
if not os.path.isdir(path.rsplit("/", 1)[0]):
os.mkdir(path.rsplit("/", 1)[0])
if not os.path.isdir(path):
......@@ -196,13 +197,10 @@ if not os.path.isdir(path):
# Plot confusion matrice
sns.heatmap(confMatrix, annot=annot, fmt="10", cmap="Blues", vmin=0, vmax=350, xticklabels=labels, yticklabels=labels)
plt.title("Model: {}{}{}_".format(model_type, model_2nd[1:], freeze) + str(config["speaker_number"]) +
"emo_{}batch\nepoch: {} lr: {} Data: Test-IEMOCAP {}".format(nb_batch, aepoch[-1], lr, ses_nb) +
" UAR = " + str(UARPercent) + "%")
plt.title("Model: {}{}{}_".format(model_type, model_2nd, freeze) + str(config["speaker_number"]) + "emo_{}batch\nepoch: {} lr: {} Data: Test-IEMOCAP {}".format(nb_batch, aepoch[-1], lr, ses_nb) + " UAR = " + str(UARPercent) + "%")
plt.xlabel("Prediction")
plt.ylabel("Ground truth")
plt.savefig(os.path.join(path, "confusion_matrix_{}{}_".format(model_type, model_2nd[2:-2], freeze) + str(config["speaker_number"]) +
"emo_{}batch_epoch-{}_lr-{}_Test-IEMOCAP{}.png".format(nb_batch, aepoch[-1], lr, ses_nb)))
plt.savefig(os.path.join(path, "confusion_matrix_{}{}{}_".format(model_type, model_2nd, freeze) + str(config["speaker_number"]) + "emo_{}batch_epoch-{}_lr-{}_Test-IEMOCAP{}.png".format(nb_batch, aepoch[-1], lr, ses_nb)))
plt.show()
plt.clf()
print("\nConfusion matrix done!")
......@@ -223,26 +221,16 @@ if len(reduce_lr) != 0:
plt.axvline(x=key, color=colors[i], linestyle='--', label=label)
i += 1
plt.legend()
plt.title("Model: {}{}{}_{}emo_{}batch\nEpoch: {} lr: {} Data: Test-IEMOCAP {}".format(model_type, model_2nd[1:], freeze, cates,
nb_batch, aepoch[-1], lr, ses_nb)
)
plt.savefig(os.path.join(path, "losses_{}{}{}_{}emo_{}batch_epoch-{}_lr-{}_Test-IEMOCAP{}.png".format(model_type, model_2nd[2:-2], freeze,
cates, nb_batch,
aepoch[-1], lr,
ses_nb)))
plt.title("Model: {}{}{}_{}emo_{}batch\nEpoch: {} lr: {} Data: Test-IEMOCAP {}".format(model_type, model_2nd, freeze, cates, nb_batch, aepoch[-1], lr, ses_nb))
plt.savefig(os.path.join(path, "losses_{}{}{}_{}emo_{}batch_epoch-{}_lr-{}_Test-IEMOCAP{}.png".format(model_type, model_2nd, freeze, cates, nb_batch, aepoch[-1], lr, ses_nb)))
plt.show()
plt.clf()
print("Losses plotted!\n")
### We move confusion matrix and logs to another folder ###
os.replace("model_{}/best_{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_type, cates, nb_batch, lr,
ses_nb),
os.path.join(path, "best_{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_2nd[2:-2], freeze, cates,
nb_batch, lr, ses_nb)))
os.replace("model_{}/tmp_{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_type, cates, nb_batch, lr,
ses_nb),
os.path.join(path, "tmp_{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_2nd[2:-2], freeze, cates,
nb_batch, lr, ses_nb)))
os.replace("logs/{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.log".format(model_type, cates, nb_batch, lr, ses_nb),
os.path.join(path, "{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.log".format(model_type, model_2nd[2:-2], freeze, cates,
nb_batch, lr, ses_nb)))
os.replace("model_{}/best_{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_type, cates, nb_batch, lr, ses_nb), os.path.join(path, "best_{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_2nd, freeze, cates, nb_batch, lr, ses_nb)))
os.replace("model_{}/tmp_{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_type, cates, nb_batch, lr, ses_nb), os.path.join(path, "tmp_{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, model_2nd, freeze, cates, nb_batch, lr, ses_nb)))
os.replace("logs/{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.log".format(model_type, cates, nb_batch, lr, ses_nb), os.path.join(path, "{}{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.log".format(model_type, model_2nd, freeze, cates, nb_batch, lr, ses_nb)))
......@@ -25,7 +25,12 @@ args = parser.parse_args()
labels = list(args.emotions.split(" "))
nb_batch = str(args.batchs)
model_type = args.model
lr = str(args.lr)
if "(" in model_type:
model_type, model_2nd = model_type.split("(")[0], "-" + model_type.split("(")[1]
else:
model_2nd = ""
lr = str(float(args.lr))
cates = str(args.categories)
if args.freeze is not None:
freeze = "_freeze"
......@@ -33,7 +38,11 @@ else:
freeze = ""
# Path to save the confusion matrix for cross-validation
path = "model_{}/Sess_all_cross-valid/{}emo_{}batch_lr-{}{}".format(model_type, cates, nb_batch, lr, freeze)
if model_type == "custom":
path = "model_{}/Sess_all_cross-valid/{}/{}emo_{}batch_lr-{}{}".format(model_type, ses_nb, model_2nd[1:] ,cates, nb_batch, lr, freeze)
else:
path = "model_{}/Sess_all_cross-valid/{}emo_{}batch_lr-{}{}".format(model_type, cates, nb_batch, lr, freeze)
if not os.path.isdir(path.rsplit("/", 1)[0]):
os.mkdir(path.rsplit("/", 1)[0])
if not os.path.isdir(path):
......@@ -73,7 +82,9 @@ index = open("list/emos_index.txt", "r")
index_emo = index.readlines()
index.close()
dic_index_emo = {}
dico = {} # Needed when model_type == custom
if model_type == "custom":
dico = {} # Needed when model_type == custom
for elmt in index_emo:
dic_index_emo[int(elmt.split(": ")[1].replace("\n", ""))] = elmt.split(": ")[0]
......@@ -81,10 +92,9 @@ print("\nAll emotions:", dic_index_emo, "\n\n## Beginning of extraction ##")
for i in range(1, 6):
print("Session {} is processing...".format(i))
# 1st is the model, 2nd is the weights and all
xtract, config = load_model("model_{}/Sess{}_test/{}emo_{}batch_lr-{}{}/best_{}{}_{}emo_{}batch_lr-{}_"
"Test-IEMOCAP{}.pt".format(model_type, i, cates, nb_batch, lr, freeze, model_type,
freeze, cates, nb_batch, lr, i), "cuda")
xtract, config = load_model("model_{}/Sess{}_test/{}emo_{}batch_lr-{}{}/best_{}{}_{}emo_{}batch_lr-{}_Test-IEMOCAP{}.pt".format(model_type, i, cates, nb_batch, lr, freeze, model_type, freeze, cates, nb_batch, lr, i), "cuda")
path_wav = "data/IEMOCAP/Session{}/sentences/wav".format(i)
......@@ -124,27 +134,24 @@ UAR = metrics.recall_score(gold_anno, predictions, average="macro")
UARPercent = round(UAR * 100, 2)
print("UAR:", UARPercent, "%\n")
confMatrix = metrics.confusion_matrix(gold_anno, predictions, labels=labels)
confMatrix = metrics.confusion_matrix(gold_anno, predictions)
print(confMatrix)
gold_dic = {key: 0 for key in labels}
dico = {key: gold_dic for key in labels}
for gold, pred in zip(gold_anno, predictions):
dico[gold][pred] += 1
gold_dic[gold] += 1
[print("Total", key, ":", value) for key, value in gold_dic.items()]
gold_dic = []
for i in range(len(confMatrix)):
gold_dic.append(sum(confMatrix[i]))
[print("Total", labels[i], ":", gold_dic[i]) for i in range(len(labels))]
annot = []
em = [value for value in gold_dic.values()]
for i in range(0, len(confMatrix)): # row
annot.append([])
tot = 0
for j in range(0, len(confMatrix[i])): # column
nbr = confMatrix[i][j]
percent = round(nbr/em[i], 2)*100
percent = round(nbr/gold_dic[i], 2)*100
tot += percent
if j == len(confMatrix[i])-1:
if tot > 100:
......@@ -155,13 +162,10 @@ for i in range(0, len(confMatrix)): # row
annot[i].append(full)
sns.heatmap(confMatrix, annot=annot, fmt="10", cmap="Blues", vmin=0, vmax=1000, xticklabels=labels, yticklabels=labels)
plt.title("Model: " + str(config["model_archi"]["model_type"]) + "{}_".format(freeze) + str(config["speaker_number"]) +
"emo_{}batch\nlr: {} Data: Test-IEMOCAP-cross_validation".format(nb_batch, lr) +
" UAR = " + str(UARPercent) + "%")
plt.title("Model: {}{}{}_".format(model_type, model_2nd, freeze) + str(config["speaker_number"]) + "emo_{}batch\nlr: {} Data: Test-IEMOCAP-cross_validation".format(nb_batch, lr) + " UAR = " + str(UARPercent) + "%")
plt.xlabel("Prediction")
plt.ylabel("Ground truth")
plt.savefig(os.path.join(path, "confusion_matrix_{}{}_".format(model_type, freeze) + str(config["speaker_number"]) +
"emo_{}batch_lr-{}_IEMOCAP-cross_validation.png".format(nb_batch, lr)))
plt.savefig(os.path.join(path, "confusion_matrix_{}{}{}_".format(model_type, model_2nd, freeze) + str(config["speaker_number"]) + "emo_{}batch_lr-{}_IEMOCAP-cross_validation.png".format(nb_batch, lr)))
plt.show()
plt.clf()
print("\nConfusion matrix done!")
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment