Commit bd58f691 authored by Gaëtan Caillaut's avatar Gaëtan Caillaut
Browse files

new eval script

parent 231b2aeb
from pathlib import Path
import csv
import os
from datetime import datetime, timedelta
import pandas as pd
import tokenizers
......@@ -8,18 +9,60 @@ from tokenizers import Tokenizer
from datasets import *
from evaluation import *
from train import *
import itertools
import re
import argparse
import sys
from minibert import *
def mlm_path(dir, d, att, pos):
return str(Path(dir, f"d{d}_{att}_{pos}_gelu_norm", "minibert-model.pt"))
def parse_run_path(s):
splitted = os.path.split(s)
i = 0
while not (splitted[i].startswith("t1") or splitted[i].startswith("t2")):
i = i + 1
task = splitted[i].split("_")
crps = splitted[i+1]
run_name = splitted[i+2]
def t1_path(dir, d, att, pos, e):
return str(Path(dir, f"d{d}_{att}_{pos}_norm", f"checkpoint-{e:05}.tar"))
res = {
"task": task[0],
"task_variant": task[1] if len(task) > 1 else "",
"corpus": crps,
"path": s
}
if run_name == "camembert":
res["model"] = "camembert"
res["d"] = 768
res["attention"] = "multi-head attention"
res["position"] = ""
res["architecture"] = "h12d12"
res["attention-scaling"] = "softmax"
res["frozen"] = False
else:
x = run_name.split("_")
res["model"] = "minibert"
res["d"] = int(x[0])
res["attention"] = x[1]
res["position"] = x[2]
res["architecture"] = x[5]
res["attention-scaling"] = x[6]
res["frozen"] = run_name.endswith("_frozen")
return res
def mlm_path(dir, d, att, pos, arch, scal):
return str(Path(dir, f"d{d}_{att}_{pos}_gelu_norm_{arch}_{scal}", "minibert-model.pt"))
def t1_path(dir, d, att, pos, arch, scal, frozen, e):
run_name = f"d{d}_{att}_{pos}_gelu_norm_{arch}_{scal}"
if frozen:
run_name = f"{run_name}_frozen"
return str(Path(dir, run_name, f"checkpoint-{e:05}.tar"))
def t2_path(dir, d, att, pos, e):
......@@ -44,175 +87,258 @@ def t2_camembert(path):
return model
if __name__ == "__main__":
model_types = ["cleaned", "lemmatized"]
tok_paths = ["output/tokenizer.json", "output/tokenizer_lemmatized.json"]
def eval_t1(run, e, dev_loader, test_loader):
d = run["d"]
att = run["att"]
pos = run["pos"]
arch = run["architecture"]
height, depth = [int(x) for x in arch[-1].split("d")]
scal = run["attention-scaling"]
frozen = run["frozen"]
crps = run["corpus"]
model, _, _, _ = t1_model_from_params(
mlm_path(f"models/{crps}", d, att, pos, arch, scal),
d,
parse_attention(att),
parse_position(pos),
tokenizer,
120,
"<mask>",
"<pad>",
"cuda",
checkpoint_path=str(Path(run["path"], f"checkpoint-{e:05}.tar")),
height=height,
depth=depth,
attention_scaling=parse_attention_scaling(scal)
)
model.eval()
dev_tp, dev_fp, dev_fn, dev_r, dev_p, dev_f = fmeasure_deft2018_t1(
model, dev_loader, "cuda")
tp, fp, fn, r, p, f = fmeasure_deft2018_t1(model, test_loader, "cuda")
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
return res_dict
def eval_t2(run, e, dev_loader, test_loader):
d = run["d"]
att = run["att"]
pos = run["pos"]
arch = run["architecture"]
height, depth = [int(x) for x in arch[-1].split("d")]
scal = run["attention-scaling"]
frozen = run["frozen"]
crps = run["corpus"]
model, _, _, _ = t2_model_from_params(
mlm_path(f"models/{crps}", d, att, pos, arch, scal),
d,
parse_attention(att),
parse_position(pos),
tokenizer,
120,
"<mask>",
"<pad>",
"cuda",
checkpoint_path=str(Path(run["path"], f"checkpoint-{e:05}.tar")),
height=height,
depth=depth,
attention_scaling=parse_attention_scaling(scal)
)
model.eval()
t2_classes = set(T2Dataset.labels_to_id().values())
dev_tp, dev_fp, dev_fn, dev_r, dev_p, dev_f = fmeasure_deft2018_t2(
model, dev_loader, t2_classes, "cuda")
tp, fp, fn, r, p, f = fmeasure_deft2018_t2(
model, test_loader, t2_classes, "cuda")
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
return res_dict
def eval_t1_camembert(run, e, dev_loader, test_loader):
crps = run["corpus"]
model = t1_camembert(f"models/t1/{crps}/camembert/checkpoint-{e:05}.tar")
model.eval()
dev_tp, dev_fp, dev_fn, dev_r, dev_p, dev_f = fmeasure_deft2018_t1(
model, dev_loader, "cuda")
tp, fp, fn, r, p, f = fmeasure_deft2018_t1(
model, test_loader, "cuda")
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
return res_dict
def eval_t2_camembert(run, e, dev_loader, test_loader):
crps = run["corpus"]
model = t2_camembert(f"models/t2/{crps}/camembert/checkpoint-{e:05}.tar")
model.eval()
ds = [16, 32, 64, 96, 128]
epochs = list(range(10, 101, 10))
positions = ["none", "fixed", "trained"]
attentions = ["self-attention", "non-transforming"]
t2_classes = set(T2Dataset.labels_to_id().values())
res_dict = {
"d": [],
"attention": [],
"position": [],
"epoch": [],
"corpus": [],
"task": [],
"tp": [],
"fp": [],
"fn": [],
"precision": [],
"recall": [],
"fmeasure": [],
dev_tp, dev_fp, dev_fn, dev_r, dev_p, dev_f = fmeasure_deft2018_t2(
model, dev_loader, t2_classes, "cuda")
tp, fp, fn, r, p, f = fmeasure_deft2018_t2(
model, test_loader, t2_classes, "cuda")
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
return res_dict
if __name__ == "__main__":
toks = {
"lemmatized": Tokenizer.from_file("output/tokenizer_lemmatized.json"),
"cleaned": Tokenizer.from_file("output/tokenizer.json"),
"camembert": CamembertTokenizerFast.from_pretrained("camembert/camembert-base")
}
camembert_res_dict = {
"d": [],
"epoch": [],
"corpus": [],
"task": [],
"tp": [],
"fp": [],
"fn": [],
"precision": [],
"recall": [],
"fmeasure": [],
ds = {
"t1": {
"lemmatized": {
"test": T1Dataset.from_csv(f"data/lemmatized/t2/test.csv"),
"dev": T1Dataset.from_csv(f"data/lemmatized/t2/dev.csv"),
},
"cleaned": {
"test": T1Dataset.from_csv(f"data/cleaned/t2/test.csv"),
"dev": T1Dataset.from_csv(f"data/cleaned/t2/dev.csv"),
}
},
"t2": {
"lemmatized": {
"test": T2Dataset.from_csv(f"data/lemmatized/t2/test.csv"),
"dev": T2Dataset.from_csv(f"data/lemmatized/t2/dev.csv"),
},
"cleaned": {
"test": T2Dataset.from_csv(f"data/cleaned/t1/test.csv"),
"dev": T2Dataset.from_csv(f"data/cleaned/t2/dev.csv"),
}
}
}
pad_token = "<pad>"
mask_token = "<mask>"
# TASK 1
for typ, tok_path in zip(model_types, tok_paths):
tokenizer = Tokenizer.from_file(tok_path)
pad_idx = tokenizer.token_to_id(pad_token)
deft_collater = DEFT2018Collater(
tokenizer, pad_idx, T1Dataset.labels_to_id(), max_seq_len=120)
test_dataset = T1Dataset.from_csv(f"data/{typ}/t1/test.csv")
test_loader = DataLoader(
test_dataset, collate_fn=deft_collater, batch_size=256, pin_memory=True)
for e in epochs:
# MINIBERT
for d in ds:
for pos in positions:
for att in attentions:
model, _, _, _ = t1_model_from_params(
mlm_path(f"models/{typ}", d, att, pos),
d,
parse_attention(att),
parse_position(pos),
tokenizer,
120,
"<mask>",
"<pad>",
"cuda",
checkpoint_path=t1_path(
f"models/t1/{typ}", d, att, pos, e
)
)
model.eval()
tp, fp, fn, r, p, f = fmeasure_deft2018_t1(
model, test_loader, "cuda")
res_dict["d"].append(d)
res_dict["attention"].append(att)
res_dict["position"].append(pos)
res_dict["epoch"].append(e)
res_dict["corpus"].append(typ)
res_dict["task"].append("t1")
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
# CAMEMBERT
model = t1_camembert(
f"models/t1/{typ}/camembert/checkpoint-{e:05}.tar")
model.eval()
tp, fp, fn, r, p, f = fmeasure_deft2018_t1(
model, test_loader, "cuda")
camembert_res_dict["d"].append(768)
camembert_res_dict["epoch"].append(e)
camembert_res_dict["corpus"].append(typ)
camembert_res_dict["task"].append("t1")
camembert_res_dict["tp"].append(tp)
camembert_res_dict["fp"].append(fp)
camembert_res_dict["fn"].append(fn)
camembert_res_dict["recall"].append(r)
camembert_res_dict["precision"].append(p)
camembert_res_dict["fmeasure"].append(f)
# TASK 2
for typ, tok_path in zip(model_types, tok_paths):
tokenizer = Tokenizer.from_file(tok_path)
pad_idx = tokenizer.token_to_id(pad_token)
classes = set(T2Dataset.labels_to_id().values())
deft_collater = DEFT2018Collater(
tokenizer, pad_idx, T2Dataset.labels_to_id(), max_seq_len=120)
test_dataset = T2Dataset.from_csv(f"data/{typ}/t2/test.csv")
test_loader = DataLoader(
test_dataset, collate_fn=deft_collater, batch_size=256, pin_memory=True)
for e in epochs:
# MINIBERT
for d in ds:
for pos in positions:
for att in attentions:
model, _, _, _ = t2_model_from_params(
mlm_path(f"models/{typ}", d, att, pos),
d,
parse_attention(att),
parse_position(pos),
tokenizer,
120,
"<mask>",
"<pad>",
"cuda",
checkpoint_path=t2_path(
f"models/t2/{typ}", d, att, pos, e
)
)
model.eval()
tp, fp, fn, r, p, f = fmeasure_deft2018_t2(
model, test_loader, t2_classes, "cuda")
res_dict["d"].append(d)
res_dict["attention"].append(att)
res_dict["position"].append(pos)
res_dict["epoch"].append(e)
res_dict["corpus"].append(typ)
res_dict["task"].append("t2")
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
# CAMEMBERT
model = t2_camembert(
f"models/t1/{typ}/camembert/checkpoint-{e:05}.tar")
model.eval()
tp, fp, fn, r, p, f = fmeasure_deft2018(
model, test_loader, t2_classes, "cuda")
camembert_res_dict["d"].append(768)
camembert_res_dict["epoch"].append(e)
camembert_res_dict["corpus"].append(typ)
camembert_res_dict["task"].append("t2")
camembert_res_dict["tp"].append(tp)
camembert_res_dict["fp"].append(fp)
camembert_res_dict["fn"].append(fn)
camembert_res_dict["recall"].append(r)
camembert_res_dict["precision"].append(p)
camembert_res_dict["fmeasure"].append(f)
df = pd.DataFrame(res_dict)
df.to_csv("fmeasure.csv", quoting=csv.QUOTE_NONNUMERIC, index=False)
df_camembert = pd.DataFrame(camembert_res_dict)
df_camembert.to_csv("fmeasure_camembert.csv",
quoting=csv.QUOTE_NONNUMERIC, index=False)
labels2id = {
"t1": T1Dataset.labels_to_id(),
"t2": T2Dataset.labels_to_id(),
}
runs = [str(x) for x in Path("models").glob("*/*/*")
if x.is_dir() and re.match("^models/(t1|t2)", str(x))]
res = []
for run_path in runs:
run = parse_run_path(run_path)
crps = run["corpus"]
task = run[task]
dev_dataset = ds[task][crps]["dev"]
test_dataset = ds[task][crps]["test"]
lab2id = labels2id[task]
if run["model"] == "minibert":
tokenizer = toks[crps]
pad_idx = tokenizer.token_to_id(pad_token)
collater = DEFT2018Collater(
tokenizer, pad_idx, lab2id, max_seq_len=120)
dev_loader = DataLoader(
dev_dataset, collate_fn=collater, batch_size=256, pin_memory=True)
test_loader = DataLoader(
test_dataset, collate_fn=collater, batch_size=256, pin_memory=True)
for e in range(10, 51, 10):
if task == "t1":
scores = eval_t1(run, e, dev_loader, test_loader)
elif task == "t2":
scores = eval_t2(run, e, dev_loader, test_loader)
res.append(scores)
elif run["model"] == "camembert":
tokenizer = toks["camembert"]
collater = DEFT2018CollaterForCamembert(tokenizer, lab2id)
dev_loader = DataLoader(
dev_dataset, collate_fn=collater, batch_size=256, pin_memory=True)
test_loader = DataLoader(
test_dataset, collate_fn=collater, batch_size=256, pin_memory=True)
for e in range(1, 11):
if task == "t1":
eval_t1(run, e, dev_loader, test_loader)
elif task == "t2":
eval_t2(run, e, dev_loader, test_loader)
df = pd.DataFrame.from_records(res)
df.to_csv("performances.csv", quoting=csv.QUOTE_NONNUMERIC, index=False)
......@@ -954,8 +954,7 @@ def finetune_t1_camembert(args):
pin_memory = device != "cpu"
camembert_tokenizer = CamembertTokenizerFast.from_pretrained(
"camembert/camembert-base",
)
"camembert/camembert-base")
if args.checkpoint is None:
model = CamembertForSequenceClassification.from_pretrained(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment