Commit 70b20c05 authored by Caillaut Gaetan's avatar Caillaut Gaetan
Browse files

eval.py

parent bd58f691
......@@ -18,8 +18,10 @@ from minibert import *
def parse_run_path(s):
splitted = os.path.split(s)
#splitted = os.path.split(s)
splitted = s.split("/")
i = 0
#print(splitted)
while not (splitted[i].startswith("t1") or splitted[i].startswith("t2")):
i = i + 1
......@@ -45,7 +47,7 @@ def parse_run_path(s):
else:
x = run_name.split("_")
res["model"] = "minibert"
res["d"] = int(x[0])
res["d"] = int(x[0][1:])
res["attention"] = x[1]
res["position"] = x[2]
res["architecture"] = x[5]
......@@ -89,10 +91,10 @@ def t2_camembert(path):
def eval_t1(run, e, dev_loader, test_loader):
d = run["d"]
att = run["att"]
pos = run["pos"]
att = run["attention"]
pos = run["position"]
arch = run["architecture"]
height, depth = [int(x) for x in arch[-1].split("d")]
height, depth = [int(x) for x in arch[1:].split("d")]
scal = run["attention-scaling"]
frozen = run["frozen"]
crps = run["corpus"]
......@@ -119,31 +121,31 @@ def eval_t1(run, e, dev_loader, test_loader):
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["epoch"] = e
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["dev_tp"] = dev_tp
res_dict["dev_fp"] = dev_fp
res_dict["dev_fn"] = dev_fn
res_dict["dev_recall"] = dev_r
res_dict["dev_precision"] = dev_p
res_dict["dev_fmeasure"] = dev_f
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
res_dict["tp"] = tp
res_dict["fp"] = fp
res_dict["fn"] = fn
res_dict["recall"] = r
res_dict["precision"] = p
res_dict["fmeasure"] = f
return res_dict
def eval_t2(run, e, dev_loader, test_loader):
d = run["d"]
att = run["att"]
pos = run["pos"]
att = run["attention"]
pos = run["position"]
arch = run["architecture"]
height, depth = [int(x) for x in arch[-1].split("d")]
height, depth = [int(x) for x in arch[1:].split("d")]
scal = run["attention-scaling"]
frozen = run["frozen"]
crps = run["corpus"]
......@@ -174,21 +176,21 @@ def eval_t2(run, e, dev_loader, test_loader):
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["epoch"] = e
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["dev_tp"] = dev_tp
res_dict["dev_fp"] = dev_fp
res_dict["dev_fn"] = dev_fn
res_dict["dev_recall"] = dev_r
res_dict["dev_precision"] = dev_p
res_dict["dev_fmeasure"] = dev_f
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
res_dict["tp"] = tp
res_dict["fp"] = fp
res_dict["fn"] = fn
res_dict["recall"] = r
res_dict["precision"] = p
res_dict["fmeasure"] = f
return res_dict
......@@ -206,21 +208,21 @@ def eval_t1_camembert(run, e, dev_loader, test_loader):
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["epoch"] = e
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["dev_tp"] = dev_tp
res_dict["dev_fp"] = dev_fp
res_dict["dev_fn"] = dev_fn
res_dict["dev_recall"] = dev_r
res_dict["dev_precision"] = dev_p
res_dict["dev_fmeasure"] = dev_f
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
res_dict["tp"] = tp
res_dict["fp"] = fp
res_dict["fn"] = fn
res_dict["recall"] = r
res_dict["precision"] = p
res_dict["fmeasure"] = f
return res_dict
......@@ -240,21 +242,21 @@ def eval_t2_camembert(run, e, dev_loader, test_loader):
res_dict = run.copy()
res_dict["epoch"].append(e)
res_dict["epoch"] = e
res_dict["dev_tp"].append(dev_tp)
res_dict["dev_fp"].append(dev_fp)
res_dict["dev_fn"].append(dev_fn)
res_dict["dev_recall"].append(dev_r)
res_dict["dev_precision"].append(dev_p)
res_dict["dev_fmeasure"].append(dev_f)
res_dict["dev_tp"] = dev_tp
res_dict["dev_fp"] = dev_fp
res_dict["dev_fn"] = dev_fn
res_dict["dev_recall"] = dev_r
res_dict["dev_precision"] = dev_p
res_dict["dev_fmeasure"] = dev_f
res_dict["tp"].append(tp)
res_dict["fp"].append(fp)
res_dict["fn"].append(fn)
res_dict["recall"].append(r)
res_dict["precision"].append(p)
res_dict["fmeasure"].append(f)
res_dict["tp"] = tp
res_dict["fp"] = fp
res_dict["fn"] = fn
res_dict["recall"] = r
res_dict["precision"] = p
res_dict["fmeasure"] = f
return res_dict
......@@ -297,11 +299,16 @@ if __name__ == "__main__":
runs = [str(x) for x in Path("models").glob("*/*/*")
if x.is_dir() and re.match("^models/(t1|t2)", str(x))]
#import random
#random.shuffle(runs)
#runs = runs[:10]
res = []
for run_path in runs:
print(run_path)
run = parse_run_path(run_path)
crps = run["corpus"]
task = run[task]
task = run["task"]
dev_dataset = ds[task][crps]["dev"]
test_dataset = ds[task][crps]["test"]
......@@ -309,6 +316,7 @@ if __name__ == "__main__":
if run["model"] == "minibert":
tokenizer = toks[crps]
pad_token = "<pad>"
pad_idx = tokenizer.token_to_id(pad_token)
collater = DEFT2018Collater(
......@@ -336,9 +344,9 @@ if __name__ == "__main__":
for e in range(1, 11):
if task == "t1":
eval_t1(run, e, dev_loader, test_loader)
eval_t1_camembert(run, e, dev_loader, test_loader)
elif task == "t2":
eval_t2(run, e, dev_loader, test_loader)
eval_t2_camembert(run, e, dev_loader, test_loader)
df = pd.DataFrame.from_records(res)
df.to_csv("performances.csv", quoting=csv.QUOTE_NONNUMERIC, index=False)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment