Commit 118379de authored by Gaëtan Caillaut's avatar Gaëtan Caillaut
Browse files

refactoring in slurm_scripts

parent 11b063c3
......@@ -37,14 +37,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
TB_DIR="${LOGDIR}/${RUN_NAME}"
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR} --height 1 --depth 2
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR} --height 1 --depth 2
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -37,14 +37,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
TB_DIR="${LOGDIR}/${RUN_NAME}"
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR} --height 1 --depth 2
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR} --height 1 --depth 2
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_h1d2"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_h1d2"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
T2_RUN_NAME="d${D}_${ATT}_${POS}_norm_h1d2"
TB_DIR="${LOGDIR}/${T2_RUN_NAME}"
T2_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T2_RUN_NAME}/checkpoint-${E}.tar"
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
else
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2"
T2_RUN_NAME="d${D}_${ATT}_${POS}_norm_h1d2"
TB_DIR="${LOGDIR}/${T2_RUN_NAME}"
T2_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d2_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T2_RUN_NAME}/checkpoint-${E}.tar"
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 1 --depth 2
else
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 1 --depth 2
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 1 --depth 2
fi
done
done
......
......@@ -38,13 +38,12 @@ for E in $(seq -f "%05g" 0 10 40); do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
TB_DIR="${LOGDIR}/${RUN_NAME}"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR} --height 2 --depth 1
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR} --height 2 --depth 1
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -37,14 +37,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
TB_DIR="${LOGDIR}/${RUN_NAME}"
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR} --height 2 --depth 1
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR} --height 2 --depth 1
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_h2d1"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_h2d1"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
T2_RUN_NAME="d${D}_${ATT}_${POS}_norm_h2d1"
TB_DIR="${LOGDIR}/${T2_RUN_NAME}"
T2_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T2_RUN_NAME}/checkpoint-${E}.tar"
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
else
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1"
T2_RUN_NAME="d${D}_${ATT}_${POS}_norm_h2d1"
TB_DIR="${LOGDIR}/${T2_RUN_NAME}"
T2_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h2d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T2_RUN_NAME}/checkpoint-${E}.tar"
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --height 2 --depth 1
else
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --height 2 --depth 1
python train.py t2-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --height 2 --depth 1
fi
done
done
......
......@@ -34,17 +34,16 @@ set -x
set -e
for E in $(seq -f "%05g" 0 10 40); do
for D in 16 32 64 96 128; do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed" "trained"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
TB_DIR="${LOGDIR}/${RUN_NAME}"
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR}
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR}
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR}
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -37,14 +37,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
TB_DIR="${LOGDIR}/${RUN_NAME}"
RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${RUN_NAME}/checkpoint-${E}.tar"
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${TB_DIR}
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --checkpoint ${CHECKPOINT} --logdir ${LOGDIR}
else
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${TB_DIR}
python train.py mlm ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --bs ${BS} --epochs 10 --attention ${ATT} --position ${POS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -38,15 +38,14 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT}
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT}
else
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR}
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -38,15 +38,14 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_frozen"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax_frozen"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --freeze-attention
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --freeze-attention
else
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --freeze-attention
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --freeze-attention
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT}
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT}
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR}
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -39,15 +39,13 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT}
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT}
else
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR}
python train.py t1-fs ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -38,18 +38,15 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/mi\
nibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoi\
nt ${CHECKPOINT}
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT}
else
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/mi\
nibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR}
nibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR}
fi
done
done
......
......@@ -38,15 +38,14 @@ for E in $(seq -f "%05g" 0 10 40); do
for D in 32; do
for ATT in "self-attention" "non-transforming" "semi-transforming"; do
for POS in "none" "fixed"; do
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm"
T1_RUN_NAME="d${D}_${ATT}_${POS}_norm_frozen"
TB_DIR="${LOGDIR}/${T1_RUN_NAME}"
MLM_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax"
T1_RUN_NAME="d${D}_${ATT}_${POS}_gelu_norm_h1d1_softmax_frozen"
if ((10#$E>0)); then
CHECKPOINT="${OUT_DIR}/${T1_RUN_NAME}/checkpoint-${E}.tar"
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --checkpoint ${CHECKPOINT} --freeze-attention
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --checkpoint ${CHECKPOINT} --freeze-attention
else
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${TB_DIR} --freeze-attention
python train.py t1 ${TRAIN} ${TEST} ${DEV} ${TOKENIZER} "${PRETRAINED_DIR}/${MLM_RUN_NAME}/minibert-model.pt" -o ${OUT_DIR} -d ${D} --attention ${ATT} --position ${POS} --epochs 10 --bs ${BS} --device ${DEVICE} --logdir ${LOGDIR} --freeze-attention
fi
done
done
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment