Commit 707a8dec authored by Valentin Pelloin's avatar Valentin Pelloin

commands to execute parallel and sequential on single/multi-server; examples

parent d387df81
#!/usr/bin/env sh
set -- $(get_nodes)
echo $1
#!/usr/bin/env python3
import os
if not 'SLURM_NODELIST' in os.environ:
raise EnvironmentError('Probably not in a SLURM job')
base = os.environ['SLURM_NODELIST']
if '[' in base:
types, nums = base.split('[')
nums = nums.strip(']').split('-')
print(" ".join([types + n for n in nums]))
else:
print(base)
#!/usr/bin/env bash
# Copyright 2012 Johns Hopkins University (Author: Daniel Povey);
# Arnab Ghoshal, Karel Vesely
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABLITY OR NON-INFRINGEMENT.
# See the Apache 2 License for the specific language governing permissions and
# limitations under the License.
# Parse command-line options.
# To be sourced by another script (as in ". parse_options.sh").
# Option format is: --option-name arg
# and shell variable "option_name" gets set to value "arg."
# The exception is --help, which takes no arguments, but prints the
# $help_message variable (if defined).
###
### The --config file options have lower priority to command line
### options, so we need to import them first...
###
# Now import all the configs specified by command-line, in left-to-right order
for ((argpos=1; argpos<$#; argpos++)); do
if [ "${!argpos}" == "--config" ]; then
argpos_plus1=$((argpos+1))
config=${!argpos_plus1}
[ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
. $config # source the config file.
fi
done
###
### Now we process the command line options
###
while true; do
[ -z "${1:-}" ] && break; # break if there are no arguments
case "$1" in
# If the enclosing script is called with --help option, print the help
# message and exit. Scripts should put help messages in $help_message
--help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
else printf "$help_message\n" 1>&2 ; fi;
exit 0 ;;
--*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
exit 1 ;;
# If the first command-line argument begins with "--" (e.g. --foo-bar),
# then work out the variable name as $name, which will equal "foo_bar".
--*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
# Next we test whether the variable in question is undefned-- if so it's
# an invalid option and we die. Note: $0 evaluates to the name of the
# enclosing script.
# The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
# is undefined. We then have to wrap this test inside "eval" because
# foo_bar is itself inside a variable ($name).
eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
oldval="`eval echo \\$$name`";
# Work out whether we seem to be expecting a Boolean argument.
if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
was_bool=true;
else
was_bool=false;
fi
# Set the variable to the right value-- the escaped quotes make it work if
# the option had spaces, like --cmd "queue.pl -sync y"
eval $name=\"$2\";
# Check that Boolean-valued arguments are really Boolean.
if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
exit 1;
fi
shift 2;
;;
*) break;
esac
done
# Check for an empty argument to the --cmd option, which can easily occur as a
# result of scripting errors.
[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
true; # so this script returns exit code 0.
#!/usr/bin/env sh
output=$1
shift
echo $@ >> $output
#!/bin/sh
sh $SLURM_SUBMIT_DIR/.slurm_batch_script_job_$SLURM_ARRAY_TASK_ID.sh
#!/usr/bin/env bash
set -e
mem="5G"
p="cpu"
w=
c=1
N=1
gres=
job_name=
time="01-00"
max_parallel=10
ranks=""
. /lium/buster1/vpelloin/.local/bin/parse_options.sh
# echo "mem: $mem | p: $p | w: $w | c: $c | N: $N | gres: $gres | job_name: $job_name | time: $time | max_parallel: $max_parallel"
if [ -z $job_name ]; then
job_name=$(ps -o comm= $PPID)
fi
params=()
params+=(--mem $mem)
params+=(--time $time)
params+=(--job-name $job_name)
params+=(-p $p)
params+=(-c $c)
params+=(-N $N)
# optional params
if ! [ -z $w ]; then
params+=(-w $w)
fi
if ! [ -z $gres ]; then
params+=(--gres $gres)
fi
# allow clean CTRL+C cancel
trap ctrl_c INT
cancel=false
function ctrl_c() {
cancel=true
}
if [ "$#" -gt 1 ]; then
params+=(--wait)
params+=(--parsable)
params+=(--array "1-$#%$max_parallel")
echo -n "Starting $# jobs on SLURM..."
jobid=$(sbatch "${params[@]}" slurm_array_job_start)
echo " Done!"
elif [ "$#" == 1 ]; then
stdout=.stdout$1
params+=(--output $stdout)
params+=(-K1)
if [ -f $stdout ]; then
rm $stdout
fi
touch $stdout
if ! [ -z "$ranks" ]; then
i=0
for rank in $ranks; do
text="if test \$SLURM_NODEID = $i ; then export RANK=$rank; fi"
sed -i "3 i $text" $1
i=$((i + 1))
done
fi
text="export MASTER_HOST=$\(get_master)"
sed -i "3 i $text" $1
chmod +x $1
jobid=$(srun "${params[@]}" $1 &) &
tail --pid $(jobs -p) -f $stdout
wait
else
echo "Not enough jobs given"
exit 1
fi
if $cancel; then
# we used CTRL+C
scancel $jobid
fi
i=1
for script in $@; do
rm .slurm_batch_script_job_$i.sh
if [ "$#" -gt 1 ]; then
echo -e "====\n$i\n===="
output=slurm-${jobid}_${i}.out
if [ -f $output ]; then
cat $output
rm $output
fi
echo -e "\n"
else
rm $stdout
fi
i=$((i + 1))
done
if $cancel; then
exit 0
fi
slurm_add_command
\ No newline at end of file
slurm_execute
\ No newline at end of file
slurm_prepare
\ No newline at end of file
#!/usr/bin/env bash
i=$(echo $(($(find . -name '.slurm_batch_script_job_*' | wc -l) + 1)))
output=".slurm_batch_script_job_$i.sh"
if [ -f "$output" ]; then
rm $output
fi
cat <(echo -e "#!/bin/sh \nset -e") > .slurm_batch_script_job_$i.sh
echo $output
#!/usr/bin/env python3
import numpy
megabytes=1024
tab = [numpy.random.bytes(1024*1024) for x in range(megabytes)]
print("Created a {}Mb object!".format(len(tab)))
#!/usr/bin/env bash
id=$(slurm_prepare)
slurm_add_command $id \
echo "Starting a big script..."
slurm_add_command $id \
sleep 10
slurm_add_command $id \
echo "It is done!"
slurm_execute \
--mem "200M" \
--p cpu \
--time "04:00" \
--job-name "testing_script" \
$id
#!/usr/bin/env bash
job=$(slurm_prepare)
slurm_add_command $job \
echo "Starting script..."
slurm_add_command $job \
echo '$RANK' '$CUDA_VISIBLE_DEVICES' '$HOST' '$MASTER_HOST'
slurm_add_command $job sleep 3
slurm_add_command $job \
echo "Done!"
# --gres "gpu:1" --p gpu
slurm_execute \
--mem "50M" \
--N 2 --c 1 \
--ranks "0 8" \
--job-name "parallel_computation_test" \
$job
#!/usr/bin/env bash
ids="" # list that holds every parallel script ID
for corpus in train dev test; do
job=$(slurm_list_prepare) # we get an ID for that iteration
ids="$ids $job" # we add it to our global list
# and then we add our commands for this iteration using slurm_list_add_command <jobid>
slurm_list_add_command $job \
echo "Processing $corpus..."
slurm_list_add_command $job \
sleep 5
slurm_list_add_command $job \
echo "Done!"
done
# we finally start the jobs
slurm_list_execute \
--mem "50M" \
--N 1 --c 1 --p cpu \
--max-parallel 2 \
--job-name "parallel_computation_test" \
$ids
#!/usr/bin/env bash
set -e
prefix=$HOME/.local
help_message=$(cat <<-END
Usage: $0 [OPTION]
\t--prefix\tinstall prefix path (default to $HOME/.local)
END
)
. bin/parse_options.sh
bin_dir=bin
for file in $(/bin/ls -1 bin); do
src=$PWD/bin/$file
dst=$prefix/$bin_dir/$file
echo -n "Installing $file... "
if [ -L $dst ]; then
echo -n "removing symbolic link. "
rm $dst
elif [ -f $dst ]; then
echo -n "creating a backup. "
mv $dst $dst.backup
fi
ln -s $src $dst
echo "Done"
done
echo "Installation done"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment