commands to execute parallel and sequential on single/multi-server; examples

bin/get_master

+#!/usr/bin/env sh
+
+set -- $(get_nodes)
+echo $1
+
+

bin/get_nodes

+#!/usr/bin/env python3 
+
+import os
+
+
+if not 'SLURM_NODELIST' in os.environ:
+    raise EnvironmentError('Probably not in a SLURM job')
+
+base = os.environ['SLURM_NODELIST']
+
+if '[' in base:
+    types, nums = base.split('[')
+    nums = nums.strip(']').split('-')
+    print(" ".join([types + n for n in nums]))
+else:
+    print(base)

bin/parse_options.sh

+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.

bin/slurm_add_command

+#!/usr/bin/env sh
+
+output=$1
+shift
+
+echo $@ >> $output

bin/slurm_array_job_start

+#!/bin/sh
+
+sh $SLURM_SUBMIT_DIR/.slurm_batch_script_job_$SLURM_ARRAY_TASK_ID.sh

bin/slurm_execute

+#!/usr/bin/env bash
+
+set -e
+
+mem="5G"
+p="cpu"
+w=
+c=1
+N=1
+gres=
+job_name=
+time="01-00"
+max_parallel=10
+ranks=""
+
+. /lium/buster1/vpelloin/.local/bin/parse_options.sh
+
+# echo "mem: $mem | p: $p | w: $w | c: $c | N: $N | gres: $gres | job_name: $job_name | time: $time | max_parallel: $max_parallel"
+
+if [ -z $job_name ]; then
+	job_name=$(ps -o comm= $PPID)
+fi
+
+params=()
+params+=(--mem $mem)
+params+=(--time $time)
+params+=(--job-name $job_name)
+params+=(-p $p)
+params+=(-c $c)
+params+=(-N $N)
+
+# optional params
+if ! [ -z $w ]; then
+	params+=(-w $w)
+fi
+if ! [ -z $gres ]; then
+	params+=(--gres $gres)
+fi
+
+# allow clean CTRL+C cancel
+trap ctrl_c INT
+cancel=false
+function ctrl_c() {
+	cancel=true
+}
+
+
+if [ "$#" -gt 1 ]; then
+	params+=(--wait)
+	params+=(--parsable)
+	params+=(--array "1-$#%$max_parallel")
+	
+	echo -n "Starting $# jobs on SLURM..."
+	jobid=$(sbatch "${params[@]}" slurm_array_job_start) 
+	echo " Done!"
+
+elif [ "$#" == 1 ]; then
+	stdout=.stdout$1
+	params+=(--output $stdout)
+	params+=(-K1)
+
+	if [ -f $stdout ]; then
+		rm $stdout
+	fi
+
+	touch $stdout
+	if ! [ -z "$ranks" ]; then
+		i=0
+		for rank in $ranks; do
+			text="if test \$SLURM_NODEID = $i ; then export RANK=$rank; fi"
+			sed -i "3 i $text" $1
+			i=$((i + 1))
+		done
+	fi
+	text="export MASTER_HOST=$\(get_master)"
+	sed -i "3 i $text" $1
+
+	chmod +x $1
+	jobid=$(srun "${params[@]}" $1 &) &
+	tail --pid $(jobs -p) -f $stdout
+	wait
+else
+	echo "Not enough jobs given"
+	exit 1
+fi
+
+if $cancel; then
+	# we used CTRL+C
+	scancel $jobid
+fi
+
+i=1
+for script in $@; do
+	rm .slurm_batch_script_job_$i.sh
+
+	if [ "$#" -gt 1 ]; then
+		echo -e "====\n$i\n===="
+		output=slurm-${jobid}_${i}.out
+		if [ -f $output ]; then
+			cat $output
+			rm $output
+		fi
+		echo -e "\n"
+	else
+		rm $stdout
+	fi
+	i=$((i + 1))
+done
+
+if $cancel; then
+	exit 0
+fi

bin/slurm_list_add_command

+slurm_add_command
\ No newline at end of file

bin/slurm_list_execute

+slurm_execute
\ No newline at end of file

bin/slurm_list_prepare

+slurm_prepare
\ No newline at end of file

bin/slurm_prepare

+#!/usr/bin/env bash
+
+i=$(echo $(($(find . -name '.slurm_batch_script_job_*' | wc -l) + 1)))
+output=".slurm_batch_script_job_$i.sh"
+
+if [ -f "$output" ]; then
+	rm $output
+fi
+
+cat <(echo -e "#!/bin/sh \nset -e") > .slurm_batch_script_job_$i.sh
+
+echo $output

bin/test_ram

+#!/usr/bin/env python3
+
+import numpy
+
+megabytes=1024
+
+tab = [numpy.random.bytes(1024*1024) for x in range(megabytes)]
+print("Created a {}Mb object!".format(len(tab)))
+

example/test.sh

+#!/usr/bin/env bash
+
+id=$(slurm_prepare)
+
+slurm_add_command $id \
+	echo "Starting a big script..."
+slurm_add_command $id \
+	sleep 10
+slurm_add_command $id \
+	echo "It is done!"
+
+slurm_execute \
+	--mem "200M" \
+	--p cpu \
+	--time "04:00" \
+	--job-name "testing_script" \
+	$id

example/test_multi_servers.sh

+#!/usr/bin/env bash
+
+job=$(slurm_prepare) 
+
+slurm_add_command $job \
+	echo "Starting script..."
+
+slurm_add_command $job \
+	echo '$RANK' '$CUDA_VISIBLE_DEVICES' '$HOST' '$MASTER_HOST'
+
+slurm_add_command $job sleep 3
+
+slurm_add_command $job \
+	echo "Done!"
+
+# --gres "gpu:1" --p gpu
+
+slurm_execute \
+	--mem "50M" \
+	--N 2 --c 1 \
+	--ranks "0 8" \
+	--job-name "parallel_computation_test" \
+	$job

example/test_parallel.sh

+#!/usr/bin/env bash
+
+ids="" # list that holds every parallel script ID
+
+for corpus in train dev test; do
+	job=$(slurm_list_prepare)    # we get an ID for that iteration
+	ids="$ids $job"              # we add it to our global list
+
+	# and then we add our commands for this iteration using slurm_list_add_command <jobid>
+	slurm_list_add_command $job \
+		echo "Processing $corpus..."
+
+	slurm_list_add_command $job \
+		sleep 5
+
+	slurm_list_add_command $job \
+		echo "Done!"
+
+done
+
+# we finally start the jobs
+slurm_list_execute \
+	--mem "50M" \
+	--N 1 --c 1 --p cpu \
+	--max-parallel 2 \
+	--job-name "parallel_computation_test" \
+	$ids

install.sh

+#!/usr/bin/env bash
+
+set -e
+
+prefix=$HOME/.local
+
+help_message=$(cat <<-END
+	Usage: $0 [OPTION]
+	\t--prefix\tinstall prefix path (default to $HOME/.local)
+END
+)
+
+. bin/parse_options.sh
+
+bin_dir=bin
+
+for file in $(/bin/ls -1 bin); do
+	src=$PWD/bin/$file
+	dst=$prefix/$bin_dir/$file
+
+	echo -n "Installing $file... "
+	if [ -L $dst ]; then
+		echo -n "removing symbolic link. "
+		rm $dst
+	elif [ -f $dst ]; then
+		echo -n "creating a backup. "
+		mv $dst $dst.backup
+	fi
+	ln -s $src $dst
+	echo "Done"
+done
+
+echo "Installation done"