Commit 74d354ac authored by Iker Martín Álvarez's avatar Iker Martín Álvarez
Browse files

Merge branch 'RMA-Distributions' into 'Extrae-Executions'

Update Extrae-Executions with current code

See merge request martini/malleability_benchmark!3
parents 88a1e68f 412646ab
#!/bin/bash
# Obtains the number of total cores in an homogenous partition
# Parameter 1 - Partition to use
#====== Do not modify these values =======
partition=$1
hostlist=$(sinfo -hs --partition $partition | sed 's/ */:/g' | cut -d ':' -f5)
basic_node=$(scontrol show hostname $hostlist | paste -d, -s | cut -d ',' -f1)
cores=$(scontrol show node $basic_node | grep CPUTot | cut -d '=' -f3 | cut -d ' ' -f1)
echo "$cores"
#!/bin/bash
# Obtains for a given configuration file how many nodes will be needed
# Parameter 1 - Configuration file name for the emulation.
# Parameter 2 - Base directory of the malleability benchmark
# Parameter 3 - Number of cores in the machines. The machines must be homogenous. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
if [ "$#" -lt "3" ]
then
echo "Not enough arguments"
echo "Usage -> bash getMaxNodesNeeded.sh Configuration.ini BaseDirectory NumCores"
exit -1
fi
config_file=$1
dir=$2
cores=$3
max_numP=-1
total_resizes=$(grep Total_Resizes $config_file | cut -d '=' -f2)
total_groups=$(($total_resizes + 1))
for ((j=0; j<total_groups; j++));
do
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $config_file $j)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
node_qty=$(($max_numP / $cores))
if [ $node_qty -eq 0 ]
then
node_qty=1
fi
echo $node_qty
#!/bin/bash
dir="/home/martini/malleability_benchmark" #FIXME Obtain from another way
# Runs in a given current directory all .ini files
# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
config_file=$1
group_index=$2
resize_info=$(grep "\[resize$group_index\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
echo $numP
#!/bin/bash #!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/martini/malleability_benchmark/"
cores=20 partition="P1"
# Checks if all the runs in the current working directory performed under a # Checks if all the runs in the current working directory performed under a
# Slurm manager have been performed correctly and if some runs can be corrected # Slurm manager have been performed correctly and if some runs can be corrected
# they are launched again # they are launched again
# Parameter 1 - Maximum index of the runs # Parameter 1 - Common name of the configuration files
# Parameter 2 - Amount of repetitions per index/run # Parameter 2 - Maximum index of the runs
# Parameter 3 - Total stages in all runs. #FIXME The amount of stages must be equal across all the runs, must be modified in the future. # Parameter 3 - Amount of repetitions per index/run
# Parameter 4 - Total groups of processes in all runs #FIXME The amount of groups must be equal across all the runs, must be modified in the future. # Parameter 4 - Total stages in all runs. #FIXME The amount of stages must be equal across all the runs, must be modified in the future.
# Parameter 5 - Maximum valid iteration time across all runs. If an iteration time # Parameter 5 - Total groups of processes in all runs #FIXME The amount of groups must be equal across all the runs, must be modified in the future.
# Parameter 6 - Maximum valid iteration time across all runs. If an iteration time
# is higher, that particular repetition inside the run is cleaned and # is higher, that particular repetition inside the run is cleaned and
# launched again. # launched again.
# Parameter 7(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time.
# Must be a positive integer.
#====== Do not modify the following values ======= #====== Do not modify the following values =======
codeDir="Codes/" codeDir="Codes/"
execDir="Exec/" execDir="Exec/"
ResultsDir="Results/" ResultsDir="Results/"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
maxIndex=$1 if [ "$#" -lt "6" ]
totalEjGrupo=$2 #Total de ejecuciones por grupo then
total_stages=$3 echo "Not enough arguments"
total_groups=$4 echo "Usage -> bash CheckRun Common_Name maxIndex total_repetitions total_groups total_stages max_iteration_time [limit_time]"
maxTime=$5 #Maximo tiempo que se considera válido exit -1
fi
common_name=$1
maxIndex=$2
totalEjGrupo=$3 #Total de ejecuciones por grupo
total_stages=$4
total_groups=$5
maxTime=$6 #Maximo tiempo que se considera válido
limit_time_exec=0
if [ $# -ge 7 ] #Max time per execution in seconds
then
limit_time_exec=$7
fi
exec_lines_basic=6 limit_time=0
exec_lines_basic=7
iter_lines_basic=3 iter_lines_basic=3
exec_total_lines=$(($exec_lines_basic+$total_stages+$total_groups)) exec_total_lines=$(($exec_lines_basic+$total_stages+$total_groups))
iter_total_lines=$(($iter_lines_basic+$total_stages*2+1)) iter_total_lines=$(($iter_lines_basic+$total_stages*2+1))
exec_remove=$(($exec_lines_basic+$total_stages+$total_groups-1)) exec_remove=$(($exec_lines_basic+$total_stages+$total_groups-1))
iter_remove=$(($iter_lines_basic+$total_stages-1)) iter_remove=$(($iter_lines_basic+$total_stages))
echo $#
if [ "$#" -lt "5" ]
then
echo "Not enough arguments"
echo "Usage -> bash CheckRun maxIndes total_repetitions total_groups total_stages max_iteration_time"
exit -1
fi
#Check if there are fatal errors during executions #Check if there are fatal errors during executions
grep -i -e fatal -e error -e abort -e == slurm* > errores2.txt grep -i -e fatal -e error -e abort -e == slurm* > errores2.txt
...@@ -47,13 +58,14 @@ if [ "$qty" -gt "0" ] ...@@ -47,13 +58,14 @@ if [ "$qty" -gt "0" ]
then then
echo "Found Fatal errors during execution. Aborting" echo "Found Fatal errors during execution. Aborting"
echo "Read file errors2 to see the errors and in which files" echo "Read file errors2 to see the errors and in which files"
echo "FAILURE"
exit -2 exit -2
fi fi
#Check if the number of output files is correct. #Check if the number of output files is correct.
#If the number is not correct is a fatal error and the user #If the number is not correct is a fatal error and the user
# is informed in which runs the amount does not match, and # is informed in which runs the amount does not match, and
# then the scripts exit. # then the scripts exits.
#The user must figure out what to do with those runs. #The user must figure out what to do with those runs.
qtyG=$(ls R*_Global.out | wc -l) qtyG=$(ls R*_Global.out | wc -l)
qtyG=$(($qtyG * 2)) qtyG=$(($qtyG * 2))
...@@ -76,21 +88,22 @@ else ...@@ -76,21 +88,22 @@ else
echo "Files do not match at Run $i -- diff=$diff" >> errores2.txt echo "Files do not match at Run $i -- diff=$diff" >> errores2.txt
fi fi
done done
echo "FAILURE"
exit -1 exit -1
fi fi
rm errores2.txt rm errores2.txt
# Check if there is any negative execution time # Check if there is any negative execution time
# Only invalid IDs are stored # Only invalid IDs are stored
rm -f errores.txt rm -f tmp.txt
touch errores.txt touch tmp.txt
exec_ids=($(grep -n "T_total" R*_Global.out | grep - | cut -d '_' -f1 | cut -d 'R' -f2)) exec_ids=($(grep -n "T_total" R*_Global.out | grep - | cut -d '_' -f1 | cut -d 'R' -f2))
exec_line=($(grep -n "T_total" R*_Global.out | grep - | cut -d ':' -f2)) exec_line=($(grep -n "T_total" R*_Global.out | grep - | cut -d ':' -f2))
for ((i=${#exec_ids[@]}-1; i>=0; i--)) for ((i=${#exec_ids[@]}-1; i>=0; i--))
do do
first_line=$((${exec_line[$i]}-$exec_remove)) first_line=$((${exec_line[$i]}-$exec_remove))
last_line=$(($first_line+$exec_total_lines-1)) last_line=$(($first_line+$exec_total_lines-1))
echo "${exec_ids[$i]}:$first_line:$last_line" >> errores.txt echo "${exec_ids[$i]}:$first_line:$last_line" >> tmp.txt
done done
# Check if there is any iter time higher than expected # Check if there is any iter time higher than expected
...@@ -108,25 +121,29 @@ do ...@@ -108,25 +121,29 @@ do
first_line=$(($first_line/$iter_total_lines)) first_line=$(($first_line/$iter_total_lines))
first_line=$(($first_line*$exec_total_lines+1)) first_line=$(($first_line*$exec_total_lines+1))
last_line=$(($first_line+$exec_total_lines-1)) last_line=$(($first_line+$exec_total_lines-1))
echo "${iter_ids[$i]}:$first_line:$last_line" >> errores.txt echo "${iter_ids[$i]}:$first_line:$last_line" >> tmp.txt
fi fi
done done
#Clean data from collected erroneous executions #Clean data from collected erroneous executions
qty=$(wc -l errores.txt | cut -d ' ' -f1) qty=$(wc -l tmp.txt | cut -d ' ' -f1)
if [ "$qty" -gt 0 ]; if [ "$qty" -gt 0 ];
then then
echo "Se han encontrado errores de ejecución leves. Volviendo a ejecutar" echo "Found minor execution errors. Executing again. Review file errores.txt."
echo "CHECKRUN -- Found errors" >> errores.txt
while IFS="" read -r lineRun || [ -n "$lineRun" ] while IFS="" read -r lineRun || [ -n "$lineRun" ]
do do
#Obtain data of erroneous execution #Obtain data of erroneous execution
run=$(echo $lineRun | cut -d ':' -f1) run=$(echo $lineRun | cut -d ':' -f1)
echo "Run $run had an erroneous execution, cleaning bad data." echo "Run $run had an erroneous execution, cleaning bad data."
echo "Run$run----------------------------------------------" >> errores.txt
#1 - Delete erroneous lines in Global file #1 - Delete erroneous lines in Global file
first_line=$(echo $lineRun | cut -d ':' -f2) first_line=$(echo $lineRun | cut -d ':' -f2)
last_line=$(echo $lineRun | cut -d ':' -f3) last_line=$(echo $lineRun | cut -d ':' -f3)
sed -n ''$first_line','$last_line'p' R${run}_Global.out >> errores.txt
sed -i ''$first_line','$last_line'd' R${run}_Global.out sed -i ''$first_line','$last_line'd' R${run}_Global.out
#2 - Translate line numbers to Local files type #2 - Translate line numbers to Local files type
...@@ -136,10 +153,12 @@ then ...@@ -136,10 +153,12 @@ then
#3 - Delete erroneous lines in Local files #3 - Delete erroneous lines in Local files
for ((j=0; j<total_groups; j++)); for ((j=0; j<total_groups; j++));
do do
sed -n ''$first_line','$last_line'p' R${run}_G${j}* >> errores.txt
sed -i ''$first_line','$last_line'd' R${run}_G${j}* sed -i ''$first_line','$last_line'd' R${run}_G${j}*
done done
echo "--------------------------------------------------" >> errores.txt
done < errores.txt done < tmp.txt
fi fi
#Check if all repetitions for each Run have been executed #Check if all repetitions for each Run have been executed
...@@ -153,43 +172,28 @@ do ...@@ -153,43 +172,28 @@ do
qtyEx=$(grep T_total R"$run"_Global.out | wc -l) qtyEx=$(grep T_total R"$run"_Global.out | wc -l)
if [ "$qtyEx" -ne "$totalEjGrupo" ]; if [ "$qtyEx" -ne "$totalEjGrupo" ];
then then
#1 - Obtain config file name and repetitions to perform
diff=$(($totalEjGrupo-$qtyEx)) diff=$(($totalEjGrupo-$qtyEx))
qty_missing=$(($qty_missing+$diff)) qty_missing=$(($qty_missing+$diff))
config_file="config$run.ini" config_file="$common_name$run.ini"
if [ $limit_time_exec -ne 0 ] #Max time per execution in seconds
#1 - Obtain maximum number of processes for the run
max_numP=-1
for ((j=0; j<total_groups; j++));
do
resize_info=$(grep "\[resize$j\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
#3 - Obtain needed nodes for the number of processes
node_qty=$(($max_numP / $cores))
if [ "$node_qty" -eq "0" ];
then then
node_qty=1 limit_time=$(($limit_time_exec*$diff/60+1))
fi fi
#2 - Obtain number of nodes needed
node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
#3 - Launch execution #3 - Launch execution
echo "Run$run lacks $diff repetitions" echo "Run$run lacks $diff repetitions"
use_extrae=0 use_extrae=0
sbatch -N $node_qty $dir$execDir./generalRun.sh $dir $config_file $use_extrae $run $diff sbatch -p $partition -N $node_qty -t $limit_time $dir$execDir./generalRun.sh $dir $cores $config_file $use_extrae $run $diff
fi fi
else else
echo "File R${run}_Global.out does not exist -- Could it be it must still be executed?" echo "File R${run}_Global.out does not exist -- Could it be it must still be executed?"
fi fi
done done
if [ "$qty_missing" -eq "0" ]; if [ "$qty_missing" -eq "0" ];
then then
echo "SUCCESS" echo "SUCCESS"
......
#!/bin/bash #!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/martini/malleability_benchmark"
codeDir="/Codes/build" codeDir="/Codes/build"
export EXTRAE_CONFIG_FILE=extrae.xml export EXTRAE_CONFIG_FILE=extrae.xml
export LD_PRELOAD=$EXTRAE_HOME/lib/libmpitrace.so export LD_PRELOAD=$EXTRAE_HOME/lib/libmpitrace.so
$dir$codeDir/./a.out $dir$codeDir/./a.out
#!/bin/bash #!/bin/bash
#This script should only be called by others scripts, do not call it directly
#SBATCH --exclude=c02,c01,c00 #SBATCH --exclude=c02,c01,c00
#SBATCH -p P1 #SBATCH -p P1
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters with the aid of the RMS Slurm.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec" execDir="/Exec"
ResultsDir="/Results" ResultsDir="/Results"
...@@ -10,13 +20,14 @@ ResultsDir="/Results" ...@@ -10,13 +20,14 @@ ResultsDir="/Results"
echo "START TEST" echo "START TEST"
#$1 == baseDir #$1 == baseDir
#$2 == configFile #$2 == cores
#$3 == use_extrae #$3 == configFile
#$4 == outFileIndex #$4 == use_extrae
#$5 == qty #$5 == outFileIndex
#$6 == qty
echo $@ echo $@
if [ $# -lt 3 ] if [ $# -lt 4 ]
then then
echo "Internal ERROR generalRun.sh - Not enough arguments were given" echo "Internal ERROR generalRun.sh - Not enough arguments were given"
exit -1 exit -1
...@@ -24,13 +35,14 @@ fi ...@@ -24,13 +35,14 @@ fi
#READ PARAMETERS AND ENSURE CORRECTNESS #READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1 dir=$1
configFile=$2 cores=$2
use_extrae=$3 configFile=$3
outFileIndex=$4 use_extrae=$4
outFileIndex=$5
qty=1 qty=1
if [ $# -ge 5 ] if [ $# -ge 5 ]
then then
qty=$5 qty=$6
fi fi
nodelist=$SLURM_JOB_NODELIST nodelist=$SLURM_JOB_NODELIST
...@@ -45,10 +57,8 @@ then ...@@ -45,10 +57,8 @@ then
nodes=1 nodes=1
fi fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1) numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
read -r ini fin <<<$(echo $aux) initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
diff=$(( fin - ini ))
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
#EXECUTE RUN #EXECUTE RUN
echo "Nodes=$nodelist" echo "Nodes=$nodelist"
...@@ -56,7 +66,7 @@ if [ $use_extrae -ne 1 ] ...@@ -56,7 +66,7 @@ if [ $use_extrae -ne 1 ]
then then
for ((i=0; i<qty; i++)) for ((i=0; i<qty; i++))
do do
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done done
else else
cp $dir$execDir/Extrae/extrae.xml . cp $dir$execDir/Extrae/extrae.xml .
...@@ -64,6 +74,7 @@ else ...@@ -64,6 +74,7 @@ else
cp $dir$execDir/Extrae/trace_worker.sh . cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++)) for ((i=0; i<qty; i++))
do do
#FIXME Extrae not tested keeping in mind the initial nodelist - Could have some errors
srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done done
fi fi
......
#!/bin/bash #!/bin/bash
#This script should only be called by others scripts, do not call it directly
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec" execDir="/Exec"
ResultsDir="/Results" ResultsDir="/Results"
...@@ -7,63 +17,64 @@ ResultsDir="/Results" ...@@ -7,63 +17,64 @@ ResultsDir="/Results"
echo "START TEST" echo "START TEST"
#$1 == baseDir #$1 == baseDir
#$2 == configFile #$2 == cores
#$3 == use_extrae #$3 == configFile
#$4 == outFileIndex #$4 == use_extrae
#$5 == qty #$5 == outFileIndex
#$6 == qty
echo $@ echo $@
if [ $# -lt 3 ] if [ $# -lt 3 ]
then then
echo "Internal ERROR generalRun.sh - Not enough arguments were given" echo "Internal ERROR generalRunCostum.sh - Not enough arguments were given"
exit -1 exit -1
fi fi
#READ PARAMETERS AND ENSURE CORRECTNESS #READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1 dir=$1
configFile=$2 cores=$2
configFile=$3
use_extrae=0 use_extrae=0
outFileIndex=0 outFileIndex=0
qty=1 qty=1
if [ $# -ge 3 ] if [ $# -ge 4 ]
then then
use_extrae=$3 use_extrae=$3
fi fi
if [ $# -ge 4 ] if [ $# -ge 5 ]
then then
outFileIndex=$4 outFileIndex=$4
fi fi
if [ $# -ge 5 ] if [ $# -ge 6 ]
then then
qty=$5 qty=$5
fi fi
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
nodelist=$SLURM_JOB_NODELIST nodelist=$SLURM_JOB_NODELIST
nodes=$SLURM_JOB_NUM_NODES nodes=$SLURM_JOB_NUM_NODES
if [ -z "$nodelist" ]; if [ -z "$nodelist" ];
then then
nodelist="localhost" nodelist="localhost"
initial_nodelist="localhost"
else
initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
fi fi
if [ -z "$nodes" ]; if [ -z "$nodes" ];
then then
nodes=1 nodes=1
fi fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1)
read -r ini fin <<<$(echo $aux)
diff=$(( fin - ini ))
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
#EXECUTE RUN #EXECUTE RUN
echo "Nodes=$nodelist" echo "Nodes=$nodelist"
if [ $use_extrae -ne 1 ] if [ $use_extrae -ne 1 ]
then then
for ((i=0; i<qty; i++)) for ((i=0; i<qty; i++))
do do
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done done
else else
cp $dir$execDir/Extrae/extrae.xml . cp $dir$execDir/Extrae/extrae.xml .
...@@ -71,7 +82,7 @@ else ...@@ -71,7 +82,7 @@ else
cp $dir$execDir/Extrae/trace_worker.sh . cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++)) for ((i=0; i<qty; i++))
do do
mpirun -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes mpirun -hosts $initial_nodelist -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done done
fi fi
......
#!/bin/bash #!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/martini/malleability_benchmark/"
# Creates a directory with all possible and valid combinations of configuration files
# that can be created from a given complex configuration file.
# Parameter 1: Complex configuration file name.
# Parameter 2: Common output name of the output configuration files. It will be appended an index to each of them.
#====== Do not modify these values =======
codeDir="Codes/" codeDir="Codes/"
execDir="Exec/" execDir="Exec/"
ResultsDir="Results/" ResultsDir="Results/"
......
...@@ -3,15 +3,16 @@ ...@@ -3,15 +3,16 @@
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/usuario/Documentos/malleability_benchmark"
partition="P1" partition="P1"
exclude="c00,c01,c02" exclude="c00,c01,c02"
cores=20
# Runs in a given current directory all .ini files # Runs in a given current directory all .ini files with the aid of the RMS
# Parameter 1(Optional) - Amount of executions per file. Must be a positive number # Parameter 1(Optional) - Amount of executions per file. Must be a positive number
# Parameter 2(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
#====== Do not modify these values ======= #====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec" execDir="/Exec"
ResultsDir="/Results" ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
use_extrae=0 use_extrae=0
qty=1 qty=1
...@@ -20,35 +21,18 @@ then ...@@ -20,35 +21,18 @@ then
qty=$1 qty=$1
fi fi
limit_time=$((0))
if [ $# -ge 2 ] #Max time per execution in seconds
then
limit_time=$(($2 * $qty / 60 + 1))
fi
files="./*.ini" files="./*.ini"
internalIndex=$(echo $files | tr -cd ' ' | wc -c) internalIndex=$(echo $files | tr -cd ' ' | wc -c)
index=$((0)) index=$((0))
for config_file in $files for config_file in $files
do do
# FIXME Tener en cuenta que puede ser más de un resize node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
aux=$(grep "\[resize0\]" -n $config_file | cut -d ":" -f1)
ini=$(echo $aux | cut -d " " -f1)
fin=$(echo $aux | cut -d " " -f2)
diff=$(( fin - ini ))
numP1=$(head -$fin $config_file | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
aux=$(grep "\[resize1\]" -n $config_file | cut -d ":" -f1)
ini=$(echo $aux | cut -d " " -f1)
fin=$(echo $aux | cut -d " " -f2)
diff=$(( fin - ini ))
numP2=$(head -$fin $config_file | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
echo "------------------------------------------run np=$numP1"
if [ $numP1 -lt $numP2 ]
then
numP1=$numP2
fi
node_qty=$(($numP1 / $cores))
if [ $node_qty -eq 0 ]
then
node_qty=1
fi
outFileIndex=$(echo $config_file | sed s/[^0-9]//g) outFileIndex=$(echo $config_file | sed s/[^0-9]//g)
if [[ $outFileIndex ]]; then if [[ $outFileIndex ]]; then
...@@ -60,6 +44,6 @@ do ...@@ -60,6 +44,6 @@ do
#Execute test #Execute test
echo "Execute job $index with Nodes=$node_qty and config_file=$config_file" echo "Execute job $index with Nodes=$node_qty and config_file=$config_file"
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $index $qty sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $index $qty
done done
echo "End" echo "End"
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/usuario/Documentos/malleability_benchmark"
partition="P1" partition="P1"
exclude="c00,c01,c02" exclude="c00,c01,c02"
cores=20
# Executes a given configuration file with the aid of # Executes a given configuration file with the aid of
# the RMS Slurm. # the RMS Slurm.
...@@ -11,12 +10,14 @@ cores=20 ...@@ -11,12 +10,14 @@ cores=20
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer. # Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer. # Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
# Parameter 4(Optional): Use Extrae(1) or not(0). # Parameter 4(Optional): Use Extrae(1) or not(0).
# Parameter 5(Optional): Path where the output files should be saved. # Parameter 5(Optional): Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
# Parameter 6(Optional): Path where the output files should be saved.
#====== Do not modify these values ======= #====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec" execDir="/Exec"
ResultsDir="/Results" ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
if [ $# -lt 1 ] if [ $# -lt 1 ]
then then
...@@ -29,7 +30,8 @@ fi ...@@ -29,7 +30,8 @@ fi
#$2 == outFileIndex #$2 == outFileIndex
#$3 == Qty of repetitions #$3 == Qty of repetitions
#$4 == Use extrae NO(0) YES(1) #$4 == Use extrae NO(0) YES(1)
#$5 == Output path #$5 == Max time per execution(s)
#$6 == Output path
config_file=$1 config_file=$1
outFileIndex=0 outFileIndex=0
...@@ -48,36 +50,20 @@ if [ $# -ge 4 ] ...@@ -48,36 +50,20 @@ if [ $# -ge 4 ]
then then
use_extrae=$4 use_extrae=$4
fi fi
if [ $# -ge 5 ] limit_time=$((0))
if [ $# -ge 5 ] #Max time per execution in seconds
then then
output=$5 limit_time=$(($5 * $qty / 60 + 1))
fi fi
if [ $# -ge 6 ]
#1 - Obtain maximum number of processes for the run
max_numP=-1
total_groups=$(grep Total_Resizes $config_file | cut -d '=' -f2)
for ((j=0; j<total_groups; j++));
do
resize_info=$(grep "\[resize$j\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
#2 - Obtain needed nodes for the number of processes
node_qty=$(($max_numP / $cores))
if [ "$node_qty" -eq "0" ];
then then
node_qty=1 output=$6
fi fi
#3 - Run with the expected amount of nodes #Obtain amount of nodes neeeded
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex $qty node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
#Run with the expected amount of nodes
sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
if ! [ -z "$output" ] if ! [ -z "$output" ]
then then
......
#!/bin/bash #!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark" dir="/home/martini/malleability_benchmark"
cores=20
# Executes a given configuration file # Executes a given configuration file. This script can be called with Slurm commands to
# choose the desired user configuration.
# Parameter 1: Configuration file name for the emulation. # Parameter 1: Configuration file name for the emulation.
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer. # Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer. # Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
...@@ -49,7 +51,7 @@ then ...@@ -49,7 +51,7 @@ then
output=$5 output=$5
fi fi
bash $dir$execDir/generalRunCostum.sh $dir $config_file $use_extrae $outFileIndex $qty bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
if ! [ -z "$output" ] if ! [ -z "$output" ]
then then
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment