Merge branch 'RMA-Distributions' into 'dev'

RMA functionality and refactor of many of the codes

See merge request martini/malleability_benchmark!4

Exec/generalRun.sh

 #!/bin/bash
-#This script should only be called by others scripts, do not call it directly
 
 #SBATCH --exclude=c02,c01,c00
 #SBATCH -p P1
+
+# !!!!This script should only be called by others scripts, do not call it directly!!!
+# Runs a given configuration file with the indicated parameters with the aid of the RMS Slurm.
+# Parameter 1 - Base directory of the malleability benchmark
+# Parameter 2 - Number of cores in a single machine
+# Parameter 3 - Configuration file name for the emulation.
+# Parameter 4 - Use Extrae(1) or not(0).
+# Parameter 5 - Index to use for the output files. Must be a positive integer.
+# Parameter 6 - Amount of executions per file. Must be a positive number.
+#====== Do not modify these values =======
+
 codeDir="/Codes/build"
+execDir="/Exec"
+ResultsDir="/Results"
 
 echo "START TEST"
 
 #$1 == baseDir
-#$2 == configFile
-#$3 == use_extrae
-#$4 == outFileIndex
+#$2 == cores
+#$3 == configFile
+#$4 == use_extrae
+#$5 == outFileIndex
+#$6 == qty
 
 echo $@
-if [ $# -lt 3 ]
+if [ $# -lt 4 ]
 then
   echo "Internal ERROR generalRun.sh - Not enough arguments were given"
   exit -1
 fi
 
+#READ PARAMETERS AND ENSURE CORRECTNESS
 dir=$1
-configFile=$2
-use_extrae=$3
-outFileIndex=$4
+cores=$2
+configFile=$3
+use_extrae=$4
+outFileIndex=$5
+qty=1
+if [ $# -ge 5 ]
+then
+  qty=$6
+fi
 
-aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1)
-read -r ini fin <<<$(echo $aux)
-diff=$(( fin - ini ))
-numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
+nodelist=$SLURM_JOB_NODELIST
+nodes=$SLURM_JOB_NUM_NODES
+if [ -z "$nodelist" ];
+then
+  echo "Internal ERROR in generalRun.sh - Nodelist not provided"
+  exit -1
+fi
+if [ -z "$nodes" ];
+then
+  nodes=1
+fi
 
-echo "Nodes=$SLURM_JOB_NODELIST"
+numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
+initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
+
+#EXECUTE RUN
+echo "Nodes=$nodelist"
 if [ $use_extrae -ne 1 ]
 then
-  mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $SLURM_JOB_NODELIST $SLURM_JOB_NUM_NODES 
+  for ((i=0; i<qty; i++))
+  do
+    mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes 
+  done
 else
-  srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $SLURM_JOB_NODELIST $SLURM_JOB_NUM_NODES
+  cp $dir$execDir/Extrae/extrae.xml .
+  cp $dir$execDir/Extrae/trace.sh .
+  cp $dir$execDir/Extrae/trace_worker.sh .
+  for ((i=0; i<qty; i++))
+  do
+    #FIXME Extrae not tested keeping in mind the initial nodelist - Could have some errors
+    srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
+  done
 fi
 
-
 echo "END TEST"
 sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
+sed -i 's/Abort(-100)/shrink cleaning/g' slurm-$SLURM_JOB_ID.out

Exec/generalRunCostum.sh

+#!/bin/bash
+
+# !!!!This script should only be called by others scripts, do not call it directly!!!
+# Runs a given configuration file with the indicated parameters.
+# Parameter 1 - Base directory of the malleability benchmark
+# Parameter 2 - Number of cores in a single machine
+# Parameter 3 - Configuration file name for the emulation.
+# Parameter 4 - Use Extrae(1) or not(0).
+# Parameter 5 - Index to use for the output files. Must be a positive integer.
+# Parameter 6 - Amount of executions per file. Must be a positive number.
+#====== Do not modify these values =======
+
+codeDir="/Codes/build"
+execDir="/Exec"
+ResultsDir="/Results"
+
+echo "START TEST"
+
+#$1 == baseDir
+#$2 == cores
+#$3 == configFile
+#$4 == use_extrae
+#$5 == outFileIndex
+#$6 == qty
+
+echo $@
+if [ $# -lt 3 ]
+then
+  echo "Internal ERROR generalRunCostum.sh - Not enough arguments were given"
+  exit -1
+fi
+
+#READ PARAMETERS AND ENSURE CORRECTNESS
+dir=$1
+cores=$2
+configFile=$3
+use_extrae=0
+outFileIndex=0
+qty=1
+
+if [ $# -ge 4 ]
+then
+  use_extrae=$4
+fi
+
+if [ $# -ge 5 ]
+then
+  outFileIndex=$5
+fi
+
+if [ $# -ge 6 ]
+then
+  qty=$6
+fi
+
+numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
+nodelist=$SLURM_JOB_NODELIST
+nodes=$SLURM_JOB_NUM_NODES
+if [ -z "$nodelist" ];
+then
+  nodelist="localhost"
+  initial_nodelist="localhost"
+else
+  initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
+fi
+if [ -z "$nodes" ];
+then
+  nodes=1
+fi
+
+#EXECUTE RUN
+echo "Nodes=$nodelist"
+if [ $use_extrae -ne 1 ]
+then
+  for ((i=0; i<qty; i++))
+  do
+    mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes 
+  done
+else
+  cp $dir$execDir/Extrae/extrae.xml .
+  cp $dir$execDir/Extrae/trace.sh .
+  cp $dir$execDir/Extrae/trace_worker.sh .
+  for ((i=0; i<qty; i++))
+  do
+    mpirun -hosts $initial_nodelist -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes 
+  done
+fi
+
+echo "END TEST"

Exec/multipleRuns.sh

+#!/bin/bash
+
+dir="/home/martini/malleability_benchmark/"
+
+# Creates a directory with all possible and valid combinations of configuration files
+#   that can be created from a given complex configuration file.
+# Parameter 1: Complex configuration file name.
+# Parameter 2: Common output name of the output configuration files. It will be appended an index to each of them.
+#====== Do not modify these values =======
+
+codeDir="Codes/"
+execDir="Exec/"
+ResultsDir="Results/"
+
+complex_file=$1
+output_name=$2
+
+python3 $dir$execDir/PythonCodes/read_multiple.py $complex_file $output_name
+
+echo "END TEST"
+

Exec/runAll.sh

+#!/bin/bash
+
 dir="/home/martini/malleability_benchmark"
 partition="P1"
 exclude="c00,c01,c02"
-procs=(2 10 20 40 80 120 160)
-cores=20
-
 
+# Runs in a given current directory all .ini files with the aid of the RMS
+# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
+# Parameter 2(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
 #====== Do not modify these values =======
 
 codeDir="/Codes/build"
 execDir="/Exec"
 ResultsDir="/Results"
+cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
 use_extrae=0
 
 qty=1
-outFileIndex=$2
-
 if [ $# -ge 1 ]
 then
   qty=$1
 fi
 
-for proc in "${procs[@]}"
+limit_time=$((0))
+if [ $# -ge 2 ] #Max time per execution in seconds
+then
+  limit_time=$(($2 * $qty / 60 + 1))
+fi
+
+files="./*.ini"
+internalIndex=$(echo $files | tr -cd ' ' | wc -c)
+index=$((0))
+for config_file in $files
 do
-  echo "------------------------------------------run np=$proc"
-  node_qty=$(($proc / $cores))
-  if [ $node_qty -eq 0 ]
-  then
-    node_qty=1
+  node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
+
+  outFileIndex=$(echo $config_file | sed s/[^0-9]//g)
+  if [[ $outFileIndex ]]; then 
+    index=$outFileIndex
+  else 
+    index=$internalIndex
+    ((internalIndex++))
   fi
 
-  config_file="test$proc"".ini"
-  for ((i=0; i<qty; i++))
-  do
   #Execute test
-    sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex
-  done
+  echo "Execute job $index with Nodes=$node_qty and config_file=$config_file"
+  sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $index $qty
 done
 echo "End"

Exec/runTrace.sh

-dir="/home/martini/malleability_benchmark"
-partition="P1"
-exclude="c00,c01,c02"
-procs=(2 10 20 40 80 120 160)
-cores=20
-
-
-#====== Do not modify these values =======
-
-codeDir="/Codes/build"
-execDir="/Exec"
-ResultsDir="/Results"
-use_extrae=1
-
-#Extrae config
-qty=1
-outFileIndex=$2
-
-if [ $# -ge 1 ]
-then
-  qty=$1
-fi
-
-for proc in "${procs[@]}"
-do
-  echo "------------------------------------------run np=$proc"
-
-  node_qty=$(($proc / $cores))
-  if [ $node_qty -eq 0 ]
-  then
-    node_qty=1
-  fi
-
-  config_file="test$proc"".ini"
-  upper_dir="Proc$proc"
-  mkdir $upper_dir
-  cd $upper_dir
-  for ((i=0; i<qty; i++))
-  do
-    # Move data to new directory
-    lower_dir="Run$i"
-    mkdir $lower_dir
-    cd $lower_dir
-    cp $dir$execDir/Extrae/extrae.xml .
-    cp $dir$execDir/Extrae/trace.sh .
-    cp ../../$config_file .
-
-    #Execute test
-    sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex
-    cd ..
-  done
-  cd ..
-
-done
-echo "End"

Exec/singleRun.sh

 #!/bin/bash
 
-#SBATCH --exclude=c02,c01,c00
-#SBATCH -p P1
-
 dir="/home/martini/malleability_benchmark"
-codeDir="/Codes/build"
+partition="P1"
+exclude="c00,c01,c02"
 
-nodelist=$SLURM_JOB_NODELIST
-nodes=$SLURM_JOB_NUM_NODES
+# Executes a given configuration file with the aid of
+# the RMS Slurm.
+# Parameter 1: Configuration file name for the emulation.
+# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
+# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
+# Parameter 4(Optional): Use Extrae(1) or not(0).
+# Parameter 5(Optional): Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
+# Parameter 6(Optional): Path where the output files should be saved. 
+#====== Do not modify these values =======
+
+codeDir="/Codes/build"
+execDir="/Exec"
+ResultsDir="/Results"
+cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
 
 if [ $# -lt 1 ]
 then
   echo "Not enough arguments. Usage:"
-  echo "singleRun.sh config.ini [outFileIndex] [Qty] [Output path]"
+  echo "bash singleRun.sh config.ini [outFileIndex] [Qty] [Use extrae] [Output path]"
   exit 1
 fi
 
-echo "START TEST"
-
 #$1 == configFile
 #$2 == outFileIndex
 #$3 == Qty of repetitions
-#$4 == Output path
+#$4 == Use extrae NO(0) YES(1)
+#$5 == Max time per execution(s)
+#$6 == Output path
 
-configFile=$1
-outFileIndex=$2
+config_file=$1
+outFileIndex=0
 qty=1
+use_extrae=0
 
-if [ $# -gt 2 ]
+if [ $# -ge 2 ]
+then
+  outFileIndex=$2
+fi
+if [ $# -ge 3 ]
 then
   qty=$3
-  if [ $# -gt 3 ]
-  then
-    output=$4
-  fi
+fi
+if [ $# -ge 4 ]
+then
+  use_extrae=$4
+fi
+limit_time=$((0))
+if [ $# -ge 5 ] #Max time per execution in seconds
+then
+  limit_time=$(($5 * $qty / 60 + 1))
+fi
+if [ $# -ge 6 ]
+then
+  output=$6
 fi
 
-aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1)
-read -r ini fin <<<$(echo $aux)
-diff=$(( fin - ini ))
-numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
-
-echo "Nodes=$SLURM_JOB_NODELIST"
-for ((i=0; i<qty; i++))
-do
-  echo "Iter $i -- numP=$numP"
-  mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes 
-done
-
-echo "END TEST"
-sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
+#Obtain amount of nodes neeeded
+node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
+#Run with the expected amount of nodes
+sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
 
-if [ $# -gt 3 ]
+if ! [ -z "$output" ]
 then
+  mkdir -p $output
   echo "Moving data to $output\nMoved files:"
   ls R${outFileIndex}_G*
   mv R${outFileIndex}_G* $output
+  if [ "$use_extrae" -eq 1 ]
+  then
+    mv a.out.* $output
+    mv TRACE* $output
+    mv set-0/ $output
+  fi
 fi

Exec/singleRunCostum.sh

+#!/bin/bash
+
+dir="/home/martini/malleability_benchmark"
+cores=20
+
+# Executes a given configuration file. This script can be called with Slurm commands to 
+#   choose the desired user configuration.
+# Parameter 1: Configuration file name for the emulation.
+# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
+# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
+# Parameter 4(Optional): Use Extrae(1) or not(0).
+# Parameter 5(Optional): Path where the output files should be saved. 
+#====== Do not modify these values =======
+
+codeDir="/Codes/build"
+execDir="/Exec"
+ResultsDir="/Results"
+
+if [ $# -lt 1 ]
+then
+  echo "Not enough arguments. Usage:"
+  echo "singleRunCostum.sh config.ini [outFileIndex] [Qty] [Use Extrae] [Output path]"
+  exit 1
+fi
+
+#$1 == configFile
+#$2 == outFileIndex
+#$3 == Qty of repetitions
+#$4 == Use extrae NO(0) YES(1)
+#$5 == Output path
+
+config_file=$1
+outFileIndex=0
+qty=1
+use_extrae=0
+
+if [ $# -ge 2 ]
+then
+  outFileIndex=$2
+fi
+if [ $# -ge 3 ]
+then
+  qty=$3
+fi
+if [ $# -ge 4 ]
+then
+  use_extrae=$4
+fi
+if [ $# -ge 5 ]
+then
+  output=$5
+fi
+
+bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
+
+if ! [ -z "$output" ]
+then
+  mkdir -p $output
+  echo "Moving data to $output\nMoved files:"
+  ls R${outFileIndex}_G*
+  mv R${outFileIndex}_G* $output
+  if [ "$use_extrae" -eq 1 ]
+  then
+    mv a.out.* $output
+    mv TRACE* $output
+    mv set-0/ $output
+  fi
+fi