WIP 2/2. Added option for spawn to allow wrappers around targets processes to...

WIP 2/2. Added option for spawn to allow wrappers around targets processes to use Extrae o Valgrind. Is not fully automatic, as the code has to make a MaM call to allow it correctly.

Codes/runBase.sh

@@ -20,7 +20,7 @@ then
   outIndex=$2
 fi
 
-echo "MPICH"
+echo "MPICH provider=$FI_PROVIDER"
 mpirun --version
 numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
 initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)

Codes/runValgrind.sh

@@ -4,23 +4,39 @@
 #SBATCH -N 1
 #SBATCH --exclude=c01,c00,c02
 
-scriptDir="$(dirname "$0")"
-source $scriptDir/build/config.txt
+source build/config.txt
 codeDir="/Codes"
 execDir="/Exec"
+cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
 
-nodelist="localhost"
-nodes=1
-configFile=$1
-outIndex=$2
+#configFile=$1
+#outIndex=$2
 
-echo "MPICH"
+#echo "MPICH"
 #module load mpich-3.4.1-noucx
 #export HYDRA_DEBUG=1
 
+#mpirun --version
+#numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
+#mpirun -np $numP valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --trace-children=yes --log-file=nc.vg.%p $dir$codeDir/build/a.out $configFile $outIndex
+
+nodelist=$SLURM_JOB_NODELIST
+nodes=$SLURM_JOB_NUM_NODES
+configFile=$1
+
+outIndex=0
+if [ $# -ge 2 ]
+then
+  outIndex=$2
+fi
+
+echo "MPICH provider=$FI_PROVIDER"
 mpirun --version
 numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
-mpirun -np $numP valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --trace-children=yes --log-file=nc.vg.%p $dir$codeDir/build/a.out $configFile $outIndex
+initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
+echo $initial_nodelist
+echo "Test PreRUN $numP $nodelist"
+mpirun -hosts $initial_nodelist -np $numP valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --trace-children=yes --log-file=vg.sp.%p.$SLURM_JOB_ID $dir$codeDir/build/a.out $configFile $outIndex 
 
 echo "END RUN"
 sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out

Exec/Valgrind/worker_valgrind.sh

+#!/bin/bash
+scriptDir="$(dirname "$0")"
+source $scriptDir/../../Codes/build/config.txt
+codeDir="/Codes/build"
+
+valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --log-file=vg.tp.%p $dir$codeDir/./a.out

Exec/generalRun.sh

@@ -8,7 +8,7 @@
 # Parameter 1 - Base directory of the malleability benchmark
 # Parameter 2 - Number of cores in a single machine
 # Parameter 3 - Configuration file name for the emulation.
-# Parameter 4 - Use Extrae(1) or not(0).
+# Parameter 4 - Use Valgrind(1), Extrae(2) or nothing(0).
 # Parameter 5 - Index to use for the output files. Must be a positive integer.
 # Parameter 6 - Amount of executions per file. Must be a positive number.
 #====== Do not modify these values =======
@@ -22,7 +22,7 @@ echo "START TEST"
 #$1 == baseDir
 #$2 == cores
 #$3 == configFile
-#$4 == use_extrae
+#$4 == use_external
 #$5 == outFileIndex
 #$6 == qty
 
@@ -37,7 +37,7 @@ fi
 dir=$1
 cores=$2
 configFile=$3
-use_extrae=$4
+use_external=$4
 outFileIndex=$5
 qty=1
 if [ $# -ge 5 ]
@@ -57,7 +57,7 @@ initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP
 
 #EXECUTE RUN
 echo "Nodes=$nodelist"
-if [ $use_extrae -ne 1 ]
+if [ $use_external -eq 0 ] #NORMAL
 then
   for ((i=0; i<qty; i++))
   do
@@ -65,10 +65,19 @@ then
     mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex
     echo "Run $i ends"
   done
-else
+elif [ $use_external -eq 1 ] #VALGRIND
+then
+  cp $dir$execDir/Valgrind/worker_valgrind.sh .
+  for ((i=0; i<qty; i++))
+  do
+    echo "Run $i starts"
+    mpirun -hosts $initial_nodelist -np $numP valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --trace-children=yes --log-file=vg.sp.%p.$SLURM_JOB_ID.$i $dir$codeDir/a.out $configFile $outIndex 
+    echo "Run $i ends"
+  done
+else #EXTRAE
   cp $dir$execDir/Extrae/extrae.xml .
   cp $dir$execDir/Extrae/trace.sh .
-  cp $dir$execDir/Extrae/trace_worker.sh .
+  cp $dir$execDir/Extrae/worker_extrae.sh .
   for ((i=0; i<qty; i++))
   do
     #FIXME Extrae not tested keeping in mind the initial nodelist - Could have some errors

Exec/generalRunCostum.sh

@@ -19,7 +19,7 @@ echo "START TEST"
 #$1 == baseDir
 #$2 == cores
 #$3 == configFile
-#$4 == use_extrae
+#$4 == use_external
 #$5 == outFileIndex
 #$6 == qty
 
@@ -34,13 +34,13 @@ fi
 dir=$1
 cores=$2
 configFile=$3
-use_extrae=0
+use_external=0
 outFileIndex=0
 qty=1
 
 if [ $# -ge 4 ]
 then
-  use_extrae=$4
+  use_external=$4
 fi
 
 if [ $# -ge 5 ]
@@ -65,16 +65,27 @@ fi
 
 #EXECUTE RUN
 echo "Nodes=$nodelist"
-if [ $use_extrae -ne 1 ]
+if [ $use_external -eq 0 ]
 then
   for ((i=0; i<qty; i++))
   do
+    echo "Run $i starts"
     mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex
+    echo "Run $i ends"
+  done
+elif [ $use_external -eq 1 ] #VALGRIND
+then
+  cp $dir$execDir/Valgrind/worker_valgrind.sh .
+  for ((i=0; i<qty; i++))
+  do
+    echo "Run $i starts"
+    mpirun -hosts $initial_nodelist -np $numP valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --trace-children=yes --log-file=vg.sp.%p.$SLURM_JOB_ID.$i $dir$codeDir/a.out $configFile $outIndex 
+    echo "Run $i ends"
   done
 else
   cp $dir$execDir/Extrae/extrae.xml .
   cp $dir$execDir/Extrae/trace.sh .
-  cp $dir$execDir/Extrae/trace_worker.sh .
+  cp $dir$execDir/Extrae/worker_extrae.sh .
   for ((i=0; i<qty; i++))
   do
     mpirun -hosts $initial_nodelist -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex 

Exec/singleRun.sh

@@ -8,7 +8,7 @@ exclude="c00,c01,c02"
 # Parameter 1: Configuration file name for the emulation.
 # Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
 # Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
-# Parameter 4(Optional): Use Extrae(1) or not(0).
+# Parameter 4(Optional): Use Valgrind(1), Extrae(2) or nothing(0).
 # Parameter 5(Optional): Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
 # Parameter 6(Optional): Path where the output files should be saved. 
 #====== Do not modify these values =======
@@ -30,14 +30,14 @@ fi
 #$1 == configFile
 #$2 == outFileIndex
 #$3 == Qty of repetitions
-#$4 == Use extrae NO(0) YES(1)
+#$4 == Use external NO(0) Valgrind(1), Extrae(2)
 #$5 == Max time per execution(s)
 #$6 == Output path
 
 config_file=$1
 outFileIndex=0
 qty=1
-use_extrae=0
+use_external=0
 
 if [ $# -ge 2 ]
 then
@@ -49,7 +49,7 @@ then
 fi
 if [ $# -ge 4 ]
 then
-  use_extrae=$4
+  use_external=$4
 fi
 limit_time=$((0))
 if [ $# -ge 5 ] #Max time per execution in seconds
@@ -64,7 +64,7 @@ fi
 #Obtain amount of nodes neeeded
 node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
 #Run with the expected amount of nodes
-sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
+sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_external $outFileIndex $qty
 
 if ! [ -z "$output" ]
 then
@@ -72,10 +72,13 @@ then
   echo "Moving data to $output\nMoved files:"
   ls R${outFileIndex}_G*
   mv R${outFileIndex}_G* $output
-  if [ "$use_extrae" -eq 1 ]
+  if [ "$use_external" -eq 2 ] # Extrae additional output
   then
     mv a.out.* $output
     mv TRACE* $output
     mv set-0/ $output
+  elif [ "$use_external" -eq 1 ] # Valgrind additional output
+  then
+    mv vg.* $output
   fi
 fi

Exec/singleRunCostum.sh

@@ -7,7 +7,7 @@ cores=20
 # Parameter 1: Configuration file name for the emulation.
 # Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
 # Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
-# Parameter 4(Optional): Use Extrae(1) or not(0).
+# Parameter 4(Optional): Use Valgrind(1), Extrae(2) or nothing(0).
 # Parameter 5(Optional): Path where the output files should be saved. 
 #====== Do not modify these values =======
 
@@ -27,13 +27,13 @@ fi
 #$1 == configFile
 #$2 == outFileIndex
 #$3 == Qty of repetitions
-#$4 == Use extrae NO(0) YES(1)
+#$4 == Use external NO(0) Valgrind(1), Extrae(2)
 #$5 == Output path
 
 config_file=$1
 outFileIndex=0
 qty=1
-use_extrae=0
+use_external=0
 
 if [ $# -ge 2 ]
 then
@@ -45,14 +45,14 @@ then
 fi
 if [ $# -ge 4 ]
 then
-  use_extrae=$4
+  use_external=$4
 fi
 if [ $# -ge 5 ]
 then
   output=$5
 fi
 
-bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
+bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_external $outFileIndex $qty
 
 if ! [ -z "$output" ]
 then
@@ -60,10 +60,13 @@ then
   echo "Moving data to $output\nMoved files:"
   ls R${outFileIndex}_G*
   mv R${outFileIndex}_G* $output
-  if [ "$use_extrae" -eq 1 ]
+  if [ "$use_external" -eq 2 ] # Extrae additional output
   then
     mv a.out.* $output
     mv TRACE* $output
     mv set-0/ $output
+  elif [ "$use_external" -eq 1 ] # Valgrind additional output
+  then
+    mv vg.* $output
   fi
 fi