Skip to content

GitLab

Commit 6633cd95 authored by Iker Martín Álvarez's avatar Iker Martín Álvarez
Browse files

RMA functionality and refactor of many of the codes

parent 2f81e29c
Hide whitespace changes
Inline Side-by-side
Exec/generalRun.sh 100644 → 100755
View file @ 6633cd95
#!/bin/bash #!/bin/bash
#This script should only be called by others scripts, do not call it directly
#SBATCH --exclude=c02,c01,c00 #SBATCH --exclude=c02,c01,c00
#SBATCH -p P1 #SBATCH -p P1
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters with the aid of the RMS Slurm.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
echo "START TEST" echo "START TEST"
#$1 == baseDir #$1 == baseDir
#$2 == configFile #$2 == cores
#$3 == use_extrae #$3 == configFile
#$4 == outFileIndex #$4 == use_extrae
#$5 == outFileIndex
#$6 == qty
echo $@ echo $@
if [ $# -lt 3 ] if [ $# -lt 4 ]
then then
echo "Internal ERROR generalRun.sh - Not enough arguments were given" echo "Internal ERROR generalRun.sh - Not enough arguments were given"
exit -1 exit -1
fi fi
#READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1 dir=$1
configFile=$2 cores=$2
use_extrae=$3 configFile=$3
outFileIndex=$4 use_extrae=$4
outFileIndex=$5
qty=1
if [ $# -ge 5 ]
then
qty=$6
fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1) nodelist=$SLURM_JOB_NODELIST
read -r ini fin <<<$(echo $aux) nodes=$SLURM_JOB_NUM_NODES
diff=$(( fin - ini )) if [ -z "$nodelist" ];
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2) then
echo "Internal ERROR in generalRun.sh - Nodelist not provided"
exit -1
fi
if [ -z "$nodes" ];
then
nodes=1
fi
echo "Nodes=$SLURM_JOB_NODELIST" numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
#EXECUTE RUN
echo "Nodes=$nodelist"
if [ $use_extrae -ne 1 ] if [ $use_extrae -ne 1 ]
then then
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $SLURM_JOB_NODELIST $SLURM_JOB_NUM_NODES for ((i=0; i<qty; i++))
do
mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
else else
srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $SLURM_JOB_NODELIST $SLURM_JOB_NUM_NODES cp $dir$execDir/Extrae/extrae.xml .
cp $dir$execDir/Extrae/trace.sh .
cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++))
do
#FIXME Extrae not tested keeping in mind the initial nodelist - Could have some errors
srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
fi fi
echo "END TEST" echo "END TEST"
sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
sed -i 's/Abort(-100)/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
Exec/generalRunCostum.sh 0 → 100755
View file @ 6633cd95
#!/bin/bash
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
echo "START TEST"
#$1 == baseDir
#$2 == cores
#$3 == configFile
#$4 == use_extrae
#$5 == outFileIndex
#$6 == qty
echo $@
if [ $# -lt 3 ]
then
echo "Internal ERROR generalRunCostum.sh - Not enough arguments were given"
exit -1
fi
#READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1
cores=$2
configFile=$3
use_extrae=0
outFileIndex=0
qty=1
if [ $# -ge 4 ]
then
use_extrae=$4
fi
if [ $# -ge 5 ]
then
outFileIndex=$5
fi
if [ $# -ge 6 ]
then
qty=$6
fi
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
nodelist=$SLURM_JOB_NODELIST
nodes=$SLURM_JOB_NUM_NODES
if [ -z "$nodelist" ];
then
nodelist="localhost"
initial_nodelist="localhost"
else
initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
fi
if [ -z "$nodes" ];
then
nodes=1
fi
#EXECUTE RUN
echo "Nodes=$nodelist"
if [ $use_extrae -ne 1 ]
then
for ((i=0; i<qty; i++))
do
mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
else
cp $dir$execDir/Extrae/extrae.xml .
cp $dir$execDir/Extrae/trace.sh .
cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++))
do
mpirun -hosts $initial_nodelist -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
fi
echo "END TEST"
Exec/multipleRuns.sh 0 → 100755
View file @ 6633cd95
#!/bin/bash
dir="/home/martini/malleability_benchmark/"
# Creates a directory with all possible and valid combinations of configuration files
# that can be created from a given complex configuration file.
# Parameter 1: Complex configuration file name.
# Parameter 2: Common output name of the output configuration files. It will be appended an index to each of them.
#====== Do not modify these values =======
codeDir="Codes/"
execDir="Exec/"
ResultsDir="Results/"
complex_file=$1
output_name=$2
python3 $dir$execDir/PythonCodes/read_multiple.py $complex_file $output_name
echo "END TEST"
Exec/runAll.sh 100644 → 100755
View file @ 6633cd95
#!/bin/bash
dir="/home/martini/malleability_benchmark" dir="/home/martini/malleability_benchmark"
partition="P1" partition="P1"
exclude="c00,c01,c02" exclude="c00,c01,c02"
procs=(2 10 20 40 80 120 160)
cores=20
# Runs in a given current directory all .ini files with the aid of the RMS
# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
# Parameter 2(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
#====== Do not modify these values ======= #====== Do not modify these values =======
codeDir="/Codes/build" codeDir="/Codes/build"
execDir="/Exec" execDir="/Exec"
ResultsDir="/Results" ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
use_extrae=0 use_extrae=0
qty=1 qty=1
outFileIndex=$2
if [ $# -ge 1 ] if [ $# -ge 1 ]
then then
qty=$1 qty=$1
fi fi
for proc in "${procs[@]}" limit_time=$((0))
if [ $# -ge 2 ] #Max time per execution in seconds
then
limit_time=$(($2 * $qty / 60 + 1))
fi
files="./*.ini"
internalIndex=$(echo $files | tr -cd ' ' | wc -c)
index=$((0))
for config_file in $files
do do
echo "------------------------------------------run np=$proc" node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
node_qty=$(($proc / $cores))
if [ $node_qty -eq 0 ] outFileIndex=$(echo $config_file | sed s/[^0-9]//g)
then if [[ $outFileIndex ]]; then
node_qty=1 index=$outFileIndex
else
index=$internalIndex
((internalIndex++))
fi fi
config_file="test$proc"".ini" #Execute test
for ((i=0; i<qty; i++)) echo "Execute job $index with Nodes=$node_qty and config_file=$config_file"
do sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $index $qty
#Execute test
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex
done
done done
echo "End" echo "End"
Exec/runTrace.sh deleted 100644 → 0
View file @ 2f81e29c
dir="/home/martini/malleability_benchmark"
partition="P1"
exclude="c00,c01,c02"
procs=(2 10 20 40 80 120 160)
cores=20
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
use_extrae=1
#Extrae config
qty=1
outFileIndex=$2
if [ $# -ge 1 ]
then
qty=$1
fi
for proc in "${procs[@]}"
do
echo "------------------------------------------run np=$proc"
node_qty=$(($proc / $cores))
if [ $node_qty -eq 0 ]
then
node_qty=1
fi
config_file="test$proc"".ini"
upper_dir="Proc$proc"
mkdir $upper_dir
cd $upper_dir
for ((i=0; i<qty; i++))
do
# Move data to new directory
lower_dir="Run$i"
mkdir $lower_dir
cd $lower_dir
cp $dir$execDir/Extrae/extrae.xml .
cp $dir$execDir/Extrae/trace.sh .
cp ../../$config_file .
#Execute test
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex
cd ..
done
cd ..
done
echo "End"
Exec/singleRun.sh 100644 → 100755
View file @ 6633cd95
#!/bin/bash #!/bin/bash
#SBATCH --exclude=c02,c01,c00
#SBATCH -p P1
dir="/home/martini/malleability_benchmark" dir="/home/martini/malleability_benchmark"
codeDir="/Codes/build" partition="P1"
exclude="c00,c01,c02"
nodelist=$SLURM_JOB_NODELIST # Executes a given configuration file with the aid of
nodes=$SLURM_JOB_NUM_NODES # the RMS Slurm.
# Parameter 1: Configuration file name for the emulation.
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
# Parameter 4(Optional): Use Extrae(1) or not(0).
# Parameter 5(Optional): Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
# Parameter 6(Optional): Path where the output files should be saved.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
if [ $# -lt 1 ] if [ $# -lt 1 ]
then then
echo "Not enough arguments. Usage:" echo "Not enough arguments. Usage:"
echo "singleRun.sh config.ini [outFileIndex] [Qty] [Output path]" echo "bash singleRun.sh config.ini [outFileIndex] [Qty] [Use extrae] [Output path]"
exit 1 exit 1
fi fi
echo "START TEST"
#$1 == configFile #$1 == configFile
#$2 == outFileIndex #$2 == outFileIndex
#$3 == Qty of repetitions #$3 == Qty of repetitions
#$4 == Output path #$4 == Use extrae NO(0) YES(1)
#$5 == Max time per execution(s)
#$6 == Output path
configFile=$1 config_file=$1
outFileIndex=$2 outFileIndex=0
qty=1 qty=1
use_extrae=0
if [ $# -gt 2 ] if [ $# -ge 2 ]
then
outFileIndex=$2
fi
if [ $# -ge 3 ]
then then
qty=$3 qty=$3
if [ $# -gt 3 ] fi
then if [ $# -ge 4 ]
output=$4 then
fi use_extrae=$4
fi
limit_time=$((0))
if [ $# -ge 5 ] #Max time per execution in seconds
then
limit_time=$(($5 * $qty / 60 + 1))
fi
if [ $# -ge 6 ]
then
output=$6
fi fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1) #Obtain amount of nodes neeeded
read -r ini fin <<<$(echo $aux) node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
diff=$(( fin - ini )) #Run with the expected amount of nodes
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2) sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
echo "Nodes=$SLURM_JOB_NODELIST"
for ((i=0; i<qty; i++))
do
echo "Iter $i -- numP=$numP"
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
echo "END TEST"
sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
if [ $# -gt 3 ] if ! [ -z "$output" ]
then then
mkdir -p $output
echo "Moving data to $output\nMoved files:" echo "Moving data to $output\nMoved files:"
ls R${outFileIndex}_G* ls R${outFileIndex}_G*
mv R${outFileIndex}_G* $output mv R${outFileIndex}_G* $output
if [ "$use_extrae" -eq 1 ]
then
mv a.out.* $output
mv TRACE* $output
mv set-0/ $output
fi
fi fi
Exec/singleRunCostum.sh 0 → 100755
View file @ 6633cd95
#!/bin/bash
dir="/home/martini/malleability_benchmark"
cores=20
# Executes a given configuration file. This script can be called with Slurm commands to
# choose the desired user configuration.
# Parameter 1: Configuration file name for the emulation.
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
# Parameter 4(Optional): Use Extrae(1) or not(0).
# Parameter 5(Optional): Path where the output files should be saved.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
if [ $# -lt 1 ]
then
echo "Not enough arguments. Usage:"
echo "singleRunCostum.sh config.ini [outFileIndex] [Qty] [Use Extrae] [Output path]"
exit 1
fi
#$1 == configFile
#$2 == outFileIndex
#$3 == Qty of repetitions
#$4 == Use extrae NO(0) YES(1)
#$5 == Output path
config_file=$1
outFileIndex=0
qty=1
use_extrae=0
if [ $# -ge 2 ]
then
outFileIndex=$2
fi
if [ $# -ge 3 ]
then
qty=$3
fi
if [ $# -ge 4 ]
then
use_extrae=$4
fi
if [ $# -ge 5 ]
then
output=$5
fi
bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
if ! [ -z "$output" ]
then
mkdir -p $output
echo "Moving data to $output\nMoved files:"
ls R${outFileIndex}_G*
mv R${outFileIndex}_G* $output
if [ "$use_extrae" -eq 1 ]
then
mv a.out.* $output
mv TRACE* $output
mv set-0/ $output
fi
fi
Manual.pdf 0 → 100644
View file @ 6633cd95
File added
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment