Commit 412646ab authored by Iker Martín Álvarez's avatar Iker Martín Álvarez
Browse files

Update Extrae-Executions with current code

parent 88a1e68f
#!/bin/bash
# Obtains the number of total cores in an homogenous partition
# Parameter 1 - Partition to use
#====== Do not modify these values =======
partition=$1
hostlist=$(sinfo -hs --partition $partition | sed 's/ */:/g' | cut -d ':' -f5)
basic_node=$(scontrol show hostname $hostlist | paste -d, -s | cut -d ',' -f1)
cores=$(scontrol show node $basic_node | grep CPUTot | cut -d '=' -f3 | cut -d ' ' -f1)
echo "$cores"
#!/bin/bash
# Obtains for a given configuration file how many nodes will be needed
# Parameter 1 - Configuration file name for the emulation.
# Parameter 2 - Base directory of the malleability benchmark
# Parameter 3 - Number of cores in the machines. The machines must be homogenous. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
if [ "$#" -lt "3" ]
then
echo "Not enough arguments"
echo "Usage -> bash getMaxNodesNeeded.sh Configuration.ini BaseDirectory NumCores"
exit -1
fi
config_file=$1
dir=$2
cores=$3
max_numP=-1
total_resizes=$(grep Total_Resizes $config_file | cut -d '=' -f2)
total_groups=$(($total_resizes + 1))
for ((j=0; j<total_groups; j++));
do
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $config_file $j)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
node_qty=$(($max_numP / $cores))
if [ $node_qty -eq 0 ]
then
node_qty=1
fi
echo $node_qty
#!/bin/bash
dir="/home/martini/malleability_benchmark" #FIXME Obtain from another way
# Runs in a given current directory all .ini files
# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
config_file=$1
group_index=$2
resize_info=$(grep "\[resize$group_index\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
echo $numP
#!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark"
cores=20
dir="/home/martini/malleability_benchmark/"
partition="P1"
# Checks if all the runs in the current working directory performed under a
# Slurm manager have been performed correctly and if some runs can be corrected
# they are launched again
# Parameter 1 - Maximum index of the runs
# Parameter 2 - Amount of repetitions per index/run
# Parameter 3 - Total stages in all runs. #FIXME The amount of stages must be equal across all the runs, must be modified in the future.
# Parameter 4 - Total groups of processes in all runs #FIXME The amount of groups must be equal across all the runs, must be modified in the future.
# Parameter 5 - Maximum valid iteration time across all runs. If an iteration time
# Parameter 1 - Common name of the configuration files
# Parameter 2 - Maximum index of the runs
# Parameter 3 - Amount of repetitions per index/run
# Parameter 4 - Total stages in all runs. #FIXME The amount of stages must be equal across all the runs, must be modified in the future.
# Parameter 5 - Total groups of processes in all runs #FIXME The amount of groups must be equal across all the runs, must be modified in the future.
# Parameter 6 - Maximum valid iteration time across all runs. If an iteration time
# is higher, that particular repetition inside the run is cleaned and
# launched again.
# Parameter 7(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time.
# Must be a positive integer.
#====== Do not modify the following values =======
codeDir="Codes/"
execDir="Exec/"
ResultsDir="Results/"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
maxIndex=$1
totalEjGrupo=$2 #Total de ejecuciones por grupo
total_stages=$3
total_groups=$4
maxTime=$5 #Maximo tiempo que se considera válido
if [ "$#" -lt "6" ]
then
echo "Not enough arguments"
echo "Usage -> bash CheckRun Common_Name maxIndex total_repetitions total_groups total_stages max_iteration_time [limit_time]"
exit -1
fi
common_name=$1
maxIndex=$2
totalEjGrupo=$3 #Total de ejecuciones por grupo
total_stages=$4
total_groups=$5
maxTime=$6 #Maximo tiempo que se considera válido
limit_time_exec=0
if [ $# -ge 7 ] #Max time per execution in seconds
then
limit_time_exec=$7
fi
exec_lines_basic=6
limit_time=0
exec_lines_basic=7
iter_lines_basic=3
exec_total_lines=$(($exec_lines_basic+$total_stages+$total_groups))
iter_total_lines=$(($iter_lines_basic+$total_stages*2+1))
exec_remove=$(($exec_lines_basic+$total_stages+$total_groups-1))
iter_remove=$(($iter_lines_basic+$total_stages-1))
iter_remove=$(($iter_lines_basic+$total_stages))
echo $#
if [ "$#" -lt "5" ]
then
echo "Not enough arguments"
echo "Usage -> bash CheckRun maxIndes total_repetitions total_groups total_stages max_iteration_time"
exit -1
fi
#Check if there are fatal errors during executions
grep -i -e fatal -e error -e abort -e == slurm* > errores2.txt
......@@ -47,13 +58,14 @@ if [ "$qty" -gt "0" ]
then
echo "Found Fatal errors during execution. Aborting"
echo "Read file errors2 to see the errors and in which files"
echo "FAILURE"
exit -2
fi
#Check if the number of output files is correct.
#If the number is not correct is a fatal error and the user
# is informed in which runs the amount does not match, and
# then the scripts exit.
# then the scripts exits.
#The user must figure out what to do with those runs.
qtyG=$(ls R*_Global.out | wc -l)
qtyG=$(($qtyG * 2))
......@@ -76,21 +88,22 @@ else
echo "Files do not match at Run $i -- diff=$diff" >> errores2.txt
fi
done
echo "FAILURE"
exit -1
fi
rm errores2.txt
# Check if there is any negative execution time
# Only invalid IDs are stored
rm -f errores.txt
touch errores.txt
rm -f tmp.txt
touch tmp.txt
exec_ids=($(grep -n "T_total" R*_Global.out | grep - | cut -d '_' -f1 | cut -d 'R' -f2))
exec_line=($(grep -n "T_total" R*_Global.out | grep - | cut -d ':' -f2))
for ((i=${#exec_ids[@]}-1; i>=0; i--))
do
first_line=$((${exec_line[$i]}-$exec_remove))
last_line=$(($first_line+$exec_total_lines-1))
echo "${exec_ids[$i]}:$first_line:$last_line" >> errores.txt
echo "${exec_ids[$i]}:$first_line:$last_line" >> tmp.txt
done
# Check if there is any iter time higher than expected
......@@ -108,25 +121,29 @@ do
first_line=$(($first_line/$iter_total_lines))
first_line=$(($first_line*$exec_total_lines+1))
last_line=$(($first_line+$exec_total_lines-1))
echo "${iter_ids[$i]}:$first_line:$last_line" >> errores.txt
echo "${iter_ids[$i]}:$first_line:$last_line" >> tmp.txt
fi
done
#Clean data from collected erroneous executions
qty=$(wc -l errores.txt | cut -d ' ' -f1)
qty=$(wc -l tmp.txt | cut -d ' ' -f1)
if [ "$qty" -gt 0 ];
then
echo "Se han encontrado errores de ejecución leves. Volviendo a ejecutar"
echo "Found minor execution errors. Executing again. Review file errores.txt."
echo "CHECKRUN -- Found errors" >> errores.txt
while IFS="" read -r lineRun || [ -n "$lineRun" ]
do
#Obtain data of erroneous execution
run=$(echo $lineRun | cut -d ':' -f1)
echo "Run $run had an erroneous execution, cleaning bad data."
echo "Run$run----------------------------------------------" >> errores.txt
#1 - Delete erroneous lines in Global file
first_line=$(echo $lineRun | cut -d ':' -f2)
last_line=$(echo $lineRun | cut -d ':' -f3)
sed -n ''$first_line','$last_line'p' R${run}_Global.out >> errores.txt
sed -i ''$first_line','$last_line'd' R${run}_Global.out
#2 - Translate line numbers to Local files type
......@@ -136,10 +153,12 @@ then
#3 - Delete erroneous lines in Local files
for ((j=0; j<total_groups; j++));
do
sed -n ''$first_line','$last_line'p' R${run}_G${j}* >> errores.txt
sed -i ''$first_line','$last_line'd' R${run}_G${j}*
done
echo "--------------------------------------------------" >> errores.txt
done < errores.txt
done < tmp.txt
fi
#Check if all repetitions for each Run have been executed
......@@ -153,43 +172,28 @@ do
qtyEx=$(grep T_total R"$run"_Global.out | wc -l)
if [ "$qtyEx" -ne "$totalEjGrupo" ];
then
#1 - Obtain config file name and repetitions to perform
diff=$(($totalEjGrupo-$qtyEx))
qty_missing=$(($qty_missing+$diff))
config_file="config$run.ini"
#1 - Obtain maximum number of processes for the run
max_numP=-1
for ((j=0; j<total_groups; j++));
do
resize_info=$(grep "\[resize$j\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
#3 - Obtain needed nodes for the number of processes
node_qty=$(($max_numP / $cores))
if [ "$node_qty" -eq "0" ];
config_file="$common_name$run.ini"
if [ $limit_time_exec -ne 0 ] #Max time per execution in seconds
then
node_qty=1
limit_time=$(($limit_time_exec*$diff/60+1))
fi
#2 - Obtain number of nodes needed
node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
#3 - Launch execution
echo "Run$run lacks $diff repetitions"
use_extrae=0
sbatch -N $node_qty $dir$execDir./generalRun.sh $dir $config_file $use_extrae $run $diff
sbatch -p $partition -N $node_qty -t $limit_time $dir$execDir./generalRun.sh $dir $cores $config_file $use_extrae $run $diff
fi
else
echo "File R${run}_Global.out does not exist -- Could it be it must still be executed?"
fi
done
if [ "$qty_missing" -eq "0" ];
then
echo "SUCCESS"
......
#!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark"
dir="/home/martini/malleability_benchmark"
codeDir="/Codes/build"
export EXTRAE_CONFIG_FILE=extrae.xml
export LD_PRELOAD=$EXTRAE_HOME/lib/libmpitrace.so
$dir$codeDir/./a.out
#!/bin/bash
#This script should only be called by others scripts, do not call it directly
#SBATCH --exclude=c02,c01,c00
#SBATCH -p P1
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters with the aid of the RMS Slurm.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
......@@ -10,13 +20,14 @@ ResultsDir="/Results"
echo "START TEST"
#$1 == baseDir
#$2 == configFile
#$3 == use_extrae
#$4 == outFileIndex
#$5 == qty
#$2 == cores
#$3 == configFile
#$4 == use_extrae
#$5 == outFileIndex
#$6 == qty
echo $@
if [ $# -lt 3 ]
if [ $# -lt 4 ]
then
echo "Internal ERROR generalRun.sh - Not enough arguments were given"
exit -1
......@@ -24,13 +35,14 @@ fi
#READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1
configFile=$2
use_extrae=$3
outFileIndex=$4
cores=$2
configFile=$3
use_extrae=$4
outFileIndex=$5
qty=1
if [ $# -ge 5 ]
then
qty=$5
qty=$6
fi
nodelist=$SLURM_JOB_NODELIST
......@@ -45,10 +57,8 @@ then
nodes=1
fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1)
read -r ini fin <<<$(echo $aux)
diff=$(( fin - ini ))
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
#EXECUTE RUN
echo "Nodes=$nodelist"
......@@ -56,7 +66,7 @@ if [ $use_extrae -ne 1 ]
then
for ((i=0; i<qty; i++))
do
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
else
cp $dir$execDir/Extrae/extrae.xml .
......@@ -64,6 +74,7 @@ else
cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++))
do
#FIXME Extrae not tested keeping in mind the initial nodelist - Could have some errors
srun -n$numP --mpi=pmi2 ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
fi
......
#!/bin/bash
#This script should only be called by others scripts, do not call it directly
# !!!!This script should only be called by others scripts, do not call it directly!!!
# Runs a given configuration file with the indicated parameters.
# Parameter 1 - Base directory of the malleability benchmark
# Parameter 2 - Number of cores in a single machine
# Parameter 3 - Configuration file name for the emulation.
# Parameter 4 - Use Extrae(1) or not(0).
# Parameter 5 - Index to use for the output files. Must be a positive integer.
# Parameter 6 - Amount of executions per file. Must be a positive number.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
......@@ -7,63 +17,64 @@ ResultsDir="/Results"
echo "START TEST"
#$1 == baseDir
#$2 == configFile
#$3 == use_extrae
#$4 == outFileIndex
#$5 == qty
#$2 == cores
#$3 == configFile
#$4 == use_extrae
#$5 == outFileIndex
#$6 == qty
echo $@
if [ $# -lt 3 ]
then
echo "Internal ERROR generalRun.sh - Not enough arguments were given"
echo "Internal ERROR generalRunCostum.sh - Not enough arguments were given"
exit -1
fi
#READ PARAMETERS AND ENSURE CORRECTNESS
dir=$1
configFile=$2
cores=$2
configFile=$3
use_extrae=0
outFileIndex=0
qty=1
if [ $# -ge 3 ]
if [ $# -ge 4 ]
then
use_extrae=$3
fi
if [ $# -ge 4 ]
if [ $# -ge 5 ]
then
outFileIndex=$4
fi
if [ $# -ge 5 ]
if [ $# -ge 6 ]
then
qty=$5
fi
numP=$(bash $dir$execDir/BashScripts/getNumPNeeded.sh $configFile 0)
nodelist=$SLURM_JOB_NODELIST
nodes=$SLURM_JOB_NUM_NODES
if [ -z "$nodelist" ];
then
nodelist="localhost"
initial_nodelist="localhost"
else
initial_nodelist=$(bash $dir$execDir/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
fi
if [ -z "$nodes" ];
then
nodes=1
fi
aux=$(grep "\[resize0\]" -n $configFile | cut -d ":" -f1)
read -r ini fin <<<$(echo $aux)
diff=$(( fin - ini ))
numP=$(head -$fin $configFile | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
#EXECUTE RUN
echo "Nodes=$nodelist"
if [ $use_extrae -ne 1 ]
then
for ((i=0; i<qty; i++))
do
mpirun -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
mpirun -hosts $initial_nodelist -np $numP $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
else
cp $dir$execDir/Extrae/extrae.xml .
......@@ -71,7 +82,7 @@ else
cp $dir$execDir/Extrae/trace_worker.sh .
for ((i=0; i<qty; i++))
do
mpirun -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
mpirun -hosts $initial_nodelist -np $numP ./trace.sh $dir$codeDir/a.out $configFile $outFileIndex $nodelist $nodes
done
fi
......
#!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark"
dir="/home/martini/malleability_benchmark/"
# Creates a directory with all possible and valid combinations of configuration files
# that can be created from a given complex configuration file.
# Parameter 1: Complex configuration file name.
# Parameter 2: Common output name of the output configuration files. It will be appended an index to each of them.
#====== Do not modify these values =======
codeDir="Codes/"
execDir="Exec/"
ResultsDir="Results/"
......
......@@ -3,15 +3,16 @@
dir="/home/usuario/Documentos/malleability_benchmark"
partition="P1"
exclude="c00,c01,c02"
cores=20
# Runs in a given current directory all .ini files
# Runs in a given current directory all .ini files with the aid of the RMS
# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
# Parameter 2(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
use_extrae=0
qty=1
......@@ -20,35 +21,18 @@ then
qty=$1
fi
limit_time=$((0))
if [ $# -ge 2 ] #Max time per execution in seconds
then
limit_time=$(($2 * $qty / 60 + 1))
fi
files="./*.ini"
internalIndex=$(echo $files | tr -cd ' ' | wc -c)
index=$((0))
for config_file in $files
do
# FIXME Tener en cuenta que puede ser más de un resize
aux=$(grep "\[resize0\]" -n $config_file | cut -d ":" -f1)
ini=$(echo $aux | cut -d " " -f1)
fin=$(echo $aux | cut -d " " -f2)
diff=$(( fin - ini ))
numP1=$(head -$fin $config_file | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
aux=$(grep "\[resize1\]" -n $config_file | cut -d ":" -f1)
ini=$(echo $aux | cut -d " " -f1)
fin=$(echo $aux | cut -d " " -f2)
diff=$(( fin - ini ))
numP2=$(head -$fin $config_file | tail -$diff | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
echo "------------------------------------------run np=$numP1"
if [ $numP1 -lt $numP2 ]
then
numP1=$numP2
fi
node_qty=$(($numP1 / $cores))
if [ $node_qty -eq 0 ]
then
node_qty=1
fi
node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
outFileIndex=$(echo $config_file | sed s/[^0-9]//g)
if [[ $outFileIndex ]]; then
......@@ -60,6 +44,6 @@ do
#Execute test
echo "Execute job $index with Nodes=$node_qty and config_file=$config_file"
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $index $qty
sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $index $qty
done
echo "End"
......@@ -3,7 +3,6 @@
dir="/home/usuario/Documentos/malleability_benchmark"
partition="P1"
exclude="c00,c01,c02"
cores=20
# Executes a given configuration file with the aid of
# the RMS Slurm.
......@@ -11,12 +10,14 @@ cores=20
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
# Parameter 4(Optional): Use Extrae(1) or not(0).
# Parameter 5(Optional): Path where the output files should be saved.
# Parameter 5(Optional): Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. Must be a positive integer.
# Parameter 6(Optional): Path where the output files should be saved.
#====== Do not modify these values =======
codeDir="/Codes/build"
execDir="/Exec"
ResultsDir="/Results"
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
if [ $# -lt 1 ]
then
......@@ -29,7 +30,8 @@ fi
#$2 == outFileIndex
#$3 == Qty of repetitions
#$4 == Use extrae NO(0) YES(1)
#$5 == Output path
#$5 == Max time per execution(s)
#$6 == Output path
config_file=$1
outFileIndex=0
......@@ -48,36 +50,20 @@ if [ $# -ge 4 ]
then
use_extrae=$4
fi
if [ $# -ge 5 ]
limit_time=$((0))
if [ $# -ge 5 ] #Max time per execution in seconds
then
output=$5
limit_time=$(($5 * $qty / 60 + 1))
fi
#1 - Obtain maximum number of processes for the run
max_numP=-1
total_groups=$(grep Total_Resizes $config_file | cut -d '=' -f2)
for ((j=0; j<total_groups; j++));
do
resize_info=$(grep "\[resize$j\]" -n $config_file | cut -d ":" -f1)
first_line=$(echo $resize_info | cut -d " " -f1)
last_line=$(echo $resize_info | cut -d " " -f2)
range_lines=$(( last_line - first_line ))
numP=$(head -$last_line $config_file | tail -$range_lines | cut -d ';' -f1 | grep Procs | cut -d '=' -f2)
if [ "$numP" -gt "$max_numP" ];
then
max_numP=$numP
fi
done
#2 - Obtain needed nodes for the number of processes
node_qty=$(($max_numP / $cores))
if [ "$node_qty" -eq "0" ];
if [ $# -ge 6 ]
then
node_qty=1
output=$6
fi
#3 - Run with the expected amount of nodes
sbatch -p $partition --exclude=$exclude -N $node_qty $dir$execDir/generalRun.sh $dir $config_file $use_extrae $outFileIndex $qty
#Obtain amount of nodes neeeded
node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
#Run with the expected amount of nodes
sbatch -p $partition --exclude=$exclude -N $node_qty -t $limit_time $dir$execDir/generalRun.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
if ! [ -z "$output" ]
then
......
#!/bin/bash
dir="/home/usuario/Documentos/malleability_benchmark"
dir="/home/martini/malleability_benchmark"
cores=20
# Executes a given configuration file
# Executes a given configuration file. This script can be called with Slurm commands to
# choose the desired user configuration.
# Parameter 1: Configuration file name for the emulation.
# Parameter 2(Optional): Index to use for the output files. Must be a positive integer.
# Parameter 3(Optional): Number of repetitions to perform. Must be a positive integer.
......@@ -49,7 +51,7 @@ then
output=$5
fi
bash $dir$execDir/generalRunCostum.sh $dir $config_file $use_extrae $outFileIndex $qty
bash $dir$execDir/generalRunCostum.sh $dir $cores $config_file $use_extrae $outFileIndex $qty
if ! [ -z "$output" ]
then
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment