CheckRun.sh 7.06 KB
Newer Older
1
2
3
#!/bin/bash

dir="/home/martini/malleability_benchmark/"
4
partition="P1"
5
6
7
8

# Checks if all the runs in the current working directory performed under a 
# Slurm manager have been performed correctly and if some runs can be corrected 
# they are launched again
9
10
11
12
13
14
# Parameter 1 - Common name of the configuration files
# Parameter 2 - Maximum index of the runs
# Parameter 3 - Amount of repetitions per index/run
# Parameter 4 - Total stages in all runs. #FIXME The amount of stages must be equal across all the runs, must be modified in the future.
# Parameter 5 - Total groups of processes in all runs #FIXME The amount of groups must be equal across all the runs, must be modified in the future.
# Parameter 6 - Maximum valid iteration time across all runs. If an iteration time
15
16
#               is higher, that particular repetition inside the run is cleaned and
#               launched again.
17
18
# Parameter 7(Optional) - Maximum amount of time in seconds needed by a single execution. Default value is 0, which indicates infinite time. 
#               Must be a positive integer.
19
20
#====== Do not modify the following values =======

21
22
23
codeDir="Codes/"
execDir="Exec/"
ResultsDir="Results/"
24
cores=$(bash $dir$execDir/BashScripts/getCores.sh $partition)
25

26
27
28
29
30
31
32
33
34
35
36
37
38
if [ "$#" -lt "6" ]
then
  echo "Not enough arguments"
  echo "Usage -> bash CheckRun Common_Name maxIndex total_repetitions total_groups total_stages max_iteration_time [limit_time]"
  exit -1
fi

common_name=$1
maxIndex=$2
totalEjGrupo=$3 #Total de ejecuciones por grupo
total_stages=$4
total_groups=$5
maxTime=$6 #Maximo tiempo que se considera válido
39
limit_time_exec=0
40
41
if [ $# -ge 7 ] #Max time per execution in seconds
then
42
  limit_time_exec=$7
43
fi
44

45
limit_time=0
46
47
48
49
50
exec_lines_basic=6
iter_lines_basic=3
exec_total_lines=$(($exec_lines_basic+$total_stages+$total_groups))
iter_total_lines=$(($iter_lines_basic+$total_stages*2+1))
exec_remove=$(($exec_lines_basic+$total_stages+$total_groups-1))
51
iter_remove=$(($iter_lines_basic+$total_stages))
52

53

54
55
#Check if there are fatal errors during executions
grep -i -e fatal -e error -e abort -e == slurm* > errores2.txt
56
qty=$(wc -l errores2.txt | cut -d ' ' -f1)
57
if [ "$qty" -gt "0" ]
58
then
59
60
  echo "Found Fatal errors during execution. Aborting"
  echo "Read file errors2 to see the errors and in which files"
61
  echo "FAILURE"
62
63
64
  exit -2
fi

65
66
67
#Check if the number of output files is correct.
#If the number is not correct is a fatal error and the user
# is informed in which runs the amount does not match, and
68
# then the scripts exits.
69
70
#The user must figure out what to do with those runs.
qtyG=$(ls R*_Global.out | wc -l)
71
qtyG=$(($qtyG * 2))
72
73
qtyL=$(ls R*_G*N*.out | wc -l)
if [ "$qtyG" == "$qtyL" ]
74
then
75
  echo "Number of G($qtyG) and L($qtyL) files match"
76
else
77
78
79
  echo "Lacking Local($qtyL) or global($qtyG) files. Aborting"
  echo "Lacking Local($qtyL) or global($qtyG) files. Aborting" > errores2.txt
  for ((i=0; i<$maxIndex; i++))
80
  do
81
82
    qtyEx=$(grep T_total R"$i"_Global.out | wc -l)
    qtyIt=$(grep T_iter R"$i"_G*N*.out | wc -l)
83
    qtyEx=$(($qtyEx * 2))
84
    if [ "$qtyEx" -ne "$qtyIt" ] 
85
86
    then
      diff=$(($totalEjGrupo-$qtyEx))
87
88
      echo "Files do not match at Run $i -- diff=$diff"
      echo "Files do not match at Run $i -- diff=$diff" >> errores2.txt
89
90
    fi
  done
91
  echo "FAILURE"
92
93
  exit -1
fi
94
rm errores2.txt
95

96
97
# Check if there is any negative execution time
# Only invalid IDs are stored
98
99
rm -f tmp.txt
touch tmp.txt
100
101
102
103
104
105
exec_ids=($(grep -n "T_total" R*_Global.out | grep - | cut -d '_' -f1 | cut -d 'R' -f2))
exec_line=($(grep -n "T_total" R*_Global.out | grep - | cut -d ':' -f2))
for ((i=${#exec_ids[@]}-1; i>=0; i--))
do
  first_line=$((${exec_line[$i]}-$exec_remove))
  last_line=$(($first_line+$exec_total_lines-1))
106
  echo "${exec_ids[$i]}:$first_line:$last_line" >> tmp.txt
107
done
108

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# Check if there is any iter time higher than expected
# Only invalid IDs are stored
iter_times=($(grep "T_iter" R*_G*N*.out | cut -d ' ' -f2))
iter_ids=($(grep "T_iter" R*_G*N*.out | cut -d '_' -f1 | cut -d 'R' -f2))
iter_line=($(grep -n "T_iter" R*_G*N*.out | cut -d ':' -f2))
for ((i=${#iter_times[@]}-1; i>=0; i--))
do
  is_invalid=$(echo ${iter_times[$i]}'>'$maxTime | bc -l)
  if [ $is_invalid -eq 1 ]
  then
    first_line=$((${iter_line[$i]}-$iter_remove))
    # Translate line number to Global file
    first_line=$(($first_line/$iter_total_lines))
    first_line=$(($first_line*$exec_total_lines+1))
    last_line=$(($first_line+$exec_total_lines-1))
124
    echo "${iter_ids[$i]}:$first_line:$last_line" >> tmp.txt
125
126
127
128
  fi
done

#Clean data from collected erroneous executions
129
qty=$(wc -l tmp.txt | cut -d ' ' -f1)
130
if [ "$qty" -gt 0 ];
131
then
132
133
  echo "Found minor execution errors. Executing again. Review file errores.txt."
  echo "CHECKRUN -- Found errors" >> errores.txt
134

135
  while IFS="" read -r lineRun || [ -n "$lineRun" ]
136
  do
137
138
139
    #Obtain data of erroneous execution
    run=$(echo $lineRun | cut -d ':' -f1)
    echo "Run $run had an erroneous execution, cleaning bad data."
140
    echo "Run$run----------------------------------------------" >> errores.txt
141
142
143
144

    #1 - Delete erroneous lines in Global file
    first_line=$(echo $lineRun | cut -d ':' -f2)
    last_line=$(echo $lineRun | cut -d ':' -f3)
145
146

    sed -n ''$first_line','$last_line'p' R${run}_Global.out >> errores.txt
147
148
149
150
151
152
153
154
155
    sed -i ''$first_line','$last_line'd' R${run}_Global.out

    #2 - Translate line numbers to Local files type
    first_line=$(($first_line/$exec_total_lines))
    first_line=$(($first_line*$iter_total_lines+1))
    last_line=$(($first_line+$iter_total_lines-1))
    #3 - Delete erroneous lines in Local files
    for ((j=0; j<total_groups; j++)); 
    do
156
      sed -n ''$first_line','$last_line'p' R${run}_G${j}* >> errores.txt
157
      sed -i ''$first_line','$last_line'd' R${run}_G${j}*
158
    done
159
    echo "--------------------------------------------------" >> errores.txt
160

161
  done < tmp.txt
162
fi
163

164
165
166
#Check if all repetitions for each Run have been executed
#If any run lacks repetitions, the job is automatically launched again
#If a run has even executed a repetition, is not launched as it could be in the waiting queue
167
qty_missing=0
168
for ((run=0; run<$maxIndex; run++))
169
do
170
  if [ -f "R${run}_Global.out" ]
171
  then
172
173
174
    qtyEx=$(grep T_total R"$run"_Global.out | wc -l)
    if [ "$qtyEx" -ne "$totalEjGrupo" ];
    then
175
      #1 - Obtain config file name and repetitions to perform
176
177
      diff=$(($totalEjGrupo-$qtyEx))
      qty_missing=$(($qty_missing+$diff))
178
      config_file="$common_name$run.ini"
179
180
181
182
      if [ $limit_time_exec -ne 0 ] #Max time per execution in seconds
      then
        limit_time=$(($limit_time_exec*$diff/60+1))
      fi
183
184

      #2 - Obtain number of nodes needed
185
      node_qty=$(bash $dir$execDir/BashScripts/getMaxNodesNeeded.sh $config_file $dir $cores)
186
187
188

      #3 - Launch execution
      echo "Run$run lacks $diff repetitions"
189
      use_extrae=0
190
      sbatch -p $partition -N $node_qty -t $limit_time $dir$execDir./generalRun.sh $dir $cores $config_file $use_extrae $run $diff
191
192
193
    fi
  else
    echo "File R${run}_Global.out does not exist -- Could it be it must still be executed?"
194
195
196
  fi
done

197
if [ "$qty_missing" -eq "0" ];
198
then
199
200
201
  echo "SUCCESS"
else
  echo "REPEATING - A total of $qty_missing executions are being repeated"
202
fi