Added missing files to execute the code

Exec/BashScripts/createInitialNodelist.sh

+#!/bin/bash
+# Runs in a given current directory all .ini files
+# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
+#====== Do not modify these values =======
+
+numP=$1
+cores=$2
+nodelist=$3
+
+initial_node_qty=$(($numP / $cores))
+if [ $initial_node_qty -eq 0 ]
+then
+  initial_node_qty=1
+fi
+
+common_node_name="n" #FIXME What if it uses another type of node?
+if [[ $nodelist == *"["* ]]; then
+  common_node_name=$(echo $nodelist | cut -d '[' -f1)
+fi
+
+node_array=($(echo $nodelist | sed -e 's/[\[n]//g' -e 's/\]/ /g' -e 's/,/ /g'))
+actual_node_qty=0
+for ((i=0; $actual_node_qty<$initial_node_qty; i++))
+do
+  element=($(echo ${node_array[$i]} | sed -e 's/-/ /g'))
+
+  nodes_qty=1
+  if [ "${#element[@]}" -gt 1 ];
+  then
+    nodes_qty=$((10#${element[1]}-10#${element[0]}+1))
+  fi
+
+  expected_node_qty=$(($actual_node_qty + $nodes_qty))
+  if [ "$expected_node_qty" -le "$initial_node_qty" ];
+  then
+    added_qty=$nodes_qty
+    actual_node_qty=$expected_node_qty
+  else
+    added_qty=$(($initial_node_qty - $actual_node_qty))
+    actual_node_qty=$initial_node_qty
+  fi
+
+  for ((j=0; j<$added_qty; j++))
+  do
+    index=$((10#${element[0]} + $j))
+    index=0$index # FIXME What if there are more than 9 nodes?
+    #FIXME What if less than $cores have to be spawned?
+    for ((core=0; core<$cores; core++)) # FIXME What if the user asks for a spread distribution
+    do
+      initial_nodelist="${initial_nodelist:+$initial_nodelist,}"$common_node_name$index
+    done
+  done
+done
+
+#Print result
+echo $initial_nodelist

Exec/CreateResizeDataframe.py

+import sys
+import glob
+import numpy as np
+import pandas as pd
+from enum import Enum
+
+class G_enum(Enum):
+    TOTAL_RESIZES = 0
+    TOTAL_GROUPS = 1
+    SDR = 2
+    ADR = 3
+    DR = 4
+    RED_METHOD = 5
+    RED_STRATEGY = 6
+    SPAWN_METHOD = 7
+    SPAWN_STRATEGY = 8
+    GROUPS = 9
+    ITERS = 10
+    T_SPAWN = 11
+    T_SR = 12
+    T_AR = 13
+    T_MALLEABILITY = 14
+    T_TOTAL = 15
+    #Malleability specific
+    NP = 0
+    NC = 1
+
+
+#columnsG = ["Total_Resizes", "Total_Groups", "SDR", "ADR", "DR", "Redistribution_Method", \
+#            "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", \
+#            "Iters", "T_spawn", "T_SR", "T_AR", "T_Malleability", "T_total"] #16
+
+
+columnsM = ["NP", "NC", "SDR", "ADR", "DR", "Redistribution_Method", \
+            "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy",  \
+            "Iters", "T_spawn", "T_SR", "T_AR", "T_Malleability"] #15
+
+def copy_resize(row, dataM_it, resize):
+  basic_indexes = [G_enum.SDR.value, G_enum.ADR.value, G_enum.DR.value]
+  array_actual_group = [G_enum.ITERS.value,  \
+          G_enum.T_SPAWN.value, G_enum.T_SR.value, \
+          G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]
+  array_next_group = [G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, \
+          G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value]
+
+  dataM_it[G_enum.NP.value] = row[G_enum.GROUPS.value][resize]
+  dataM_it[G_enum.NC.value] = row[G_enum.GROUPS.value][resize+1]
+
+  for index in basic_indexes:
+    dataM_it[index] = row[index]
+    
+  for index in array_actual_group:
+    dataM_it[index-1] = row[index][resize] 
+
+  for index in array_next_group:
+    dataM_it[index] = row[index][resize+1]
+
+
+#-----------------------------------------------
+
+def create_resize_dataframe(dfG, dataM):
+  it = -1
+  for row_index in range(len(dfG)):
+    row = dfG.iloc[row_index]
+    resizes = row[G_enum.TOTAL_RESIZES.value]
+
+    for resize in range(resizes):
+      it += 1
+      dataM.append( [None] * len(columnsM) )
+      copy_resize(row, dataM[it], resize)
+
+#-----------------------------------------------
+if len(sys.argv) < 2:
+    print("The files name is missing\nUsage: python3 CreateResizeDataframe.py input_file.pkl output_name")
+    exit(1)
+
+input_name = sys.argv[1]
+if len(sys.argv) > 2:
+  name = sys.argv[2]
+else:
+  name = "dataM"
+print("File name will be: " + name + ".pkl")
+
+
+dfG = pd.read_pickle(input_name)
+dataM = []
+create_resize_dataframe(dfG, dataM)
+
+dfM = pd.DataFrame(dataM, columns=columnsM)
+dfM.to_pickle(name + '.pkl')
+
+print(dfG)
+print(dfM)

Exec/MallTimes.py

+import sys
+import glob
+import numpy as np
+import pandas as pd
+from enum import Enum
+
+class G_enum(Enum):
+    TOTAL_RESIZES = 0
+    TOTAL_GROUPS = 1
+    SDR = 2
+    ADR = 3
+    DR = 4
+    RED_METHOD = 5
+    RED_STRATEGY = 6
+    SPAWN_METHOD = 7
+    SPAWN_STRATEGY = 8
+    GROUPS = 9
+    ITERS = 10
+    T_SPAWN = 11
+    T_SR = 12
+    T_AR = 13
+    T_MALLEABILITY = 14
+    T_TOTAL = 15
+    #Malleability specific
+    NP = 0
+    NC = 1
+
+
+columnsG = ["Total_Resizes", "Total_Groups", "SDR", "ADR", "DR", "Redistribution_Method", \
+            "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", \
+            "Iters", "T_spawn", "T_SR", "T_AR", "T_Malleability", "T_total"] #16
+
+#-----------------------------------------------
+# Obtains the value of a given index in a splited line
+# and returns it as a float values if possible, string otherwise
+def get_value(line, index, separator=True):
+  if separator:
+    value = line[index].split('=')[1].split(',')[0]
+  else:
+    value = line[index]
+
+  try:
+    value = float(value)
+    if value.is_integer():
+      value = int(value)
+  except ValueError:
+    return value
+  return value
+
+#-----------------------------------------------
+def record_config_line(lineS, aux_data):
+  aux_data[0] = int(lineS[1].split("=")[1])
+  aux_data[1] = int(lineS[2].split("=")[1])
+  aux_data[2] = 1
+  aux_data[3] = 0
+  aux_data[4] = 0
+#          0        1   
+#Test numP=160 numC=120 -- qty=5
+#  0      1        2         3
+
+#-----------------------------------------------
+# Obtains the general parameters of an execution and
+# stores them for creating a global dataframe
+def record_new_line(lineS, dataG_it, data_aux):
+  offset_line = 2
+  sdr = 3550186388
+  adr = sdr * 0.9981
+
+  dataG_it[G_enum.TOTAL_RESIZES.value] = 1
+  dataG_it[G_enum.TOTAL_GROUPS.value] = dataG_it[G_enum.TOTAL_RESIZES.value]+1
+
+  # Init lists for each column
+  array_groups = [G_enum.GROUPS.value, G_enum.ITERS.value, G_enum.RED_METHOD.value, \
+          G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value]
+
+  array_resizes = [ G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]
+
+  for index in array_groups:
+    dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_GROUPS.value]
+  for index in array_resizes:
+    dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_RESIZES.value]
+
+  dataG_it[G_enum.GROUPS.value][0] = data_aux[0]
+  dataG_it[G_enum.GROUPS.value][1] = data_aux[1] 
+  dataG_it[G_enum.ITERS.value][0] = dataG_it[G_enum.ITERS.value][1] = 500
+
+  dataG_it[G_enum.SPAWN_METHOD.value][0] = dataG_it[G_enum.RED_METHOD.value][0] = 0
+  dataG_it[G_enum.SPAWN_METHOD.value][1] = data_aux[2]
+  dataG_it[G_enum.RED_METHOD.value][1] = data_aux[3]
+  dataG_it[G_enum.SPAWN_STRATEGY.value][0] = dataG_it[G_enum.RED_STRATEGY.value][0] = data_aux[4]
+  dataG_it[G_enum.SPAWN_STRATEGY.value][1] = dataG_it[G_enum.RED_STRATEGY.value][1] = data_aux[4]
+
+  dataG_it[G_enum.SDR.value] = sdr
+  dataG_it[G_enum.ADR.value] = 0
+  dataG_it[G_enum.DR.value] = dataG_it[G_enum.SDR.value]
+
+#-----------------------------------------------
+def record_time_line(lineS, dataG_it):
+  T_names = ["T_spawn:", "T_SR:", "T_AR:", "T_Malleability:", "Time_loop:"]
+  T_values = [G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value, G_enum.T_TOTAL.value]
+  if not (lineS[0] in T_names): # Execute only if line represents a Time
+      return
+
+  index = T_names.index(lineS[0])
+  index = T_values[index]
+  offset_lines = 1
+
+  len_index = 1
+  if dataG_it[index] != None:
+    len_index = len(dataG_it[index])
+    for i in range(len_index):
+      dataG_it[index][i] = get_value(lineS, i+offset_lines, False)
+  else:
+      dataG_it[index] = get_value(lineS, offset_lines, False)
+
+  sdr = 3550186388
+  adr = sdr * 0.9981
+  if index == G_enum.T_AR.value and dataG_it[index][0] != 0:
+    dataG_it[G_enum.ADR.value] = adr
+    dataG_it[G_enum.SDR.value] -= adr
+
+
+#-----------------------------------------------
+def read_global_file(f, dataG, it):
+  runs_in_file=0
+  aux_data = [0,0,0,0,0]
+  for line in f: 
+    lineS = line.split()
+
+    if len(lineS) > 0:
+      if lineS[0] == "Test": # CONFIG LINE
+        record_config_line(lineS, aux_data)
+      elif lineS[0] == "Start":
+        it += 1
+        runs_in_file += 1
+        group = 0
+
+        dataG.append([None]*len(columnsG))
+        record_new_line(lineS, dataG[it], aux_data)
+      elif it>-1:
+        record_time_line(lineS, dataG[it])
+
+  return it,runs_in_file
+
+#-----------------------------------------------
+def convert_to_tuples(dfG):
+  array_list_items = [G_enum.GROUPS.value, G_enum.ITERS.value, \
+          G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, \
+          G_enum.SPAWN_STRATEGY.value, G_enum.T_SPAWN.value, G_enum.T_SR.value, \
+          G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]
+
+  for item in array_list_items:
+    name = columnsG[item]
+    values = dfG[name].copy()
+    for index in range(len(values)):
+      values[index] = tuple(values[index])
+    dfG[name] = values
+
+#-----------------------------------------------
+
+if len(sys.argv) < 2:
+    print("The files name is missing\nUsage: python3 MallTimes.py commonName directory [OutName]")
+    exit(1)
+
+common_name = sys.argv[1]
+if len(sys.argv) >= 3:
+    BaseDir = sys.argv[2]
+    print("Searching in directory: "+ BaseDir)
+else:
+    BaseDir = "./"
+
+if len(sys.argv) >= 4:
+  name = sys.argv[3]
+else:
+  name = "data"
+print("File name will be: " + name + "G.pkl")
+
+lista = (glob.glob(BaseDir + common_name + "*.out")) # Se utiliza cuando solo hay un nivel de directorios
+print("Number of files found: "+ str(len(lista)));
+
+it = -1
+dataG = []
+
+for elem in lista:
+  f = open(elem, "r")
+  it,runs_in_file = read_global_file(f, dataG, it)
+  f.close()
+
+dfG = pd.DataFrame(dataG, columns=columnsG)
+convert_to_tuples(dfG)
+print(dfG)
+dfG.to_pickle(name + 'G.pkl')
+
+#dfM = pd.DataFrame(dataM, columns=columnsM)
+
+#Poner en TC el valor real y en TH el necesario para la app
+#cond = dfM.TH != 0
+#dfM.loc[cond, ['TC', 'TH']] = dfM.loc[cond, ['TH', 'TC']].values
+#dfM.to_csv(name + 'M.csv')

Exec/generalRun.sh

+#!/bin/bash
+
+#SBATCH -N 1
+#SBATCH -p P1
+#SBATCH -t 10:00:00
+
+
+
+nodelist=$SLURM_JOB_NODELIST
+nodes=$SLURM_JOB_NUM_NODES
+cores=20
+use_mat_file=1
+numP=$1
+matrix=$2
+numC=$3
+qty=$4
+initial_nodelist=$(bash $dirBI/Exec/BashScripts/createInitialNodelist.sh $numP $cores $nodelist)
+echo "Nodelist $nodelist -- Matrix = $matrix"
+echo "Test numP=$numP numC=$numC -- qty=$qty"
+for ((i=0; i<qty; i++))
+do
+  echo "Start BICG"
+  mpirun -hosts $initial_nodelist -np $numP $dirBI/BiCGStab_Iker/BiCGStab $matrix $use_mat_file $numC
+done
+echo "End"
+
+sed -i 's/application called MPI_Abort(MPI_COMM_WORLD, -100) - process/shrink cleaning/g' slurm-$SLURM_JOB_ID.out
+sed -i 's/Abort(-100)/shrink cleaning/g' slurm-$SLURM_JOB_ID.out

Exec/joinDf.py

+import sys
+import glob
+import numpy as numpy
+import pandas as pd
+
+if len(sys.argv) < 3:
+    print("The files name is missing\nUsage: python3 joinDf.py resultsName1.pkl resultsName2.pkl OutName")
+    exit(1)
+
+if len(sys.argv) >= 4:
+  name = sys.argv[3]
+else:
+  name = "dataJOINED"
+
+print("File name will be: " + name + ".pkl")
+df1 = pd.read_pickle( sys.argv[1] )
+df2 = pd.read_pickle( sys.argv[2] )
+frames = [df1, df2]
+df3 = pd.concat(frames)
+
+df3.to_pickle(name + '.pkl')

Exec/runAll.sh

+#!/bin/bash
+
+scriptDir="$(dirname "$0")"
+source $scriptDir/config.txt
+dirM="${dirBI}../SparseMatrix"
+export dirBI
+
+#matrix="tub100.rb"
+#matrix="sherman2.rb"
+matrix="HV15R.rb"
+
+
+procs=(2 20 40 80 160)
+msm=(0 1)
+mss=(1 2)
+mrm=(0 1)
+is_syncs=(1 0)
+qty=1
+if [ $# -ge 1 ]
+then
+  qty=$1
+fi
+
+echo "$dirM/$matrix"
+for proc in "${procs[@]}"
+do
+  echo "------------------------------------------run np=$proc next=0"
+  for proc_c in "${procs[@]}"
+  do
+    if [ $proc -ne $proc_c ]
+    then
+
+      max_numP=$proc
+      if [ "$proc_c" -gt "$proc" ];
+      then
+        max_numP=$proc_c
+      fi
+      node_qty=$(($max_numP / 20))
+      if [ $node_qty -eq 0 ]
+      then
+        node_qty=1
+      fi
+      sbatch -p P1 -N $node_qty $dirBI/Exec/generalRun.sh $proc $dirM/$matrix $proc_c $qty
+    fi
+  done
+done
+
+echo "End"

Exec/runAll_nonMaM.sh

+#!/bin/bash
+
+scriptDir="$(dirname "$0")"
+source $scriptDir/config.txt
+dirM="${dirBI}../SparseMatrix"
+export dirBI
+
+#matrix="tub100.rb"
+#matrix="sherman2.rb"
+matrix="HV15R.rb"
+
+
+procs=(2 20 40 80 160)
+msm=(0 1)
+mss=(1 2)
+mrm=(0 1)
+is_syncs=(1 0)
+qty=1
+if [ $# -ge 1 ]
+then
+  qty=$1
+fi
+
+proc_c=200
+echo "$dirM/$matrix"
+for proc in "${procs[@]}"
+do
+  echo "------------------------------------------run np=$proc next=0"
+  node_qty=$(($proc / 20))
+  if [ $node_qty -eq 0 ]
+  then
+    node_qty=1
+  fi
+  sbatch -p P1 -N $node_qty $dirBI/Exec/generalRun.sh $proc $dirM/$matrix $proc_c $qty
+done
+
+echo "End"

Exec/runBase.sh

+#!/bin/bash
+
+scriptDir="$(dirname "$0")"
+source $scriptDir/config.txt
+dirM="${dirBI}../SparseMatrix"
+export dirBI
+#matrix="tub100.rb"
+#matrix="sherman2.rb"
+#matrix="audikw_1.rb"
+matrix="HV15R.rb"
+
+numP=$1
+numC=$2
+use_mat_file=1
+qty=1
+if [ $# -gt 2 ]
+then
+  qty=$3
+fi
+
+sbatch -p P1 -N 8 $dirBI/Exec/generalRun.sh $numP $dirM/$matrix $numC $qty
+
+echo "End"