WIP. Tests must be performed to ensure correctness. Added new feature to test...

WIP. Tests must be performed to ensure correctness. Added new feature to test all combinations of redistributions+spawn techniques. Now is possible to create multiple configuration files with one complex configuration file.

Exec/PythonCodes/read_multiple.py

+import sys
+import glob
+import os
+from datetime import date
+from enum import Enum
+
+GENERAL_SECTION = "[general]"
+RESIZE_SECTION = "[resize"
+STAGE_SECTION = "[stage"
+END_SECTION_DELIMITER = ";end"
+
+class Config_section(Enum):
+    INVALID=0
+    GENERAL=1
+    RESIZE=2
+    STAGE=3
+
+    P_TOTAL_RESIZES="Total_Resizes"
+    P_TOTAL_STAGES="Total_Stages"
+    P_GRANULARITY="Granularity"
+    P_SDR="SDR"
+    P_ADR="ADR"
+    P_RIGID="Rigid"
+
+    P_STAGE_TYPE="Stage_Type"
+    P_STAGE_BYTES="Stage_Bytes"
+    P_STAGE_TIME_CAPPED="Stage_Time_Capped"
+    P_STAGE_TIME="Stage_Time"
+
+    P_RESIZE_ITERS="Iters"
+    P_RESIZE_PROCS="Procs"
+    P_RESIZE_FACTORS="FactorS"
+    P_RESIZE_DIST="Dist"
+    P_RESIZE_REDISTRIBUTION_METHOD="Redistribution_Method"
+    P_RESIZE_REDISTRIBUTION_STRATEGY="Redistribution_Strategy"
+    P_RESIZE_SPAWN_METHOD="Spawn_Method"
+    P_RESIZE_SPAWN_STRATEGY="Spawn_Strategy"
+
+    @classmethod
+    def has_key(cls, name):
+      return any(x.value == name for x in cls)
+
+def is_ending_of_section(line):
+  if(END_SECTION_DELIMITER in line):
+    return True
+  return False
+
+def is_a_general_section(line):
+  if(line == GENERAL_SECTION):
+    return True
+  return False
+
+def is_a_resize_section(line):
+  if(RESIZE_SECTION in line and not is_ending_of_section(line)):
+      return True
+  return False
+
+def is_a_stage_section(line):
+  if(STAGE_SECTION in line and not is_ending_of_section(line)):
+      return True
+  return False
+
+def process_line(line, data):
+  key,value = line.split('=')
+  if(not Config_section.has_key(key)):
+    print("Unknown parameter " + key)
+    return False
+  
+  if(',' in value):
+    value = value.split(',')
+    for i in range(len(value)):
+      try:
+        value[i] = float(value[i])
+      except ValueError:
+        print("Unable to convert to float - Not a fatal error")
+  else:
+    try:
+      value = float(value)
+    except ValueError:
+      print("Unable to convert to float - Not a fatal error")
+    
+
+  data[key]=value
+  return True
+
+def process_file(file_name):
+  f = open(file_name, "r")
+  lines = f.read().splitlines()
+  section_type = Config_section.INVALID
+  general_data = {}
+  stages_data=[]
+  resizes_data=[]
+  processing=0
+  for line in lines:
+    if(section_type != Config_section.INVALID):
+      if(is_ending_of_section(line)):
+        section_type = Config_section.INVALID
+      else:
+        process_line(line, processing)
+    elif(is_a_general_section(line)):
+      section_type = Config_section.GENERAL
+      processing = general_data
+    elif(is_a_resize_section(line)):
+      section_type = Config_section.RESIZE
+      resizes_data.append({})
+      processing = resizes_data[len(resizes_data)-1]
+    elif(is_a_stage_section(line)):
+      section_type = Config_section.STAGE
+      stages_data.append({})
+      processing = stages_data[len(stages_data)-1]
+
+#  print(general_data)
+#  print(stages_data)
+#  print(resizes_data)
+  f.close()
+  return general_data,stages_data,resizes_data
+
+def general_section_write(f, general_data):
+    f.write(GENERAL_SECTION + "\n")
+    keys = list(general_data.keys())
+    values = list(general_data.values())
+    for i in range(len(keys)):
+        f.write(keys[i] + "=" + str(values[i]) + "\n")
+    f.write(END_SECTION_DELIMITER + " " + GENERAL_SECTION + "\n")
+
+def stage_section_write(f, stage_data, section_index):
+    f.write(STAGE_SECTION + str(section_index) + "]\n")
+    keys = list(stage_data.keys())
+    values = list(stage_data.values())
+    for i in range(len(keys)):
+        f.write(keys[i] + "=" + str(values[i]) + "\n")
+    f.write(END_SECTION_DELIMITER + " " + STAGE_SECTION + str(section_index) + "]\n")
+
+def resize_section_write(f, resize_data, section_index):
+    f.write(RESIZE_SECTION + str(section_index) + "]\n")
+    keys = list(resize_data.keys())
+    values = list(resize_data.values())
+    for i in range(len(keys)):
+        f.write(keys[i] + "=" + str(values[i]) + "\n")
+    f.write(END_SECTION_DELIMITER + " " + RESIZE_SECTION + str(section_index) + "]\n")
+
+
+def write_output_file(datasets, common_output_name, output_index):
+    file_name = common_output_name + str(output_index) + ".ini"
+    total_stages=int(datasets[0][Config_section.P_TOTAL_STAGES.value])
+    total_resizes=int(datasets[0][Config_section.P_TOTAL_RESIZES.value])+1
+
+    f = open(file_name, "w")
+    general_section_write(f, datasets[0])
+
+    for i in range(total_stages):
+        stage_section_write(f, datasets[i+1], i)
+    for i in range(total_resizes):
+        resize_section_write(f, datasets[i+1+total_stages], i)
+    f.close()
+    
+
+def check_sections_assumptions(datasets):
+    total_resizes=int(datasets[0][Config_section.P_TOTAL_RESIZES.value])+1
+    total_stages=int(datasets[0][Config_section.P_TOTAL_STAGES.value])
+
+    adr = datasets[0][Config_section.P_ADR.value]
+    for i in range(total_resizes):
+        #Not valid if trying to use thread strategy and adr(Async data) is 0
+        if adr==0 and (datasets[total_stages+1+i][Config_section.P_RESIZE_SPAWN_STRATEGY.value] == 2 or datasets[total_stages+1+i][Config_section.P_RESIZE_REDISTRIBUTION_STRATEGY.value] == 2):
+            return False
+        #Not valid if the strategies are different
+        if datasets[total_stages+1+i][Config_section.P_RESIZE_SPAWN_STRATEGY.value] != datasets[total_stages+1+i][Config_section.P_RESIZE_REDISTRIBUTION_STRATEGY.value]:
+            return False
+        #Not valid if resize is to the same amount of processes
+        if i>0:
+            if datasets[total_stages+1+i][Config_section.P_RESIZE_PROCS.value] == datasets[total_stages+i][Config_section.P_RESIZE_PROCS.value]:
+                return False
+    return True
+
+def correct_adr(general_dataset):
+    sdr = general_dataset[Config_section.P_SDR.value]
+    adr_percentage = general_dataset[Config_section.P_ADR.value]
+    if (adr_percentage != 0):
+        general_dataset[Config_section.P_ADR.value] = sdr * (adr_percentage/100)
+
+
+def create_output_files(common_output_name, general_data, resize_data, stage_data):
+
+    def read_parameter(level_index):
+        dictionary = write_datasets[ds_indexes[level_index]]
+        key = keys[level_index]
+        index = indexes[level_index]
+        max_index = mindexes[level_index]
+        values = lists[level_index]
+        finished=False
+
+        if(index == max_index):
+            index = 0
+            if(level_index+1 == len(lists)):
+                finished = True
+            else:
+                finished = read_parameter(level_index+1)
+
+        dictionary[key] = values[index]
+        if(key == Config_section.P_RESIZE_PROCS.value):
+            original_dictionary = datasets[ds_indexes[level_index]]
+            dictionary[Config_section.P_RESIZE_FACTORS.value] = original_dictionary[Config_section.P_RESIZE_FACTORS.value][index]
+        indexes[level_index] = index + 1
+        return finished
+
+
+    datasets=[general_data]
+    write_datasets=[general_data.copy()]
+    for dataset in resize_data:
+        datasets.append(dataset)
+        write_datasets.append(dataset.copy())
+    for dataset in stage_data:
+        datasets.append(dataset)
+        write_datasets.append(dataset.copy())
+
+    directory = "/Desglosed-" + str(date.today())
+    path = os.getcwd() + directory
+    os.mkdir(path, mode=0o775)
+    os.chdir(path)
+
+    lists=[]
+    keys=[]
+    indexes=[]
+    mindexes=[]
+    ds_indexes=[]
+    for i in range(len(datasets)):
+        values_aux = list(datasets[i].values())
+        keys_aux = list(datasets[i].keys())
+        for j in range(len(values_aux)):
+            if type(values_aux[j]) == list and keys_aux[j] != Config_section.P_RESIZE_FACTORS.value:
+                keys.append(keys_aux[j])
+                lists.append(values_aux[j])
+                ds_indexes.append(i)
+                indexes.append(0)
+                mindexes.append(len(values_aux[j]))
+
+    for i in range(len(lists)):
+        read_parameter(i)
+    output_index=0
+    while True:
+        if(check_sections_assumptions(write_datasets)):
+            correct_adr(write_datasets[0])
+            write_output_file(write_datasets, common_output_name, output_index)
+#            for i in range(len(write_datasets)):
+#                print(write_datasets[i])
+#            print("\n\n\n------------------------------------------" + str(output_index))
+            output_index+=1
+        finished = read_parameter(0)
+        if finished:
+            break
+    
+
+if(len(sys.argv) < 3):
+    print("Not enough arguments given.\nExpected usage: python3 read_multiple.py file.ini output_name")
+name = sys.argv[1]
+common_output_name = sys.argv[2]
+
+general_data, resize_data, stage_data = process_file(name)
+create_output_files(common_output_name, general_data, resize_data, stage_data)
+
+exit(1)

Exec/multipleRuns.sh

+#!/bin/bash
+
+dir="/home/martini/malleability_benchmark/"
+codeDir="Codes/"
+execDir="Exec/"
+ResultsDir="Results/"
+
+$comples_file=$1
+$output_name=$2
+
+python3 $dir$execDir/PythonCodes/read_multiple.py $complex_file $output_name
+
+echo "END TEST"
+

Exec/runAll.sh

@@ -4,7 +4,10 @@ exclude="c00,c01,c02"
 procs=(2 10 20 40 80 120 160)
 cores=20
 
-
+#Runs in a given current directory all .ini files that contains 
+# in the name an element of the array "procs".
+# Parameter 1(Optional) - Amount of executions per file. Must be a positive number
+# Parameter 2(Optional) - Index to use for the resulting files of the execution. Must be a number.
 #====== Do not modify these values =======
 
 codeDir="/Codes/build"