import sys import glob import numpy as np import pandas as pd from enum import Enum class G_enum(Enum): TOTAL_RESIZES = 0 TOTAL_GROUPS = 1 SDR = 2 ADR = 3 DR = 4 RED_METHOD = 5 RED_STRATEGY = 6 SPAWN_METHOD = 7 SPAWN_STRATEGY = 8 GROUPS = 9 ITERS = 10 T_SPAWN = 11 T_SR = 12 T_AR = 13 T_MALLEABILITY = 14 T_TOTAL = 15 #Malleability specific NP = 0 NC = 1 columnsG = ["Total_Resizes", "Total_Groups", "SDR", "ADR", "DR", "Redistribution_Method", \ "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", \ "Iters", "T_spawn", "T_SR", "T_AR", "T_Malleability", "T_total"] #16 #----------------------------------------------- # Obtains the value of a given index in a splited line # and returns it as a float values if possible, string otherwise def get_value(line, index, separator=True): if separator: value = line[index].split('=')[1].split(',')[0] else: value = line[index] try: value = float(value) if value.is_integer(): value = int(value) except ValueError: return value return value #----------------------------------------------- def record_config_line(lineS, aux_data): aux_data[0] = int(lineS[1].split("=")[1]) aux_data[1] = int(lineS[2].split("=")[1]) aux_data[2] = int(lineS[3].split("=")[1]) aux_data[3] = int(lineS[4]) aux_data[4] = int(lineS[5]) aux_data[5] = int(lineS[7].split("=")[1]) # SM RM SS # 0 1 2 3 4 5 #Test numP=160 numC=120 Meths=0 0 1 -- Is_synch=1 qty=5 # 0 1 2 3 4 5 6 7 8 #----------------------------------------------- # Obtains the general parameters of an execution and # stores them for creating a global dataframe def record_new_line(lineS, dataG_it, data_aux): offset_line = 2 sdr = 3947883504 adr = sdr * 0.966 dataG_it[G_enum.TOTAL_RESIZES.value] = 1 dataG_it[G_enum.TOTAL_GROUPS.value] = dataG_it[G_enum.TOTAL_RESIZES.value]+1 # Init lists for each column array_groups = [G_enum.GROUPS.value, G_enum.ITERS.value, G_enum.RED_METHOD.value, \ G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value] array_resizes = [ G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value] for index in array_groups: dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_GROUPS.value] for index in array_resizes: dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_RESIZES.value] dataG_it[G_enum.GROUPS.value][0] = data_aux[0] dataG_it[G_enum.GROUPS.value][1] = data_aux[1] dataG_it[G_enum.ITERS.value][0] = dataG_it[G_enum.ITERS.value][1] = 500 dataG_it[G_enum.SPAWN_METHOD.value][0] = dataG_it[G_enum.RED_METHOD.value][0] = 0 dataG_it[G_enum.SPAWN_METHOD.value][1] = data_aux[2] dataG_it[G_enum.RED_METHOD.value][1] = data_aux[3] dataG_it[G_enum.SPAWN_STRATEGY.value][0] = dataG_it[G_enum.RED_STRATEGY.value][0] = 1 dataG_it[G_enum.SPAWN_STRATEGY.value][1] = dataG_it[G_enum.RED_STRATEGY.value][1] = data_aux[4] dataG_it[G_enum.SDR.value] = sdr dataG_it[G_enum.ADR.value] = 0 if data_aux[5] == 0: #Is asynch send dataG_it[G_enum.ADR.value] = adr dataG_it[G_enum.SDR.value] -= adr dataG_it[G_enum.DR.value] = dataG_it[G_enum.SDR.value] + dataG_it[G_enum.ADR.value] #----------------------------------------------- def record_time_line(lineS, dataG_it): T_names = ["T_spawn:", "T_SR:", "T_AR:", "T_Malleability:", "T_total:"] T_values = [G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value, G_enum.T_TOTAL.value] if not (lineS[0] in T_names): # Execute only if line represents a Time return index = T_names.index(lineS[0]) index = T_values[index] offset_lines = 1 len_index = 1 if dataG_it[index] != None: len_index = len(dataG_it[index]) for i in range(len_index): dataG_it[index][i] = get_value(lineS, i+offset_lines, False) else: dataG_it[index] = get_value(lineS, offset_lines, False) #----------------------------------------------- def read_global_file(f, dataG, it): runs_in_file=0 aux_data = [0,0,0,0,0,0] for line in f: lineS = line.split() if len(lineS) > 0: if lineS[0] == "Test": # CONFIG LINE record_config_line(lineS, aux_data) elif lineS[0] == "Start": it += 1 runs_in_file += 1 group = 0 dataG.append([None]*len(columnsG)) record_new_line(lineS, dataG[it], aux_data) elif it>-1: print(lineS) print("== "+ str(it) + " "+str(runs_in_file) ) record_time_line(lineS, dataG[it]) return it,runs_in_file #----------------------------------------------- def convert_to_tuples(dfG): array_list_items = [G_enum.GROUPS.value, G_enum.ITERS.value, \ G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, \ G_enum.SPAWN_STRATEGY.value, G_enum.T_SPAWN.value, G_enum.T_SR.value, \ G_enum.T_AR.value, G_enum.T_MALLEABILITY.value] for item in array_list_items: name = columnsG[item] values = dfG[name].copy() for index in range(len(values)): values[index] = tuple(values[index]) dfG[name] = values #----------------------------------------------- if len(sys.argv) < 2: print("The files name is missing\nUsage: python3 MallTimes.py commonName directory OutName") exit(1) common_name = sys.argv[1] if len(sys.argv) >= 3: BaseDir = sys.argv[2] print("Searching in directory: "+ BaseDir) else: BaseDir = "./" if len(sys.argv) >= 4: name = sys.argv[3] else: name = "data" print("File name will be: " + name + "G.pkl") lista = (glob.glob(BaseDir + common_name + "*.out")) # Se utiliza cuando solo hay un nivel de directorios print("Number of files found: "+ str(len(lista))); it = -1 dataG = [] for elem in lista: f = open(elem, "r") it,runs_in_file = read_global_file(f, dataG, it) f.close() dfG = pd.DataFrame(dataG, columns=columnsG) convert_to_tuples(dfG) print(dfG) dfG.to_pickle(name + 'G.pkl') #dfM = pd.DataFrame(dataM, columns=columnsM) #Poner en TC el valor real y en TH el necesario para la app #cond = dfM.TH != 0 #dfM.loc[cond, ['TC', 'TH']] = dfM.loc[cond, ['TH', 'TC']].values #dfM.to_csv(name + 'M.csv')