MallTimes.py

import sys
import glob
import numpy as np
import pandas as pd
from enum import Enum

class G_enum(Enum):
    TOTAL_RESIZES = 0
    TOTAL_GROUPS = 1
    SDR = 2
    ADR = 3
    DR = 4
    RED_METHOD = 5
    RED_STRATEGY = 6
    SPAWN_METHOD = 7
    SPAWN_STRATEGY = 8
    GROUPS = 9
    ITERS = 10
    T_SPAWN = 11
    T_SR = 12
    T_AR = 13
    T_MALLEABILITY = 14
    T_TOTAL = 15
    #Malleability specific
    NP = 0
    NC = 1


columnsG = ["Total_Resizes", "Total_Groups", "SDR", "ADR", "DR", "Redistribution_Method", \
            "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", \
            "Iters", "T_spawn", "T_SR", "T_AR", "T_Malleability", "T_total"] #16

#-----------------------------------------------
# Obtains the value of a given index in a splited line
# and returns it as a float values if possible, string otherwise
def get_value(line, index, separator=True):
  if separator:
    value = line[index].split('=')[1].split(',')[0]
  else:
    value = line[index]

  try:
    value = float(value)
    if value.is_integer():
      value = int(value)
  except ValueError:
    return value
  return value

#-----------------------------------------------
def record_config_line(lineS, aux_data):
  aux_data[0] = int(lineS[1].split("=")[1])
  aux_data[1] = int(lineS[2].split("=")[1])
  aux_data[2] = int(lineS[3].split("=")[1])
  aux_data[3] = int(lineS[4])
  aux_data[4] = int(lineS[5])
  aux_data[5] = int(lineS[7].split("=")[1])
#                           SM RM SS
#          0        1         2 3 4             5
#Test numP=160 numC=120 Meths=0 0 1 -- Is_synch=1 qty=5
#  0      1        2         3  4 5 6  7          8


#-----------------------------------------------
# Obtains the general parameters of an execution and
# stores them for creating a global dataframe
def record_new_line(lineS, dataG_it, data_aux):
  offset_line = 2
  sdr = 3947883504
  adr = sdr * 0.966

  dataG_it[G_enum.TOTAL_RESIZES.value] = 1
  dataG_it[G_enum.TOTAL_GROUPS.value] = dataG_it[G_enum.TOTAL_RESIZES.value]+1

  # Init lists for each column
  array_groups = [G_enum.GROUPS.value, G_enum.ITERS.value, G_enum.RED_METHOD.value, \
          G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value]

  array_resizes = [ G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]

  for index in array_groups:
    dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_GROUPS.value]
  for index in array_resizes:
    dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_RESIZES.value]

  dataG_it[G_enum.GROUPS.value][0] = data_aux[0]
  dataG_it[G_enum.GROUPS.value][1] = data_aux[1] 
  dataG_it[G_enum.ITERS.value][0] = dataG_it[G_enum.ITERS.value][1] = 500

  dataG_it[G_enum.SPAWN_METHOD.value][0] = dataG_it[G_enum.RED_METHOD.value][0] = 0
  dataG_it[G_enum.SPAWN_METHOD.value][1] = data_aux[2]
  dataG_it[G_enum.RED_METHOD.value][1] = data_aux[3]
  dataG_it[G_enum.SPAWN_STRATEGY.value][0] = dataG_it[G_enum.RED_STRATEGY.value][0] = 1
  dataG_it[G_enum.SPAWN_STRATEGY.value][1] = dataG_it[G_enum.RED_STRATEGY.value][1] = data_aux[4]

  dataG_it[G_enum.SDR.value] = sdr
  dataG_it[G_enum.ADR.value] = 0
  if data_aux[5] == 0: #Is asynch send
    dataG_it[G_enum.ADR.value] = adr
    dataG_it[G_enum.SDR.value] -= adr
  dataG_it[G_enum.DR.value] = dataG_it[G_enum.SDR.value] + dataG_it[G_enum.ADR.value]

#-----------------------------------------------
def record_time_line(lineS, dataG_it):
  T_names = ["T_spawn:", "T_SR:", "T_AR:", "T_Malleability:", "T_total:"]
  T_values = [G_enum.T_SPAWN.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value, G_enum.T_TOTAL.value]
  if not (lineS[0] in T_names): # Execute only if line represents a Time
      return

  index = T_names.index(lineS[0])
  index = T_values[index]
  offset_lines = 1

  len_index = 1
  if dataG_it[index] != None:
    len_index = len(dataG_it[index])
    for i in range(len_index):
      dataG_it[index][i] = get_value(lineS, i+offset_lines, False)
  else:
      dataG_it[index] = get_value(lineS, offset_lines, False)

#-----------------------------------------------
def read_global_file(f, dataG, it):
  runs_in_file=0
  aux_data = [0,0,0,0,0,0]
  for line in f: 
    lineS = line.split()

    if len(lineS) > 0:
      if lineS[0] == "Test": # CONFIG LINE
        record_config_line(lineS, aux_data)
      elif lineS[0] == "Start":
        it += 1
        runs_in_file += 1
        group = 0

        dataG.append([None]*len(columnsG))
        record_new_line(lineS, dataG[it], aux_data)
      elif it>-1:
        record_time_line(lineS, dataG[it])

  return it,runs_in_file

#-----------------------------------------------
def convert_to_tuples(dfG):
  array_list_items = [G_enum.GROUPS.value, G_enum.ITERS.value, \
          G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, \
          G_enum.SPAWN_STRATEGY.value, G_enum.T_SPAWN.value, G_enum.T_SR.value, \
          G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]

  for item in array_list_items:
    name = columnsG[item]
    values = dfG[name].copy()
    for index in range(len(values)):
      values[index] = tuple(values[index])
    dfG[name] = values

#-----------------------------------------------

if len(sys.argv) < 2:
    print("The files name is missing\nUsage: python3 MallTimes.py commonName directory OutName")
    exit(1)

common_name = sys.argv[1]
if len(sys.argv) >= 3:
    BaseDir = sys.argv[2]
    print("Searching in directory: "+ BaseDir)
else:
    BaseDir = "./"

if len(sys.argv) >= 4:
  name = sys.argv[3]
else:
  name = "data"
print("File name will be: " + name + "G.pkl")

lista = (glob.glob(BaseDir + common_name + "*.out")) # Se utiliza cuando solo hay un nivel de directorios
print("Number of files found: "+ str(len(lista)));

it = -1
dataG = []

for elem in lista:
  f = open(elem, "r")
  it,runs_in_file = read_global_file(f, dataG, it)
  f.close()

dfG = pd.DataFrame(dataG, columns=columnsG)
convert_to_tuples(dfG)
print(dfG)
dfG.to_pickle(name + 'G.pkl')

#dfM = pd.DataFrame(dataM, columns=columnsM)

#Poner en TC el valor real y en TH el necesario para la app
#cond = dfM.TH != 0
#dfM.loc[cond, ['TC', 'TH']] = dfM.loc[cond, ['TH', 'TC']].values
#dfM.to_csv(name + 'M.csv')