Cambios sobre recopiladores para obtener tiempos de creacion de procesos -...

Cambios sobre recopiladores para obtener tiempos de creacion de procesos - Analizador (ipynb) tiene graficas automatizadas

Analysis/MallTimes.py

@@ -60,31 +60,38 @@ def read_file(f, dataA, dataB, it):
       if lineS[0] == "Config": # CONFIG LINE
         recording = True
         it += 1
-        dataA.append([None]*9)
-        dataB.append([None]*13)
-        resizes = int(lineS[2].split('=')[1].split(',')[0])
+        dataA.append([None]*13)
+        dataB.append([None]*15)
+        #resizes = int(lineS[2].split('=')[1].split(',')[0])
+        resizes = 2
         compute_tam = int(lineS[3].split('=')[1].split(',')[0])
         comm_tam = int(lineS[4].split('=')[1].split(',')[0])
         sdr = int(lineS[5].split('=')[1].split(',')[0])
         adr = int(lineS[6].split('=')[1].split(',')[0]) #TODO Que lo tome como porcentaje
+        css = int(lineS[8].split('=')[1].split(',')[0])
+        cst = int(lineS[9].split('=')[1].split(',')[0])
         # TODO Que obtenga Aib
-        time = float(lineS[8].split('=')[1])
+        time = float(lineS[10].split('=')[1])
 
         dataB[it][0] = sdr
         dataB[it][1] = adr 
         dataB[it][4] = "" 
         dataB[it][5] = compute_tam
         dataB[it][6] = comm_tam
-        dataB[it][7] = time
-        dataB[it][8] = "" 
+        dataB[it][7] = cst
+        dataB[it][8] = css
+        dataB[it][9] = time
+        dataB[it][10] = "" 
 
         dataA[it][0] = sdr
         dataA[it][1] = adr 
-        dataA[it][3] = ""
-        dataA[it][4] = compute_tam
-        dataA[it][5] = comm_tam
-        dataA[it][6] = time
-        dataA[it][7] = ""
+        dataA[it][5] = ""
+        dataA[it][6] = compute_tam
+        dataA[it][7] = comm_tam
+        dataA[it][8] = cst
+        dataA[it][9] = css
+        dataA[it][10] = time
+        dataA[it][11] = ""
 
       elif recording and resizes != 0: # RESIZE LINE
         iters = int(lineS[2].split('=')[1].split(',')[0])
@@ -95,44 +102,49 @@ def read_file(f, dataA, dataB, it):
         if resizes == 0:
           dataB[it][3] = npr
           dataB[it][4] += dist
-          dataB[it][8] += str(iters)
+          dataB[it][10] += str(iters)
 
+          dataA[it][4] = npr #FIXME No sera correcta si hay mas de una reconfig
           dataA[it][2] = str(previousNP) + "," + str(npr)
-          dataA[it][3] += dist
-          dataA[it][7] += str(iters)
+          dataA[it][5] += dist
+          dataA[it][11] += str(iters)
           timer = 4
         else:
           dataB[it][2] = npr
           dataB[it][4] += dist + ","
-          dataB[it][8] += str(iters) + ","
+          dataB[it][10] += str(iters) + ","
 
-          dataA[it][3] += dist + ","
-          dataA[it][7] += str(iters) + ","
+          dataA[it][3] = npr
+          dataA[it][5] += dist + ","
+          dataA[it][11] += str(iters) + ","
           previousNP = npr
 
       else: # SAVE TIMES
         if timer == 4:
-          dataB[it][9] = float(lineS[1])
+          dataB[it][11] = float(lineS[1])
         elif timer == 3:
-          dataB[it][10] = float(lineS[1])
+          dataB[it][12] = float(lineS[1])
         elif timer == 2:
-          dataB[it][11] = float(lineS[1])
+          dataB[it][13] = float(lineS[1])
         elif timer == 1:
-          dataB[it][12] = float(lineS[1])
+          dataB[it][14] = float(lineS[1])
         else:
-          dataA[it][8] = float(lineS[1])
+          dataA[it][12] = float(lineS[1])
         timer = timer - 1
           
   return it
-#columnsA1 = ["N", "%Async", "Groups", "Dist", "Matrix", "CommTam", "Time", "Iters", "TE"] #8
-#columnsB1 = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Time", "Iters", "TC", "TS", "TA"] #12
+#columnsA1 = ["N", "%Async", "Groups", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TE"] #8
+#columnsB1 = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TC", "TS", "TA"] #12
 #Config loaded: resizes=2, matrix=1000, sdr=1000000000, adr=0, aib=0, time=2.000000 || grp=1
 #Resize 0: Iters=100, Procs=2, Factors=1.000000, Phy=2
 #Resize 1: Iters=100, Procs=4, Factors=0.500000, Phy=2
 #Tspawn: 0.249393 
+#Tthread: 0 
 #Tsync: 0.330391 
 #Tasync: 0
 #Tex: 301.428615
+
+#Config loaded: resizes=1, matrix=0, comm_tam=0, sdr=0, adr=0, aib=0, cst=3, css=1, time=1 || grp=1
 #-----------------------------------------------
 
 if len(sys.argv) < 2:
@@ -158,8 +170,8 @@ print("Number of files found: "+ str(len(lista)));
 it = -1
 dataA = []
 dataB = []
-columnsA = ["N", "%Async", "Groups", "Dist", "Matrix", "CommTam", "Time", "Iters", "TE"] #9
-columnsB = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Time", "Iters", "TC", "TH", "TS", "TA"] #13
+columnsA = ["N", "%Async", "Groups", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TE"] #13
+columnsB = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TC", "TH", "TS", "TA"] #15
 
 for elem in lista:
   f = open(elem, "r")

Analysis/iterTimes.py

 import sys
 import glob
-import numpy as np
+import numpy as numpy
 import pandas as pd
 
 #-----------------------------------------------
-def read_file(f, data, it):
+def read_file(f, dataA, dataB, itA, itB):
   compute_tam = 0
   comm_tam = 0
   sdr = 0
   adr = 0
   dist = 0
+  css = 0
+  cst = 0
   time = 0
   recording = False
   it_line = 0
-  aux_it = 0
+  aux_itA = 0
+  aux_itB = 0
   iters = 0
   np = 0
   np_par = 0
   ns = 0
+  array = []
   columnas = ['Titer','Ttype','Top']
 
   #print(f)
@@ -26,45 +30,72 @@ def read_file(f, data, it):
     if len(lineS) > 1:
 
         if recording and lineS[0].split(':')[0] in columnas: #Record data
-          aux_it = 0
+          aux_itA = 0
           lineS.pop(0)
 
           if it_line==0:
             for observation in lineS:
-              data.append([None]*13)
-              data[it+aux_it][0] = sdr
-              data[it+aux_it][1] = adr
-              data[it+aux_it][2] = np
-              data[it+aux_it][3] = np_par
-              data[it+aux_it][4] = ns
-              data[it+aux_it][5] = dist
-              data[it+aux_it][6] = compute_tam
-              data[it+aux_it][7] = comm_tam
-              data[it+aux_it][8] = time
-              data[it+aux_it][9] = iters
-              data[it+aux_it][10] = float(observation)
-              aux_it+=1
+              dataA.append([None]*15)
+              dataA[itA+aux_itA][0] = sdr
+              dataA[itA+aux_itA][1] = adr
+              dataA[itA+aux_itA][2] = np
+              dataA[itA+aux_itA][3] = np_par
+              dataA[itA+aux_itA][4] = ns
+              dataA[itA+aux_itA][5] = dist
+              dataA[itA+aux_itA][6] = compute_tam
+              dataA[itA+aux_itA][7] = comm_tam
+              dataA[itA+aux_itA][8] = cst
+              dataA[itA+aux_itA][9] = css
+              dataA[itA+aux_itA][10] = time
+              dataA[itA+aux_itA][11] = iters
+              dataA[itA+aux_itA][12] = float(observation)
+              array.append(float(observation))
+              aux_itA+=1
           elif it_line==1:
+            deleted = 0
             for observation in lineS:
-              data[it+aux_it][11] = float(observation)
-              aux_it+=1
+              dataA[itA+aux_itA][13] = float(observation)
+              if float(observation) == 0:
+                  array.pop(aux_itA - deleted)
+                  deleted+=1
+              aux_itA+=1
           else:
             for observation in lineS:
-              data[it+aux_it][12] = float(observation)
-              aux_it+=1
+              dataA[itA+aux_itA][14] = float(observation)
+              aux_itA+=1
 
           it_line += 1
           if(it_line % 3 == 0): # Comprobar si se ha terminado de mirar esta ejecucion
             recording = False
             it_line = 0
-            it = it + aux_it
+            itA = itA + aux_itA
+            if ns != 0: # Solo obtener datos de grupos con hijos
+                dataB.append([None]*14)
+                dataB[itB][0] = sdr
+                dataB[itB][1] = adr
+                dataB[itB][2] = np
+                dataB[itB][3] = np_par
+                dataB[itB][4] = ns
+                dataB[itB][5] = dist
+                dataB[itB][6] = compute_tam
+                dataB[itB][7] = comm_tam
+                dataB[itB][8] = cst
+                dataB[itB][9] = css
+                dataB[itB][10] = time
+                dataB[itB][11] = iters
+                dataB[itB][12] = tuple(array)
+                dataB[itB][13] = numpy.sum(array)
+                itB+=1
+                array = []
 
         if lineS[0] == "Config:":
           compute_tam = int(lineS[1].split('=')[1].split(',')[0])
           comm_tam = int(lineS[2].split('=')[1].split(',')[0])
           sdr = int(lineS[3].split('=')[1].split(',')[0])
           adr = int(lineS[4].split('=')[1].split(',')[0]) 
-          time = float(lineS[6].split('=')[1])
+          css = int(lineS[6].split('=')[1].split(',')[0])
+          cst = int(lineS[7].split('=')[1].split(',')[0])
+          time = float(lineS[8].split('=')[1])
         elif lineS[0] == "Config":
           recording = True
           iters = int(lineS[2].split('=')[1].split(',')[0])
@@ -73,7 +104,7 @@ def read_file(f, data, it):
           np_par = int(lineS[6].split('=')[1].split(',')[0])
           ns = int(float(lineS[7].split('=')[1]))
 
-  return it
+  return itA,itB
 #-----------------------------------------------
 #Config: matrix=1000, sdr=1000000000, adr=0, aib=0 time=2.000000
 #Config Group: iters=100, factor=1.000000, phy=2, procs=2, parents=0, sons=4
@@ -90,7 +121,7 @@ else:
     BaseDir = sys.argv[2]
 
 if len(sys.argv) >= 4:
-  print("Csv name will be: " + sys.argv[3] + ".csv")
+  print("Csv name will be: " + sys.argv[3] + ".csv and "+ sys.argv[3] + "_Total.csv")
   name = sys.argv[3]
 else:
   name = "data"
@@ -99,18 +130,25 @@ insideDir = "Run"
 lista = glob.glob("./" + BaseDir + insideDir + "*/" + sys.argv[1]+ "*ID*.o*")
 print("Number of files found: "+ str(len(lista)));
 
-it = 0
-data = []  #0   #1        #2    #3       #4     #5      #6             #7          #8       #9   #10    #11    #12
-columns = ["N", "%Async", "NP", "N_par", "NS", "Dist", "Compute_tam", "Comm_tam", "Time", "Iters", "Ti", "Tt", "To"] #13
+itA = itB = 0
+dataA = []
+dataB = []  #0   #1        #2    #3       #4     #5      #6             #7          #8     #9   #10    #11       #12   #13   #14
+columnsA = ["N", "%Async", "NP", "N_par", "NS", "Dist", "Compute_tam", "Comm_tam", "Cst", "Css","Time", "Iters", "Ti", "Tt", "To"] #15
+columnsB = ["N", "%Async", "NP", "N_par", "NS", "Dist", "Compute_tam", "Comm_tam", "Cst", "Css","Time", "Iters", "Ti", "Sum"] #14
 
 for elem in lista:
   f = open(elem, "r")
-  it = read_file(f, data, it)
+  itA,itB = read_file(f, dataA, dataB, itA, itB)
   f.close()
 
 #print(data)
-df = pd.DataFrame(data, columns=columns)
+dfA = pd.DataFrame(dataA, columns=columnsA)
+dfB = pd.DataFrame(dataB, columns=columnsB)
 
-df['N'] += df['%Async']
-df['%Async'] = (df['%Async'] / df['N']) * 100
-df.to_csv(name + '.csv')
+dfA['N'] += dfA['%Async']
+dfA['%Async'] = (dfA['%Async'] / dfA['N']) * 100
+dfA.to_csv(name + '.csv')
+
+dfB['N'] += dfB['%Async']
+dfB['%Async'] = (dfB['%Async'] / dfB['N']) * 100
+dfB.to_csv(name + '_Total.csv')