Commit 2639ab13 authored by Iker Martín Álvarez's avatar Iker Martín Álvarez
Browse files

Merge branch 'dev' into 'master'

New version of Proteo

See merge request martini/malleability_benchmark!6
parents 26305fac e83b5922
#Ignore ini files
*.ini
#Ignore build files
/Codes/build/
/Codes/MaM/build
/Codes/SAM/build
#Ignore Results files
/Results/*
import sys
import glob
import numpy as np
import pandas as pd
from enum import Enum
class G_enum(Enum):
TOTAL_RESIZES = 0
TOTAL_GROUPS = 1
TOTAL_STAGES = 2
GRANULARITY = 3
SDR = 4
ADR = 5
DR = 6
RED_METHOD = 7
RED_STRATEGY = 8
SPAWN_METHOD = 9
SPAWN_STRATEGY = 10
GROUPS = 11
FACTOR_S = 12
DIST = 13
STAGE_TYPES = 14
STAGE_TIMES = 15
STAGE_BYTES = 16
ITERS = 17
ASYNCH_ITERS = 18
T_ITER = 19
T_STAGES = 20
T_SPAWN = 21
T_SPAWN_REAL = 22
T_SR = 23
T_AR = 24
T_MALLEABILITY = 25
T_TOTAL = 26
#Malleability specific
NP = 0
NC = 1
#Iteration specific
IS_DYNAMIC = 11
N_PARENTS = 17
#columnsG = ["Total_Resizes", "Total_Groups", "Total_Stages", "Granularity", "SDR", "ADR", "DR", "Redistribution_Method", \
# "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", "FactorS", "Dist", "Stage_Types", "Stage_Times", \
# "Stage_Bytes", "Iters", "Asynch_Iters", "T_iter", "T_stages", "T_spawn", "T_spawn_real", "T_SR", "T_AR", "T_Malleability", "T_total"] #27
columnsL = ["NP", "NC", "Total_Stages", "Granularity", "SDR", "ADR", "DR", "Redistribution_Method", \
"Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Is_Dynamic", "FactorS", "Dist", "Stage_Types", "Stage_Times", \
"Stage_Bytes", "N_Parents", "Asynch_Iters", "T_iter", "T_stages"] #20
def copy_iteration(row, dataL_it, group, iteration, is_asynch):
basic_indexes = [G_enum.TOTAL_STAGES.value, G_enum.GRANULARITY.value, \
G_enum.STAGE_TYPES.value, G_enum.STAGE_TIMES.value, G_enum.STAGE_BYTES.value]
basic_asynch = [G_enum.SDR.value, G_enum.ADR.value, G_enum.DR.value]
array_asynch_group = [G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, \
G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value, G_enum.DIST.value]
dataL_it[G_enum.FACTOR_S.value] = row[G_enum.FACTOR_S.value][group]
dataL_it[G_enum.NP.value] = row[G_enum.GROUPS.value][group]
dataL_it[G_enum.ASYNCH_ITERS.value] = is_asynch
dataL_it[G_enum.T_ITER.value] = row[G_enum.T_ITER.value][group][iteration]
dataL_it[G_enum.T_STAGES.value] = list(row[G_enum.T_STAGES.value][group][iteration])
dataL_it[G_enum.IS_DYNAMIC.value] = True if group > 0 else False
for index in basic_indexes:
dataL_it[index] = row[index]
for index in array_asynch_group:
dataL_it[index] = [None, -1]
dataL_it[index][0] = row[index][group]
dataL_it[G_enum.N_PARENTS.value] = -1
if group > 0:
dataL_it[G_enum.N_PARENTS.value] = row[G_enum.GROUPS.value][group-1]
if is_asynch:
dataL_it[G_enum.NC.value] = row[G_enum.GROUPS.value][group+1]
for index in basic_asynch:
dataL_it[index] = row[index]
for index in array_asynch_group:
dataL_it[index][1] = row[index][group+1]
for index in array_asynch_group: # Convert to tuple
dataL_it[index] = tuple(dataL_it[index])
#-----------------------------------------------
def write_iter_dataframe(dataL, name, i, first=False):
dfL = pd.DataFrame(dataL, columns=columnsL)
dfL.to_pickle(name + str(i) + '.pkl')
if first:
print(dfL)
#-----------------------------------------------
def create_iter_dataframe(dfG, name, max_it_L):
it = -1
file_i = 0
first = True
dataL = []
for row_index in range(len(dfG)):
row = dfG.iloc[row_index]
groups = row[G_enum.TOTAL_GROUPS.value]
for group in range(groups):
real_iterations = len(row[G_enum.T_ITER.value][group])
real_asynch = row[G_enum.ASYNCH_ITERS.value][group]
is_asynch = False
for iteration in range(real_iterations-real_asynch):
it += 1
dataL.append( [None] * len(columnsL) )
copy_iteration(row, dataL[it], group, iteration, is_asynch)
is_asynch = True
for iteration in range(real_iterations-real_asynch, real_iterations):
it += 1
dataL.append( [None] * len(columnsL) )
copy_iteration(row, dataL[it], group, iteration, is_asynch)
if it >= max_it_L-1: #Var "it" starts at -1, so one more must be extracted for precise cut
write_iter_dataframe(dataL, name, file_i, first)
dataL = []
file_i += 1
first = False
it = -1
if it != -1:
write_iter_dataframe(dataL, name, file_i)
#-----------------------------------------------
if len(sys.argv) < 2:
print("The files name is missing\nUsage: python3 CreateIterDataframe.py input_file.pkl output_name [max_rows_per_file]")
exit(1)
input_name = sys.argv[1]
if len(sys.argv) > 2:
name = sys.argv[2]
else:
name = "dataL"
print("File names will be: " + name + ".pkl")
if len(sys.argv) > 3:
max_it_L = int(sys.argv[3])
else:
max_it_L = 100000
dfG = pd.read_pickle(input_name)
print(dfG)
create_iter_dataframe(dfG, name, max_it_L)
import sys
import glob
import numpy as np
import pandas as pd
from enum import Enum
class G_enum(Enum):
TOTAL_RESIZES = 0
TOTAL_GROUPS = 1
TOTAL_STAGES = 2
GRANULARITY = 3
SDR = 4
ADR = 5
DR = 6
RED_METHOD = 7
RED_STRATEGY = 8
SPAWN_METHOD = 9
SPAWN_STRATEGY = 10
GROUPS = 11
FACTOR_S = 12
DIST = 13
STAGE_TYPES = 14
STAGE_TIMES = 15
STAGE_BYTES = 16
ITERS = 17
ASYNCH_ITERS = 18
T_ITER = 19
T_STAGES = 20
T_SPAWN = 21
T_SPAWN_REAL = 22
T_SR = 23
T_AR = 24
T_MALLEABILITY = 25
T_TOTAL = 26
#Malleability specific
NP = 0
NC = 1
#Iteration specific
IS_DYNAMIC = 11
N_PARENTS = 17
#columnsG = ["Total_Resizes", "Total_Groups", "Total_Stages", "Granularity", "SDR", "ADR", "DR", "Redistribution_Method", \
# "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", "FactorS", "Dist", "Stage_Types", "Stage_Times", \
# "Stage_Bytes", "Iters", "Asynch_Iters", "T_iter", "T_stages", "T_spawn", "T_spawn_real", "T_SR", "T_AR", "T_Malleability", "T_total"] #27
columnsM = ["NP", "NC", "Total_Stages", "Granularity", "SDR", "ADR", "DR", "Redistribution_Method", \
"Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "FactorS", "Dist", "Stage_Type", "Stage_Time", \
"Stage_Bytes", "Iters", "Asynch_Iters", "T_iter", "T_stages", "T_spawn", "T_spawn_real", "T_SR", "T_AR", "T_Malleability"] #25
def copy_resize(row, dataM_it, resize):
basic_indexes = [G_enum.TOTAL_STAGES.value, G_enum.GRANULARITY.value, G_enum.SDR.value, \
G_enum.ADR.value, G_enum.DR.value]
basic_group = [G_enum.STAGE_TYPES.value, G_enum.STAGE_TIMES.value, G_enum.STAGE_BYTES.value]
array_actual_group = [G_enum.FACTOR_S.value, G_enum.ITERS.value, G_enum.ASYNCH_ITERS.value, \
G_enum.T_SPAWN.value, G_enum.T_SPAWN_REAL.value, G_enum.T_SR.value, \
G_enum.T_AR.value, G_enum.T_MALLEABILITY.value, G_enum.T_ITER.value, G_enum.T_STAGES.value]
array_next_group = [G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, \
G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value]
dataM_it[G_enum.NP.value] = row[G_enum.GROUPS.value][resize]
dataM_it[G_enum.NC.value] = row[G_enum.GROUPS.value][resize+1]
dataM_it[G_enum.DIST.value-1] = [None, None]
dataM_it[G_enum.DIST.value-1][0] = row[G_enum.DIST.value][resize]
dataM_it[G_enum.DIST.value-1][1] = row[G_enum.DIST.value][resize+1]
for index in basic_indexes:
dataM_it[index] = row[index]
for index in basic_group:
dataM_it[index-1] = row[index]
for index in array_actual_group:
dataM_it[index-1] = row[index][resize]
for index in array_next_group:
dataM_it[index] = row[index][resize+1]
#-----------------------------------------------
def create_resize_dataframe(dfG, dataM):
it = -1
for row_index in range(len(dfG)):
row = dfG.iloc[row_index]
resizes = row[G_enum.TOTAL_RESIZES.value]
for resize in range(resizes):
it += 1
dataM.append( [None] * len(columnsM) )
copy_resize(row, dataM[it], resize)
#-----------------------------------------------
if len(sys.argv) < 2:
print("The files name is missing\nUsage: python3 CreateResizeDataframe.py input_file.pkl output_name")
exit(1)
input_name = sys.argv[1]
if len(sys.argv) > 2:
name = sys.argv[2]
else:
name = "dataM"
print("File name will be: " + name + ".pkl")
dfG = pd.read_pickle(input_name)
dataM = []
create_resize_dataframe(dfG, dataM)
dfM = pd.DataFrame(dataM, columns=columnsM)
dfM.to_pickle(name + '.pkl')
print(dfG)
print(dfM)
...@@ -2,189 +2,299 @@ import sys ...@@ -2,189 +2,299 @@ import sys
import glob import glob
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from enum import Enum
def getData(lineS, outData, tp, hasIter = False): class G_enum(Enum):
for data in lineS: TOTAL_RESIZES = 0
k_v = data.split('=') TOTAL_GROUPS = 1
if k_v[0] == "time": TOTAL_STAGES = 2
time = float(k_v[1]) GRANULARITY = 3
elif k_v[0] == "iters" and hasIter: SDR = 4
iters = int(k_v[1]) ADR = 5
DR = 6
RED_METHOD = 7
RED_STRATEGY = 8
SPAWN_METHOD = 9
SPAWN_STRATEGY = 10
GROUPS = 11
FACTOR_S = 12
DIST = 13
STAGE_TYPES = 14
STAGE_TIMES = 15
STAGE_BYTES = 16
ITERS = 17
ASYNCH_ITERS = 18
T_ITER = 19
T_STAGES = 20
T_SPAWN = 21
T_SPAWN_REAL = 22
T_SR = 23
T_AR = 24
T_MALLEABILITY = 25
T_TOTAL = 26
#Malleability specific
NP = 0
NC = 1
#Iteration specific
IS_DYNAMIC = 11
N_PARENTS = 17
outData[tp] = time
if hasIter: columnsG = ["Total_Resizes", "Total_Groups", "Total_Stages", "Granularity", "SDR", "ADR", "DR", "Redistribution_Method", \
outData[tp+1] = iters "Redistribution_Strategy", "Spawn_Method", "Spawn_Strategy", "Groups", "FactorS", "Dist", "Stage_Types", "Stage_Times", \
"Stage_Bytes", "Iters", "Asynch_Iters", "T_iter", "T_stages", "T_spawn", "T_spawn_real", "T_SR", "T_AR", "T_Malleability", "T_total"] #27
#-----------------------------------------------
# Obtains the value of a given index in a splited line
# and returns it as a float values if possible, string otherwise
def get_value(line, index, separator=True):
if separator:
value = line[index].split('=')[1].split(',')[0]
else:
value = line[index]
try:
value = float(value)
if value.is_integer():
value = int(value)
except ValueError:
return value
return value
#-----------------------------------------------
# Obtains the general parameters of an execution and
# stores them for creating a global dataframe
def record_config_line(lineS, dataG_it):
ordered_indexes = [G_enum.TOTAL_RESIZES.value, G_enum.TOTAL_STAGES.value, \
G_enum.GRANULARITY.value, G_enum.SDR.value, G_enum.ADR.value]
offset_line = 2
for i in range(len(ordered_indexes)):
value = get_value(lineS, i+offset_line)
index = ordered_indexes[i]
dataG_it[index] = value
dataG_it[G_enum.TOTAL_GROUPS.value] = dataG_it[G_enum.TOTAL_RESIZES.value]+1
#FIXME Modificar cuando ADR ya no sea un porcentaje
dataG_it[G_enum.DR.value] = dataG_it[G_enum.SDR.value] + dataG_it[G_enum.ADR.value]
# Init lists for each column
array_groups = [G_enum.GROUPS.value, G_enum.FACTOR_S.value, G_enum.DIST.value, G_enum.ITERS.value, \
G_enum.ASYNCH_ITERS.value, G_enum.T_ITER.value, G_enum.T_STAGES.value, G_enum.RED_METHOD.value, \
G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, G_enum.SPAWN_STRATEGY.value,]
array_resizes = [ G_enum.T_SPAWN.value, G_enum.T_SPAWN_REAL.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value]
array_stages = [G_enum.STAGE_TYPES.value, \
G_enum.STAGE_TIMES.value, G_enum.STAGE_BYTES.value]
for index in array_groups:
dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_GROUPS.value]
for group in range(dataG_it[G_enum.TOTAL_GROUPS.value]):
dataG_it[G_enum.T_ITER.value][group] = []
for index in array_resizes:
dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_RESIZES.value]
for index in array_stages:
dataG_it[index] = [None]*dataG_it[G_enum.TOTAL_STAGES.value]
#-----------------------------------------------
# Obtains the parameters of a stage line
# and stores it in the dataframe
# Is needed to indicate in which stage is
# being performed
def record_stage_line(lineS, dataG_it, stage):
array_stages = [G_enum.STAGE_TYPES.value, \
G_enum.STAGE_TIMES.value, G_enum.STAGE_BYTES.value]
offset_lines = 2
for i in range(len(array_stages)):
value = get_value(lineS, i+offset_lines)
index = array_stages[i]
dataG_it[index][stage] = value
#-----------------------------------------------
# Obtains the parameters of a resize line
# and stores them in the dataframe
# Is needed to indicate to which group refers
# the resize line
# Group 0: Iters=3, Procs=80, Factors=0.037500, Dist=2, RM=0, SM=0, RS=0, SS=0
def record_group_line(lineS, dataG_it, group):
array_groups = [G_enum.ITERS.value, G_enum.GROUPS.value, G_enum.FACTOR_S.value, G_enum.DIST.value, \
G_enum.RED_METHOD.value, G_enum.SPAWN_METHOD.value, G_enum.RED_STRATEGY.value, G_enum.SPAWN_STRATEGY.value]
offset_lines = 2
for i in range(len(array_groups)):
value = get_value(lineS, i+offset_lines)
index = array_groups[i]
dataG_it[index][group] = value
#-----------------------------------------------
def record_time_line(lineS, dataG_it):
T_names = ["T_spawn:", "T_spawn_real:", "T_SR:", "T_AR:", "T_Malleability:", "T_total:"]
T_values = [G_enum.T_SPAWN.value, G_enum.T_SPAWN_REAL.value, G_enum.T_SR.value, G_enum.T_AR.value, G_enum.T_MALLEABILITY.value, G_enum.T_TOTAL.value]
if not (lineS[0] in T_names): # Execute only if line represents a Time
return
index = T_names.index(lineS[0])
index = T_values[index]
offset_lines = 1
len_index = 1
if dataG_it[index] != None:
len_index = len(dataG_it[index])
for i in range(len_index):
dataG_it[index][i] = get_value(lineS, i+offset_lines, False)
else:
dataG_it[index] = get_value(lineS, offset_lines, False)
#----------------------------------------------- #-----------------------------------------------
def record(f, observation, line): def record_multiple_times_line(lineS, dataG_it, group):
# Record first line - General info T_names = ["T_iter:", "T_stage"]
lineS = line.split() T_values = [G_enum.T_ITER.value, G_enum.T_STAGES.value]
for j in range(1,7): if not (lineS[0] in T_names): # Execute only if line represents a Time
observation[j] = int(lineS[j].split('=')[1]) return
# Record procces number
line = next(f)
lineS = line.split()
j = 7
for key_values in lineS:
k_v = key_values.split('=')
observation[j] = int(k_v[1])
j+=1
# Record data
j = 9
for j in range(9, 13):
line = next(f)
lineS = line.split()
getData(lineS, observation, j)
line = next(f)
lineS = line.split()
#if observation[0] == "A":
getData(lineS, observation, 13, True)
#else:
# getData(lineS, observation, 13)
index = T_names.index(lineS[0])
index = T_values[index]
offset_lines = 1
if index == G_enum.T_STAGES.value:
offset_lines += 1
total_iters = len(lineS)-offset_lines
stage = int(lineS[1].split(":")[0])
if stage == 0:
dataG_it[index][group] = [None] * total_iters
for i in range(total_iters):
dataG_it[index][group][i] = [None] * dataG_it[G_enum.TOTAL_STAGES.value]
for i in range(total_iters):
dataG_it[index][group][i][stage] = get_value(lineS, i+offset_lines, False)
else:
total_iters = len(lineS)-offset_lines
for i in range(total_iters):
dataG_it[index][group].append(get_value(lineS, i+offset_lines, False))
#----------------------------------------------- #-----------------------------------------------
def read_file(f, dataA, dataB, it): def read_local_file(f, dataG, it, runs_in_file):
recording = False offset = 0
resizes = 0 real_it = 0
timer = 0 group = 0
previousNP = 0
for line in f:
lineS = line.split()
if len(lineS) > 0:
if lineS[0] == "Group": # GROUP number
offset += 1
real_it = it - (runs_in_file-offset)
group = int(lineS[1].split(":")[0])
elif lineS[0] == "Async_Iters:":
offset_line = 1
dataG[real_it][G_enum.ASYNCH_ITERS.value][group] = get_value(lineS, offset_line, False)
else:
record_multiple_times_line(lineS, dataG[real_it], group)
#-----------------------------------------------
def read_global_file(f, dataG, it):
runs_in_file=0
for line in f: for line in f:
lineS = line.split() lineS = line.split()
if len(lineS) > 0: if len(lineS) > 0:
if lineS[0] == "Config": # CONFIG LINE if lineS[0] == "Config": # CONFIG LINE
recording = True
it += 1 it += 1
dataA.append([None]*13) runs_in_file += 1
dataB.append([None]*15) group = 0
#resizes = int(lineS[2].split('=')[1].split(',')[0]) stage = 0
resizes = 2
compute_tam = int(lineS[3].split('=')[1].split(',')[0]) dataG.append([None]*len(columnsG))
comm_tam = int(lineS[4].split('=')[1].split(',')[0]) record_config_line(lineS, dataG[it])
sdr = int(lineS[5].split('=')[1].split(',')[0])
adr = int(lineS[6].split('=')[1].split(',')[0]) #TODO Que lo tome como porcentaje elif lineS[0] == "Stage":
css = int(lineS[8].split('=')[1].split(',')[0]) record_stage_line(lineS, dataG[it], stage)
cst = int(lineS[9].split('=')[1].split(',')[0]) stage+=1
# TODO Que obtenga Aib elif lineS[0] == "Group":
time = float(lineS[10].split('=')[1]) record_group_line(lineS, dataG[it], group)
group+=1
dataB[it][0] = sdr else:
dataB[it][1] = adr record_time_line(lineS, dataG[it])
dataB[it][4] = ""
dataB[it][5] = compute_tam return it,runs_in_file
dataB[it][6] = comm_tam
dataB[it][7] = cst #-----------------------------------------------
dataB[it][8] = css
dataB[it][9] = time
dataB[it][10] = "" #-----------------------------------------------
def convert_to_tuples(dfG):
dataA[it][0] = sdr array_list_items = [G_enum.GROUPS.value, G_enum.FACTOR_S.value, G_enum.DIST.value, G_enum.ITERS.value, \
dataA[it][1] = adr G_enum.ASYNCH_ITERS.value, G_enum.RED_METHOD.value, G_enum.RED_STRATEGY.value, G_enum.SPAWN_METHOD.value, \
dataA[it][5] = "" G_enum.SPAWN_STRATEGY.value, G_enum.T_SPAWN.value, G_enum.T_SPAWN_REAL.value, G_enum.T_SR.value, \
dataA[it][6] = compute_tam G_enum.T_AR.value, G_enum.STAGE_TYPES.value, G_enum.STAGE_TIMES.value, G_enum.STAGE_BYTES.value]
dataA[it][7] = comm_tam #TODO Falta T_malleability?
dataA[it][8] = cst array_multiple_list_items = [G_enum.T_ITER.value, G_enum.T_STAGES.value]
dataA[it][9] = css for item in array_list_items:
dataA[it][10] = time name = columnsG[item]
dataA[it][11] = "" values = dfG[name].copy()
for index in range(len(values)):
elif recording and resizes != 0: # RESIZE LINE values[index] = tuple(values[index])
iters = int(lineS[2].split('=')[1].split(',')[0]) dfG[name] = values
npr = int(lineS[3].split('=')[1].split(',')[0])
dist = lineS[5].split('=')[1] for item in array_multiple_list_items:
name = columnsG[item]
resizes = resizes - 1 values = dfG[name].copy()
if resizes == 0: for i in range(len(values)):
dataB[it][3] = npr for j in range(len(values[i])):
dataB[it][4] += dist if(type(values[i][j][0]) == list):
dataB[it][10] += str(iters) for r in range(len(values[i][j])):
values[i][j][r] = tuple(values[i][j][r])
dataA[it][4] = npr #FIXME No sera correcta si hay mas de una reconfig values[i][j] = tuple(values[i][j])
dataA[it][2] = str(previousNP) + "," + str(npr) values[i] = tuple(values[i])
dataA[it][5] += dist dfG[name] = values
dataA[it][11] += str(iters)
timer = 4
else:
dataB[it][2] = npr
dataB[it][4] += dist + ","
dataB[it][10] += str(iters) + ","
dataA[it][3] = npr
dataA[it][5] += dist + ","
dataA[it][11] += str(iters) + ","
previousNP = npr
else: # SAVE TIMES
if timer == 4:
dataB[it][11] = float(lineS[1])
elif timer == 3:
dataB[it][12] = float(lineS[1])
elif timer == 2:
dataB[it][13] = float(lineS[1])
elif timer == 1:
dataB[it][14] = float(lineS[1])
else:
dataA[it][12] = float(lineS[1])
timer = timer - 1
return it
#columnsA1 = ["N", "%Async", "Groups", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TE"] #8
#columnsB1 = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TC", "TS", "TA"] #12
#Config loaded: resizes=2, matrix=1000, sdr=1000000000, adr=0, aib=0, time=2.000000 || grp=1
#Resize 0: Iters=100, Procs=2, Factors=1.000000, Phy=2
#Resize 1: Iters=100, Procs=4, Factors=0.500000, Phy=2
#Tspawn: 0.249393
#Tthread: 0
#Tsync: 0.330391
#Tasync: 0
#Tex: 301.428615
#Config loaded: resizes=1, matrix=0, comm_tam=0, sdr=0, adr=0, aib=0, cst=3, css=1, time=1 || grp=1
#----------------------------------------------- #-----------------------------------------------
if len(sys.argv) < 2: if len(sys.argv) < 2:
print("The files name is missing\nUsage: python3 iterTimes.py resultsName directory csvOutName") print("The files name is missing\nUsage: python3 MallTimes.py commonName directory OutName")
exit(1) exit(1)
common_name = sys.argv[1]
if len(sys.argv) >= 3: if len(sys.argv) >= 3:
BaseDir = sys.argv[2] BaseDir = sys.argv[2]
print("Searching in directory: "+ BaseDir) print("Searching in directory: "+ BaseDir)
else: else:
BaseDir = sys.argv[2] BaseDir = "./"
if len(sys.argv) >= 4: if len(sys.argv) >= 4:
print("Csv name will be: " + sys.argv[3] + "G.csv & " + sys.argv[3] + "M.csv")
name = sys.argv[3] name = sys.argv[3]
else: else:
name = "data" name = "data"
print("File name will be: " + name + "G.pkl")
insideDir = "Run" insideDir = "Run"
lista = glob.glob("./" + BaseDir + insideDir + "*/" + sys.argv[1]+ "*Global.o*") lista = glob.glob(BaseDir + insideDir + "*/" + common_name + "*_Global.out")
lista += (glob.glob(BaseDir + common_name + "*_Global.out")) # Se utiliza cuando solo hay un nivel de directorios
print("Number of files found: "+ str(len(lista))); print("Number of files found: "+ str(len(lista)));
it = -1 it = -1
dataA = [] dataG = []
dataB = []
columnsA = ["N", "%Async", "Groups", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TE"] #13
columnsB = ["N", "%Async", "NP", "NS", "Dist", "Matrix", "CommTam", "Cst", "Css", "Time", "Iters", "TC", "TH", "TS", "TA"] #15
for elem in lista: for elem in lista:
f = open(elem, "r") f = open(elem, "r")
it = read_file(f, dataA, dataB, it) id_run = elem.split("_Global.out")[0].split(common_name)[-1]
lista_local = glob.glob(BaseDir + common_name + id_run + "_G*NP*.out")
it,runs_in_file = read_global_file(f, dataG, it)
f.close() f.close()
for elem_local in lista_local:
f_local = open(elem_local, "r")
read_local_file(f_local, dataG, it, runs_in_file)
f_local.close()
#print(data) dfG = pd.DataFrame(dataG, columns=columnsG)
dfA = pd.DataFrame(dataA, columns=columnsA) convert_to_tuples(dfG)
dfA.to_csv(name + 'G.csv') print(dfG)
dfG.to_pickle(name + 'G.pkl')
dfB = pd.DataFrame(dataB, columns=columnsB) #dfM = pd.DataFrame(dataM, columns=columnsM)
#Poner en TC el valor real y en TH el necesario para la app #Poner en TC el valor real y en TH el necesario para la app
cond = dfB.TH != 0 #cond = dfM.TH != 0
dfB.loc[cond, ['TC', 'TH']] = dfB.loc[cond, ['TH', 'TC']].values #dfM.loc[cond, ['TC', 'TH']] = dfM.loc[cond, ['TH', 'TC']].values
dfB.to_csv(name + 'M.csv') #dfM.to_csv(name + 'M.csv')
This source diff could not be displayed because it is too large. You can view the blob instead.
'''
Created on Oct 24, 2016
@author: David Llorens (dllorens@uji.es)
(c) Universitat Jaume I 2016
@license: GPL2
'''
from abc import ABCMeta, abstractmethod
infinity = float("infinity")
## Esquema para BT básico --------------------------------------------------------------------------
class PartialSolution(metaclass=ABCMeta):
@abstractmethod
def is_solution(self)-> "bool":
pass
@abstractmethod
def get_solution(self) -> "solution":
pass
@abstractmethod
def successors(self) -> "IEnumerable<PartialSolution>":
pass
class BacktrackingSolver(metaclass=ABCMeta):
@staticmethod
def solve(initial_ps : "PartialSolution") -> "IEnumerable<Solution>":
def bt(ps):
if ps.is_solution():
yield ps.get_solution()
else:
for new_ps in ps.successors():
yield from bt(new_ps)
yield from bt(initial_ps)
class BacktrackingSolverOld(metaclass=ABCMeta):
def solve(self, initial_ps : "PartialSolution") -> "IEnumerable<Solution>":
def bt(ps):
if ps.is_solution():
return [ps.get_solution()]
else:
solutions = []
for new_ps in ps.successors():
solutions.extend(bt(new_ps))
return solutions
return bt(initial_ps)
## Esquema para BT con control de visitados --------------------------------------------------------
class PartialSolutionWithVisitedControl(PartialSolution):
@abstractmethod
def state(self)-> "state":
# the returned object must be of an inmutable type
pass
class BacktrackingVCSolver(metaclass=ABCMeta):
@staticmethod
def solve(initial_ps : "PartialSolutionWithVisitedControl") -> "IEnumerable<Solution>":
def bt(ps):
seen.add(ps.state())
if ps.is_solution():
yield ps.get_solution()
else:
for new_ps in ps.successors():
state = new_ps.state()
if state not in seen:
yield from bt(new_ps)
seen = set()
yield from bt(initial_ps)
## Esquema para BT para optimización ----------------------------------------------------------------
class PartialSolutionWithOptimization(PartialSolutionWithVisitedControl):
@abstractmethod
def f(self)-> "int or double":
# result of applying the objective function to the partial solution
pass
class BacktrackingOptSolver(metaclass=ABCMeta):
@staticmethod
def solve(initial_ps : "PartialSolutionWithOptimization") -> "IEnumerable<Solution>":
def bt(ps):
nonlocal best_solution_found_score
ps_score = ps.f()
best_seen[ps.state()] = ps_score
if ps.is_solution() and ps_score < best_solution_found_score: #sólo muestra una solución si mejora la última mostrada
best_solution_found_score = ps_score
yield ps.get_solution()
else:
for new_ps in ps.successors():
state = new_ps.state()
if state not in best_seen or new_ps.f() < best_seen[state]:
yield from bt(new_ps)
best_seen = {}
best_solution_found_score = infinity
yield from bt(initial_ps)
import sys
import glob
import numpy as numpy
import pandas as pd
#-----------------------------------------------
def read_file(f, dataA, dataB, itA, itB):
compute_tam = 0
comm_tam = 0
sdr = 0
adr = 0
dist = 0
css = 0
cst = 0
time = 0
recording = False
it_line = 0
aux_itA = 0
aux_itB = 0
iters = 0
np = 0
np_par = 0
ns = 0
array = []
columnas = ['Titer','Ttype','Top']
#print(f)
for line in f:
lineS = line.split()
if len(lineS) > 1:
if recording and lineS[0].split(':')[0] in columnas: #Record data
aux_itA = 0
lineS.pop(0)
if it_line==0:
for observation in lineS:
dataA.append([None]*15)
dataA[itA+aux_itA][0] = sdr
dataA[itA+aux_itA][1] = adr
dataA[itA+aux_itA][2] = np
dataA[itA+aux_itA][3] = np_par
dataA[itA+aux_itA][4] = ns
dataA[itA+aux_itA][5] = dist
dataA[itA+aux_itA][6] = compute_tam
dataA[itA+aux_itA][7] = comm_tam
dataA[itA+aux_itA][8] = cst
dataA[itA+aux_itA][9] = css
dataA[itA+aux_itA][10] = time
dataA[itA+aux_itA][11] = iters
dataA[itA+aux_itA][12] = float(observation)
array.append(float(observation))
aux_itA+=1
elif it_line==1:
deleted = 0
for observation in lineS:
dataA[itA+aux_itA][13] = float(observation)
if float(observation) == 0:
array.pop(aux_itA - deleted)
deleted+=1
aux_itA+=1
else:
for observation in lineS:
dataA[itA+aux_itA][14] = float(observation)
aux_itA+=1
it_line += 1
if(it_line % 3 == 0): # Comprobar si se ha terminado de mirar esta ejecucion
recording = False
it_line = 0
itA = itA + aux_itA
if ns != 0: # Solo obtener datos de grupos con hijos
dataB.append([None]*14)
dataB[itB][0] = sdr
dataB[itB][1] = adr
dataB[itB][2] = np
dataB[itB][3] = np_par
dataB[itB][4] = ns
dataB[itB][5] = dist
dataB[itB][6] = compute_tam
dataB[itB][7] = comm_tam
dataB[itB][8] = cst
dataB[itB][9] = css
dataB[itB][10] = time
dataB[itB][11] = iters
dataB[itB][12] = tuple(array)
dataB[itB][13] = numpy.sum(array)
itB+=1
array = []
if lineS[0] == "Config:":
compute_tam = int(lineS[1].split('=')[1].split(',')[0])
comm_tam = int(lineS[2].split('=')[1].split(',')[0])
sdr = int(lineS[3].split('=')[1].split(',')[0])
adr = int(lineS[4].split('=')[1].split(',')[0])
css = int(lineS[6].split('=')[1].split(',')[0])
cst = int(lineS[7].split('=')[1].split(',')[0])
time = float(lineS[8].split('=')[1])
elif lineS[0] == "Config":
recording = True
iters = int(lineS[2].split('=')[1].split(',')[0])
dist = int(lineS[4].split('=')[1].split(',')[0])
np = int(lineS[5].split('=')[1].split(',')[0])
np_par = int(lineS[6].split('=')[1].split(',')[0])
ns = int(float(lineS[7].split('=')[1]))
return itA,itB
#-----------------------------------------------
#Config: matrix=1000, sdr=1000000000, adr=0, aib=0 time=2.000000
#Config Group: iters=100, factor=1.000000, phy=2, procs=2, parents=0, sons=4
#Ttype: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
if len(sys.argv) < 2:
print("The files name is missing\nUsage: python3 iterTimes.py resultsName directory csvOutName")
exit(1)
if len(sys.argv) >= 3:
BaseDir = sys.argv[2]
print("Searching in directory: "+ BaseDir)
else: #FIXME
BaseDir = sys.argv[2]
if len(sys.argv) >= 4:
print("Csv name will be: " + sys.argv[3] + ".csv and "+ sys.argv[3] + "_Total.csv")
name = sys.argv[3]
else:
name = "data"
insideDir = "Run"
lista = glob.glob("./" + BaseDir + insideDir + "*/" + sys.argv[1]+ "*ID*.o*")
print("Number of files found: "+ str(len(lista)));
itA = itB = 0
dataA = []
dataB = [] #0 #1 #2 #3 #4 #5 #6 #7 #8 #9 #10 #11 #12 #13 #14
columnsA = ["N", "%Async", "NP", "N_par", "NS", "Dist", "Compute_tam", "Comm_tam", "Cst", "Css","Time", "Iters", "Ti", "Tt", "To"] #15
columnsB = ["N", "%Async", "NP", "N_par", "NS", "Dist", "Compute_tam", "Comm_tam", "Cst", "Css","Time", "Iters", "Ti", "Sum"] #14
for elem in lista:
f = open(elem, "r")
itA,itB = read_file(f, dataA, dataB, itA, itB)
f.close()
#print(data)
dfA = pd.DataFrame(dataA, columns=columnsA)
dfB = pd.DataFrame(dataB, columns=columnsB)
dfA['N'] += dfA['%Async']
dfA['%Async'] = (dfA['%Async'] / dfA['N']) * 100
dfA.to_csv(name + '.csv')
dfB['N'] += dfB['%Async']
dfB['%Async'] = (dfB['%Async'] / dfB['N']) * 100
dfB.to_csv(name + '_Total.csv')
...@@ -3,22 +3,19 @@ import glob ...@@ -3,22 +3,19 @@ import glob
import numpy as numpy import numpy as numpy
import pandas as pd import pandas as pd
if len(sys.argv) < 3: if len(sys.argv) < 3:
print("The files name is missing\nUsage: python3 joinDf.py resultsName1.csv resultsName2.csv csvOutName") print("The files name is missing\nUsage: python3 joinDf.py resultsName1.pkl resultsName2.pkl OutName")
exit(1) exit(1)
if len(sys.argv) >= 4: if len(sys.argv) >= 4:
print("Csv name will be: " + sys.argv[3] + ".csv")
name = sys.argv[3] name = sys.argv[3]
else: else:
name = "dataJOINED" name = "dataJOINED"
df1 = pd.read_csv( sys.argv[1] ) print("File name will be: " + name + ".pkl")
df2 = pd.read_csv( sys.argv[2] ) df1 = pd.read_pickle( sys.argv[1] )
df2 = pd.read_pickle( sys.argv[2] )
frames = [df1, df2] frames = [df1, df2]
df3 = pd.concat(frames) df3 = pd.concat(frames)
df3 = df3.drop(columns=df3.columns[0])
df3.to_csv(name + '.csv') df3.to_pickle(name + '.pkl')
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import pandas as pd\n",
"from pandas import DataFrame, Series\n",
"import numpy as np\n",
"import math\n",
"\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.patches as mpatches\n",
"import matplotlib.colors as colors\n",
"from matplotlib.legend_handler import HandlerLine2D, HandlerTuple\n",
"from matplotlib.colors import LinearSegmentedColormap\n",
"from scipy import stats\n",
"import scikit_posthocs as sp\n",
"import sys\n",
"\n",
"from mpl_toolkits.mplot3d import axes3d"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"AllName=\"dataG.pkl\"\n",
"ResizesName=\"dataM.pkl\"\n",
"ItersName=\"dataL\"\n",
"NameExtension=\".pkl\"\n",
"n_cores=20\n",
"repet = 5 #CAMBIAR EL NUMERO SEGUN NUMERO DE EJECUCIONES POR CONFIG\n",
"\n",
"significance_value = 0.05\n",
"processes = [2,10,20,40,80,120,160]\n",
"\n",
"positions = [321, 322, 323, 324, 325]\n",
"positions_small = [221, 222, 223, 224]\n",
"\n",
"labels = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n",
" '(10,1)', '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n",
" '(20,1)', '(20,10)', '(20,40)', '(20,80)', '(20,120)','(20,160)',\n",
" '(40,1)', '(40,10)', '(40,20)', '(40,80)', '(40,120)','(40,160)',\n",
" '(80,1)', '(80,10)', '(80,20)', '(80,40)', '(80,120)','(80,160)',\n",
" '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)','(120,160)',\n",
" '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n",
"\n",
"labelsExpand = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n",
" '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n",
" '(20,40)', '(20,80)', '(20,120)','(20,160)',\n",
" '(40,80)', '(40,120)','(40,160)',\n",
" '(80,120)','(80,160)',\n",
" '(120,160)']\n",
"labelsShrink = ['(10,1)', \n",
" '(20,1)', '(20,10)',\n",
" '(40,1)', '(40,10)', '(40,20)',\n",
" '(80,1)', '(80,10)', '(80,20)', '(80,40)',\n",
" '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)',\n",
" '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n",
"\n",
"# WORST BEST\n",
"labels_dist = ['null', 'SpreadFit', 'CompactFit']\n",
" #0 #1 #2 #3\n",
"labelsMethods = ['Baseline', 'Baseline single','Baseline - Asynchronous','Baseline single - Asynchronous',\n",
" 'Merge','Merge single','Merge - Asynchronous','Merge single - Asynchronous']\n",
" #4 #5 #6 #7\n",
" \n",
"colors_m = ['green','darkgreen','red','darkred','mediumseagreen','seagreen','palegreen','springgreen','indianred','firebrick','darkgoldenrod','saddlebrown']\n",
"linestyle_m = ['-', '--', '-.', ':']\n",
"markers_m = ['.','v','s','p', 'h','d','X','P','^']\n",
"\n",
"OrMult_patch = mpatches.Patch(hatch='', facecolor='green', label='Baseline')\n",
"OrSing_patch = mpatches.Patch(hatch='', facecolor='springgreen', label='Baseline single')\n",
"OrPthMult_patch = mpatches.Patch(hatch='//', facecolor='blue', label='Baseline - Asyncrhonous')\n",
"OrPthSing_patch = mpatches.Patch(hatch='\\\\', facecolor='darkblue', label='Baseline single - Asyncrhonous')\n",
"MergeMult_patch = mpatches.Patch(hatch='||', facecolor='red', label='Merge')\n",
"MergeSing_patch = mpatches.Patch(hatch='...', facecolor='darkred', label='Merge single')\n",
"MergePthMult_patch = mpatches.Patch(hatch='xx', facecolor='yellow', label='Merge - Asyncrhonous')\n",
"MergePthSing_patch = mpatches.Patch(hatch='++', facecolor='olive', label='Merge single - Asyncrhonous')\n",
"\n",
"handles_spawn = [OrMult_patch,OrSing_patch,OrPthMult_patch,OrPthSing_patch,MergeMult_patch,MergeSing_patch,MergePthMult_patch,MergePthSing_patch]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dfG = pd.read_pickle( AllName )\n",
"\n",
"dfG['ADR'] = round((dfG['ADR'] / dfG['DR']) * 100,1)\n",
"dfG['SDR'] = round((dfG['SDR'] / dfG['DR']) * 100,1)\n",
" \n",
"out_group = dfG.groupby(['Groups', 'ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_total']\n",
"group = dfG.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','Groups'])['T_total']\n",
"\n",
"grouped_aggG = group.agg(['median'])\n",
"grouped_aggG.rename(columns={'median':'T_total'}, inplace=True) \n",
"\n",
"out_grouped_G = out_group.agg(['median'])\n",
"out_grouped_G.rename(columns={'median':'T_total'}, inplace=True) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dfM = pd.read_pickle( ResizesName )\n",
"\n",
"dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)\n",
"dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)\n",
"dfM['T_Redistribution'] = dfM['T_SR'] + dfM['T_AR']\n",
"dfM.loc[dfM['T_Malleability']==0,'T_Malleability'] = dfM['T_spawn'] + dfM['T_Redistribution']\n",
" \n",
"out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']\n",
"group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']\n",
"\n",
"grouped_aggM = group.agg(['median'])\n",
"grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)\n",
"\n",
"out_grouped_M = out_group.agg(['median'])\n",
"out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"grouped_aggLAsynch = pd.DataFrame()\n",
"grouped_aggLSynch = pd.DataFrame()\n",
"grouped_aggLDyn = pd.DataFrame()\n",
"grouped_aggLNDyn = pd.DataFrame()\n",
"\n",
"for i in range(26):\n",
" aux_name = ItersName + str(i) + NameExtension\n",
" dfL = pd.read_pickle( aux_name )\n",
"\n",
" #Fixme comprobar si hay iters asincronas antes de esto\n",
" aux_df = dfL[(dfL.Asynch_Iters == True)]\n",
" #aux_df['ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)\n",
" dfL.loc[(dfL.Asynch_Iters == True),'ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)\n",
" #dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)\n",
" dfL.loc[(dfL.Asynch_Iters == True),'SDR'] = round((aux_df['SDR'] / aux_df['DR']) * 100,1)\n",
" dfL['ADR'].fillna(-1, inplace=True)\n",
" dfL['SDR'].fillna(-1, inplace=True)\n",
" dfL['DR'].fillna(-1, inplace=True)\n",
" \n",
" aux_df = dfL[(dfL.Asynch_Iters == True)]\n",
" group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter']\n",
" grouped_aux = group.agg(['median','count'])\n",
" grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n",
" grouped_aux.rename(columns={'median':'T_iter'}, inplace=True) \n",
" group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_stages']\n",
" aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n",
" grouped_aux['T_stages'] = aux_column\n",
" if grouped_aggLAsynch.empty:\n",
" grouped_aggLAsynch = grouped_aux\n",
" else:\n",
" grouped_aggLAsynch = pd.concat([grouped_aggLAsynch, grouped_aux], ignore_index=False, sort=True).sort_index()\n",
"\n",
" aux_df = dfL[(dfL.Asynch_Iters == False)]\n",
" group = aux_df.groupby('NP')['T_iter']\n",
" grouped_aux = group.agg(['median'])\n",
" grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n",
" group = aux_df.groupby(['NP'])['T_stages']\n",
" aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n",
" grouped_aux['T_stages'] = aux_column\n",
" if grouped_aggLSynch.empty:\n",
" grouped_aggLSynch = grouped_aux\n",
" else:\n",
" grouped_aggLSynch = pd.concat([grouped_aggLSynch, grouped_aux], ignore_index=False, sort=True).sort_index()\n",
"\n",
" aux_df2 = aux_df[(aux_df.Is_Dynamic == True)]\n",
" group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_iter']\n",
" grouped_aux = group.agg(['median'])\n",
" grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n",
" group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_stages']\n",
" aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n",
" grouped_aux['T_stages'] = aux_column\n",
" if grouped_aggLDyn.empty:\n",
" grouped_aggLDyn = grouped_aux\n",
" else:\n",
" grouped_aggLDyn = pd.concat([grouped_aggLDyn, grouped_aux], ignore_index=False, sort=True).sort_index()\n",
"\n",
" aux_df2 = aux_df[(aux_df.Is_Dynamic == False)]\n",
" group = aux_df2.groupby('NP')['T_iter']\n",
" grouped_aux = group.agg(['median'])\n",
" grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n",
" group = aux_df2.groupby(['NP'])['T_stages']\n",
" aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n",
" grouped_aux['T_stages'] = aux_column\n",
" if grouped_aggLNDyn.empty:\n",
" grouped_aggLNDyn = grouped_aux\n",
" else:\n",
" grouped_aggLNDyn = pd.concat([grouped_aggLNDyn, grouped_aux], ignore_index=False, sort=True).sort_index()\n",
"dfL = None\n",
"\n",
"group = grouped_aggLAsynch.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])\n",
"grouped_aux = group.agg(['mean'])\n",
"grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n",
"grouped_aux['T_sum'] = grouped_aux['count'] * grouped_aux['T_iter']\n",
"grouped_aggLAsynch = grouped_aux\n",
"\n",
"group = grouped_aggLSynch.groupby('NP')\n",
"grouped_aux = group.agg(['mean'])\n",
"grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n",
"grouped_aggLSynch = grouped_aux\n",
"\n",
"group = grouped_aggLDyn.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])\n",
"grouped_aux = group.agg(['mean'])\n",
"grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n",
"grouped_aggLDyn = grouped_aux\n",
"\n",
"group = grouped_aggLNDyn.groupby('NP')\n",
"grouped_aux = group.agg(['mean'])\n",
"grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n",
"grouped_aggLNDyn = grouped_aux"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from bt_scheme import PartialSolution, BacktrackingSolver\n",
"def elegirConf(parameters):\n",
" class StatePS(PartialSolution):\n",
" def __init__(self, config):\n",
" self.config= config\n",
" self.n= len(config) #Indica el valor a añadir\n",
"\n",
" def is_solution(self):\n",
" return self.n == len(parameters)\n",
"\n",
" def get_solution(self):\n",
" return tuple(self.config)\n",
"\n",
" def successors(self):\n",
" array = parameters[self.n]\n",
" for parameter_value in array: #Test all values of the next parameter\n",
" self.config.append(parameter_value)\n",
" yield StatePS(self.config)\n",
" self.config.pop()\n",
"\n",
" initialPs= StatePS([])\n",
" return BacktrackingSolver().solve(initialPs)\n",
"\n",
"\n",
"def obtenerConfs(parameters):\n",
" soluciones=[]\n",
" for solucion in elegirConf(parameters):\n",
" soluciones.append(solucion)\n",
" return soluciones\n",
"\n",
"def modifyToGlobal(parameters, len_parameters, configuration):\n",
" usable_configuration = []\n",
" for i in range(len(parameters)):\n",
" if len_parameters[i] > 1:\n",
" aux = (parameters[i][0], configuration[i])\n",
" else:\n",
" aux = (configuration[i])\n",
" usable_configuration.append(aux)\n",
" \n",
" return usable_configuration\n",
"\n",
"def modifyToLocalDynamic(parameters, len_parameters, configuration):\n",
" usable_configuration = []\n",
" for i in range(len(parameters)):\n",
" if len_parameters[i] > 1:\n",
" aux = (configuration[i], -1)\n",
" else:\n",
" aux = (-1)\n",
" usable_configuration.append(aux)\n",
" \n",
" return tuple(usable_configuration)\n",
"\n",
"def CheckConfExists(configuration, dataSet, type_conf='global'):\n",
" exists = False\n",
" config = list(configuration)\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" \n",
" if type_conf == 'global':\n",
" config.append((np_aux, ns_aux))\n",
" elif type_conf == 'malleability':\n",
" config.append(np_aux)\n",
" config.append(ns_aux)\n",
" elif type_conf == 'local':\n",
" config.append(np_aux)\n",
" \n",
" if tuple(config) in dataSet.index: \n",
" exists = True # FIXME Return here true?\n",
" config.pop()\n",
" \n",
" if type_conf == 'malleability':\n",
" config.pop()\n",
" return exists"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"adr = [0,96.6]\n",
"sp_method = [0,1]\n",
"rd_method = [0,1]\n",
"rd_strat = [1,2]\n",
"parameters = [adr, sp_method, rd_method, rd_strat]\n",
"parameters_names = ['ADR', 'Spawn_Method', 'Redistribution_Method', 'Redistribution_Strategy']\n",
"len_parameters = [1,2,2,2]\n",
"configurations_aux = obtenerConfs(parameters)\n",
"configurations = []\n",
"configurations_local_dynamic = set()\n",
"configurations_local = set()\n",
"configurations_simple = []\n",
"for checked_conf in configurations_aux:\n",
" aux_conf = modifyToGlobal(parameters, len_parameters, checked_conf)\n",
" if CheckConfExists(aux_conf, grouped_aggG):\n",
" configurations.append(aux_conf)\n",
"\n",
" if CheckConfExists(checked_conf, grouped_aggM, 'malleability'):\n",
" configurations_simple.append(list(checked_conf))\n",
" \n",
" aux_conf = modifyToLocalDynamic(parameters, len_parameters, checked_conf)\n",
" if CheckConfExists(aux_conf, grouped_aggLDyn, 'local'):\n",
" configurations_local_dynamic.add(aux_conf)\n",
"\n",
"configurations_local_dynamic = list(configurations_local_dynamic)\n",
"for index in range(len(configurations_local_dynamic)):\n",
" configurations_local_dynamic[index] = list(configurations_local_dynamic[index])\n",
"\n",
"print(configurations_simple)\n",
"print(configurations_local_dynamic)\n",
"print(configurations)\n",
"print(len(configurations))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#ALPHA COMPUTATION\n",
"def compute_alpha(config_a, config_b):\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" config_a.append(np_aux)\n",
" config_a.append(ns_aux)\n",
" config_b.append(np_aux)\n",
" config_b.append(ns_aux)\n",
" grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']\n",
" #grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Redistribution'] / grouped_aggM.loc[tuple(config_a),'T_Redistribution']\n",
" config_a.pop()\n",
" config_a.pop()\n",
" config_b.pop()\n",
" config_b.pop()\n",
" \n",
" \n",
" config_a.insert(0,ns_aux)\n",
" config_a.insert(0,np_aux)\n",
" config_b.insert(0,ns_aux)\n",
" config_b.insert(0,np_aux)\n",
" out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']\n",
" #out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Redistribution'] / out_grouped_M.loc[tuple(config_a),'T_Redistribution']\n",
" \n",
" config_a.pop(0)\n",
" config_a.pop(0)\n",
" config_b.pop(0)\n",
" config_b.pop(0)\n",
"\n",
"if not ('Alpha' in grouped_aggM.columns):\n",
" for config_a in configurations_simple:\n",
" for config_b in configurations_simple:\n",
" if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:\n",
" compute_alpha(config_a, config_b)\n",
"else:\n",
" print(\"ALPHA already exists\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#OMEGA COMPUTATION\n",
"def compute_omega(config):\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" if len(config) > len(parameters):\n",
" config.pop()\n",
" config.pop()\n",
" config.append(np_aux)\n",
" config.append(ns_aux)\n",
" value = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']\n",
" grouped_aggLAsynch.at[tuple(config),'Omega'] = value\n",
" \n",
" #grouped_aggLAsynch.at[tuple(config),'Omega'] = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']\n",
" value = grouped_aggLAsynch.at[tuple(config),'T_stages'] / grouped_aggLSynch.at[np_aux,'T_stages']\n",
" grouped_aggLAsynch.at[tuple(config),'Omega_Stages'] = value.astype(object)\n",
" config.pop()\n",
" config.pop()\n",
"\n",
"if not ('Omega' in grouped_aggLAsynch.columns):\n",
" grouped_aggLAsynch['Omega'] = 1.0\n",
" grouped_aggLAsynch['Omega_Stages'] = None\n",
" for config in configurations:\n",
" if config[0] != 0:\n",
" compute_omega(config)\n",
"else:\n",
" print(\"OMEGA already exists\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Dynamic Coherence COMPUTATION\n",
"def compute_dyn_coherency(config):\n",
" for np_aux in processes:\n",
" for n_parents_aux in processes:\n",
" if np_aux != n_parents_aux:\n",
" config.append(np_aux)\n",
" config.append(n_parents_aux)\n",
" value = grouped_aggLDyn.at[tuple(config),'T_iter'] / grouped_aggLNDyn.at[np_aux,'T_iter']\n",
" grouped_aggLDyn.at[tuple(config),'Dyn_Coherency'] = value\n",
" value = grouped_aggLDyn.at[tuple(config),'T_stages'] / grouped_aggLNDyn.at[np_aux,'T_stages']\n",
" grouped_aggLDyn.at[tuple(config),'Dyn_Coherency_Stages'] = value.astype(object)\n",
" config.pop()\n",
" config.pop()\n",
"\n",
"if not ('Dyn_Coherency' in grouped_aggLDyn.columns):\n",
" grouped_aggLDyn['Dyn_Coherency'] = 1.0\n",
" grouped_aggLDyn['Dyn_Coherency_Stages'] = None\n",
" for config in configurations_local_dynamic:\n",
" compute_dyn_coherency(config)\n",
"else:\n",
" print(\"Dyn_Coherency already exists\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Malleability Coherence COMPUTATION\n",
"test=dfM[(dfM.Asynch_Iters > 0)]\n",
"coherence_boundary = 1.01 # Allows a 1% error\n",
"total_aux = 0\n",
"show_all = False\n",
"\n",
"# Coherence of synchronous inner times + asynchronous iterations\n",
"for index in range(len(test)):\n",
" time_malleability_aux = test[\"T_Malleability\"].values[index]\n",
" \n",
" total_asynch_iters = int(test[\"Asynch_Iters\"].values[index])\n",
" asynch_iters = test[\"T_iter\"].values[index][-total_asynch_iters:]\n",
" time_iters_aux = np.sum(asynch_iters)\n",
" \n",
" time_synch_aux = test[\"T_SR\"].values[index]\n",
" time_spawn_aux = test[\"T_spawn\"].values[index]\n",
" is_synch_spawn = (test[\"Spawn_Strategy\"].values[index] % 2 != 0)\n",
" \n",
" expected_time = time_iters_aux + time_spawn_aux * is_synch_spawn + time_synch_aux\n",
" time_malleability_aux = time_malleability_aux * coherence_boundary\n",
" \n",
" if time_malleability_aux < expected_time:\n",
" total_aux += 1\n",
" if show_all:\n",
" print(test.iloc[index])\n",
" print(asynch_iters)\n",
" print(time_iters_aux)\n",
" print(time_malleability_aux)\n",
" print(\"\")\n",
"if total_aux > 0:\n",
" print(\"Coherence fails for inner synchronous times + asynchornous iterations in \" + str(total_aux) + \" cases\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Malleability Coherence COMPUTATION\n",
"test=dfM[(dfM.Asynch_Iters > 0)]\n",
"coherence_boundary = 1.01 # Allows a 1% error\n",
"# Coherence of inner times\n",
"aux_df = test[\"T_Malleability\"] * coherence_boundary < (test[\"T_spawn\"] + test[\"T_Redistribution\"])\n",
"if not test[aux_df].empty:\n",
" print(\"Coherence fails for inner times of malleability in \" + str(len(test[aux_df])) + \" cases\")\n",
" display(test[aux_df])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"out_grouped_G.to_excel(\"resultG.xlsx\") \n",
"out_grouped_M.to_excel(\"resultM.xlsx\") \n",
"grouped_aggLAsynch.to_excel(\"AsynchIters.xlsx\")\n",
"grouped_aggLSynch.to_excel(\"SynchIters.xlsx\")\n",
"grouped_aggLNDyn.to_excel(\"StaticCoherence.xlsx\")\n",
"grouped_aggLDyn.to_excel(\"DynCoherence.xlsx\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"dfL.columns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def create_group_boundary(rms_boundary, np_aux, ns_aux):\n",
" tc_boundary = 0\n",
" boundaries = None\n",
" if rms_boundary != 0:\n",
" # El porcentaje de tc_boundary se tiene en cuenta para eliminar aquellos\n",
" # tiempos demasiado grandes en su malleability time respecto al más pequeño\n",
" boundaries = get_np_ns_data(\"T_Malleability\", grouped_aggM, configurations_simple, np_aux, ns_aux)\n",
" tc_boundary = min(boundaries)\n",
" tc_boundary = tc_boundary + tc_boundary*rms_boundary\n",
" return tc_boundary, boundaries\n",
"\n",
"# Aquellos grupos que tengán valores por encima del límite no se considerarán\n",
"def check_groups_boundaries(dataLists, boundaries, tc_boundary):\n",
" for index in range(len(boundaries)):\n",
" if boundaries[index] > tc_boundary:\n",
" dataLists[index] = float('infinity')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def get_perc_differences(dataLists, boundaries, tc_boundary):\n",
" perc = 1.05\n",
" if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos\n",
" check_groups_boundaries(dataLists, boundaries, tc_boundary) \n",
" indexes = np.argsort(dataLists)\n",
" \n",
" best = -1\n",
" bestMax = -1\n",
" otherBest=[]\n",
" for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en media/mediana\n",
" if best == -1:\n",
" best = index\n",
" bestMax = dataLists[best] * perc\n",
" elif dataLists[index] <= bestMax: # Media/Medianas i < Media/Mediana best\n",
" otherBest.append(index)\n",
" \n",
" otherBest.insert(0,best)\n",
" return otherBest\n",
"\n",
"def get_stat_differences(dataLists, df_Res, boundaries, tc_boundary):\n",
" if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos\n",
" check_groups_boundaries(dataLists, boundaries, tc_boundary) \n",
" indexes = np.argsort(dataLists)\n",
" \n",
" best = -1\n",
" otherBest=[] \n",
" for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en mediana\n",
" if dataLists[index] != float('infinity'):\n",
" if best == -1:\n",
" best = index\n",
" elif not df_Res.iat[best,index]: # df_Res == False indicates 'index' and 'best' have the same mean/median\n",
" otherBest.append(index)\n",
" \n",
" otherBest.insert(0,best)\n",
" return otherBest"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"grouped_np = [\"T_total\"]\n",
"separated_np = [\"T_Malleability\", \"T_Redistribution\", \"T_spawn\", \"T_SR\", \"T_AR\", \"Alpha\", \"Omega\", \"count\"]\n",
"\n",
"def get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux):\n",
" dataLists=[]\n",
" for config in used_config:\n",
" if tipo in grouped_np:\n",
" config.append((np_aux,ns_aux))\n",
" elif tipo in separated_np:\n",
" config.append(np_aux)\n",
" config.append(ns_aux)\n",
" \n",
" if tuple(config) in data_aux.index:\n",
" aux_value = data_aux.loc[tuple(config),tipo]\n",
" if isinstance(aux_value, pd.Series):\n",
" aux_value = aux_value.values[0]\n",
" if aux_value == 0: #Values of zero indicate it was not performed\n",
" aux_value = float('infinity')\n",
" else: # This configuration is not present in the dataset\n",
" aux_value = float('infinity')\n",
" dataLists.append(aux_value)\n",
" config.pop()\n",
" if tipo in separated_np:\n",
" config.pop()\n",
" return dataLists\n",
"\n",
"def get_config_data(tipo, data_aux, config):\n",
" dataLists=[]\n",
" procsLists=[]\n",
" for ns_aux in processes:\n",
" for np_aux in processes:\n",
" if np_aux != ns_aux:\n",
" \n",
" if tipo in grouped_np:\n",
" config.append((np_aux,ns_aux))\n",
" elif tipo in separated_np:\n",
" config.append(np_aux)\n",
" config.append(ns_aux)\n",
" if tuple(config) in data_aux.index:\n",
" procsLists.append((np_aux,ns_aux))\n",
" aux_value = data_aux.loc[tuple(config),tipo]\n",
" if isinstance(aux_value, pd.Series):\n",
" aux_value = aux_value.values[0]\n",
" if aux_value == 0: #Values of zero indicate it was not performed\n",
" aux_value = float('infinity')\n",
" else: # This configuration is not present in the dataset\n",
" aux_value = float('infinity')\n",
" dataLists.append(aux_value)\n",
" config.pop()\n",
" if tipo in separated_np:\n",
" config.pop()\n",
" return dataLists, procsLists\n",
"\n",
"def get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux):\n",
" dataLists=[]\n",
" if tipo in grouped_np:\n",
" tuple_data = (np_aux, ns_aux)\n",
" df_npns_aux = df_check.loc[(df_check['Groups']==tuple_data)]\n",
" elif tipo in separated_np:\n",
" df_npns_aux = df_check.loc[(df_check['NP']==np_aux)]\n",
" df_npns_aux = df_npns_aux.loc[(df_npns_aux['NC']==ns_aux)]\n",
" \n",
" for config in used_config:\n",
" df_config_aux = df_npns_aux\n",
" for index in range(len(config)):\n",
" aux_name = parameters_names[index]\n",
" aux_value = config[index]\n",
" df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]\n",
" \n",
" aux_value = list(df_config_aux[tipo])\n",
" if len(aux_value) > 0:\n",
" dataLists.append(aux_value)\n",
" return dataLists\n",
"\n",
"def get_df_config_data(df_check, tipo, config):\n",
" dataLists=[]\n",
" df_config_aux = df_check\n",
" for index in range(len(config)):\n",
" aux_name = parameters_names[index]\n",
" aux_value = config[index]\n",
" df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]\n",
" \n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" if tipo in grouped_np:\n",
" tuple_data = (np_aux, ns_aux)\n",
" df_aux = df_config_aux.loc[(df_config_aux['Groups']==tuple_data)]\n",
" elif tipo in separated_np:\n",
" df_aux = df_config_aux.loc[(df_config_aux['NP']==np_aux)]\n",
" df_aux = df_aux.loc[(df_aux['NC']==ns_aux)]\n",
" aux_value = list(df_aux[tipo])\n",
" if len(aux_value) > 0:\n",
" dataLists.append(aux_value)\n",
" return dataLists\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def check_normality(df_check, tipo, used_config, fast=True):\n",
" normality_array=[True] * (len(processes) * (len(processes)-1) * len(used_config))\n",
" normality = True\n",
" total=0\n",
" i=-1\n",
" #Comprobar para cada configuración si se sigue una distribución normal/gaussiana\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" i+=1\n",
" dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n",
" for data_aux in dataLists:\n",
" st,p = stats.shapiro(data_aux) # Tendrían que ser al menos 20 datos y menos de 50\n",
" if p < significance_value: # Reject H0\n",
" if fast:\n",
" return False\n",
" normality_array[i] = False\n",
" normality = False\n",
" total+=1\n",
" print(\"Se sigue una distribución guassiana: \" + str(normality) + \"\\nUn total de: \" + str(total) + \" no siguen una gaussiana\")\n",
" print(normality_array)\n",
" return normality\n",
"\n",
"def check_homoscedasticity(df_check, tipo, used_config, fast=True):\n",
" homoscedasticity_array=[True] * (len(processes) * (len(processes)-1))\n",
" homoscedasticity = True\n",
" total=0\n",
" i=-1\n",
" #Comprobar para cada configuración es homoestatica\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" i+=1\n",
" dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n",
" st,p = stats.levene(*dataLists) # Tendrían que ser al menos 20 datos y menos de 50\n",
" if p < significance_value: # Reject H0\n",
" if fast:\n",
" return False\n",
" homoscedasticity_array[i] = False\n",
" homoscedasticity = False\n",
" total+=1\n",
" print(\"Se sigue una distribución de datos Homocedastica: \" + str(homoscedasticity) + \"\\nUn total de: \" + str(total) + \" no siguen una homocedastica\")\n",
" print(homoscedasticity_array)\n",
" return homoscedasticity\n",
"\n",
"def compute_global_stat_difference(dataLists, parametric, np_aux, ns_aux):\n",
" if parametric:\n",
" st,p=stats.f_oneway(*dataLists)\n",
" else:\n",
" st,p=stats.kruskal(*dataLists)\n",
" if p > significance_value:\n",
" print(\"For NP \" + str(np_aux) + \" and \" + str(ns_aux) + \" is accepted H0\")\n",
" return True # Equal values || Accept H0\n",
" return False # Some groups are different || Reject H0\n",
"\n",
"def compute_global_posthoc(dataLists, parametric): #TODO Comprobar CDF de los grupos\n",
" data_stats=[]\n",
" data_stats2=[]\n",
" ini=0\n",
" end=len(dataLists)\n",
" if parametric:\n",
" df_aux = sp.posthoc_ttest(dataLists)\n",
" df_Res = df_aux.copy()\n",
" for i in range(ini,end):\n",
" data_stats.append(np.mean(dataLists[i]))\n",
" \n",
" for j in range(ini,end):\n",
" if df_Res.iat[i,j] < significance_value: # Different means || Reject H0\n",
" df_Res.iat[i, j] = True\n",
" else:\n",
" df_Res.iat[i, j] = False\n",
" else:\n",
" df_aux = sp.posthoc_conover(dataLists)\n",
" df_Res = df_aux.copy()\n",
" for i in range(ini,end):\n",
" data_stats.append(np.median(dataLists[i]))\n",
" #data_stats2.append(stats.iqr(dataLists[i],axis=0))\n",
" for j in range(ini,end):\n",
" if df_Res.iat[i,j] < significance_value: # Different medians || Reject H0\n",
" df_Res.iat[i, j] = True # Not equal medians\n",
" else:\n",
" df_Res.iat[i, j] = False # Equal medians\n",
" #print(df_Res)\n",
" #print(df_aux)\n",
" #print(data_stats)\n",
" #print(data_stats2)\n",
" #aux_value = min(data_stats)\n",
" #print(data_stats.index(aux_value))\n",
" return df_Res, data_stats"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def results_with_perc(tipo, data_aux, used_config, rms_boundary=0):\n",
" results = []\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)\n",
" \n",
" #Get all values for particular config with these number of processes\n",
" dataLists = get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux)\n",
"\n",
" aux_data = get_perc_differences(dataLists, boundaries, tc_boundary)\n",
" results.append(aux_data)\n",
" return results\n",
"\n",
"def results_with_stats(tipo, df_check, used_config, rms_boundary=0):\n",
" results = []\n",
" use_parametric = check_normality(df_check, tipo, used_config)\n",
" if use_parametric:\n",
" use_parametric = check_homoscedasticity(df_check, tipo, used_config)\n",
" print(\"Se usan tests parametricos: \"+str(use_parametric))\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)\n",
" \n",
" #Get all values for particular config with these number of processes\n",
" dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n",
" equal_set = compute_global_stat_difference(dataLists, use_parametric, np_aux, ns_aux)\n",
" if equal_set:\n",
" aux_data = list(range(len(used_config))) # All data is equal\n",
" else:\n",
" res_aux, times_aux = compute_global_posthoc(dataLists, use_parametric)\n",
" aux_data = get_stat_differences(times_aux, res_aux, boundaries, tc_boundary)\n",
" \n",
" results.append(aux_data)\n",
" \n",
" return results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"checked_type='T_total'\n",
"use_perc = False\n",
"select_first_winner = False\n",
"prefer_first_winner = False\n",
"rms_boundary=0 # Poner a 0 para perspectiva de app. Valor >0 y <1 para perspectiva de RMS\n",
"if checked_type=='T_total':\n",
" tipo=\"T_total\"\n",
" if use_perc:\n",
" data_aux = grouped_aggG\n",
" else:\n",
" data_aux = dfG\n",
" used_config = configurations\n",
"elif checked_type=='T_Malleability' or checked_type=='T_spawn' or checked_type=='T_SR' or checked_type=='T_AR' or checked_type=='T_Redistribution':\n",
" tipo=checked_type\n",
" \n",
" if use_perc:\n",
" data_aux = grouped_aggM\n",
" else:\n",
" data_aux = dfM\n",
" if tipo == 'T_AR':\n",
" data_aux = data_aux[(data_aux.ADR > 0)]\n",
" elif tipo == 'T_SR':\n",
" data_aux = data_aux[(data_aux.ADR == 0)]\n",
" \n",
" used_config = configurations_simple\n",
" \n",
"if use_perc:\n",
" results = results_with_perc(tipo, data_aux, used_config, rms_boundary)\n",
"else:\n",
" results = results_with_stats(tipo, data_aux, used_config, rms_boundary)\n",
" \n",
"if not use_perc and tipo == 'T_AR': #FIXME!!!! No tiene en cuenta total de configuraciones sincronos\n",
" for res_index in range(len(results)):\n",
" for inner_index in range(len(results[res_index])):\n",
" results[res_index][inner_index]+=4\n",
"\n",
"#Results is a 2 dimensional array. First dimension indicates lists of winners of a particulal number of processes (NP->NC). \n",
"#Second dimension is an ordered preference of indexes in the array configurations.\n",
"print(results)\n",
"print(len(results))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Lista de indices de mayor a menor según el total de ocurrencias\n",
"aux_array = []\n",
"for data in results:\n",
" aux_array+=data\n",
"aux_keys, aux_counts = np.unique(aux_array, return_counts=True)\n",
"aux_ordered_index=list(reversed(np.argsort(aux_counts)))\n",
"\n",
"#Lista de indices de mayor a menor según el total de ocurrencias del primero de cada grupo\n",
"aux_array = [0] * len(results)\n",
"for index in range(len(results)):\n",
" aux_array[index] = results[index][0]\n",
"aux_keys_best, aux_counts_best = np.unique(aux_array, return_counts = True)\n",
"aux_ordered_best_index=list(reversed(np.argsort(aux_counts_best)))\n",
"\n",
"def heatmap_get_best(index, ordered_array, keys_array, counts_array, prefer_winner=False):\n",
" valid_candidates_indexes = []\n",
" prev_counts = -1\n",
" for tested_index in ordered_array:\n",
" if keys_array[tested_index] in results[index]:\n",
" if counts_array[tested_index] >= prev_counts:\n",
" prev_counts = counts_array[tested_index]\n",
" valid_candidates_indexes.append(tested_index)\n",
" else:\n",
" break\n",
" \n",
" if prefer_winner: # Si esta activo, en caso de empate en ocurrencias se selecciona el menor tiempo\n",
" for tested_index in results[index]:\n",
" if tested_index in valid_candidates_indexes:\n",
" return tested_index\n",
" return min(valid_candidates_indexes) # En caso de empate se devuelve el que tiene menor valor (Suele ser la config más simple)\n",
"\n",
"i=0\n",
"j=0\n",
"used_aux=0\n",
"heatmap=np.zeros((len(processes),len(processes))).astype(int)\n",
"\n",
"if select_first_winner:\n",
" for i in range(len(processes)):\n",
" for j in range(len(processes)):\n",
" if i==j:\n",
" heatmap[i][j]=-1\n",
" used_aux+=1\n",
" else:\n",
" results_index = i*len(processes) + j - used_aux\n",
" heatmap[i][j] = results[results_index][0]\n",
"else:\n",
" for i in range(len(processes)):\n",
" for j in range(len(processes)):\n",
" if i==j:\n",
" heatmap[i][j]=-1\n",
" used_aux+=1\n",
" else:\n",
" results_index = i*len(processes) + j - used_aux\n",
" index = heatmap_get_best(results_index, aux_ordered_index, aux_keys, aux_counts, prefer_first_winner)\n",
" heatmap[i][j]=aux_keys[index]\n",
" #index = heatmap_get_best(results_index, aux_ordered_best_index, aux_keys_best, aux_counts_best, prefer_first_winner)\n",
" #heatmap[i][j]=aux_keys_best[index]\n",
"heatmap[-1][-1]=len(used_config)\n",
"print(aux_keys)\n",
"print(aux_counts)\n",
"print(heatmap)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Adapta results a una cadena asegurando que cada cadena no se sale de su celda\n",
"def get_heatmap_multiple_strings(results): #FIXME Deprecated\n",
" results_str = []\n",
" max_counts = 1\n",
" max_per_line = 3\n",
" for i in range(len(results)):\n",
" results_str.append(list())\n",
" count = len(results[i])\n",
" results_aux = results[i]\n",
" if count > max_counts:\n",
" count = max_counts\n",
" results_aux = results[i][:count]\n",
" \n",
" remainder = count%max_per_line\n",
" if count <= max_per_line:\n",
" aux_str = str(results_aux).replace('[','').replace(']','')\n",
" results_str[i].append(aux_str)\n",
" else:\n",
" if remainder == 0:\n",
" index = count//2\n",
" else:\n",
" index = count - ((remainder-1)*max_per_line + 1)\n",
" aux_str = str(results_aux[:index]).replace('[','').replace(']','')\n",
" results_str[i].append(aux_str)\n",
" aux_str = str(results_aux[index:]).replace('[','').replace(']','')\n",
" results_str[i].append(aux_str)\n",
" return results_str\n",
"\n",
"def get_heatmap_strings(heatmap):\n",
" results_str = []\n",
" for i in range(len(processes)):\n",
" for j in range(len(processes)):\n",
" if i!=j:\n",
" results_str.append(list())\n",
" results_str[-1].append(heatmap[i][j])\n",
" return results_str"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#Crea un heatmap teniendo en cuenta los colores anteriores\n",
"f=plt.figure(figsize=(24, 12))\n",
"ax=f.add_subplot(111)\n",
"\n",
"myColors = (colors.to_rgba(\"white\"), \n",
" colors.to_rgba(\"green\"), #BCOLS\n",
" colors.to_rgba(\"darkgreen\"), #BP2PS\n",
" #colors.to_rgba(\"blue\"), #BRMA1S\n",
" #colors.to_rgba(\"royalblue\"), #BRMA2S\n",
" colors.to_rgba(\"red\"), #MCOLS\n",
" colors.to_rgba(\"darkred\"), #MP2PS\n",
" #colors.to_rgba(\"mediumblue\"), #MRMA1S\n",
" #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n",
" #colors.to_rgba(\"blue\"), #BIntraCOLS\n",
" #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n",
" colors.to_rgba(\"mediumseagreen\"), #BCOLA\n",
" colors.to_rgba(\"seagreen\"), #BCOLT\n",
" colors.to_rgba(\"palegreen\"), #BP2PA\n",
" colors.to_rgba(\"springgreen\"), #BP2PT\n",
" #colors.to_rgba(\"purple\"), #BRMA1A\n",
" #colors.to_rgba(\"darkviolet\"), #BRMA1T\n",
" #colors.to_rgba(\"indigo\"), #BRMA2A\n",
" #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n",
" colors.to_rgba(\"indianred\"), #MCOLA \n",
" colors.to_rgba(\"firebrick\"), #MCOLT\n",
" colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n",
" colors.to_rgba(\"saddlebrown\"), #MP2PT\n",
" #colors.to_rgba(\"magenta\"), #MRMA1A\n",
" #colors.to_rgba(\"violet\"), #MRMA1T\n",
" #colors.to_rgba(\"deeppink\"), #MRMA2A\n",
" #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n",
" #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n",
" #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n",
" colors.to_rgba(\"white\"))\n",
"cmap = LinearSegmentedColormap.from_list('Custom', myColors, len(myColors))\n",
"\n",
"im = ax.imshow(heatmap,cmap=cmap,interpolation='nearest')\n",
"\n",
"# Loop over data dimensions and create text annotations.\n",
"used_aux=0\n",
"results_str = get_heatmap_strings(heatmap)\n",
"for i in range(len(processes)):\n",
" for j in range(len(processes)):\n",
" if i!=j:\n",
" aux_color=\"white\"\n",
" if 0 <= heatmap[i, j] <= 1 or 4 <= heatmap[i, j] <= 7: # El 1 puede necesitar texto en negro\n",
" aux_color=\"black\"\n",
" results_index = i*len(processes) +j-used_aux\n",
" if len(results_str[results_index]) == 1:\n",
" text = results_str[results_index][0]\n",
" ax.text(j, i, text, ha=\"center\", va=\"center\", color=aux_color, fontsize=36)\n",
" else:\n",
" add_aux = 0.33\n",
" for line in range(len(results_str[results_index])):\n",
" i_range = i - 0.5 + add_aux\n",
" ax.text(j, i_range, results_str[results_index][line],\n",
" ha=\"center\", va=\"center\", color=aux_color, fontsize=36)\n",
" add_aux+=0.33\n",
" else:\n",
" used_aux+=1\n",
"\n",
"ax.set_ylabel(\"NS\", fontsize=36)\n",
"ax.set_xlabel(\"NT\", fontsize=36)\n",
"\n",
"ax.set_xticklabels(['']+processes, fontsize=36)\n",
"ax.set_yticklabels(['']+processes, fontsize=36)\n",
"\n",
"\n",
"labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',\n",
" 'Merge - COLS (2)','Merge - P2PS (3)',\n",
" 'Baseline - COLA (4)', 'Baseline - COLT (5)','Baseline - P2PA (6)','Baseline - P2PT (7)',\n",
" 'Merge - COLA (8)','Merge - COLT (9)','Merge - P2PA (10)','Merge - P2PT (11)']\n",
"\n",
"#labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',\n",
"# 'Baseline - RMA1S (2)', 'Baseline - RMA2S (3)',\n",
"# 'Merge - COLS (4)','Merge - P2PS (5)',\n",
"# 'Merge - RMA1S (6)', 'Merge - RMA2S (7)',\n",
"# 'Baseline - COLA (8)', 'Baseline - COLT (9)','Baseline - P2PA (10)','Baseline - P2PT (11)',\n",
"# 'Baseline - RMA1A (12)', 'Baseline - RMA1T (13)','Baseline - RMA2A (14)','Baseline - RMA2T (15)',\n",
"# 'Merge - COLA (16)','Merge - COLT (17)','Merge - P2PA (18)','Merge - P2PT (19)',\n",
"# 'Merge - RMA1A (20)','Merge - RMA1T (21)','Merge - RMA2A (22)','Merge - RMA2T (23)']\n",
"\n",
"colorbar=f.colorbar(im, ax=ax)\n",
"tick_bar = []\n",
"for i in range(len(used_config)):\n",
" #tick_bar.append(0.41 + i*0.96) #Config de 24 valores\n",
" tick_bar.append(0.37 + i*0.92) #Config de 12 valores\n",
" #tick_bar.append(0.35 + i*0.89) #Config de 8 valores\n",
"colorbar.set_ticks(tick_bar) \n",
"colorbar.set_ticklabels(labelsMethods_aux)\n",
"colorbar.ax.tick_params(labelsize=32)\n",
"#\n",
"\n",
"f.tight_layout()\n",
"print(\"Filename: Heatmap_\"+tipo+\".png\")\n",
"f.savefig(\"Images/Heatmap_\"+tipo+\".jpeg\", format=\"jpeg\", dpi=300)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"aux_array = [] #Counts all\n",
"for data in results:\n",
" aux_array+=data\n",
"aux_results, aux_counts = np.unique(aux_array, return_counts=True)\n",
"print(aux_results)\n",
"print(aux_counts)\n",
"\n",
"aux_array = [0] * len(results) # Counts ganador celda\n",
"for index in range(len(results)):\n",
" aux_array[index] = results[index][0]\n",
"aux_results, aux_counts = np.unique(aux_array, return_counts = True)\n",
"print(aux_results)\n",
"print(aux_counts)\n"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"El siguiente código asume que para cada número de procesos padre/hijo existen valores en todas las configuraciones que se van a probar"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"def normalize_arrays(arrays, norm_array):\n",
" nef normalize_arrays(arrays, norm_array):\n",
" new_arrays = arrays.copy()\n",
" for index in range(len(new_arrays)):\n",
" new_arrays[index] = np.divide(norm_array, new_arrays[index])\n",
" return new_arrays\n",
"\n",
"def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):\n",
" labels_aux = []\n",
" if used_direction == 's':\n",
" for ns_aux in processes:\n",
" for np_aux in processes:\n",
" if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n",
" new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n",
" labels_aux.append(new_label)\n",
" else:\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n",
" new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n",
" labels_aux.append(new_label)\n",
" return labels_aux\n",
"\n",
"def reorder_data(plot_data, actual_order, expected_order):\n",
" ordered_indexes = []\n",
" len_order = len(actual_order)\n",
" for index in range(len_order):\n",
" actual_order[index] = str(actual_order[index]).replace(\" \", \"\")\n",
" for index in range(len_order):\n",
" ordered_indexes.append(actual_order.index(expected_order[index]))\n",
"\n",
" for index in range(len(plot_data)):\n",
" old_array = plot_data[index]\n",
" new_array = []\n",
" for i in ordered_indexes:\n",
" new_array.append(old_array[i])\n",
" plot_data[index] = new_array\n",
"\n",
" return plot_dataew_arrays = arrays.copy()\n",
" for index in range(len(new_arrays)):\n",
" new_arrays[index] = np.divide(norm_array, new_arrays[index])\n",
" return new_arrays\n",
"\n",
"def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):\n",
" labels_aux = []\n",
" if used_direction == 's':\n",
" for ns_aux in processes:\n",
" for np_aux in processes:\n",
" if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n",
" new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n",
" labels_aux.append(new_label)\n",
" else:\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n",
" new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n",
" labels_aux.append(new_label)\n",
" return labels_aux\n",
"\n",
"def reorder_data(plot_data, actual_order, expected_order):\n",
" ordered_indexes = []\n",
" len_order = len(actual_order)\n",
" for index in range(len_order):\n",
" actual_order[index] = str(actual_order[index]).replace(\" \", \"\")\n",
" for index in range(len_order):\n",
" ordered_indexes.append(actual_order.index(expected_order[index]))\n",
"\n",
" for index in range(len(plot_data)):\n",
" old_array = plot_data[index]\n",
" new_array = []\n",
" for i in ordered_indexes:\n",
" new_array.append(old_array[i])\n",
" plot_data[index] = new_array\n",
"\n",
" return plot_data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"used_direction='s'\n",
"test_parameter='T_Malleability' #Valores son \"alpha\" o \"omega\"\n",
"\n",
"if test_parameter == 'alpha':\n",
" name_fig=\"Alpha_\"\n",
" real_parameter='Alpha'\n",
" name_legend = \"Valores de α\"\n",
" normalize = True\n",
" allow_all = False\n",
" used_config = configurations_simple\n",
" data_aux = grouped_aggM[grouped_aggM[real_parameter] > 0]\n",
"elif test_parameter == 'T_Malleability':\n",
" name_fig=\"Malleability_\"\n",
" real_parameter='T_Malleability'\n",
" name_legend = \"Time(s)\"\n",
" normalize = False\n",
" allow_all = True\n",
" used_config = configurations_simple\n",
" data_aux = grouped_aggM\n",
"elif test_parameter == 'T_Redistribution':\n",
" name_fig=\"Redistribution_\"\n",
" real_parameter='T_Redistribution'\n",
" name_legend = \"Tiempo(s)\"\n",
" normalize = False\n",
" allow_all = True\n",
" used_config = configurations_simple\n",
" data_aux = grouped_aggM\n",
"elif test_parameter == 'omega':\n",
" name_fig=\"Omega_\"\n",
" real_parameter='Omega'\n",
" name_legend = \"Values of ω\"\n",
" normalize = True\n",
" allow_all = False\n",
" used_config = configurations\n",
" data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]\n",
"elif test_parameter == 'iters_count':\n",
" name_fig=\"Iters_\"\n",
" real_parameter='count'\n",
" name_legend = \"Asynchronous iterations\"\n",
" normalize = False\n",
" allow_all = True\n",
" used_config = configurations\n",
" data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]\n",
" \n",
"if used_direction=='s':\n",
" data_aux=data_aux.query('NP > NC')\n",
" name_fig= name_fig+\"Shrink\"\n",
"elif used_direction=='e':\n",
" data_aux=data_aux.query('NP < NC')\n",
" name_fig= name_fig+\"Expand\"\n",
"elif used_direction=='a':\n",
" name_fig= name_fig+\"All\" \n",
"#data_aux=data_aux.query('NP == 160 or NC == 160')\n",
"\n",
"plot_data = []\n",
"for config in used_config:\n",
" if config[0] > 0 or allow_all:\n",
" dataLists,procsLists = get_config_data(real_parameter, data_aux, config)\n",
" dataLists = list(filter(lambda x: x != float('infinity'), dataLists))\n",
" plot_data.append(dataLists)\n",
" \n",
"#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)\n",
"labels_aux = create_labels_lineplot(used_direction)\n",
"plot_data = reorder_data(plot_data, procsLists, labels_aux)\n",
" \n",
"#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n",
"# 'Baseline - RMA1S', 'Baseline - RMA2S',\n",
"# 'Merge - COLS','Merge - P2PS',\n",
"# 'Merge - RMA1S', 'Merge - RMA2S',\n",
"# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',\n",
"# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',\n",
"# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',\n",
"# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'\n",
"# ]\n",
"labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n",
" 'Merge - COLS','Merge - P2PS',\n",
" 'Baseline - COLA', 'Baseline - COLT', 'Baseline - P2PA', 'Baseline - P2PT',\n",
" 'Merge - COLA', 'Merge - COLT', 'Merge - P2PA', 'Merge - P2PT']\n",
"colors_m = ( \n",
" colors.to_rgba(\"green\"), #BCOLS\n",
" colors.to_rgba(\"darkgreen\"), #BP2PS\n",
" #colors.to_rgba(\"blue\"), #BRMA1S\n",
" #colors.to_rgba(\"royalblue\"), #BRMA2S\n",
" colors.to_rgba(\"red\"), #MCOLS\n",
" colors.to_rgba(\"darkred\"), #MP2PS\n",
" #colors.to_rgba(\"mediumblue\"), #MRMA1S\n",
" #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n",
" #colors.to_rgba(\"blue\"), #BIntraCOLS\n",
" #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n",
" colors.to_rgba(\"mediumseagreen\"), #BCOLA\n",
" colors.to_rgba(\"seagreen\"), #BCOLT\n",
" colors.to_rgba(\"palegreen\"), #BP2PA\n",
" colors.to_rgba(\"springgreen\"), #BP2PT\n",
" #colors.to_rgba(\"purple\"), #BRMA1A\n",
" #colors.to_rgba(\"darkviolet\"), #BRMA1T\n",
" #colors.to_rgba(\"indigo\"), #BRMA2A\n",
" #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n",
" colors.to_rgba(\"indianred\"), #MCOLA \n",
" colors.to_rgba(\"firebrick\"), #MCOLT\n",
" colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n",
" colors.to_rgba(\"saddlebrown\"), #MP2PT\n",
" #colors.to_rgba(\"magenta\"), #MRMA1A\n",
" #colors.to_rgba(\"violet\"), #MRMA1T\n",
" #colors.to_rgba(\"deeppink\"), #MRMA2A\n",
" #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n",
" #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n",
" #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n",
" )\n",
"\n",
"f=plt.figure(figsize=(22, 14))\n",
"ax=f.add_subplot(111)\n",
"x = np.arange(len(labels_aux))\n",
"for index in range(len(plot_data)):\n",
" array_aux = plot_data[index]\n",
" plot_index = index\n",
" if real_parameter == 'Alpha' or real_parameter == 'Omega' or real_parameter == 'count': #FIXME This line is a lie...\n",
" plot_index = 4 + index #FIXME This line is a lie...\n",
" print(array_aux)\n",
" ax.plot(x, array_aux, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n",
" marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[plot_index])\n",
"\n",
"ax.set_xlabel(\"(NS,NT)\", fontsize=36)\n",
"ax.set_ylabel(name_legend, fontsize=36)\n",
"if normalize:\n",
" ax.axhline(y=1, color='black', linestyle='--')\n",
"plt.xticks(x, labels_aux,rotation=90)\n",
"ax.tick_params(axis='both', which='major', labelsize=36)\n",
"ax.tick_params(axis='both', which='minor', labelsize=36)\n",
"plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n",
" \n",
"f.tight_layout()\n",
"f.savefig(\"Images/LinePlot_\"+name_fig+\".png\", format=\"png\")"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"Gráfica de lineas para generar tiempos del grupo G."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"used_direction='e'\n",
"test_parameter='T_total'\n",
"intranode=False\n",
"\n",
"if test_parameter == 'T_total':\n",
" name_fig=\"Ttotal\"\n",
" real_parameter='T_total'\n",
" name_legend = \"Time(s)\"\n",
" used_config = configurations\n",
" data_aux = grouped_aggG\n",
" #data_aux = data_aux[data_aux.index.isin(df1.index)]\n",
" \n",
"if used_direction=='s':\n",
" data_aux_cmp=grouped_aggM.reset_index().query('NP > NC')\n",
" name_fig= name_fig+\"Shrink\"\n",
"elif used_direction=='e':\n",
" data_aux_cmp=grouped_aggM.reset_index().query('NP < NC')\n",
" name_fig= name_fig+\"Expand\"\n",
"elif used_direction=='a':\n",
" data_aux_cmp=grouped_aggM.reset_index()\n",
" name_fig= name_fig+\"All\"\n",
" \n",
"if intranode:\n",
" data_aux_cmp = data_aux_cmp.query('NP <= 20 and NC <= 20')\n",
"#else:\n",
" #data_aux_cmp = data_aux_cmp.query('NP > 20 and NC > 20')\n",
"#data_aux_cmp = data_aux_cmp.query('NP == 160 or NC == 160')\n",
"\n",
"if used_direction!='a' or True:\n",
" pruebaG = data_aux.reset_index()\n",
" pruebaG = pruebaG.loc[pruebaG.index.intersection(data_aux_cmp.index)]\n",
" data_aux = data_aux[(data_aux.T_total.isin(pruebaG.T_total))]\n",
"\n",
"plot_data = []\n",
"for config in used_config:\n",
" #if config[0] == 0:\n",
" dataLists,procsLists = get_config_data(real_parameter, data_aux, config)\n",
" dataLists = list(filter(lambda x: x != float('infinity'), dataLists))\n",
" plot_data.append(dataLists)\n",
"for array_aux in plot_data:\n",
" print(array_aux)\n",
" \n",
"\n",
"#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)\n",
"labels_aux = create_labels_lineplot(used_direction)\n",
"plot_data = reorder_data(plot_data, procsLists, labels_aux)\n",
"plot_data_normalized = normalize_arrays(plot_data[1:], plot_data[0])\n",
"name_legend=\"SpeedUp over Baseline\"\n",
"\n",
"#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']\n",
"labelsMethods_aux = ['Baseline - AllS', 'Baseline - P2PS',\n",
" 'Merge - AllS','Merge - P2PS',\n",
" 'Baseline - AllA', 'Baseline - AllT','Baseline - P2PA','Baseline - P2PT',\n",
" 'Merge - AllA','Merge - AllT','Merge - P2PA','Merge - P2PT']\n",
"#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n",
"# 'Baseline - RMA1S', 'Baseline - RMA2S',\n",
"# 'Merge - COLS','Merge - P2PS',\n",
"# 'Merge - RMA1S', 'Merge - RMA2S',\n",
"# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',\n",
"# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',\n",
"# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',\n",
"# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'\n",
"# ]\n",
"\n",
"colors_m = ( \n",
" colors.to_rgba(\"green\"), #BCOLS\n",
" colors.to_rgba(\"darkgreen\"), #BP2PS\n",
" #colors.to_rgba(\"blue\"), #BRMA1S\n",
" #colors.to_rgba(\"royalblue\"), #BRMA2S\n",
" colors.to_rgba(\"red\"), #MCOLS\n",
" colors.to_rgba(\"darkred\"), #MP2PS\n",
" #colors.to_rgba(\"mediumblue\"), #MRMA1S\n",
" #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n",
" #colors.to_rgba(\"blue\"), #BIntraCOLS\n",
" #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n",
" colors.to_rgba(\"mediumseagreen\"), #BCOLA\n",
" colors.to_rgba(\"seagreen\"), #BCOLT\n",
" colors.to_rgba(\"palegreen\"), #BP2PA\n",
" colors.to_rgba(\"springgreen\"), #BP2PT\n",
" #colors.to_rgba(\"purple\"), #BRMA1A\n",
" #colors.to_rgba(\"darkviolet\"), #BRMA1T\n",
" #colors.to_rgba(\"indigo\"), #BRMA2A\n",
" #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n",
" colors.to_rgba(\"indianred\"), #MCOLA \n",
" colors.to_rgba(\"firebrick\"), #MCOLT\n",
" colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n",
" colors.to_rgba(\"saddlebrown\"), #MP2PT\n",
" #colors.to_rgba(\"magenta\"), #MRMA1A\n",
" #colors.to_rgba(\"violet\"), #MRMA1T\n",
" #colors.to_rgba(\"deeppink\"), #MRMA2A\n",
" #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n",
" #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n",
" #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n",
" )\n",
"\n",
"\n",
"f=plt.figure(figsize=(22, 14))\n",
"ax=f.add_subplot(111)\n",
"ax2 = ax.twinx()\n",
"x = np.arange(len(labels_aux))\n",
"for index in range(len(plot_data_normalized)):\n",
" array_aux = plot_data_normalized[index]\n",
" index= index+1\n",
" ax.plot(x, array_aux, color=colors_m[index%len(colors_m)], linestyle=linestyle_m[index%len(linestyle_m)], \\\n",
" marker=markers_m[index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n",
"ax2.plot(x, plot_data[0], color='black', linestyle=linestyle_m[0], \\\n",
" marker=markers_m[0], markersize=18, label=labelsMethods_aux[0])\n",
"ax.axhline(y=1, color='black', linestyle='--')\n",
"\n",
"ax.set_xlabel(\"(NP,NC)\", fontsize=36)\n",
"ax.set_ylabel(name_legend, fontsize=36)\n",
"ax.tick_params(axis='both', which='both', labelsize=36)\n",
"ax.set_xticks(x)\n",
"ax.set_xticklabels(labels_aux, rotation=90)\n",
"#ax.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n",
"\n",
"ax2.set_ylabel('Baseline Time(s)', fontsize=36)\n",
"ax2.tick_params(axis='y', which='both', labelsize=36)\n",
"#ax2.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n",
"\n",
"f.legend(bbox_to_anchor=(0.5, 0.98), fontsize=26,ncol=2,framealpha=0.8)\n",
"\n",
" \n",
"f.tight_layout()\n",
"f.savefig(\"Images/LinePlot_\"+name_fig+\".png\", format=\"png\")"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"El siguiente generá una imagen en 3d de T_total para cada una de las diferentes configuraciones."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def generate_3d_image(config, name):\n",
" fig, ax = plt.subplots(1, 1, subplot_kw={'projection': '3d'}, figsize=(15, 15))\n",
"\n",
" Z = [None] * len(processes)\n",
" X, Y = np.meshgrid(processes, processes)\n",
" for i in range(len(processes)):\n",
" np_aux = processes[i]\n",
" Z[i] = [0] * len(processes)\n",
" Z[i][i] = grouped_aggLSynch.loc[np_aux, 'T_iter'] * 1000\n",
" for j in range(len(processes)):\n",
" if i!=j:\n",
" ns_aux = processes[j]\n",
" config.append((np_aux,ns_aux))\n",
" aux = grouped_aggG.loc[tuple(config),'T_total']\n",
" config.pop()\n",
" \n",
" Z[i][j] = aux.values[0]\n",
" #Z[i][j] = Z[i][j] / Z[i][i]\n",
" #Z[i][i] = 1\n",
"\n",
" Z = np.array(Z)\n",
"\n",
" ax.plot_surface(X, Y, Z, rstride=1, cstride=1,\n",
" cmap='viridis', edgecolor='none')\n",
" ax.view_init(15, 25)\n",
" ax.set_xlabel(\"NC\", fontsize=16)\n",
" ax.set_ylabel(\"NP\", fontsize=16)\n",
" ax.set_zlabel(\"Normalized time\", fontsize=16)\n",
" ax.set_title(name, fontsize=10)\n",
" plt.show()\n",
" \n",
"for index in range(len(configurations)):\n",
" used_config = configurations[index]\n",
" generate_3d_image(used_config,str(index))"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"El siguiente código es computar la coherencia de T_malleability respecto a los tiempos internos de maleabilidad (Coherency1)\n",
"y por otro lado de T_malleability respecto a iteraciones asíncronas más los pasos síncronos."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test=dfM[(dfM.Asynch_Iters > 0)]\n",
"\n",
"# Seguro con barriers en Malleability\n",
"test[\"Resize_Coherency\"] = test[\"T_Malleability\"] >= (test[\"T_spawn\"] + test[\"T_SR\"] + test[\"T_AR\"])\n",
"# Seguro al usar Rigid para iteraciones\n",
"test[\"Resize_Coherency2\"] = test[\"T_Malleability\"] >= 0\n",
"\n",
"for index in range(len(test)):\n",
" time_malleability_aux = test[\"T_Malleability\"].values[index]\n",
" time_synch_aux = test[\"T_SR\"].values[index]\n",
" time_spawn_aux = test[\"T_spawn\"].values[index]\n",
" is_asynch_spawn = (test[\"Spawn_Strategy\"].values[index] % 2 == 0)\n",
" \n",
" total_asynch_iters = int(test[\"Asynch_Iters\"].values[index])\n",
" asynch_iters = test[\"T_iter\"].values[index][-total_asynch_iters:]\n",
" time_iters_aux = np.sum(asynch_iters[:])\n",
" \n",
" sum_times = time_synch_aux + is_asynch_spawn * time_spawn_aux + time_iters_aux\n",
" \n",
" if time_malleability_aux < sum_times:\n",
" real_index = test.index.values[index]\n",
" test.at[real_index, \"Resize_Coherency2\"] = False\n",
"test[(test.Resize_Coherency == False)].query('Redistribution_Method > 1')"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"El siguiente código es para utilizar Dask. Una versión que paraleliza una serie de tareas de Pandas.\n",
"Tras llamar a compute se realizan todas las tareas que se han pedido."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import dask.dataframe as dd\n",
"ddf = dd.from_pandas(dfL[(dfL.Asynch_Iters == False)], npartitions=10)\n",
"group = ddf.groupby('NP')['T_iter']\n",
"grouped_aggLSynch = group.agg(['mean'])\n",
"grouped_aggLSynch = grouped_aggLSynch.rename(columns={'mean':'T_iter'}) \n",
"grouped_aggLSynch = grouped_aggLSynch.compute()"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"Las siguientes dos celdas de código son para comprobar los Empirical Comulative distribution functions(ecdf).\n",
"Si dos ecdf crean una intersección el test de Conover-Iman no es fiable.\n",
"#TODO No esta terminado. Falta obtener cual ha sido el metodo ganador antes de llamar a \"check_specific_ecdf\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def test_ecdf(data1, data2):\n",
" diff_aux = data1[0] - data2[0]\n",
" cmp_f = (lambda a, b: a-b < 0) # Se esperan siempre diferencias positivas, si una es negativa se cruzan los ecdf's\n",
" if diff_aux < 0:\n",
" cmp_f = (lambda a, b: a-b > 0) # Se esperan siempre diferencias negativas, si una es positivas se cruzan los ecdf's\n",
" \n",
" for index_value in range(1,len(data1)):\n",
" if cmp_f(data1[index_value], data2[index_value]):\n",
" return False\n",
" return True\n",
"\n",
"def plot_ecdf(data1, data2, ecdf1, ecdf2, title):\n",
" f=plt.figure(figsize=(22, 14))\n",
" ax=f.add_subplot(111)\n",
"\n",
" ax.plot(data1, ecdf1, color=colors_m[0], linestyle=linestyle_m[0], \\\n",
" marker=markers_m[0], markersize=18)\n",
" ax.plot(data2, ecdf2, color=colors_m[2], linestyle=linestyle_m[2], \\\n",
" marker=markers_m[2], markersize=18)\n",
"\n",
" ax.set_xlabel(\"Values\", fontsize=36)\n",
" ax.set_ylabel(\"Bins\", fontsize=36)\n",
" ax.tick_params(axis='both', which='major', labelsize=36)\n",
" ax.tick_params(axis='both', which='minor', labelsize=36)\n",
" ax.set_title(title, fontsize=40)\n",
" \n",
" f.tight_layout()\n",
"\n",
"def check_specific_ecdf(np_aux, ns_aux, configuration_index):\n",
" df_check = dfM\n",
" tipo = 'T_Redistribution'\n",
" used_config = configurations_simple\n",
" dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n",
" ecdfLists = []\n",
" for index_i in range(len(dataLists)):\n",
" dataLists[index_i].sort()\n",
" ecdfLists.append( np.arange(1, len(dataLists[index_i])+1)/float(len(dataLists[index_i])) )\n",
"\n",
" index_i = configuration_index\n",
" data_i = dataLists[index_i]\n",
" ecdf_i = ecdfLists[index_i]\n",
" for index_j in range(len(dataLists)):\n",
" if index_i != index_j:\n",
" data_j = dataLists[index_j]\n",
" ecdf_j = ecdfLists[index_j]\n",
" res_aux = test_ecdf(data_i, data_j)\n",
" if not res_aux:\n",
" aux_title = \"NP=\"+str(np_aux)+\"/NC=\"+str(ns_aux)+\" -- Comparing C\"+str(index_i) + \" vs C\"+str(index_j)\n",
" plot_ecdf(data_i, data_j, ecdf_i, ecdf_j, aux_title)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux and not (np_aux==2 and ns_aux==20):\n",
" check_winner_ecdf(np_aux, ns_aux, 9)\n",
"\n",
" \n",
" "
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"a = 5\n",
"b = 3\n",
"c = False\n",
"d = a + b * c\n",
"print(d)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Ethernet\n",
"b1_aux = [82.747868, 83.191989, 95.520019, 87.435987, 90.843995, 150.356100]\n",
"b2_aux = [75.238174, 74.380054, 74.755995, 42.656109, 21.588040, 17.843997]\n",
"m1_aux = [74.654167, 74.357901, 74.351350, 43.599915, 21.637509, 15.128712]\n",
"m2_aux = [74.353249, 74.359214, 74.356160, 43.874266, 21.511082, 14.969010]\n",
"\n",
"b3_aux = [105.128014, 110.004008, 126.552019, 116.312400, 95.752019, 151.023994]\n",
"b4_aux = [83.021885, 77.632630, 75.396010, 43.076039, 24.028075, 19.556047]\n",
"m3_aux = [118.275992, 83.027866, 81.008479, 46.432212, 24.247949, 17.725083]\n",
"m4_aux = [119.286457, 84.205993, 80.741585, 47.144632, 24.206617, 17.738496]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Infiniband\n",
"b1_aux = [64.669525, 35.171971, 38.916010, 47.456630, 56.288048, 119.428020]\n",
"b2_aux = [36.538361, 15.536046, 13.396083, 9.652013, 5.772058, 5.615009]\n",
"m1_aux = [61.664380, 18.400559, 19.112526, 22.155880, 11.712381, 30.775627]\n",
"m2_aux = [33.428639, 13.905561, 14.691367, 7.363081, 6.629037, 12.150872]\n",
"\n",
"b3_aux = [91.368664, 60.648074, 53.663981, 49.152031, 64.752057, 118.243807]\n",
"b4_aux = [84.941260, 34.039990, 26.008021, 12.298989, 7.916004, 5.736054]\n",
"m3_aux = [105.839726, 26.822071, 23.834452, 12.876862, 9.063136, 7.007535]\n",
"m4_aux = [75.412319, 25.566336, 22.129483, 12.491161, 7.903744, 6.534291]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"plot_data_e = [b1_aux, m1_aux, b2_aux, m2_aux] #Expand\n",
"plot_data_s = [b3_aux, m3_aux, b4_aux, m4_aux] #Shrink\n",
"\n",
"labels_aux_e = create_labels_lineplot('e', lambda a, b: a == 160 or b == 160)\n",
"labels_aux_s = create_labels_lineplot('s', lambda a, b: a == 160 or b == 160)\n",
"#labels_aux = create_labels_lineplot(used_direction)\n",
"\n",
"labelsMethods_aux = ['Baseline - AllS', 'Merge - AllS',\n",
" 'Baseline - AllA','Merge - AllA']\n",
"\n",
"#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']\n",
"\n",
"f, (axe, axs)=plt.subplots(1,2,figsize=(22, 14))\n",
"x = np.arange(len(labels_aux_e))\n",
"for index in range(len(plot_data_e)):\n",
" array_aux_e = plot_data_e[index]\n",
" array_aux_s = plot_data_s[index]\n",
" plot_index = index\n",
" if index > 0:\n",
" plot_index = 2**plot_index\n",
" print(array_aux_e)\n",
" axe.plot(x, array_aux_e, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n",
" marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n",
" axs.plot(x, array_aux_s, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n",
" marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n",
"\n",
"axe.set_xlabel(\"(NP,NC)\", fontsize=36)\n",
"axe.set_ylabel(\"Time(s)\", fontsize=36)\n",
"axe.set_xticks(x)\n",
"axe.set_xticklabels(labels_aux_e, rotation=90)\n",
"axe.tick_params(axis='both', which='major', labelsize=36)\n",
"axe.tick_params(axis='both', which='minor', labelsize=36)\n",
"\n",
"axs.set_xlabel(\"(NP,NC)\", fontsize=36)\n",
"axs.set_ylabel(\"Time(s)\", fontsize=36)\n",
"axs.set_xticks(x)\n",
"axs.set_xticklabels(labels_aux_s, rotation=90)\n",
"axs.tick_params(axis='both', which='major', labelsize=36)\n",
"axs.tick_params(axis='both', which='minor', labelsize=36)\n",
"\n",
"plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n",
" \n",
"f.tight_layout()\n",
"f.savefig(\"Images/LinePlot_100Gb.png\", format=\"png\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import pandas as pd\n",
"from pandas import DataFrame, Series\n",
"import numpy as np\n",
"\n",
"\n",
"new_df = pd.read_pickle( \"dataBasic.pkl\" )\n",
"new_df.columns"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"grouped_aggG[\"T_total\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"100000 ** 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
%% Cell type:code id: tags:
``` python
%matplotlib inline
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import math
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.colors as colors
from matplotlib.legend_handler import HandlerLine2D, HandlerTuple
from matplotlib.colors import LinearSegmentedColormap
from scipy import stats
import scikit_posthocs as sp
import sys
from mpl_toolkits.mplot3d import axes3d
```
%% Cell type:code id: tags:
``` python
AllName="dataG.pkl"
ResizesName="dataM.pkl"
ItersName="dataL"
NameExtension=".pkl"
n_cores=20
repet = 5 #CAMBIAR EL NUMERO SEGUN NUMERO DE EJECUCIONES POR CONFIG
significance_value = 0.05
processes = [2,10,20,40,80,120,160]
positions = [321, 322, 323, 324, 325]
positions_small = [221, 222, 223, 224]
labels = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',
'(10,1)', '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',
'(20,1)', '(20,10)', '(20,40)', '(20,80)', '(20,120)','(20,160)',
'(40,1)', '(40,10)', '(40,20)', '(40,80)', '(40,120)','(40,160)',
'(80,1)', '(80,10)', '(80,20)', '(80,40)', '(80,120)','(80,160)',
'(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)','(120,160)',
'(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']
labelsExpand = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',
'(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',
'(20,40)', '(20,80)', '(20,120)','(20,160)',
'(40,80)', '(40,120)','(40,160)',
'(80,120)','(80,160)',
'(120,160)']
labelsShrink = ['(10,1)',
'(20,1)', '(20,10)',
'(40,1)', '(40,10)', '(40,20)',
'(80,1)', '(80,10)', '(80,20)', '(80,40)',
'(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)',
'(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']
# WORST BEST
labels_dist = ['null', 'SpreadFit', 'CompactFit']
#0 #1 #2 #3
labelsMethods = ['Baseline', 'Baseline single','Baseline - Asynchronous','Baseline single - Asynchronous',
'Merge','Merge single','Merge - Asynchronous','Merge single - Asynchronous']
#4 #5 #6 #7
colors_m = ['green','darkgreen','red','darkred','mediumseagreen','seagreen','palegreen','springgreen','indianred','firebrick','darkgoldenrod','saddlebrown']
linestyle_m = ['-', '--', '-.', ':']
markers_m = ['.','v','s','p', 'h','d','X','P','^']
OrMult_patch = mpatches.Patch(hatch='', facecolor='green', label='Baseline')
OrSing_patch = mpatches.Patch(hatch='', facecolor='springgreen', label='Baseline single')
OrPthMult_patch = mpatches.Patch(hatch='//', facecolor='blue', label='Baseline - Asyncrhonous')
OrPthSing_patch = mpatches.Patch(hatch='\\', facecolor='darkblue', label='Baseline single - Asyncrhonous')
MergeMult_patch = mpatches.Patch(hatch='||', facecolor='red', label='Merge')
MergeSing_patch = mpatches.Patch(hatch='...', facecolor='darkred', label='Merge single')
MergePthMult_patch = mpatches.Patch(hatch='xx', facecolor='yellow', label='Merge - Asyncrhonous')
MergePthSing_patch = mpatches.Patch(hatch='++', facecolor='olive', label='Merge single - Asyncrhonous')
handles_spawn = [OrMult_patch,OrSing_patch,OrPthMult_patch,OrPthSing_patch,MergeMult_patch,MergeSing_patch,MergePthMult_patch,MergePthSing_patch]
```
%% Cell type:code id: tags:
``` python
dfG = pd.read_pickle( AllName )
dfG['ADR'] = round((dfG['ADR'] / dfG['DR']) * 100,1)
dfG['SDR'] = round((dfG['SDR'] / dfG['DR']) * 100,1)
out_group = dfG.groupby(['Groups', 'ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_total']
group = dfG.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','Groups'])['T_total']
grouped_aggG = group.agg(['median'])
grouped_aggG.rename(columns={'median':'T_total'}, inplace=True)
out_grouped_G = out_group.agg(['median'])
out_grouped_G.rename(columns={'median':'T_total'}, inplace=True)
```
%% Cell type:code id: tags:
``` python
dfM = pd.read_pickle( ResizesName )
dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)
dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)
dfM['T_Redistribution'] = dfM['T_SR'] + dfM['T_AR']
dfM.loc[dfM['T_Malleability']==0,'T_Malleability'] = dfM['T_spawn'] + dfM['T_Redistribution']
out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']
group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']
grouped_aggM = group.agg(['median'])
grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)
out_grouped_M = out_group.agg(['median'])
out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)
```
%% Cell type:code id: tags:
``` python
grouped_aggLAsynch = pd.DataFrame()
grouped_aggLSynch = pd.DataFrame()
grouped_aggLDyn = pd.DataFrame()
grouped_aggLNDyn = pd.DataFrame()
for i in range(26):
aux_name = ItersName + str(i) + NameExtension
dfL = pd.read_pickle( aux_name )
#Fixme comprobar si hay iters asincronas antes de esto
aux_df = dfL[(dfL.Asynch_Iters == True)]
#aux_df['ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)
dfL.loc[(dfL.Asynch_Iters == True),'ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)
#dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)
dfL.loc[(dfL.Asynch_Iters == True),'SDR'] = round((aux_df['SDR'] / aux_df['DR']) * 100,1)
dfL['ADR'].fillna(-1, inplace=True)
dfL['SDR'].fillna(-1, inplace=True)
dfL['DR'].fillna(-1, inplace=True)
aux_df = dfL[(dfL.Asynch_Iters == True)]
group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter']
grouped_aux = group.agg(['median','count'])
grouped_aux.columns = grouped_aux.columns.get_level_values(0)
grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)
group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_stages']
aux_column = group.apply(list).apply(lambda x: np.median(x,0))
grouped_aux['T_stages'] = aux_column
if grouped_aggLAsynch.empty:
grouped_aggLAsynch = grouped_aux
else:
grouped_aggLAsynch = pd.concat([grouped_aggLAsynch, grouped_aux], ignore_index=False, sort=True).sort_index()
aux_df = dfL[(dfL.Asynch_Iters == False)]
group = aux_df.groupby('NP')['T_iter']
grouped_aux = group.agg(['median'])
grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)
group = aux_df.groupby(['NP'])['T_stages']
aux_column = group.apply(list).apply(lambda x: np.median(x,0))
grouped_aux['T_stages'] = aux_column
if grouped_aggLSynch.empty:
grouped_aggLSynch = grouped_aux
else:
grouped_aggLSynch = pd.concat([grouped_aggLSynch, grouped_aux], ignore_index=False, sort=True).sort_index()
aux_df2 = aux_df[(aux_df.Is_Dynamic == True)]
group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_iter']
grouped_aux = group.agg(['median'])
grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)
group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_stages']
aux_column = group.apply(list).apply(lambda x: np.median(x,0))
grouped_aux['T_stages'] = aux_column
if grouped_aggLDyn.empty:
grouped_aggLDyn = grouped_aux
else:
grouped_aggLDyn = pd.concat([grouped_aggLDyn, grouped_aux], ignore_index=False, sort=True).sort_index()
aux_df2 = aux_df[(aux_df.Is_Dynamic == False)]
group = aux_df2.groupby('NP')['T_iter']
grouped_aux = group.agg(['median'])
grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)
group = aux_df2.groupby(['NP'])['T_stages']
aux_column = group.apply(list).apply(lambda x: np.median(x,0))
grouped_aux['T_stages'] = aux_column
if grouped_aggLNDyn.empty:
grouped_aggLNDyn = grouped_aux
else:
grouped_aggLNDyn = pd.concat([grouped_aggLNDyn, grouped_aux], ignore_index=False, sort=True).sort_index()
dfL = None
group = grouped_aggLAsynch.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])
grouped_aux = group.agg(['mean'])
grouped_aux.columns = grouped_aux.columns.get_level_values(0)
grouped_aux['T_sum'] = grouped_aux['count'] * grouped_aux['T_iter']
grouped_aggLAsynch = grouped_aux
group = grouped_aggLSynch.groupby('NP')
grouped_aux = group.agg(['mean'])
grouped_aux.columns = grouped_aux.columns.get_level_values(0)
grouped_aggLSynch = grouped_aux
group = grouped_aggLDyn.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])
grouped_aux = group.agg(['mean'])
grouped_aux.columns = grouped_aux.columns.get_level_values(0)
grouped_aggLDyn = grouped_aux
group = grouped_aggLNDyn.groupby('NP')
grouped_aux = group.agg(['mean'])
grouped_aux.columns = grouped_aux.columns.get_level_values(0)
grouped_aggLNDyn = grouped_aux
```
%% Cell type:code id: tags:
``` python
from bt_scheme import PartialSolution, BacktrackingSolver
def elegirConf(parameters):
class StatePS(PartialSolution):
def __init__(self, config):
self.config= config
self.n= len(config) #Indica el valor a añadir
def is_solution(self):
return self.n == len(parameters)
def get_solution(self):
return tuple(self.config)
def successors(self):
array = parameters[self.n]
for parameter_value in array: #Test all values of the next parameter
self.config.append(parameter_value)
yield StatePS(self.config)
self.config.pop()
initialPs= StatePS([])
return BacktrackingSolver().solve(initialPs)
def obtenerConfs(parameters):
soluciones=[]
for solucion in elegirConf(parameters):
soluciones.append(solucion)
return soluciones
def modifyToGlobal(parameters, len_parameters, configuration):
usable_configuration = []
for i in range(len(parameters)):
if len_parameters[i] > 1:
aux = (parameters[i][0], configuration[i])
else:
aux = (configuration[i])
usable_configuration.append(aux)
return usable_configuration
def modifyToLocalDynamic(parameters, len_parameters, configuration):
usable_configuration = []
for i in range(len(parameters)):
if len_parameters[i] > 1:
aux = (configuration[i], -1)
else:
aux = (-1)
usable_configuration.append(aux)
return tuple(usable_configuration)
def CheckConfExists(configuration, dataSet, type_conf='global'):
exists = False
config = list(configuration)
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
if type_conf == 'global':
config.append((np_aux, ns_aux))
elif type_conf == 'malleability':
config.append(np_aux)
config.append(ns_aux)
elif type_conf == 'local':
config.append(np_aux)
if tuple(config) in dataSet.index:
exists = True # FIXME Return here true?
config.pop()
if type_conf == 'malleability':
config.pop()
return exists
```
%% Cell type:code id: tags:
``` python
adr = [0,96.6]
sp_method = [0,1]
rd_method = [0,1]
rd_strat = [1,2]
parameters = [adr, sp_method, rd_method, rd_strat]
parameters_names = ['ADR', 'Spawn_Method', 'Redistribution_Method', 'Redistribution_Strategy']
len_parameters = [1,2,2,2]
configurations_aux = obtenerConfs(parameters)
configurations = []
configurations_local_dynamic = set()
configurations_local = set()
configurations_simple = []
for checked_conf in configurations_aux:
aux_conf = modifyToGlobal(parameters, len_parameters, checked_conf)
if CheckConfExists(aux_conf, grouped_aggG):
configurations.append(aux_conf)
if CheckConfExists(checked_conf, grouped_aggM, 'malleability'):
configurations_simple.append(list(checked_conf))
aux_conf = modifyToLocalDynamic(parameters, len_parameters, checked_conf)
if CheckConfExists(aux_conf, grouped_aggLDyn, 'local'):
configurations_local_dynamic.add(aux_conf)
configurations_local_dynamic = list(configurations_local_dynamic)
for index in range(len(configurations_local_dynamic)):
configurations_local_dynamic[index] = list(configurations_local_dynamic[index])
print(configurations_simple)
print(configurations_local_dynamic)
print(configurations)
print(len(configurations))
```
%% Cell type:code id: tags:
``` python
#ALPHA COMPUTATION
def compute_alpha(config_a, config_b):
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
config_a.append(np_aux)
config_a.append(ns_aux)
config_b.append(np_aux)
config_b.append(ns_aux)
grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']
#grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Redistribution'] / grouped_aggM.loc[tuple(config_a),'T_Redistribution']
config_a.pop()
config_a.pop()
config_b.pop()
config_b.pop()
config_a.insert(0,ns_aux)
config_a.insert(0,np_aux)
config_b.insert(0,ns_aux)
config_b.insert(0,np_aux)
out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']
#out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Redistribution'] / out_grouped_M.loc[tuple(config_a),'T_Redistribution']
config_a.pop(0)
config_a.pop(0)
config_b.pop(0)
config_b.pop(0)
if not ('Alpha' in grouped_aggM.columns):
for config_a in configurations_simple:
for config_b in configurations_simple:
if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:
compute_alpha(config_a, config_b)
else:
print("ALPHA already exists")
```
%% Cell type:code id: tags:
``` python
#OMEGA COMPUTATION
def compute_omega(config):
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
if len(config) > len(parameters):
config.pop()
config.pop()
config.append(np_aux)
config.append(ns_aux)
value = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']
grouped_aggLAsynch.at[tuple(config),'Omega'] = value
#grouped_aggLAsynch.at[tuple(config),'Omega'] = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']
value = grouped_aggLAsynch.at[tuple(config),'T_stages'] / grouped_aggLSynch.at[np_aux,'T_stages']
grouped_aggLAsynch.at[tuple(config),'Omega_Stages'] = value.astype(object)
config.pop()
config.pop()
if not ('Omega' in grouped_aggLAsynch.columns):
grouped_aggLAsynch['Omega'] = 1.0
grouped_aggLAsynch['Omega_Stages'] = None
for config in configurations:
if config[0] != 0:
compute_omega(config)
else:
print("OMEGA already exists")
```
%% Cell type:code id: tags:
``` python
#Dynamic Coherence COMPUTATION
def compute_dyn_coherency(config):
for np_aux in processes:
for n_parents_aux in processes:
if np_aux != n_parents_aux:
config.append(np_aux)
config.append(n_parents_aux)
value = grouped_aggLDyn.at[tuple(config),'T_iter'] / grouped_aggLNDyn.at[np_aux,'T_iter']
grouped_aggLDyn.at[tuple(config),'Dyn_Coherency'] = value
value = grouped_aggLDyn.at[tuple(config),'T_stages'] / grouped_aggLNDyn.at[np_aux,'T_stages']
grouped_aggLDyn.at[tuple(config),'Dyn_Coherency_Stages'] = value.astype(object)
config.pop()
config.pop()
if not ('Dyn_Coherency' in grouped_aggLDyn.columns):
grouped_aggLDyn['Dyn_Coherency'] = 1.0
grouped_aggLDyn['Dyn_Coherency_Stages'] = None
for config in configurations_local_dynamic:
compute_dyn_coherency(config)
else:
print("Dyn_Coherency already exists")
```
%% Cell type:code id: tags:
``` python
#Malleability Coherence COMPUTATION
test=dfM[(dfM.Asynch_Iters > 0)]
coherence_boundary = 1.01 # Allows a 1% error
total_aux = 0
show_all = False
# Coherence of synchronous inner times + asynchronous iterations
for index in range(len(test)):
time_malleability_aux = test["T_Malleability"].values[index]
total_asynch_iters = int(test["Asynch_Iters"].values[index])
asynch_iters = test["T_iter"].values[index][-total_asynch_iters:]
time_iters_aux = np.sum(asynch_iters)
time_synch_aux = test["T_SR"].values[index]
time_spawn_aux = test["T_spawn"].values[index]
is_synch_spawn = (test["Spawn_Strategy"].values[index] % 2 != 0)
expected_time = time_iters_aux + time_spawn_aux * is_synch_spawn + time_synch_aux
time_malleability_aux = time_malleability_aux * coherence_boundary
if time_malleability_aux < expected_time:
total_aux += 1
if show_all:
print(test.iloc[index])
print(asynch_iters)
print(time_iters_aux)
print(time_malleability_aux)
print("")
if total_aux > 0:
print("Coherence fails for inner synchronous times + asynchornous iterations in " + str(total_aux) + " cases")
```
%% Cell type:code id: tags:
``` python
#Malleability Coherence COMPUTATION
test=dfM[(dfM.Asynch_Iters > 0)]
coherence_boundary = 1.01 # Allows a 1% error
# Coherence of inner times
aux_df = test["T_Malleability"] * coherence_boundary < (test["T_spawn"] + test["T_Redistribution"])
if not test[aux_df].empty:
print("Coherence fails for inner times of malleability in " + str(len(test[aux_df])) + " cases")
display(test[aux_df])
```
%% Cell type:code id: tags:
``` python
out_grouped_G.to_excel("resultG.xlsx")
out_grouped_M.to_excel("resultM.xlsx")
grouped_aggLAsynch.to_excel("AsynchIters.xlsx")
grouped_aggLSynch.to_excel("SynchIters.xlsx")
grouped_aggLNDyn.to_excel("StaticCoherence.xlsx")
grouped_aggLDyn.to_excel("DynCoherence.xlsx")
```
%% Cell type:code id: tags:
``` python
dfL.columns
```
%% Cell type:code id: tags:
``` python
def create_group_boundary(rms_boundary, np_aux, ns_aux):
tc_boundary = 0
boundaries = None
if rms_boundary != 0:
# El porcentaje de tc_boundary se tiene en cuenta para eliminar aquellos
# tiempos demasiado grandes en su malleability time respecto al más pequeño
boundaries = get_np_ns_data("T_Malleability", grouped_aggM, configurations_simple, np_aux, ns_aux)
tc_boundary = min(boundaries)
tc_boundary = tc_boundary + tc_boundary*rms_boundary
return tc_boundary, boundaries
# Aquellos grupos que tengán valores por encima del límite no se considerarán
def check_groups_boundaries(dataLists, boundaries, tc_boundary):
for index in range(len(boundaries)):
if boundaries[index] > tc_boundary:
dataLists[index] = float('infinity')
```
%% Cell type:code id: tags:
``` python
def get_perc_differences(dataLists, boundaries, tc_boundary):
perc = 1.05
if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos
check_groups_boundaries(dataLists, boundaries, tc_boundary)
indexes = np.argsort(dataLists)
best = -1
bestMax = -1
otherBest=[]
for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en media/mediana
if best == -1:
best = index
bestMax = dataLists[best] * perc
elif dataLists[index] <= bestMax: # Media/Medianas i < Media/Mediana best
otherBest.append(index)
otherBest.insert(0,best)
return otherBest
def get_stat_differences(dataLists, df_Res, boundaries, tc_boundary):
if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos
check_groups_boundaries(dataLists, boundaries, tc_boundary)
indexes = np.argsort(dataLists)
best = -1
otherBest=[]
for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en mediana
if dataLists[index] != float('infinity'):
if best == -1:
best = index
elif not df_Res.iat[best,index]: # df_Res == False indicates 'index' and 'best' have the same mean/median
otherBest.append(index)
otherBest.insert(0,best)
return otherBest
```
%% Cell type:code id: tags:
``` python
grouped_np = ["T_total"]
separated_np = ["T_Malleability", "T_Redistribution", "T_spawn", "T_SR", "T_AR", "Alpha", "Omega", "count"]
def get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux):
dataLists=[]
for config in used_config:
if tipo in grouped_np:
config.append((np_aux,ns_aux))
elif tipo in separated_np:
config.append(np_aux)
config.append(ns_aux)
if tuple(config) in data_aux.index:
aux_value = data_aux.loc[tuple(config),tipo]
if isinstance(aux_value, pd.Series):
aux_value = aux_value.values[0]
if aux_value == 0: #Values of zero indicate it was not performed
aux_value = float('infinity')
else: # This configuration is not present in the dataset
aux_value = float('infinity')
dataLists.append(aux_value)
config.pop()
if tipo in separated_np:
config.pop()
return dataLists
def get_config_data(tipo, data_aux, config):
dataLists=[]
procsLists=[]
for ns_aux in processes:
for np_aux in processes:
if np_aux != ns_aux:
if tipo in grouped_np:
config.append((np_aux,ns_aux))
elif tipo in separated_np:
config.append(np_aux)
config.append(ns_aux)
if tuple(config) in data_aux.index:
procsLists.append((np_aux,ns_aux))
aux_value = data_aux.loc[tuple(config),tipo]
if isinstance(aux_value, pd.Series):
aux_value = aux_value.values[0]
if aux_value == 0: #Values of zero indicate it was not performed
aux_value = float('infinity')
else: # This configuration is not present in the dataset
aux_value = float('infinity')
dataLists.append(aux_value)
config.pop()
if tipo in separated_np:
config.pop()
return dataLists, procsLists
def get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux):
dataLists=[]
if tipo in grouped_np:
tuple_data = (np_aux, ns_aux)
df_npns_aux = df_check.loc[(df_check['Groups']==tuple_data)]
elif tipo in separated_np:
df_npns_aux = df_check.loc[(df_check['NP']==np_aux)]
df_npns_aux = df_npns_aux.loc[(df_npns_aux['NC']==ns_aux)]
for config in used_config:
df_config_aux = df_npns_aux
for index in range(len(config)):
aux_name = parameters_names[index]
aux_value = config[index]
df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]
aux_value = list(df_config_aux[tipo])
if len(aux_value) > 0:
dataLists.append(aux_value)
return dataLists
def get_df_config_data(df_check, tipo, config):
dataLists=[]
df_config_aux = df_check
for index in range(len(config)):
aux_name = parameters_names[index]
aux_value = config[index]
df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
if tipo in grouped_np:
tuple_data = (np_aux, ns_aux)
df_aux = df_config_aux.loc[(df_config_aux['Groups']==tuple_data)]
elif tipo in separated_np:
df_aux = df_config_aux.loc[(df_config_aux['NP']==np_aux)]
df_aux = df_aux.loc[(df_aux['NC']==ns_aux)]
aux_value = list(df_aux[tipo])
if len(aux_value) > 0:
dataLists.append(aux_value)
return dataLists
```
%% Cell type:code id: tags:
``` python
def check_normality(df_check, tipo, used_config, fast=True):
normality_array=[True] * (len(processes) * (len(processes)-1) * len(used_config))
normality = True
total=0
i=-1
#Comprobar para cada configuración si se sigue una distribución normal/gaussiana
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
i+=1
dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)
for data_aux in dataLists:
st,p = stats.shapiro(data_aux) # Tendrían que ser al menos 20 datos y menos de 50
if p < significance_value: # Reject H0
if fast:
return False
normality_array[i] = False
normality = False
total+=1
print("Se sigue una distribución guassiana: " + str(normality) + "\nUn total de: " + str(total) + " no siguen una gaussiana")
print(normality_array)
return normality
def check_homoscedasticity(df_check, tipo, used_config, fast=True):
homoscedasticity_array=[True] * (len(processes) * (len(processes)-1))
homoscedasticity = True
total=0
i=-1
#Comprobar para cada configuración es homoestatica
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
i+=1
dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)
st,p = stats.levene(*dataLists) # Tendrían que ser al menos 20 datos y menos de 50
if p < significance_value: # Reject H0
if fast:
return False
homoscedasticity_array[i] = False
homoscedasticity = False
total+=1
print("Se sigue una distribución de datos Homocedastica: " + str(homoscedasticity) + "\nUn total de: " + str(total) + " no siguen una homocedastica")
print(homoscedasticity_array)
return homoscedasticity
def compute_global_stat_difference(dataLists, parametric, np_aux, ns_aux):
if parametric:
st,p=stats.f_oneway(*dataLists)
else:
st,p=stats.kruskal(*dataLists)
if p > significance_value:
print("For NP " + str(np_aux) + " and " + str(ns_aux) + " is accepted H0")
return True # Equal values || Accept H0
return False # Some groups are different || Reject H0
def compute_global_posthoc(dataLists, parametric): #TODO Comprobar CDF de los grupos
data_stats=[]
data_stats2=[]
ini=0
end=len(dataLists)
if parametric:
df_aux = sp.posthoc_ttest(dataLists)
df_Res = df_aux.copy()
for i in range(ini,end):
data_stats.append(np.mean(dataLists[i]))
for j in range(ini,end):
if df_Res.iat[i,j] < significance_value: # Different means || Reject H0
df_Res.iat[i, j] = True
else:
df_Res.iat[i, j] = False
else:
df_aux = sp.posthoc_conover(dataLists)
df_Res = df_aux.copy()
for i in range(ini,end):
data_stats.append(np.median(dataLists[i]))
#data_stats2.append(stats.iqr(dataLists[i],axis=0))
for j in range(ini,end):
if df_Res.iat[i,j] < significance_value: # Different medians || Reject H0
df_Res.iat[i, j] = True # Not equal medians
else:
df_Res.iat[i, j] = False # Equal medians
#print(df_Res)
#print(df_aux)
#print(data_stats)
#print(data_stats2)
#aux_value = min(data_stats)
#print(data_stats.index(aux_value))
return df_Res, data_stats
```
%% Cell type:code id: tags:
``` python
def results_with_perc(tipo, data_aux, used_config, rms_boundary=0):
results = []
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)
#Get all values for particular config with these number of processes
dataLists = get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux)
aux_data = get_perc_differences(dataLists, boundaries, tc_boundary)
results.append(aux_data)
return results
def results_with_stats(tipo, df_check, used_config, rms_boundary=0):
results = []
use_parametric = check_normality(df_check, tipo, used_config)
if use_parametric:
use_parametric = check_homoscedasticity(df_check, tipo, used_config)
print("Se usan tests parametricos: "+str(use_parametric))
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)
#Get all values for particular config with these number of processes
dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)
equal_set = compute_global_stat_difference(dataLists, use_parametric, np_aux, ns_aux)
if equal_set:
aux_data = list(range(len(used_config))) # All data is equal
else:
res_aux, times_aux = compute_global_posthoc(dataLists, use_parametric)
aux_data = get_stat_differences(times_aux, res_aux, boundaries, tc_boundary)
results.append(aux_data)
return results
```
%% Cell type:code id: tags:
``` python
checked_type='T_total'
use_perc = False
select_first_winner = False
prefer_first_winner = False
rms_boundary=0 # Poner a 0 para perspectiva de app. Valor >0 y <1 para perspectiva de RMS
if checked_type=='T_total':
tipo="T_total"
if use_perc:
data_aux = grouped_aggG
else:
data_aux = dfG
used_config = configurations
elif checked_type=='T_Malleability' or checked_type=='T_spawn' or checked_type=='T_SR' or checked_type=='T_AR' or checked_type=='T_Redistribution':
tipo=checked_type
if use_perc:
data_aux = grouped_aggM
else:
data_aux = dfM
if tipo == 'T_AR':
data_aux = data_aux[(data_aux.ADR > 0)]
elif tipo == 'T_SR':
data_aux = data_aux[(data_aux.ADR == 0)]
used_config = configurations_simple
if use_perc:
results = results_with_perc(tipo, data_aux, used_config, rms_boundary)
else:
results = results_with_stats(tipo, data_aux, used_config, rms_boundary)
if not use_perc and tipo == 'T_AR': #FIXME!!!! No tiene en cuenta total de configuraciones sincronos
for res_index in range(len(results)):
for inner_index in range(len(results[res_index])):
results[res_index][inner_index]+=4
#Results is a 2 dimensional array. First dimension indicates lists of winners of a particulal number of processes (NP->NC).
#Second dimension is an ordered preference of indexes in the array configurations.
print(results)
print(len(results))
```
%% Cell type:code id: tags:
``` python
#Lista de indices de mayor a menor según el total de ocurrencias
aux_array = []
for data in results:
aux_array+=data
aux_keys, aux_counts = np.unique(aux_array, return_counts=True)
aux_ordered_index=list(reversed(np.argsort(aux_counts)))
#Lista de indices de mayor a menor según el total de ocurrencias del primero de cada grupo
aux_array = [0] * len(results)
for index in range(len(results)):
aux_array[index] = results[index][0]
aux_keys_best, aux_counts_best = np.unique(aux_array, return_counts = True)
aux_ordered_best_index=list(reversed(np.argsort(aux_counts_best)))
def heatmap_get_best(index, ordered_array, keys_array, counts_array, prefer_winner=False):
valid_candidates_indexes = []
prev_counts = -1
for tested_index in ordered_array:
if keys_array[tested_index] in results[index]:
if counts_array[tested_index] >= prev_counts:
prev_counts = counts_array[tested_index]
valid_candidates_indexes.append(tested_index)
else:
break
if prefer_winner: # Si esta activo, en caso de empate en ocurrencias se selecciona el menor tiempo
for tested_index in results[index]:
if tested_index in valid_candidates_indexes:
return tested_index
return min(valid_candidates_indexes) # En caso de empate se devuelve el que tiene menor valor (Suele ser la config más simple)
i=0
j=0
used_aux=0
heatmap=np.zeros((len(processes),len(processes))).astype(int)
if select_first_winner:
for i in range(len(processes)):
for j in range(len(processes)):
if i==j:
heatmap[i][j]=-1
used_aux+=1
else:
results_index = i*len(processes) + j - used_aux
heatmap[i][j] = results[results_index][0]
else:
for i in range(len(processes)):
for j in range(len(processes)):
if i==j:
heatmap[i][j]=-1
used_aux+=1
else:
results_index = i*len(processes) + j - used_aux
index = heatmap_get_best(results_index, aux_ordered_index, aux_keys, aux_counts, prefer_first_winner)
heatmap[i][j]=aux_keys[index]
#index = heatmap_get_best(results_index, aux_ordered_best_index, aux_keys_best, aux_counts_best, prefer_first_winner)
#heatmap[i][j]=aux_keys_best[index]
heatmap[-1][-1]=len(used_config)
print(aux_keys)
print(aux_counts)
print(heatmap)
```
%% Cell type:code id: tags:
``` python
#Adapta results a una cadena asegurando que cada cadena no se sale de su celda
def get_heatmap_multiple_strings(results): #FIXME Deprecated
results_str = []
max_counts = 1
max_per_line = 3
for i in range(len(results)):
results_str.append(list())
count = len(results[i])
results_aux = results[i]
if count > max_counts:
count = max_counts
results_aux = results[i][:count]
remainder = count%max_per_line
if count <= max_per_line:
aux_str = str(results_aux).replace('[','').replace(']','')
results_str[i].append(aux_str)
else:
if remainder == 0:
index = count//2
else:
index = count - ((remainder-1)*max_per_line + 1)
aux_str = str(results_aux[:index]).replace('[','').replace(']','')
results_str[i].append(aux_str)
aux_str = str(results_aux[index:]).replace('[','').replace(']','')
results_str[i].append(aux_str)
return results_str
def get_heatmap_strings(heatmap):
results_str = []
for i in range(len(processes)):
for j in range(len(processes)):
if i!=j:
results_str.append(list())
results_str[-1].append(heatmap[i][j])
return results_str
```
%% Cell type:code id: tags:
``` python
#Crea un heatmap teniendo en cuenta los colores anteriores
f=plt.figure(figsize=(24, 12))
ax=f.add_subplot(111)
myColors = (colors.to_rgba("white"),
colors.to_rgba("green"), #BCOLS
colors.to_rgba("darkgreen"), #BP2PS
#colors.to_rgba("blue"), #BRMA1S
#colors.to_rgba("royalblue"), #BRMA2S
colors.to_rgba("red"), #MCOLS
colors.to_rgba("darkred"), #MP2PS
#colors.to_rgba("mediumblue"), #MRMA1S
#colors.to_rgba("mediumslateblue"), #MRMA2S
#colors.to_rgba("blue"), #BIntraCOLS
#colors.to_rgba("royalblue"), #BIntraP2PS
colors.to_rgba("mediumseagreen"), #BCOLA
colors.to_rgba("seagreen"), #BCOLT
colors.to_rgba("palegreen"), #BP2PA
colors.to_rgba("springgreen"), #BP2PT
#colors.to_rgba("purple"), #BRMA1A
#colors.to_rgba("darkviolet"), #BRMA1T
#colors.to_rgba("indigo"), #BRMA2A
#colors.to_rgba("rebeccapurple"), #BRMA2T
colors.to_rgba("indianred"), #MCOLA
colors.to_rgba("firebrick"), #MCOLT
colors.to_rgba("darkgoldenrod"), #MP2PA
colors.to_rgba("saddlebrown"), #MP2PT
#colors.to_rgba("magenta"), #MRMA1A
#colors.to_rgba("violet"), #MRMA1T
#colors.to_rgba("deeppink"), #MRMA2A
#colors.to_rgba("mediumvioletred"), #MRMA2T
#colors.to_rgba("mediumblue"), #BIntraCOLA
#colors.to_rgba("mediumslateblue"), #BIntraP2PA
colors.to_rgba("white"))
cmap = LinearSegmentedColormap.from_list('Custom', myColors, len(myColors))
im = ax.imshow(heatmap,cmap=cmap,interpolation='nearest')
# Loop over data dimensions and create text annotations.
used_aux=0
results_str = get_heatmap_strings(heatmap)
for i in range(len(processes)):
for j in range(len(processes)):
if i!=j:
aux_color="white"
if 0 <= heatmap[i, j] <= 1 or 4 <= heatmap[i, j] <= 7: # El 1 puede necesitar texto en negro
aux_color="black"
results_index = i*len(processes) +j-used_aux
if len(results_str[results_index]) == 1:
text = results_str[results_index][0]
ax.text(j, i, text, ha="center", va="center", color=aux_color, fontsize=36)
else:
add_aux = 0.33
for line in range(len(results_str[results_index])):
i_range = i - 0.5 + add_aux
ax.text(j, i_range, results_str[results_index][line],
ha="center", va="center", color=aux_color, fontsize=36)
add_aux+=0.33
else:
used_aux+=1
ax.set_ylabel("NS", fontsize=36)
ax.set_xlabel("NT", fontsize=36)
ax.set_xticklabels(['']+processes, fontsize=36)
ax.set_yticklabels(['']+processes, fontsize=36)
labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',
'Merge - COLS (2)','Merge - P2PS (3)',
'Baseline - COLA (4)', 'Baseline - COLT (5)','Baseline - P2PA (6)','Baseline - P2PT (7)',
'Merge - COLA (8)','Merge - COLT (9)','Merge - P2PA (10)','Merge - P2PT (11)']
#labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',
# 'Baseline - RMA1S (2)', 'Baseline - RMA2S (3)',
# 'Merge - COLS (4)','Merge - P2PS (5)',
# 'Merge - RMA1S (6)', 'Merge - RMA2S (7)',
# 'Baseline - COLA (8)', 'Baseline - COLT (9)','Baseline - P2PA (10)','Baseline - P2PT (11)',
# 'Baseline - RMA1A (12)', 'Baseline - RMA1T (13)','Baseline - RMA2A (14)','Baseline - RMA2T (15)',
# 'Merge - COLA (16)','Merge - COLT (17)','Merge - P2PA (18)','Merge - P2PT (19)',
# 'Merge - RMA1A (20)','Merge - RMA1T (21)','Merge - RMA2A (22)','Merge - RMA2T (23)']
colorbar=f.colorbar(im, ax=ax)
tick_bar = []
for i in range(len(used_config)):
#tick_bar.append(0.41 + i*0.96) #Config de 24 valores
tick_bar.append(0.37 + i*0.92) #Config de 12 valores
#tick_bar.append(0.35 + i*0.89) #Config de 8 valores
colorbar.set_ticks(tick_bar)
colorbar.set_ticklabels(labelsMethods_aux)
colorbar.ax.tick_params(labelsize=32)
#
f.tight_layout()
print("Filename: Heatmap_"+tipo+".png")
f.savefig("Images/Heatmap_"+tipo+".jpeg", format="jpeg", dpi=300)
```
%% Cell type:code id: tags:
``` python
aux_array = [] #Counts all
for data in results:
aux_array+=data
aux_results, aux_counts = np.unique(aux_array, return_counts=True)
print(aux_results)
print(aux_counts)
aux_array = [0] * len(results) # Counts ganador celda
for index in range(len(results)):
aux_array[index] = results[index][0]
aux_results, aux_counts = np.unique(aux_array, return_counts = True)
print(aux_results)
print(aux_counts)
```
%% Cell type:raw id: tags:
El siguiente código asume que para cada número de procesos padre/hijo existen valores en todas las configuraciones que se van a probar
%% Cell type:code id: tags:
``` python
def normalize_arrays(arrays, norm_array):
nef normalize_arrays(arrays, norm_array):
new_arrays = arrays.copy()
for index in range(len(new_arrays)):
new_arrays[index] = np.divide(norm_array, new_arrays[index])
return new_arrays
def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):
labels_aux = []
if used_direction == 's':
for ns_aux in processes:
for np_aux in processes:
if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):
new_label = "(" + str(np_aux) + "," + str(ns_aux) + ")"
labels_aux.append(new_label)
else:
for np_aux in processes:
for ns_aux in processes:
if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):
new_label = "(" + str(np_aux) + "," + str(ns_aux) + ")"
labels_aux.append(new_label)
return labels_aux
def reorder_data(plot_data, actual_order, expected_order):
ordered_indexes = []
len_order = len(actual_order)
for index in range(len_order):
actual_order[index] = str(actual_order[index]).replace(" ", "")
for index in range(len_order):
ordered_indexes.append(actual_order.index(expected_order[index]))
for index in range(len(plot_data)):
old_array = plot_data[index]
new_array = []
for i in ordered_indexes:
new_array.append(old_array[i])
plot_data[index] = new_array
return plot_dataew_arrays = arrays.copy()
for index in range(len(new_arrays)):
new_arrays[index] = np.divide(norm_array, new_arrays[index])
return new_arrays
def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):
labels_aux = []
if used_direction == 's':
for ns_aux in processes:
for np_aux in processes:
if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):
new_label = "(" + str(np_aux) + "," + str(ns_aux) + ")"
labels_aux.append(new_label)
else:
for np_aux in processes:
for ns_aux in processes:
if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):
new_label = "(" + str(np_aux) + "," + str(ns_aux) + ")"
labels_aux.append(new_label)
return labels_aux
def reorder_data(plot_data, actual_order, expected_order):
ordered_indexes = []
len_order = len(actual_order)
for index in range(len_order):
actual_order[index] = str(actual_order[index]).replace(" ", "")
for index in range(len_order):
ordered_indexes.append(actual_order.index(expected_order[index]))
for index in range(len(plot_data)):
old_array = plot_data[index]
new_array = []
for i in ordered_indexes:
new_array.append(old_array[i])
plot_data[index] = new_array
return plot_data
```
%% Cell type:code id: tags:
``` python
used_direction='s'
test_parameter='T_Malleability' #Valores son "alpha" o "omega"
if test_parameter == 'alpha':
name_fig="Alpha_"
real_parameter='Alpha'
name_legend = "Valores de α"
normalize = True
allow_all = False
used_config = configurations_simple
data_aux = grouped_aggM[grouped_aggM[real_parameter] > 0]
elif test_parameter == 'T_Malleability':
name_fig="Malleability_"
real_parameter='T_Malleability'
name_legend = "Time(s)"
normalize = False
allow_all = True
used_config = configurations_simple
data_aux = grouped_aggM
elif test_parameter == 'T_Redistribution':
name_fig="Redistribution_"
real_parameter='T_Redistribution'
name_legend = "Tiempo(s)"
normalize = False
allow_all = True
used_config = configurations_simple
data_aux = grouped_aggM
elif test_parameter == 'omega':
name_fig="Omega_"
real_parameter='Omega'
name_legend = "Values of ω"
normalize = True
allow_all = False
used_config = configurations
data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]
elif test_parameter == 'iters_count':
name_fig="Iters_"
real_parameter='count'
name_legend = "Asynchronous iterations"
normalize = False
allow_all = True
used_config = configurations
data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]
if used_direction=='s':
data_aux=data_aux.query('NP > NC')
name_fig= name_fig+"Shrink"
elif used_direction=='e':
data_aux=data_aux.query('NP < NC')
name_fig= name_fig+"Expand"
elif used_direction=='a':
name_fig= name_fig+"All"
#data_aux=data_aux.query('NP == 160 or NC == 160')
plot_data = []
for config in used_config:
if config[0] > 0 or allow_all:
dataLists,procsLists = get_config_data(real_parameter, data_aux, config)
dataLists = list(filter(lambda x: x != float('infinity'), dataLists))
plot_data.append(dataLists)
#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)
labels_aux = create_labels_lineplot(used_direction)
plot_data = reorder_data(plot_data, procsLists, labels_aux)
#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',
# 'Baseline - RMA1S', 'Baseline - RMA2S',
# 'Merge - COLS','Merge - P2PS',
# 'Merge - RMA1S', 'Merge - RMA2S',
# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',
# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',
# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',
# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'
# ]
labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',
'Merge - COLS','Merge - P2PS',
'Baseline - COLA', 'Baseline - COLT', 'Baseline - P2PA', 'Baseline - P2PT',
'Merge - COLA', 'Merge - COLT', 'Merge - P2PA', 'Merge - P2PT']
colors_m = (
colors.to_rgba("green"), #BCOLS
colors.to_rgba("darkgreen"), #BP2PS
#colors.to_rgba("blue"), #BRMA1S
#colors.to_rgba("royalblue"), #BRMA2S
colors.to_rgba("red"), #MCOLS
colors.to_rgba("darkred"), #MP2PS
#colors.to_rgba("mediumblue"), #MRMA1S
#colors.to_rgba("mediumslateblue"), #MRMA2S
#colors.to_rgba("blue"), #BIntraCOLS
#colors.to_rgba("royalblue"), #BIntraP2PS
colors.to_rgba("mediumseagreen"), #BCOLA
colors.to_rgba("seagreen"), #BCOLT
colors.to_rgba("palegreen"), #BP2PA
colors.to_rgba("springgreen"), #BP2PT
#colors.to_rgba("purple"), #BRMA1A
#colors.to_rgba("darkviolet"), #BRMA1T
#colors.to_rgba("indigo"), #BRMA2A
#colors.to_rgba("rebeccapurple"), #BRMA2T
colors.to_rgba("indianred"), #MCOLA
colors.to_rgba("firebrick"), #MCOLT
colors.to_rgba("darkgoldenrod"), #MP2PA
colors.to_rgba("saddlebrown"), #MP2PT
#colors.to_rgba("magenta"), #MRMA1A
#colors.to_rgba("violet"), #MRMA1T
#colors.to_rgba("deeppink"), #MRMA2A
#colors.to_rgba("mediumvioletred"), #MRMA2T
#colors.to_rgba("mediumblue"), #BIntraCOLA
#colors.to_rgba("mediumslateblue"), #BIntraP2PA
)
f=plt.figure(figsize=(22, 14))
ax=f.add_subplot(111)
x = np.arange(len(labels_aux))
for index in range(len(plot_data)):
array_aux = plot_data[index]
plot_index = index
if real_parameter == 'Alpha' or real_parameter == 'Omega' or real_parameter == 'count': #FIXME This line is a lie...
plot_index = 4 + index #FIXME This line is a lie...
print(array_aux)
ax.plot(x, array_aux, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \
marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[plot_index])
ax.set_xlabel("(NS,NT)", fontsize=36)
ax.set_ylabel(name_legend, fontsize=36)
if normalize:
ax.axhline(y=1, color='black', linestyle='--')
plt.xticks(x, labels_aux,rotation=90)
ax.tick_params(axis='both', which='major', labelsize=36)
ax.tick_params(axis='both', which='minor', labelsize=36)
plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)
f.tight_layout()
f.savefig("Images/LinePlot_"+name_fig+".png", format="png")
```
%% Cell type:raw id: tags:
================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================
%% Cell type:raw id: tags:
Gráfica de lineas para generar tiempos del grupo G.
%% Cell type:code id: tags:
``` python
used_direction='e'
test_parameter='T_total'
intranode=False
if test_parameter == 'T_total':
name_fig="Ttotal"
real_parameter='T_total'
name_legend = "Time(s)"
used_config = configurations
data_aux = grouped_aggG
#data_aux = data_aux[data_aux.index.isin(df1.index)]
if used_direction=='s':
data_aux_cmp=grouped_aggM.reset_index().query('NP > NC')
name_fig= name_fig+"Shrink"
elif used_direction=='e':
data_aux_cmp=grouped_aggM.reset_index().query('NP < NC')
name_fig= name_fig+"Expand"
elif used_direction=='a':
data_aux_cmp=grouped_aggM.reset_index()
name_fig= name_fig+"All"
if intranode:
data_aux_cmp = data_aux_cmp.query('NP <= 20 and NC <= 20')
#else:
#data_aux_cmp = data_aux_cmp.query('NP > 20 and NC > 20')
#data_aux_cmp = data_aux_cmp.query('NP == 160 or NC == 160')
if used_direction!='a' or True:
pruebaG = data_aux.reset_index()
pruebaG = pruebaG.loc[pruebaG.index.intersection(data_aux_cmp.index)]
data_aux = data_aux[(data_aux.T_total.isin(pruebaG.T_total))]
plot_data = []
for config in used_config:
#if config[0] == 0:
dataLists,procsLists = get_config_data(real_parameter, data_aux, config)
dataLists = list(filter(lambda x: x != float('infinity'), dataLists))
plot_data.append(dataLists)
for array_aux in plot_data:
print(array_aux)
#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)
labels_aux = create_labels_lineplot(used_direction)
plot_data = reorder_data(plot_data, procsLists, labels_aux)
plot_data_normalized = normalize_arrays(plot_data[1:], plot_data[0])
name_legend="SpeedUp over Baseline"
#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']
labelsMethods_aux = ['Baseline - AllS', 'Baseline - P2PS',
'Merge - AllS','Merge - P2PS',
'Baseline - AllA', 'Baseline - AllT','Baseline - P2PA','Baseline - P2PT',
'Merge - AllA','Merge - AllT','Merge - P2PA','Merge - P2PT']
#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',
# 'Baseline - RMA1S', 'Baseline - RMA2S',
# 'Merge - COLS','Merge - P2PS',
# 'Merge - RMA1S', 'Merge - RMA2S',
# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',
# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',
# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',
# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'
# ]
colors_m = (
colors.to_rgba("green"), #BCOLS
colors.to_rgba("darkgreen"), #BP2PS
#colors.to_rgba("blue"), #BRMA1S
#colors.to_rgba("royalblue"), #BRMA2S
colors.to_rgba("red"), #MCOLS
colors.to_rgba("darkred"), #MP2PS
#colors.to_rgba("mediumblue"), #MRMA1S
#colors.to_rgba("mediumslateblue"), #MRMA2S
#colors.to_rgba("blue"), #BIntraCOLS
#colors.to_rgba("royalblue"), #BIntraP2PS
colors.to_rgba("mediumseagreen"), #BCOLA
colors.to_rgba("seagreen"), #BCOLT
colors.to_rgba("palegreen"), #BP2PA
colors.to_rgba("springgreen"), #BP2PT
#colors.to_rgba("purple"), #BRMA1A
#colors.to_rgba("darkviolet"), #BRMA1T
#colors.to_rgba("indigo"), #BRMA2A
#colors.to_rgba("rebeccapurple"), #BRMA2T
colors.to_rgba("indianred"), #MCOLA
colors.to_rgba("firebrick"), #MCOLT
colors.to_rgba("darkgoldenrod"), #MP2PA
colors.to_rgba("saddlebrown"), #MP2PT
#colors.to_rgba("magenta"), #MRMA1A
#colors.to_rgba("violet"), #MRMA1T
#colors.to_rgba("deeppink"), #MRMA2A
#colors.to_rgba("mediumvioletred"), #MRMA2T
#colors.to_rgba("mediumblue"), #BIntraCOLA
#colors.to_rgba("mediumslateblue"), #BIntraP2PA
)
f=plt.figure(figsize=(22, 14))
ax=f.add_subplot(111)
ax2 = ax.twinx()
x = np.arange(len(labels_aux))
for index in range(len(plot_data_normalized)):
array_aux = plot_data_normalized[index]
index= index+1
ax.plot(x, array_aux, color=colors_m[index%len(colors_m)], linestyle=linestyle_m[index%len(linestyle_m)], \
marker=markers_m[index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])
ax2.plot(x, plot_data[0], color='black', linestyle=linestyle_m[0], \
marker=markers_m[0], markersize=18, label=labelsMethods_aux[0])
ax.axhline(y=1, color='black', linestyle='--')
ax.set_xlabel("(NP,NC)", fontsize=36)
ax.set_ylabel(name_legend, fontsize=36)
ax.tick_params(axis='both', which='both', labelsize=36)
ax.set_xticks(x)
ax.set_xticklabels(labels_aux, rotation=90)
#ax.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)
ax2.set_ylabel('Baseline Time(s)', fontsize=36)
ax2.tick_params(axis='y', which='both', labelsize=36)
#ax2.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)
f.legend(bbox_to_anchor=(0.5, 0.98), fontsize=26,ncol=2,framealpha=0.8)
f.tight_layout()
f.savefig("Images/LinePlot_"+name_fig+".png", format="png")
```
%% Cell type:raw id: tags:
El siguiente generá una imagen en 3d de T_total para cada una de las diferentes configuraciones.
%% Cell type:code id: tags:
``` python
def generate_3d_image(config, name):
fig, ax = plt.subplots(1, 1, subplot_kw={'projection': '3d'}, figsize=(15, 15))
Z = [None] * len(processes)
X, Y = np.meshgrid(processes, processes)
for i in range(len(processes)):
np_aux = processes[i]
Z[i] = [0] * len(processes)
Z[i][i] = grouped_aggLSynch.loc[np_aux, 'T_iter'] * 1000
for j in range(len(processes)):
if i!=j:
ns_aux = processes[j]
config.append((np_aux,ns_aux))
aux = grouped_aggG.loc[tuple(config),'T_total']
config.pop()
Z[i][j] = aux.values[0]
#Z[i][j] = Z[i][j] / Z[i][i]
#Z[i][i] = 1
Z = np.array(Z)
ax.plot_surface(X, Y, Z, rstride=1, cstride=1,
cmap='viridis', edgecolor='none')
ax.view_init(15, 25)
ax.set_xlabel("NC", fontsize=16)
ax.set_ylabel("NP", fontsize=16)
ax.set_zlabel("Normalized time", fontsize=16)
ax.set_title(name, fontsize=10)
plt.show()
for index in range(len(configurations)):
used_config = configurations[index]
generate_3d_image(used_config,str(index))
```
%% Cell type:raw id: tags:
El siguiente código es computar la coherencia de T_malleability respecto a los tiempos internos de maleabilidad (Coherency1)
y por otro lado de T_malleability respecto a iteraciones asíncronas más los pasos síncronos.
%% Cell type:code id: tags:
``` python
test=dfM[(dfM.Asynch_Iters > 0)]
# Seguro con barriers en Malleability
test["Resize_Coherency"] = test["T_Malleability"] >= (test["T_spawn"] + test["T_SR"] + test["T_AR"])
# Seguro al usar Rigid para iteraciones
test["Resize_Coherency2"] = test["T_Malleability"] >= 0
for index in range(len(test)):
time_malleability_aux = test["T_Malleability"].values[index]
time_synch_aux = test["T_SR"].values[index]
time_spawn_aux = test["T_spawn"].values[index]
is_asynch_spawn = (test["Spawn_Strategy"].values[index] % 2 == 0)
total_asynch_iters = int(test["Asynch_Iters"].values[index])
asynch_iters = test["T_iter"].values[index][-total_asynch_iters:]
time_iters_aux = np.sum(asynch_iters[:])
sum_times = time_synch_aux + is_asynch_spawn * time_spawn_aux + time_iters_aux
if time_malleability_aux < sum_times:
real_index = test.index.values[index]
test.at[real_index, "Resize_Coherency2"] = False
test[(test.Resize_Coherency == False)].query('Redistribution_Method > 1')
```
%% Cell type:raw id: tags:
El siguiente código es para utilizar Dask. Una versión que paraleliza una serie de tareas de Pandas.
Tras llamar a compute se realizan todas las tareas que se han pedido.
%% Cell type:code id: tags:
``` python
import dask.dataframe as dd
ddf = dd.from_pandas(dfL[(dfL.Asynch_Iters == False)], npartitions=10)
group = ddf.groupby('NP')['T_iter']
grouped_aggLSynch = group.agg(['mean'])
grouped_aggLSynch = grouped_aggLSynch.rename(columns={'mean':'T_iter'})
grouped_aggLSynch = grouped_aggLSynch.compute()
```
%% Cell type:raw id: tags:
Las siguientes dos celdas de código son para comprobar los Empirical Comulative distribution functions(ecdf).
Si dos ecdf crean una intersección el test de Conover-Iman no es fiable.
#TODO No esta terminado. Falta obtener cual ha sido el metodo ganador antes de llamar a "check_specific_ecdf"
%% Cell type:code id: tags:
``` python
def test_ecdf(data1, data2):
diff_aux = data1[0] - data2[0]
cmp_f = (lambda a, b: a-b < 0) # Se esperan siempre diferencias positivas, si una es negativa se cruzan los ecdf's
if diff_aux < 0:
cmp_f = (lambda a, b: a-b > 0) # Se esperan siempre diferencias negativas, si una es positivas se cruzan los ecdf's
for index_value in range(1,len(data1)):
if cmp_f(data1[index_value], data2[index_value]):
return False
return True
def plot_ecdf(data1, data2, ecdf1, ecdf2, title):
f=plt.figure(figsize=(22, 14))
ax=f.add_subplot(111)
ax.plot(data1, ecdf1, color=colors_m[0], linestyle=linestyle_m[0], \
marker=markers_m[0], markersize=18)
ax.plot(data2, ecdf2, color=colors_m[2], linestyle=linestyle_m[2], \
marker=markers_m[2], markersize=18)
ax.set_xlabel("Values", fontsize=36)
ax.set_ylabel("Bins", fontsize=36)
ax.tick_params(axis='both', which='major', labelsize=36)
ax.tick_params(axis='both', which='minor', labelsize=36)
ax.set_title(title, fontsize=40)
f.tight_layout()
def check_specific_ecdf(np_aux, ns_aux, configuration_index):
df_check = dfM
tipo = 'T_Redistribution'
used_config = configurations_simple
dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)
ecdfLists = []
for index_i in range(len(dataLists)):
dataLists[index_i].sort()
ecdfLists.append( np.arange(1, len(dataLists[index_i])+1)/float(len(dataLists[index_i])) )
index_i = configuration_index
data_i = dataLists[index_i]
ecdf_i = ecdfLists[index_i]
for index_j in range(len(dataLists)):
if index_i != index_j:
data_j = dataLists[index_j]
ecdf_j = ecdfLists[index_j]
res_aux = test_ecdf(data_i, data_j)
if not res_aux:
aux_title = "NP="+str(np_aux)+"/NC="+str(ns_aux)+" -- Comparing C"+str(index_i) + " vs C"+str(index_j)
plot_ecdf(data_i, data_j, ecdf_i, ecdf_j, aux_title)
```
%% Cell type:code id: tags:
``` python
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux and not (np_aux==2 and ns_aux==20):
check_winner_ecdf(np_aux, ns_aux, 9)
```
%% Cell type:raw id: tags:
================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================
%% Cell type:code id: tags:
``` python
a = 5
b = 3
c = False
d = a + b * c
print(d)
```
%% Cell type:code id: tags:
``` python
#Ethernet
b1_aux = [82.747868, 83.191989, 95.520019, 87.435987, 90.843995, 150.356100]
b2_aux = [75.238174, 74.380054, 74.755995, 42.656109, 21.588040, 17.843997]
m1_aux = [74.654167, 74.357901, 74.351350, 43.599915, 21.637509, 15.128712]
m2_aux = [74.353249, 74.359214, 74.356160, 43.874266, 21.511082, 14.969010]
b3_aux = [105.128014, 110.004008, 126.552019, 116.312400, 95.752019, 151.023994]
b4_aux = [83.021885, 77.632630, 75.396010, 43.076039, 24.028075, 19.556047]
m3_aux = [118.275992, 83.027866, 81.008479, 46.432212, 24.247949, 17.725083]
m4_aux = [119.286457, 84.205993, 80.741585, 47.144632, 24.206617, 17.738496]
```
%% Cell type:code id: tags:
``` python
#Infiniband
b1_aux = [64.669525, 35.171971, 38.916010, 47.456630, 56.288048, 119.428020]
b2_aux = [36.538361, 15.536046, 13.396083, 9.652013, 5.772058, 5.615009]
m1_aux = [61.664380, 18.400559, 19.112526, 22.155880, 11.712381, 30.775627]
m2_aux = [33.428639, 13.905561, 14.691367, 7.363081, 6.629037, 12.150872]
b3_aux = [91.368664, 60.648074, 53.663981, 49.152031, 64.752057, 118.243807]
b4_aux = [84.941260, 34.039990, 26.008021, 12.298989, 7.916004, 5.736054]
m3_aux = [105.839726, 26.822071, 23.834452, 12.876862, 9.063136, 7.007535]
m4_aux = [75.412319, 25.566336, 22.129483, 12.491161, 7.903744, 6.534291]
```
%% Cell type:code id: tags:
``` python
plot_data_e = [b1_aux, m1_aux, b2_aux, m2_aux] #Expand
plot_data_s = [b3_aux, m3_aux, b4_aux, m4_aux] #Shrink
labels_aux_e = create_labels_lineplot('e', lambda a, b: a == 160 or b == 160)
labels_aux_s = create_labels_lineplot('s', lambda a, b: a == 160 or b == 160)
#labels_aux = create_labels_lineplot(used_direction)
labelsMethods_aux = ['Baseline - AllS', 'Merge - AllS',
'Baseline - AllA','Merge - AllA']
#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']
f, (axe, axs)=plt.subplots(1,2,figsize=(22, 14))
x = np.arange(len(labels_aux_e))
for index in range(len(plot_data_e)):
array_aux_e = plot_data_e[index]
array_aux_s = plot_data_s[index]
plot_index = index
if index > 0:
plot_index = 2**plot_index
print(array_aux_e)
axe.plot(x, array_aux_e, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \
marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])
axs.plot(x, array_aux_s, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \
marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])
axe.set_xlabel("(NP,NC)", fontsize=36)
axe.set_ylabel("Time(s)", fontsize=36)
axe.set_xticks(x)
axe.set_xticklabels(labels_aux_e, rotation=90)
axe.tick_params(axis='both', which='major', labelsize=36)
axe.tick_params(axis='both', which='minor', labelsize=36)
axs.set_xlabel("(NP,NC)", fontsize=36)
axs.set_ylabel("Time(s)", fontsize=36)
axs.set_xticks(x)
axs.set_xticklabels(labels_aux_s, rotation=90)
axs.tick_params(axis='both', which='major', labelsize=36)
axs.tick_params(axis='both', which='minor', labelsize=36)
plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)
f.tight_layout()
f.savefig("Images/LinePlot_100Gb.png", format="png")
```
%% Cell type:code id: tags:
``` python
%matplotlib inline
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
new_df = pd.read_pickle( "dataBasic.pkl" )
new_df.columns
```
%% Cell type:code id: tags:
``` python
grouped_aggG["T_total"]
```
%% Cell type:code id: tags:
``` python
100000 ** 2
```
%% Cell type:code id: tags:
``` python
```
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
#include "read_ini.h"
#include "../malleability/ProcessDist.h"
#include "ini.h"
void malloc_config_arrays(configuration *user_config, int resizes);
void def_struct_config_file(configuration *config_file, MPI_Datatype *config_type);
void def_struct_config_file_array(configuration *config_file, MPI_Datatype *config_type);
/*
* Funcion utilizada para leer el fichero de configuracion
* y guardarlo en una estructura para utilizarlo en el futuro.
*
* Primero lee la seccion "general" y a continuacion cada una
* de las secciones "resize%d".
*/
static int handler(void* user, const char* section, const char* name,
const char* value) {
configuration* pconfig = (configuration*)user;
char *resize_name = malloc(10 * sizeof(char));
int act_resize = pconfig->actual_resize;
snprintf(resize_name, 10, "resize%d", act_resize);
#define MATCH(s, n) strcmp(section, s) == 0 && strcmp(name, n) == 0
if (MATCH("general", "resizes")) {
pconfig->resizes = atoi(value) + 1;
malloc_config_arrays(pconfig, pconfig->resizes);
} else if (MATCH("general", "matrix_tam")) {
pconfig->matrix_tam = atoi(value);
} else if (MATCH("general", "comm_tam")) {
pconfig->comm_tam = atoi(value);
} else if (MATCH("general", "SDR")) {
pconfig->sdr = atoi(value);
} else if (MATCH("general", "ADR")) {
pconfig->adr = atoi(value);
} else if (MATCH("general", "AIB")) { //TODO Refactor cambiar nombre
pconfig->aib = atoi(value);
} else if (MATCH("general", "CST")) {
pconfig->cst = atoi(value);
} else if (MATCH("general", "CSS")) {
pconfig->css = atoi(value);
} else if (MATCH("general", "time")) {
pconfig->general_time = atof(value);
// Resize
} else if (MATCH(resize_name, "iters")) {
pconfig->iters[act_resize] = atoi(value);
} else if (MATCH(resize_name, "procs")) {
pconfig->procs[act_resize] = atoi(value);
} else if (MATCH(resize_name, "factor")) {
pconfig->factors[act_resize] = atof(value);
} else if (MATCH(resize_name, "physical_dist")) {
char *aux = strdup(value);
if (strcmp(aux, "node") == 0) {
pconfig->phy_dist[act_resize] = COMM_PHY_NODES;
} else {
pconfig->phy_dist[act_resize] = COMM_PHY_CPU;
}
free(aux);
pconfig->actual_resize = pconfig->actual_resize+1; // Ultimo elemento del grupo
} else {
return 0; /* unknown section or name, error */
}
free(resize_name);
return 1;
}
/*
* Crea y devuelve una estructura de configuracion a traves
* de un nombre de fichero dado.
*
* La memoria de la estructura se reserva en la funcion y es conveniente
* liberarla con la funcion "free_config()"
*/
configuration *read_ini_file(char *file_name) {
configuration *config = NULL;
config = malloc(sizeof(configuration) * 1);
if(config == NULL) {
printf("Error when reserving configuration structure\n");
return NULL;
}
config->actual_resize=0;
if(ini_parse(file_name, handler, config) < 0) { // Obtener configuracion
printf("Can't load '%s'\n", file_name);
return NULL;
}
return config;
}
/*
* Reserva de memoria para los vectores de la estructura de configuracion
*
* Si se llama desde fuera de este fichero, la memoria de la estructura
* tiene que reservarse con la siguiente linea:
* "configuration *config = malloc(sizeof(configuration));"
*
* Sin embargo se puede obtener a traves de las funciones
* - read_ini_file
* - recv_config_file
*/
void malloc_config_arrays(configuration *user_config, int resizes) {
if(user_config != NULL) {
user_config->iters = malloc(sizeof(int) * resizes);
user_config->procs = malloc(sizeof(int) * resizes);
user_config->factors = malloc(sizeof(float) * resizes);
user_config->phy_dist = malloc(sizeof(int) * resizes);
}
}
/*
* Libera toda la memoria de una estructura de configuracion
*/
void free_config(configuration *user_config) {
if(user_config != NULL) {
free(user_config->iters);
free(user_config->procs);
free(user_config->factors);
free(user_config->phy_dist);
free(user_config);
}
}
/*
* Imprime por salida estandar toda la informacion que contiene
* la configuracion pasada como argumento
*/
void print_config(configuration *user_config, int grp) {
if(user_config != NULL) {
int i;
printf("Config loaded: resizes=%d, matrix=%d, comm_tam=%d, sdr=%d, adr=%d, aib=%d, css=%d, cst=%d, time=%f || grp=%d\n",
user_config->resizes, user_config->matrix_tam, user_config->comm_tam, user_config->sdr, user_config->adr, user_config->aib, user_config->css, user_config->cst, user_config->general_time, grp);
for(i=0; i<user_config->resizes; i++) {
printf("Resize %d: Iters=%d, Procs=%d, Factors=%f, Phy=%d\n",
i, user_config->iters[i], user_config->procs[i], user_config->factors[i], user_config->phy_dist[i]);
}
}
}
/*
* Imprime por salida estandar la informacion relacionada con un
* solo grupo de procesos en su configuracion.
*/
void print_config_group(configuration *user_config, int grp) {
if(user_config != NULL) {
int parents, sons;
parents = sons = 0;
if(grp > 0) {
parents = user_config->procs[grp-1];
}
if(grp < user_config->resizes - 1) {
sons = user_config->procs[grp+1];
}
printf("Config: matrix=%d, comm_tam=%d, sdr=%d, adr=%d, aib=%d, css=%d, cst=%d, time=%f\n",
user_config->matrix_tam, user_config->comm_tam, user_config->sdr, user_config->adr, user_config->aib, user_config->css, user_config->cst, user_config->general_time);
printf("Config Group: iters=%d, factor=%f, phy=%d, procs=%d, parents=%d, sons=%d\n",
user_config->iters[grp], user_config->factors[grp], user_config->phy_dist[grp], user_config->procs[grp], parents, sons);
}
}
//||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ||
//||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ||
//| FUNCIONES DE INTERCOMUNICACION DE ESTRUCTURA DE CONFIGURACION ||
//||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ||
//||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| |/
/*
* Envia una estructura de configuracion al grupo de procesos al que se
* enlaza este grupo a traves del intercomunicador pasado como argumento.
*
* Esta funcion tiene que ser llamada por todos los procesos del mismo grupo
* e indicar cual es el proceso raiz que se encargara de enviar la
* configuracion al otro grupo.
*/
void send_config_file(configuration *config_file, int root, MPI_Comm intercomm) {
MPI_Datatype config_type, config_type_array;
// Obtener un tipo derivado para enviar todos los
// datos escalares con una sola comunicacion
def_struct_config_file(config_file, &config_type);
// Obtener un tipo derivado para enviar los tres vectores
// de enteros con una sola comunicacion
def_struct_config_file_array(config_file, &config_type_array);
MPI_Bcast(config_file, 1, config_type, root, intercomm);
MPI_Bcast(config_file, 1, config_type_array, root, intercomm);
MPI_Bcast(config_file->factors, config_file->resizes, MPI_FLOAT, root, intercomm);
//Liberar tipos derivados
MPI_Type_free(&config_type);
MPI_Type_free(&config_type_array);
}
/*
* Recibe una estructura de configuracion desde otro grupo de procesos
* y la devuelve. La memoria de la estructura se reserva en esta funcion.
*
* Esta funcion tiene que ser llamada por todos los procesos del mismo grupo
* e indicar cual es el proceso raiz del otro grupo que se encarga de enviar
* la configuracion a este grupo.
*
* La memoria de la configuracion devuelta tiene que ser liberada con
* la funcion "free_config".
*/
configuration *recv_config_file(int root, MPI_Comm intercomm) {
MPI_Datatype config_type, config_type_array;
configuration *config_file = malloc(sizeof(configuration) * 1);
// Obtener un tipo derivado para recibir todos los
// datos escalares con una sola comunicacion
def_struct_config_file(config_file, &config_type);
MPI_Bcast(config_file, 1, config_type, root, intercomm);
// Obtener un tipo derivado para enviar los tres vectores
// de enteros con una sola comunicacion
malloc_config_arrays(config_file, config_file->resizes); // Reserva de memoria de los vectores
def_struct_config_file_array(config_file, &config_type_array);
MPI_Bcast(config_file, 1, config_type_array, root, intercomm);
MPI_Bcast(config_file->factors, config_file->resizes, MPI_FLOAT, root, intercomm);
//Liberar tipos derivados
MPI_Type_free(&config_type);
MPI_Type_free(&config_type_array);
return config_file;
}
/*
* Tipo derivado para enviar 6 elementos especificos
* de la estructura de configuracion con una sola comunicacion.
*/
void def_struct_config_file(configuration *config_file, MPI_Datatype *config_type) {
int i, counts = 11;
int blocklengths[11] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
MPI_Aint displs[counts], dir;
MPI_Datatype types[counts];
// Rellenar vector types
types[0] = types[1] = types[2] = types[3] = types[4] = types[5] = types[6] = types[7] = types[8] = MPI_INT;
types[9] = MPI_FLOAT;
types[10] = MPI_DOUBLE;
// Rellenar vector displs
MPI_Get_address(config_file, &dir);
MPI_Get_address(&(config_file->resizes), &displs[0]);
MPI_Get_address(&(config_file->actual_resize), &displs[1]);
MPI_Get_address(&(config_file->matrix_tam), &displs[2]);
MPI_Get_address(&(config_file->comm_tam), &displs[3]);
MPI_Get_address(&(config_file->sdr), &displs[4]);
MPI_Get_address(&(config_file->adr), &displs[5]);
MPI_Get_address(&(config_file->aib), &displs[6]);
MPI_Get_address(&(config_file->css), &displs[7]);
MPI_Get_address(&(config_file->cst), &displs[8]);
MPI_Get_address(&(config_file->general_time), &displs[9]);
MPI_Get_address(&(config_file->Top), &displs[10]);
for(i=0;i<counts;i++) displs[i] -= dir;
MPI_Type_create_struct(counts, blocklengths, displs, types, config_type);
MPI_Type_commit(config_type);
}
/*
* Tipo derivado para enviar tres vectores de enteros
* de la estructura de configuracion con una sola comunicacion.
*/
void def_struct_config_file_array(configuration *config_file, MPI_Datatype *config_type) {
int i, counts = 3;
int blocklengths[3] = {1, 1, 1};
MPI_Aint displs[counts], dir;
MPI_Datatype aux, types[counts];
// Rellenar vector types
types[0] = types[1] = types[2] = MPI_INT;
// Modificar blocklengths al valor adecuado
blocklengths[0] = blocklengths[1] = blocklengths[2] = config_file->resizes;
//Rellenar vector displs
MPI_Get_address(config_file, &dir);
MPI_Get_address(config_file->iters, &displs[0]);
MPI_Get_address(config_file->procs, &displs[1]);
MPI_Get_address(config_file->phy_dist, &displs[2]);
for(i=0;i<counts;i++) displs[i] -= dir;
// Tipo derivado para enviar un solo elemento de tres vectores
MPI_Type_create_struct(counts, blocklengths, displs, types, &aux);
// Tipo derivado para enviar N elementos de tres vectores(3N en total)
MPI_Type_create_resized(aux, 0, 1*sizeof(int), config_type);
MPI_Type_commit(config_type);
}
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#define RESULTS_INIT_DATA_QTY 100
typedef struct {
// Iters data
double *iters_time;
int *iters_type, iter_index, iters_size;
// Spawn, Thread, Sync, Async and Exec time
double spawn_start, *spawn_time, *spawn_real_time;
double sync_start, sync_end, *sync_time;
double async_start, async_end, *async_time;
double exec_start, exec_time;
//Overcharge time is time spent in malleability that is from IO modules
} results_data;
void send_results(results_data *results, int root, int resizes, MPI_Comm intercomm);
void recv_results(results_data *results, int root, int resizes, MPI_Comm intercomm);
void set_results_post_reconfig(results_data *results, int grp, int sdr, int adr);
void reset_results_index(results_data *results);
void compute_results_iter(results_data *results, int myId, int root, MPI_Comm comm);
void print_iter_results(results_data results, int last_normal_iter_index);
void print_global_results(results_data results, int resizes);
void init_results_data(results_data *results, int resizes, int iters_size);
void realloc_results_iters(results_data *results, int needed);
void free_results_data(results_data *results);
#ifndef MAM_H
#define MAM_H
#include "MAM_Constants.h"
#include "MAM_Manager.h"
#include "MAM_Configuration.h"
#include "MAM_Times_retrieve.h"
#endif
#include "MAM_Configuration.h"
#include "MAM_Init_Configuration.h"
#include "MAM_DataStructures.h"
#include <limits.h>
typedef struct {
unsigned int *value, default_value;
int config_max_length;
union {
int (*set_config_simple)(unsigned int, unsigned int *);
int (*set_config_complex)(unsigned int);
};
char *env_name;
} mam_config_setting_t;
int MAM_I_set_method(unsigned int new_method, unsigned int *method);
int MAM_I_set_spawn_strat(unsigned int strategy, unsigned int *strategies);
int MAM_I_set_red_strat(unsigned int strategy, unsigned int *strategies);
int MAM_I_set_target_number(unsigned int new_numC);
int MAM_I_configuration_get_defaults();
int MAM_I_contains_strat(unsigned int comm_strategies, unsigned int strategy);
int MAM_I_add_strat(unsigned int *comm_strategies, unsigned int strategy);
int MAM_I_remove_strat(unsigned int *comm_strategies, unsigned int strategy);
mam_config_setting_t configSettings[] = {
{NULL, MAM_SPAWN_MERGE, MAM_METHODS_SPAWN_LEN, {.set_config_simple = MAM_I_set_method }, MAM_SPAWN_METHOD_ENV},
{NULL, MAM_STRAT_SPAWN_CLEAR, MAM_STRATS_SPAWN_LEN, {.set_config_simple = MAM_I_set_spawn_strat }, MAM_SPAWN_STRATS_ENV},
{NULL, MAM_PHY_DIST_COMPACT, MAM_METHODS_PHYSICAL_DISTRIBUTION_LEN, {.set_config_simple = MAM_I_set_method }, MAM_PHYSICAL_DISTRIBUTION_METHOD_ENV},
{NULL, MAM_RED_BASELINE, MAM_METHODS_RED_LEN, {.set_config_simple = MAM_I_set_method }, MAM_RED_METHOD_ENV},
{NULL, MAM_STRAT_RED_CLEAR, MAM_STRATS_RED_LEN, {.set_config_simple = MAM_I_set_red_strat }, MAM_RED_STRATS_ENV},
{NULL, 1, INT_MAX, {.set_config_complex = MAM_I_set_target_number }, MAM_NUM_TARGETS_ENV}
};
unsigned int masks_spawn[] = {MAM_STRAT_CLEAR_VALUE, MAM_MASK_PTHREAD, MAM_MASK_SPAWN_SINGLE, MAM_MASK_SPAWN_INTERCOMM, MAM_MASK_SPAWN_MULTIPLE, MAM_MASK_SPAWN_PARALLEL};
unsigned int masks_red[] = {MAM_STRAT_CLEAR_VALUE, MAM_MASK_PTHREAD, MAM_MASK_RED_WAIT_SOURCES, MAM_MASK_RED_WAIT_TARGETS};
/**
* @brief Set configuration parameters for MAM.
*
* This function allows setting various configuration parameters for MAM
* such as spawn method, spawn strategies, spawn physical distribution,
* redistribution method, and red strategies.
*
* @param spawn_method The spawn method reconfiguration.
* @param spawn_strategies The spawn strategies reconfiguration.
* @param spawn_dist The spawn physical distribution method reconfiguration.
* @param red_method The redistribution method reconfiguration.
* @param red_strategies The redesitribution strategy for reconfiguration.
*/
void MAM_Set_configuration(int spawn_method, int spawn_strategies, int spawn_dist, int red_method, int red_strategies) {
int i, aux;
int aux_array[] = {spawn_method, spawn_strategies, spawn_dist, red_method, red_strategies};
if(state > MAM_I_NOT_STARTED) return;
mam_config_setting_t *config = NULL;
for (i = 0; i < MAM_KEY_COUNT-1; i++) { //FIXME Numero magico para no cambiar num_targets
aux = aux_array[i];
config = &configSettings[i];
if (0 <= aux && aux < config->config_max_length) {
if(i == MAM_NUM_TARGETS) {
config->set_config_complex(aux);
} else {
config->set_config_simple(aux, config->value);
}
}
}
}
/*
* @brief Set the configuration value for a specific key in MAM.
*
* Modifies the configuration value associated with the given key
* to the specified "required" value. The final value set is returned in the
* "provided" parameter.
*
* @param key The key for which the configuration value is to be modified.
* @param required The required value to set for the specified key.
* @param provided Pointer to an integer where the final value set will be stored.
* This parameter is updated with the actual value after modification.
* For strategy keys the value is "MAM_STRATS_ADDED" if "required" has
* been added, or "MAM_STRATS_MODIFIED" if multiple strategies of the
* key have been modified.
*/
void MAM_Set_key_configuration(int key, int required, int *provided) {
int i, aux;
if(provided == NULL) provided = &aux;
*provided = MAM_DENIED;
if(required < 0 || state > MAM_I_NOT_STARTED) return;
mam_config_setting_t *config = NULL;
for (i = 0; i < MAM_KEY_COUNT; i++) { //FIXME A for is not needed -- Check if key < MAM_KEY_COUNT and then just use key as index
if (key == i) {
config = &configSettings[i];
break;
}
}
if (config != NULL) {
if (required < config->config_max_length) {
if(i == MAM_NUM_TARGETS) {
*provided = config->set_config_complex(required);
} else {
*provided = config->set_config_simple(required, config->value);
}
} else {*provided = *(config->value); }
} else { printf("MAM: Key %d does not exist\n", key); }
}
/*
* Retorna si una estrategia aparece o no
*/
int MAM_Contains_strat(int key, unsigned int strategy, int *result) {
int strategies, aux = MAM_OK;
unsigned int len = 0, mask;
switch(key) {
case MAM_SPAWN_STRATEGIES:
strategies = mall_conf->spawn_strategies;
mask = masks_spawn[strategy];
len = MAM_STRATS_SPAWN_LEN;
break;
case MAM_RED_STRATEGIES:
strategies = mall_conf->red_strategies;
mask = masks_red[strategy];
len = MAM_STRATS_RED_LEN;
break;
default:
aux = MAM_DENIED;
break;
}
if(aux == MAM_OK && strategy < len) {
aux = MAM_I_contains_strat(strategies, mask);
} else {
aux = 0;
}
if(result != NULL) *result = aux;
return aux;
}
/*
* //TODO
* Tiene que ser llamado despues de setear la config
*/
int MAM_Set_target_number(unsigned int numC){
return MAM_I_set_target_number(numC);
}
/*
* //TODO
* Tiene que ser llamado fuera de reconfig
*/
void MAM_Use_valgrind(int flag) {
if(state > MAM_I_NOT_STARTED) return;
mall_conf->external_usage = flag ? MAM_USE_VALGRIND: 0;
#if MAM_DEBUG
if(mall->myId == mall->root && flag) DEBUG_FUNC("Settled Valgrind Wrapper", mall->myId, mall->numP); fflush(stdout);
#endif
}
/*
* //TODO
* Tiene que ser llamado fuera de reconfig
*/
void MAM_Use_extrae(int flag) {
if(state > MAM_I_NOT_STARTED) return;
mall_conf->external_usage = flag ? MAM_USE_EXTRAE: 0;
#if MAM_DEBUG
if(mall->myId == mall->root && flag) DEBUG_FUNC("Settled Extrae Wrapper", mall->myId, mall->numP); fflush(stdout);
#endif
}
//======================================================||
//===============MAM_INIT FUNCTIONS=====================||
//======================================================||
//======================================================||
void MAM_Init_configuration() {
if(mall == NULL || mall_conf == NULL) {
printf("MAM FATAL ERROR: Setting initial config without previous mallocs\n");
fflush(stdout);
MPI_Abort(MPI_COMM_WORLD, -50);
}
mall_conf->spawn_method = MAM_STRAT_CLEAR_VALUE;
mall_conf->spawn_strategies = MAM_STRAT_CLEAR_VALUE;
mall_conf->red_method = MAM_STRAT_CLEAR_VALUE;
mall_conf->red_strategies = MAM_STRAT_CLEAR_VALUE;
mall_conf->external_usage = 0;
configSettings[MAM_SPAWN_METHOD].value = &mall_conf->spawn_method;
configSettings[MAM_SPAWN_STRATEGIES].value = &mall_conf->spawn_strategies;
configSettings[MAM_PHYSICAL_DISTRIBUTION].value = &mall_conf->spawn_dist;
configSettings[MAM_RED_METHOD].value = &mall_conf->red_method;
configSettings[MAM_RED_STRATEGIES].value = &mall_conf->red_strategies;
}
void MAM_Set_initial_configuration() {
int not_filled = 1;
not_filled = MAM_I_configuration_get_defaults();
if(not_filled) {
if(mall->myId == mall->root) printf("MAM WARNING: Starting configuration not set\n");
fflush(stdout);
MPI_Abort(mall->comm, -50);
}
#if MAM_DEBUG >= 2
if(mall->myId == mall->root) {
DEBUG_FUNC("Initial configuration settled", mall->myId, mall->numP);
fflush(stdout);
}
#endif
}
void MAM_Check_configuration() {
int global_internodes;
if(mall->numC == mall->numP) { // Migrate
MAM_Set_key_configuration(MAM_SPAWN_METHOD, MAM_SPAWN_BASELINE, NULL);
}
MPI_Allreduce(&mall->internode_group, &global_internodes, 1, MPI_INT, MPI_MAX, mall->comm);
if((MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_MULTIPLE, NULL)
|| MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_PARALLEL, NULL) )
&& global_internodes) { // Remove internode MPI_COMM_WORLDs
MAM_Set_key_configuration(MAM_SPAWN_METHOD, MAM_SPAWN_BASELINE, NULL);
}
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) {
if(MAM_I_contains_strat(mall_conf->spawn_strategies, MAM_MASK_SPAWN_INTERCOMM)) {
MAM_I_remove_strat(&mall_conf->spawn_strategies, MAM_MASK_SPAWN_INTERCOMM);
}
// FIXME This should not be required to be removed for that case...
if(mall->numP > mall->numC && MAM_I_contains_strat(mall_conf->spawn_strategies, MAM_MASK_SPAWN_SINGLE)) {
MAM_I_remove_strat(&mall_conf->spawn_strategies, MAM_MASK_SPAWN_SINGLE);
}
}
if(mall_conf->red_method == MAM_RED_RMA_LOCK || mall_conf->red_method == MAM_RED_RMA_LOCKALL) {
if(MAM_I_contains_strat(mall_conf->spawn_strategies, MAM_MASK_SPAWN_INTERCOMM)) {
MAM_I_remove_strat(&mall_conf->spawn_strategies, MAM_MASK_SPAWN_INTERCOMM);
}
if(!MAM_I_contains_strat(mall_conf->red_strategies, MAM_MASK_RED_WAIT_TARGETS) &&
!MAM_I_contains_strat(mall_conf->red_strategies, MAM_MASK_PTHREAD)) {
MAM_I_set_red_strat(MAM_STRAT_RED_WAIT_TARGETS, &mall_conf->red_strategies);
}
}
#if MAM_DEBUG >= 2
if(mall->myId == mall->root) {
DEBUG_FUNC("MaM configuration", mall->myId, mall->numP);
printf("Spawn M=%d S=%d D=%d Redist M=%d S=%d\n",
mall_conf->spawn_method, mall_conf->spawn_strategies, mall_conf->spawn_dist, mall_conf->red_method, mall_conf->red_strategies);
fflush(stdout);
}
#endif
}
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//======================================================||
//======================================================||
int MAM_I_configuration_get_defaults() {
size_t i;
int set_value;
char *tmp = NULL;
mam_config_setting_t *config = NULL;
for (i = 0; i < MAM_KEY_COUNT; i++) {
config = &configSettings[i];
tmp = getenv(config->env_name);
if(tmp != NULL) {
set_value = atoi(tmp);
} else {
set_value = config->default_value;
}
if (0 <= set_value && set_value < config->config_max_length) {
if(i == MAM_NUM_TARGETS) {
config->set_config_complex(set_value);
} else {
config->set_config_simple(set_value, config->value);
}
}
tmp = NULL;
}
return 0;
}
int MAM_I_set_method(unsigned int new_method, unsigned int *method) {
*method = new_method;
return *method;
}
//TODO Se podría hacer un par de arrays o dict para obtener la mascara sin un switch
int MAM_I_set_spawn_strat(unsigned int strategy, unsigned int *strategies) {
int result = 0;
int strat_removed = 0;
switch(strategy) {
case MAM_STRAT_SPAWN_CLEAR:
*strategies = MAM_STRAT_CLEAR_VALUE;
result = MAM_STRATS_MODIFIED;
break;
case MAM_STRAT_SPAWN_PTHREAD:
result = MAM_I_add_strat(strategies, MAM_MASK_PTHREAD);
break;
case MAM_STRAT_SPAWN_SINGLE:
result = MAM_I_add_strat(strategies, MAM_MASK_SPAWN_SINGLE);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_SPAWN_PARALLEL);
}
break;
case MAM_STRAT_SPAWN_INTERCOMM:
result = MAM_I_add_strat(strategies, MAM_MASK_SPAWN_INTERCOMM);
break;
case MAM_STRAT_SPAWN_MULTIPLE:
result = MAM_I_add_strat(strategies, MAM_MASK_SPAWN_MULTIPLE);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_SPAWN_PARALLEL);
}
break;
case MAM_STRAT_SPAWN_PARALLEL:
result = MAM_I_add_strat(strategies, MAM_MASK_SPAWN_PARALLEL);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_SPAWN_MULTIPLE);
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_SPAWN_SINGLE);
}
break;
default:
//Unkown strategy
result = MAM_DENIED;
break;
}
if(strat_removed) {
result = MAM_STRATS_MODIFIED;
}
return result;
}
int MAM_I_set_red_strat(unsigned int strategy, unsigned int *strategies) {
int result = 0;
int strat_removed = 0;
switch(strategy) {
case MAM_STRAT_RED_CLEAR:
*strategies = MAM_STRAT_CLEAR_VALUE;
result = MAM_STRATS_MODIFIED;
break;
case MAM_STRAT_RED_PTHREAD: //TODO - IMPROVEMENT - This could be done with a single operation instead of 3.
result = MAM_I_add_strat(strategies, MAM_MASK_PTHREAD);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_RED_WAIT_SOURCES);
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_RED_WAIT_TARGETS);
}
break;
case MAM_STRAT_RED_WAIT_SOURCES:
result = MAM_I_add_strat(strategies, MAM_MASK_RED_WAIT_SOURCES);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_RED_WAIT_TARGETS);
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_PTHREAD);
}
break;
case MAM_STRAT_RED_WAIT_TARGETS:
result = MAM_I_add_strat(strategies, MAM_MASK_RED_WAIT_TARGETS);
if(result == MAM_STRATS_ADDED) {
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_RED_WAIT_SOURCES);
strat_removed += MAM_I_remove_strat(strategies, MAM_MASK_PTHREAD);
}
break;
default:
//Unkown strategy
result = MAM_DENIED;
break;
}
if(strat_removed) {
result = MAM_STRATS_MODIFIED;
}
return result;
}
int MAM_I_set_target_number(unsigned int new_numC) {
if(state > MAM_I_NOT_STARTED || new_numC == 0) return MAM_DENIED;
mall->numC = (int) new_numC;
return new_numC;
}
/*
* Returns 1 if strategy is applied, 0 otherwise
*/
int MAM_I_contains_strat(unsigned int comm_strategies, unsigned int strategy) {
return comm_strategies & strategy;
}
int MAM_I_add_strat(unsigned int *comm_strategies, unsigned int strategy) {
if(MAM_I_contains_strat(*comm_strategies, strategy)) return MAM_OK;
*comm_strategies |= strategy;
return MAM_STRATS_ADDED;
}
int MAM_I_remove_strat(unsigned int *comm_strategies, unsigned int strategy) {
if(!MAM_I_contains_strat(*comm_strategies, strategy)) return MAM_OK;
*comm_strategies &= ~strategy;
return MAM_STRATS_MODIFIED;
}
#ifndef MAM_CONFIGURATION_H
#define MAM_CONFIGURATION_H
#include <mpi.h>
#include "MAM_Constants.h"
#define MAM_STRAT_CLEAR_VALUE 0
#define MAM_STRATS_ADDED 1
#define MAM_STRATS_MODIFIED 2
#define MAM_MASK_PTHREAD 0x01
#define MAM_MASK_SPAWN_SINGLE 0x02
#define MAM_MASK_SPAWN_INTERCOMM 0x04
#define MAM_MASK_SPAWN_MULTIPLE 0x08
#define MAM_MASK_SPAWN_PARALLEL 0x10
#define MAM_MASK_RED_WAIT_SOURCES 0x02
#define MAM_MASK_RED_WAIT_TARGETS 0x04
int MAM_Contains_strat(int key, unsigned int strategy, int *result);
void MAM_Set_configuration(int spawn_method, int spawn_strategies, int spawn_dist, int red_method, int red_strategies);
void MAM_Set_key_configuration(int key, int required, int *provided);
int MAM_Set_target_number(unsigned int numC);
void MAM_Use_valgrind(int flag);
void MAM_Use_extrae(int flag);
#endif
#ifndef MAM_CONSTANTS_H
#define MAM_CONSTANTS_H
//States
#define MAM_DENIED -1
#define MAM_OK 0
enum mam_states{MAM_UNRESERVED, MAM_NOT_STARTED, MAM_PENDING, MAM_USER_PENDING, MAM_COMPLETED};
enum mam_proc_states{MAM_PROC_CONTINUE, MAM_PROC_NEW_RANK, MAM_PROC_ZOMBIE};
enum mam_spawn_methods{MAM_SPAWN_BASELINE, MAM_SPAWN_MERGE, MAM_METHODS_SPAWN_LEN};
enum mam_spawn_strategies{MAM_STRAT_SPAWN_CLEAR, MAM_STRAT_SPAWN_PTHREAD, MAM_STRAT_SPAWN_SINGLE, MAM_STRAT_SPAWN_INTERCOMM, MAM_STRAT_SPAWN_MULTIPLE, MAM_STRAT_SPAWN_PARALLEL, MAM_STRATS_SPAWN_LEN};
enum mam_phy_dist_methods{MAM_PHY_DIST_SPREAD = 1, MAM_PHY_DIST_COMPACT, MAM_METHODS_PHYSICAL_DISTRIBUTION_LEN};
enum mam_phy_info_methods{MAM_PHY_TYPE_STRING = 1, MAM_PHY_TYPE_HOSTFILE};
enum mam_redistribution_methods{MAM_RED_BASELINE, MAM_RED_POINT, MAM_RED_RMA_LOCK, MAM_RED_RMA_LOCKALL, MAM_METHODS_RED_LEN};
enum mam_red_strategies{MAM_STRAT_RED_CLEAR, MAM_STRAT_RED_PTHREAD, MAM_STRAT_RED_WAIT_SOURCES, MAM_STRAT_RED_WAIT_TARGETS, MAM_STRATS_RED_LEN};
/* KEYS & VALUES for config*/
enum mam_key_values{MAM_SPAWN_METHOD=0, MAM_SPAWN_STRATEGIES, MAM_PHYSICAL_DISTRIBUTION, MAM_RED_METHOD, MAM_RED_STRATEGIES, MAM_NUM_TARGETS, MAM_KEY_COUNT};
#define MAM_SPAWN_METHOD_ENV "MAM_SPAWN_METHOD"
#define MAM_SPAWN_STRATS_ENV "MAM_SPAWN_STRATS"
#define MAM_PHYSICAL_DISTRIBUTION_METHOD_ENV "MAM_PHYSICAL_DISTRIBUTION_METHOD"
#define MAM_RED_METHOD_ENV "MAM_RED_METHOD"
#define MAM_RED_STRATS_ENV "MAM_RED_STRATS"
#define MAM_NUM_TARGETS_ENV "MAM_NUM_TARGETS"
#define MAM_CHECK_COMPLETION 0
#define MAM_WAIT_COMPLETION 1
#define MAM_SOURCES 0
#define MAM_TARGETS 1
#define MAM_DATA_DISTRIBUTED 0
#define MAM_DATA_REPLICATED 1
#define MAM_DATA_VARIABLE 0
#define MAM_DATA_CONSTANT 1
// Tags for messages in spawn strategies
#define MAM_MPITAG_STRAT_SINGLE 130
#define MAM_MPITAG_STRAT_MULTIPLE 131
#endif
#include "MAM_DataStructures.h"
malleability_config_t *mall_conf = NULL;
malleability_t *mall = NULL;
int state = MAM_I_UNRESERVED;
/*
* Crea un tipo derivado para mandar las dos estructuras principales
* de MaM.
*/
void MAM_Def_main_datatype() {
int i, counts = 12;
int blocklengths[counts];
MPI_Aint displs[counts];
MPI_Datatype types[counts];
for(i=0; i<5; i++) {
blocklengths[i] = 1;
types[i] = MPI_UNSIGNED;
}
for(i=5; i<counts; i++) {
blocklengths[i] = 1;
types[i] = MPI_INT;
}
// Obtain base direction
MPI_Get_address(&(mall_conf->spawn_method), &displs[0]);
MPI_Get_address(&(mall_conf->spawn_strategies), &displs[1]);
MPI_Get_address(&(mall_conf->spawn_dist), &displs[2]);
MPI_Get_address(&(mall_conf->red_method), &displs[3]);
MPI_Get_address(&(mall_conf->red_strategies), &displs[4]);
MPI_Get_address(&(mall->root_parents), &displs[5]);
MPI_Get_address(&(mall->num_parents), &displs[6]); //TODO Add only when Single strat active?
MPI_Get_address(&(mall->numC), &displs[7]); //TODO Add only when MultipleSpawn strat active?
MPI_Get_address(&(mall->gid), &displs[8]); //TODO Add only when ParallelSpawn strat active?
MPI_Get_address(&(mall->num_cpus), &displs[9]);
MPI_Get_address(&(mall->num_nodes), &displs[10]);
MPI_Get_address(&(mall->nodelist_len), &displs[11]);
MPI_Type_create_struct(counts, blocklengths, displs, types, &mall->struct_type);
MPI_Type_commit(&mall->struct_type);
}
void MAM_Free_main_datatype() {
if(mall->struct_type != MPI_DATATYPE_NULL) {
MPI_Type_free(&mall->struct_type);
}
}
/*
* Comunica datos necesarios de las estructuras
* principales de MAM de sources a targets.
*/
void MAM_Comm_main_structures(MPI_Comm comm, int rootBcast) {
MPI_Bcast(MPI_BOTTOM, 1, mall->struct_type, rootBcast, comm);
if(mall->nodelist == NULL) {
mall->nodelist = malloc((mall->nodelist_len) * sizeof(char));
mall->nodelist[mall->nodelist_len-1] = '\0';
}
MPI_Bcast(mall->nodelist, mall->nodelist_len, MPI_CHAR, rootBcast, comm);
}
/*
* Muestra por pantalla el estado actual de todos los comunicadores
*/
void MAM_print_comms_state() {
int tester;
char *comm_name = malloc(MPI_MAX_OBJECT_NAME * sizeof(char));
MPI_Comm_get_name(mall->comm, comm_name, &tester);
printf("P%d Comm=%d Name=%s\n", mall->myId, mall->comm, comm_name);
MPI_Comm_get_name(*(mall->user_comm), comm_name, &tester);
printf("P%d Comm=%d Name=%s\n", mall->myId, *(mall->user_comm), comm_name);
if(mall->intercomm != MPI_COMM_NULL) {
MPI_Comm_get_name(mall->intercomm, comm_name, &tester);
printf("P%d Comm=%d Name=%s\n", mall->myId, mall->intercomm, comm_name);
}
free(comm_name);
}
/*
* Función para modificar los comunicadores principales de MaM
*/
void MAM_comms_update(MPI_Comm comm) {
if(mall->thread_comm != MPI_COMM_WORLD) MPI_Comm_disconnect(&(mall->thread_comm));
if(mall->comm != MPI_COMM_WORLD) MPI_Comm_disconnect(&(mall->comm));
MPI_Comm_dup(comm, &(mall->thread_comm));
MPI_Comm_dup(comm, &(mall->comm));
MPI_Comm_set_name(mall->thread_comm, "MAM_THREAD");
MPI_Comm_set_name(mall->comm, "MAM_MAIN");
}
#ifndef MAM_DATA_STRUCTURES_H
#define MAM_DATA_STRUCTURES_H
/*
* Shows available data structures for inner ussage.
*/
#include <stdlib.h>
#include <stdio.h>
#include <mpi.h>
#include <pthread.h>
#include "MAM_Constants.h"
#define DEBUG_FUNC(debug_string, rank, numP) printf("MaM [P%d/%d]: %s -- %s:%s:%d\n", rank, numP, debug_string, __FILE__, __func__, __LINE__)
/* --- MAM INNER CONSTANTS --- */
#define MAM_ROOT 0
enum mam_inner_states{MAM_I_UNRESERVED, MAM_I_NOT_STARTED, MAM_I_RMS_COMPLETED, MAM_I_SPAWN_PENDING, MAM_I_SPAWN_SINGLE_PENDING,
MAM_I_SPAWN_SINGLE_COMPLETED, MAM_I_SPAWN_ADAPT_POSTPONE, MAM_I_SPAWN_COMPLETED, MAM_I_DIST_PENDING, MAM_I_DIST_COMPLETED,
MAM_I_SPAWN_ADAPT_PENDING, MAM_I_USER_START, MAM_I_USER_PENDING, MAM_I_USER_COMPLETED, MAM_I_SPAWN_ADAPTED, MAM_I_COMPLETED};
#define MAM_USE_VALGRIND 1
#define MAM_USE_EXTRAE 2
#define MAM_VALGRIND_SCRIPT "./worker_valgrind.sh"
#define MAM_EXTRAE_SCRIPT "./worker_extrae.sh"
/* --- TIME CAPTURE STRUCTURE --- */
typedef struct {
// Spawn, Sync and Async time
double spawn_start, spawn_time;
double sync_start, sync_end;
double async_start, async_end;
double user_start, user_end;
double malleability_start, malleability_end;
MPI_Datatype times_type;
} malleability_times_t;
/* --- GLOBAL STRUCTURES --- */
typedef struct {
unsigned int spawn_method;
unsigned int spawn_dist;
unsigned int spawn_strategies;
unsigned int red_method;
unsigned int red_strategies;
int external_usage; // Whether a different application should be called by Spawn and which
malleability_times_t *times;
} malleability_config_t;
typedef struct {
int myId, numP, numC, zombie;
int root, root_collectives;
int num_parents, root_parents, gid;
pthread_t async_thread;
MPI_Comm comm, thread_comm, original_comm;
MPI_Comm intercomm, tmp_comm;
MPI_Comm *user_comm;
MPI_Datatype struct_type;
// Specific vars for Wait_targets strat
int wait_targets_posted;
MPI_Request wait_targets;
char *name_exec, *nodelist;
int num_cpus, num_nodes, nodelist_len;
int internode_group;
} malleability_t;
/* --- VARIABLES --- */
extern malleability_config_t *mall_conf;
extern malleability_t *mall;
extern int state;
/* --- FUNCTIONS --- */
void MAM_Def_main_datatype();
void MAM_Free_main_datatype();
void MAM_Comm_main_structures(MPI_Comm comm, int rootBcast);
void MAM_print_comms_state();
void MAM_comms_update(MPI_Comm comm);
#endif
#ifndef MAM_INIT_CONFIGURATION_H
#define MAM_INIT_CONFIGURATION_H
#include <mpi.h>
#include "MAM_Constants.h"
void MAM_Init_configuration();
void MAM_Set_initial_configuration();
void MAM_Check_configuration();
#endif
#include <pthread.h>
#include <string.h>
#include "MAM.h"
#include "MAM_Constants.h"
#include "MAM_DataStructures.h"
#include "MAM_Types.h"
#include "MAM_Zombies.h"
#include "MAM_Times.h"
#include "MAM_RMS.h"
#include "MAM_Init_Configuration.h"
#include "spawn_methods/GenericSpawn.h"
#include "distribution_methods/Distributed_CommDist.h"
#define MAM_USE_SYNCHRONOUS 0
#define MAM_USE_ASYNCHRONOUS 1
void MAM_Commit(int *mam_state);
void send_data(int numP_children, malleability_data_t *data_struct, int is_asynchronous);
void recv_data(int numP_parents, malleability_data_t *data_struct, int is_asynchronous);
int MAM_St_rms(int *mam_state);
int MAM_St_spawn_start();
int MAM_St_spawn_pending(int wait_completed);
int MAM_St_red_start();
int MAM_St_red_pending(int wait_completed);
int MAM_St_user_start(int *mam_state);
int MAM_St_user_pending(int *mam_state, int wait_completed, void (*user_function)(void *), void *user_args);
int MAM_St_user_completed();
int MAM_St_spawn_adapt_pending(int wait_completed);
int MAM_St_spawn_adapted(int *mam_state);
int MAM_St_red_completed(int *mam_state);
int MAM_St_completed(int *mam_state);
void Children_init(void (*user_function)(void *), void *user_args);
int spawn_step();
int start_redistribution();
int check_redistribution(int wait_completed);
int end_redistribution();
int shrink_redistribution();
int thread_creation();
int thread_check(int wait_completed);
void* thread_async_work();
int MAM_I_convert_key(char *key);
void MAM_I_create_user_struct(int is_children_group);
malleability_data_t *rep_s_data;
malleability_data_t *dist_s_data;
malleability_data_t *rep_a_data;
malleability_data_t *dist_a_data;
mam_user_reconf_t *user_reconf;
/*
* Inicializa la reserva de memoria para el modulo de maleabilidad
* creando todas las estructuras necesarias y copias de comunicadores
* para no interferir en la aplicación.
*
* Si es llamada por un grupo de procesos creados de forma dinámica,
* inicializan la comunicacion con sus padres. En este caso, al terminar
* la comunicacion los procesos hijo estan preparados para ejecutar la
* aplicacion.
*/
int MAM_Init(int root, MPI_Comm *comm, char *name_exec, void (*user_function)(void *), void *user_args) {
MPI_Comm dup_comm, thread_comm, original_comm;
mall_conf = (malleability_config_t *) malloc(sizeof(malleability_config_t));
mall = (malleability_t *) malloc(sizeof(malleability_t));
user_reconf = (mam_user_reconf_t *) malloc(sizeof(mam_user_reconf_t));
MPI_Comm_rank(*comm, &(mall->myId));
MPI_Comm_size(*comm, &(mall->numP));
#if MAM_DEBUG
DEBUG_FUNC("Initializing MaM", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(*comm);
#endif
rep_s_data = (malleability_data_t *) malloc(sizeof(malleability_data_t));
dist_s_data = (malleability_data_t *) malloc(sizeof(malleability_data_t));
rep_a_data = (malleability_data_t *) malloc(sizeof(malleability_data_t));
dist_a_data = (malleability_data_t *) malloc(sizeof(malleability_data_t));
MPI_Comm_dup(*comm, &dup_comm);
MPI_Comm_dup(*comm, &thread_comm);
MPI_Comm_dup(*comm, &original_comm);
MPI_Comm_set_name(dup_comm, "MAM_MAIN");
MPI_Comm_set_name(thread_comm, "MAM_THREAD");
MPI_Comm_set_name(original_comm, "MAM_ORIGINAL");
mall->root = root;
mall->root_parents = root;
mall->zombie = 0;
mall->comm = dup_comm;
mall->thread_comm = thread_comm;
mall->original_comm = original_comm;
mall->user_comm = comm;
mall->tmp_comm = MPI_COMM_NULL;
mall->name_exec = name_exec;
mall->nodelist = NULL;
mall->nodelist_len = 0;
rep_s_data->entries = 0;
rep_a_data->entries = 0;
dist_s_data->entries = 0;
dist_a_data->entries = 0;
state = MAM_I_NOT_STARTED;
MAM_Init_configuration();
MAM_Zombies_service_init();
init_malleability_times();
MAM_Def_main_datatype();
// Si son el primer grupo de procesos, obtienen los datos de los padres
MPI_Comm_get_parent(&(mall->intercomm));
if(mall->intercomm != MPI_COMM_NULL) {
Children_init(user_function, user_args);
return MAM_TARGETS;
}
//TODO Check potential improvement - If check_hosts does not use slurm, internode_group could be obtained there
MAM_check_hosts();
mall->internode_group = MAM_Is_internode_group();
MAM_Set_initial_configuration();
#if MAM_USE_BARRIERS && MAM_DEBUG
if(mall->myId == mall->root)
printf("MaM: Using barriers to record times.\n");
#endif
#if MAM_DEBUG
DEBUG_FUNC("MaM has been initialized correctly as parents", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(*comm);
#endif
return MAM_SOURCES;
}
/*
* Elimina toda la memoria reservado por el modulo
* de maleabilidad y asegura que los zombies
* despierten si los hubiese.
*/
int MAM_Finalize() {
int request_abort;
free_malleability_data_struct(rep_s_data);
free_malleability_data_struct(rep_a_data);
free_malleability_data_struct(dist_s_data);
free_malleability_data_struct(dist_a_data);
free(rep_s_data);
free(rep_a_data);
free(dist_s_data);
free(dist_a_data);
if(mall->nodelist != NULL) free(mall->nodelist);
MAM_Free_main_datatype();
request_abort = MAM_Zombies_service_free();
free_malleability_times();
if(mall->comm != MPI_COMM_WORLD && mall->comm != MPI_COMM_NULL) MPI_Comm_disconnect(&(mall->comm));
if(mall->thread_comm != MPI_COMM_WORLD && mall->thread_comm != MPI_COMM_NULL) MPI_Comm_disconnect(&(mall->thread_comm));
if(mall->intercomm != MPI_COMM_WORLD && mall->intercomm != MPI_COMM_NULL) { MPI_Comm_disconnect(&(mall->intercomm)); } //FIXME Error en OpenMPI + Merge
if(mall->original_comm != MPI_COMM_WORLD && mall->original_comm != MPI_COMM_NULL) MPI_Comm_free(&(mall->original_comm));
free(mall);
free(mall_conf);
free(user_reconf);
state = MAM_I_UNRESERVED;
return request_abort;
}
/*
* TODO Reescribir
* Comprueba el estado de la maleabilidad. Intenta avanzar en la misma
* si es posible. Funciona como una máquina de estados.
* Retorna el estado de la maleabilidad concreto y modifica el argumento
* "mam_state" a uno generico.
*
* El argumento "wait_completed" se utiliza para esperar a la finalización de
* las tareas llevadas a cabo por parte de MAM.
*
*/
int MAM_Checkpoint(int *mam_state, int wait_completed, void (*user_function)(void *), void *user_args) {
int call_checkpoint = 0;
//TODO This could be changed to an array with the functions to call in each case
switch(state) {
case MAM_I_UNRESERVED:
*mam_state = MAM_UNRESERVED;
break;
case MAM_I_NOT_STARTED:
call_checkpoint = MAM_St_rms(mam_state);
break;
case MAM_I_RMS_COMPLETED:
call_checkpoint = MAM_St_spawn_start();
break;
case MAM_I_SPAWN_PENDING: // Comprueba si el spawn ha terminado
case MAM_I_SPAWN_SINGLE_PENDING:
call_checkpoint = MAM_St_spawn_pending(wait_completed);
break;
case MAM_I_SPAWN_ADAPT_POSTPONE:
case MAM_I_SPAWN_COMPLETED:
call_checkpoint = MAM_St_red_start();
break;
case MAM_I_DIST_PENDING:
call_checkpoint = MAM_St_red_pending(wait_completed);
break;
case MAM_I_USER_START:
call_checkpoint = MAM_St_user_start(mam_state);
break;
case MAM_I_USER_PENDING:
call_checkpoint = MAM_St_user_pending(mam_state, wait_completed, user_function, user_args);
break;
case MAM_I_USER_COMPLETED:
call_checkpoint = MAM_St_user_completed();
break;
case MAM_I_SPAWN_ADAPT_PENDING:
call_checkpoint = MAM_St_spawn_adapt_pending(wait_completed);
break;
case MAM_I_SPAWN_ADAPTED:
case MAM_I_DIST_COMPLETED:
call_checkpoint = MAM_St_completed(mam_state);
break;
}
if(call_checkpoint) { MAM_Checkpoint(mam_state, wait_completed, user_function, user_args); }
if(state > MAM_I_NOT_STARTED && state < MAM_I_COMPLETED) *mam_state = MAM_PENDING;
return state;
}
/*
* TODO
*/
void MAM_Resume_redistribution(int *mam_state) {
state = MAM_I_USER_COMPLETED;
if(mam_state != NULL) *mam_state = MAM_PENDING;
}
/*
* TODO
*/
void MAM_Commit(int *mam_state) {
int request_abort;
#if MAM_DEBUG
if(mall->myId == mall->root){ DEBUG_FUNC("Trying to commit", mall->myId, mall->numP); } fflush(stdout);
#endif
// Get times before commiting
if(mall_conf->spawn_method == MAM_SPAWN_BASELINE) {
// This communication is only needed when the root process will become a zombie
malleability_times_broadcast(mall->root_collectives);
}
// Free unneded communicators
if(mall->tmp_comm != MPI_COMM_WORLD && mall->tmp_comm != MPI_COMM_NULL) MPI_Comm_disconnect(&(mall->tmp_comm));
if(*(mall->user_comm) != MPI_COMM_WORLD && *(mall->user_comm) != MPI_COMM_NULL) MPI_Comm_disconnect(mall->user_comm);
// Zombies Treatment
MAM_Zombies_update();
if(mall->zombie) {
#if MAM_DEBUG >= 1
DEBUG_FUNC("Is terminating as zombie", mall->myId, mall->numP); fflush(stdout);
#endif
request_abort = MAM_Finalize();
if(request_abort) { MPI_Abort(MPI_COMM_WORLD, -101); }
MPI_Finalize();
exit(0);
}
// Reset/Free communicators
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) { MAM_comms_update(mall->intercomm); }
if(mall->intercomm != MPI_COMM_NULL && mall->intercomm != MPI_COMM_WORLD) { MPI_Comm_disconnect(&(mall->intercomm)); } //FIXME Error en OpenMPI + Merge
MPI_Comm_rank(mall->comm, &mall->myId);
MPI_Comm_size(mall->comm, &mall->numP);
mall->root = mall_conf->spawn_method == MAM_SPAWN_BASELINE ? mall->root : mall->root_parents;
mall->root_parents = mall->root;
state = MAM_I_NOT_STARTED;
if(mam_state != NULL) *mam_state = MAM_COMPLETED;
// Set new communicator
MPI_Comm_dup(mall->comm, mall->user_comm);
#if MAM_DEBUG
if(mall->myId == mall->root) DEBUG_FUNC("Reconfiguration has been commited", mall->myId, mall->numP); fflush(stdout);
#endif
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->malleability_end = MPI_Wtime();
}
/*
* This function adds data to a data structure based on whether the operation is synchronous or asynchronous,
* and whether the data is replicated or distributed. It takes the following parameters:
* - data: a pointer to the data to be added
* - index: a pointer to a size_t variable where the index of the added data will be stored
* - total_qty: the amount of elements in data
* - type: the MPI datatype of the data
* - is_replicated: a flag indicating whether the data is replicated (MAM_DATA_REPLICATED) or not (MAM_DATA_DISTRIBUTED)
* - is_constant: a flag indicating whether the operation is asynchronous (MAM_DATA_CONSTANT) or synchronous (MAM_DATA_VARIABLE)
* Finally, it updates the index with the index of the last added data if index is not NULL.
*/
void MAM_Data_add(void *data, size_t *index, size_t total_qty, MPI_Datatype type, int is_replicated, int is_constant) {
size_t total_reqs = 0, returned_index;
if(is_constant) { //Async
if(is_replicated) {
total_reqs = 1;
add_data(data, total_qty, type, total_reqs, rep_a_data);
returned_index = rep_a_data->entries-1;
} else {
if(mall_conf->red_method == MAM_RED_BASELINE) {
total_reqs = 1;
} else if(mall_conf->red_method == MAM_RED_POINT || mall_conf->red_method == MAM_RED_RMA_LOCK || mall_conf->red_method == MAM_RED_RMA_LOCKALL) {
total_reqs = mall->numC;
}
add_data(data, total_qty, type, total_reqs, dist_a_data);
returned_index = dist_a_data->entries-1;
}
} else { //Sync
if(is_replicated) {
add_data(data, total_qty, type, total_reqs, rep_s_data);
returned_index = rep_s_data->entries-1;
} else {
add_data(data, total_qty, type, total_reqs, dist_s_data);
returned_index = dist_s_data->entries-1;
}
}
if(index != NULL) *index = returned_index;
}
/*
* This function modifies a data entry to a data structure based on whether the operation is synchronous or asynchronous,
* and whether the data is replicated or distributed. It takes the following parameters:
* - data: a pointer to the data to be added
* - index: a value indicating which entry will be modified
* - total_qty: the amount of elements in data
* - type: the MPI datatype of the data
* - is_replicated: a flag indicating whether the data is replicated (MAM_DATA_REPLICATED) or not (MAM_DATA_DISTRIBUTED)
* - is_constant: a flag indicating whether the operation is asynchronous (MAM_DATA_CONSTANT) or synchronous (MAM_DATA_VARIABLE)
*/
void MAM_Data_modify(void *data, size_t index, size_t total_qty, MPI_Datatype type, int is_replicated, int is_constant) {
size_t total_reqs = 0;
if(is_constant) {
if(is_replicated) {
total_reqs = 1;
modify_data(data, index, total_qty, type, total_reqs, rep_a_data); //FIXME total_reqs==0 ???
} else {
if(mall_conf->red_method == MAM_RED_BASELINE) {
total_reqs = 1;
} else if(mall_conf->red_method == MAM_RED_POINT || mall_conf->red_method == MAM_RED_RMA_LOCK || mall_conf->red_method == MAM_RED_RMA_LOCKALL) {
total_reqs = mall->numC;
}
modify_data(data, index, total_qty, type, total_reqs, dist_a_data);
}
} else {
if(is_replicated) {
modify_data(data, index, total_qty, type, total_reqs, rep_s_data);
} else {
modify_data(data, index, total_qty, type, total_reqs, dist_s_data);
}
}
}
/*
* This functions returns how many data entries are available for one of the specific data structures.
* It takes the following parameters:
* - is_replicated: a flag indicating whether the structure is replicated (MAM_DATA_REPLICATED) or not (MAM_DATA_DISTRIBUTED)
* - is_constant: a flag indicating whether the operation is asynchronous (MAM_DATA_CONSTANT) or synchronous (MAM_DATA_VARIABLE)
* - entries: a pointer where the amount of entries will be stored
*/
void MAM_Data_get_entries(int is_replicated, int is_constant, size_t *entries){
if(is_constant) {
if(is_replicated) {
*entries = rep_a_data->entries;
} else {
*entries = dist_a_data->entries;
}
} else {
if(is_replicated) {
*entries = rep_s_data->entries;
} else {
*entries = dist_s_data->entries;
}
}
}
/*
* This function returns a data entry to a data structure based on whether the operation is synchronous or asynchronous,
* and whether the data is replicated or distributed. It takes the following parameters:
* - index: a value indicating which entry will be modified
* - is_replicated: a flag indicating whether the data is replicated (MAM_DATA_REPLICATED) or not (MAM_DATA_DISTRIBUTED)
* - is_constant: a flag indicating whether the operation is asynchronous (MAM_DATA_CONSTANT) or synchronous (MAM_DATA_VARIABLE)
* - data: a pointer where the data will be stored. The user must free it
* - total_qty: the amount of elements in data for all ranks
* - local_qty: the amount of elements in data for this rank
*/
void MAM_Data_get_pointer(void **data, size_t index, size_t *total_qty, MPI_Datatype *type, int is_replicated, int is_constant) {
malleability_data_t *data_struct;
if(is_constant) {
if(is_replicated) {
data_struct = rep_a_data;
} else {
data_struct = dist_a_data;
}
} else {
if(is_replicated) {
data_struct = rep_s_data;
} else {
data_struct = dist_s_data;
}
}
*data = data_struct->arrays[index];
if(total_qty != NULL) *total_qty = data_struct->qty[index];
if(type != NULL) *type = data_struct->types[index];
//get_block_dist(qty, mall->myId, mall->numP, &dist_data); //FIXME Asegurar que numP es correcto
}
/*
* @brief Returns a structure to perform data redistribution during a reconfiguration.
*
* This function is intended to be called when the state of MaM is MAM_I_USER_PENDING only.
* It is designed to provide the necessary information for the user to perform data redistribution.
*
* Parameters:
* - mam_user_reconf_t *reconf_info: A pointer to a mam_user_reconf_t structure where the function will store the required information for data redistribution.
*
* Return Value:
* - MAM_OK: If the function successfully retrieves the reconfiguration information.
* - MAM_DENIED: If the function is called when the state of the MaM is not MAM_I_USER_PENDING.
*/
int MAM_Get_Reconf_Info(mam_user_reconf_t *reconf_info) {
if(state != MAM_I_USER_PENDING) return MAM_DENIED;
*reconf_info = *user_reconf;
return MAM_OK;
}
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//================DATA COMMUNICATION====================||
//======================================================||
//======================================================||
/*
* Funcion generalizada para enviar datos desde los hijos.
* La asincronizidad se refiere a si el hilo padre e hijo lo hacen
* de forma bloqueante o no. El padre puede tener varios hilos.
*/
void send_data(int numP_children, malleability_data_t *data_struct, int is_asynchronous) {
size_t i;
void *aux_send, *aux_recv;
if(is_asynchronous) {
for(i=0; i < data_struct->entries; i++) {
aux_send = data_struct->arrays[i];
aux_recv = NULL;
async_communication_start(aux_send, &aux_recv, data_struct->qty[i], data_struct->types[i], mall->numP, numP_children, MAM_SOURCES,
mall->intercomm, &(data_struct->requests[i]), &(data_struct->request_qty[i]), &(data_struct->windows[i]));
if(aux_recv != NULL) data_struct->arrays[i] = aux_recv;
}
} else {
for(i=0; i < data_struct->entries; i++) {
aux_send = data_struct->arrays[i];
aux_recv = NULL;
sync_communication(aux_send, &aux_recv, data_struct->qty[i], data_struct->types[i], mall->numP, numP_children, MAM_SOURCES, mall->intercomm);
if(aux_recv != NULL) data_struct->arrays[i] = aux_recv;
}
}
}
/*
* Funcion generalizada para recibir datos desde los hijos.
* La asincronizidad se refiere a si el hilo padre e hijo lo hacen
* de forma bloqueante o no. El padre puede tener varios hilos.
*/
void recv_data(int numP_parents, malleability_data_t *data_struct, int is_asynchronous) {
size_t i;
void *aux, *aux_s = NULL;
if(is_asynchronous) {
for(i=0; i < data_struct->entries; i++) {
aux = data_struct->arrays[i];
async_communication_start(aux_s, &aux, data_struct->qty[i], data_struct->types[i], mall->numP, numP_parents, MAM_TARGETS,
mall->intercomm, &(data_struct->requests[i]), &(data_struct->request_qty[i]), &(data_struct->windows[i]));
data_struct->arrays[i] = aux;
}
} else {
for(i=0; i < data_struct->entries; i++) {
aux = data_struct->arrays[i];
sync_communication(aux_s, &aux, data_struct->qty[i], data_struct->types[i], mall->numP, numP_parents, MAM_TARGETS, mall->intercomm);
data_struct->arrays[i] = aux;
}
}
}
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//====================MAM STAGES========================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
int MAM_St_rms(int *mam_state) {
reset_malleability_times();
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->malleability_start = MPI_Wtime();
MAM_Check_configuration();
*mam_state = MAM_NOT_STARTED;
state = MAM_I_RMS_COMPLETED;
mall->wait_targets_posted = 0;
//if(CHECK_RMS()) {return MAM_DENIED;}
return 1;
}
int MAM_St_spawn_start() {
mall->num_parents = mall->numP;
state = spawn_step();
//FIXME Esto es necesario pero feo
if(mall_conf->spawn_method == MAM_SPAWN_MERGE && mall->myId >= mall->numC){ mall->zombie = 1; }
else if(mall_conf->spawn_method == MAM_SPAWN_BASELINE){ mall->zombie = 1; }
if (state == MAM_I_SPAWN_COMPLETED || state == MAM_I_SPAWN_ADAPT_POSTPONE){
return 1;
}
return 0;
}
int MAM_St_spawn_pending(int wait_completed) {
state = check_spawn_state(&(mall->intercomm), mall->comm, wait_completed);
if (state == MAM_I_SPAWN_COMPLETED || state == MAM_I_SPAWN_ADAPTED) {
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_time = MPI_Wtime() - mall_conf->times->malleability_start;
return 1;
}
return 0;
}
int MAM_St_red_start() {
if(MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_INTERCOMM, NULL)) {
mall->root_collectives = mall->myId == mall->root ? MPI_ROOT : MPI_PROC_NULL;
} else {
mall->root_collectives = mall->root;
}
state = start_redistribution();
return 1;
}
int MAM_St_red_pending(int wait_completed) {
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_PTHREAD, NULL)) {
state = thread_check(wait_completed);
} else {
state = check_redistribution(wait_completed);
}
if(state != MAM_I_DIST_PENDING) {
state = MAM_I_USER_START;
return 1;
}
return 0;
}
int MAM_St_user_start(int *mam_state) {
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->user_start = MPI_Wtime(); // Obtener timestamp de cuando termina user redist
if(MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_INTERCOMM, NULL)) {
MPI_Intercomm_merge(mall->intercomm, MAM_SOURCES, &mall->tmp_comm); //El que pone 0 va primero
} else {
MPI_Comm_dup(mall->intercomm, &mall->tmp_comm);
}
MPI_Comm_set_name(mall->tmp_comm, "MAM_USER_TMP");
state = MAM_I_USER_PENDING;
*mam_state = MAM_USER_PENDING;
return 1;
}
int MAM_St_user_pending(int *mam_state, int wait_completed, void (*user_function)(void *), void *user_args) {
#if MAM_DEBUG
if(mall->myId == mall->root) DEBUG_FUNC("Starting USER redistribution", mall->myId, mall->numP); fflush(stdout);
#endif
if(user_function != NULL) {
MAM_I_create_user_struct(MAM_SOURCES);
do {
user_function(user_args);
} while(wait_completed && state == MAM_I_USER_PENDING);
} else {
MAM_Resume_redistribution(mam_state);
}
if(state != MAM_I_USER_PENDING) {
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) mall_conf->times->user_end = MPI_Wtime(); // Obtener timestamp de cuando termina user redist
#if MAM_DEBUG
if(mall->myId == mall->root) DEBUG_FUNC("Ended USER redistribution", mall->myId, mall->numP); fflush(stdout);
#endif
return 1;
}
return 0;
}
int MAM_St_user_completed() {
state = end_redistribution();
return 1;
}
int MAM_St_spawn_adapt_pending(int wait_completed) {
wait_completed = MAM_WAIT_COMPLETION;
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_start = MPI_Wtime();
unset_spawn_postpone_flag(state);
state = check_spawn_state(&(mall->intercomm), mall->comm, wait_completed);
/* TODO Comentar problema, basicamente indicar que no es posible de la forma actual
* Ademas es solo para una operación que hemos visto como "extremadamente" rápida
* NO es posible debido a que solo se puede hacer tras enviar los datos variables
* y por tanto pierden validez dichos datos
if(!MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_PTHREAD, NULL)) {
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_time = MPI_Wtime() - mall_conf->times->spawn_start;
return 1;
}
return 0;
*/
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_time = MPI_Wtime() - mall_conf->times->spawn_start;
return 1;
}
int MAM_St_completed(int *mam_state) {
MAM_Commit(mam_state);
return 0;
}
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//=====================CHILDREN=========================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
/*
* Inicializacion de los datos de los hijos.
* En la misma se reciben datos de los padres: La configuracion
* de la ejecucion a realizar; y los datos a recibir de los padres
* ya sea de forma sincrona, asincrona o ambas.
*/
void Children_init(void (*user_function)(void *), void *user_args) {
size_t i;
#if MAM_DEBUG
DEBUG_FUNC("MaM will now initialize spawned processes", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
malleability_connect_children(&(mall->intercomm));
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) { // For Merge Method, these processes will be added
MPI_Comm_rank(mall->intercomm, &mall->myId);
MPI_Comm_size(mall->intercomm, &mall->numP);
}
mall->root_collectives = mall->root_parents;
if(MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_MULTIPLE, NULL)
|| MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_PARALLEL, NULL)) {
mall->internode_group = 0;
} else {
mall->internode_group = MAM_Is_internode_group();
}
#if MAM_DEBUG
DEBUG_FUNC("Spawned have completed spawn step", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
comm_data_info(rep_a_data, dist_a_data, MAM_TARGETS);
if(dist_a_data->entries || rep_a_data->entries) { // Recibir datos asincronos
#if MAM_DEBUG >= 2
DEBUG_FUNC("Spawned start asynchronous redistribution", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_PTHREAD, NULL)) {
recv_data(mall->num_parents, dist_a_data, MAM_USE_SYNCHRONOUS);
for(i=0; i<rep_a_data->entries; i++) {
MPI_Bcast(rep_a_data->arrays[i], rep_a_data->qty[i], rep_a_data->types[i], mall->root_collectives, mall->intercomm);
}
} else {
recv_data(mall->num_parents, dist_a_data, MAM_USE_ASYNCHRONOUS);
for(i=0; i<rep_a_data->entries; i++) {
MPI_Ibcast(rep_a_data->arrays[i], rep_a_data->qty[i], rep_a_data->types[i], mall->root_collectives, mall->intercomm, &(rep_a_data->requests[i][0]));
}
#if MAM_DEBUG >= 2
DEBUG_FUNC("Spawned started asynchronous redistribution", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
for(i=0; i<rep_a_data->entries; i++) {
async_communication_wait(rep_a_data->requests[i], rep_a_data->request_qty[i]);
}
for(i=0; i<dist_a_data->entries; i++) {
async_communication_wait(dist_a_data->requests[i], dist_a_data->request_qty[i]);
}
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_WAIT_TARGETS, NULL)) {
MPI_Ibarrier(mall->intercomm, &mall->wait_targets);
mall->wait_targets_posted = 1;
MPI_Wait(&mall->wait_targets, MPI_STATUS_IGNORE);
}
#if MAM_DEBUG >= 2
DEBUG_FUNC("Spawned waited for all asynchronous redistributions", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
for(i=0; i<dist_a_data->entries; i++) {
async_communication_end(dist_a_data->requests[i], dist_a_data->request_qty[i], &(dist_a_data->windows[i]));
}
for(i=0; i<rep_a_data->entries; i++) {
async_communication_end(rep_a_data->requests[i], rep_a_data->request_qty[i], &(rep_a_data->windows[i]));
}
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->async_end= MPI_Wtime(); // Obtener timestamp de cuando termina comm asincrona
}
#if MAM_DEBUG
DEBUG_FUNC("Spawned have completed asynchronous data redistribution step", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_INTERCOMM, NULL)) {
MPI_Intercomm_merge(mall->intercomm, MAM_TARGETS, &mall->tmp_comm); //El que pone 0 va primero
} else {
MPI_Comm_dup(mall->intercomm, &mall->tmp_comm);
}
MPI_Comm_set_name(mall->tmp_comm, "MAM_USER_TMP");
if(user_function != NULL) {
state = MAM_I_USER_PENDING;
MAM_I_create_user_struct(MAM_TARGETS);
user_function(user_args);
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->user_end = MPI_Wtime(); // Obtener timestamp de cuando termina user redist
comm_data_info(rep_s_data, dist_s_data, MAM_TARGETS);
if(dist_s_data->entries || rep_s_data->entries) { // Recibir datos sincronos
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
recv_data(mall->num_parents, dist_s_data, MAM_USE_SYNCHRONOUS);
for(i=0; i<rep_s_data->entries; i++) {
MPI_Bcast(rep_s_data->arrays[i], rep_s_data->qty[i], rep_s_data->types[i], mall->root_collectives, mall->intercomm);
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->sync_end = MPI_Wtime(); // Obtener timestamp de cuando termina comm sincrona
}
#if MAM_DEBUG
DEBUG_FUNC("Targets have completed synchronous data redistribution step", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
MAM_Commit(NULL);
#if MAM_DEBUG
DEBUG_FUNC("MaM has been initialized correctly for new ranks", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
}
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//=====================PARENTS==========================||
//======================================================||
//======================================================||
//======================================================||
//======================================================||
/*
* Se encarga de realizar la creacion de los procesos hijos.
* Si se pide en segundo plano devuelve el estado actual.
*/
int spawn_step(){
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_start = MPI_Wtime();
state = init_spawn(mall->thread_comm, &(mall->intercomm));
if(!MAM_Contains_strat(MAM_SPAWN_STRATEGIES, MAM_STRAT_SPAWN_PTHREAD, NULL)) {
#if MAM_USE_BARRIERS
MPI_Barrier(mall->comm);
#endif
mall_conf->times->spawn_time = MPI_Wtime() - mall_conf->times->malleability_start;
}
return state;
}
/*
* Comienza la redistribucion de los datos con el nuevo grupo de procesos.
*
* Primero se envia la configuracion a utilizar al nuevo grupo de procesos y a continuacion
* se realiza el envio asincrono y/o sincrono si lo hay.
*
* En caso de que haya comunicacion asincrona, se comienza y se termina la funcion
* indicando que se ha comenzado un envio asincrono.
*
* Si no hay comunicacion asincrono se pasa a realizar la sincrona si la hubiese.
*
* Finalmente se envian datos sobre los resultados a los hijos y se desconectan ambos
* grupos de procesos.
*/
int start_redistribution() {
size_t i;
if(mall->intercomm == MPI_COMM_NULL) {
// Si no tiene comunicador creado, se debe a que se ha pospuesto el Spawn
// y se trata del spawn Merge Shrink
MPI_Comm_dup(mall->comm, &(mall->intercomm));
}
comm_data_info(rep_a_data, dist_a_data, MAM_SOURCES);
if(dist_a_data->entries || rep_a_data->entries) { // Enviar datos asincronos
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->async_start = MPI_Wtime();
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_PTHREAD, NULL)) {
return thread_creation();
} else {
send_data(mall->numC, dist_a_data, MAM_USE_ASYNCHRONOUS);
for(i=0; i<rep_a_data->entries; i++) {
MPI_Ibcast(rep_a_data->arrays[i], rep_a_data->qty[i], rep_a_data->types[i], mall->root_collectives, mall->intercomm, &(rep_a_data->requests[i][0]));
}
if(mall->zombie && MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_WAIT_TARGETS, NULL)) {
MPI_Ibarrier(mall->intercomm, &mall->wait_targets);
mall->wait_targets_posted = 1;
}
return MAM_I_DIST_PENDING;
}
}
return MAM_I_USER_START;
}
/*
* Comprueba si la redistribucion asincrona ha terminado.
* Si no ha terminado la funcion termina indicandolo, en caso contrario,
* se continua con la comunicacion sincrona, el envio de resultados y
* se desconectan los grupos de procesos.
*
* Esta funcion permite dos modos de funcionamiento al comprobar si la
* comunicacion asincrona ha terminado.
* Si se utiliza el modo "MAL_USE_NORMAL" o "MAL_USE_POINT", se considera
* terminada cuando los padres terminan de enviar.
* Si se utiliza el modo "MAL_USE_IBARRIER", se considera terminada cuando
* los hijos han terminado de recibir.
* //FIXME Modificar para que se tenga en cuenta rep_a_data
*/
int check_redistribution(int wait_completed) {
int completed, local_completed, all_completed;
size_t i, req_qty;
MPI_Request *req_completed;
MPI_Win window;
local_completed = 1;
#if MAM_DEBUG >= 2
DEBUG_FUNC("Sources are testing for all asynchronous redistributions", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
if(wait_completed) {
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_WAIT_TARGETS, NULL) && !mall->wait_targets_posted) {
MPI_Ibarrier(mall->intercomm, &mall->wait_targets);
mall->wait_targets_posted = 1;
}
for(i=0; i<dist_a_data->entries; i++) {
req_completed = dist_a_data->requests[i];
req_qty = dist_a_data->request_qty[i];
async_communication_wait(req_completed, req_qty);
}
for(i=0; i<rep_a_data->entries; i++) {
req_completed = rep_a_data->requests[i];
req_qty = rep_a_data->request_qty[i];
async_communication_wait(req_completed, req_qty);
}
if(MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_WAIT_TARGETS, NULL)) { MPI_Wait(&mall->wait_targets, MPI_STATUS_IGNORE); }
} else {
if(mall->wait_targets_posted) {
MPI_Test(&mall->wait_targets, &local_completed, MPI_STATUS_IGNORE);
} else {
for(i=0; i<dist_a_data->entries; i++) {
req_completed = dist_a_data->requests[i];
req_qty = dist_a_data->request_qty[i];
completed = async_communication_check(MAM_SOURCES, req_completed, req_qty);
local_completed = local_completed && completed;
}
for(i=0; i<rep_a_data->entries; i++) {
req_completed = rep_a_data->requests[i];
req_qty = rep_a_data->request_qty[i];
completed = async_communication_check(MAM_SOURCES, req_completed, req_qty);
local_completed = local_completed && completed;
}
if(local_completed && MAM_Contains_strat(MAM_RED_STRATEGIES, MAM_STRAT_RED_WAIT_TARGETS, NULL)) {
MPI_Ibarrier(mall->intercomm, &mall->wait_targets);
mall->wait_targets_posted = 1;
MPI_Test(&mall->wait_targets, &local_completed, MPI_STATUS_IGNORE); //TODO - Figure out if last process takes profit from calling here
}
}
#if MAM_DEBUG >= 2
DEBUG_FUNC("Sources will now check a global decision", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
MPI_Allreduce(&local_completed, &all_completed, 1, MPI_INT, MPI_MIN, mall->comm);
if(!all_completed) return MAM_I_DIST_PENDING; // Continue only if asynchronous send has ended
}
#if MAM_DEBUG >= 2
DEBUG_FUNC("Sources sent asynchronous redistributions", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD);
#endif
for(i=0; i<dist_a_data->entries; i++) {
req_completed = dist_a_data->requests[i];
req_qty = dist_a_data->request_qty[i];
window = dist_a_data->windows[i];
async_communication_end(req_completed, req_qty, &window);
}
for(i=0; i<rep_a_data->entries; i++) {
req_completed = rep_a_data->requests[i];
req_qty = rep_a_data->request_qty[i];
window = rep_a_data->windows[i];
async_communication_end(req_completed, req_qty, &window);
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) mall_conf->times->async_end = MPI_Wtime(); // Merge method only
return MAM_I_USER_START;
}
/*
* Termina la redistribución de los datos con los hijos, comprobando
* si se han realizado iteraciones con comunicaciones en segundo plano
* y enviando cuantas iteraciones se han realizado a los hijos.
*
* Además se realizan las comunicaciones síncronas se las hay.
* Finalmente termina enviando los datos temporales a los hijos.
*/
int end_redistribution() {
size_t i;
int local_state;
comm_data_info(rep_s_data, dist_s_data, MAM_SOURCES);
if(dist_s_data->entries || rep_s_data->entries) { // Enviar datos sincronos
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
mall_conf->times->sync_start = MPI_Wtime();
send_data(mall->numC, dist_s_data, MAM_USE_SYNCHRONOUS);
for(i=0; i<rep_s_data->entries; i++) {
MPI_Bcast(rep_s_data->arrays[i], rep_s_data->qty[i], rep_s_data->types[i], mall->root_collectives, mall->intercomm);
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) mall_conf->times->sync_end = MPI_Wtime(); // Merge method only
}
#if MAM_DEBUG
DEBUG_FUNC("Sources have completed synchronous data redistribution step", mall->myId, mall->numP); fflush(stdout); MPI_Barrier(mall->comm);
#endif
local_state = MAM_I_DIST_COMPLETED;
if(mall_conf->spawn_method == MAM_SPAWN_MERGE && mall->numP > mall->numC) { // Merge Shrink
local_state = MAM_I_SPAWN_ADAPT_PENDING;
}
return local_state;
}
// TODO MOVER A OTRO LADO??
//======================================================||
//================PRIVATE FUNCTIONS=====================||
//===============COMM PARENTS THREADS===================||
//======================================================||
//======================================================||
int comm_state; //FIXME Usar un handler
/*
* Crea una hebra para ejecutar una comunicación en segundo plano.
*/
int thread_creation() {
comm_state = MAM_I_DIST_PENDING;
if(pthread_create(&(mall->async_thread), NULL, thread_async_work, NULL)) {
printf("Error al crear el hilo\n");
MPI_Abort(MPI_COMM_WORLD, -1);
return -1;
}
return comm_state;
}
/*
* Comprobación por parte de una hebra maestra que indica
* si una hebra esclava ha terminado su comunicación en segundo plano.
*
* El estado de la comunicación es devuelto al finalizar la función.
*/
int thread_check(int wait_completed) {
int all_completed = 0;
if(wait_completed && comm_state == MAM_I_DIST_PENDING) {
if(pthread_join(mall->async_thread, NULL)) {
printf("Error al esperar al hilo\n");
MPI_Abort(MPI_COMM_WORLD, -1);
return -2;
}
}
// Comprueba que todos los hilos han terminado la distribucion (Mismo valor en commAsync)
MPI_Allreduce(&comm_state, &all_completed, 1, MPI_INT, MPI_MAX, mall->comm);
if(all_completed != MAM_I_DIST_COMPLETED) return MAM_I_DIST_PENDING; // Continue only if asynchronous send has ended
if(pthread_join(mall->async_thread, NULL)) {
printf("Error al esperar al hilo\n");
MPI_Abort(MPI_COMM_WORLD, -1);
return -2;
}
#if MAM_USE_BARRIERS
MPI_Barrier(mall->intercomm);
#endif
if(mall_conf->spawn_method == MAM_SPAWN_MERGE) mall_conf->times->async_end = MPI_Wtime(); // Merge method only
return MAM_I_USER_START;
}
/*
* Función ejecutada por una hebra.
* Ejecuta una comunicación síncrona con los hijos que
* para el usuario se puede considerar como en segundo plano.
*
* Cuando termina la comunicación la hebra maestra puede comprobarlo
* por el valor "commAsync".
*/
void* thread_async_work() {
size_t i;
send_data(mall->numC, dist_a_data, MAM_USE_SYNCHRONOUS);
for(i=0; i<rep_a_data->entries; i++) {
MPI_Bcast(rep_a_data->arrays[i], rep_a_data->qty[i], rep_a_data->types[i], mall->root_collectives, mall->intercomm);
}
comm_state = MAM_I_DIST_COMPLETED;
pthread_exit(NULL);
}
//==============================================================================
/*
* TODO Por hacer
*/
void MAM_I_create_user_struct(int is_children_group) {
user_reconf->comm = mall->tmp_comm;
if(is_children_group) {
user_reconf->rank_state = MAM_PROC_NEW_RANK;
user_reconf->numS = mall->num_parents;
user_reconf->numT = mall->numP;
} else {
user_reconf->numS = mall->numP;
user_reconf->numT = mall->numC;
if(mall->zombie) user_reconf->rank_state = MAM_PROC_ZOMBIE;
else user_reconf->rank_state = MAM_PROC_CONTINUE;
}
}
#ifndef MAM_MANAGER_H
#define MAM_MANAGER_H
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <mpi.h>
typedef struct {
int numS, numT;
int rank_state;
MPI_Comm comm;
} mam_user_reconf_t;
int MAM_Init(int root, MPI_Comm *comm, char *name_exec, void (*user_function)(void *), void *user_args);
int MAM_Finalize();
int MAM_Checkpoint(int *mam_state, int wait_completed, void (*user_function)(void *), void *user_args);
void MAM_Resume_redistribution(int *mam_state);
int MAM_Get_Reconf_Info(mam_user_reconf_t *reconf_info);
void MAM_Data_add(void *data, size_t *index, size_t total_qty, MPI_Datatype type, int is_replicated, int is_constant);
void MAM_Data_modify(void *data, size_t index, size_t total_qty, MPI_Datatype type, int is_replicated, int is_constant);
void MAM_Data_get_entries(int is_replicated, int is_constant, size_t *entries);
void MAM_Data_get_pointer(void **data, size_t index, size_t *total_qty, MPI_Datatype *type, int is_replicated, int is_constant);
#endif
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment