{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "%matplotlib inline\n", "import pandas as pd\n", "from pandas import DataFrame, Series\n", "import numpy as np\n", "import math\n", "\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "import matplotlib.patches as mpatches\n", "import matplotlib.colors as colors\n", "from matplotlib.legend_handler import HandlerLine2D, HandlerTuple\n", "from matplotlib.colors import LinearSegmentedColormap\n", "from scipy import stats\n", "import scikit_posthocs as sp\n", "import sys\n", "\n", "from mpl_toolkits.mplot3d import axes3d" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "AllName=\"dataG.pkl\"\n", "ResizesName=\"dataM.pkl\"\n", "ItersName=\"dataL\"\n", "NameExtension=\".pkl\"\n", "n_cores=20\n", "repet = 5 #CAMBIAR EL NUMERO SEGUN NUMERO DE EJECUCIONES POR CONFIG\n", "\n", "significance_value = 0.05\n", "processes = [2,10,20,40,80,120,160]\n", "\n", "positions = [321, 322, 323, 324, 325]\n", "positions_small = [221, 222, 223, 224]\n", "\n", "labels = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n", " '(10,1)', '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n", " '(20,1)', '(20,10)', '(20,40)', '(20,80)', '(20,120)','(20,160)',\n", " '(40,1)', '(40,10)', '(40,20)', '(40,80)', '(40,120)','(40,160)',\n", " '(80,1)', '(80,10)', '(80,20)', '(80,40)', '(80,120)','(80,160)',\n", " '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)','(120,160)',\n", " '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n", "\n", "labelsExpand = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n", " '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n", " '(20,40)', '(20,80)', '(20,120)','(20,160)',\n", " '(40,80)', '(40,120)','(40,160)',\n", " '(80,120)','(80,160)',\n", " '(120,160)']\n", "labelsShrink = ['(10,1)', \n", " '(20,1)', '(20,10)',\n", " '(40,1)', '(40,10)', '(40,20)',\n", " '(80,1)', '(80,10)', '(80,20)', '(80,40)',\n", " '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)',\n", " '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n", "\n", "# WORST BEST\n", "labels_dist = ['null', 'SpreadFit', 'CompactFit']\n", " #0 #1 #2 #3\n", "labelsMethods = ['Baseline', 'Baseline single','Baseline - Asynchronous','Baseline single - Asynchronous',\n", " 'Merge','Merge single','Merge - Asynchronous','Merge single - Asynchronous']\n", " #4 #5 #6 #7\n", " \n", "colors_m = ['green','darkgreen','red','darkred','mediumseagreen','seagreen','palegreen','springgreen','indianred','firebrick','darkgoldenrod','saddlebrown']\n", "linestyle_m = ['-', '--', '-.', ':']\n", "markers_m = ['.','v','s','p', 'h','d','X','P','^']\n", "\n", "OrMult_patch = mpatches.Patch(hatch='', facecolor='green', label='Baseline')\n", "OrSing_patch = mpatches.Patch(hatch='', facecolor='springgreen', label='Baseline single')\n", "OrPthMult_patch = mpatches.Patch(hatch='//', facecolor='blue', label='Baseline - Asyncrhonous')\n", "OrPthSing_patch = mpatches.Patch(hatch='\\\\', facecolor='darkblue', label='Baseline single - Asyncrhonous')\n", "MergeMult_patch = mpatches.Patch(hatch='||', facecolor='red', label='Merge')\n", "MergeSing_patch = mpatches.Patch(hatch='...', facecolor='darkred', label='Merge single')\n", "MergePthMult_patch = mpatches.Patch(hatch='xx', facecolor='yellow', label='Merge - Asyncrhonous')\n", "MergePthSing_patch = mpatches.Patch(hatch='++', facecolor='olive', label='Merge single - Asyncrhonous')\n", "\n", "handles_spawn = [OrMult_patch,OrSing_patch,OrPthMult_patch,OrPthSing_patch,MergeMult_patch,MergeSing_patch,MergePthMult_patch,MergePthSing_patch]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "dfG = pd.read_pickle( AllName )\n", "\n", "dfG['ADR'] = round((dfG['ADR'] / dfG['DR']) * 100,1)\n", "dfG['SDR'] = round((dfG['SDR'] / dfG['DR']) * 100,1)\n", " \n", "out_group = dfG.groupby(['Groups', 'ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_total']\n", "group = dfG.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','Groups'])['T_total']\n", "\n", "grouped_aggG = group.agg(['median'])\n", "grouped_aggG.rename(columns={'median':'T_total'}, inplace=True) \n", "\n", "out_grouped_G = out_group.agg(['median'])\n", "out_grouped_G.rename(columns={'median':'T_total'}, inplace=True) " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "dfM = pd.read_pickle( ResizesName )\n", "\n", "dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)\n", "dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)\n", "dfM['T_Redistribution'] = dfM['T_SR'] + dfM['T_AR']\n", "dfM.loc[dfM['T_Malleability']==0,'T_Malleability'] = dfM['T_spawn'] + dfM['T_Redistribution']\n", " \n", "out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']\n", "group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_Redistribution','T_spawn','T_spawn_real','T_SR','T_AR']\n", "\n", "grouped_aggM = group.agg(['median'])\n", "grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)\n", "\n", "out_grouped_M = out_group.agg(['median'])\n", "out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "grouped_aggLAsynch = pd.DataFrame()\n", "grouped_aggLSynch = pd.DataFrame()\n", "grouped_aggLDyn = pd.DataFrame()\n", "grouped_aggLNDyn = pd.DataFrame()\n", "\n", "for i in range(26):\n", " aux_name = ItersName + str(i) + NameExtension\n", " dfL = pd.read_pickle( aux_name )\n", "\n", " #Fixme comprobar si hay iters asincronas antes de esto\n", " aux_df = dfL[(dfL.Asynch_Iters == True)]\n", " #aux_df['ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)\n", " dfL.loc[(dfL.Asynch_Iters == True),'ADR'] = round((aux_df['ADR'] / aux_df['DR']) * 100,1)\n", " #dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)\n", " dfL.loc[(dfL.Asynch_Iters == True),'SDR'] = round((aux_df['SDR'] / aux_df['DR']) * 100,1)\n", " dfL['ADR'].fillna(-1, inplace=True)\n", " dfL['SDR'].fillna(-1, inplace=True)\n", " dfL['DR'].fillna(-1, inplace=True)\n", " \n", " aux_df = dfL[(dfL.Asynch_Iters == True)]\n", " group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter']\n", " grouped_aux = group.agg(['median','count'])\n", " grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n", " grouped_aux.rename(columns={'median':'T_iter'}, inplace=True) \n", " group = aux_df.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_stages']\n", " aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n", " grouped_aux['T_stages'] = aux_column\n", " if grouped_aggLAsynch.empty:\n", " grouped_aggLAsynch = grouped_aux\n", " else:\n", " grouped_aggLAsynch = pd.concat([grouped_aggLAsynch, grouped_aux], ignore_index=False, sort=True).sort_index()\n", "\n", " aux_df = dfL[(dfL.Asynch_Iters == False)]\n", " group = aux_df.groupby('NP')['T_iter']\n", " grouped_aux = group.agg(['median'])\n", " grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n", " group = aux_df.groupby(['NP'])['T_stages']\n", " aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n", " grouped_aux['T_stages'] = aux_column\n", " if grouped_aggLSynch.empty:\n", " grouped_aggLSynch = grouped_aux\n", " else:\n", " grouped_aggLSynch = pd.concat([grouped_aggLSynch, grouped_aux], ignore_index=False, sort=True).sort_index()\n", "\n", " aux_df2 = aux_df[(aux_df.Is_Dynamic == True)]\n", " group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_iter']\n", " grouped_aux = group.agg(['median'])\n", " grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n", " group = aux_df2.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])['T_stages']\n", " aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n", " grouped_aux['T_stages'] = aux_column\n", " if grouped_aggLDyn.empty:\n", " grouped_aggLDyn = grouped_aux\n", " else:\n", " grouped_aggLDyn = pd.concat([grouped_aggLDyn, grouped_aux], ignore_index=False, sort=True).sort_index()\n", "\n", " aux_df2 = aux_df[(aux_df.Is_Dynamic == False)]\n", " group = aux_df2.groupby('NP')['T_iter']\n", " grouped_aux = group.agg(['median'])\n", " grouped_aux.rename(columns={'median':'T_iter'}, inplace=True)\n", " group = aux_df2.groupby(['NP'])['T_stages']\n", " aux_column = group.apply(list).apply(lambda x: np.median(x,0))\n", " grouped_aux['T_stages'] = aux_column\n", " if grouped_aggLNDyn.empty:\n", " grouped_aggLNDyn = grouped_aux\n", " else:\n", " grouped_aggLNDyn = pd.concat([grouped_aggLNDyn, grouped_aux], ignore_index=False, sort=True).sort_index()\n", "dfL = None\n", "\n", "group = grouped_aggLAsynch.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])\n", "grouped_aux = group.agg(['mean'])\n", "grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n", "grouped_aux['T_sum'] = grouped_aux['count'] * grouped_aux['T_iter']\n", "grouped_aggLAsynch = grouped_aux\n", "\n", "group = grouped_aggLSynch.groupby('NP')\n", "grouped_aux = group.agg(['mean'])\n", "grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n", "grouped_aggLSynch = grouped_aux\n", "\n", "group = grouped_aggLDyn.groupby(['ADR', 'Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','N_Parents'])\n", "grouped_aux = group.agg(['mean'])\n", "grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n", "grouped_aggLDyn = grouped_aux\n", "\n", "group = grouped_aggLNDyn.groupby('NP')\n", "grouped_aux = group.agg(['mean'])\n", "grouped_aux.columns = grouped_aux.columns.get_level_values(0)\n", "grouped_aggLNDyn = grouped_aux" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "from bt_scheme import PartialSolution, BacktrackingSolver\n", "def elegirConf(parameters):\n", " class StatePS(PartialSolution):\n", " def __init__(self, config):\n", " self.config= config\n", " self.n= len(config) #Indica el valor a añadir\n", "\n", " def is_solution(self):\n", " return self.n == len(parameters)\n", "\n", " def get_solution(self):\n", " return tuple(self.config)\n", "\n", " def successors(self):\n", " array = parameters[self.n]\n", " for parameter_value in array: #Test all values of the next parameter\n", " self.config.append(parameter_value)\n", " yield StatePS(self.config)\n", " self.config.pop()\n", "\n", " initialPs= StatePS([])\n", " return BacktrackingSolver().solve(initialPs)\n", "\n", "\n", "def obtenerConfs(parameters):\n", " soluciones=[]\n", " for solucion in elegirConf(parameters):\n", " soluciones.append(solucion)\n", " return soluciones\n", "\n", "def modifyToGlobal(parameters, len_parameters, configuration):\n", " usable_configuration = []\n", " for i in range(len(parameters)):\n", " if len_parameters[i] > 1:\n", " aux = (parameters[i][0], configuration[i])\n", " else:\n", " aux = (configuration[i])\n", " usable_configuration.append(aux)\n", " \n", " return usable_configuration\n", "\n", "def modifyToLocalDynamic(parameters, len_parameters, configuration):\n", " usable_configuration = []\n", " for i in range(len(parameters)):\n", " if len_parameters[i] > 1:\n", " aux = (configuration[i], -1)\n", " else:\n", " aux = (-1)\n", " usable_configuration.append(aux)\n", " \n", " return tuple(usable_configuration)\n", "\n", "def CheckConfExists(configuration, dataSet, type_conf='global'):\n", " exists = False\n", " config = list(configuration)\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " \n", " if type_conf == 'global':\n", " config.append((np_aux, ns_aux))\n", " elif type_conf == 'malleability':\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " elif type_conf == 'local':\n", " config.append(np_aux)\n", " \n", " if tuple(config) in dataSet.index: \n", " exists = True # FIXME Return here true?\n", " config.pop()\n", " \n", " if type_conf == 'malleability':\n", " config.pop()\n", " return exists" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "adr = [0,96.6]\n", "sp_method = [0,1]\n", "rd_method = [0,1]\n", "rd_strat = [1,2]\n", "parameters = [adr, sp_method, rd_method, rd_strat]\n", "parameters_names = ['ADR', 'Spawn_Method', 'Redistribution_Method', 'Redistribution_Strategy']\n", "len_parameters = [1,2,2,2]\n", "configurations_aux = obtenerConfs(parameters)\n", "configurations = []\n", "configurations_local_dynamic = set()\n", "configurations_local = set()\n", "configurations_simple = []\n", "for checked_conf in configurations_aux:\n", " aux_conf = modifyToGlobal(parameters, len_parameters, checked_conf)\n", " if CheckConfExists(aux_conf, grouped_aggG):\n", " configurations.append(aux_conf)\n", "\n", " if CheckConfExists(checked_conf, grouped_aggM, 'malleability'):\n", " configurations_simple.append(list(checked_conf))\n", " \n", " aux_conf = modifyToLocalDynamic(parameters, len_parameters, checked_conf)\n", " if CheckConfExists(aux_conf, grouped_aggLDyn, 'local'):\n", " configurations_local_dynamic.add(aux_conf)\n", "\n", "configurations_local_dynamic = list(configurations_local_dynamic)\n", "for index in range(len(configurations_local_dynamic)):\n", " configurations_local_dynamic[index] = list(configurations_local_dynamic[index])\n", "\n", "print(configurations_simple)\n", "print(configurations_local_dynamic)\n", "print(configurations)\n", "print(len(configurations))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#ALPHA COMPUTATION\n", "def compute_alpha(config_a, config_b):\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " config_a.append(np_aux)\n", " config_a.append(ns_aux)\n", " config_b.append(np_aux)\n", " config_b.append(ns_aux)\n", " grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']\n", " #grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Redistribution'] / grouped_aggM.loc[tuple(config_a),'T_Redistribution']\n", " config_a.pop()\n", " config_a.pop()\n", " config_b.pop()\n", " config_b.pop()\n", " \n", " \n", " config_a.insert(0,ns_aux)\n", " config_a.insert(0,np_aux)\n", " config_b.insert(0,ns_aux)\n", " config_b.insert(0,np_aux)\n", " out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']\n", " #out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Redistribution'] / out_grouped_M.loc[tuple(config_a),'T_Redistribution']\n", " \n", " config_a.pop(0)\n", " config_a.pop(0)\n", " config_b.pop(0)\n", " config_b.pop(0)\n", "\n", "if not ('Alpha' in grouped_aggM.columns):\n", " for config_a in configurations_simple:\n", " for config_b in configurations_simple:\n", " if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:\n", " compute_alpha(config_a, config_b)\n", "else:\n", " print(\"ALPHA already exists\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#OMEGA COMPUTATION\n", "def compute_omega(config):\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " if len(config) > len(parameters):\n", " config.pop()\n", " config.pop()\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " value = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']\n", " grouped_aggLAsynch.at[tuple(config),'Omega'] = value\n", " \n", " #grouped_aggLAsynch.at[tuple(config),'Omega'] = grouped_aggLAsynch.at[tuple(config),'T_iter'] / grouped_aggLSynch.at[np_aux,'T_iter']\n", " value = grouped_aggLAsynch.at[tuple(config),'T_stages'] / grouped_aggLSynch.at[np_aux,'T_stages']\n", " grouped_aggLAsynch.at[tuple(config),'Omega_Stages'] = value.astype(object)\n", " config.pop()\n", " config.pop()\n", "\n", "if not ('Omega' in grouped_aggLAsynch.columns):\n", " grouped_aggLAsynch['Omega'] = 1.0\n", " grouped_aggLAsynch['Omega_Stages'] = None\n", " for config in configurations:\n", " if config[0] != 0:\n", " compute_omega(config)\n", "else:\n", " print(\"OMEGA already exists\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Dynamic Coherence COMPUTATION\n", "def compute_dyn_coherency(config):\n", " for np_aux in processes:\n", " for n_parents_aux in processes:\n", " if np_aux != n_parents_aux:\n", " config.append(np_aux)\n", " config.append(n_parents_aux)\n", " value = grouped_aggLDyn.at[tuple(config),'T_iter'] / grouped_aggLNDyn.at[np_aux,'T_iter']\n", " grouped_aggLDyn.at[tuple(config),'Dyn_Coherency'] = value\n", " value = grouped_aggLDyn.at[tuple(config),'T_stages'] / grouped_aggLNDyn.at[np_aux,'T_stages']\n", " grouped_aggLDyn.at[tuple(config),'Dyn_Coherency_Stages'] = value.astype(object)\n", " config.pop()\n", " config.pop()\n", "\n", "if not ('Dyn_Coherency' in grouped_aggLDyn.columns):\n", " grouped_aggLDyn['Dyn_Coherency'] = 1.0\n", " grouped_aggLDyn['Dyn_Coherency_Stages'] = None\n", " for config in configurations_local_dynamic:\n", " compute_dyn_coherency(config)\n", "else:\n", " print(\"Dyn_Coherency already exists\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Malleability Coherence COMPUTATION\n", "test=dfM[(dfM.Asynch_Iters > 0)]\n", "coherence_boundary = 1.01 # Allows a 1% error\n", "total_aux = 0\n", "show_all = False\n", "\n", "# Coherence of synchronous inner times + asynchronous iterations\n", "for index in range(len(test)):\n", " time_malleability_aux = test[\"T_Malleability\"].values[index]\n", " \n", " total_asynch_iters = int(test[\"Asynch_Iters\"].values[index])\n", " asynch_iters = test[\"T_iter\"].values[index][-total_asynch_iters:]\n", " time_iters_aux = np.sum(asynch_iters)\n", " \n", " time_synch_aux = test[\"T_SR\"].values[index]\n", " time_spawn_aux = test[\"T_spawn\"].values[index]\n", " is_synch_spawn = (test[\"Spawn_Strategy\"].values[index] % 2 != 0)\n", " \n", " expected_time = time_iters_aux + time_spawn_aux * is_synch_spawn + time_synch_aux\n", " time_malleability_aux = time_malleability_aux * coherence_boundary\n", " \n", " if time_malleability_aux < expected_time:\n", " total_aux += 1\n", " if show_all:\n", " print(test.iloc[index])\n", " print(asynch_iters)\n", " print(time_iters_aux)\n", " print(time_malleability_aux)\n", " print(\"\")\n", "if total_aux > 0:\n", " print(\"Coherence fails for inner synchronous times + asynchornous iterations in \" + str(total_aux) + \" cases\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Malleability Coherence COMPUTATION\n", "test=dfM[(dfM.Asynch_Iters > 0)]\n", "coherence_boundary = 1.01 # Allows a 1% error\n", "# Coherence of inner times\n", "aux_df = test[\"T_Malleability\"] * coherence_boundary < (test[\"T_spawn\"] + test[\"T_Redistribution\"])\n", "if not test[aux_df].empty:\n", " print(\"Coherence fails for inner times of malleability in \" + str(len(test[aux_df])) + \" cases\")\n", " display(test[aux_df])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "out_grouped_G.to_excel(\"resultG.xlsx\") \n", "out_grouped_M.to_excel(\"resultM.xlsx\") \n", "grouped_aggLAsynch.to_excel(\"AsynchIters.xlsx\")\n", "grouped_aggLSynch.to_excel(\"SynchIters.xlsx\")\n", "grouped_aggLNDyn.to_excel(\"StaticCoherence.xlsx\")\n", "grouped_aggLDyn.to_excel(\"DynCoherence.xlsx\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "dfL.columns" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "def create_group_boundary(rms_boundary, np_aux, ns_aux):\n", " tc_boundary = 0\n", " boundaries = None\n", " if rms_boundary != 0:\n", " # El porcentaje de tc_boundary se tiene en cuenta para eliminar aquellos\n", " # tiempos demasiado grandes en su malleability time respecto al más pequeño\n", " boundaries = get_np_ns_data(\"T_Malleability\", grouped_aggM, configurations_simple, np_aux, ns_aux)\n", " tc_boundary = min(boundaries)\n", " tc_boundary = tc_boundary + tc_boundary*rms_boundary\n", " return tc_boundary, boundaries\n", "\n", "# Aquellos grupos que tengán valores por encima del límite no se considerarán\n", "def check_groups_boundaries(dataLists, boundaries, tc_boundary):\n", " for index in range(len(boundaries)):\n", " if boundaries[index] > tc_boundary:\n", " dataLists[index] = float('infinity')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "def get_perc_differences(dataLists, boundaries, tc_boundary):\n", " perc = 1.05\n", " if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos\n", " check_groups_boundaries(dataLists, boundaries, tc_boundary) \n", " indexes = np.argsort(dataLists)\n", " \n", " best = -1\n", " bestMax = -1\n", " otherBest=[]\n", " for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en media/mediana\n", " if best == -1:\n", " best = index\n", " bestMax = dataLists[best] * perc\n", " elif dataLists[index] <= bestMax: # Media/Medianas i < Media/Mediana best\n", " otherBest.append(index)\n", " \n", " otherBest.insert(0,best)\n", " return otherBest\n", "\n", "def get_stat_differences(dataLists, df_Res, boundaries, tc_boundary):\n", " if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos\n", " check_groups_boundaries(dataLists, boundaries, tc_boundary) \n", " indexes = np.argsort(dataLists)\n", " \n", " best = -1\n", " otherBest=[] \n", " for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en mediana\n", " if dataLists[index] != float('infinity'):\n", " if best == -1:\n", " best = index\n", " elif not df_Res.iat[best,index]: # df_Res == False indicates 'index' and 'best' have the same mean/median\n", " otherBest.append(index)\n", " \n", " otherBest.insert(0,best)\n", " return otherBest" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "grouped_np = [\"T_total\"]\n", "separated_np = [\"T_Malleability\", \"T_Redistribution\", \"T_spawn\", \"T_SR\", \"T_AR\", \"Alpha\", \"Omega\", \"count\"]\n", "\n", "def get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux):\n", " dataLists=[]\n", " for config in used_config:\n", " if tipo in grouped_np:\n", " config.append((np_aux,ns_aux))\n", " elif tipo in separated_np:\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " \n", " if tuple(config) in data_aux.index:\n", " aux_value = data_aux.loc[tuple(config),tipo]\n", " if isinstance(aux_value, pd.Series):\n", " aux_value = aux_value.values[0]\n", " if aux_value == 0: #Values of zero indicate it was not performed\n", " aux_value = float('infinity')\n", " else: # This configuration is not present in the dataset\n", " aux_value = float('infinity')\n", " dataLists.append(aux_value)\n", " config.pop()\n", " if tipo in separated_np:\n", " config.pop()\n", " return dataLists\n", "\n", "def get_config_data(tipo, data_aux, config):\n", " dataLists=[]\n", " procsLists=[]\n", " for ns_aux in processes:\n", " for np_aux in processes:\n", " if np_aux != ns_aux:\n", " \n", " if tipo in grouped_np:\n", " config.append((np_aux,ns_aux))\n", " elif tipo in separated_np:\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " if tuple(config) in data_aux.index:\n", " procsLists.append((np_aux,ns_aux))\n", " aux_value = data_aux.loc[tuple(config),tipo]\n", " if isinstance(aux_value, pd.Series):\n", " aux_value = aux_value.values[0]\n", " if aux_value == 0: #Values of zero indicate it was not performed\n", " aux_value = float('infinity')\n", " else: # This configuration is not present in the dataset\n", " aux_value = float('infinity')\n", " dataLists.append(aux_value)\n", " config.pop()\n", " if tipo in separated_np:\n", " config.pop()\n", " return dataLists, procsLists\n", "\n", "def get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux):\n", " dataLists=[]\n", " if tipo in grouped_np:\n", " tuple_data = (np_aux, ns_aux)\n", " df_npns_aux = df_check.loc[(df_check['Groups']==tuple_data)]\n", " elif tipo in separated_np:\n", " df_npns_aux = df_check.loc[(df_check['NP']==np_aux)]\n", " df_npns_aux = df_npns_aux.loc[(df_npns_aux['NC']==ns_aux)]\n", " \n", " for config in used_config:\n", " df_config_aux = df_npns_aux\n", " for index in range(len(config)):\n", " aux_name = parameters_names[index]\n", " aux_value = config[index]\n", " df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]\n", " \n", " aux_value = list(df_config_aux[tipo])\n", " if len(aux_value) > 0:\n", " dataLists.append(aux_value)\n", " return dataLists\n", "\n", "def get_df_config_data(df_check, tipo, config):\n", " dataLists=[]\n", " df_config_aux = df_check\n", " for index in range(len(config)):\n", " aux_name = parameters_names[index]\n", " aux_value = config[index]\n", " df_config_aux = df_config_aux.loc[(df_config_aux[aux_name] == aux_value)]\n", " \n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " if tipo in grouped_np:\n", " tuple_data = (np_aux, ns_aux)\n", " df_aux = df_config_aux.loc[(df_config_aux['Groups']==tuple_data)]\n", " elif tipo in separated_np:\n", " df_aux = df_config_aux.loc[(df_config_aux['NP']==np_aux)]\n", " df_aux = df_aux.loc[(df_aux['NC']==ns_aux)]\n", " aux_value = list(df_aux[tipo])\n", " if len(aux_value) > 0:\n", " dataLists.append(aux_value)\n", " return dataLists\n", " \n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "def check_normality(df_check, tipo, used_config, fast=True):\n", " normality_array=[True] * (len(processes) * (len(processes)-1) * len(used_config))\n", " normality = True\n", " total=0\n", " i=-1\n", " #Comprobar para cada configuración si se sigue una distribución normal/gaussiana\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " i+=1\n", " dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n", " for data_aux in dataLists:\n", " st,p = stats.shapiro(data_aux) # Tendrían que ser al menos 20 datos y menos de 50\n", " if p < significance_value: # Reject H0\n", " if fast:\n", " return False\n", " normality_array[i] = False\n", " normality = False\n", " total+=1\n", " print(\"Se sigue una distribución guassiana: \" + str(normality) + \"\\nUn total de: \" + str(total) + \" no siguen una gaussiana\")\n", " print(normality_array)\n", " return normality\n", "\n", "def check_homoscedasticity(df_check, tipo, used_config, fast=True):\n", " homoscedasticity_array=[True] * (len(processes) * (len(processes)-1))\n", " homoscedasticity = True\n", " total=0\n", " i=-1\n", " #Comprobar para cada configuración es homoestatica\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " i+=1\n", " dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n", " st,p = stats.levene(*dataLists) # Tendrían que ser al menos 20 datos y menos de 50\n", " if p < significance_value: # Reject H0\n", " if fast:\n", " return False\n", " homoscedasticity_array[i] = False\n", " homoscedasticity = False\n", " total+=1\n", " print(\"Se sigue una distribución de datos Homocedastica: \" + str(homoscedasticity) + \"\\nUn total de: \" + str(total) + \" no siguen una homocedastica\")\n", " print(homoscedasticity_array)\n", " return homoscedasticity\n", "\n", "def compute_global_stat_difference(dataLists, parametric, np_aux, ns_aux):\n", " if parametric:\n", " st,p=stats.f_oneway(*dataLists)\n", " else:\n", " st,p=stats.kruskal(*dataLists)\n", " if p > significance_value:\n", " print(\"For NP \" + str(np_aux) + \" and \" + str(ns_aux) + \" is accepted H0\")\n", " return True # Equal values || Accept H0\n", " return False # Some groups are different || Reject H0\n", "\n", "def compute_global_posthoc(dataLists, parametric): #TODO Comprobar CDF de los grupos\n", " data_stats=[]\n", " data_stats2=[]\n", " ini=0\n", " end=len(dataLists)\n", " if parametric:\n", " df_aux = sp.posthoc_ttest(dataLists)\n", " df_Res = df_aux.copy()\n", " for i in range(ini,end):\n", " data_stats.append(np.mean(dataLists[i]))\n", " \n", " for j in range(ini,end):\n", " if df_Res.iat[i,j] < significance_value: # Different means || Reject H0\n", " df_Res.iat[i, j] = True\n", " else:\n", " df_Res.iat[i, j] = False\n", " else:\n", " df_aux = sp.posthoc_conover(dataLists)\n", " df_Res = df_aux.copy()\n", " for i in range(ini,end):\n", " data_stats.append(np.median(dataLists[i]))\n", " #data_stats2.append(stats.iqr(dataLists[i],axis=0))\n", " for j in range(ini,end):\n", " if df_Res.iat[i,j] < significance_value: # Different medians || Reject H0\n", " df_Res.iat[i, j] = True # Not equal medians\n", " else:\n", " df_Res.iat[i, j] = False # Equal medians\n", " #print(df_Res)\n", " #print(df_aux)\n", " #print(data_stats)\n", " #print(data_stats2)\n", " #aux_value = min(data_stats)\n", " #print(data_stats.index(aux_value))\n", " return df_Res, data_stats" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "def results_with_perc(tipo, data_aux, used_config, rms_boundary=0):\n", " results = []\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)\n", " \n", " #Get all values for particular config with these number of processes\n", " dataLists = get_np_ns_data(tipo, data_aux, used_config, np_aux, ns_aux)\n", "\n", " aux_data = get_perc_differences(dataLists, boundaries, tc_boundary)\n", " results.append(aux_data)\n", " return results\n", "\n", "def results_with_stats(tipo, df_check, used_config, rms_boundary=0):\n", " results = []\n", " use_parametric = check_normality(df_check, tipo, used_config)\n", " if use_parametric:\n", " use_parametric = check_homoscedasticity(df_check, tipo, used_config)\n", " print(\"Se usan tests parametricos: \"+str(use_parametric))\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " tc_boundary, boundaries = create_group_boundary(rms_boundary, np_aux, ns_aux)\n", " \n", " #Get all values for particular config with these number of processes\n", " dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n", " equal_set = compute_global_stat_difference(dataLists, use_parametric, np_aux, ns_aux)\n", " if equal_set:\n", " aux_data = list(range(len(used_config))) # All data is equal\n", " else:\n", " res_aux, times_aux = compute_global_posthoc(dataLists, use_parametric)\n", " aux_data = get_stat_differences(times_aux, res_aux, boundaries, tc_boundary)\n", " \n", " results.append(aux_data)\n", " \n", " return results" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "checked_type='T_total'\n", "use_perc = False\n", "select_first_winner = False\n", "prefer_first_winner = False\n", "rms_boundary=0 # Poner a 0 para perspectiva de app. Valor >0 y <1 para perspectiva de RMS\n", "if checked_type=='T_total':\n", " tipo=\"T_total\"\n", " if use_perc:\n", " data_aux = grouped_aggG\n", " else:\n", " data_aux = dfG\n", " used_config = configurations\n", "elif checked_type=='T_Malleability' or checked_type=='T_spawn' or checked_type=='T_SR' or checked_type=='T_AR' or checked_type=='T_Redistribution':\n", " tipo=checked_type\n", " \n", " if use_perc:\n", " data_aux = grouped_aggM\n", " else:\n", " data_aux = dfM\n", " if tipo == 'T_AR':\n", " data_aux = data_aux[(data_aux.ADR > 0)]\n", " elif tipo == 'T_SR':\n", " data_aux = data_aux[(data_aux.ADR == 0)]\n", " \n", " used_config = configurations_simple\n", " \n", "if use_perc:\n", " results = results_with_perc(tipo, data_aux, used_config, rms_boundary)\n", "else:\n", " results = results_with_stats(tipo, data_aux, used_config, rms_boundary)\n", " \n", "if not use_perc and tipo == 'T_AR': #FIXME!!!! No tiene en cuenta total de configuraciones sincronos\n", " for res_index in range(len(results)):\n", " for inner_index in range(len(results[res_index])):\n", " results[res_index][inner_index]+=4\n", "\n", "#Results is a 2 dimensional array. First dimension indicates lists of winners of a particulal number of processes (NP->NC). \n", "#Second dimension is an ordered preference of indexes in the array configurations.\n", "print(results)\n", "print(len(results))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Lista de indices de mayor a menor según el total de ocurrencias\n", "aux_array = []\n", "for data in results:\n", " aux_array+=data\n", "aux_keys, aux_counts = np.unique(aux_array, return_counts=True)\n", "aux_ordered_index=list(reversed(np.argsort(aux_counts)))\n", "\n", "#Lista de indices de mayor a menor según el total de ocurrencias del primero de cada grupo\n", "aux_array = [0] * len(results)\n", "for index in range(len(results)):\n", " aux_array[index] = results[index][0]\n", "aux_keys_best, aux_counts_best = np.unique(aux_array, return_counts = True)\n", "aux_ordered_best_index=list(reversed(np.argsort(aux_counts_best)))\n", "\n", "def heatmap_get_best(index, ordered_array, keys_array, counts_array, prefer_winner=False):\n", " valid_candidates_indexes = []\n", " prev_counts = -1\n", " for tested_index in ordered_array:\n", " if keys_array[tested_index] in results[index]:\n", " if counts_array[tested_index] >= prev_counts:\n", " prev_counts = counts_array[tested_index]\n", " valid_candidates_indexes.append(tested_index)\n", " else:\n", " break\n", " \n", " if prefer_winner: # Si esta activo, en caso de empate en ocurrencias se selecciona el menor tiempo\n", " for tested_index in results[index]:\n", " if tested_index in valid_candidates_indexes:\n", " return tested_index\n", " return min(valid_candidates_indexes) # En caso de empate se devuelve el que tiene menor valor (Suele ser la config más simple)\n", "\n", "i=0\n", "j=0\n", "used_aux=0\n", "heatmap=np.zeros((len(processes),len(processes))).astype(int)\n", "\n", "if select_first_winner:\n", " for i in range(len(processes)):\n", " for j in range(len(processes)):\n", " if i==j:\n", " heatmap[i][j]=-1\n", " used_aux+=1\n", " else:\n", " results_index = i*len(processes) + j - used_aux\n", " heatmap[i][j] = results[results_index][0]\n", "else:\n", " for i in range(len(processes)):\n", " for j in range(len(processes)):\n", " if i==j:\n", " heatmap[i][j]=-1\n", " used_aux+=1\n", " else:\n", " results_index = i*len(processes) + j - used_aux\n", " index = heatmap_get_best(results_index, aux_ordered_index, aux_keys, aux_counts, prefer_first_winner)\n", " heatmap[i][j]=aux_keys[index]\n", " #index = heatmap_get_best(results_index, aux_ordered_best_index, aux_keys_best, aux_counts_best, prefer_first_winner)\n", " #heatmap[i][j]=aux_keys_best[index]\n", "heatmap[-1][-1]=len(used_config)\n", "print(aux_keys)\n", "print(aux_counts)\n", "print(heatmap)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Adapta results a una cadena asegurando que cada cadena no se sale de su celda\n", "def get_heatmap_multiple_strings(results): #FIXME Deprecated\n", " results_str = []\n", " max_counts = 1\n", " max_per_line = 3\n", " for i in range(len(results)):\n", " results_str.append(list())\n", " count = len(results[i])\n", " results_aux = results[i]\n", " if count > max_counts:\n", " count = max_counts\n", " results_aux = results[i][:count]\n", " \n", " remainder = count%max_per_line\n", " if count <= max_per_line:\n", " aux_str = str(results_aux).replace('[','').replace(']','')\n", " results_str[i].append(aux_str)\n", " else:\n", " if remainder == 0:\n", " index = count//2\n", " else:\n", " index = count - ((remainder-1)*max_per_line + 1)\n", " aux_str = str(results_aux[:index]).replace('[','').replace(']','')\n", " results_str[i].append(aux_str)\n", " aux_str = str(results_aux[index:]).replace('[','').replace(']','')\n", " results_str[i].append(aux_str)\n", " return results_str\n", "\n", "def get_heatmap_strings(heatmap):\n", " results_str = []\n", " for i in range(len(processes)):\n", " for j in range(len(processes)):\n", " if i!=j:\n", " results_str.append(list())\n", " results_str[-1].append(heatmap[i][j])\n", " return results_str" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "#Crea un heatmap teniendo en cuenta los colores anteriores\n", "f=plt.figure(figsize=(24, 12))\n", "ax=f.add_subplot(111)\n", "\n", "myColors = (colors.to_rgba(\"white\"), \n", " colors.to_rgba(\"green\"), #BCOLS\n", " colors.to_rgba(\"darkgreen\"), #BP2PS\n", " #colors.to_rgba(\"blue\"), #BRMA1S\n", " #colors.to_rgba(\"royalblue\"), #BRMA2S\n", " colors.to_rgba(\"red\"), #MCOLS\n", " colors.to_rgba(\"darkred\"), #MP2PS\n", " #colors.to_rgba(\"mediumblue\"), #MRMA1S\n", " #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n", " #colors.to_rgba(\"blue\"), #BIntraCOLS\n", " #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n", " colors.to_rgba(\"mediumseagreen\"), #BCOLA\n", " colors.to_rgba(\"seagreen\"), #BCOLT\n", " colors.to_rgba(\"palegreen\"), #BP2PA\n", " colors.to_rgba(\"springgreen\"), #BP2PT\n", " #colors.to_rgba(\"purple\"), #BRMA1A\n", " #colors.to_rgba(\"darkviolet\"), #BRMA1T\n", " #colors.to_rgba(\"indigo\"), #BRMA2A\n", " #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n", " colors.to_rgba(\"indianred\"), #MCOLA \n", " colors.to_rgba(\"firebrick\"), #MCOLT\n", " colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n", " colors.to_rgba(\"saddlebrown\"), #MP2PT\n", " #colors.to_rgba(\"magenta\"), #MRMA1A\n", " #colors.to_rgba(\"violet\"), #MRMA1T\n", " #colors.to_rgba(\"deeppink\"), #MRMA2A\n", " #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n", " #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n", " #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n", " colors.to_rgba(\"white\"))\n", "cmap = LinearSegmentedColormap.from_list('Custom', myColors, len(myColors))\n", "\n", "im = ax.imshow(heatmap,cmap=cmap,interpolation='nearest')\n", "\n", "# Loop over data dimensions and create text annotations.\n", "used_aux=0\n", "results_str = get_heatmap_strings(heatmap)\n", "for i in range(len(processes)):\n", " for j in range(len(processes)):\n", " if i!=j:\n", " aux_color=\"white\"\n", " if 0 <= heatmap[i, j] <= 1 or 4 <= heatmap[i, j] <= 7: # El 1 puede necesitar texto en negro\n", " aux_color=\"black\"\n", " results_index = i*len(processes) +j-used_aux\n", " if len(results_str[results_index]) == 1:\n", " text = results_str[results_index][0]\n", " ax.text(j, i, text, ha=\"center\", va=\"center\", color=aux_color, fontsize=36)\n", " else:\n", " add_aux = 0.33\n", " for line in range(len(results_str[results_index])):\n", " i_range = i - 0.5 + add_aux\n", " ax.text(j, i_range, results_str[results_index][line],\n", " ha=\"center\", va=\"center\", color=aux_color, fontsize=36)\n", " add_aux+=0.33\n", " else:\n", " used_aux+=1\n", "\n", "ax.set_ylabel(\"NS\", fontsize=36)\n", "ax.set_xlabel(\"NT\", fontsize=36)\n", "\n", "ax.set_xticklabels(['']+processes, fontsize=36)\n", "ax.set_yticklabels(['']+processes, fontsize=36)\n", "\n", "\n", "labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',\n", " 'Merge - COLS (2)','Merge - P2PS (3)',\n", " 'Baseline - COLA (4)', 'Baseline - COLT (5)','Baseline - P2PA (6)','Baseline - P2PT (7)',\n", " 'Merge - COLA (8)','Merge - COLT (9)','Merge - P2PA (10)','Merge - P2PT (11)']\n", "\n", "#labelsMethods_aux = ['Baseline - COLS (0)', 'Baseline - P2PS (1)',\n", "# 'Baseline - RMA1S (2)', 'Baseline - RMA2S (3)',\n", "# 'Merge - COLS (4)','Merge - P2PS (5)',\n", "# 'Merge - RMA1S (6)', 'Merge - RMA2S (7)',\n", "# 'Baseline - COLA (8)', 'Baseline - COLT (9)','Baseline - P2PA (10)','Baseline - P2PT (11)',\n", "# 'Baseline - RMA1A (12)', 'Baseline - RMA1T (13)','Baseline - RMA2A (14)','Baseline - RMA2T (15)',\n", "# 'Merge - COLA (16)','Merge - COLT (17)','Merge - P2PA (18)','Merge - P2PT (19)',\n", "# 'Merge - RMA1A (20)','Merge - RMA1T (21)','Merge - RMA2A (22)','Merge - RMA2T (23)']\n", "\n", "colorbar=f.colorbar(im, ax=ax)\n", "tick_bar = []\n", "for i in range(len(used_config)):\n", " #tick_bar.append(0.41 + i*0.96) #Config de 24 valores\n", " tick_bar.append(0.37 + i*0.92) #Config de 12 valores\n", " #tick_bar.append(0.35 + i*0.89) #Config de 8 valores\n", "colorbar.set_ticks(tick_bar) \n", "colorbar.set_ticklabels(labelsMethods_aux)\n", "colorbar.ax.tick_params(labelsize=32)\n", "#\n", "\n", "f.tight_layout()\n", "print(\"Filename: Heatmap_\"+tipo+\".png\")\n", "f.savefig(\"Images/Heatmap_\"+tipo+\".jpeg\", format=\"jpeg\", dpi=300)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "aux_array = [] #Counts all\n", "for data in results:\n", " aux_array+=data\n", "aux_results, aux_counts = np.unique(aux_array, return_counts=True)\n", "print(aux_results)\n", "print(aux_counts)\n", "\n", "aux_array = [0] * len(results) # Counts ganador celda\n", "for index in range(len(results)):\n", " aux_array[index] = results[index][0]\n", "aux_results, aux_counts = np.unique(aux_array, return_counts = True)\n", "print(aux_results)\n", "print(aux_counts)\n" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "El siguiente código asume que para cada número de procesos padre/hijo existen valores en todas las configuraciones que se van a probar" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "def normalize_arrays(arrays, norm_array):\n", " nef normalize_arrays(arrays, norm_array):\n", " new_arrays = arrays.copy()\n", " for index in range(len(new_arrays)):\n", " new_arrays[index] = np.divide(norm_array, new_arrays[index])\n", " return new_arrays\n", "\n", "def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):\n", " labels_aux = []\n", " if used_direction == 's':\n", " for ns_aux in processes:\n", " for np_aux in processes:\n", " if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n", " new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n", " labels_aux.append(new_label)\n", " else:\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n", " new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n", " labels_aux.append(new_label)\n", " return labels_aux\n", "\n", "def reorder_data(plot_data, actual_order, expected_order):\n", " ordered_indexes = []\n", " len_order = len(actual_order)\n", " for index in range(len_order):\n", " actual_order[index] = str(actual_order[index]).replace(\" \", \"\")\n", " for index in range(len_order):\n", " ordered_indexes.append(actual_order.index(expected_order[index]))\n", "\n", " for index in range(len(plot_data)):\n", " old_array = plot_data[index]\n", " new_array = []\n", " for i in ordered_indexes:\n", " new_array.append(old_array[i])\n", " plot_data[index] = new_array\n", "\n", " return plot_dataew_arrays = arrays.copy()\n", " for index in range(len(new_arrays)):\n", " new_arrays[index] = np.divide(norm_array, new_arrays[index])\n", " return new_arrays\n", "\n", "def create_labels_lineplot(used_direction, user_condition=lambda a, b: True):\n", " labels_aux = []\n", " if used_direction == 's':\n", " for ns_aux in processes:\n", " for np_aux in processes:\n", " if used_direction=='s' and np_aux > ns_aux and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n", " new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n", " labels_aux.append(new_label)\n", " else:\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if ((used_direction=='e' and np_aux < ns_aux) or used_direction=='a') and np_aux != ns_aux and user_condition(np_aux, ns_aux):\n", " new_label = \"(\" + str(np_aux) + \",\" + str(ns_aux) + \")\"\n", " labels_aux.append(new_label)\n", " return labels_aux\n", "\n", "def reorder_data(plot_data, actual_order, expected_order):\n", " ordered_indexes = []\n", " len_order = len(actual_order)\n", " for index in range(len_order):\n", " actual_order[index] = str(actual_order[index]).replace(\" \", \"\")\n", " for index in range(len_order):\n", " ordered_indexes.append(actual_order.index(expected_order[index]))\n", "\n", " for index in range(len(plot_data)):\n", " old_array = plot_data[index]\n", " new_array = []\n", " for i in ordered_indexes:\n", " new_array.append(old_array[i])\n", " plot_data[index] = new_array\n", "\n", " return plot_data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "used_direction='s'\n", "test_parameter='T_Malleability' #Valores son \"alpha\" o \"omega\"\n", "\n", "if test_parameter == 'alpha':\n", " name_fig=\"Alpha_\"\n", " real_parameter='Alpha'\n", " name_legend = \"Valores de α\"\n", " normalize = True\n", " allow_all = False\n", " used_config = configurations_simple\n", " data_aux = grouped_aggM[grouped_aggM[real_parameter] > 0]\n", "elif test_parameter == 'T_Malleability':\n", " name_fig=\"Malleability_\"\n", " real_parameter='T_Malleability'\n", " name_legend = \"Time(s)\"\n", " normalize = False\n", " allow_all = True\n", " used_config = configurations_simple\n", " data_aux = grouped_aggM\n", "elif test_parameter == 'T_Redistribution':\n", " name_fig=\"Redistribution_\"\n", " real_parameter='T_Redistribution'\n", " name_legend = \"Tiempo(s)\"\n", " normalize = False\n", " allow_all = True\n", " used_config = configurations_simple\n", " data_aux = grouped_aggM\n", "elif test_parameter == 'omega':\n", " name_fig=\"Omega_\"\n", " real_parameter='Omega'\n", " name_legend = \"Values of ω\"\n", " normalize = True\n", " allow_all = False\n", " used_config = configurations\n", " data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]\n", "elif test_parameter == 'iters_count':\n", " name_fig=\"Iters_\"\n", " real_parameter='count'\n", " name_legend = \"Asynchronous iterations\"\n", " normalize = False\n", " allow_all = True\n", " used_config = configurations\n", " data_aux = grouped_aggLAsynch[grouped_aggLAsynch[real_parameter] > 0]\n", " \n", "if used_direction=='s':\n", " data_aux=data_aux.query('NP > NC')\n", " name_fig= name_fig+\"Shrink\"\n", "elif used_direction=='e':\n", " data_aux=data_aux.query('NP < NC')\n", " name_fig= name_fig+\"Expand\"\n", "elif used_direction=='a':\n", " name_fig= name_fig+\"All\" \n", "#data_aux=data_aux.query('NP == 160 or NC == 160')\n", "\n", "plot_data = []\n", "for config in used_config:\n", " if config[0] > 0 or allow_all:\n", " dataLists,procsLists = get_config_data(real_parameter, data_aux, config)\n", " dataLists = list(filter(lambda x: x != float('infinity'), dataLists))\n", " plot_data.append(dataLists)\n", " \n", "#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)\n", "labels_aux = create_labels_lineplot(used_direction)\n", "plot_data = reorder_data(plot_data, procsLists, labels_aux)\n", " \n", "#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n", "# 'Baseline - RMA1S', 'Baseline - RMA2S',\n", "# 'Merge - COLS','Merge - P2PS',\n", "# 'Merge - RMA1S', 'Merge - RMA2S',\n", "# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',\n", "# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',\n", "# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',\n", "# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'\n", "# ]\n", "labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n", " 'Merge - COLS','Merge - P2PS',\n", " 'Baseline - COLA', 'Baseline - COLT', 'Baseline - P2PA', 'Baseline - P2PT',\n", " 'Merge - COLA', 'Merge - COLT', 'Merge - P2PA', 'Merge - P2PT']\n", "colors_m = ( \n", " colors.to_rgba(\"green\"), #BCOLS\n", " colors.to_rgba(\"darkgreen\"), #BP2PS\n", " #colors.to_rgba(\"blue\"), #BRMA1S\n", " #colors.to_rgba(\"royalblue\"), #BRMA2S\n", " colors.to_rgba(\"red\"), #MCOLS\n", " colors.to_rgba(\"darkred\"), #MP2PS\n", " #colors.to_rgba(\"mediumblue\"), #MRMA1S\n", " #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n", " #colors.to_rgba(\"blue\"), #BIntraCOLS\n", " #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n", " colors.to_rgba(\"mediumseagreen\"), #BCOLA\n", " colors.to_rgba(\"seagreen\"), #BCOLT\n", " colors.to_rgba(\"palegreen\"), #BP2PA\n", " colors.to_rgba(\"springgreen\"), #BP2PT\n", " #colors.to_rgba(\"purple\"), #BRMA1A\n", " #colors.to_rgba(\"darkviolet\"), #BRMA1T\n", " #colors.to_rgba(\"indigo\"), #BRMA2A\n", " #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n", " colors.to_rgba(\"indianred\"), #MCOLA \n", " colors.to_rgba(\"firebrick\"), #MCOLT\n", " colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n", " colors.to_rgba(\"saddlebrown\"), #MP2PT\n", " #colors.to_rgba(\"magenta\"), #MRMA1A\n", " #colors.to_rgba(\"violet\"), #MRMA1T\n", " #colors.to_rgba(\"deeppink\"), #MRMA2A\n", " #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n", " #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n", " #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n", " )\n", "\n", "f=plt.figure(figsize=(22, 14))\n", "ax=f.add_subplot(111)\n", "x = np.arange(len(labels_aux))\n", "for index in range(len(plot_data)):\n", " array_aux = plot_data[index]\n", " plot_index = index\n", " if real_parameter == 'Alpha' or real_parameter == 'Omega' or real_parameter == 'count': #FIXME This line is a lie...\n", " plot_index = 4 + index #FIXME This line is a lie...\n", " print(array_aux)\n", " ax.plot(x, array_aux, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n", " marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[plot_index])\n", "\n", "ax.set_xlabel(\"(NS,NT)\", fontsize=36)\n", "ax.set_ylabel(name_legend, fontsize=36)\n", "if normalize:\n", " ax.axhline(y=1, color='black', linestyle='--')\n", "plt.xticks(x, labels_aux,rotation=90)\n", "ax.tick_params(axis='both', which='major', labelsize=36)\n", "ax.tick_params(axis='both', which='minor', labelsize=36)\n", "plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n", " \n", "f.tight_layout()\n", "f.savefig(\"Images/LinePlot_\"+name_fig+\".png\", format=\"png\")" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "Gráfica de lineas para generar tiempos del grupo G." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "used_direction='e'\n", "test_parameter='T_total'\n", "intranode=False\n", "\n", "if test_parameter == 'T_total':\n", " name_fig=\"Ttotal\"\n", " real_parameter='T_total'\n", " name_legend = \"Time(s)\"\n", " used_config = configurations\n", " data_aux = grouped_aggG\n", " #data_aux = data_aux[data_aux.index.isin(df1.index)]\n", " \n", "if used_direction=='s':\n", " data_aux_cmp=grouped_aggM.reset_index().query('NP > NC')\n", " name_fig= name_fig+\"Shrink\"\n", "elif used_direction=='e':\n", " data_aux_cmp=grouped_aggM.reset_index().query('NP < NC')\n", " name_fig= name_fig+\"Expand\"\n", "elif used_direction=='a':\n", " data_aux_cmp=grouped_aggM.reset_index()\n", " name_fig= name_fig+\"All\"\n", " \n", "if intranode:\n", " data_aux_cmp = data_aux_cmp.query('NP <= 20 and NC <= 20')\n", "#else:\n", " #data_aux_cmp = data_aux_cmp.query('NP > 20 and NC > 20')\n", "#data_aux_cmp = data_aux_cmp.query('NP == 160 or NC == 160')\n", "\n", "if used_direction!='a' or True:\n", " pruebaG = data_aux.reset_index()\n", " pruebaG = pruebaG.loc[pruebaG.index.intersection(data_aux_cmp.index)]\n", " data_aux = data_aux[(data_aux.T_total.isin(pruebaG.T_total))]\n", "\n", "plot_data = []\n", "for config in used_config:\n", " #if config[0] == 0:\n", " dataLists,procsLists = get_config_data(real_parameter, data_aux, config)\n", " dataLists = list(filter(lambda x: x != float('infinity'), dataLists))\n", " plot_data.append(dataLists)\n", "for array_aux in plot_data:\n", " print(array_aux)\n", " \n", "\n", "#labels_aux = create_labels_lineplot(used_direction, lambda a, b: a == 160 or b == 160)\n", "labels_aux = create_labels_lineplot(used_direction)\n", "plot_data = reorder_data(plot_data, procsLists, labels_aux)\n", "plot_data_normalized = normalize_arrays(plot_data[1:], plot_data[0])\n", "name_legend=\"SpeedUp over Baseline\"\n", "\n", "#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']\n", "labelsMethods_aux = ['Baseline - AllS', 'Baseline - P2PS',\n", " 'Merge - AllS','Merge - P2PS',\n", " 'Baseline - AllA', 'Baseline - AllT','Baseline - P2PA','Baseline - P2PT',\n", " 'Merge - AllA','Merge - AllT','Merge - P2PA','Merge - P2PT']\n", "#labelsMethods_aux = ['Baseline - COLS', 'Baseline - P2PS',\n", "# 'Baseline - RMA1S', 'Baseline - RMA2S',\n", "# 'Merge - COLS','Merge - P2PS',\n", "# 'Merge - RMA1S', 'Merge - RMA2S',\n", "# 'Baseline - COLA', 'Baseline - COLT','Baseline - P2PA','Baseline - P2PT',\n", "# 'Baseline - RMA1A', 'Baseline - RMA1T','Baseline - RMA2A','Baseline - RMA2T',\n", "# 'Merge - COLA','Merge - COLT','Merge - P2PA','Merge - P2PT',\n", "# 'Merge - RMA1A','Merge - RMA1T','Merge - RMA2A','Merge - RMA2T'\n", "# ]\n", "\n", "colors_m = ( \n", " colors.to_rgba(\"green\"), #BCOLS\n", " colors.to_rgba(\"darkgreen\"), #BP2PS\n", " #colors.to_rgba(\"blue\"), #BRMA1S\n", " #colors.to_rgba(\"royalblue\"), #BRMA2S\n", " colors.to_rgba(\"red\"), #MCOLS\n", " colors.to_rgba(\"darkred\"), #MP2PS\n", " #colors.to_rgba(\"mediumblue\"), #MRMA1S\n", " #colors.to_rgba(\"mediumslateblue\"), #MRMA2S\n", " #colors.to_rgba(\"blue\"), #BIntraCOLS\n", " #colors.to_rgba(\"royalblue\"), #BIntraP2PS\n", " colors.to_rgba(\"mediumseagreen\"), #BCOLA\n", " colors.to_rgba(\"seagreen\"), #BCOLT\n", " colors.to_rgba(\"palegreen\"), #BP2PA\n", " colors.to_rgba(\"springgreen\"), #BP2PT\n", " #colors.to_rgba(\"purple\"), #BRMA1A\n", " #colors.to_rgba(\"darkviolet\"), #BRMA1T\n", " #colors.to_rgba(\"indigo\"), #BRMA2A\n", " #colors.to_rgba(\"rebeccapurple\"), #BRMA2T\n", " colors.to_rgba(\"indianred\"), #MCOLA \n", " colors.to_rgba(\"firebrick\"), #MCOLT\n", " colors.to_rgba(\"darkgoldenrod\"), #MP2PA\n", " colors.to_rgba(\"saddlebrown\"), #MP2PT\n", " #colors.to_rgba(\"magenta\"), #MRMA1A\n", " #colors.to_rgba(\"violet\"), #MRMA1T\n", " #colors.to_rgba(\"deeppink\"), #MRMA2A\n", " #colors.to_rgba(\"mediumvioletred\"), #MRMA2T\n", " #colors.to_rgba(\"mediumblue\"), #BIntraCOLA\n", " #colors.to_rgba(\"mediumslateblue\"), #BIntraP2PA\n", " )\n", "\n", "\n", "f=plt.figure(figsize=(22, 14))\n", "ax=f.add_subplot(111)\n", "ax2 = ax.twinx()\n", "x = np.arange(len(labels_aux))\n", "for index in range(len(plot_data_normalized)):\n", " array_aux = plot_data_normalized[index]\n", " index= index+1\n", " ax.plot(x, array_aux, color=colors_m[index%len(colors_m)], linestyle=linestyle_m[index%len(linestyle_m)], \\\n", " marker=markers_m[index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n", "ax2.plot(x, plot_data[0], color='black', linestyle=linestyle_m[0], \\\n", " marker=markers_m[0], markersize=18, label=labelsMethods_aux[0])\n", "ax.axhline(y=1, color='black', linestyle='--')\n", "\n", "ax.set_xlabel(\"(NP,NC)\", fontsize=36)\n", "ax.set_ylabel(name_legend, fontsize=36)\n", "ax.tick_params(axis='both', which='both', labelsize=36)\n", "ax.set_xticks(x)\n", "ax.set_xticklabels(labels_aux, rotation=90)\n", "#ax.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n", "\n", "ax2.set_ylabel('Baseline Time(s)', fontsize=36)\n", "ax2.tick_params(axis='y', which='both', labelsize=36)\n", "#ax2.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n", "\n", "f.legend(bbox_to_anchor=(0.5, 0.98), fontsize=26,ncol=2,framealpha=0.8)\n", "\n", " \n", "f.tight_layout()\n", "f.savefig(\"Images/LinePlot_\"+name_fig+\".png\", format=\"png\")" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "El siguiente generá una imagen en 3d de T_total para cada una de las diferentes configuraciones." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def generate_3d_image(config, name):\n", " fig, ax = plt.subplots(1, 1, subplot_kw={'projection': '3d'}, figsize=(15, 15))\n", "\n", " Z = [None] * len(processes)\n", " X, Y = np.meshgrid(processes, processes)\n", " for i in range(len(processes)):\n", " np_aux = processes[i]\n", " Z[i] = [0] * len(processes)\n", " Z[i][i] = grouped_aggLSynch.loc[np_aux, 'T_iter'] * 1000\n", " for j in range(len(processes)):\n", " if i!=j:\n", " ns_aux = processes[j]\n", " config.append((np_aux,ns_aux))\n", " aux = grouped_aggG.loc[tuple(config),'T_total']\n", " config.pop()\n", " \n", " Z[i][j] = aux.values[0]\n", " #Z[i][j] = Z[i][j] / Z[i][i]\n", " #Z[i][i] = 1\n", "\n", " Z = np.array(Z)\n", "\n", " ax.plot_surface(X, Y, Z, rstride=1, cstride=1,\n", " cmap='viridis', edgecolor='none')\n", " ax.view_init(15, 25)\n", " ax.set_xlabel(\"NC\", fontsize=16)\n", " ax.set_ylabel(\"NP\", fontsize=16)\n", " ax.set_zlabel(\"Normalized time\", fontsize=16)\n", " ax.set_title(name, fontsize=10)\n", " plt.show()\n", " \n", "for index in range(len(configurations)):\n", " used_config = configurations[index]\n", " generate_3d_image(used_config,str(index))" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "El siguiente código es computar la coherencia de T_malleability respecto a los tiempos internos de maleabilidad (Coherency1)\n", "y por otro lado de T_malleability respecto a iteraciones asíncronas más los pasos síncronos." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "test=dfM[(dfM.Asynch_Iters > 0)]\n", "\n", "# Seguro con barriers en Malleability\n", "test[\"Resize_Coherency\"] = test[\"T_Malleability\"] >= (test[\"T_spawn\"] + test[\"T_SR\"] + test[\"T_AR\"])\n", "# Seguro al usar Rigid para iteraciones\n", "test[\"Resize_Coherency2\"] = test[\"T_Malleability\"] >= 0\n", "\n", "for index in range(len(test)):\n", " time_malleability_aux = test[\"T_Malleability\"].values[index]\n", " time_synch_aux = test[\"T_SR\"].values[index]\n", " time_spawn_aux = test[\"T_spawn\"].values[index]\n", " is_asynch_spawn = (test[\"Spawn_Strategy\"].values[index] % 2 == 0)\n", " \n", " total_asynch_iters = int(test[\"Asynch_Iters\"].values[index])\n", " asynch_iters = test[\"T_iter\"].values[index][-total_asynch_iters:]\n", " time_iters_aux = np.sum(asynch_iters[:])\n", " \n", " sum_times = time_synch_aux + is_asynch_spawn * time_spawn_aux + time_iters_aux\n", " \n", " if time_malleability_aux < sum_times:\n", " real_index = test.index.values[index]\n", " test.at[real_index, \"Resize_Coherency2\"] = False\n", "test[(test.Resize_Coherency == False)].query('Redistribution_Method > 1')" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "El siguiente código es para utilizar Dask. Una versión que paraleliza una serie de tareas de Pandas.\n", "Tras llamar a compute se realizan todas las tareas que se han pedido." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import dask.dataframe as dd\n", "ddf = dd.from_pandas(dfL[(dfL.Asynch_Iters == False)], npartitions=10)\n", "group = ddf.groupby('NP')['T_iter']\n", "grouped_aggLSynch = group.agg(['mean'])\n", "grouped_aggLSynch = grouped_aggLSynch.rename(columns={'mean':'T_iter'}) \n", "grouped_aggLSynch = grouped_aggLSynch.compute()" ] }, { "cell_type": "raw", "metadata": {}, "source": [ "Las siguientes dos celdas de código son para comprobar los Empirical Comulative distribution functions(ecdf).\n", "Si dos ecdf crean una intersección el test de Conover-Iman no es fiable.\n", "#TODO No esta terminado. Falta obtener cual ha sido el metodo ganador antes de llamar a \"check_specific_ecdf\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def test_ecdf(data1, data2):\n", " diff_aux = data1[0] - data2[0]\n", " cmp_f = (lambda a, b: a-b < 0) # Se esperan siempre diferencias positivas, si una es negativa se cruzan los ecdf's\n", " if diff_aux < 0:\n", " cmp_f = (lambda a, b: a-b > 0) # Se esperan siempre diferencias negativas, si una es positivas se cruzan los ecdf's\n", " \n", " for index_value in range(1,len(data1)):\n", " if cmp_f(data1[index_value], data2[index_value]):\n", " return False\n", " return True\n", "\n", "def plot_ecdf(data1, data2, ecdf1, ecdf2, title):\n", " f=plt.figure(figsize=(22, 14))\n", " ax=f.add_subplot(111)\n", "\n", " ax.plot(data1, ecdf1, color=colors_m[0], linestyle=linestyle_m[0], \\\n", " marker=markers_m[0], markersize=18)\n", " ax.plot(data2, ecdf2, color=colors_m[2], linestyle=linestyle_m[2], \\\n", " marker=markers_m[2], markersize=18)\n", "\n", " ax.set_xlabel(\"Values\", fontsize=36)\n", " ax.set_ylabel(\"Bins\", fontsize=36)\n", " ax.tick_params(axis='both', which='major', labelsize=36)\n", " ax.tick_params(axis='both', which='minor', labelsize=36)\n", " ax.set_title(title, fontsize=40)\n", " \n", " f.tight_layout()\n", "\n", "def check_specific_ecdf(np_aux, ns_aux, configuration_index):\n", " df_check = dfM\n", " tipo = 'T_Redistribution'\n", " used_config = configurations_simple\n", " dataLists = get_df_np_ns_data(df_check, tipo, used_config, np_aux, ns_aux)\n", " ecdfLists = []\n", " for index_i in range(len(dataLists)):\n", " dataLists[index_i].sort()\n", " ecdfLists.append( np.arange(1, len(dataLists[index_i])+1)/float(len(dataLists[index_i])) )\n", "\n", " index_i = configuration_index\n", " data_i = dataLists[index_i]\n", " ecdf_i = ecdfLists[index_i]\n", " for index_j in range(len(dataLists)):\n", " if index_i != index_j:\n", " data_j = dataLists[index_j]\n", " ecdf_j = ecdfLists[index_j]\n", " res_aux = test_ecdf(data_i, data_j)\n", " if not res_aux:\n", " aux_title = \"NP=\"+str(np_aux)+\"/NC=\"+str(ns_aux)+\" -- Comparing C\"+str(index_i) + \" vs C\"+str(index_j)\n", " plot_ecdf(data_i, data_j, ecdf_i, ecdf_j, aux_title)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux and not (np_aux==2 and ns_aux==20):\n", " check_winner_ecdf(np_aux, ns_aux, 9)\n", "\n", " \n", " " ] }, { "cell_type": "raw", "metadata": {}, "source": [ "================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "a = 5\n", "b = 3\n", "c = False\n", "d = a + b * c\n", "print(d)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#Ethernet\n", "b1_aux = [82.747868, 83.191989, 95.520019, 87.435987, 90.843995, 150.356100]\n", "b2_aux = [75.238174, 74.380054, 74.755995, 42.656109, 21.588040, 17.843997]\n", "m1_aux = [74.654167, 74.357901, 74.351350, 43.599915, 21.637509, 15.128712]\n", "m2_aux = [74.353249, 74.359214, 74.356160, 43.874266, 21.511082, 14.969010]\n", "\n", "b3_aux = [105.128014, 110.004008, 126.552019, 116.312400, 95.752019, 151.023994]\n", "b4_aux = [83.021885, 77.632630, 75.396010, 43.076039, 24.028075, 19.556047]\n", "m3_aux = [118.275992, 83.027866, 81.008479, 46.432212, 24.247949, 17.725083]\n", "m4_aux = [119.286457, 84.205993, 80.741585, 47.144632, 24.206617, 17.738496]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#Infiniband\n", "b1_aux = [64.669525, 35.171971, 38.916010, 47.456630, 56.288048, 119.428020]\n", "b2_aux = [36.538361, 15.536046, 13.396083, 9.652013, 5.772058, 5.615009]\n", "m1_aux = [61.664380, 18.400559, 19.112526, 22.155880, 11.712381, 30.775627]\n", "m2_aux = [33.428639, 13.905561, 14.691367, 7.363081, 6.629037, 12.150872]\n", "\n", "b3_aux = [91.368664, 60.648074, 53.663981, 49.152031, 64.752057, 118.243807]\n", "b4_aux = [84.941260, 34.039990, 26.008021, 12.298989, 7.916004, 5.736054]\n", "m3_aux = [105.839726, 26.822071, 23.834452, 12.876862, 9.063136, 7.007535]\n", "m4_aux = [75.412319, 25.566336, 22.129483, 12.491161, 7.903744, 6.534291]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "\n", "plot_data_e = [b1_aux, m1_aux, b2_aux, m2_aux] #Expand\n", "plot_data_s = [b3_aux, m3_aux, b4_aux, m4_aux] #Shrink\n", "\n", "labels_aux_e = create_labels_lineplot('e', lambda a, b: a == 160 or b == 160)\n", "labels_aux_s = create_labels_lineplot('s', lambda a, b: a == 160 or b == 160)\n", "#labels_aux = create_labels_lineplot(used_direction)\n", "\n", "labelsMethods_aux = ['Baseline - AllS', 'Merge - AllS',\n", " 'Baseline - AllA','Merge - AllA']\n", "\n", "#labelsMethods_aux = ['Baseline - All', 'Baseline - P2P','Merge - All','Merge - P2P']\n", "\n", "f, (axe, axs)=plt.subplots(1,2,figsize=(22, 14))\n", "x = np.arange(len(labels_aux_e))\n", "for index in range(len(plot_data_e)):\n", " array_aux_e = plot_data_e[index]\n", " array_aux_s = plot_data_s[index]\n", " plot_index = index\n", " if index > 0:\n", " plot_index = 2**plot_index\n", " print(array_aux_e)\n", " axe.plot(x, array_aux_e, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n", " marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n", " axs.plot(x, array_aux_s, color=colors_m[plot_index%len(colors_m)], linestyle=linestyle_m[plot_index%len(linestyle_m)], \\\n", " marker=markers_m[plot_index%len(markers_m)], markersize=18, label=labelsMethods_aux[index])\n", "\n", "axe.set_xlabel(\"(NP,NC)\", fontsize=36)\n", "axe.set_ylabel(\"Time(s)\", fontsize=36)\n", "axe.set_xticks(x)\n", "axe.set_xticklabels(labels_aux_e, rotation=90)\n", "axe.tick_params(axis='both', which='major', labelsize=36)\n", "axe.tick_params(axis='both', which='minor', labelsize=36)\n", "\n", "axs.set_xlabel(\"(NP,NC)\", fontsize=36)\n", "axs.set_ylabel(\"Time(s)\", fontsize=36)\n", "axs.set_xticks(x)\n", "axs.set_xticklabels(labels_aux_s, rotation=90)\n", "axs.tick_params(axis='both', which='major', labelsize=36)\n", "axs.tick_params(axis='both', which='minor', labelsize=36)\n", "\n", "plt.legend(loc='best', fontsize=30,ncol=2,framealpha=0.8)\n", " \n", "f.tight_layout()\n", "f.savefig(\"Images/LinePlot_100Gb.png\", format=\"png\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "%matplotlib inline\n", "import pandas as pd\n", "from pandas import DataFrame, Series\n", "import numpy as np\n", "\n", "\n", "new_df = pd.read_pickle( \"dataBasic.pkl\" )\n", "new_df.columns" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "grouped_aggG[\"T_total\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "100000 ** 2" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.3" } }, "nbformat": 4, "nbformat_minor": 4 }