{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "import pandas as pd\n", "from pandas import DataFrame, Series\n", "import numpy as np\n", "import math\n", "\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "import matplotlib.patches as mpatches\n", "import matplotlib.colors as colors\n", "from matplotlib.legend_handler import HandlerLine2D, HandlerTuple\n", "from matplotlib.colors import LinearSegmentedColormap\n", "from scipy import stats\n", "import scikit_posthocs as sp\n", "import sys\n", "\n", "from mpl_toolkits.mplot3d import axes3d" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "AllName=\"dataG.pkl\"\n", "ResizesName=\"dataM.pkl\"\n", "ItersName=\"dataL.pkl\"\n", "matrixIt_Total=\"data_L_Total.csv\"\n", "n_cores=20\n", "repet = 5 #CAMBIAR EL NUMERO SEGUN NUMERO DE EJECUCIONES POR CONFIG\n", "\n", "p_value = 0.05\n", "processes = [2,10,20,40,80,120,160]\n", "\n", "positions = [321, 322, 323, 324, 325]\n", "positions_small = [221, 222, 223, 224]\n", "\n", "labels = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n", " '(10,1)', '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n", " '(20,1)', '(20,10)', '(20,40)', '(20,80)', '(20,120)','(20,160)',\n", " '(40,1)', '(40,10)', '(40,20)', '(40,80)', '(40,120)','(40,160)',\n", " '(80,1)', '(80,10)', '(80,20)', '(80,40)', '(80,120)','(80,160)',\n", " '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)','(120,160)',\n", " '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n", "\n", "labelsExpand = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',\n", " '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',\n", " '(20,40)', '(20,80)', '(20,120)','(20,160)',\n", " '(40,80)', '(40,120)','(40,160)',\n", " '(80,120)','(80,160)',\n", " '(120,160)']\n", "labelsShrink = ['(10,1)', \n", " '(20,1)', '(20,10)',\n", " '(40,1)', '(40,10)', '(40,20)',\n", " '(80,1)', '(80,10)', '(80,20)', '(80,40)',\n", " '(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)',\n", " '(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']\n", "\n", "# WORST BEST\n", "labels_dist = ['null', 'SpreadFit', 'CompactFit']\n", " #0 #1 #2 #3\n", "labelsMethods = ['Baseline', 'Baseline single','Baseline - Asynchronous','Baseline single - Asynchronous',\n", " 'Merge','Merge single','Merge - Asynchronous','Merge single - Asynchronous']\n", " #4 #5 #6 #7\n", " \n", "colors_spawn = ['green','springgreen','blue','darkblue','red','darkred','darkgoldenrod','olive','violet']\n", "linestyle_spawn = ['-', '--', '-.', ':']\n", "markers_spawn = ['.','v','s','p', 'h','d','X','P','^']\n", "\n", "OrMult_patch = mpatches.Patch(hatch='', facecolor='green', label='Baseline')\n", "OrSing_patch = mpatches.Patch(hatch='', facecolor='springgreen', label='Baseline single')\n", "OrPthMult_patch = mpatches.Patch(hatch='//', facecolor='blue', label='Baseline - Asyncrhonous')\n", "OrPthSing_patch = mpatches.Patch(hatch='\\\\', facecolor='darkblue', label='Baseline single - Asyncrhonous')\n", "MergeMult_patch = mpatches.Patch(hatch='||', facecolor='red', label='Merge')\n", "MergeSing_patch = mpatches.Patch(hatch='...', facecolor='darkred', label='Merge single')\n", "MergePthMult_patch = mpatches.Patch(hatch='xx', facecolor='yellow', label='Merge - Asyncrhonous')\n", "MergePthSing_patch = mpatches.Patch(hatch='++', facecolor='olive', label='Merge single - Asyncrhonous')\n", "\n", "handles_spawn = [OrMult_patch,OrSing_patch,OrPthMult_patch,OrPthSing_patch,MergeMult_patch,MergeSing_patch,MergePthMult_patch,MergePthSing_patch]" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "dfG = pd.read_pickle( AllName )\n", "\n", "dfG['ADR'] = round((dfG['ADR'] / dfG['DR']) * 100,1)\n", "dfG['SDR'] = round((dfG['SDR'] / dfG['DR']) * 100,1)\n", " \n", "out_group = dfG.groupby(['Groups', 'ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_total']\n", "group = dfG.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','Groups'])['T_total']\n", "\n", "grouped_aggG = group.agg(['median'])\n", "grouped_aggG.rename(columns={'median':'T_total'}, inplace=True) \n", "\n", "out_grouped_G = out_group.agg(['median'])\n", "out_grouped_G.rename(columns={'median':'T_total'}, inplace=True) " ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_7488/535752050.py:7: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n", " out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n", "/tmp/ipykernel_7488/535752050.py:8: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n", " group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n" ] } ], "source": [ "dfM = pd.read_pickle( ResizesName )\n", "\n", "dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)\n", "dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)\n", "dfM['T_Malleability'] = dfM['T_spawn'] + dfM['T_SR'] + dfM['T_AR']\n", " \n", "out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n", "group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n", "\n", "grouped_aggM = group.agg(['median'])\n", "grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)\n", "\n", "out_grouped_M = out_group.agg(['median'])\n", "out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_7488/998390630.py:6: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n", " group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']\n", "/tmp/ipykernel_7488/998390630.py:7: FutureWarning: ['T_stages'] did not aggregate successfully. If any error is raised this will raise in a future version of pandas. Drop these columns/ops to avoid this warning.\n", " grouped_aggLAsynch = group.agg(['mean'])\n" ] } ], "source": [ "dfL = pd.read_pickle( ItersName )\n", "\n", "dfL['ADR'] = round((dfL['ADR'] / dfL['DR']) * 100,1)\n", "dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)\n", " \n", "group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']\n", "grouped_aggLAsynch = group.agg(['mean'])\n", "grouped_aggLAsynch.columns = grouped_aggLAsynch.columns.get_level_values(0)\n", "\n", "group = dfL.groupby('NP')['T_iter']\n", "grouped_aggLSynch = group.agg(['mean'])\n", "grouped_aggLSynch.rename(columns={'mean':'T_iter'}, inplace=True) " ] }, { "cell_type": "code", "execution_count": 81, "metadata": {}, "outputs": [], "source": [ "from bt_scheme import PartialSolution, BacktrackingSolver\n", "def elegirConf(parameters):\n", " class StatePS(PartialSolution):\n", " def __init__(self, config):\n", " self.config= config\n", " self.n= len(config) #Indica el valor a añadir\n", "\n", " def is_solution(self):\n", " return self.n == len(parameters)\n", "\n", " def get_solution(self):\n", " return tuple(self.config)\n", "\n", " def successors(self):\n", " array = parameters[self.n]\n", " for parameter_value in array: #Test all values of the next parameter\n", " self.config.append(parameter_value)\n", " yield StatePS(self.config)\n", " self.config.pop()\n", "\n", " initialPs= StatePS([])\n", " return BacktrackingSolver().solve(initialPs)\n", "\n", "\n", "def obtenerConfs(parameters):\n", " soluciones=[]\n", " for solucion in elegirConf(parameters):\n", " soluciones.append(solucion)\n", " return soluciones\n", "\n", "def modifyToUsable(parameters, len_parameters, configuration):\n", " usable_configuration = []\n", " for i in range(len(parameters)):\n", " if len_parameters[i] > 1:\n", " aux = (parameters[i][0], configuration[i])\n", " else:\n", " aux = (configuration[i])\n", " usable_configuration.append(aux)\n", " \n", " return usable_configuration\n", "\n", "def CheckConfExists(configuration, dataSet, type_conf='global'):\n", " remove = 0\n", " config = list(configuration)\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " \n", " if type_conf == 'global':\n", " config.append((np_aux, ns_aux))\n", " elif type_conf == 'malleability':\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " \n", " if tuple(config) in dataSet.index: \n", " remove = 1\n", " elif remove != 1:\n", " remove = -1\n", " config.pop()\n", " \n", " if type_conf == 'malleability':\n", " config.pop()\n", " if remove == 1:\n", " return True\n", " return False" ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[0, 0, 0, 1], [0, 0, 1, 1], [0, 1, 0, 1], [0, 1, 1, 1], [96.6, 0, 0, 1], [96.6, 0, 0, 2], [96.6, 0, 1, 1], [96.6, 0, 1, 2], [96.6, 1, 0, 1], [96.6, 1, 0, 2], [96.6, 1, 1, 1], [96.6, 1, 1, 2]]\n", "[[0, (0, 0), (0, 0), (1, 1)], [0, (0, 0), (0, 1), (1, 1)], [0, (0, 1), (0, 0), (1, 1)], [0, (0, 1), (0, 1), (1, 1)], [96.6, (0, 0), (0, 0), (1, 1)], [96.6, (0, 0), (0, 0), (1, 2)], [96.6, (0, 0), (0, 1), (1, 1)], [96.6, (0, 0), (0, 1), (1, 2)], [96.6, (0, 1), (0, 0), (1, 1)], [96.6, (0, 1), (0, 0), (1, 2)], [96.6, (0, 1), (0, 1), (1, 1)], [96.6, (0, 1), (0, 1), (1, 2)]]\n", "12\n" ] } ], "source": [ "sp_method = [0,1]\n", "rd_method = [0,1]\n", "rd_strat = [1,2]\n", "adr = [0,96.6]\n", "parameters = [adr, sp_method, rd_method, rd_strat]\n", "len_parameters = [1,2,2,2]\n", "configurations_aux = obtenerConfs(parameters)\n", "configurations = []\n", "configurations_simple = []\n", "for index in range(len(configurations_aux)):\n", " aux_conf = modifyToUsable(parameters, len_parameters, configurations_aux[index])\n", " if CheckConfExists(aux_conf, grouped_aggG):\n", " configurations.append(aux_conf)\n", " if CheckConfExists(configurations_aux[index], grouped_aggM, 'malleability'):\n", " configurations_simple.append(list(configurations_aux[index]))\n", "\n", "print(configurations_simple)\n", "print(configurations)\n", "print(len(configurations))" ] }, { "cell_type": "code", "execution_count": 86, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ALPHA already exists\n" ] } ], "source": [ "#ALPHA COMPUTATION\n", "def compute_alpha(config_a, config_b):\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " config_a.append(np_aux)\n", " config_a.append(ns_aux)\n", " config_b.append(np_aux)\n", " config_b.append(ns_aux)\n", " grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']\n", " config_a.pop()\n", " config_a.pop()\n", " config_b.pop()\n", " config_b.pop()\n", " \n", " \n", " config_a.insert(0,ns_aux)\n", " config_a.insert(0,np_aux)\n", " config_b.insert(0,ns_aux)\n", " config_b.insert(0,np_aux)\n", " out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']\n", " config_a.pop(0)\n", " config_a.pop(0)\n", " config_b.pop(0)\n", " config_b.pop(0)\n", "\n", "if not ('Alpha' in grouped_aggM.columns):\n", " for config_a in configurations_simple:\n", " for config_b in configurations_simple:\n", " #FIXME/TODO If the last index of configurations is not the strategy or different from pthreads may fail this computation\n", " if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:\n", " compute_alpha(config_a, config_b)\n", "else:\n", " print(\"ALPHA already exists\")" ] }, { "cell_type": "code", "execution_count": 87, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "OMEGA already exists\n" ] } ], "source": [ "#OMEGA COMPUTATION\n", "def compute_omega(config):\n", " for np_aux in processes:\n", " for ns_aux in processes:\n", " if np_aux != ns_aux:\n", " config.append(np_aux)\n", " config.append(ns_aux)\n", " grouped_aggLAsynch.loc[tuple(config),'Omega'] = grouped_aggLAsynch.loc[tuple(config),'T_iter'] / grouped_aggLSynch.loc[np_aux,'T_iter']\n", " config.pop()\n", " config.pop()\n", "\n", "if not ('Omega' in grouped_aggLAsynch.columns):\n", " for config in configurations_simple:\n", " if config[0] != 0:\n", " compute_omega(config)\n", "else:\n", " print(\"OMEGA already exists\")" ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [], "source": [ "out_grouped_G.to_excel(\"resultG.xlsx\") \n", "out_grouped_M.to_excel(\"resultM.xlsx\") \n", "grouped_aggLAsynch.to_excel(\"AsynchIters.xlsx\")" ] }, { "cell_type": "code", "execution_count": 84, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | T_iter | \n", "Omega | \n", "
---|---|---|---|---|---|---|---|
ADR | \n", "Spawn_Method | \n", "Redistribution_Method | \n", "Redistribution_Strategy | \n", "NP | \n", "NC | \n", "\n", " | \n", " |
96.6 | \n", "0.0 | \n", "0.0 | \n", "1.0 | \n", "2 | \n", "10.0 | \n", "1.075937 | \n", "1.789700 | \n", "
20.0 | \n", "1.207753 | \n", "2.008962 | \n", "|||||
40.0 | \n", "1.256070 | \n", "2.089332 | \n", "|||||
80.0 | \n", "1.169895 | \n", "1.945990 | \n", "|||||
120.0 | \n", "1.225243 | \n", "2.038054 | \n", "|||||
... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "|
1.0 | \n", "1.0 | \n", "2.0 | \n", "160 | \n", "10.0 | \n", "0.896296 | \n", "5.711017 | \n", "|
20.0 | \n", "0.708625 | \n", "4.515218 | \n", "|||||
40.0 | \n", "0.533604 | \n", "3.400016 | \n", "|||||
80.0 | \n", "0.482244 | \n", "3.072765 | \n", "|||||
120.0 | \n", "0.468189 | \n", "2.983204 | \n", "
336 rows × 2 columns
\n", "