Skip to content

GitLab

Commit c531b6ca authored by Iker Martín Álvarez's avatar Iker Martín Álvarez
Browse files

WIP. The analyser is a huge monster to be converted into a python code...

parent 44cfed8a
Hide whitespace changes
Inline Side-by-side
Analysis/new_analyser.ipynb
View file @ c531b6ca
......@@ -11,6 +11,7 @@
"from pandas import DataFrame, Series\n",
"import numpy as np\n",
"import math\n",
"\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.patches as mpatches\n",
......@@ -26,7 +27,7 @@
},
{
"cell_type": "code",
"execution_count": 130,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
......@@ -89,7 +90,7 @@
},
{
"cell_type": "code",
"execution_count": 107,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
......@@ -105,22 +106,21 @@
"grouped_aggG.rename(columns={'median':'T_total'}, inplace=True) \n",
"\n",
"out_grouped_G = out_group.agg(['median'])\n",
"out_grouped_G.rename(columns={'median':'T_total'}, inplace=True) \n",
"out_grouped_G.to_excel(\"resultG.xlsx\") "
"out_grouped_G.rename(columns={'median':'T_total'}, inplace=True) "
]
},
{
"cell_type": "code",
"execution_count": 108,
"execution_count": 43,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_4155/1340609544.py:7: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
"/tmp/ipykernel_7488/535752050.py:7: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
" out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n",
"/tmp/ipykernel_4155/1340609544.py:8: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
"/tmp/ipykernel_7488/535752050.py:8: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
" group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n"
]
}
......@@ -139,489 +139,43 @@
"grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)\n",
"\n",
"out_grouped_M = out_group.agg(['median'])\n",
"out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)\n",
"out_grouped_M.to_excel(\"resultM.xlsx\") "
"out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)"
]
},
{
"cell_type": "code",
"execution_count": 133,
"metadata": {},
"outputs": [],
"source": [
"dfL = pd.read_pickle( ItersName )\n",
"\n",
"#dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)\n",
"#dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)\n",
"#dfM['T_Malleability'] = dfM['T_spawn'] + dfM['T_SR'] + dfM['T_AR']\n",
" \n",
"#out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n",
"#group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']\n",
"\n",
"#grouped_aggM = group.agg(['median'])\n",
"#grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)\n",
"\n",
"#out_grouped_M = out_group.agg(['median'])\n",
"#out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)\n",
"#out_grouped_M.to_excel(\"resultM.xlsx\") "
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [],
"source": [
"out_data = dfG.loc[(dfG.Spawn_Method==(0,1)) & (dfG.ADR==0) & (dfG.Redistribution_Method==(0,0)) & (dfG.Redistribution_Strategy==(1,1)) & (dfG.Groups==(10,80))]\n",
"out_data.to_excel(\"review2.xlsx\") "
]
},
{
"cell_type": "code",
"execution_count": 138,
"execution_count": 75,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>NP</th>\n",
" <th>NC</th>\n",
" <th>Total_Stages</th>\n",
" <th>Granularity</th>\n",
" <th>SDR</th>\n",
" <th>ADR</th>\n",
" <th>DR</th>\n",
" <th>Redistribution_Method</th>\n",
" <th>Redistribution_Strategy</th>\n",
" <th>Spawn_Method</th>\n",
" <th>Spawn_Strategy</th>\n",
" <th>Is_Dynamic</th>\n",
" <th>FactorS</th>\n",
" <th>Dist</th>\n",
" <th>Stage_Types</th>\n",
" <th>Stage_Times</th>\n",
" <th>Stage_Bytes</th>\n",
" <th>Asynch_Iters</th>\n",
" <th>T_iter</th>\n",
" <th>T_stages</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>420500</th>\n",
" <td>160</td>\n",
" <td>40.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.016612</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>2.234976</td>\n",
" <td>(0.037385, 0.118837, 0.086114, 1.792483)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>421500</th>\n",
" <td>160</td>\n",
" <td>40.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.016612</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>2.416995</td>\n",
" <td>(0.02672, 0.125521, 0.081292, 1.784105)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>422500</th>\n",
" <td>160</td>\n",
" <td>40.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.016612</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>2.187999</td>\n",
" <td>(0.131866, 0.032969, 0.051701, 1.742342)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>423500</th>\n",
" <td>160</td>\n",
" <td>40.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.016612</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>2.244867</td>\n",
" <td>(0.010719, 0.119629, 0.07511, 1.789791)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>424500</th>\n",
" <td>160</td>\n",
" <td>40.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.016612</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>2.259979</td>\n",
" <td>(0.034214, 0.061322, 0.063906, 1.861178)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2510500</th>\n",
" <td>80</td>\n",
" <td>160.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.030397</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>0.250689</td>\n",
" <td>(0.01873, 0.084494, 0.001038, 0.143586)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2511500</th>\n",
" <td>80</td>\n",
" <td>160.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.030397</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>0.281567</td>\n",
" <td>(0.018751, 0.039149, 0.01087, 0.212796)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2512500</th>\n",
" <td>80</td>\n",
" <td>160.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.030397</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>0.313766</td>\n",
" <td>(0.018743, 0.080215, 0.018336, 0.191758)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2513500</th>\n",
" <td>80</td>\n",
" <td>160.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.030397</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>0.398377</td>\n",
" <td>(0.018734, 0.054583, 0.03555, 0.289509)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2514500</th>\n",
" <td>80</td>\n",
" <td>160.0</td>\n",
" <td>4</td>\n",
" <td>100000</td>\n",
" <td>134228039.0</td>\n",
" <td>3.813655e+09</td>\n",
" <td>3.947884e+09</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>False</td>\n",
" <td>0.030397</td>\n",
" <td>[2, 2]</td>\n",
" <td>(0, 6, 6, 4)</td>\n",
" <td>(0.608531, 0, 0, 0)</td>\n",
" <td>(0, 8, 8, 33176880)</td>\n",
" <td>True</td>\n",
" <td>0.509275</td>\n",
" <td>(0.018742, 0.086675, 0.018152, 0.382491)</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1398 rows × 20 columns</p>\n",
"</div>"
],
"text/plain": [
" NP NC Total_Stages Granularity SDR ADR \\\n",
"420500 160 40.0 4 100000 134228039.0 3.813655e+09 \n",
"421500 160 40.0 4 100000 134228039.0 3.813655e+09 \n",
"422500 160 40.0 4 100000 134228039.0 3.813655e+09 \n",
"423500 160 40.0 4 100000 134228039.0 3.813655e+09 \n",
"424500 160 40.0 4 100000 134228039.0 3.813655e+09 \n",
"... ... ... ... ... ... ... \n",
"2510500 80 160.0 4 100000 134228039.0 3.813655e+09 \n",
"2511500 80 160.0 4 100000 134228039.0 3.813655e+09 \n",
"2512500 80 160.0 4 100000 134228039.0 3.813655e+09 \n",
"2513500 80 160.0 4 100000 134228039.0 3.813655e+09 \n",
"2514500 80 160.0 4 100000 134228039.0 3.813655e+09 \n",
"\n",
" DR Redistribution_Method Redistribution_Strategy \\\n",
"420500 3.947884e+09 1.0 1.0 \n",
"421500 3.947884e+09 1.0 1.0 \n",
"422500 3.947884e+09 1.0 1.0 \n",
"423500 3.947884e+09 1.0 1.0 \n",
"424500 3.947884e+09 1.0 1.0 \n",
"... ... ... ... \n",
"2510500 3.947884e+09 1.0 1.0 \n",
"2511500 3.947884e+09 1.0 1.0 \n",
"2512500 3.947884e+09 1.0 1.0 \n",
"2513500 3.947884e+09 1.0 1.0 \n",
"2514500 3.947884e+09 1.0 1.0 \n",
"\n",
" Spawn_Method Spawn_Strategy Is_Dynamic FactorS Dist \\\n",
"420500 0.0 1.0 False 0.016612 [2, 2] \n",
"421500 0.0 1.0 False 0.016612 [2, 2] \n",
"422500 0.0 1.0 False 0.016612 [2, 2] \n",
"423500 0.0 1.0 False 0.016612 [2, 2] \n",
"424500 0.0 1.0 False 0.016612 [2, 2] \n",
"... ... ... ... ... ... \n",
"2510500 1.0 1.0 False 0.030397 [2, 2] \n",
"2511500 1.0 1.0 False 0.030397 [2, 2] \n",
"2512500 1.0 1.0 False 0.030397 [2, 2] \n",
"2513500 1.0 1.0 False 0.030397 [2, 2] \n",
"2514500 1.0 1.0 False 0.030397 [2, 2] \n",
"\n",
" Stage_Types Stage_Times Stage_Bytes Asynch_Iters \\\n",
"420500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"421500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"422500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"423500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"424500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"... ... ... ... ... \n",
"2510500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"2511500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"2512500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"2513500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"2514500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True \n",
"\n",
" T_iter T_stages \n",
"420500 2.234976 (0.037385, 0.118837, 0.086114, 1.792483) \n",
"421500 2.416995 (0.02672, 0.125521, 0.081292, 1.784105) \n",
"422500 2.187999 (0.131866, 0.032969, 0.051701, 1.742342) \n",
"423500 2.244867 (0.010719, 0.119629, 0.07511, 1.789791) \n",
"424500 2.259979 (0.034214, 0.061322, 0.063906, 1.861178) \n",
"... ... ... \n",
"2510500 0.250689 (0.01873, 0.084494, 0.001038, 0.143586) \n",
"2511500 0.281567 (0.018751, 0.039149, 0.01087, 0.212796) \n",
"2512500 0.313766 (0.018743, 0.080215, 0.018336, 0.191758) \n",
"2513500 0.398377 (0.018734, 0.054583, 0.03555, 0.289509) \n",
"2514500 0.509275 (0.018742, 0.086675, 0.018152, 0.382491) \n",
"\n",
"[1398 rows x 20 columns]"
]
},
"execution_count": 138,
"metadata": {},
"output_type": "execute_result"
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_7488/998390630.py:6: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
" group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']\n",
"/tmp/ipykernel_7488/998390630.py:7: FutureWarning: ['T_stages'] did not aggregate successfully. If any error is raised this will raise in a future version of pandas. Drop these columns/ops to avoid this warning.\n",
" grouped_aggLAsynch = group.agg(['mean'])\n"
]
}
],
"source": [
"dfG"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"used_direction='s'\n",
"test_parameter='T_total'\n",
" \n",
"if used_direction=='s':\n",
" df_aux=grouped_aggG.query('NP > NS')\n",
" used_labels=labelsShrink\n",
" name_fig=\"Shrink\"\n",
" np_aux = [10, 20,20, 40,40,40, 80,80,80,80, 120,120,120,120,120]\n",
" nc_aux = [1, 1,10, 1,10,20, 1,10,20,40, 1,10,20,40,80]\n",
"elif used_direction=='e':\n",
" df_aux=grouped_aggM.query('NP < NS')\n",
" used_labels=labelsExpand\n",
" name_fig=\"Expand\"\n",
" np_aux = [1,1,1,1,1, 10,10,10,10, 20,20,20, 40,40, 80 ]\n",
" nc_aux = [10,20,40,80,120, 20,40,80,120, 40,80,120, 80,120, 120]\n",
"elif used_direction=='a':\n",
" df_aux=grouped_aggM\n",
" used_labels=labels\n",
" name_fig=\"All\"\n",
" np_aux = [1,1,1,1,1, 10,10,10,10,10, 20,20,20,20,20, 40,40,40,40,40, 80,80,80,80,80, 120,120,120,120,120]\n",
" nc_aux = [10,20,40,80,120, 1,20,40,80,120, 1,10,40,80,120, 1,10,20,80,120, 1,10,20,40,120, 1,10,20,40,80]\n",
" \n",
"x = np.arange(len(used_labels))\n",
"handles = []\n",
"dfL = pd.read_pickle( ItersName )\n",
"\n",
"f=plt.figure(figsize=(20, 12))\n",
"#ax=f.add_subplot(111)\n",
"ax = plt.axes(projection='3d')\n",
"ax.azim = -60\n",
"ax.dist = 10\n",
"ax.elev = 10\n",
"ax.set_xlabel(\"NP\", fontsize=20)\n",
"ax.set_ylabel(\"NC\", fontsize=20)\n",
"ax.set_zlabel(\"Alpha\", fontsize=20)\n",
"ax.tick_params(axis='both', which='major', labelsize=24)\n",
"ax.tick_params(axis='both', which='minor', labelsize=22)\n",
"dfL['ADR'] = round((dfL['ADR'] / dfL['DR']) * 100,1)\n",
"dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)\n",
" \n",
"group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']\n",
"grouped_aggLAsynch = group.agg(['mean'])\n",
"grouped_aggLAsynch.columns = grouped_aggLAsynch.columns.get_level_values(0)\n",
"\n",
"for cst_aux in [1,3]:\n",
" df_aux2 = df_aux.query('Cst == @cst_aux')\n",
" for css_aux in [0,1]:\n",
" array_aux = df_aux2.query('Css == @css_aux')['alpha'].values\n",
" ax.plot3D(np_aux, nc_aux, array_aux, colors_spawn[cst_aux*2 + css_aux])\n",
" \n",
" handles.append(handles_spawn[cst_aux*2 + css_aux])\n",
" \n",
"#ax.set_zlim(0,4)\n",
"plt.legend(handles=handles, loc='best', fontsize=20,ncol=2,framealpha=1)\n",
" \n",
"f.tight_layout()\n",
"f.savefig(\"Images/Spawn/3dPlot_\"+name_fig+'_'+test_parameter+\".png\", format=\"png\")"
"group = dfL.groupby('NP')['T_iter']\n",
"grouped_aggLSynch = group.agg(['mean'])\n",
"grouped_aggLSynch.rename(columns={'mean':'T_iter'}, inplace=True) "
]
},
{
"cell_type": "code",
"execution_count": 87,
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
......@@ -694,7 +248,7 @@
},
{
"cell_type": "code",
"execution_count": 88,
"execution_count": 82,
"metadata": {},
"outputs": [
{
......@@ -729,6 +283,260 @@
"print(len(configurations))"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ALPHA already exists\n"
]
}
],
"source": [
"#ALPHA COMPUTATION\n",
"def compute_alpha(config_a, config_b):\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" config_a.append(np_aux)\n",
" config_a.append(ns_aux)\n",
" config_b.append(np_aux)\n",
" config_b.append(ns_aux)\n",
" grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']\n",
" config_a.pop()\n",
" config_a.pop()\n",
" config_b.pop()\n",
" config_b.pop()\n",
" \n",
" \n",
" config_a.insert(0,ns_aux)\n",
" config_a.insert(0,np_aux)\n",
" config_b.insert(0,ns_aux)\n",
" config_b.insert(0,np_aux)\n",
" out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']\n",
" config_a.pop(0)\n",
" config_a.pop(0)\n",
" config_b.pop(0)\n",
" config_b.pop(0)\n",
"\n",
"if not ('Alpha' in grouped_aggM.columns):\n",
" for config_a in configurations_simple:\n",
" for config_b in configurations_simple:\n",
" #FIXME/TODO If the last index of configurations is not the strategy or different from pthreads may fail this computation\n",
" if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:\n",
" compute_alpha(config_a, config_b)\n",
"else:\n",
" print(\"ALPHA already exists\")"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"OMEGA already exists\n"
]
}
],
"source": [
"#OMEGA COMPUTATION\n",
"def compute_omega(config):\n",
" for np_aux in processes:\n",
" for ns_aux in processes:\n",
" if np_aux != ns_aux:\n",
" config.append(np_aux)\n",
" config.append(ns_aux)\n",
" grouped_aggLAsynch.loc[tuple(config),'Omega'] = grouped_aggLAsynch.loc[tuple(config),'T_iter'] / grouped_aggLSynch.loc[np_aux,'T_iter']\n",
" config.pop()\n",
" config.pop()\n",
"\n",
"if not ('Omega' in grouped_aggLAsynch.columns):\n",
" for config in configurations_simple:\n",
" if config[0] != 0:\n",
" compute_omega(config)\n",
"else:\n",
" print(\"OMEGA already exists\")"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {},
"outputs": [],
"source": [
"out_grouped_G.to_excel(\"resultG.xlsx\") \n",
"out_grouped_M.to_excel(\"resultM.xlsx\") \n",
"grouped_aggLAsynch.to_excel(\"AsynchIters.xlsx\")"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>T_iter</th>\n",
" <th>Omega</th>\n",
" </tr>\n",
" <tr>\n",
" <th>ADR</th>\n",
" <th>Spawn_Method</th>\n",
" <th>Redistribution_Method</th>\n",
" <th>Redistribution_Strategy</th>\n",
" <th>NP</th>\n",
" <th>NC</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"11\" valign=\"top\">96.6</th>\n",
" <th rowspan=\"5\" valign=\"top\">0.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">0.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">1.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">2</th>\n",
" <th>10.0</th>\n",
" <td>1.075937</td>\n",
" <td>1.789700</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20.0</th>\n",
" <td>1.207753</td>\n",
" <td>2.008962</td>\n",
" </tr>\n",
" <tr>\n",
" <th>40.0</th>\n",
" <td>1.256070</td>\n",
" <td>2.089332</td>\n",
" </tr>\n",
" <tr>\n",
" <th>80.0</th>\n",
" <td>1.169895</td>\n",
" <td>1.945990</td>\n",
" </tr>\n",
" <tr>\n",
" <th>120.0</th>\n",
" <td>1.225243</td>\n",
" <td>2.038054</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <th>...</th>\n",
" <th>...</th>\n",
" <th>...</th>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"5\" valign=\"top\">1.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">1.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">2.0</th>\n",
" <th rowspan=\"5\" valign=\"top\">160</th>\n",
" <th>10.0</th>\n",
" <td>0.896296</td>\n",
" <td>5.711017</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20.0</th>\n",
" <td>0.708625</td>\n",
" <td>4.515218</td>\n",
" </tr>\n",
" <tr>\n",
" <th>40.0</th>\n",
" <td>0.533604</td>\n",
" <td>3.400016</td>\n",
" </tr>\n",
" <tr>\n",
" <th>80.0</th>\n",
" <td>0.482244</td>\n",
" <td>3.072765</td>\n",
" </tr>\n",
" <tr>\n",
" <th>120.0</th>\n",
" <td>0.468189</td>\n",
" <td>2.983204</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>336 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" T_iter \\\n",
"ADR Spawn_Method Redistribution_Method Redistribution_Strategy NP NC \n",
"96.6 0.0 0.0 1.0 2 10.0 1.075937 \n",
" 20.0 1.207753 \n",
" 40.0 1.256070 \n",
" 80.0 1.169895 \n",
" 120.0 1.225243 \n",
"... ... \n",
" 1.0 1.0 2.0 160 10.0 0.896296 \n",
" 20.0 0.708625 \n",
" 40.0 0.533604 \n",
" 80.0 0.482244 \n",
" 120.0 0.468189 \n",
"\n",
" Omega \n",
"ADR Spawn_Method Redistribution_Method Redistribution_Strategy NP NC \n",
"96.6 0.0 0.0 1.0 2 10.0 1.789700 \n",
" 20.0 2.008962 \n",
" 40.0 2.089332 \n",
" 80.0 1.945990 \n",
" 120.0 2.038054 \n",
"... ... \n",
" 1.0 1.0 2.0 160 10.0 5.711017 \n",
" 20.0 4.515218 \n",
" 40.0 3.400016 \n",
" 80.0 3.072765 \n",
" 120.0 2.983204 \n",
"\n",
"[336 rows x 2 columns]"
]
},
"execution_count": 84,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"grouped_aggLAsynch"
]
},
{
"cell_type": "code",
"execution_count": 113,
......@@ -1073,29 +881,6 @@
"================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================"
]
},
{
"cell_type": "code",
"execution_count": 253,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[0, 1, 100, 3, 4]\n",
"[0, 1, 100, 3, 4]\n"
]
}
],
"source": [
"a=[0,1,2,3,4]\n",
"b=a\n",
"count=3\n",
"a[2]=100\n",
"print(a)\n",
"print(b)"
]
},
{
"cell_type": "code",
"execution_count": 114,
......@@ -1142,6 +927,80 @@
"plt.show()"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"El siguiente código es para utilizar Dask. Una versión que paraleliza una serie de tareas de Pandas.\n",
"Tras llamar a compute se realizan todas las tareas que se han pedido."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import dask.dataframe as dd\n",
"ddf = dd.from_pandas(dfL[(dfL.Asynch_Iters == False)], npartitions=10)\n",
"group = ddf.groupby('NP')['T_iter']\n",
"grouped_aggLSynch = group.agg(['mean'])\n",
"grouped_aggLSynch = grouped_aggLSynch.rename(columns={'mean':'T_iter'}) \n",
"grouped_aggLSynch = grouped_aggLSynch.compute()"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": [
"================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"True\n",
"False\n",
"(3, 5, 6)\n"
]
}
],
"source": [
"a = (4, 3, 5, 6)\n",
"b = (4, 3, 5, 6)\n",
"d = (4, 3, 5, 7)\n",
"c = (3, 4, 5, 6)\n",
"\n",
"print(a[1:]==b[1:])\n",
"print(a[1:]==c[1:])\n",
"print(a[1:])"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[3]\n"
]
}
],
"source": [
"a = [2, 3]\n",
"a.pop(0)\n",
"print(a)"
]
},
{
"cell_type": "code",
"execution_count": null,
......
%% Cell type:code id: tags:
``` python
%matplotlib inline
import pandas as pd
from pandas import DataFrame, Series
import numpy as np
import math
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.colors as colors
from matplotlib.legend_handler import HandlerLine2D, HandlerTuple
from matplotlib.colors import LinearSegmentedColormap
from scipy import stats
import scikit_posthocs as sp
import sys
from mpl_toolkits.mplot3d import axes3d
```
%% Cell type:code id: tags:
``` python
AllName="dataG.pkl"
ResizesName="dataM.pkl"
ItersName="dataL.pkl"
matrixIt_Total="data_L_Total.csv"
n_cores=20
repet = 5 #CAMBIAR EL NUMERO SEGUN NUMERO DE EJECUCIONES POR CONFIG
p_value = 0.05
processes = [2,10,20,40,80,120,160]
positions = [321, 322, 323, 324, 325]
positions_small = [221, 222, 223, 224]
labels = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',
'(10,1)', '(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',
'(20,1)', '(20,10)', '(20,40)', '(20,80)', '(20,120)','(20,160)',
'(40,1)', '(40,10)', '(40,20)', '(40,80)', '(40,120)','(40,160)',
'(80,1)', '(80,10)', '(80,20)', '(80,40)', '(80,120)','(80,160)',
'(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)','(120,160)',
'(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']
labelsExpand = ['(1,10)', '(1,20)', '(1,40)', '(1,80)', '(1,120)','(1,160)',
'(10,20)', '(10,40)', '(10,80)', '(10,120)','(10,160)',
'(20,40)', '(20,80)', '(20,120)','(20,160)',
'(40,80)', '(40,120)','(40,160)',
'(80,120)','(80,160)',
'(120,160)']
labelsShrink = ['(10,1)',
'(20,1)', '(20,10)',
'(40,1)', '(40,10)', '(40,20)',
'(80,1)', '(80,10)', '(80,20)', '(80,40)',
'(120,1)','(120,10)', '(120,20)','(120,40)','(120,80)',
'(160,1)','(160,10)', '(160,20)','(160,40)','(160,80)','(160,120)']
# WORST BEST
labels_dist = ['null', 'SpreadFit', 'CompactFit']
#0 #1 #2 #3
labelsMethods = ['Baseline', 'Baseline single','Baseline - Asynchronous','Baseline single - Asynchronous',
'Merge','Merge single','Merge - Asynchronous','Merge single - Asynchronous']
#4 #5 #6 #7
colors_spawn = ['green','springgreen','blue','darkblue','red','darkred','darkgoldenrod','olive','violet']
linestyle_spawn = ['-', '--', '-.', ':']
markers_spawn = ['.','v','s','p', 'h','d','X','P','^']
OrMult_patch = mpatches.Patch(hatch='', facecolor='green', label='Baseline')
OrSing_patch = mpatches.Patch(hatch='', facecolor='springgreen', label='Baseline single')
OrPthMult_patch = mpatches.Patch(hatch='//', facecolor='blue', label='Baseline - Asyncrhonous')
OrPthSing_patch = mpatches.Patch(hatch='\\', facecolor='darkblue', label='Baseline single - Asyncrhonous')
MergeMult_patch = mpatches.Patch(hatch='||', facecolor='red', label='Merge')
MergeSing_patch = mpatches.Patch(hatch='...', facecolor='darkred', label='Merge single')
MergePthMult_patch = mpatches.Patch(hatch='xx', facecolor='yellow', label='Merge - Asyncrhonous')
MergePthSing_patch = mpatches.Patch(hatch='++', facecolor='olive', label='Merge single - Asyncrhonous')
handles_spawn = [OrMult_patch,OrSing_patch,OrPthMult_patch,OrPthSing_patch,MergeMult_patch,MergeSing_patch,MergePthMult_patch,MergePthSing_patch]
```
%% Cell type:code id: tags:
``` python
dfG = pd.read_pickle( AllName )
dfG['ADR'] = round((dfG['ADR'] / dfG['DR']) * 100,1)
dfG['SDR'] = round((dfG['SDR'] / dfG['DR']) * 100,1)
out_group = dfG.groupby(['Groups', 'ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_total']
group = dfG.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','Groups'])['T_total']
grouped_aggG = group.agg(['median'])
grouped_aggG.rename(columns={'median':'T_total'}, inplace=True)
out_grouped_G = out_group.agg(['median'])
out_grouped_G.rename(columns={'median':'T_total'}, inplace=True)
out_grouped_G.to_excel("resultG.xlsx")
```
%% Cell type:code id: tags:
``` python
dfM = pd.read_pickle( ResizesName )
dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)
dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)
dfM['T_Malleability'] = dfM['T_spawn'] + dfM['T_SR'] + dfM['T_AR']
out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
grouped_aggM = group.agg(['median'])
grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)
out_grouped_M = out_group.agg(['median'])
out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)
out_grouped_M.to_excel("resultM.xlsx")
```
%%%% Output: stream
/tmp/ipykernel_4155/1340609544.py:7: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
/tmp/ipykernel_7488/535752050.py:7: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
/tmp/ipykernel_4155/1340609544.py:8: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
/tmp/ipykernel_7488/535752050.py:8: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
%% Cell type:code id: tags:
``` python
dfL = pd.read_pickle( ItersName )
#dfM['ADR'] = round((dfM['ADR'] / dfM['DR']) * 100,1)
#dfM['SDR'] = round((dfM['SDR'] / dfM['DR']) * 100,1)
#dfM['T_Malleability'] = dfM['T_spawn'] + dfM['T_SR'] + dfM['T_AR']
#out_group = dfM.groupby(['NP','NC','ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
#group = dfM.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_Malleability','T_spawn','T_spawn_real','T_SR','T_AR']
dfL['ADR'] = round((dfL['ADR'] / dfL['DR']) * 100,1)
dfL['SDR'] = round((dfL['SDR'] / dfL['DR']) * 100,1)
#grouped_aggM = group.agg(['median'])
#grouped_aggM.columns = grouped_aggM.columns.get_level_values(0)
#out_grouped_M = out_group.agg(['median'])
#out_grouped_M.columns = out_grouped_M.columns.get_level_values(0)
#out_grouped_M.to_excel("resultM.xlsx")
```
%% Cell type:code id: tags:
``` python
out_data = dfG.loc[(dfG.Spawn_Method==(0,1)) & (dfG.ADR==0) & (dfG.Redistribution_Method==(0,0)) & (dfG.Redistribution_Strategy==(1,1)) & (dfG.Groups==(10,80))]
out_data.to_excel("review2.xlsx")
```
%% Cell type:code id: tags:
``` python
dfG
group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']
grouped_aggLAsynch = group.agg(['mean'])
grouped_aggLAsynch.columns = grouped_aggLAsynch.columns.get_level_values(0)
group = dfL.groupby('NP')['T_iter']
grouped_aggLSynch = group.agg(['mean'])
grouped_aggLSynch.rename(columns={'mean':'T_iter'}, inplace=True)
```
%%%% Output: execute_result
NP NC Total_Stages Granularity SDR ADR \
420500 160 40.0 4 100000 134228039.0 3.813655e+09
421500 160 40.0 4 100000 134228039.0 3.813655e+09
422500 160 40.0 4 100000 134228039.0 3.813655e+09
423500 160 40.0 4 100000 134228039.0 3.813655e+09
424500 160 40.0 4 100000 134228039.0 3.813655e+09
... ... ... ... ... ... ...
2510500 80 160.0 4 100000 134228039.0 3.813655e+09
2511500 80 160.0 4 100000 134228039.0 3.813655e+09
2512500 80 160.0 4 100000 134228039.0 3.813655e+09
2513500 80 160.0 4 100000 134228039.0 3.813655e+09
2514500 80 160.0 4 100000 134228039.0 3.813655e+09
DR Redistribution_Method Redistribution_Strategy \
420500 3.947884e+09 1.0 1.0
421500 3.947884e+09 1.0 1.0
422500 3.947884e+09 1.0 1.0
423500 3.947884e+09 1.0 1.0
424500 3.947884e+09 1.0 1.0
... ... ... ...
2510500 3.947884e+09 1.0 1.0
2511500 3.947884e+09 1.0 1.0
2512500 3.947884e+09 1.0 1.0
2513500 3.947884e+09 1.0 1.0
2514500 3.947884e+09 1.0 1.0
Spawn_Method Spawn_Strategy Is_Dynamic FactorS Dist \
420500 0.0 1.0 False 0.016612 [2, 2]
421500 0.0 1.0 False 0.016612 [2, 2]
422500 0.0 1.0 False 0.016612 [2, 2]
423500 0.0 1.0 False 0.016612 [2, 2]
424500 0.0 1.0 False 0.016612 [2, 2]
... ... ... ... ... ...
2510500 1.0 1.0 False 0.030397 [2, 2]
2511500 1.0 1.0 False 0.030397 [2, 2]
2512500 1.0 1.0 False 0.030397 [2, 2]
2513500 1.0 1.0 False 0.030397 [2, 2]
2514500 1.0 1.0 False 0.030397 [2, 2]
Stage_Types Stage_Times Stage_Bytes Asynch_Iters \
420500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
421500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
422500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
423500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
424500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
... ... ... ... ...
2510500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
2511500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
2512500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
2513500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
2514500 (0, 6, 6, 4) (0.608531, 0, 0, 0) (0, 8, 8, 33176880) True
T_iter T_stages
420500 2.234976 (0.037385, 0.118837, 0.086114, 1.792483)
421500 2.416995 (0.02672, 0.125521, 0.081292, 1.784105)
422500 2.187999 (0.131866, 0.032969, 0.051701, 1.742342)
423500 2.244867 (0.010719, 0.119629, 0.07511, 1.789791)
424500 2.259979 (0.034214, 0.061322, 0.063906, 1.861178)
... ... ...
2510500 0.250689 (0.01873, 0.084494, 0.001038, 0.143586)
2511500 0.281567 (0.018751, 0.039149, 0.01087, 0.212796)
2512500 0.313766 (0.018743, 0.080215, 0.018336, 0.191758)
2513500 0.398377 (0.018734, 0.054583, 0.03555, 0.289509)
2514500 0.509275 (0.018742, 0.086675, 0.018152, 0.382491)
[1398 rows x 20 columns]
%% Cell type:code id: tags:
``` python
used_direction='s'
test_parameter='T_total'
if used_direction=='s':
df_aux=grouped_aggG.query('NP > NS')
used_labels=labelsShrink
name_fig="Shrink"
np_aux = [10, 20,20, 40,40,40, 80,80,80,80, 120,120,120,120,120]
nc_aux = [1, 1,10, 1,10,20, 1,10,20,40, 1,10,20,40,80]
elif used_direction=='e':
df_aux=grouped_aggM.query('NP < NS')
used_labels=labelsExpand
name_fig="Expand"
np_aux = [1,1,1,1,1, 10,10,10,10, 20,20,20, 40,40, 80 ]
nc_aux = [10,20,40,80,120, 20,40,80,120, 40,80,120, 80,120, 120]
elif used_direction=='a':
df_aux=grouped_aggM
used_labels=labels
name_fig="All"
np_aux = [1,1,1,1,1, 10,10,10,10,10, 20,20,20,20,20, 40,40,40,40,40, 80,80,80,80,80, 120,120,120,120,120]
nc_aux = [10,20,40,80,120, 1,20,40,80,120, 1,10,40,80,120, 1,10,20,80,120, 1,10,20,40,120, 1,10,20,40,80]
x = np.arange(len(used_labels))
handles = []
f=plt.figure(figsize=(20, 12))
#ax=f.add_subplot(111)
ax = plt.axes(projection='3d')
ax.azim = -60
ax.dist = 10
ax.elev = 10
ax.set_xlabel("NP", fontsize=20)
ax.set_ylabel("NC", fontsize=20)
ax.set_zlabel("Alpha", fontsize=20)
ax.tick_params(axis='both', which='major', labelsize=24)
ax.tick_params(axis='both', which='minor', labelsize=22)
for cst_aux in [1,3]:
df_aux2 = df_aux.query('Cst == @cst_aux')
for css_aux in [0,1]:
array_aux = df_aux2.query('Css == @css_aux')['alpha'].values
ax.plot3D(np_aux, nc_aux, array_aux, colors_spawn[cst_aux*2 + css_aux])
handles.append(handles_spawn[cst_aux*2 + css_aux])
#ax.set_zlim(0,4)
plt.legend(handles=handles, loc='best', fontsize=20,ncol=2,framealpha=1)
%%%% Output: stream
f.tight_layout()
f.savefig("Images/Spawn/3dPlot_"+name_fig+'_'+test_parameter+".png", format="png")
```
/tmp/ipykernel_7488/998390630.py:6: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
group = dfL.groupby(['ADR','Spawn_Method','Redistribution_Method', 'Redistribution_Strategy','NP','NC'])['T_iter', 'T_stages']
/tmp/ipykernel_7488/998390630.py:7: FutureWarning: ['T_stages'] did not aggregate successfully. If any error is raised this will raise in a future version of pandas. Drop these columns/ops to avoid this warning.
grouped_aggLAsynch = group.agg(['mean'])
%% Cell type:code id: tags:
``` python
from bt_scheme import PartialSolution, BacktrackingSolver
def elegirConf(parameters):
class StatePS(PartialSolution):
def __init__(self, config):
self.config= config
self.n= len(config) #Indica el valor a añadir
def is_solution(self):
return self.n == len(parameters)
def get_solution(self):
return tuple(self.config)
def successors(self):
array = parameters[self.n]
for parameter_value in array: #Test all values of the next parameter
self.config.append(parameter_value)
yield StatePS(self.config)
self.config.pop()
initialPs= StatePS([])
return BacktrackingSolver().solve(initialPs)
def obtenerConfs(parameters):
soluciones=[]
for solucion in elegirConf(parameters):
soluciones.append(solucion)
return soluciones
def modifyToUsable(parameters, len_parameters, configuration):
usable_configuration = []
for i in range(len(parameters)):
if len_parameters[i] > 1:
aux = (parameters[i][0], configuration[i])
else:
aux = (configuration[i])
usable_configuration.append(aux)
return usable_configuration
def CheckConfExists(configuration, dataSet, type_conf='global'):
remove = 0
config = list(configuration)
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
if type_conf == 'global':
config.append((np_aux, ns_aux))
elif type_conf == 'malleability':
config.append(np_aux)
config.append(ns_aux)
if tuple(config) in dataSet.index:
remove = 1
elif remove != 1:
remove = -1
config.pop()
if type_conf == 'malleability':
config.pop()
if remove == 1:
return True
return False
```
%% Cell type:code id: tags:
``` python
sp_method = [0,1]
rd_method = [0,1]
rd_strat = [1,2]
adr = [0,96.6]
parameters = [adr, sp_method, rd_method, rd_strat]
len_parameters = [1,2,2,2]
configurations_aux = obtenerConfs(parameters)
configurations = []
configurations_simple = []
for index in range(len(configurations_aux)):
aux_conf = modifyToUsable(parameters, len_parameters, configurations_aux[index])
if CheckConfExists(aux_conf, grouped_aggG):
configurations.append(aux_conf)
if CheckConfExists(configurations_aux[index], grouped_aggM, 'malleability'):
configurations_simple.append(list(configurations_aux[index]))
print(configurations_simple)
print(configurations)
print(len(configurations))
```
%%%% Output: stream
[[0, 0, 0, 1], [0, 0, 1, 1], [0, 1, 0, 1], [0, 1, 1, 1], [96.6, 0, 0, 1], [96.6, 0, 0, 2], [96.6, 0, 1, 1], [96.6, 0, 1, 2], [96.6, 1, 0, 1], [96.6, 1, 0, 2], [96.6, 1, 1, 1], [96.6, 1, 1, 2]]
[[0, (0, 0), (0, 0), (1, 1)], [0, (0, 0), (0, 1), (1, 1)], [0, (0, 1), (0, 0), (1, 1)], [0, (0, 1), (0, 1), (1, 1)], [96.6, (0, 0), (0, 0), (1, 1)], [96.6, (0, 0), (0, 0), (1, 2)], [96.6, (0, 0), (0, 1), (1, 1)], [96.6, (0, 0), (0, 1), (1, 2)], [96.6, (0, 1), (0, 0), (1, 1)], [96.6, (0, 1), (0, 0), (1, 2)], [96.6, (0, 1), (0, 1), (1, 1)], [96.6, (0, 1), (0, 1), (1, 2)]]
12
%% Cell type:code id: tags:
``` python
#ALPHA COMPUTATION
def compute_alpha(config_a, config_b):
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
config_a.append(np_aux)
config_a.append(ns_aux)
config_b.append(np_aux)
config_b.append(ns_aux)
grouped_aggM.loc[tuple(config_b),'Alpha'] = grouped_aggM.loc[tuple(config_b),'T_Malleability'] / grouped_aggM.loc[tuple(config_a),'T_Malleability']
config_a.pop()
config_a.pop()
config_b.pop()
config_b.pop()
config_a.insert(0,ns_aux)
config_a.insert(0,np_aux)
config_b.insert(0,ns_aux)
config_b.insert(0,np_aux)
out_grouped_M.loc[tuple(config_b),'Alpha'] = out_grouped_M.loc[tuple(config_b),'T_Malleability'] / out_grouped_M.loc[tuple(config_a),'T_Malleability']
config_a.pop(0)
config_a.pop(0)
config_b.pop(0)
config_b.pop(0)
if not ('Alpha' in grouped_aggM.columns):
for config_a in configurations_simple:
for config_b in configurations_simple:
#FIXME/TODO If the last index of configurations is not the strategy or different from pthreads may fail this computation
if config_a[1:-1] == config_b[1:-1] and config_a[0] == 0 and config_b[0] != 0:
compute_alpha(config_a, config_b)
else:
print("ALPHA already exists")
```
%%%% Output: stream
ALPHA already exists
%% Cell type:code id: tags:
``` python
#OMEGA COMPUTATION
def compute_omega(config):
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
config.append(np_aux)
config.append(ns_aux)
grouped_aggLAsynch.loc[tuple(config),'Omega'] = grouped_aggLAsynch.loc[tuple(config),'T_iter'] / grouped_aggLSynch.loc[np_aux,'T_iter']
config.pop()
config.pop()
if not ('Omega' in grouped_aggLAsynch.columns):
for config in configurations_simple:
if config[0] != 0:
compute_omega(config)
else:
print("OMEGA already exists")
```
%%%% Output: stream
OMEGA already exists
%% Cell type:code id: tags:
``` python
out_grouped_G.to_excel("resultG.xlsx")
out_grouped_M.to_excel("resultM.xlsx")
grouped_aggLAsynch.to_excel("AsynchIters.xlsx")
```
%% Cell type:code id: tags:
``` python
grouped_aggLAsynch
```
%%%% Output: execute_result
T_iter \
ADR Spawn_Method Redistribution_Method Redistribution_Strategy NP NC
96.6 0.0 0.0 1.0 2 10.0 1.075937
20.0 1.207753
40.0 1.256070
80.0 1.169895
120.0 1.225243
... ...
1.0 1.0 2.0 160 10.0 0.896296
20.0 0.708625
40.0 0.533604
80.0 0.482244
120.0 0.468189
Omega
ADR Spawn_Method Redistribution_Method Redistribution_Strategy NP NC
96.6 0.0 0.0 1.0 2 10.0 1.789700
20.0 2.008962
40.0 2.089332
80.0 1.945990
120.0 2.038054
... ...
1.0 1.0 2.0 160 10.0 5.711017
20.0 4.515218
40.0 3.400016
80.0 3.072765
120.0 2.983204
[336 rows x 2 columns]
%% Cell type:code id: tags:
``` python
# Aquellos grupos que tengán valores por encima del límite no se considerarán
# Con sumar a si mismos su valor actual estarán fuera
def check_groups_boundaries(dataLists, boundaries, tc_boundary):
for index in range(len(boundaries)):
if boundaries[index] > tc_boundary:
dataLists[index]= float('infinity')
```
%% Cell type:code id: tags:
``` python
def get_perc_differences(dataLists, boundaries, tc_boundary):
if boundaries != None: # Si se usa perspectiva de RMS, se desconsideran valores muy altos
check_groups_boundaries(dataLists, boundaries, tc_boundary)
indexes = np.argsort(dataLists)
best = -1
bestMax = -1
otherBest=[]
for index in indexes: # Para cada metodo -- Empezando por el tiempo más bajo en media/mediana
if best == -1:
best = index
bestMax = dataLists[best] * 1.05
elif dataLists[index] <= bestMax: # Medias/Medianas diferentes && Media/Medianas i < Media/Mediana best
otherBest.append(index)
otherBest.insert(0,best)
return otherBest
```
%% Cell type:code id: tags:
``` python
def get_config_data(tipo, data_aux, used_config, np_aux, ns_aux):
dataLists=[]
for config in used_config:
if tipo == "T_total":
config.append((np_aux,ns_aux))
elif tipo == "T_Malleability":
config.append(np_aux)
config.append(ns_aux)
if tuple(config) in data_aux.index:
aux_value = data_aux.loc[tuple(config),tipo]
if isinstance(aux_value, pd.Series):
aux_value = aux_value.values[0]
else: # This configuration is not present in the dataset
aux_value = float('infinity')
dataLists.append(aux_value)
config.pop()
if tipo == "T_Malleability":
config.pop()
return dataLists
```
%% Cell type:code id: tags:
``` python
def results_with_perc(tipo, data_aux, used_config, rms_boundary=0):
results = []
boundaries = None
for np_aux in processes:
for ns_aux in processes:
if np_aux != ns_aux:
# 1 - Get all values for particular config with these number of processes
dataLists = get_config_data(tipo, data_aux, used_config, np_aux, ns_aux)
tc_boundary = 0
if rms_boundary != 0:
# El porcentaje de tc_boundary se tiene en cuenta para eliminar aquellos
# tiempos demasiado grandes en su malleability time respecto al más pequeño
boundaries = get_config_data("T_Malleability", grouped_aggM, configurations_simple, np_aux, ns_aux)
tc_boundary = min(boundaries)
tc_boundary = tc_boundary + tc_boundary*rms_boundary
aux_data = get_perc_differences(dataLists, boundaries, tc_boundary)
results.append(aux_data)
return results
```
%% Cell type:code id: tags:
``` python
checked_type='T_total'
use_perc = True
rms_boundary=0.1 # Poner a 0 para perspectiva de app. Valor >0 y <1 para perspectiva de RMS
if checked_type=='T_total':
tipo="T_total"
data_aux=grouped_aggG
used_config = configurations
elif checked_type=='T_Malleability':
tipo="T_Malleability"
data_aux=grouped_aggM
used_config = configurations_simple
if use_perc:
results = results_with_perc(tipo, data_aux, used_config, rms_boundary)
else:
#results = results_with_st(tipo, data_aux)
results = None
#Results is a 2 dimensional array. First dimensional indicates winners of a particulal number of processes (NP->NC).
#Second dimension is an ordered preference of indexes in the array configurations.
print(results)
print(len(results))
```
%%%% Output: stream
[[1], [2], [2], [4, 6], [6, 4], [4], [2], [10, 3, 2, 8], [2, 8, 3], [2], [3, 2], [2, 10, 3], [8], [2], [2], [2, 10, 3], [3, 10], [3], [10], [8, 2], [8, 2], [2], [2, 3], [3], [10], [10, 2], [10, 2], [3], [2, 3], [3, 2], [10], [10], [10, 2], [3, 2], [3, 2], [2], [10], [2], [2, 3], [3, 2], [3, 2], [3, 2]]
42
%% Cell type:code id: tags:
``` python
#Lista de indices de mayor a menor de los valores
aux_array = []
for data in results:
aux_array+=data
unique, counts = np.unique(aux_array, return_counts=True)
aux_dict = dict(zip(unique, counts))
aux_keys=list(aux_dict.keys())
aux_values=list(aux_dict.values())
aux_ordered_index=list(reversed(list(np.argsort(aux_values))))
i=0
j=0
used_aux=0
heatmap=np.zeros((len(processes),len(processes))).astype(int)
if use_perc:
for i in range(len(processes)):
for j in range(len(processes)):
if i==j:
heatmap[i][j]=-1
used_aux+=1
else:
results_index = i*len(processes) +j-used_aux
heatmap[i][j] = results[results_index][0]
else:
for i in range(len(processes)):
for j in range(len(processes)):
if i==j:
heatmap[i][j]=-1
used_aux+=1
else:
results_index = i*len(processes) +j-used_aux
for index in aux_ordered_index:
if aux_keys[index] in results[results_index]:
heatmap[i][j]=aux_keys[index]
break
heatmap[-1][-1]=len(used_config)
print(heatmap)
```
%%%% Output: stream
[[-1 1 2 2 4 6 4]
[ 2 -1 10 2 2 3 2]
[ 8 2 -1 2 2 3 3]
[10 8 8 -1 2 2 3]
[10 10 10 3 -1 2 3]
[10 10 10 3 3 -1 2]
[10 2 2 3 3 3 12]]
%% Cell type:code id: tags:
``` python
#Adapta results a una cadena asegurando que cada cadena no se sale de su celda
results_str = []
max_counts = 1
max_per_line = 3
for i in range(len(results)):
results_str.append(list())
count = len(results[i])
results_aux = results[i]
if count > max_counts:
count = max_counts
results_aux = results[i][:count]
remainder = count%max_per_line
if count <= max_per_line:
aux_str = str(results_aux).replace('[','').replace(']','')
results_str[i].append(aux_str)
else:
if remainder == 0:
index = count//2
else:
index = count - ((remainder-1)*max_per_line + 1)
aux_str = str(results_aux[:index]).replace('[','').replace(']','')
results_str[i].append(aux_str)
aux_str = str(results_aux[index:]).replace('[','').replace(']','')
results_str[i].append(aux_str)
#print(results_str)
```
%% Cell type:code id: tags:
``` python
#Crea un heatmap teniendo en cuenta los colores anteriores
f=plt.figure(figsize=(24, 12))
ax=f.add_subplot(111)
myColors = (colors.to_rgba("white"),colors.to_rgba("green"), colors.to_rgba("springgreen"),colors.to_rgba("blue"),colors.to_rgba("darkblue"),
colors.to_rgba("red"),colors.to_rgba("darkred"),colors.to_rgba("darkgoldenrod"),colors.to_rgba("olive"),
colors.to_rgba("green"), colors.to_rgba("springgreen"),colors.to_rgba("blue"),colors.to_rgba("darkblue"),colors.to_rgba("white"))
cmap = LinearSegmentedColormap.from_list('Custom', myColors, len(myColors))
im = ax.imshow(heatmap,cmap=cmap,interpolation='nearest')
# Loop over data dimensions and create text annotations.
used_aux=0
for i in range(len(processes)):
for j in range(len(processes)):
if i!=j:
aux_color="white"
if heatmap[i, j] == 1 or heatmap[i, j] == 9: # El 1 puede necesitar texto en negro
aux_color="black"
results_index = i*len(processes) +j-used_aux
if len(results_str[results_index]) == 1:
text = results_str[results_index][0]
ax.text(j, i, text, ha="center", va="center", color=aux_color, fontsize=36)
else:
add_aux = 0.33
for line in range(len(results_str[results_index])):
i_range = i - 0.5 + add_aux
ax.text(j, i_range, results_str[results_index][line],
ha="center", va="center", color=aux_color, fontsize=36)
add_aux+=0.33
else:
used_aux+=1
ax.set_ylabel("NP", fontsize=36)
ax.set_xlabel("NC", fontsize=36)
ax.set_xticklabels(['']+processes, fontsize=36)
ax.set_yticklabels(['']+processes, fontsize=36)
labelsMethods_aux = ['Baseline - AllS (0)', 'Baseline - P2PS (1)',
'Merge - AllS (2)','Merge - P2PS (3)',
'Baseline - AllA (4)', 'Baseline - AllT (5)','Baseline - P2PA (6)','Baseline - P2PT (7)',
'Merge - AllA (8)','Merge - AllT (9)','Merge - P2PA (10)','Merge - P2PT (11)']
colorbar=f.colorbar(im, ax=ax)
tick_bar = []
for i in range(len(used_config)):
tick_bar.append(0.37 + i*0.92) #TE
colorbar.set_ticks(tick_bar)
colorbar.set_ticklabels(labelsMethods_aux)
colorbar.ax.tick_params(labelsize=32)
#
f.tight_layout()
f.savefig("Images/Spawn/Heatmap_"+tipo+".png", format="png")
```
%%%% Output: stream
/tmp/ipykernel_4155/3389487544.py:37: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_xticklabels(['']+processes, fontsize=36)
/tmp/ipykernel_4155/3389487544.py:38: UserWarning: FixedFormatter should only be used together with FixedLocator
ax.set_yticklabels(['']+processes, fontsize=36)
%%%% Output: display_data
[Hidden Image Output]
%% Cell type:code id: tags:
``` python
aux_array = [0] * len(results)
for index in range(len(results)):
aux_array[index] = results[index][0]
aux_results, aux_counts = np.unique(aux_array, return_counts = True)
print(aux_results)
print(aux_counts)
```
%%%% Output: stream
[ 1 2 3 4 6 8 10]
[ 1 21 8 1 2 2 7]
%% Cell type:raw id: tags:
================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================
%% Cell type:code id: tags:
``` python
a=[0,1,2,3,4]
b=a
count=3
a[2]=100
print(a)
print(b)
```
%%%% Output: stream
[0, 1, 100, 3, 4]
[0, 1, 100, 3, 4]
%% Cell type:code id: tags:
``` python
fig, ax = plt.subplots(1, 1, subplot_kw={'projection': '3d'})
# Get the test data
#X, Y, Z = axes3d.get_test_data(0.05)
aux = grouped_aggG.loc[u_sols[0],'T_total']
Z = [None] * len(processes)
X, Y = np.meshgrid(processes, processes)
removed_index = 0
for i in range(len(processes)):
Z[i] = [0] * len(processes)
for j in range(len(processes)):
if i!=j:
real_i = i - removed_index
real_j = j - removed_index
Z[i][j] = aux.values[real_i*len(processes)+real_j]
else:
Z[i][j] = 0
removed_index += 1
Z = np.array(Z)
ax.plot_wireframe(X, Y, Z, rstride=20, cstride=10)
ax.set_proj_type('ortho') # FOV = 0 deg
ax.set_title("'ortho'\nfocal_length = ∞", fontsize=10)
plt.show()
```
%%%% Output: display_data
[Hidden Image Output]
%% Cell type:raw id: tags:
El siguiente código es para utilizar Dask. Una versión que paraleliza una serie de tareas de Pandas.
Tras llamar a compute se realizan todas las tareas que se han pedido.
%% Cell type:code id: tags:
``` python
import dask.dataframe as dd
ddf = dd.from_pandas(dfL[(dfL.Asynch_Iters == False)], npartitions=10)
group = ddf.groupby('NP')['T_iter']
grouped_aggLSynch = group.agg(['mean'])
grouped_aggLSynch = grouped_aggLSynch.rename(columns={'mean':'T_iter'})
grouped_aggLSynch = grouped_aggLSynch.compute()
```
%% Cell type:raw id: tags:
================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================
%% Cell type:code id: tags:
``` python
a = (4, 3, 5, 6)
b = (4, 3, 5, 6)
d = (4, 3, 5, 7)
c = (3, 4, 5, 6)
print(a[1:]==b[1:])
print(a[1:]==c[1:])
print(a[1:])
```
%%%% Output: stream
True
False
(3, 5, 6)
%% Cell type:code id: tags:
``` python
a = [2, 3]
a.pop(0)
print(a)
```
%%%% Output: stream
[3]
%% Cell type:code id: tags:
``` python
```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment