Skip to content

GitLab

Commit f732b56c authored by iker_martin's avatar iker_martin
Browse files

Added fully Synch redistribution. Prctl may no longer be needed.

parent 9b3fed07
Show whitespace changes
Inline Side-by-side
BiCGStab_Iker/BiCGStab.c
View file @ f732b56c
......@@ -8,7 +8,7 @@
#include <mpi.h>
#include <hb_io.h>
//#include <vector>
#include <sys/prctl.h>
//#include <sys/prctl.h>
#include "reloj.h"
#include "ScalarVectors.h"
......@@ -179,7 +179,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
int IONE = 1;
double DONE = 1.0, DMONE = -1.0, DZERO = 0.0;
int n, n_dist;
int maxiter, myId, reconfigure, rec_iter, state;
int maxiter, myId, reconfigure, rec_iter, state, flag;
double beta, alpha, umbral, omega, tmp;
double t3, t4;
double reduce[2];
......@@ -187,9 +187,11 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
n = size; n_dist = sizeR; maxiter = 16 * size; rec_iter = maxiter / 2; umbral = 1.0e-8;
myId = computeData->myId;
state = -1;
reconfigure = 0; rec_iter = 1; maxiter=100;
reconfigure = 0; rec_iter= 500;
// flag = (computeData->rho == 0.0)? -1: 1;
while ((computeData->iter < maxiter) && (computeData->tol > umbral)) {
// while ((computeData->iter < maxiter) && (flag == 1)) {
#if PRECOND
VvecDoubles (DONE, computeData->diags, computeData->p, DZERO, computeData->p_hat, n_dist); // p_hat = D^-1 * p
......@@ -212,6 +214,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
printf ("PD=%d %d \t %g \t %g \t %g \n", computeData->numP, computeData->iter, computeData->tol, umbral, computeData->direct_err);
#else
printf ("%d \t %g \n", computeData->iter, computeData->tol);
// printf ("%d \t %20.10e \n", computeData->iter, computeData->tol);
#endif // DIRECT_ERROR
alpha = rdot (&n_dist, computeData->r0, &IONE, computeData->s, &IONE);
MPI_Allreduce (MPI_IN_PLACE, &alpha, 1, MPI_DOUBLE, MPI_SUM, computeData->comm);
......@@ -395,7 +398,7 @@ int main (int argc, char **argv) {
user_data = empty_user_data;
user_data.comm = computeData.comm;
prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
//prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
isTarget = MAM_Init(root, &computeData.comm, argv[0], user_func, (void *) &user_data);
if(isTarget) {
......@@ -566,9 +569,12 @@ void originals_set_data(Compute_data *computeData, user_redist_t *user_data, int
MAM_Set_target_number(num_target);
MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
//MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
//MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
//MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
MAM_Data_add(&(computeData->iter), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
MAM_Data_add(&(computeData->tol), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
......@@ -577,14 +583,19 @@ void originals_set_data(Compute_data *computeData, user_redist_t *user_data, int
MAM_Data_add(&(computeData->direct_err), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
#endif
MAM_Data_add(computeData->vlen, NULL, computeData->n, MPI_INT, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
//MAM_Data_add(computeData->vlen, NULL, computeData->n, MPI_INT, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
//MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
//MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
#if PRECOND
MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
//MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
#endif
#if DIRECT_ERROR
MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
//MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
#endif
MAM_Data_add(computeData->p, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
......@@ -612,14 +623,22 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
MPI_Comm_rank(computeData->comm, &computeData->myId);
entry = 0;
/*
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
computeData->n = *((int *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
computeData->tol0 = *((double *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
computeData->t1 = *((double *)value);
*/
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
computeData->n = *((int *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
computeData->tol0 = *((double *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
computeData->t1 = *((double *)value);
entry = 0;
// entry = 0;
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
computeData->iter = *((int *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
......@@ -632,6 +651,7 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
#endif
entry = 0;
/*
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
computeData->vlen = ((int *)value);
//computeData->vlen = user_data->recv_vlen;
......@@ -647,8 +667,22 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
computeData->x_exact = ((double *)value);
#endif
*/
computeData->vlen = user_data->recv_vlen;
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
computeData->r0 = ((double *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
computeData->b = ((double *)value);
#if PRECOND
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
computeData->diags = ((double *)value);
#endif
#if DIRECT_ERROR
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
computeData->x_exact = ((double *)value);
#endif
entry = 0;
// entry = 0;
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
computeData->p = ((double *)value);
MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
......@@ -695,9 +729,9 @@ void user_func(void *args) {
MAM_Get_Reconf_Info(&user_reconf);
user_redist_t *user_data = (user_redist_t *) args;
if(!user_data->initiated) {
//targets_distribution_synch(user_reconf, user_data);
//flag = 1;
targets_distribution_synch(user_reconf, user_data);
flag = 1;
/*
targets_distribution(user_reconf, user_data);
user_data->initiated = 1;
......@@ -705,6 +739,7 @@ void user_func(void *args) {
MPI_Waitall(2, user_data->reqs, MPI_STATUSES_IGNORE);
flag = 1;
}
*/
} else {
MPI_Testall(2, user_data->reqs, &local_flag, MPI_STATUSES_IGNORE);
MPI_Allreduce(&local_flag, &flag, 1, MPI_INT, MPI_MIN, user_data->comm);
......
BiCGStab_Iker/ToolsMAM.c
View file @ f732b56c
......@@ -116,6 +116,118 @@ void targets_distribution(mam_user_reconf_t user_reconf, user_redist_t *user_dat
free(scounts); free(sdispls); free(rcounts); free(rdispls);
}
void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data) {
int i, n, offset, elems, rank, numP, *rank_states;
int *scounts, *rcounts, *sdispls, *rdispls;
//size_t total_qty;
//void *value = NULL;
struct Dist_data dist_data;
//MPI_Datatype type;
int aux_int;
int *recv_vpos = &aux_int;
double aux_double;
double *recv_vval = &aux_double;
user_data->recv_vlen = &aux_int;
MPI_Comm_rank(user_reconf.comm, &rank);
MPI_Comm_size(user_reconf.comm, &numP);
scounts = calloc(numP, sizeof(int));
sdispls = calloc(numP, sizeof(int));
rcounts = calloc(numP, sizeof(int));
rdispls = calloc(numP, sizeof(int));
offset = 0;
rank_states = (int *) malloc(numP * sizeof(int));
MPI_Allgather(&user_reconf.rank_state, 1, MPI_INT, rank_states, 1, MPI_INT, user_reconf.comm);
if(rank == 0) n = user_data->n;
MPI_Bcast(&n, 1, MPI_INT, 0, user_reconf.comm);
if(user_reconf.rank_state != MAM_PROC_ZOMBIE) {
MPI_Comm_rank(user_data->comm, &dist_data.myId);
dist_data.numP = user_reconf.numT;
if(user_reconf.rank_state == MAM_PROC_NEW_RANK) {
user_data->array_vlen = &aux_int;
for(i=0; i<user_reconf.numS; i++) {
if(rank_states[i] == MAM_PROC_CONTINUE) {
dist_data.myId += user_reconf.numS;
break;
}
}
}
get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
CreateInts(&user_data->recv_vlen, dist_data.tamBl);
prepare_redist_counts_vlen(rcounts, rdispls, user_reconf.numS, offset, dist_data);
// print_counts2(dist_data, rcounts, rdispls, numP, 0, "TARGETS");
}
if(user_reconf.rank_state != MAM_PROC_NEW_RANK) {
MPI_Comm_rank(user_data->comm, &dist_data.myId);
dist_data.numP = user_reconf.numS;
get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
offset = (user_reconf.numS + user_reconf.numT) == numP ?
user_reconf.numS : 0;
prepare_redist_counts_vlen(scounts, sdispls, user_reconf.numT, offset, dist_data);
// print_counts2(dist_data, scounts, sdispls, numP, 0, "SOURCES");
}
// COMUNICACION DE DATOS //
MPI_Alltoallv(user_data->array_vlen, scounts, sdispls, MPI_INT, user_data->recv_vlen, rcounts, rdispls, MPI_INT, user_reconf.comm);
free(scounts); free(sdispls); free(rcounts); free(rdispls);
scounts = calloc(numP, sizeof(int));
sdispls = calloc(numP, sizeof(int));
rcounts = calloc(numP, sizeof(int));
rdispls = calloc(numP, sizeof(int));
offset = 0;
if(user_reconf.rank_state != MAM_PROC_ZOMBIE) {
MPI_Comm_rank(user_data->comm, &dist_data.myId);
dist_data.numP = user_reconf.numT;
if(user_reconf.rank_state == MAM_PROC_NEW_RANK) {
user_data->array_vlen = &aux_int;
for(i=0; i<user_reconf.numS; i++) {
if(rank_states[i] == MAM_PROC_CONTINUE) {
dist_data.myId += user_reconf.numS;
break;
}
}
}
get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
CreateSparseMatrixVptr(&user_data->other_subm, dist_data.tamBl, n, 0);
user_data->other_subm.vptr[0] = 0;
//memcpy(user_data->other_subm.vptr+1, vlen, dist_data.tamBl * sizeof(int));
for(i=0; i<dist_data.tamBl; i++) {
user_data->other_subm.vptr[i+1] = user_data->recv_vlen[i];
}
TransformLengthtoHeader(user_data->other_subm.vptr, user_data->other_subm.dim1); // The array is converted from vlen to vptr
elems = user_data->other_subm.vptr[dist_data.tamBl];
CreateSparseMatrixValues(&user_data->other_subm, dist_data.tamBl, n, elems, 0);
recv_vpos = user_data->other_subm.vpos;
recv_vval = user_data->other_subm.vval;
prepare_redist_counts(rcounts, rdispls, user_reconf.numS, offset, dist_data, user_data->other_subm.vptr);
}
if(user_reconf.rank_state != MAM_PROC_NEW_RANK) {
MPI_Comm_rank(user_data->comm, &dist_data.myId);
dist_data.numP = user_reconf.numS;
get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
offset = (user_reconf.numS + user_reconf.numT) == numP ?
user_reconf.numS : 0;
prepare_redist_counts(scounts, sdispls, user_reconf.numT, offset, dist_data, user_data->array_vptr);
}
// COMUNICACION DE DATOS //
MPI_Alltoallv(user_data->array_vpos, scounts, sdispls, MPI_INT, recv_vpos, rcounts, rdispls, MPI_INT, user_reconf.comm);
MPI_Alltoallv(user_data->array_vval, scounts, sdispls, MPI_DOUBLE, recv_vval, rcounts, rdispls, MPI_DOUBLE, user_reconf.comm);
free(rank_states);
free(scounts); free(sdispls); free(rcounts); free(rdispls);
}
/*
* ========================================================================================
* ========================================================================================
......
BiCGStab_Iker/ToolsMAM.h
View file @ f732b56c
......@@ -29,7 +29,7 @@ static const user_redist_t empty_user_data = {
};
extern void targets_distribution(mam_user_reconf_t user_reconf, user_redist_t *user_data);
//extern void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data);
extern void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data);
extern void print_global_results(double start_time);
#endif
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment