Added fully Synch redistribution. Prctl may no longer be needed.

BiCGStab_Iker/BiCGStab.c

@@ -8,7 +8,7 @@
 #include <mpi.h>
 #include <hb_io.h>
 //#include <vector>
-#include <sys/prctl.h>
+//#include <sys/prctl.h>
 
 #include "reloj.h"
 #include "ScalarVectors.h"
@@ -179,7 +179,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
     int IONE = 1; 
     double DONE = 1.0, DMONE = -1.0, DZERO = 0.0;
     int n, n_dist;
-    int maxiter, myId, reconfigure, rec_iter, state;
+    int maxiter, myId, reconfigure, rec_iter, state, flag;
     double beta, alpha, umbral, omega, tmp;
     double t3, t4;
     double reduce[2];
@@ -187,9 +187,11 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
     n = size; n_dist = sizeR; maxiter = 16 * size; rec_iter = maxiter / 2; umbral = 1.0e-8;
     myId = computeData->myId;
     state = -1;
-    reconfigure = 0; rec_iter = 1; maxiter=100;
+    reconfigure = 0; rec_iter= 500;
+//	  flag = (computeData->rho == 0.0)? -1: 1;
 
     while ((computeData->iter < maxiter) && (computeData->tol > umbral)) {
+//    while ((computeData->iter < maxiter) && (flag == 1)) {
 
 #if PRECOND
         VvecDoubles (DONE, computeData->diags, computeData->p, DZERO, computeData->p_hat, n_dist);              // p_hat = D^-1 * p
@@ -212,6 +214,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
             printf ("PD=%d %d \t %g \t %g \t %g \n", computeData->numP, computeData->iter, computeData->tol, umbral, computeData->direct_err);
 #else        
         printf ("%d \t %g \n", computeData->iter, computeData->tol);
+//        printf ("%d \t %20.10e \n", computeData->iter, computeData->tol);
 #endif // DIRECT_ERROR
         alpha = rdot (&n_dist, computeData->r0, &IONE, computeData->s, &IONE);
         MPI_Allreduce (MPI_IN_PLACE, &alpha, 1, MPI_DOUBLE, MPI_SUM, computeData->comm);
@@ -395,7 +398,7 @@ int main (int argc, char **argv) {
     user_data = empty_user_data;
     user_data.comm = computeData.comm;
 
-    prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
+    //prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
     isTarget = MAM_Init(root, &computeData.comm, argv[0], user_func, (void *) &user_data);
 
     if(isTarget) {
@@ -566,9 +569,12 @@ void originals_set_data(Compute_data *computeData, user_redist_t *user_data, int
 
     MAM_Set_target_number(num_target);
 
-    MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
-    MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
-    MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
+    MAM_Data_add(&(computeData->n), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
+    MAM_Data_add(&(computeData->tol0), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
+    MAM_Data_add(&(computeData->t1), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
 
     MAM_Data_add(&(computeData->iter), NULL, 1, MPI_INT, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
     MAM_Data_add(&(computeData->tol), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
@@ -577,14 +583,19 @@ void originals_set_data(Compute_data *computeData, user_redist_t *user_data, int
     MAM_Data_add(&(computeData->direct_err), NULL, 1, MPI_DOUBLE, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
 #endif
 
-    MAM_Data_add(computeData->vlen, NULL, computeData->n, MPI_INT, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT); 
-    MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
-    MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(computeData->vlen, NULL, computeData->n, MPI_INT, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT); 
+    //MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+
+    MAM_Data_add(computeData->r0, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
+    MAM_Data_add(computeData->b, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
 #if PRECOND
-    MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    MAM_Data_add(computeData->diags, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
 #endif
 #if DIRECT_ERROR
-    MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    //MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
+    MAM_Data_add(computeData->x_exact, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
 #endif
 
     MAM_Data_add(computeData->p, NULL, computeData->n, MPI_DOUBLE, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
@@ -612,14 +623,22 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
     MPI_Comm_rank(computeData->comm, &computeData->myId);
 
     entry = 0;
+    /*
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
     computeData->n = *((int *)value);
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
     computeData->tol0 = *((double *)value);
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_CONSTANT);
     computeData->t1 = *((double *)value);
+    */
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
+    computeData->n = *((int *)value);
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
+    computeData->tol0 = *((double *)value);
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
+    computeData->t1 = *((double *)value);
 
-    entry = 0;
+//    entry = 0;
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
     computeData->iter = *((int *)value); 
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_REPLICATED, MAM_DATA_VARIABLE);
@@ -632,6 +651,7 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
 #endif
 
     entry = 0;
+    /*
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
     computeData->vlen = ((int *)value);
     //computeData->vlen = user_data->recv_vlen;
@@ -647,8 +667,22 @@ void targets_update(Compute_data *computeData, user_redist_t *user_data) {
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_CONSTANT);
     computeData->x_exact = ((double *)value);
 #endif
+    */
+    computeData->vlen = user_data->recv_vlen;
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
+    computeData->r0 = ((double *)value);
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
+    computeData->b = ((double *)value);
+#if PRECOND
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
+    computeData->diags = ((double *)value);
+#endif
+#if DIRECT_ERROR
+    MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
+    computeData->x_exact = ((double *)value);
+#endif
 
-    entry = 0;
+//    entry = 0;
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
     computeData->p = ((double *)value);
     MAM_Data_get_pointer(&value, entry++, &total_qty, &type, MAM_DATA_DISTRIBUTED, MAM_DATA_VARIABLE);
@@ -695,9 +729,9 @@ void user_func(void *args) {
     MAM_Get_Reconf_Info(&user_reconf);
     user_redist_t *user_data = (user_redist_t *) args;
     if(!user_data->initiated) {
-      //targets_distribution_synch(user_reconf, user_data);
-      //flag = 1;
-
+      targets_distribution_synch(user_reconf, user_data);
+      flag = 1;
+      /*
       targets_distribution(user_reconf, user_data);
       user_data->initiated = 1;
 
@@ -705,6 +739,7 @@ void user_func(void *args) {
         MPI_Waitall(2, user_data->reqs, MPI_STATUSES_IGNORE);
 	flag = 1;
       }
+      */
     } else {
       MPI_Testall(2, user_data->reqs, &local_flag, MPI_STATUSES_IGNORE);
       MPI_Allreduce(&local_flag, &flag, 1, MPI_INT, MPI_MIN, user_data->comm);

BiCGStab_Iker/ToolsMAM.c

@@ -116,6 +116,118 @@ void targets_distribution(mam_user_reconf_t user_reconf, user_redist_t *user_dat
     free(scounts); free(sdispls); free(rcounts); free(rdispls);
 }
 
+
+
+void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data) {
+    int i, n, offset, elems, rank, numP, *rank_states;
+    int *scounts, *rcounts, *sdispls, *rdispls;
+    //size_t total_qty;
+    //void *value = NULL;
+    struct Dist_data dist_data;
+    //MPI_Datatype type;
+
+    int aux_int;
+    int *recv_vpos = &aux_int;
+    double aux_double;
+    double *recv_vval = &aux_double;
+    user_data->recv_vlen = &aux_int;
+
+
+    MPI_Comm_rank(user_reconf.comm, &rank);
+    MPI_Comm_size(user_reconf.comm, &numP);
+    scounts = calloc(numP, sizeof(int));
+    sdispls = calloc(numP, sizeof(int));
+    rcounts = calloc(numP, sizeof(int));
+    rdispls = calloc(numP, sizeof(int));
+    offset = 0;
+	  
+    rank_states = (int *) malloc(numP * sizeof(int));
+    MPI_Allgather(&user_reconf.rank_state, 1, MPI_INT, rank_states, 1, MPI_INT, user_reconf.comm);
+
+    if(rank == 0) n = user_data->n;
+    MPI_Bcast(&n, 1, MPI_INT, 0, user_reconf.comm);
+
+    if(user_reconf.rank_state != MAM_PROC_ZOMBIE) {
+      MPI_Comm_rank(user_data->comm, &dist_data.myId);
+      dist_data.numP = user_reconf.numT;
+      if(user_reconf.rank_state == MAM_PROC_NEW_RANK) {
+	user_data->array_vlen = &aux_int;
+	for(i=0; i<user_reconf.numS; i++) {
+          if(rank_states[i] == MAM_PROC_CONTINUE) {
+            dist_data.myId += user_reconf.numS;
+	    break;
+	  }
+	}
+      }
+      get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
+      CreateInts(&user_data->recv_vlen, dist_data.tamBl);
+
+      prepare_redist_counts_vlen(rcounts, rdispls, user_reconf.numS, offset, dist_data);
+    //  print_counts2(dist_data, rcounts, rdispls, numP, 0, "TARGETS");
+    } 
+
+    if(user_reconf.rank_state != MAM_PROC_NEW_RANK) {
+      MPI_Comm_rank(user_data->comm, &dist_data.myId);
+      dist_data.numP = user_reconf.numS;
+      get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
+      offset = (user_reconf.numS + user_reconf.numT) == numP ? 
+	      user_reconf.numS : 0; 
+      prepare_redist_counts_vlen(scounts, sdispls, user_reconf.numT, offset, dist_data);
+    //  print_counts2(dist_data, scounts, sdispls, numP, 0, "SOURCES");
+    }
+
+    // COMUNICACION DE DATOS //
+    MPI_Alltoallv(user_data->array_vlen, scounts, sdispls, MPI_INT, user_data->recv_vlen, rcounts, rdispls, MPI_INT, user_reconf.comm);
+    free(scounts); free(sdispls); free(rcounts); free(rdispls);
+    scounts = calloc(numP, sizeof(int));
+    sdispls = calloc(numP, sizeof(int));
+    rcounts = calloc(numP, sizeof(int));
+    rdispls = calloc(numP, sizeof(int));
+    offset = 0;
+
+    if(user_reconf.rank_state != MAM_PROC_ZOMBIE) {
+      MPI_Comm_rank(user_data->comm, &dist_data.myId);
+      dist_data.numP = user_reconf.numT;
+      if(user_reconf.rank_state == MAM_PROC_NEW_RANK) {
+	user_data->array_vlen = &aux_int;
+	for(i=0; i<user_reconf.numS; i++) {
+          if(rank_states[i] == MAM_PROC_CONTINUE) {
+            dist_data.myId += user_reconf.numS;
+	    break;
+	  }
+	}
+      }
+      get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
+      CreateSparseMatrixVptr(&user_data->other_subm, dist_data.tamBl, n, 0);
+      user_data->other_subm.vptr[0] = 0;
+      //memcpy(user_data->other_subm.vptr+1, vlen, dist_data.tamBl * sizeof(int));
+      for(i=0; i<dist_data.tamBl; i++) {
+        user_data->other_subm.vptr[i+1] = user_data->recv_vlen[i];
+      }
+      TransformLengthtoHeader(user_data->other_subm.vptr, user_data->other_subm.dim1); // The array is converted from vlen to vptr
+      elems = user_data->other_subm.vptr[dist_data.tamBl];
+      CreateSparseMatrixValues(&user_data->other_subm, dist_data.tamBl, n, elems, 0);
+      recv_vpos = user_data->other_subm.vpos;
+      recv_vval = user_data->other_subm.vval;
+
+      prepare_redist_counts(rcounts, rdispls, user_reconf.numS, offset, dist_data, user_data->other_subm.vptr);
+    }
+    if(user_reconf.rank_state != MAM_PROC_NEW_RANK) {
+      MPI_Comm_rank(user_data->comm, &dist_data.myId);
+      dist_data.numP = user_reconf.numS;
+      get_dist(n, dist_data.myId, dist_data.numP, &dist_data);
+      offset = (user_reconf.numS + user_reconf.numT) == numP ? 
+	      user_reconf.numS : 0; 
+      prepare_redist_counts(scounts, sdispls, user_reconf.numT, offset, dist_data, user_data->array_vptr);
+    }
+    // COMUNICACION DE DATOS //
+    MPI_Alltoallv(user_data->array_vpos, scounts, sdispls, MPI_INT,    recv_vpos, rcounts, rdispls, MPI_INT,    user_reconf.comm);
+    MPI_Alltoallv(user_data->array_vval, scounts, sdispls, MPI_DOUBLE, recv_vval, rcounts, rdispls, MPI_DOUBLE, user_reconf.comm);
+
+    free(rank_states);
+    free(scounts); free(sdispls); free(rcounts); free(rdispls);
+}
+
 /*
  * ========================================================================================
  * ========================================================================================

BiCGStab_Iker/ToolsMAM.h

@@ -29,7 +29,7 @@ static const user_redist_t empty_user_data = {
 };
 
 extern void targets_distribution(mam_user_reconf_t user_reconf, user_redist_t *user_data);
-//extern void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data);
+extern void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data);
 extern void print_global_results(double start_time);
 
 #endif