First working version for Ethernet network. Found bug around MPICH+OFI for function MPI_Probe.

BiCGStab_Iker/BiCGStab.c

@@ -187,7 +187,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
     n = size; n_dist = sizeR; maxiter = 16 * size; rec_iter = maxiter / 2; umbral = 1.0e-8;
     myId = computeData->myId;
     state = -1;
-    reconfigure = 0; rec_iter = 1;
+    reconfigure = 0; rec_iter = 1; maxiter=100;
 
     while ((computeData->iter < maxiter) && (computeData->tol > umbral)) {
 
@@ -290,7 +290,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
         computeData->iter++;
         if (computeData->iter == rec_iter) { reconfigure = 1;}
 	if (reconfigure) {
-          //dump(computeData);
+//          dump(computeData);
 	  MAM_Checkpoint(&state, MAM_CHECK_COMPLETION, user_func, (void *) user_data);
 	  if(state == MAM_COMPLETED) {
 	    reconfigure = 0; 
@@ -298,7 +298,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
             targets_update(computeData, user_data);
             sizeR = computeData->matL.dim1; 
             n_dist = sizeR;
-            //dump(computeData);
+//            dump(computeData);
 	    }
 	}
     }
@@ -335,6 +335,7 @@ void BiCGStab_compute (Compute_data *computeData, user_redist_t *user_data) {
 
 void BiCGStab_free (Compute_data *computeData) {
 
+    RemoveDoubles (&computeData->x); RemoveDoubles (&computeData->b);
     RemoveDoubles (&computeData->aux); RemoveDoubles (&computeData->s); 
     RemoveDoubles (&computeData->q); RemoveDoubles (&computeData->r); 
     RemoveDoubles (&computeData->p); RemoveDoubles (&computeData->r0); RemoveDoubles (&computeData->y);
@@ -346,8 +347,6 @@ void BiCGStab_free (Compute_data *computeData) {
     RemoveDoubles (&computeData->res_err); RemoveDoubles (&computeData->x_exact); 
 #endif
 
-    RemoveDoubles (&computeData->x); 
-    RemoveDoubles (&computeData->b);
     RemoveInts (&computeData->sizes); RemoveInts (&computeData->dspls); 
     RemoveInts (&computeData->vlen);
     if (computeData->isOriginal) {
@@ -366,7 +365,8 @@ int main (int argc, char **argv) {
     int index = 0, indexL = 0;
     SparseMatrix mat  = {0, 0, NULL, NULL, NULL}, sym = {0, 0, NULL, NULL, NULL};
 
-    int root = 0, myId, nProcs, isTarget, numTarget, req;
+    int root = 0, myId, nProcs;
+    int isTarget, numTarget, req, initNumP;
     int dimL, dspL, *vdimL = NULL, *vdspL = NULL;
     SparseMatrix matL = {0, 0, NULL, NULL, NULL};
     double *sol1L = NULL, *sol2L = NULL;
@@ -391,6 +391,7 @@ int main (int argc, char **argv) {
     computeData.myId = myId;
     computeData.numP = nProcs;
     computeData.comm = MPI_COMM_WORLD;
+    initNumP = computeData.numP; 
     user_data = empty_user_data;
     user_data.comm = computeData.comm;
 
@@ -400,7 +401,7 @@ int main (int argc, char **argv) {
     if(isTarget) {
       targets_update(&computeData, &user_data);
       computeData.isOriginal = 0;
-            //dump(&computeData);
+//            dump(&computeData);
     } else {
     /***************************************/
       if (argc == 4) {
@@ -414,7 +415,7 @@ int main (int argc, char **argv) {
       }
     /***************************************/
 
-      printf ("A\n");
+//      printf ("A\n");
       CreateInts (&vdimL, nProcs); CreateInts (&vdspL, nProcs); 
       if(mat_from_file) {
           if (myId == root) {
@@ -431,7 +432,7 @@ int main (int argc, char **argv) {
             RemoveSparseMatrix (&mat);
             RemoveSparseMatrix (&sym);
           } 
-          printf ("B\n");
+//          printf ("B\n");
       }
       else {
           dim = size_param * size_param * size_param;
@@ -466,7 +467,7 @@ int main (int argc, char **argv) {
 
     /***************************************/
 
-      printf ("C\n");
+//      printf ("C\n");
 
       beta = 1.0 / sqrt(dim);
       if(mat_from_file) {
@@ -486,11 +487,11 @@ int main (int argc, char **argv) {
           }
       }
 
-      printf ("D\n");
+//      printf ("D\n");
 
 //      MPI_Scatterv (sol2, vdimL, vdspL, MPI_DOUBLE, sol2L, dimL, MPI_DOUBLE, root, MPI_COMM_WORLD); //FIXME It does not seem to do anything
 
-      printf ("E\n");
+//      printf ("E\n");
       computeData.sizes = vdimL;
       computeData.my_size = dimL;
       computeData.dspls = vdspL;
@@ -529,16 +530,22 @@ int main (int argc, char **argv) {
 //    } 
 
     beta = sqrt(beta);
-    if (computeData.myId == 0) 
+    if (computeData.myId == 0) {
         printf ("Error: %20.10e\n", beta);
+        print_global_results(computeData.t1);
+    }
 
     /***************************************/
     // Freeing memory
     BiCGStab_free (&computeData);
     RemoveDoubles (&sol2); 
+    MAM_Finalize ();
+
+    if(initNumP > numTarget && computeData.myId == 0) {
+      MPI_Abort(MPI_COMM_WORLD, -100);
+    }
     if(computeData.comm != MPI_COMM_WORLD && computeData.comm != MPI_COMM_NULL) MPI_Comm_free(&(computeData.comm));
 
-    MAM_Finalize ();
     MPI_Finalize ();
 
     return 0;

BiCGStab_Iker/ToolsMAM.c

@@ -277,6 +277,6 @@ void print_global_results(double start_time) {
   printf("T_spawn: %lf", sp_time);
   printf("\nT_SR: %lf", sy_time);
   printf("\nT_AR: %lf", asy_time);
-  printf("\nT_Malleability: %lf", mall_time);
-  printf("\nT_total: %lf\n", global_time);
+  printf("\nT_Malleability: %lf\n", mall_time);
+  //printf("T_total: %lf\n", global_time);
 }

BiCGStab_Iker/ToolsMAM.h

@@ -30,5 +30,6 @@ static const user_redist_t empty_user_data = {
 
 extern void targets_distribution(mam_user_reconf_t user_reconf, user_redist_t *user_data);
 //extern void targets_distribution_synch(mam_user_reconf_t user_reconf, user_redist_t *user_data);
+extern void print_global_results(double start_time);
 
 #endif

BiCGStab_Iker/ToolsMPI.c

@@ -154,13 +154,13 @@ int ComputeSprMatrixRecvWeights (int prc_src, int dimL, MPI_Comm comm) {
 	int tam, tam_int, tam_double, tam_ub;
 	MPI_Status sta;
 
+
 	// Definition of sizes
 	MPI_Type_size(MPI_INT, &tam_int);
 	MPI_Type_size(MPI_DOUBLE, &tam_double);
 	MPI_Type_size(MPI_UB, &tam_ub);
 	MPI_Probe (prc_src, Tag_Send_Packet_Matrix_To_Leaf, comm, &sta);
 	MPI_Get_count (&sta, MPI_BYTE, &tam);
-
 	// Return the number of nonzeros included in a packet
 	return (tam - (dimL*tam_int + tam_ub)) / (tam_int + tam_double);
 }
@@ -175,7 +175,7 @@ int DistributeMatrix (SparseMatrix spr, int index, ptr_SparseMatrix sprL, int in
 	// Getiing the parameter of the communicator
 	MPI_Comm_rank(comm, &myId); MPI_Comm_size(comm, &nProcs); 
 	// Broadcasting the matrix dimension
-	MPI_Bcast (&dim, 1, MPI_INT, root, MPI_COMM_WORLD); 
+	MPI_Bcast (&dim, 1, MPI_INT, root, comm); 
 
 	// Calculating the vectors of sizes (vdimL) and displacements (vdspl)
 	divL = (dim / nProcs); rstL = (dim % nProcs);

BiCGStab_Iker/makefile

@@ -23,8 +23,7 @@
 # ============================================================
 DIR = /home/martini/mam_bicgstab
 APP = malleability/build
-MAMFLAGS = -I$(DIR)/malleability -L$(DIR)/$(APP) -lmam
-#-Wl,-rpath=$(DIR)/$(APP) 
+MAMFLAGS = -I$(DIR)/malleability -L$(DIR)/$(APP) -Wl,-rpath=$(DIR)/$(APP) -lmam
 
 CC = mpicc
 CFLAGS = -std=c++11 -mavx -fabi-version=0 -Wall -fopenmp -I. -I${MKLROOT}/include -I${HOME}/libs