// asynch version - illustrates various forms of message passing in MPI

#include<stdio.h>
#include<stdlib.h>
#include<mpi.h>

// timings added.

#define SN 100019

int myid;

int sbuf[200000];  // for buffered send.

/* this program adds up an array of SN randomly generated integers */

void genarray(int A[], int iv,int procs);
void suminterval(int iv, int *sum);
void collect(int A[], int procs, int *sum);
int csum(int A[]);

int main(int argc, char* argv[])
{
  double t1, t2, t3, t4;  // for timing measurements
  int A[SN]; // array
  int sum =0;   // the overall sum
  int rank, size, iv;
  int regsum;  // conventional result
  MPI_Init(&argc,&argv);
  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
  MPI_Comm_size(MPI_COMM_WORLD,&size);
  
  if (argc>1) srandom(atoi(argv[1]));
  myid = rank;
  iv = (SN/(size-1)); // size of interval sent to procs 1+
  // remainder will be sumed by rank 0 process.

  if (rank==0) 
    { 
      genarray(A,iv,size); // generate and broadcast array
        t1 = MPI_Wtime();
      regsum = csum(A);
        t2 = MPI_Wtime();
    } 
  else suminterval(iv, &sum);
  if (rank==0)
    {
        t3 = MPI_Wtime();
      collect(A,size,&sum);
        t4 = MPI_Wtime();
      printf("correct sum is %d\n",regsum);
      printf("mp sum is %d\n",sum);
      printf("time for conventional sum: %f\n",t2-t1);
      printf("time for parallel sum: %f\n",t4-t3);
    }
  MPI_Finalize();
}

void genarray(int A[],int iv,int procs)
{
  int i, x;
  MPI_Status stat;
  MPI_Request sreq[128];   // assume procs is less than 128
  for(i=0;i<SN;i++) A[i] = random()%10000;
  //for(i=0;i<SN;i++) printf("%d ",A[i]); printf("\n");
  //  MPI_Buffer_attach(sbuf,200000); // effect is process-global
  for(i=1;i<=procs-1;i++)
    {
      // MPI_Send(A+(iv*(i-1)),iv,MPI_INT,i,i,MPI_COMM_WORLD); // synched
 
     //  MPI_Bsend(A+(iv*(i-1)),iv,MPI_INT,i,i,MPI_COMM_WORLD);
      // returns MPI_SUCCESS if ok
      // note that this is synchronized send
      MPI_Send_init(A+(iv*(i-1)),iv,MPI_INT,i,i,MPI_COMM_WORLD,sreq+i);
    }
  for(i=1;i<=procs-1;i++)  MPI_Start(sreq+i);      // like start thread
  for(i=1;i<=procs-1;i++)  MPI_Wait(sreq+i,&stat); // like join

  /* Yet another way to do non-blocking IO is Isend and Wait
  for(i=1;i<=procs-1;i++)
    MPI_Isend(A+(iv*(i-1)),iv,MPI_INT,i,i,MPI_COMM_WORLD,sreq+i);
  for(i=1;i<=procs-1;i++) MPI_Waitany(procs,sreq,&x,&stat); // select
  // There's also MPI_Waitall    
  */

} // genarray


void suminterval(int iv, int *sum)  // executed by all by root
{ int i, ax;
  int A[50000];
  MPI_Status stat;

  MPI_Request rreq;
  MPI_Recv_init(A,iv,MPI_INT,0,MPI_ANY_TAG,MPI_COMM_WORLD,&rreq);
  MPI_Start(&rreq);  
  MPI_Wait(&rreq,&stat);
  //  printf("source is %d\n",stat.MPI_SOURCE);
  //  MPI_Recv(A,iv,MPI_INT,0,MPI_ANY_TAG,MPI_COMM_WORLD,&stat);
  // stat.MPI_SOURCE is source proc, stat.MPI_TAG is TAG value
  // Can also use MPI_ANY_SOURCE for a form of *select*.

  // alternative using Irecv:
  //  MPI_Irecv(A,iv,MPI_INT,0,MPI_ANY_TAG,MPI_COMM_WORLD,&rreq);
  //  MPI_Wait(&rreq,&stat);

  ax = 0;
  for(i=0;i<iv;i++) ax += A[i];
  // ax now contains sum of interval.
  // reduce - add local ax to *sum on process rank 0:
  MPI_Reduce(&ax,sum,1,MPI_INT,MPI_SUM,0,MPI_COMM_WORLD);
//  printf("... %d has called reduced with ax %d\n",myid,ax);
}

// called by rank 0 process  - misnamed
void collect(int A[], int procs, int *sum)
{ int iv, i;
  int ax = 0;
  iv = SN/(procs-1);
  for (i=(iv*(procs-1));i<SN;i++)
    ax += A[i];
  MPI_Reduce(&ax,sum,1,MPI_INT,MPI_SUM,0,MPI_COMM_WORLD);
  // *sum +=ax;  apparently overwrites MPI_Reduce of other procs.
//  printf("... %d has called reduced with ax %d\n",myid,ax);
}

// conventional sum
int csum(int A[])
{  int i, ax;
   ax = 0;
   for(i=0;i<SN;i++) ax+=A[i];
   return ax;
}