22 6 22 MPI MPI 1 1 2 2 3 MPI 3 4 7 4.1.................................. 7 4.2 ( )................................ 10 4.3 (Allreduce )................................. 12 5 14 5.1........................................ 14 5.2..................................... 18 6 18 1 CPU 1
2 T 1 N n T n α = T 1 nt n (1) α = 1 100% 4 4 1 OpenMP MPI OpenMP OpenMP MPI (Message Passing Interface) MPI MPICH OpenMPI 1 OpenMP MPI MPI (trivial parallelization) 100% MPI Mac OS X 2 $ echo Hello Hello Hello mpirun $ mpirun -np 2 echo Hello Hello Hello Hello mpirun -np -np 2 Hello -np 4 4 Hello 4 1 OpenMP OpenMPI OpenMPI MPI OpenMP 2
MPI SPMD (Single Program Multiple Data) mpitest.cc main( argc,char **argv){ rank; MPI_Init(&argc,&argv); MPI_Comm_rank(MPI_COMM_WORLD,&rank); prf("my rank = %d\n",rank); mpic++ $ mpic++ mpitest.cc My rank = 0 My rank = 1 My rank = 2 My rank = 3 mpi.h MPI C/C++ MPI_Init MPI_Finalize MPI_Init main MPI MPI_ MPI_Comm_rank MPI_COMM_WORLD (Rank) MPI OpenMP OpenMP MPI MPI MPI mpic++(macos X) mpcc(aix IBM ) mpic++ g++ icc icpc $ icpc mpitest.cc -I/usr/local/include -L/usr/local/lib -lmpich -lrt ( ) 3
3 MPI MPI MPI MPI List 1: ( ) #include <stdlib.h> double myrand(void){ return (double)rand()/(double)rand_max; double calc_pi( seed, trial){ srand(seed); n = 0; for( i=0;i<trial;i++){ double x = myrand(); double y = myrand(); if(x*x + y*y < 1.0){ n++; return 4.0*(double)n/(double)trial; main( argc, char **argv){ double pi = calc_pi(1,1000000); prf("%f \n",pi); calc_pi seed trial $./a.out 3.142096 1. 2. main MPI_Init MPI_Finalize 3. MPI_Comm_rank 4. calc_pi 4
List 2: ( 1) #include <stdlib.h> double myrand(void){ return (double)rand()/(double)rand_max; double calc_pi( seed, trial){ srand(seed); n = 0; for( i=0;i<trial;i++){ double x = myrand(); double y = myrand(); if(x*x + y*y < 1.0){ n++; return 4.0*(double)n/(double)trial; main( argc, char **argv){ rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); double pi = calc_pi(rank,1000000); prf("rank=%d: pi = %f \n",rank,pi); 4 rank=0: pi = 3.139268 rank=3: pi = 3.142288 rank=1: pi = 3.142096 rank=2: pi = 3.139256 rank=1 MPI_Comm_size main List 3: ( ) main( argc, char **argv){ rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); double pi = calc_pi(rank,1000000); prf("rank=%d:/%d pi = %f \n",rank,size,pi); 5
2 4 $ mpirun -np 2./a.out rank=0/2 pi = 3.139268 rank=1/2 pi = 3.142096 rank=1/4 pi = 3.142096 rank=0/4 pi = 3.139268 rank=2/4 pi = 3.139256 rank=3/4 pi = 3.142288 1,2,3 0 0 MPI MPI_Allreduce List 4: ( 3) main( argc, char **argv){ rank, procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); double pi = calc_pi(rank,1000000); prf("rank=%d/%d pi = %f \n",rank,procs,pi); MPI_Barrier(MPI_COMM_WORLD); double sum = 0; MPI_Allreduce(&pi, &sum, 1, MPI_DOUBLE, MPI_SUM,MPI_COMM_WORLD); sum = sum / (double)procs; if (0==rank){ prf("average = %f\n",sum); rank=0/4 pi = 3.139268 rank=1/4 pi = 3.142096 rank=2/4 pi = 3.139256 rank=3/4 pi = 3.142288 average = 3.140727 0 ( ) MPI_Allreduce MPI_Allreduce(void* senddata, void* recvdata, count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) senddata recvdata count datatype ( double MPI_DOUBLE) op ( MPI_SUM) MPI_COMM_WORLD MPI_Allreduce 6
pi ( ) sum sum MPI_Reduce MPI_Reduce MPI_Allreduce MPI_Reduce List 5: ( 3) main( argc, char **argv){ rank, procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); double pi = calc_pi(rank,1000000); prf("rank=%d/%d pi = %f \n",rank,procs,pi); MPI_Barrier(MPI_COMM_WORLD); double sum = 0; MPI_Reduce(&pi, &sum, 1, MPI_DOUBLE, MPI_SUM, 0,MPI_COMM_WORLD); if (0==rank){ sum = sum / (double)procs; prf("average = %f\n",sum); ( 0 ) sum 0 MPI_Allreduce MPI 2 4 4.1 C List 6: (C ) main( argc, char **argv){ 2 C/C++ ( & ) Fortran ( ) 7
value = 0; scanf("%d",&value); prf("value = %d\n",value); C++ List 7: (C++ ) #include <iostream> main( argc, char **argv){ value = 0; std::cin >> value; std::cout << "value = " << value << std::endl; $./a.out 123 value = 123 123 0 List 8: main( argc, char **argv){ rank = 0; MPI_Comm_rank(MPI_COMM_WORLD,&rank); value = 0; if(0 == rank){ scanf("%d",&value); MPI_Bcast(&value, 1, MPI_INT, 0, MPI_COMM_WORLD); prf("rank = %d: value = %d\n",rank, value); MPI_Bcast MPI_Bcast(void *buffer, count, MPI_Datatype datatype, root, MPI_Comm comm ) buffer ( ) count root ( 0 ) 123 0 123 rank = 0: value = 123 8
rank = 1: value = 123 rank = 2: value = 123 rank = 3: value = 123 ( ) (double ) parameter parameter List 9: struct parameter{ seed; double temperature; ; main( argc, char **argv){ rank = 0; MPI_Comm_rank(MPI_COMM_WORLD,&rank); parameter param; if(0 == rank){ scanf("%d",¶m.seed); scanf("%lf",¶m.temperature); MPI_Bcast(¶m, sizeof(param), MPI_BYTE, 0, MPI_COMM_WORLD); prf("rank = %d: seed = %d temperature = %f\n",rank, param.seed, param. temperature); sizeof MPI_BYTE 123 0.7 rank = 0: seed = 123 temperature = 0.700000 rank = 1: seed = 123 temperature = 0.700000 rank = 2: seed = 123 temperature = 0.700000 rank = 3: seed = 123 temperature = 0.700000 123 0.7 input.cfg $ cat input.cfg 123 0.7 < input.cfg rank = 0: seed = 123 temperature = 0.700000 rank = 1: seed = 123 temperature = 0.700000 rank = 2: seed = 123 temperature = 0.700000 rank = 3: seed = 123 temperature = 0.700000 C C++ 9
List 10: (C++ ) #include <iostream> struct parameter{ seed; double temperature; ; main( argc, char **argv){ rank = 0; MPI_Comm_rank(MPI_COMM_WORLD,&rank); parameter param; if(0 == rank){ std::cin >> param.seed; std::cin >> param.temperature; MPI_Bcast(¶m, sizeof(param), MPI_BYTE, 0, MPI_COMM_WORLD); std::cout << "rank = " << rank; std::cout << " seed = " << param.seed; std::cout << " temperature = " << param.temperature << std::endl; 4.2 ( ) MPI conf.000.dat,conf.001.dat List 11: (C ) main(void){ const SIZE = 10; array[size]; for( i=0;i<size;i++){ array[i] = i; FILE *fp = fopen("data.dat","w"); for( i=0;i<size;i++){ fprf(fp,"%d\n",array[i]); fclose(fp); #include <iostream> #include <fstream> main(void){ List 12: (C++ ) 10
const SIZE = 10; array[size]; for( i=0;i<size;i++){ array[i] = i; std::ofstream ofs("data.dat"); for( i=0;i<size;i++){ ofs << array[i] << std::endl; $./a.out $ cat data.dat 0 1 2 3 4 5 6 7 8 9 array data.dat ( 0 ) ( SIZE 2 ) List 13: (C ) main( argc, char **argv){ rank, procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); const SIZE = 2; array[size]; for( i=0;i<size;i++){ array[i] = rank; FILE *fp; if(0 == rank){ fp = fopen("data.dat","w"); fclose(fp); for( j=0;j<procs;j++){ MPI_Barrier(MPI_COMM_WORLD); if(j!= rank)continue; fp = fopen("data.dat","a"); for( i=0;i<size;i++){ fprf(fp,"%d\n",array[i]); fclose(fp); 11
$ cat data.dat 0 0 1 1 2 2 3 3 0 data.dat C++ List 14: (C++ ) #include <iostream> #include <fstream> main( argc, char** argv){ rank, procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); const SIZE = 2; array[size]; for( i=0;i<size;i++){ array[i] = rank; if(0==rank){ std::ofstream ofs("data.dat"); ofs.close(); for( j=0;j<procs;j++){ MPI_Barrier(MPI_COMM_WORLD); if(j!=rank)continue; std::ofstream ofs("data.dat",std::ios::app); for( i=0;i<size;i++){ ofs << array[i] << std::endl; ofs.close(); 4.3 (Allreduce ) 12
double double List 15: (C ) main( argc, char **argv){ const SIZE = 10; double data[size]; for( i=0;i<size;i++){ data[i] = (double)i; FILE *fp = fopen("data.dat","wb"); fwrite(data,sizeof(double),size,fp); fclose(fp); data.dat hexdump 3 $./a.out $ hexdump -v -e "%f\n" data.dat 0.000000 1.000000 2.000000 3.000000 4.000000 5.000000 6.000000 7.000000 8.000000 9.000000 data 0 0 List 16: Gather(C ) main( argc, char **argv){ rank,procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); const SIZE = 2; double data[size]; for( i=0;i<size;i++){ data[i] = (double)rank; double *buf; if(0==rank){ buf = new double[size*procs]; MPI_Gather(data, SIZE, MPI_DOUBLE, buf, SIZE, MPI_DOUBLE, 0, MPI_COMM_WORLD); if(0==rank){ 3 hexdump c prf hexdump *( ) -v 13
FILE *fp = fopen("data.dat","wb"); fwrite(buf,sizeof(double),size*procs,fp); fclose(fp); delete [] buf; 0 MPI_Gather data buf MPI_Gather MPI_Gather(void *sendbuffer, sendcount, MPI_Datatype sendtype, void *recvbuffer, recvcount, MPI_Datatype recvtype, root, MPI_Comm comm ) sendcount recvcount sendtype recvtype root $ hexdump -v -e "%f\n" data.dat 0.000000 0.000000 1.000000 1.000000 2.000000 2.000000 3.000000 3.000000 C++ #include <iostream> #include <fstream> List 17: (C++ ) main( argc, char **argv){ const SIZE = 10; double data[size]; for( i=0;i<size;i++){ data[i] = (double)i; std::ofstream ofs("data.dat", std::ios::binary); ofs.write((char*)data,sizeof(double)*size); #include <iostream> #include <fstream> List 18: Gather(C++ ) main( argc, char **argv){ rank,procs; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &procs); const SIZE = 2; double data[size]; 14
for( i=0;i<size;i++){ data[i] = (double)rank; double *buf; if(0==rank){ buf = new double[size*procs]; MPI_Gather(data, SIZE, MPI_DOUBLE, buf, SIZE, MPI_DOUBLE, 0, MPI_COMM_WORLD); if(0==rank){ std::ofstream ofs("data.dat",std::ios::binary); ofs.write((char*)buf,sizeof(double)*size*procs); delete [] buf; 5 5.1 MPI_Send MPI_Recv 0 1 List 19: main( argc, char **argv){ rank; MPI_Comm_rank(MPI_COMM_WORLD,&rank); send_value = rank; recv_value = -1; const TAG = 0; MPI_Status st; if(0==rank){ MPI_Send(&send_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD); else if(1==rank){ MPI_Recv(&recv_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD,&st); prf("rank = %d: recv_value = %d\n",rank, recv_value); $ mpirun -np 2./a.out rank = 0: recv_value = -1 rank = 1: recv_value = 0 MPI_Send MPI_Recv MPI_Send(void *sendbuffer, sendcount, MPI_Datatype sendtype, sendtag, dest, MPI_Comm comm ) MPI_Recv(void *recvbuffer, recvcount, MPI_Datatype recvtype, recvtag, src, MPI_Status *st, MPI_Comm comm ) sendbuffer recvbuffer sendcount recvcount sendtype recvtype tag 15
0 MPI_Status MPI_Send MPI_Recv 4 rank = 0: recv_value = -1 rank = 1: recv_value = 0 rank = 2: recv_value = -1 rank = 3: recv_value = -1 2 3 0 1 4 List 20: main( argc, char **argv){ rank; MPI_Comm_rank(MPI_COMM_WORLD,&rank); send_value = rank; recv_value = -1; const TAG = 0; MPI_Status st; if(0==rank){ MPI_Send(&send_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD); MPI_Recv(&recv_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD,&st); else if(1==rank){ MPI_Send(&send_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD); MPI_Recv(&recv_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD,&st); prf("rank = %d: recv_value = %d\n",rank, recv_value); MPI_Send MPI_Recv 0 1 List 21: main( argc, char **argv){ rank; MPI_Comm_rank(MPI_COMM_WORLD,&rank); send_value = rank; recv_value = -1; const TAG = 0; MPI_Status st; if(0==rank){ MPI_Send(&send_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD); 4 MPI 16
MPI_Recv(&recv_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD,&st); else if(1==rank){ MPI_Recv(&recv_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD,&st); MPI_Send(&send_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD); prf("rank = %d: recv_value = %d\n",rank, recv_value); $ mpirun -np 2./a.out rank = 0: recv_value = 1 rank = 1: recv_value = 0 MPI_Sendrecv MPI_Sendrecv MPI_Sendrecv(void *sendbuf, sendcount, MPI_Datatype sendtype, dest, sendtag, void *recvbuf, recvcount, MPI_Datatype recvtype, src, recvtag, MPI_Comm comm, MPI_Status *status ) MPI_Send MPI_Recv MPI_Sendrecv List 22: Sendrecv main( argc, char **argv){ rank; MPI_Comm_rank(MPI_COMM_WORLD,&rank); send_value = rank; recv_value = -1; const TAG = 0; MPI_Status st; if(0==rank){ MPI_Sendrecv(&send_value, 1, MPI_INT, 1, TAG, &recv_value, 1, MPI_INT, 1, TAG, MPI_COMM_WORLD,&st); else if(1==rank){ MPI_Sendrecv(&send_value, 1, MPI_INT, 0, TAG, &recv_value, 1, MPI_INT, 0, TAG, MPI_COMM_WORLD,&st); prf("rank = %d: recv_value = %d\n",rank, recv_value); MPI_Sendrecv List 23: Sendrecv main( argc, char **argv){ rank, procs; MPI_Comm_rank(MPI_COMM_WORLD,&rank); 17
MPI_Comm_size(MPI_COMM_WORLD,&procs); send_value = rank; recv_value = -1; dest_rank = (rank+1)%procs; src_rank = (rank-1+procs)%procs; const TAG = 0; MPI_Status st; MPI_Sendrecv(&send_value, 1, MPI_INT, dest_rank, TAG, &recv_value, 1, MPI_INT, src_rank, TAG, MPI_COMM_WORLD,&st); prf("rank = %d: recv_value = %d\n",rank, recv_value); if if if rank = 0: recv_value = 3 rank = 1: recv_value = 0 rank = 2: recv_value = 1 rank = 3: recv_value = 2 2 3 1 MPI_Sendrecv MPI_Send MPI_Recv MPI_Sendrecv 5.2 MPI_Send MPI_Recv MPI_Isend MPI_Irecv MPI_REQUEST_MAX MPI_Request_free 6 MPI 18