Affinity and Localization of a 2D block distrubed array

Phil Merkey

Introduction

A question that always come up is how to handle multi-dimensional arrays. This examples gives a 2d array which has essentially a blocked checkerboard distribution. The array is declared 

shared int A[N][M][THREADS];
since it cyclic distributed (block size one). A[i][j][0] have affinity to thread zero for all i,j, A[i][j][1] have affinity to thread one, etc. To do the localization trick we implement the local array as an array of pointers, *B[N]. 

<blocked2d1.c>=
#include<stdio.h>
#include<upc.h>

#define N 3
#define M 4

shared int A[N][M][THREADS];

main()
{
   int k, i,j,t;
   int *B[N];

   if( MYTHREAD == 0 ){
     for(k=0; k<N*M*THREADS; k++) {
         *(&A[0][0][0]+k) = 10000*(k/(THREADS*M)) + 100*((k/THREADS)%M) + (k%THREADS);
     }
     for(i=0; i<N; i++) { for(j=0; j<M; j++) {
         for(t=0; t<THREADS; t++) 
            printf("  %06d", A[i][j][t]);
         printf("\n");
     } } 
   }
   upc_barrier;

   for(i=0; i< N; i++)
     B[i] = (int *)&A[i][0][MYTHREAD];

   for( t=0; t< THREADS; t++ ){
      upc_barrier;
      if( t != MYTHREAD) continue;
      printf("In THREAD %d, B[][] is\n", MYTHREAD);
      for(i=0; i<N; i++) {
        for(j=0; j<M; j++) 
           printf("  %06d", B[i][j]);
        printf("\n");
      }
   } 
}

There is nothing special about declaring the array as a 3D array. It is really just a block of memory and compiler keeps track of indices to give the the illusion that it is 3 dimensional. We could replace 

shared int A[N][M][THREADS];
with 
shared int C[N*M*THREADS];
Then we would have the following picture:
Local pointer to shared array [*] 

<blocked2d2.c>=
#include<stdio.h>
#include<upc.h>

#define N 3
#define M 4

shared int C[N*M*THREADS];

main()
{
   int k, i,j,t;
   int *B[N];

   if( MYTHREAD == 0 ){
     for(k=0; k<N*M*THREADS; k++) {
         *(C + k) = 10000*(k/(THREADS*M)) + 100*((k/THREADS)%M) + (k%THREADS);
     }
     for(i=0; i<N; i++) { for(j=0; j<M; j++) {
         for(t=0; t<THREADS; t++)
            printf("  %06d", C[i*M*THREADS + j*THREADS + t] );
         printf("\n");
     } }
   }
   upc_barrier;

   for(i=0; i< N; i++)
     B[i] = (int *)&C[i*M*THREADS + MYTHREAD];

   for( t=0; t< THREADS; t++ ){
      upc_barrier;
      if( t != MYTHREAD) continue;
      printf("In THREAD %d, B[][] is\n", MYTHREAD);
      for(i=0; i<N; i++) {
        for(j=0; j<M; j++)
           printf("  %06d", B[i][j]);
        printf("\n");
      }
   }
}