Faster array copy when using fewer threads in CUDA - cuda

Question

I tested two different approaches to copy a 2D array in a CUDA kernel.
The first one launchs blocks of TILE_DIM x TILE_DIM threads. Each block copy a tile of the array assigning one thread per element:
__global__ void simple_copy(float *outdata, const float *indata){
int x = blockIdx.x * TILE_DIM + threadIdx.x;
int y = blockIdx.y * TILE_DIM + threadIdx.y;
int width = gridDim.x * TILE_DIM;
outdata[y*width + x] = indata[y*width + x];
}
The second one is taken from the NVIDIA Blog. It is similar to the previous kernel but use TILE_DIM x BLOCK_ROWS threads per block. Each thread loops over multiple elements of the matrix:
__global__ void fast_copy(float *outdata, const float *indata)
{
int x = blockIdx.x * TILE_DIM + threadIdx.x;
int y = blockIdx.y * TILE_DIM + threadIdx.y;
int width = gridDim.x * TILE_DIM;
for (int k = 0 ; k < TILE_DIM ; k += BLOCK_ROWS)
outdata[(y+k)*width + x] = indata[(y+k)*width + x];
}
I run a test to compare these two approaches.
Both kernels perform coalescent access to the global memory, yet the second one seems to be noticeably faster.
The NVIDIA visual profiler confirms this test.
So how the second kernel manages to achieve a faster copy?
This is the complete code I used to test the kernels:
#include <stdio.h>
#include <stdlib.h>
#include <cuda.h>
#include <conio.h>
#define TILE_DIM 32
#define BLOCK_ROWS 8
/* KERNELS */
__global__ void simple_copy(float *outdata, const float *indata){
int x = blockIdx.x * TILE_DIM + threadIdx.x;
int y = blockIdx.y * TILE_DIM + threadIdx.y;
int width = gridDim.x * TILE_DIM;
outdata[y*width + x] = indata[y*width + x];
}
//###########################################################################
__global__ void fast_copy(float *outdata, const float *indata)
{
int x = blockIdx.x * TILE_DIM + threadIdx.x;
int y = blockIdx.y * TILE_DIM + threadIdx.y;
int width = gridDim.x * TILE_DIM;
for (int k = 0 ; k < TILE_DIM ; k += BLOCK_ROWS)
outdata[(y+k)*width + x] = indata[(y+k)*width + x];
}
//###########################################################################
/* MAIN */
int main(){
float *indata,*dev_indata,*outdata1,*dev_outdata1,*outdata2,*dev_outdata2;
cudaEvent_t start, stop;
float time1,time2;
int i,j,k;
int n_iter = 100;
int N = 2048;
cudaEventCreate(&start);
cudaEventCreate(&stop);
dim3 grid(N/TILE_DIM, N/TILE_DIM);
dim3 threads1(TILE_DIM,TILE_DIM);
dim3 threads2(TILE_DIM,BLOCK_ROWS);
// Allocations
indata = (float *)malloc(N*N*sizeof(float));
outdata1 = (float *)malloc(N*N*sizeof(float));
outdata2 = (float *)malloc(N*N*sizeof(float));
cudaMalloc( (void**)&dev_indata,N*N*sizeof(float) );
cudaMalloc( (void**)&dev_outdata1,N*N*sizeof(float) );
cudaMalloc( (void**)&dev_outdata2,N*N*sizeof(float) );
// Initialisation
for(j=0 ; j<N ; j++){
for(i=0 ; i<N ; i++){
indata[i + N*j] = i + N*j;
}
}
// Transfer to Device
cudaMemcpy( dev_indata, indata, N*N*sizeof(float),cudaMemcpyHostToDevice );
// Simple copy
cudaEventRecord( start, 0 );
for(k=0 ; k<n_iter ; k++){
simple_copy<<<grid, threads1>>>(dev_outdata1,dev_indata);
}
cudaEventRecord( stop, 0 );
cudaEventSynchronize( stop );
cudaEventElapsedTime( &time1, start, stop );
printf("Elapsed time with simple copy: %f\n",time1);
// Fast copy
cudaEventRecord( start, 0 );
for(k=0 ; k<n_iter ; k++){
fast_copy<<<grid, threads2>>>(dev_outdata2,dev_indata);
}
cudaEventRecord( stop, 0 );
cudaEventSynchronize( stop );
cudaEventElapsedTime( &time2, start, stop );
printf("Elapsed time with fast copy: %f\n",time2);
// Transfer to Host
cudaMemcpy( outdata1, dev_outdata1, N*N*sizeof(float),cudaMemcpyDeviceToHost );
cudaMemcpy( outdata2, dev_outdata2, N*N*sizeof(float),cudaMemcpyDeviceToHost );
// Check for error
float error = 0;
for(j=0 ; j<N ; j++){
for(i=0 ; i<N ; i++){
error += outdata1[i + N*j] - outdata2[i + N*j];
}
}
printf("error: %f\n",error);
/*// Print the copied matrix
printf("Copy\n");
for(j=0 ; j<N ; j++){
for(i=0 ; i<N ; i++){
printf("%f\t",outdata1[i + N*j]);
}
printf("\n");
}*/
cudaEventDestroy( start );
cudaEventDestroy( stop );
free(indata);
free(outdata1);
free(outdata2);
cudaFree(dev_indata);
cudaFree(dev_outdata1);
cudaFree(dev_outdata2);
cudaDeviceReset();
getch();
return 0;
}
//###########################################################################

Answer 1

I think you will find the answer by comparing the microcode for the two kernels.
When I compile these kernels for SM 3.0, the compiler completely unrolls the loop in the second kernel (since it knows it will iterate 4x). That probably explains the performance difference - CUDA hardware can use registers to cover memory latency as well as instruction latency. Vasily Volkov did a terrific presentation "Better Performance At Low Occupancy" on the topic a couple years ago (https://www.nvidia.com/content/GTC-2010/pdfs/2238_GTC2010.pdf).

Answer 2

Launching threads costs some GPU time. Less threads and more work per thread means less overhead of launching thread. That's why fast_copy() is faster.
But of course you still need enough number of threads and blocks to fully utilize the GPU.
In fact the following blog expands this idea further. It uses fixed number of blocks/threads to do work with arbitrary size by using Grid-stride loops. Several advantages of this method are discussed.
https://developer.nvidia.com/content/cuda-pro-tip-write-flexible-kernels-grid-stride-loops