Multiple Host threads launch CUDA kernels together - cuda

Question

I have encountered a very strange situation. Here is our code:
#include <omp.h>
#include <stdio.h>
#include <stdlib.h>
#include <cuda.h>
void initCuda(int g)
{
cudaDeviceProp prop;
if(cudaGetDeviceProperties(&prop, g) == cudaSuccess) printf("MP cnt: %d ,Concurrent Kernels:%d , AsyncEngineCount:%d , ThrdPerMP: %d\n",
prop.multiProcessorCount,prop.concurrentKernels,prop.asyncEngineCount,192);
cudaSetDevice(g);
}
__global__ void cudaJob(float *mem){
unsigned int tid=threadIdx.x+blockIdx.x*blockDim.x;
mem[tid]=-1e5;
while(mem[tid]<1.0e5){
mem[tid]=mem[tid]+1e-2;
}
}
void wrapper(int n,int b){
float** dmem=(float**)malloc(n*(sizeof(float*)));
cudaStream_t* stream=(cudaStream_t*)malloc(sizeof(cudaStream_t)*n);
dim3 grid=dim3(b,1,1);
dim3 block=dim3(192,1,1);//2496/13=192
for(int i=0;i<n;i++) {
cudaMalloc((void**)&dmem[i],192*b*sizeof(float));
cudaStreamCreate(&stream[i]);
}
for(int i=0;i<n;i++) cudaJob<<<grid,block,0,stream[i]>>>(dmem[i]);
for(int i=0;i<n;i++) {
cudaStreamDestroy(stream[i]);
cudaFree(dmem[i]);
}
free(stream);
free(dmem);
}
int main(int argc,char* argv[]){
initCuda(0);
int n=atoi(argv[1]);
int nthreads=atoi(argv[2]);
int b=atoi(argv[3]);
float t1=omp_get_wtime();
#pragma omp parallel num_threads(nthreads) firstprivate(nthreads,n,b)
{
#pragma omp barrier
float time=omp_get_wtime();
int id=omp_get_thread_num();
wrapper(n,b);
time=omp_get_wtime()-time;
printf("Num Threads: %d, Time: %f\n",id,time);
}
printf("total: %f\n",omp_get_wtime()-t1);
return 0;
}
So if we run ./main 1 8 1. It means that their will be 8 threads and each of them will launch one kernel. However sometimes the actual run time suggest that the kernels are not launch simultaneously:
MP cnt: 13 ,Concurrent Kernels:1 , AsyncEngineCount:2 , ThrdPerMP: 192
Num Threads: 0, Time: 3.788108
Num Threads: 6, Time: 6.661960
Num Threads: 7, Time: 9.535245
Num Threads: 2, Time: 12.408561
Num Threads: 5, Time: 12.410481
Num Threads: 1, Time: 12.411650
Num Threads: 4, Time: 12.412888
Num Threads: 3, Time: 12.414572
total: 12.414601
After some debuging we found that the problem may be caused by the cleaning up of the memory and stream. If we comment out all the cudaFree and StreamDestroy and free. Then the run time will suggest that everything is concurrent:
MP cnt: 13 ,Concurrent Kernels:1 , AsyncEngineCount:2 , ThrdPerMP: 192
Num Threads: 7, Time: 3.805691
Num Threads: 1, Time: 3.806201
Num Threads: 3, Time: 3.806624
Num Threads: 2, Time: 3.806695
Num Threads: 6, Time: 3.807018
Num Threads: 5, Time: 3.807456
Num Threads: 0, Time: 3.807486
Num Threads: 4, Time: 3.807792
total: 3.807799
At last we found that if we add an omp barrier right behind the kernel launching call. Then the cleaning up will not cause any problem:
for(int i=0;i<n;i++) cudaJob<<<grid,block,0,stream[i]>>>(dmem[i]);
#pragma omp barrier
for(int i=0;i<n;i++) {
cudaStreamDestroy(stream[i]);
cudaFree(dmem[i]);
}
So, we think that when multiple host threads are trying to clean up the memory and streams on the device, they may compete with each other. But we are not sure.
Is that right? Can any one help us remove the omp barrier? Because we don't think it is necessary for our problem.

Answer 1

Yes, cudaMalloc, cudaFree, and cudaStreamCreate are all synchronous, which means they will tend to serialize activity, by forcing any cuda calls issued before them to complete, before they execute.
The usual recommendation is to do all such allocations outside of time-critical code. Figure out how many allocations you need, allocate them up-front, then use (and perhaps re-use) them during your main processing loop, then free whatever is needed at the end.