cudaEventQuery() blocks - cuda

Question

I am calling cudaEventQuery() in a periodic ITIMER callback register by the main program.
The thread is at cudaDeviceSynchronize() waiting for a GPU kernel to finish.
I am seeing that cudaEventQuery() does not return and gets blocked.
I have attached the program to this file and the callstack when cudaEventQuery() gets stuck.
I appreciate any information/help on eliminating this problem/bug.
Configuration
CUDA 4.1 on Nvidia Tesla 2070 GPU.
My program
#include <stdio.h>
#include <cuda.h>
#include <unistd.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#define CHECK_CU_ERROR(err, cufunc) \
if (err != CUDA_SUCCESS) \
{ \
printf ("Error %d for CUDA Driver API function '%s'.\n", \
err, cufunc); \
exit(-1); \
}
#define N 100000
static CUcontext context;
static CUdevice device;
cudaEvent_t event;
void event_handler(int signum)
{
printf("\n Timer triggered!");
if (cudaEventQuery(event) == cudaSuccess) {
printf("\n Event finished");
fflush(stdout);
} else {
printf("\n Event NOT finished");
fflush(stdout);
}
}
// Device code
__global__ void VecAdd(const int *A, const int *B, int *C, int size)
{
int i = blockDim.x * blockIdx.x + threadIdx.x;
for (long long m = 0; m < 10; m++)
for (long long n = 0; n < 100000; n++)
if (i < size)
C[i] = A[i] + B[i];
}
static void initVec(int *vec, int n)
{
for (int i = 0; i < n; i++)
vec[i] = i;
}
int *d_A;
int *d_B;
int *d_C;
static void *compute(void *ip)
{
size_t size = N * sizeof(int);
int threadsPerBlock = 0;
int blocksPerGrid = 0;
int *h_A, *h_B, *h_C;
//int id = (int) pthread_self() + 1;
// Allocate input vectors h_A and h_B in host memory
h_A = (int *) malloc(size);
h_B = (int *) malloc(size);
h_C = (int *) malloc(size);
// Initialize input vectors
initVec(h_A, N);
initVec(h_B, N);
memset(h_C, 0, size);
// Allocate vectors in device memory
cudaMalloc((void **) &d_A, size);
cudaMalloc((void **) &d_B, size);
cudaMalloc((void **) &d_C, size);
// Copy vectors from host memory to device memory
cudaMemcpyAsync(d_A, h_A, size, cudaMemcpyHostToDevice);
cudaMemcpyAsync(d_B, h_B, size, cudaMemcpyHostToDevice);
threadsPerBlock = 256;
blocksPerGrid = (N + threadsPerBlock - 1) / threadsPerBlock;
VecAdd<<<blocksPerGrid, threadsPerBlock,0>>>(d_A, d_B, d_C, N);
cudaEventCreate(&event);
cudaEventRecord(event);
printf("\n Record");
fflush(stdout);
struct sigaction sa;
struct itimerval timer;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = &event_handler;
sigaction(SIGALRM, &sa, NULL);
timer.it_value.tv_sec = 0;
timer.it_value.tv_usec = 250;
timer.it_interval.tv_sec = 1;
timer.it_interval.tv_usec = 250;
setitimer(ITIMER_REAL, &timer, NULL);
return 0;
}
int main(int argc, char *argv[])
{
CUresult err;
int deviceNum = 0;
int deviceCount = 0;
#if 0
// Try different flags
if (cudaSetDeviceFlags(cudaDeviceScheduleSpin) != cudaSuccess) {
printf("\n failed cudaSetDeviceFlags");
exit(-1);
}
#endif
err = cuInit(0);
CHECK_CU_ERROR(err, "cuInit");
err = cuDeviceGetCount(&deviceCount);
CHECK_CU_ERROR(err, "cuDeviceGetCount");
if (deviceCount == 0) {
printf("There is no device supporting CUDA.\n");
exit(-1);
}
err = cuDeviceGet(&device, deviceNum);
CHECK_CU_ERROR(err, "cuDeviceGet");
err = cuCtxCreate(&context, 0, device);
CHECK_CU_ERROR(err, "cuCtxCreate");
compute(0);
cudaDeviceSynchronize();
printf("\n SYNCed");
while (1);
return 0;
}
CALLSTACK where cudaEventQuery gets blocked
(gdb) bt
#0 0x00000037f520e034 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000037f5209345 in _L_lock_868 () from /lib64/libpthread.so.0
#2 0x00000037f5209217 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f7bb6fd75b7 in ?? () from /usr/lib64/libcuda.so.1
#4 0x00007f7bb6fd575a in ?? () from /usr/lib64/libcuda.so.1
#5 0x00007f7bb70062e3 in ?? () from /usr/lib64/libcuda.so.1
#6 0x00007f7bb700c3ec in ?? () from /usr/lib64/libcuda.so.1
#7 0x00007f7bb6fc95d8 in ?? () from /usr/lib64/libcuda.so.1
#8 0x00007f7bb6fb9c35 in ?? () from /usr/lib64/libcuda.so.1
#9 0x00007f7bb6a5ad57 in ?? () from /usr/local/cuda/lib64/libcudart.so.4
#10 0x00007f7bb6a8c4f2 in cudaEventQuery () from /usr/local/cuda/lib64/libcudart.so.4
#11 0x0000000000400e8d in event_handler (signum=14) at event_sampling.cu:40
#12 <signal handler called>
#13 0x00007f7bb7003791 in ?? () from /usr/lib64/libcuda.so.1
#14 0x00007f7bb6fd5786 in ?? () from /usr/lib64/libcuda.so.1
#15 0x00007f7bb70062e3 in ?? () from /usr/lib64/libcuda.so.1
#16 0x00007f7bb7006646 in ?? () from /usr/lib64/libcuda.so.1
#17 0x00007f7bb6fd5839 in ?? () from /usr/lib64/libcuda.so.1
#18 0x00007f7bb6fc86e0 in ?? () from /usr/lib64/libcuda.so.1
#19 0x00007f7bb6fa7d62 in ?? () from /usr/lib64/libcuda.so.1
#20 0x00007f7bb6a5e9d3 in ?? () from /usr/local/cuda/lib64/libcudart.so.4
#21 0x00007f7bb6a9318c in cudaDeviceSynchronize () from /usr/local/cuda/lib64/libcudart.so.4
#22 0x00000000004012b3 in main (argc=1, argv=0x7fff20bff048) at event_sampling.cu:157
(gdb)
Here is the same code after removing Driver APIs
/*
* Copyright 2011 NVIDIA Corporation. All rights reserved
*
* Sample app to demonstrate use of CUPTI library to obtain profiler
* event values by sampling.
*/
#include <stdio.h>
#include <cuda.h>
#include <unistd.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#define CHECK_CU_ERROR(err, cufunc) \
if (err != CUDA_SUCCESS) \
{ \
printf ("Error %d for CUDA Driver API function '%s'.\n", \
err, cufunc); \
exit(-1); \
}
#define N 100000
cudaEvent_t event;
void event_handler(int signum)
{
printf("\n Timer triggered!");
if (cudaEventQuery(event) == cudaSuccess) {
printf("\n Event finished");
fflush(stdout);
} else {
printf("\n Event NOT finished");
fflush(stdout);
}
}
// Device code
__global__ void VecAdd(const int *A, const int *B, int *C, int size)
{
int i = blockDim.x * blockIdx.x + threadIdx.x;
for (long long m = 0; m < 10; m++)
for (long long n = 0; n < 100000; n++)
if (i < size)
C[i] = A[i] + B[i];
}
static void initVec(int *vec, int n)
{
for (int i = 0; i < n; i++)
vec[i] = i;
}
int *d_A;
int *d_B;
int *d_C;
static void *compute(void *ip)
{
size_t size = N * sizeof(int);
int threadsPerBlock = 0;
int blocksPerGrid = 0;
int *h_A, *h_B, *h_C;
//int id = (int) pthread_self() + 1;
// Allocate input vectors h_A and h_B in host memory
h_A = (int *) malloc(size);
h_B = (int *) malloc(size);
h_C = (int *) malloc(size);
// Initialize input vectors
initVec(h_A, N);
initVec(h_B, N);
memset(h_C, 0, size);
// Allocate vectors in device memory
cudaMalloc((void **) &d_A, size);
cudaMalloc((void **) &d_B, size);
cudaMalloc((void **) &d_C, size);
// Copy vectors from host memory to device memory
cudaMemcpyAsync(d_A, h_A, size, cudaMemcpyHostToDevice);
cudaMemcpyAsync(d_B, h_B, size, cudaMemcpyHostToDevice);
threadsPerBlock = 256;
blocksPerGrid = (N + threadsPerBlock - 1) / threadsPerBlock;
VecAdd<<<blocksPerGrid, threadsPerBlock,0>>>(d_A, d_B, d_C, N);
cudaEventCreate(&event);
cudaEventRecord(event);
printf("\n Record");
fflush(stdout);
struct sigaction sa;
struct itimerval timer;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = &event_handler;
sigaction(SIGALRM, &sa, NULL);
timer.it_value.tv_sec = 0;
timer.it_value.tv_usec = 250;
timer.it_interval.tv_sec = 1;
timer.it_interval.tv_usec = 250;
setitimer(ITIMER_REAL, &timer, NULL);
return 0;
}
int main(int argc, char *argv[])
{
CUresult err;
#if 0
// Try different flags
if (cudaSetDeviceFlags(cudaDeviceScheduleSpin) != cudaSuccess) {
printf("\n failed cudaSetDeviceFlags");
exit(-1);
}
#endif
compute(0);
cudaDeviceSynchronize();
printf("\n SYNCed");
fflush(stdout);
while (1)sleep(10);
return 0;
}

Answer 1

You have a single thread which launches a kernel, schedules an event to be reached after the kernel, and then calls cudaDeviceSynchronize(). When your signal handler is reached, it tries to call into another CUDA API call which blocks.
You've ignored one of the basic principles of signal handlers which is to do as little as possible!
The real question is what are you actually trying to achieve? You could just wait on the event (cudaEventSynchronize()) for what you're doing here but if you're goal is something more complex you should think in more detail about how to achieve it, signal handlers are not the right way.