Pthreads and CudaMemcpyAsync - cuda

Question

I wrote a test program to test the following idea: (1) a cuda stream copies data to gpu. The copy is done in a pthread. (2) a second cuda stream reads and processes data. (3) One more data is copied by first stream only if the previous data is processed by the second stream.
However, it does not work: only copy first data and then waiting there.
#include "cuda.h"
#include <iostream>
#include <pthread.h>
const int UNPROCESSED = 1;
const int PROCESSED = 2;
const int DONE = 3;
const int RUNNING= 0;
const int NUM_OF_DATA = 100;
const int NUM_OF_BLOCKS = 1;
const int THREADS_PER_BLOCK = 1;
//int data_states[NUM_OF_DATA];
cudaStream_t cuda_stream[2];
volatile int* process_state;
volatile int* d_process_state;
volatile int* d_copier_state;
int* d_data_state;
int* h_data_states;
cudaError_t cuda_status;
using namespace std;
void* copy_data(void* arg){
int i=0;
//cout << "in copy_data" << endl;
while(i < NUM_OF_DATA){
if (*process_state != UNPROCESSED){
cout << "Now copy data " << i << " with state = " << h_data_states[i] << endl;
*process_state = UNPROCESSED;
cuda_status = cudaMemcpyAsync(d_data_state, &h_data_states[i], sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
if (cuda_status != cudaSuccess){
cout << "Error when allocating pinned host memory (full_instance_states)" << endl;
}
i++;
}
}
int copier_state = DONE;
cudaMemcpyAsync((void*) d_copier_state, &copier_state, sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
}
__global__ void process_data(volatile int* data_state, volatile int* process_state, volatile int* copier_state){
int i = 0;
printf(" i = %d\n", i);
while(*copier_state != DONE){
printf(" i = %d, copier_state = %d, data_state = %d\n", i, *copier_state, *data_state);
if(*data_state == UNPROCESSED){
printf("now processing data %d\n", i);
i++;
// process data here, skipped
*process_state = PROCESSED;
*data_state = PROCESSED;
//__threadfence_system();
}
}
printf("process_data is done\n");
}
int main(int argc, char **argv){
int i;
cudaSetDeviceFlags(cudaDeviceMapHost);
cuda_status = cudaMallocHost((void**) &process_state, NUM_OF_BLOCKS*sizeof(int), cudaHostAllocMapped);
if (cuda_status != cudaSuccess){
cout << "Error when allocating pinned host memory (full_instance_states)" << endl;
}
cudaHostGetDevicePointer((int**) &d_process_state, (int*) process_state, 0);
cuda_status = cudaMalloc((void**) &d_copier_state, NUM_OF_BLOCKS*sizeof(int));
if (cuda_status != cudaSuccess){
cout << "Error when allocating pinned host memory (full_instance_states)" << endl;
}
cudaMemset((void*)d_copier_state, RUNNING, sizeof(int));
cuda_status = cudaMallocHost((void**) &h_data_states, NUM_OF_DATA*sizeof(int), 0);
if (cuda_status != cudaSuccess){
cout << "Error when allocating pinned host memory (full_instance_states)" << endl;
}
for(i = 0; i < NUM_OF_DATA; i++){
h_data_states[i] = UNPROCESSED;
}
cudaStreamCreate(&cuda_stream[0]);
cudaStreamCreate(&cuda_stream[1]);
pthread_t thread;
int thread_state = pthread_create(&thread, NULL, &copy_data, h_data_states);
if(thread_state){
cout << "Error: unable to create thread (produce_instances), "<< thread_state << endl;
exit(-1);
}
//cout << "Starting kernel" << endl;
process_data<<<NUM_OF_BLOCKS, THREADS_PER_BLOCK, 0, cuda_stream[1]>>>(d_data_state, d_process_state, d_copier_state);
cudaDeviceSynchronize();
cudaFree(d_data_state);
cudaFree((void*) d_copier_state);
cudaFreeHost((void*) process_state);
return 0;
}

Answer 1

You never allocate d_data_state in any way. It is a NULL pointer throughout your program.
Therefore the usage here is invalid:
cuda_status = cudaMemcpyAsync(d_data_state, &h_data_states[i], sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
And when I run your program, I get the error printout from the next line of code.
Since your kernel also uses d_data_state (which is an invalid pointer) I get various invalid global read errors if I run your code with cuda-memcheck.
Since you have not allocated anything for d_data_state, your code cannot possibly work.
You had several other issues in your code as well. As just one example:
int copier_state = DONE;
cudaMemcpyAsync((void*) d_copier_state, &copier_state, sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
In order for cudaMemcpyAsync to work as expected (i.e. be asynchronous, and overlap with other stream activity) the host memory must be a pinned memory area. int copier_state = DONE; does not create a pinned allocation, so copying from that breaks the asynchronous overlap of the cudaMemcpyAsync operation.
Here is a version of your code that works correctly for me (now updated with some additional guards against race conditions):
#include <iostream>
#include <pthread.h>
#define cudaCheckErrors(msg) \
do { \
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
fprintf(stderr, "Fatal error: %s (%s at %s:%d)\n", \
msg, cudaGetErrorString(__err), \
__FILE__, __LINE__); \
fprintf(stderr, "*** FAILED - ABORTING\n"); \
exit(1); \
} \
} while (0)
const int UNPROCESSED = 1;
const int PROCESSED = 2;
const int DONE = 3;
const int RUNNING= 0;
const int NUM_OF_DATA = 100;
const int NUM_OF_BLOCKS = 1;
const int THREADS_PER_BLOCK = 1;
//int data_states[NUM_OF_DATA];
cudaStream_t cuda_stream[2];
volatile int* process_state;
volatile int* d_process_state;
volatile int* d_copier_state;
int* d_data_state;
int* h_data_states;
int* h_copier_state;
cudaError_t cuda_status;
using namespace std;
void* copy_data(void* arg){
int i=0;
cudaSetDevice(0);
//cout << "in copy_data" << endl;
while(i < NUM_OF_DATA){
if (*process_state != UNPROCESSED){
// cout << "Now copy data " << i << " with state = " << h_data_states[i] << endl;
*process_state = UNPROCESSED;
cudaMemcpyAsync(d_data_state, &(h_data_states[i]), sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
cudaStreamSynchronize(cuda_stream[0]);
cudaCheckErrors("thread cudaMemcpyAsync fail");
//*process_state = UNPROCESSED;
i++;
}
}
*h_copier_state = DONE;
cudaMemcpyAsync((void *)d_copier_state, h_copier_state, sizeof(int), cudaMemcpyHostToDevice, cuda_stream[0]);
cudaCheckErrors("thread cudaMemcpyAsync 2 fail");
// cout << "Thread finished" << endl;
return NULL;
}
__global__ void process_data(volatile int* data_state, volatile int* process_state, volatile int* copier_state){
int i = 0;
//printf(" i = %d\n", i);
while(*copier_state != DONE){
//printf(" i = %d, copier_state = %d, data_state = %d\n", i, *copier_state, *data_state);
if(*data_state == UNPROCESSED){
//printf("now processing data %d\n", i);
i++;
// process data here, skipped
*data_state = PROCESSED;
__threadfence_system();
*process_state = PROCESSED;
__threadfence_system();
}
}
// printf("process_data is done\n");
}
int main(int argc, char **argv){
int i;
cudaSetDevice(0);
cudaSetDeviceFlags(cudaDeviceMapHost);
cudaMallocHost((void**) &process_state, NUM_OF_BLOCKS*sizeof(int), cudaHostAllocMapped);
cudaCheckErrors("cudaMallocHost 1 fail");
cudaHostGetDevicePointer((int**) &d_process_state, (int*) process_state, 0);
cudaMalloc((void**) &d_copier_state, sizeof(int));
cudaCheckErrors("cudaMalloc 1 fail");
cudaMemset((void*)d_copier_state, RUNNING, sizeof(int));
cudaMallocHost((void**) &h_copier_state, sizeof(int), 0);
cudaCheckErrors("cudaMallocHost 3 fail");
*h_copier_state = RUNNING;
cudaMallocHost((void**) &h_data_states, NUM_OF_DATA*sizeof(int), 0);
cudaCheckErrors("cudaMallocHost 2 fail");
for(i = 0; i < NUM_OF_DATA; i++){
h_data_states[i] = UNPROCESSED;
}
cudaMalloc((void**) &d_data_state, sizeof(int));
cudaCheckErrors("cudaMalloc 2 fail");
cudaMemcpy((void*)d_data_state, &(h_data_states[0]), sizeof(int), cudaMemcpyHostToDevice);
cudaStreamCreate(&cuda_stream[0]);
cudaStreamCreate(&cuda_stream[1]);
pthread_t thread;
int thread_state = pthread_create(&thread, NULL, &copy_data, NULL);
if(thread_state){
cout << "Error: unable to create thread (produce_instances), "<< thread_state << endl;
exit(-1);
}
//cout << "Starting kernel" << endl;
process_data<<<NUM_OF_BLOCKS, THREADS_PER_BLOCK, 0, cuda_stream[1]>>>(d_data_state, d_process_state, d_copier_state);
cudaDeviceSynchronize();
return 0;
}
As an aside, it's not necessary to have all the complexity of pthreads to run one extra thread. After the cuda kernel launch, all of your pthread-code could have been inserted in the main host thread, and your program would still work correctly. The host thread runs asynchronously to, and in parallel to, the device kernel, after a kernel launch.