Invalid device error returned by cuPointerGetAttribute() - cuda

Question

This question is the same that was previously asked here without being answered:
https://stackoverflow.com/questions/22996075/invalid-device-error-return-by-cupointergetattribute.
I use a GTX680, CUDA 6.5 toolkit, NVIDIA 340.46 kernel. The GPU has unified addressing capability and compute capability 3.0.
The following code returns CUDA_ERROR_INVALID_DEVICE:
CUDA_DR_ASSERT(cuMemAlloc(&dev_ptr, size));
CUDA_DR_ASSERT(cuPointerGetAttribute(&tokens, CU_POINTER_ATTRIBUTE_P2P_TOKENS, dev_ptr));
Has anyone (Sankar?) had similar problems and found the reason?
edit: this is the code I get the errors from:
CUDA_DR_ASSERT( cuInit(0) );
CUdevice dev;
CUDA_DR_ASSERT( cuDeviceGet(&dev, 0) );
CUDA_ASSERT(cudaSetDevice(dev));
CUdeviceptr dev_ptr;
std::size_t size = 2*65536;
CUDA_DR_ASSERT( cuMemAlloc( &dev_ptr, size ) );
uint flag = 1; // set CU_POINTER_ATTRIBUTE_SYNC_MEMOPS (set to 0 for unsetting this option)
CUDA_DR_ASSERT( cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, dev_ptr) );
CUDA_POINTER_ATTRIBUTE_P2P_TOKENS tokens;
CUDA_DR_ASSERT( cuPointerGetAttribute( &tokens, CU_POINTER_ATTRIBUTE_P2P_TOKENS, dev_ptr ) );

Answer 1

I have a system with a Quadro GPU at device 0 and a GeForce GPU at device 1.
Here's a fully worked example:
$ cat t642.cpp
#include <cuda.h>
#include <helper_cuda_drvapi.h>
#include <drvapi_error_string.h>
int main(int argc, char *argv[]){
int my_dev = 0;
int dev_count = 0;
if (argc > 1) my_dev=atoi(argv[1]);
CUcontext my_ctx;
checkCudaErrors(cuInit(0));
checkCudaErrors(cuDeviceGetCount(&dev_count));
if (my_dev > dev_count-1) {printf("device does not exist\n"); return 1;}
char deviceName[256];
checkCudaErrors(cuDeviceGetName(deviceName, 256, my_dev));
printf("using device %d, %s\n", my_dev, deviceName);
checkCudaErrors(cuCtxCreate(&my_ctx, 0, my_dev));
CUdeviceptr dev_ptr;
size_t size = 256;
CUDA_POINTER_ATTRIBUTE_P2P_TOKENS tokens;
checkCudaErrors(cuMemAlloc(&dev_ptr, size));
checkCudaErrors(cuPointerGetAttribute(&tokens, CU_POINTER_ATTRIBUTE_P2P_TOKENS, dev_ptr));
printf("success!\n");
return 0;
}
$ g++ -I/usr/local/cuda/include -I/usr/local/cuda/samples/common/inc t642.cpp -lcuda -o t642
$ ./t642 0
using device 0, Quadro 5000
success!
$ ./t642 1
using device 1, GeForce GT 640
checkCudaErrors() Driver API error = 0101 "CUDA_ERROR_INVALID_DEVICE (device specified is not a valid CUDA device)" from file <t642.cpp>, line 22.
$
Using a GeForce GPU with this mechanism (which is designed in support of GPUDirect RDMA) is not suppoorted. This is documented in the GPUDirect RDMA documentation, which states:
GPUDirect RDMA is available on both Tesla and Quadro GPUs.
And while it is not the crux of your issue, you may also wish to read the GPUDirect RDMA release notes, that indicate that this token mechanism was deprecated in CUDA 6.0.