CUDA 5.0: CUBIN and CUBLAS_device, compute capability 3.5 - cuda

Question

I'm trying to compile a kernel that uses dynamic parallelism to run CUBLAS to a cubin file.
When I try to compile the code using the command
nvcc -cubin -m64 -lcudadevrt -lcublas_device -gencode arch=compute_35,code=sm_35 -o test.cubin -c test.cu
I get ptxas fatal : Unresolved extern function 'cublasCreate_v2
If I add the -rdc=true compile option it compiles fine, but when I try to load the module using cuModuleLoad I get error 500: CUDA_ERROR_NOT_FOUND. From cuda.h:
/**
* This indicates that a named symbol was not found. Examples of symbols
* are global/constant variable names, texture names, and surface names.
*/
CUDA_ERROR_NOT_FOUND = 500,
The kernel code:
#include <stdio.h>
#include <cublas_v2.h>
extern "C" {
__global__ void a() {
cublasHandle_t cb_handle = NULL;
cudaStream_t stream;
if( threadIdx.x == 0 ) {
cublasStatus_t status = cublasCreate_v2(&cb_handle);
cublasSetPointerMode_v2(cb_handle, CUBLAS_POINTER_MODE_HOST);
if (status != CUBLAS_STATUS_SUCCESS) {
return;
}
cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking);
cublasSetStream_v2(cb_handle, stream);
}
__syncthreads();
int jp;
double A[3];
A[0] = 4.0f;
A[1] = 5.0f;
A[2] = 6.0f;
cublasIdamax_v2(cb_handle, 3, A, 1, &jp );
}
}
NOTE: The scope of A is local, so the data at the pointer given to cublasIdamax_v2 is undefined, and so jp ends up as a more or less random value in this code. The correct way to do it would be to have A in global memory.
Host code:
#include <stdio.h>
#include <cuda.h>
#include <cuda_runtime_api.h>
int main() {
CUresult error;
CUdevice cuDevice;
CUcontext cuContext;
CUmodule cuModule;
CUfunction testkernel;
// Initialize
error = cuInit(0);
if (error != CUDA_SUCCESS) printf("ERROR: cuInit, %i\n", error);
error = cuDeviceGet(&cuDevice, 0);
if (error != CUDA_SUCCESS) printf("ERROR: cuInit, %i\n", error);
error = cuCtxCreate(&cuContext, 0, cuDevice);
if (error != CUDA_SUCCESS) printf("ERROR: cuCtxCreate, %i\n", error);
error = cuModuleLoad(&cuModule, "test.cubin");
if (error != CUDA_SUCCESS) printf("ERROR: cuModuleLoad, %i\n", error);
error = cuModuleGetFunction(&testkernel, cuModule, "a");
if (error != CUDA_SUCCESS) printf("ERROR: cuModuleGetFunction, %i\n", error);
return 0;
}
The host code is compiled using nvcc -lcuda test.cpp.
If I replace the kernel with a simple kernel (below) and compile it without -rdc=true, it works fine.
Simple working kernel
#include <stdio.h>
extern "C" {
__global__ void a() {
printf("hello\n");
}
}
Thanks in advance
Soren

Answer 1

You are just missing -dlink in your first approach:
nvcc -cubin -m64 -lcudadevrt -lcublas_device -gencode arch=compute_35,code=sm_35 -o test.cubin -c test.cu -dlink
You can also do that in two steps:
nvcc -m64 test.cu -gencode arch=compute_35,code=sm_35 -o test.o -dc
nvcc -dlink test.o -arch sm_35 -lcublas_device -lcudadevrt -cubin -o test.cubin