How to compile/link code with different compute capabilities against cublas_device? - cuda

Question

I'm working with dynamic parallelism (and cublas) in one of my kernels and want to provide a fallback-kernel for sm_20. In maxentropy_cuda.cu I wrote both kernels and used CUDA_ARCH to compile the dynamic parallelism kernel only for the architecture>=3.5. This works fine.
Part of the Makefile:
nvcc $(NVCCFLAGS) -gencode arch=compute_35,code=sm_35 -gencode arch=compute_20,code=sm_20 $(CINCL) -M maxentropy_cuda.cu -o maxentropy_cuda.d
nvcc --device-c $(NVCCFLAGS) -gencode arch=compute_35,code=sm_35 -gencode arch=compute_20,code=sm_20 -x cu maxentropy_cuda.cu -o maxentropy_cuda.o
When I link this to the kernels in another file:
nvcc --cudart static --relocatable-device-code=true -link -gencode arch=compute_35,code=sm_35 -gencode arch=compute_20,code=sm_20 $(LIBPATHS) -o main main.o selgen.o maxentropy.o maxentropy_omp.o maxentropy_cuda.o maxentropy_kernels.o $(OBJINFRA) $(LIBS) -lcublas_device -lcudadevrt
I get the following error:
nvlink fatal : could not find compatible device code in /opt/cuda/lib64/libcublas_device.a
make: *** [main] Error 255
Of course I don't need libcublas_device for the fallback-kernel...
Is there a way to get both compute-capabilities in one binary? (I'm using CUDA 5.5)
EDIT: Example (haven't tested the output...):
#include <stdio.h>
#include <cuda_runtime.h>
#include <cublas_v2.h>
__global__ void calc_dev(double* u, double* s, double* r) {
#if __CUDA_ARCH__<350
printf("should not have been called - compiled for the wrong CUDA architecture");
#else
cublasHandle_t cnpHandle;
cublasStatus_t status = cublasCreate(&cnpHandle);
if (status != CUBLAS_STATUS_SUCCESS) {
printf("error while initializing cublas\n");
return;
}
status = cublasDdot(cnpHandle,5,u,1,s,1,r);
cudaDeviceSynchronize();
if (status != CUBLAS_STATUS_SUCCESS) {
printf("cublas error: u x s\n");
return;
}
#endif
}
void calc_host(double* u, double* s, double* r) {
cublasHandle_t cnpHandle;
cublasStatus_t status = cublasCreate(&cnpHandle);
cublasSetPointerMode(cnpHandle, CUBLAS_POINTER_MODE_DEVICE);
if (status != CUBLAS_STATUS_SUCCESS) {
printf("error while initializing cublas\n");
return;
}
status = cublasDdot(cnpHandle,5,u,1,s,1,r);
cudaThreadSynchronize();
if (status != CUBLAS_STATUS_SUCCESS) {
printf("cublas error: u x s\n");
return;
}
}
int main(int argc, char** argv) {
const int n = 5;
double u[n] = {0,1,2,4,8};
double s[n] = {1, 0.64570312500000004,
0.44203125000000004, 0.65804687500000003, 0.71976562500000008};
double r = 0.0;
double *dev_s,*dev_u,*dev_r;
cudaMalloc( (void**)&dev_s, sizeof(double)*n);
cudaMalloc( (void**)&dev_u, sizeof(double)*n);
cudaMalloc( (void**)&dev_r, sizeof(double));
cudaMemcpy( dev_s, s, n*sizeof(double), cudaMemcpyHostToDevice);
cudaMemcpy( dev_u, u, n*sizeof(double), cudaMemcpyHostToDevice);
#if __CUDA_ARCH__>=350
calc_dev<<<1,1>>>(dev_s,dev_u,dev_r);
#else
calc_host(dev_s,dev_u,dev_r);
#endif
cudaMemcpy( &r, dev_r, sizeof(double), cudaMemcpyDeviceToHost);
printf("%.3f\n",r);
return 0;
}
Nsight:
make all
Building file: ../main.cu
Invoking: NVCC Compiler
/usr/local/cuda-5.5/bin/nvcc -G -g -O0 -gencode arch=compute_20,code=sm_20 -gencode arch=compute_35,code=sm_35 -odir "" -M -o "main.d" "../main.cu"
/usr/local/cuda-5.5/bin/nvcc --device-c -G -O0 -g -gencode arch=compute_20,code=sm_20 -gencode arch=compute_35,code=sm_35 -x cu -o "main.o" "../main.cu"
Finished building: ../main.cu
Building target: test_sm_compatibility
Invoking: NVCC Linker
/usr/local/cuda-5.5/bin/nvcc --cudart static --relocatable-device-code=true -gencode arch=compute_20,code=sm_20 -gencode arch=compute_35,code=sm_35 -link -o "test_sm_compatibility" ./main.o -lcublas -lcublas_device
nvlink fatal : could not find compatible device code in /usr/local/cuda-5.5/bin/../targets/x86_64-linux/lib/libcublas_device.a
make: *** [test_sm_compatibility] Error 255

Answer 1

This is possible now in CUDA 6.0
The example you have posted compiles as-is using the compile commands you have shown.
The only difficulty is that there are a variety of nvlink warnings that will have to be ignored:
nvlink warning : SM Arch ('sm_20') not found in '/usr/local/cuda/bin/..//lib64/libcublas_device.a'
However a proper executable is built correctly.