NVProf for NCCL program - cuda

Question

When I want to use NVProf for NCCL problem with --metrics all, The profiling results always return me like
==2781== NVPROF is profiling process 2781, command: ./nccl_example 2 16
==2781== Profiling application: ./nccl_example 2 16
==2781== Profiling result:
No events/metrics were profiled.
My simple nccl program
#include <stdio.h>
#include "cuda_runtime.h"
#include "nccl.h"
int main(int argc, char* argv[])
{
ncclComm_t comms[4];
// managing 4 devices
int nDev = 3;
int size = 32*1024*1024;
int devs[4] = {0, 1, 2};
//allocating and initializing device buffers
float** sendbuff = (float**)malloc(nDev * sizeof(float*));
float** recvbuff = (float**)malloc(nDev * sizeof(float*));
cudaStream_t* s = (cudaStream_t*)malloc(sizeof(cudaStream_t)*nDev);
for (int i = 0; i < nDev; ++i) {
CUDACHECK(cudaSetDevice(i));
CUDACHECK(cudaMalloc(sendbuff + i, size * sizeof(float)));
CUDACHECK(cudaMalloc(recvbuff + i, size * sizeof(float)));
CUDACHECK(cudaMemset(sendbuff[i], 1, size * sizeof(float)));
CUDACHECK(cudaMemset(recvbuff[i], 0, size * sizeof(float)));
CUDACHECK(cudaStreamCreate(s+i));
}
//initializing NCCL
NCCLCHECK(ncclCommInitAll(comms, nDev, devs));
//calling NCCL communication API. Group API is required when using
//multiple devices per thread
NCCLCHECK(ncclGroupStart());
for (int i = 0; i < nDev; ++i)
NCCLCHECK(ncclAllReduce((const void*)sendbuff[i], (void*)recvbuff[i],
size, ncclFloat, ncclSum, comms[i], s[i]));
NCCLCHECK(ncclGroupEnd());
//synchronizing on CUDA streams to wait for completion of NCCL operation
for (int i = 0; i < nDev; ++i) {
CUDACHECK(cudaSetDevice(i));
CUDACHECK(cudaStreamSynchronize(s[i]));
}
//free device buffers
for (int i = 0; i < nDev; ++i) {
CUDACHECK(cudaSetDevice(i));
CUDACHECK(cudaFree(sendbuff[i]));
CUDACHECK(cudaFree(recvbuff[i]));
}
//finalizing NCCL
for(int i = 0; i < nDev; ++i)
ncclCommDestroy(comms[i]);
printf("Success \n");
return 0;
}
Because I need to know the detailed metrics of NCCL APIs such that I can get more insights of its performance.

Answer 1

That behavior is expected.
events, metrics, that are gathered by default pertain to CUDA device code activity. To see something that might be instructive, try profiling with --print-gpu-trace switch (and remove --metrics all).
The documented "metrics" don't apply to the operations (data copying) that NCCL is doing. They apply to CUDA kernels (i.e. CUDA device code activity).
nvprof does seem to have metrics that can be collected for NVLink activity. To see these, on a system that is applicable (e.g. has NVLink), run a command such as:
nvprof --query-metrics
or
nvprof --query-metrics |grep -i nvlink