Measuring of const memory access latency gives strange results - cuda

Question

I have 3 kernels that sum two numbers. The first adds two numbers in registers. The second takes one number from const memory and another one from register. The third takes two numbers from const memory.
According to article "Dissecting the NVIDIA Volta GPU Architecture via Microbenchmarking" latency of access to L1 constant cache is ~24, and latency of IADD instruction is 6 cycles for Pascal. Therefore for the first kernel I expected to get duration greater than 6, for the second greater than 24, and for the third greater than 24.
But, when I measure time with clock() I get 13, 12 and 214 respectively.
My videocard is GeForce GTX 1050 Mobile. CUDA 10.1.
Compilation command: nvcc -arch=sm_61 main.cu -o main
Below I present code of program and fragments of SASS code.
Program code:
#include <iostream>
#define RES_SIZE 10
__global__ void measureReg(int *res){
int a = res[0], b = res[1], c;
__shared__ int shdata[1];
for(int i=0;i<150;++i) __syncthreads(); // Covers latencies of accesses to global memory
unsigned int t1,t2;
t1 = clock();
asm volatile("add.s32 %0, %1, %2;": "=r"(c) : "r"(a), "r"(b) : "memory");
shdata[0] = c; //Prevents execution of clock() before add has finished
t2 = clock();
res[0] = t2 - t1;
}
__global__ void measureConst1(int *res, int n1){
int a,b;
a = res[0];
__shared__ int shdata[1];
for(int i=0;i<150;++i) __syncthreads();
unsigned int t1,t2;
t1 = clock();
asm volatile("add.s32 %0, %1, %2;": "=r"(b) : "r"(a), "r"(n1) : "memory");
shdata[0] = b;
t2 = clock();
res[0] = t2 - t1;
}
__global__ void measureConst2(int *res, int n1, int n2){
int a;
__shared__ int shdata[1];
unsigned int t1,t2;
t1 = clock();
asm volatile("add.s32 %0, %1, %2;": "=r"(a) : "r"(n1), "r"(n2) : "memory");
shdata[0] = a;
t2 = clock();
res[0] = t2 - t1;
}
int main(int argc, char** argv){
int hostRes[RES_SIZE], *devRes;
std::fill(hostRes, hostRes + RES_SIZE, 1);
cudaMalloc(&devRes, RES_SIZE*sizeof(int));
cudaMemcpy(devRes, hostRes, RES_SIZE*sizeof(int), cudaMemcpyHostToDevice);
measureReg<<<1,1>>>(devRes);
cudaMemcpy(hostRes, devRes, RES_SIZE*sizeof(int), cudaMemcpyDeviceToHost);
std::cout<<"IADD with registers: "<<hostRes[0]<<std::endl;
measureConst1<<<1,1>>>(devRes, 10);
cudaMemcpy(hostRes, devRes, RES_SIZE*sizeof(int), cudaMemcpyDeviceToHost);
std::cout<<"IADD with register and const mem: "<<hostRes[0]<<std::endl;
measureConst2<<<1,1>>>(devRes, 10, 20);
cudaMemcpy(hostRes, devRes, RES_SIZE*sizeof(int), cudaMemcpyDeviceToHost);
std::cout<<"IADD with const mem: "<<hostRes[0]<<std::endl;
cudaFree(devRes);
return 0;
}
Fragments of SASS code:
/* measureReg */
CS2R R4, SR_CLOCKLO ;
IADD R0, R0, R5 ;
STS [RZ], R0 ;
CS2R R5, SR_CLOCKLO ;
/* measureConst1 */
CS2R R4, SR_CLOCKLO ;
IADD R0, R0, c[0x0][0x148] ;
STS [RZ], R0 ;
CS2R R5, SR_CLOCKLO ;
/* measureConst2 */
CS2R R2, SR_CLOCKLO ;
MOV R0, c[0x0][0x148] ;
IADD R0, R0, c[0x0][0x14c] ;
STS [RZ], R0 ;
CS2R R0, SR_CLOCKLO ;

Answer 1

This is broken:
int hostRes[RES_SIZE], *devRes;
std::fill(hostRes, hostRes + RES_SIZE, 1);
cudaMemcpy(devRes, hostRes, RES_SIZE*sizeof(int), cudaMemcpyHostToDevice);
cudaMalloc(&devRes, RES_SIZE*sizeof(int));
The compiler issues a warning to the effect that devRes is used before its value is set. You should not ignore these warnings. The correct sequence is:
int hostRes[RES_SIZE], *devRes;
std::fill(hostRes, hostRes + RES_SIZE, 1);
cudaMalloc(&devRes, RES_SIZE*sizeof(int));
cudaMemcpy(devRes, hostRes, RES_SIZE*sizeof(int), cudaMemcpyHostToDevice);
With that change, and compiling your code for sm_61 using CUDA 10.2, and focused on your measureConst2 function (although the loop unrolling behavior is the same for all) I observe SASS that looks like this:
Fatbin elf code:
================
arch = sm_61
code version = [1,7]
producer = <unknown>
host = linux
compile_size = 64bit
code for sm_61
Function : _Z13measureConst2Piii
.headerflags #"EF_CUDA_SM61 EF_CUDA_PTX_SM(EF_CUDA_SM61)"
/* 0x001fbc00fde007f6 */
/*0008*/ MOV R1, c[0x0][0x20] ; /* 0x4c98078000870001 */
/*0010*/ NOP ; /* 0x50b0000000070f00 */
/*0018*/ NOP ; /* 0x50b0000000070f00 */
/* 0x001fd400ffe007ed */
/*0028*/ NOP ; /* 0x50b0000000070f00 */
/*0030*/ BAR.SYNC 0x0 ; /* 0xf0a81b8000070000 */
/*0038*/ MEMBAR.CTA ; /* 0xef98000000070000 */
/* 0x001fb800fde007ef */
/*0048*/ NOP ; /* 0x50b0000000070f00 */
/*0050*/ NOP ; /* 0x50b0000000070f00 */
/*0058*/ NOP ; /* 0x50b0000000070f00 */
/* 0x001fbc00fea007ff */
/*0068*/ BAR.SYNC 0x0 ; /* 0xf0a81b8000070000 */
/*0070*/ MEMBAR.CTA ; /* 0xef98000000070000 */
The above sequence repeats 149 times due to compiler unrolling and optimization...
/*1f18*/ NOP ; /* 0x50b0000000070f00 */
/* 0x001ffc00fdc007ef */
/*1f28*/ NOP ; /* 0x50b0000000070f00 */
/*1f30*/ NOP ; /* 0x50b0000000070f00 */
/*1f38*/ BAR.SYNC 0x0 ; /* 0xf0a81b8000070000 */
/* 0x001fd800fcc007f5 */
/*1f48*/ MEMBAR.CTA ; /* 0xef98000000070000 */
/*1f50*/ CS2R R2, SR_CLOCKLO ; /* 0x50c8000005070002 */
/*1f58*/ MOV R0, c[0x0][0x148] ; /* 0x4c98078005270000 */
/* 0x003f98001e4007f4 */
/*1f68*/ IADD R0, R0, c[0x0][0x14c] ; /* 0x4c10000005370000 */
/*1f70*/ STS [RZ], R0 ; /* 0xef5c00000007ff00 */
/*1f78*/ CS2R R0, SR_CLOCKLO ; /* 0x50c8000005070000 */
/* 0x001fc800fe2007f1 */
/*1f88*/ IADD R0, -R2, R0 ; /* 0x5c12000000070200 */
/*1f90*/ MOV R2, c[0x0][0x140] ; /* 0x4c98078005070002 */
/*1f98*/ MOV R3, c[0x0][0x144] ; /* 0x4c98078005170003 */
/* 0x001ffc00fde007f1 */
/*1fa8*/ STG.E [R2], R0 ; /* 0xeedc200000070200 */
/*1fb0*/ NOP ; /* 0x50b0000000070f00 */
/*1fb8*/ EXIT ; /* 0xe30000000007000f */
/* 0x001f8000fc0007ff */
/*1fc8*/ BRA 0x1fc0 ; /* 0xe2400fffff07000f */
/*1fd0*/ NOP; /* 0x50b0000000070f00 */
What we note is that the compiler has:
The compiler has unrolled your loop of 150 iterations (not sure what you were hoping to accomplish with that)
Your load of n1 and n1 (kernel arguments) is happening only once.
These loads occur here:
/*1f58*/ MOV R0, c[0x0][0x148] ; /* 0x4c98078005270000 */
/* 0x003f98001e4007f4 */
/*1f68*/ IADD R0, R0, c[0x0][0x14c] ; /* 0x4c10000005370000
These are loading kernel parameters through the __constant__ memory system (this is expected behavior). The __constant__ memory system is not the same as the L1 cache or the "L1 constant cache". Even if we ignore that point, it only makes sense to talk about the latency associated with cache access if the item requested is already in the cache. In the above SASS code, there is no reason to assume this is true. You are accessing the items only once, and therefore you are witnessing the latency associated with populating the __constant__ cache with items from global memory (which is ultimately how all data moves from host to device -- even kernel arguments).
You might now ask "why don't I see that long latency with measureConst1 ? In that case, your kernel design is a bit different and we see that the load behavior is also a bit different. The SASS looks like this:
Function : _Z13measureConst1Pii
.headerflags #"EF_CUDA_SM61 EF_CUDA_PTX_SM(EF_CUDA_SM61)"
/* 0x001fc800fe2007f6 */
/*0008*/ MOV R1, c[0x0][0x20] ; /* 0x4c98078000870001 */
/*0010*/ MOV R2, c[0x0][0x140] ; /* 0x4c98078005070002 */
/*0018*/ MOV R3, c[0x0][0x144] ; /* 0x4c98078005170003 */
/* 0x001fbc00fde000b1 */
/*0028*/ LDG.E R0, [R2] ; /* 0xeed4200000070200 */
/*0030*/ NOP ; /* 0x50b0000000070f00 */
/*0038*/ NOP ; /* 0x50b0000000070f00 */
/* 0x001fd400ffe007e9 */
/*0048*/ NOP ; /* 0x50b0000000070f00 */
/*0050*/ BAR.SYNC 0x0 ; /* 0xf0a81b8000070000 */
/*0058*/ MEMBAR.CTA ; /* 0xef98000000070000 */
(repeating ...)
/*1f68*/ MEMBAR.CTA ; /* 0xef98000000070000 */
/*1f70*/ CS2R R4, SR_CLOCKLO ; /* 0x50c8000005070004 */
/*1f78*/ IADD R0, R0, c[0x0][0x148] ; /* 0x4c10000005270000 */
/* 0x003fc800fcc000f2 */
/*1f88*/ STS [RZ], R0 ; /* 0xef5c00000007ff00 */
/*1f90*/ CS2R R5, SR_CLOCKLO ; /* 0x50c8000005070005 */
/*1f98*/ IADD R0, -R4, R5 ; /* 0x5c12000000570400 */
/* 0x001fa000fde007f1 */
/*1fa8*/ STG.E [R2], R0 ; /* 0xeedc200000070200 */
/*1fb0*/ NOP ; /* 0x50b0000000070f00 */
/*1fb8*/ NOP ; /* 0x50b0000000070f00 */
/* 0x001f8000ffe007ff */
/*1fc8*/ EXIT ; /* 0xe30000000007000f */
/*1fd0*/ BRA 0x1fd0 ; /* 0xe2400fffff87000f */
/*1fd8*/ NOP; /* 0x50b0000000070f00 */
Here we see at the very beginning of your kernel (due to your specific kernel design), the SASS is loading items from __constant__ that are immediately adjacent to the items you are loading in your timing region. Therefore its reasonable to assume that with some cache line load granularity, your timing region is now measuring something like cache latency, rather than global load latency.

Answer 2

Seems I have found answer on my question.
According to this question, there is immediate constant (IMC) and index constant (INC) caches. INC cache handles accesses created by LDC instruction, and IMC handles others.
I believe, that latency of L1 constant cache that is pointed out in the mentioned article "Dissecting the NVIDIA Volta GPU Architecture via Microbenchmarking" is average latency of these two caches. And latency of IMC cache is no longer than latency of access to register file.
To check these hypothesis I have written small benchmark, that contains 6 kernels. Each of these kernel consequentially read array of integers situated in "const" memory and measure latency of reads.
There is difference between these kernels. The first kernel ("imc") read data through IMC cache without any prefetching. The second one ("imc_with_imc_prefetch") also reads data through IMC, but at first prefetch data to IMC. The third one ("imc_with_inc_prefetch") prefetches data to INC and then reads from IMC. There are another 3 kernels: "inc", "inc_with_imc_prefetch", "inc_with_inc_prefetch".
According to obtained results, I have came to the conclusions:
IMC latency is 12, INC latency is 40. Average latency is 26 that is very close to latency of L1 constant cache in the mentioned article.
Cache line size of IMC and INC is 64 bytes.
There is L1.5 const cache with line size 256 bytes and average latency of 78. If it is accessed through IMC its latency is 60. And if it is accessed through INC its latency is 96.
Full code of the benchmark:
#include <iostream>
#define SMPL_COUNT 128
#define CONST_SIZE 10000 // Count of elements in constant array
__constant__ int carr[CONST_SIZE];
__global__ void imc(int* arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
// Reading from immediate constant cache
int sum = 0;
for(int i=0;i<SMPL_COUNT;++i) {
t1 = clock();
sum += carr[i];
temp[i] = sum;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = sum;
}
__global__ void imc_with_imc_prefetch(int* arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
const int stride = 32/4; // Make assumption that cache line is more or equal to 32 bytes
// Prefetch data to immediate constant cache
for(int i=0; i<(SMPL_COUNT+stride-1)/stride; ++i) arr[i] = carr[i*stride];
// Reading from immediate constant cache
int sum = 0;
for(int i=0;i<SMPL_COUNT;++i) {
t1 = clock();
sum += carr[i];
temp[i] = sum;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = sum;
}
__global__ void imc_with_inc_prefetch(int* arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
// Prefetch data to index constant cache
int index = carr[CONST_SIZE-1];
for(int i=0;i<SMPL_COUNT;++i)
index = *((int*)(((char*)carr)+index)); //Subtle way to avoid calling of shift instruction
arr[0] = index;
__syncthreads();
//Reading from immediate constant cache
index = 0;
for(int i=0;i<SMPL_COUNT;++i){
t1 = clock();
index += carr[i];
temp[i]=index;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = index;
}
__global__ void inc(int* arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
int index = carr[CONST_SIZE-1];
arr[SMPL_COUNT - 1] = index;
__syncthreads();
//Reading from index constant cache
for(int i=0;i<SMPL_COUNT;++i){
t1 = clock();
index = *((int*)(((char*)carr)+index));
temp[i] = index;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = index;
}
__global__ void inc_with_imc_prefetch(int* arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
//Prefetch data to index constant cache
const int stride = 32/4; // Make assumption that cache line is more or equal to 32 bytes
// Prefetch data to immediate constant cache
for(int i=0; i<(SMPL_COUNT+stride-1)/stride; ++i)
arr[i] = carr[i*stride];
int index = carr[CONST_SIZE-1];
arr[SMPL_COUNT - 1] = index;
__syncthreads();
//Reading from index constant cache
for(int i=0;i<SMPL_COUNT;++i){
t1 = clock();
index = *((int*)(((char*)carr)+index));
temp[i] = index;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = index;
}
__global__ void inc_with_inc_prefetch(int*arr, int* t){
unsigned int t1, t2;
__shared__ int temp[SMPL_COUNT];
__shared__ int times[SMPL_COUNT];
int index = carr[CONST_SIZE-1];
for(int i=0;i<SMPL_COUNT;++i){
index = carr[index/4];
}
arr[0] = index;
index = carr[CONST_SIZE-1];
arr[SMPL_COUNT - 1] = index;
__syncthreads();
for(int i=0;i<SMPL_COUNT;++i){
t1 = clock();
index = *((int*)(((char*)carr)+index));
temp[i] = index;
t2 = clock();
times[i] = t2-t1;
__syncthreads();
}
for(int i=0;i<SMPL_COUNT;++i) t[i] = times[i];
arr[0] = t2-t1;
arr[1] = index+2;
}
int main(int argc, char** argv){
int hostArr[SMPL_COUNT], *devArr, *devTimes;
int imc_times[SMPL_COUNT], imc_imc_times[SMPL_COUNT], imc_inc_times[SMPL_COUNT];
int inc_times[SMPL_COUNT], inc_imc_times[SMPL_COUNT], inc_inc_times[SMPL_COUNT];
cudaMalloc(&devArr, SMPL_COUNT*sizeof(int));
cudaMalloc(&devTimes, SMPL_COUNT*sizeof(int));
cudaMemset (carr, 0, CONST_SIZE*sizeof(int));
cudaMemset (devArr, 0, SMPL_COUNT*sizeof(int));
cudaMemset (devTimes, 0, SMPL_COUNT*sizeof(int));
for(int i=0;i<SMPL_COUNT;++i) hostArr[i]=4*(i+1);
cudaMemcpyToSymbol(carr, hostArr, SMPL_COUNT*sizeof(int));
imc<<<1,1>>>(devArr, devTimes);
cudaMemcpy(imc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
imc_with_imc_prefetch<<<1,1>>>(devArr, devTimes);
cudaMemcpy(imc_imc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
imc_with_inc_prefetch<<<1,1>>>(devArr, devTimes);
cudaMemcpy(imc_inc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
inc<<<1,1>>>(devArr, devTimes);
cudaMemcpy(inc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
inc_with_imc_prefetch<<<1,1>>>(devArr, devTimes);
cudaMemcpy(inc_imc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
inc_with_inc_prefetch<<<1,1>>>(devArr, devTimes);
cudaMemcpy(inc_inc_times, devTimes, SMPL_COUNT*sizeof(int), cudaMemcpyDeviceToHost);
cudaFree(devArr);
cudaFree(devTimes);
std::cout<<"IMC\tIMC(IMC)\tIMC(INC)\tINC\tINC(IMC)\tINC(INC)\n";
for(int i=0;i<SMPL_COUNT;++i){
std::cout<<imc_times[i]<<"\t"<<imc_imc_times[i]<<"\t"<<imc_inc_times[i]<<"\t";
std::cout<<inc_times[i]<<"\t"<<inc_imc_times[i]<<"\t"<<inc_inc_times[i]<<"\n";
}
return 0;
}