Reverse engineering the rendering of The Witcher 3, part 13b - witcher senses (outline map)

This post is a part of the series "Reverse engineering the rendering of The Witcher 3".


Welcome,

This is the second part of demystifying Witcher Senses effect from The Witcher 3: Wild Hunt.

Once again, example scene we are working on:

In the first post I showed a bit how "intensity map" is being generated.
We have one full-resolution R11G11B10_FLOAT texture which can look like this:

The green channel represents "traces" and red one - interesting objects Geralt can interact with.

Having this we can move to the next stage - I called it "outline map".

This is a bit strange 512x512 R16G16_FLOAT texture. What's important here, it's implemented in ping-pong fashion. That means, outline map from previous frame is input (along with intensity map) for generating a new outline map in current frame.

You can implement ping-pong buffers in many ways probably but my personal like is as follows (pseudocode):

 // Declarations  
 Texture2D m_texOutlineMap[2];  
 uint m_outlineIndex = 0;  
   
 // Rendering  
 void Render()  
 {  
   pDevCon->SetInputTexture( m_texOutlineMap[m_outlineIndex] );  
   pDevCon->SetOutputTexture( m_texOutlineMap[!m_outlineIndex] );  
   ...  
   pDevCon->Draw(...);  
   
   // after draw  
   m_outlineIndex = !m_outlineIndex;  
 }  

Such approach, when input is always [m_outlineIndex] and output is always [!m_outlineIndex] allows for nice flexibility in terms of applying postFXs in general.

Let's take a look at pixel shader:

 ps_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer cb3[1], immediateIndexed  
    dcl_sampler s0, mode_default  
    dcl_sampler s1, mode_default  
    dcl_resource_texture2d (float,float,float,float) t0  
    dcl_resource_texture2d (float,float,float,float) t1  
    dcl_input_ps linear v2.xy  
    dcl_output o0.xyzw  
    dcl_temps 4  
   0: add r0.xyzw, v2.xyxy, v2.xyxy  
   1: round_ni r1.xy, r0.zwzz  
   2: frc r0.xyzw, r0.xyzw  
   3: add r1.zw, r1.xxxy, l(0.000000, 0.000000, -1.000000, -1.000000)  
   4: dp2 r1.z, r1.zwzz, r1.zwzz  
   5: add r1.z, -r1.z, l(1.000000)  
   6: max r2.w, r1.z, l(0)  
   7: dp2 r1.z, r1.xyxx, r1.xyxx  
   8: add r3.xyzw, r1.xyxy, l(-1.000000, -0.000000, -0.000000, -1.000000)  
   9: add r1.x, -r1.z, l(1.000000)  
  10: max r2.x, r1.x, l(0)  
  11: dp2 r1.x, r3.xyxx, r3.xyxx  
  12: dp2 r1.y, r3.zwzz, r3.zwzz  
  13: add r1.xy, -r1.xyxx, l(1.000000, 1.000000, 0.000000, 0.000000)  
  14: max r2.yz, r1.xxyx, l(0, 0, 0, 0)  
  15: sample_indexable(texture2d)(float,float,float,float) r1.xyzw, r0.zwzz, t1.xyzw, s1  
  16: dp4 r1.x, r1.xyzw, r2.xyzw  
  17: add r2.xyzw, r0.zwzw, l(0.003906, 0.000000, -0.003906, 0.000000)  
  18: add r0.xyzw, r0.xyzw, l(0.000000, 0.003906, 0.000000, -0.003906)  
  19: sample_indexable(texture2d)(float,float,float,float) r1.yz, r2.xyxx, t1.zxyw, s1  
  20: sample_indexable(texture2d)(float,float,float,float) r2.xy, r2.zwzz, t1.xyzw, s1  
  21: add r1.yz, r1.yyzy, -r2.xxyx  
  22: sample_indexable(texture2d)(float,float,float,float) r0.xy, r0.xyxx, t1.xyzw, s1  
  23: sample_indexable(texture2d)(float,float,float,float) r0.zw, r0.zwzz, t1.zwxy, s1  
  24: add r0.xy, -r0.zwzz, r0.xyxx  
  25: max r0.xy, abs(r0.xyxx), abs(r1.yzyy)  
  26: min r0.xy, r0.xyxx, l(1.000000, 1.000000, 0.000000, 0.000000)  
  27: mul r0.xy, r0.xyxx, r1.xxxx  
  28: sample_indexable(texture2d)(float,float,float,float) r0.zw, v2.xyxx, t0.zwxy, s0  
  29: mad r0.w, r1.x, l(0.150000), r0.w  
  30: mad r0.x, r0.x, l(0.350000), r0.w  
  31: mad r0.x, r0.y, l(0.350000), r0.x  
  32: mul r0.yw, cb3[0].zzzw, l(0.000000, 300.000000, 0.000000, 300.000000)  
  33: mad r0.yw, v2.xxxy, l(0.000000, 150.000000, 0.000000, 150.000000), r0.yyyw  
  34: ftoi r0.yw, r0.yyyw  
  35: bfrev r0.w, r0.w  
  36: iadd r0.y, r0.w, r0.y  
  37: ishr r0.w, r0.y, l(13)  
  38: xor r0.y, r0.y, r0.w  
  39: imul null, r0.w, r0.y, r0.y  
  40: imad r0.w, r0.w, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  41: imad r0.y, r0.y, r0.w, l(146956042240.000000)  
  42: and r0.y, r0.y, l(0x7fffffff)  
  43: itof r0.y, r0.y  
  44: mad r0.y, r0.y, l(0.000000001), l(0.650000)  
  45: add_sat r1.xyzw, v2.xyxy, l(0.001953, 0.000000, -0.001953, 0.000000)  
  46: sample_indexable(texture2d)(float,float,float,float) r0.w, r1.xyxx, t0.yzwx, s0  
  47: sample_indexable(texture2d)(float,float,float,float) r1.x, r1.zwzz, t0.xyzw, s0  
  48: add r0.w, r0.w, r1.x  
  49: add_sat r1.xyzw, v2.xyxy, l(0.000000, 0.001953, 0.000000, -0.001953)  
  50: sample_indexable(texture2d)(float,float,float,float) r1.x, r1.xyxx, t0.xyzw, s0  
  51: sample_indexable(texture2d)(float,float,float,float) r1.y, r1.zwzz, t0.yxzw, s0  
  52: add r0.w, r0.w, r1.x  
  53: add r0.w, r1.y, r0.w  
  54: mad r0.w, r0.w, l(0.250000), -r0.z  
  55: mul r0.w, r0.y, r0.w  
  56: mul r0.y, r0.y, r0.z  
  57: mad r0.x, r0.w, l(0.900000), r0.x  
  58: mad r0.y, r0.y, l(-0.240000), r0.x  
  59: add r0.x, r0.y, r0.z  
  60: mov_sat r0.z, cb3[0].x  
  61: log r0.z, r0.z  
  62: mul r0.z, r0.z, l(100.000000)  
  63: exp r0.z, r0.z  
  64: mad r0.z, r0.z, l(0.160000), l(0.700000)  
  65: mul o0.xy, r0.zzzz, r0.xyxx  
  66: mov o0.zw, l(0, 0, 0, 0)  
  67: ret  

As you can see, output of outline map is divided to four equal squares and this is the first thing we need to look at:

   0: add r0.xyzw, v2.xyxy, v2.xyxy  
   1: round_ni r1.xy, r0.zwzz  
   2: frc r0.xyzw, r0.xyzw  
   3: add r1.zw, r1.xxxy, l(0.000000, 0.000000, -1.000000, -1.000000)  
   4: dp2 r1.z, r1.zwzz, r1.zwzz  
   5: add r1.z, -r1.z, l(1.000000)  
   6: max r2.w, r1.z, l(0)  
   7: dp2 r1.z, r1.xyxx, r1.xyxx  
   8: add r3.xyzw, r1.xyxy, l(-1.000000, -0.000000, -0.000000, -1.000000)  
   9: add r1.x, -r1.z, l(1.000000)  
  10: max r2.x, r1.x, l(0)  
  11: dp2 r1.x, r3.xyxx, r3.xyxx  
  12: dp2 r1.y, r3.zwzz, r3.zwzz  
  13: add r1.xy, -r1.xyxx, l(1.000000, 1.000000, 0.000000, 0.000000)  
  14: max r2.yz, r1.xxyx, l(0, 0, 0, 0)  

We start by calculating floor( TextureUV * 2.0 ), which gives:

To determine individual squares, a small function is used:

 float getParams(float2 uv)  
 {  
      float d = dot(uv, uv);  
      d = 1.0 - d;  
      d = max( d, 0.0 );  
   
      return d;  
 }  

Note that this function returns 1.0 when input is float2(0.0, 0.0).
We have this case in upper left corner. To have the same situtation for upper right corner, we have to subtract float2(1, 0) from floored texcoords, for green square subtract float2(0, 1) and for yellow one - float2(1.0, 1.0).

So:

   float2 flooredTextureUV = floor( 2.0 * TextureUV );  
   ...
     
   float2 uv1 = flooredTextureUV;  
   float2 uv2 = flooredTextureUV + float2(-1.0, -0.0);   
   float2 uv3 = flooredTextureUV + float2( -0.0, -1.0);  
   float2 uv4 = flooredTextureUV + float2(-1.0, -1.0);  
   
   float4 mask;  
   mask.x = getParams( uv1 );  
   mask.y = getParams( uv2 );  
   mask.z = getParams( uv3 );  
   mask.w = getParams( uv4 );  

Each of mask components is equal to one or zero and is responsible for one square within texture. For instance mask.r and mask.w:

mask.r

mask.w

Once we have obtainted mask, let's move further. Line 15 samples intensity map. Please note that intensity texture is R11G11B10_FLOAT, while we sample all rgba components. In this scenario, .a is set implicitly to 1.0f.

Texcoords used for this operation can be calculated as frac( TextureUV * 2.0 ). So result of this operations looks for example like this:

Do you see similarity?

The next step is really smart - a 4-components dot product (dp4) is performed:

  16: dp4 r1.x, r1.xyzw, r2.xyzw   

This way, in upper left square we have only red channel (therefore, only interesting objects), in upper right - only green channel (only traces) and in lower right - everything (because .w component of intensity was implicitly set to 1.0). Brilliant idea. The result of dot product looks this way:

Having this masterFilter, we are ready to determine outlines of objects. This is not that hard as one can expect. The algorithm is quite similar to the one applied to sharpen - we have to obtain max abs difference of values.

Here's what happens, we sample four texels near currently processsed one (important: texel size in this case is 1.0/256.0 !) and calculate maximum absolute differences for both red and green channels:

   float fTexel = 1.0 / 256;  
     
   float2 sampling1 = TextureUV + float2( fTexel, 0 );  
   float2 sampling2 = TextureUV + float2( -fTexel, 0 );  
   float2 sampling3 = TextureUV + float2( 0, fTexel );  
   float2 sampling4 = TextureUV + float2( 0, -fTexel );  
     
   float2 intensity_x0 = texIntensityMap.Sample( sampler1, sampling1 ).xy;  
   float2 intensity_x1 = texIntensityMap.Sample( sampler1, sampling2 ).xy;  
   float2 intensity_diff_x = intensity_x0 - intensity_x1;  
     
   float2 intensity_y0 = texIntensityMap.Sample( sampler1, sampling3 ).xy;  
   float2 intensity_y1 = texIntensityMap.Sample( sampler1, sampling4 ).xy;  
   float2 intensity_diff_y = intensity_y0 - intensity_y1;  
     
   float2 maxAbsDifference = max( abs(intensity_diff_x), abs(intensity_diff_y) );  
   maxAbsDifference = saturate(maxAbsDifference);  

Now - if we multiply filter and maxAbsDifference...

So simple and so effective.

Once we have outlines, we sample outline map from previous frame.
Then, to have "ghosting" effect we take a bit of parameters calculated with current pass and values from outline map.

Say "hi" to our old friend - integer noise. It's present here as well. Animation parameters ( cb3[0].zw ) are from constant buffer and they change with time.

   float2 outlines = masterFilter * maxAbsDifference;  
     
   // Sample outline map  
   float2 outlineMap = texOutlineMap.Sample( samplerLinearWrap, uv ).xy;  
     
   // I guess it's related with ghosting   
   float paramOutline = masterFilter*0.15 + outlineMap.y;  
   paramOutline += 0.35 * outlines.r;  
   paramOutline += 0.35 * outlines.g;  
     
   // input for integer noise  
   float2 noiseWeights = cb3_v0.zw;
   float2 noiseInputs = 150.0*uv + 300.0*noiseWeights;  
   int2 iNoiseInputs = (int2) noiseInputs;  
     
   float noise0 = clamp( integerNoise( iNoiseInputs.x + reversebits(iNoiseInputs.y) ), -1, 1 ) + 0.65; // r0.y  
     

Side note: If you would like to implement Witcher Senses on your own I suggest to clamp integer noise to [-1;1] range (as its website says). There is no clamp in original TW3 shader but without clamping I had awful artifacts and whole outline map was unstable.

Then, we sample outline map the same way as intensity map before (this time size of texel is 1.0/512.0) and calculate average value of  .x component:

  // sampling of outline map  
   fTexel = 1.0 / 512.0;  
     
   sampling1 = saturate( uv + float2( fTexel, 0 ) );  
   sampling2 = saturate( uv + float2( -fTexel, 0 ) );  
   sampling3 = saturate( uv + float2( 0, fTexel ) );  
   sampling4 = saturate( uv + float2( 0, -fTexel ) );  
     
   float outline_x0 = texOutlineMap.Sample( sampler0, sampling1 ).x;  
   float outline_x1 = texOutlineMap.Sample( sampler0, sampling2 ).x;  
   float outline_y0 = texOutlineMap.Sample( sampler0, sampling3 ).x;  
   float outline_y1 = texOutlineMap.Sample( sampler0, sampling4 ).x;  
   float averageOutline = (outline_x0+outline_x1+outline_y0+outline_y1) / 4.0;  

Then, following the assembly, a difference between average and value in that particular pixel is computed and perturbed with integer noise:

   // perturb with noise  
   float frameOutlineDifference = averageOutline - outlineMap.x;  
   frameOutlineDifference *= noise0;  

The next step is to perturb value from "old" outline map with noise - this is main line which gives blocky look to output texture.

There are some more calculations later and, at the very end, "damping" is caculated.

   // the main place with gives blocky look of texture  
   float newNoise = outlineMap.x * noise0;  
     
   float newOutline = frameOutlineDifference * 0.9 + paramOutline;  
   newOutline -= 0.24*newNoise;  
     
   // 59: add r0.x, r0.y, r0.z  
   float2 finalOutline = float2( outlineMap.x + newOutline, newOutline);  
     
   // * calculate damping  
   float dampingParam = saturate( cb3_v0.x );  
   dampingParam = pow( dampingParam, 100 );    
     
   float damping = 0.7 + 0.16*dampingParam;  
   
   
   // * final multiplication  
   float2 finalColor = finalOutline * damping;  
   return float4(finalColor, 0, 0);

Here is a small video which shows outline map in action:



If you are interested with complete pixel shader, it's here. It's compatible with RenderDoc.
What's interesting (and, to be honest, slightly frustrating) despite its assembly is the same as the original shader from Witcher 3, the final look of outline map in RenderDoc changes!

On a side note - in the last pass (link below) you will see that only .r channel of outline map is being used. So why do we need .g channel then? I guess it's some sort of ping-pong buffer within the texture - please note that .r contains .g channel + some new value.

We have arrived to the end of the second part. Go to the last one here.


I hope you enjoyed it.
Thanks for reading!

Reverse engineering the rendering of The Witcher 3, part 13a - witcher senses (objects & intensity map)

This post is a part of the series "Reverse engineering the rendering of The Witcher 3".


Welcome back!

So far, almost every effect/technique explained in this series was not really Witcher 3-related. I mean, you can find things like tonemapping, vignette or calculating average luminance almost in every modern video game. Even drunk effect is something quite widespread.

That's why I decided to take a closer look at "witcher senses" rendering mechanics. Since Geralt is a witcher, his senses are much more sensitive comparing to an ordinary human. Therefore, he can see and hear more than other people which greatly helps him with solving his investigations. Witcher senses mechanics allows player to visualize these traces.

Here is a demonstration of this effect:

And one more, in better lighting:

As you can see, there are 2 types of objects: the ones Geralt can interact with (yellow outline) and traces related with investigating (red outline). Once Geralt investigates red trace, it can turn to yellow one (video #1). Please note that whole screen gets more grey-ish and a fish-eye effect is applied (video #2)

The effect is quite complex so I decided to split it across three blog posts.
In the first one I will describe selection of objects, in the second one - generation of outline and the third one will focus on final combining all this together.

Selection of objects

As I mentioned, there are two types of objects so we want to distinguish between them. In The Witcher 3 it's done by using stencil buffer. While generating GBuffer meshes which are meant to be marked as "traces" (red) are rendered with stencil = 8. Meshes which are marked with yellow color as "interesting" are rendered with stencil = 4.

For example, the following two textures show example frame with visible witcher senses and corresponding stencil buffer:

Stencil buffer - a brief refreshment

Stencil buffer is generally quite often used to identify meshes drawn by assigning the same ID to certain categories of meshes.

The idea is to use Always function with Replace operator once stencil test passes and Keep operator in other cases.

Here's how to implement it with D3D11:

 D3D11_DEPTH_STENCIL_DESC depthstencilState;  
 // Set depth parameters....  
   
 // Enable stencil  
 depthstencilState.StencilEnable = TRUE;  
   
 // Read & write all bits  
 depthstencilState.StencilReadMask = 0xFF;  
 depthstencilState.StencilWriteMask = 0xFF;  
   
 // Stencil operator for front face  
 depthstencilState.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;  
 depthstencilState.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;  
 depthstencilState.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;  
 depthstencilState.FrontFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE;  
   
 // Stencil operator for back face.  
 depthstencilState.BackFace.StencilFunc = D3D11_COMPARISON_ALWAYS;  
 depthstencilState.BackFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;  
 depthstencilState.BackFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;  
 depthstencilState.BackFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE;  
   
 pDevice->CreateDepthStencilState( &depthstencilState, &m_pDS_AssignValue );  

Stencil value to write to the buffer is passed as StencilRef via API call:

 // from now on set stencil buffer values to 8  
 pDevCon->OMSetDepthStencilState( m_pDS_AssignValue, 8 );  
 ...  
 pDevCon->DrawIndexed( ... );  

Rendering intensities

For this pass, in terms of implementation there is one R11G11B10_FLOAT full-screen texture which will be used to save interesting objects and traces in R and G channel, respectively.

In terms of intensity - why exactly do we need it? It turns out that Geralt's senses have, well, limitied radius so the particular object starts to be outlined when player is close enough.

See this aspect in action:

We start by clearing the intensity texture with black color.
Then two fullscreen draw calls are peformed: the first one for "traces" and the second one for interesting objects:

The first draw call is for traces - green channel:

And the second one for interesting objects - red channel:

Okay, but how we distinguish here which pixels will be considered? We have to use stencil buffer!
During each of these calls a stencil test is performed to accept only these pixels which were marked before with "8" (first Draw call) or "4".

Visualization of stencil test for traces:

...and for interesting objects:

How the test is performed in this case? For basics about stencil test, here is good blog post about it. The general stencil test formula is as follows:

 if (StencilRef & StencilReadMask OP StencilValue & StencilReadMask)  
   accept pixel  
 else  
   discard pixel  

where:
StencilRef is value passed with API call,
StencilReadMask is a mask used to read stencil value (note it's present on both left and right side),
OP is operator used for comparing, it's set through API,
StencilValue is value of stencil buffer in currently processed pixel.

It's important to be aware that we use binary ANDs to calculate operands.

Knowing the basics, let's see settings used during these drawcalls:

Stencil state for traces

Stencil state for interesing objects

Ha! As we can see, ReadMask is the only difference. Let's try it! Let's subsitute these values to stencil test equation:

 Let StencilReadMask = 0x08 and StencilRef = 0:  
   
 For a pixel with stencil = 8:  
 0 & 0x08 < 8 & 0x08  
 0 < 8
 TRUE  
   
 For a pixel with stencil = 4:  
 0 & 0x08 < 4 & 0x08  
 0 < 0  
 FALSE  
   

Ha! Clever. As you can see, in this scenario we don't compare stencil value but rather we check if particular bit of stencil buffer value is set on. Every pixel of stencil buffer is uint8, so we have [0-255].

Side note: All DrawIndexed(36) calls are related with rendering footsteps as traces, so the final look of intensity map in this particular frame is:

But before stencil test there is a pixel shader. Both 28738 and 28748 use the same pixel shader:

 ps_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer cb0[2], immediateIndexed  
    dcl_constantbuffer cb3[8], immediateIndexed  
    dcl_constantbuffer cb12[214], immediateIndexed  
    dcl_sampler s15, mode_default  
    dcl_resource_texture2d (float,float,float,float) t15  
    dcl_input_ps_siv v0.xy, position  
    dcl_output o0.xyzw  
    dcl_output o1.xyzw  
    dcl_output o2.xyzw  
    dcl_output o3.xyzw  
    dcl_temps 2  
   0: mul r0.xy, v0.xyxx, cb0[1].zwzz  
   1: sample_indexable(texture2d)(float,float,float,float) r0.x, r0.xyxx, t15.xyzw, s15  
   2: mul r1.xyzw, v0.yyyy, cb12[211].xyzw  
   3: mad r1.xyzw, cb12[210].xyzw, v0.xxxx, r1.xyzw  
   4: mad r0.xyzw, cb12[212].xyzw, r0.xxxx, r1.xyzw  
   5: add r0.xyzw, r0.xyzw, cb12[213].xyzw  
   6: div r0.xyz, r0.xyzx, r0.wwww  
   7: add r0.xyz, r0.xyzx, -cb3[7].xyzx  
   8: dp3 r0.x, r0.xyzx, r0.xyzx  
   9: sqrt r0.x, r0.x  
  10: mul r0.y, r0.x, l(0.120000)  
  11: log r1.x, abs(cb3[6].y)  
  12: mul r1.xy, r1.xxxx, l(2.800000, 0.800000, 0.000000, 0.000000)  
  13: exp r1.xy, r1.xyxx  
  14: mad r0.zw, r1.xxxy, l(0.000000, 0.000000, 120.000000, 120.000000), l(0.000000, 0.000000, 1.000000, 1.000000)  
  15: lt r1.x, l(0.030000), cb3[6].y  
  16: movc r0.xy, r1.xxxx, r0.yzyy, r0.xwxx  
  17: div r0.x, r0.x, r0.y  
  18: log r0.x, r0.x  
  19: mul r0.x, r0.x, l(1.600000)  
  20: exp r0.x, r0.x  
  21: add r0.x, -r0.x, l(1.000000)  
  22: max r0.x, r0.x, l(0)  
  23: mul o0.xyz, r0.xxxx, cb3[0].xyzx  
  24: mov o0.w, cb3[0].w  
  25: mov o1.xyzw, cb3[1].xyzw  
  26: mov o2.xyzw, cb3[2].xyzw  
  27: mov o3.xyzw, cb3[3].xyzw  
  28: ret  

This pixel shader writes to only one render target, so lines 24-27 are redundant.

The first thing which takes place here is sampling depth (with point clamp sampler), line 1. This value is used to reconstruct world position by multiplication with special matrix and, after that, perspective division (lines 2-6).

Having Geralt's position ( cb3[7].xyz - please note this is not a camera position! ) distance from Geralt to this particular point is calculated (lines 7-9).

Inputs which are important for this shader are:
- cb3[0].rgb - color of output. This can be float3(0, 1, 0) (traces) or float3(1, 0, 0)  (interesting objects),
- cb3[6].y - distance scaling factor. This directly affects radius and intensity of final output.

Later we have a bit tricky formulas to calculate intensity depending on distance from Geralt to object. My guess is that all coefficents were selected experimentally.
Final output is color*intensity.


The HLSL would be something like this:

 struct FSInput  
 {  
      float4 param0 : SV_Position;  
 };  
   
 struct FSOutput  
 {  
      float4 param0 : SV_Target0;  
      float4 param1 : SV_Target1;  
      float4 param2 : SV_Target2;  
      float4 param3 : SV_Target3;  
 };  
   
 float3 getWorldPos( float2 screenPos, float depth )  
 {  
   float4 worldPos = float4(screenPos, depth, 1.0);  
   worldPos = mul( worldPos, screenToWorld );  
     
   return worldPos.xyz / worldPos.w;  
 }  
   
 FSOutput EditedShaderPS(in FSInput IN)  
 {  
   // * Inputs    
   // Directly affects radius of the effect  
   float distanceScaling = cb3_v6.y;  
     
   // Color of output at the end  
   float3 color = cb3_v0.rgb;  
        

   // Sample depth  
   float2 uv = IN.param0.xy * cb0_v1.zw;  
   float depth = texture15.Sample( sampler15, uv ).x;  
     
   // Reconstruct world position  
   float3 worldPos = getWorldPos( IN.param0.xy, depth );  
   
   // Calculate distance from Geralt to world position of particular object  
   float dist_geraltToWorld = length( worldPos - cb3_v7.xyz );  
     
   // Calculate two squeezing params  
   float t0 = 1.0 + 120*pow( abs(distanceScaling), 2.8 );  
   float t1 = 1.0 + 120*pow( abs(distanceScaling), 0.8 );  
     
   // Determine nominator and denominator  
   float2 params;  
   params = (distanceScaling > 0.03) ? float2(dist_geraltToWorld * 0.12, t0) : float2(dist_geraltToWorld, t1);  
     
   // Distance Geralt <-> Object  
   float nominator = params.x;   
     
   // Hiding factor  
   float denominator = params.y;  
     
   // Raise to power of 1.6  
   float param = pow( params.x / params.y, 1.6 );  
     
   // Calculate final intensity  
   float intensity = max(0.0, 1.0 - param );   
     
     
   // * Final outputs.  
   // *  
   // * This PS outputs only one color, the rest  
   // * is redundant. I just added this to keep 1-1 ratio with  
   // * original assembly.  
   FSOutput OUT = (FSOutput)0;  
   OUT.param0.xyz = color * intensity;  
     
   // == redundant ==  
   OUT.param0.w = cb3_v0.w;  
   OUT.param1 = cb3_v1;  
   OUT.param2 = cb3_v2;  
   OUT.param3 = cb3_v3;  
   // ===============  
   
   return OUT;  
 }  

And small comparison between original (left) and my (right) shader assembly.

This was the first stage of witcher senses effect. Actually, the easiest one.
Go to the second one here.

Feel free to comment and share.
Thanks for reading!

PS. This is my first post as BSc in Computer Science. Feels great! :)

Reverse engineering the rendering of The Witcher 3, part 12 - stupid sky tricks

This post is a part of the series "Reverse engineering the rendering of The Witcher 3".


Welcome,

This part will be slightly different comparing to the previous ones. Today I'd like to show you some aspects of sky shaders from The Witcher 3.

Why some "stupid tricks" instead of full shader? Well, there are a few reasons. First of all, sky shader in The Witcher 3 is quite a complex beast. Pixel Shader of 2015 version has 267 lines of assembly while PS from "Blood & Wine" DLC - 385.
Moreover, they have quite a lot of inputs which doesn't really help in struggles to reverse engineer complete (and readable!) HLSL code.

Therefore, I decided to show you some tricks from these shaders only. If I find anything new, this post will be updated.

The differences between 2015 version of the game and B&W (2016) addon are quite notable. This includes, for instance, different calculation of stars and their blinking, different approach to rendering of the Sun... Blood & Wine shader also calculates Milky Way during the night.

I'll start with some basics and switch to stupid tricks later.

At first, let's take a look at a clear night sky in The Witcher 3 (2015):

Basics

As most of modern video games, The Witcher 3 uses skydome to represent sky. Take a look at hemisphere used for this in The Witcher 3 (2015). On a side note, in this case bounding box of this mesh ranges from [0,0,0] to [1,1,1] (Z is up-axis) and has smoothly distributed UVs. We'll use them later.

The idea behind skydome is similar to skybox (mesh being used is the only difference). During vertex shader we translate a skydome with respect to observer (usually by camera position) which gives an illusion that sky is really far away - we'll never go there.

If you have been following the series for a while you know that The Witcher 3 uses reversed depth - that means, far plane is represented by 0.0f while near plane - by 1.0f. To make sure that output of skydome will be completely on far plane, we set MinDepth the same as MaxDepth of viewport parameters:

To learn how MinDepth and MaxDepth fields are used during viewport transform click here (docs.microsoft.com).

Vertex Shader

Let's start with vertex shader. In The Witcher 3 (2015) assembly of VS is as follows:

 vs_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer cb1[4], immediateIndexed  
    dcl_constantbuffer cb2[6], immediateIndexed  
    dcl_input v0.xyz  
    dcl_input v1.xy  
    dcl_output o0.xy  
    dcl_output o1.xyz  
    dcl_output_siv o2.xyzw, position  
    dcl_temps 2  
   0: mov o0.xy, v1.xyxx  
   1: mad r0.xyz, v0.xyzx, cb2[4].xyzx, cb2[5].xyzx  
   2: mov r0.w, l(1.000000)  
   3: dp4 o1.x, r0.xyzw, cb2[0].xyzw  
   4: dp4 o1.y, r0.xyzw, cb2[1].xyzw  
   5: dp4 o1.z, r0.xyzw, cb2[2].xyzw  
   6: mul r1.xyzw, cb1[0].yyyy, cb2[1].xyzw  
   7: mad r1.xyzw, cb2[0].xyzw, cb1[0].xxxx, r1.xyzw  
   8: mad r1.xyzw, cb2[2].xyzw, cb1[0].zzzz, r1.xyzw  
   9: mad r1.xyzw, cb1[0].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r1.xyzw  
  10: dp4 o2.x, r0.xyzw, r1.xyzw  
  11: mul r1.xyzw, cb1[1].yyyy, cb2[1].xyzw  
  12: mad r1.xyzw, cb2[0].xyzw, cb1[1].xxxx, r1.xyzw  
  13: mad r1.xyzw, cb2[2].xyzw, cb1[1].zzzz, r1.xyzw  
  14: mad r1.xyzw, cb1[1].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r1.xyzw  
  15: dp4 o2.y, r0.xyzw, r1.xyzw  
  16: mul r1.xyzw, cb1[2].yyyy, cb2[1].xyzw  
  17: mad r1.xyzw, cb2[0].xyzw, cb1[2].xxxx, r1.xyzw  
  18: mad r1.xyzw, cb2[2].xyzw, cb1[2].zzzz, r1.xyzw  
  19: mad r1.xyzw, cb1[2].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r1.xyzw  
  20: dp4 o2.z, r0.xyzw, r1.xyzw  
  21: mul r1.xyzw, cb1[3].yyyy, cb2[1].xyzw  
  22: mad r1.xyzw, cb2[0].xyzw, cb1[3].xxxx, r1.xyzw  
  23: mad r1.xyzw, cb2[2].xyzw, cb1[3].zzzz, r1.xyzw  
  24: mad r1.xyzw, cb1[3].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r1.xyzw  
  25: dp4 o2.w, r0.xyzw, r1.xyzw  
  26: ret  

In this scenario VS outputs only texcoords and world-space position. In Blood & Wine it also outputs normalized normal vector. I'll stay with the 2015 version as it's simpler.

Take a look at constant buffer marked as cb2:

Here we have world matrix (uniform scaling by 100 and translation by camera position). Nothing fancy. cb2_v4 and cb2_v5 are scale/bias factors which serve to transform positions of vertices from [0-1] range to [-1;1] one. But here, in terms of Z-axis (up) these coefficents will 'squeeze' it.

We have already seen similar VS in previous parts of the series. The general algorithm is to pass texcoords further, then calculate Position with scale/bias factors, then calculate PositionW in world space, then calculate final clip space position by multiplying matWorld and matViewProj matrices together -> use their product to multiply with Position to get final SV_Position.

So, the HLSL for this vertex shader would be something like this:

 struct InputStruct {  
      float3 param0 : POSITION;  
      float2 param1 : TEXCOORD;  
      float3 param2 : NORMAL;  
      float4 param3 : TANGENT;  
 };  
   
 struct OutputStruct {  
      float2 param0 : TEXCOORD0;  
      float3 param1 : TEXCOORD1;  
      float4 param2 : SV_Position;  
 };  
   
 OutputStruct EditedShaderVS(in InputStruct IN)  
 {  
      OutputStruct OUT = (OutputStruct)0;  
        
      // Simple texcoords passing  
      OUT.param0 = IN.param1;  
        
        
      // * Manually construct world and viewProj martices from float4s:  
      row_major matrix matWorld = matrix(cb2_v0, cb2_v1, cb2_v2, float4(0,0,0,1) );  
      matrix matViewProj = matrix(cb1_v0, cb1_v1, cb1_v2, cb1_v3);  
   
      // * Some optional fun with worldMatrix  
      // a) Scale  
      //matWorld._11 = matWorld._22 = matWorld._33 = 0.225f;  
   
      // b) Translate  
      // X Y Z  
      //matWorld._14 = 520.0997;  
      //matWorld._24 = 74.4226;  
      //matWorld._34 = 113.9;  
   
      // Local space - note the scale+bias here!  
      //float3 meshScale = float3(2.0, 2.0, 2.0);  
      //float3 meshBias = float3(-1.0, -1.0, -0.4);  
      float3 meshScale = cb2_v4.xyz;  
      float3 meshBias = cb2_v5.xyz;  
   
      float3 Position = IN.param0 * meshScale + meshBias;  
        
      // World space  
      float4 PositionW = mul(float4(Position, 1.0), transpose(matWorld) );  
      OUT.param1 = PositionW.xyz;  
   
      // Clip space - original approach from The Witcher 3  
      matrix matWorldViewProj = mul(matViewProj, matWorld);  
      OUT.param2 = mul( float4(Position, 1.0), transpose(matWorldViewProj) );  
        
      return OUT;  
 }  

Comparison of the my shader (left) and the original one (right):

The great thing about RenderDoc is that it allows to inject your own shader instead of original one and your changes do affect the pipeline until the very end of a frame. As you can see in HLSL code, I gave you some options to change scaling and translation of the final geometry. You can play with it and achieve some funny results:

Hail to the skydome!

Optimizing the vertex shader

Do you see a problem with the original vertex shader? Per-vertex matrix-matrix multiplication is completely redundant! I found it in at least few vertex shaders (for instance, in distant rain shafts).  We could optimize it by multiplying PositionW with matViewProj immediately!

So, we can replace HLSL code:

      // Clip space - original approach from The Witcher 3  
      matrix matWorldViewProj = mul(matViewProj, matWorld);  
      OUT.param2 = mul( float4(Position, 1.0), transpose(matWorldViewProj) );  

with this one:

      // Clip space - optimized version  
      OUT.param2 = mul( matViewProj, PositionW );  

An optimized version produces the following assembly:

    vs_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer CB1[4], immediateIndexed  
    dcl_constantbuffer CB2[6], immediateIndexed  
    dcl_input v0.xyz  
    dcl_input v1.xy  
    dcl_output o0.xy  
    dcl_output o1.xyz  
    dcl_output_siv o2.xyzw, position  
    dcl_temps 2  
   0: mov o0.xy, v1.xyxx  
   1: mad r0.xyz, v0.xyzx, cb2[4].xyzx, cb2[5].xyzx  
   2: mov r0.w, l(1.000000)  
   3: dp4 r1.x, r0.xyzw, cb2[0].xyzw  
   4: dp4 r1.y, r0.xyzw, cb2[1].xyzw  
   5: dp4 r1.z, r0.xyzw, cb2[2].xyzw  
   6: mov o1.xyz, r1.xyzx  
   7: mov r1.w, l(1.000000)  
   8: dp4 o2.x, cb1[0].xyzw, r1.xyzw  
   9: dp4 o2.y, cb1[1].xyzw, r1.xyzw  
  10: dp4 o2.z, cb1[2].xyzw, r1.xyzw  
  11: dp4 o2.w, cb1[3].xyzw, r1.xyzw  
  12: ret

As you can see, we reduced number of instructions from 26 to 12 - that's quite a change. I don't know how widespread this problem is in the game but c'mon CD Projekt Red, maybe a patch or something? :)

I'm not kidding here. You can inject my optimized shader instead of original one in RenderDoc and see for yourself that this optimization changes nothing in terms of visuals. Honestly, I don't know why CD Projekt Red decided to do per-vertex matrix-matrix multiplication...

The Sun

In The Witcher 3 (2015) calculating of atmospheric scattering and the Sun consists of two separate draw calls:

The Witcher 3 (2015) - before

The Witcher 3 (2015) - with sky

The Witcher 3 (2015) - with sky + the Sun

Rendering the Sun in 2015 version is pretty similar to the Moon in terms of geometry and blend/depth states.

On the other hand, in Blood & Wine sky with the Sun is rendered in one pass:

The Witcher 3: Blood & Wine (2016) - before sky

The Witcher 3: Blood & Wine (2016) - with sky and the Sun

No matter how you want to render the Sun at some point you will need (normalized) direction of sunlight. The most intuitive way to obtain this vector is to use spherical coordinates. Basically you need only two values representing two angles (in radians!): phi and theta. Once you have them you can assume r = 1, so it cancels, so for y-up Cartesian coordinate system we can write HLSL code like this:

 float3 vSunDir;  
 vSunDir.x = sin(fTheta)*cos(fPhi);  
 vSunDir.y = sin(fTheta)*sin(fPhi);  
 vSunDir.z = cos(fTheta);  
 vSunDir = normalize(vSunDir);  

Normally you calculate sunlight direction in your application, then pass it to constant buffer for further use.

Once we have sunlight direction we can dive into assembly of pixel shader from Blood & Wine....

  ...   
  100: add r1.xyw, -r0.xyxz, cb12[0].xyxz  
  101: dp3 r2.x, r1.xywx, r1.xywx  
  102: rsq r2.x, r2.x  
  103: mul r1.xyw, r1.xyxw, r2.xxxx  
  104: mov_sat r2.xy, cb12[205].yxyy  
  105: dp3 r2.z, -r1.xywx, -r1.xywx  
  106: rsq r2.z, r2.z  
  107: mul r1.xyw, -r1.xyxw, r2.zzzz  
  ...  

Okay. To start, cb12[0].xyz is a position of camera, while in r0.xyz we store vertex position (it's an output from vertex shader). Therefore, line 100 calculates worldToCamera vector. But take a look at lines 105-107. We could write it as normalize( -worldToCamera), which means we calculate normalized cameraToWorld vector.

  120: dp3_sat r1.x, cb12[203].yzwy, r1.xywx  

Then we calculate dot product between cameraToWorld and sunDirection vectors! Remember they have to be normalized. Also we saturate whole expression to clamp it within [0-1] range.

Cool! We have this dot product in r1.x. Let's find the next use of it...

  152: log r1.x, r1.x  
  153: mul r1.x, r1.x, cb12[203].x  
  154: exp r1.x, r1.x  
  155: mul r1.x, r2.y, r1.x  

The "log, mul, exp" triple is, simply speaking, exponentation. As you can see, we raise our cosine (dot product of normalized vectors) to some power. You may ask, why? This way we can produce gradient which will mimic our Sun. (And line 155 affects opacity of this gradient, so you can for instance set this to zero to completely hide the Sun). See some examples:

exponent = 54

exponent = 2400

Having this gradient, we use it to interpolate between skyColor and sunColor! To make sure there will be no artifacts we had to saturate in line 120.

Please take a note that this trick can be used to mimic corona phenomenon for the Moon (with lower values of the exponent). For this you will need moonDirection vector - which can be easily calculated with spherical coordinates.

Final HLSL can look similar to the following snippet:

 float3 vCamToWorld = normalize( PosW – CameraPos );  
   
 float cosTheta = saturate( dot(vSunDir, vCamToWorld) );  
 float sunGradient = pow( cosTheta, sunExponent );  
   
 float3 color = lerp( skyColor, sunColor, sunGradient );  

Moving stars

If you would make a timelapse during the night on a clear sky in The Witcher 3 you would notice that stars are not static - they slightly move across the sky with time! I noticed this quite accidentally and wanted to see how this was done.

Let's start with fact that stars in The Witcher 3 are represented with 1024x1024x6 cubemap. If you think about it, it's very handy solution as it easily allows to map directions to sample the cubemap.

Consider the following piece of assembly:

  159: add r1.xyz, -v1.xyzx, cb1[8].xyzx  
  160: dp3 r0.w, r1.xyzx, r1.xyzx  
  161: rsq r0.w, r0.w  
  162: mul r1.xyz, r0.wwww, r1.xyzx  
  163: mul r2.xyz, cb12[204].zwyz, l(0.000000, 0.000000, 1.000000, 0.000000)  
  164: mad r2.xyz, cb12[204].yzwy, l(0.000000, 1.000000, 0.000000, 0.000000), -r2.xyzx  
  165: mul r4.xyz, r2.xyzx, cb12[204].zwyz  
  166: mad r4.xyz, r2.zxyz, cb12[204].wyzw, -r4.xyzx  
  167: dp3 r4.x, r1.xyzx, r4.xyzx  
  168: dp2 r4.y, r1.xyxx, r2.yzyy  
  169: dp3 r4.z, r1.xyzx, cb12[204].yzwy  
  170: dp3 r0.w, r4.xyzx, r4.xyzx  
  171: rsq r0.w, r0.w  
  172: mul r2.xyz, r0.wwww, r4.xyzx  
  173: sample_indexable(texturecube)(float,float,float,float) r4.xyz, r2.xyzx, t0.xyzw, s0  

To calculate final sampling vector (line 173), we start by calculating normalized worldToCamera vector (lines 159-162).

Then we calculate 2 cross products (163-164, 165-166) with moonDirection and later perform 3 dot products to get final sampling vector. HLSL:

 float3 vWorldToCamera = normalize( g_CameraPos.xyz - Input.PositionW.xyz );  
 float3 vMoonDirection = cb12_v204.yzw;  
   
 float3 vStarsSamplingDir = cross( vMoonDirection, float3(0, 0, 1) );  
 float3 vStarsSamplingDir2 = cross( vStarsSamplingDir, vMoonDirection );  
   
 float dirX = dot( vWorldToCamera, vStarsSamplingDir2 );  
 float dirY = dot( vWorldToCamera, vStarsSamplingDir );  
 float dirZ = dot( vWorldToCamera, vMoonDirection);  
 float3 dirXYZ = normalize( float3(dirX, dirY, dirZ) );  
   
 float3 starsColor = texNightStars.Sample( samplerAnisoWrap, dirXYZ ).rgb;  

Note to self: This is really well-thought and I definitely have to investigate it in more details.
Note to readers: If you know more about this operation, let me know!

Blinking stars

Another nice trick I wanted to investigate in more details is blinking of stars. If you walk around, let's say, outskirts of Novigrad City and sky is clear you can notice that stars are blinking.

I was curious how this was implemented. So the difference is quite big between the 2015 version and Blood & Wine. For simplicity I'll stay with 2015 version.

So we start just after sampling starsColor from the previous section:

  174: mul r0.w, v0.x, l(100.000000)  
  175: round_ni r1.w, r0.w  
  176: mad r2.w, v0.y, l(50.000000), cb0[0].x  
  177: round_ni r4.w, r2.w  
  178: bfrev r4.w, r4.w  
  179: iadd r5.x, r1.w, r4.w  
  180: ishr r5.y, r5.x, l(13)  
  181: xor r5.x, r5.x, r5.y  
  182: imul null, r5.y, r5.x, r5.x  
  183: imad r5.y, r5.y, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  184: imad r5.x, r5.x, r5.y, l(146956042240.000000)  
  185: and r5.x, r5.x, l(0x7fffffff)  
  186: itof r5.x, r5.x  
  187: mad r5.y, v0.x, l(100.000000), l(-1.000000)  
  188: round_ni r5.y, r5.y  
  189: iadd r4.w, r4.w, r5.y  
  190: ishr r5.z, r4.w, l(13)  
  191: xor r4.w, r4.w, r5.z  
  192: imul null, r5.z, r4.w, r4.w  
  193: imad r5.z, r5.z, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  194: imad r4.w, r4.w, r5.z, l(146956042240.000000)  
  195: and r4.w, r4.w, l(0x7fffffff)  
  196: itof r4.w, r4.w  
  197: add r5.z, r2.w, l(-1.000000)  
  198: round_ni r5.z, r5.z  
  199: bfrev r5.z, r5.z  
  200: iadd r1.w, r1.w, r5.z  
  201: ishr r5.w, r1.w, l(13)  
  202: xor r1.w, r1.w, r5.w  
  203: imul null, r5.w, r1.w, r1.w  
  204: imad r5.w, r5.w, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  205: imad r1.w, r1.w, r5.w, l(146956042240.000000)  
  206: and r1.w, r1.w, l(0x7fffffff)  
  207: itof r1.w, r1.w  
  208: mul r1.w, r1.w, l(0.000000001)  
  209: iadd r5.y, r5.z, r5.y  
  210: ishr r5.z, r5.y, l(13)  
  211: xor r5.y, r5.y, r5.z  
  212: imul null, r5.z, r5.y, r5.y  
  213: imad r5.z, r5.z, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  214: imad r5.y, r5.y, r5.z, l(146956042240.000000)  
  215: and r5.y, r5.y, l(0x7fffffff)  
  216: itof r5.y, r5.y  
  217: frc r0.w, r0.w  
  218: add r0.w, -r0.w, l(1.000000)  
  219: mul r5.z, r0.w, r0.w  
  220: mul r0.w, r0.w, r5.z  
  221: mul r5.xz, r5.xxzx, l(0.000000001, 0.000000, 3.000000, 0.000000)  
  222: mad r0.w, r0.w, l(-2.000000), r5.z  
  223: frc r2.w, r2.w  
  224: add r2.w, -r2.w, l(1.000000)  
  225: mul r5.z, r2.w, r2.w  
  226: mul r2.w, r2.w, r5.z  
  227: mul r5.z, r5.z, l(3.000000)  
  228: mad r2.w, r2.w, l(-2.000000), r5.z  
  229: mad r4.w, r4.w, l(0.000000001), -r5.x  
  230: mad r4.w, r0.w, r4.w, r5.x  
  231: mad r5.x, r5.y, l(0.000000001), -r1.w  
  232: mad r0.w, r0.w, r5.x, r1.w  
  233: add r0.w, -r4.w, r0.w  
  234: mad r0.w, r2.w, r0.w, r4.w  
  235: mad r2.xyz, r0.wwww, l(0.000500, 0.000500, 0.000500, 0.000000), r2.xyzx  
  236: sample_indexable(texturecube)(float,float,float,float) r2.xyz, r2.xyzx, t0.xyzw, s0  
  237: log r4.xyz, r4.xyzx  
  238: mul r4.xyz, r4.xyzx, l(2.200000, 2.200000, 2.200000, 0.000000)  
  239: exp r4.xyz, r4.xyzx  
  240: log r2.xyz, r2.xyzx  
  241: mul r2.xyz, r2.xyzx, l(2.200000, 2.200000, 2.200000, 0.000000)  
  242: exp r2.xyz, r2.xyzx  
  243: mul r2.xyz, r2.xyzx, r4.xyzx  

Huh. Let's take a look at the very end of this quite big piece of assembly.

Once we sampled starsColor in line 173 we calculate some offset value. This offset is used to perturb first sampling direction (r2.xyz, line 235), then we sample stars cubemap again, perform gamma correction on these two values (237-242) and multiply them together (243).

Simple, isn't it? Well, not really. Think about this offset for a while. It must be different across whole skydome - stars blinking the same way would look very unrealistic.

To make sure that offset will be as diverse as possible we will take advantage of UVs wrapped across skydome (v0.xy) and elapsed time from constant buffer (cb[0].x).

If you are unfamiliar with this intimidating ishr/xor/and thing, take a look at lightnings effect to learn more about integer noise.

So as you can see, integer noise is called here four times, but it's different now comparing to lightnings. To make results even more random the input integer for noise is a sum (iadd) and reversing bits is performed (reversebits instrinsic; bfrev instruction).

Okay, easy now. Let's start from start.
We have 4 "iterations" of integer noise. I analyzed the assembly and calculation of all 4 iterations looks like this:

 int getInt( float x )
 {
     return asint( floor(x) );
 }
 
 int getReverseInt( float x )
 {
     return reversebits( getInt(x) );
 }

 // * Inputs - UV and elapsed time in seconds  
 float2 starsUV;  
 starsUV.x = 100.0 * Input.TextureUV.x;       
 starsUV.y = 50.0  * Input.TextureUV.y + g_fTime;  
             
 // * Iteration 1  
 int iStars1_A = getReverseInt( starsUV.y );
 int iStars1_B = getInt( starsUV.x );            
   
 float fStarsNoise1 = integerNoise( iStars1_A + iStars1_B );  
             
   
 // * Iteration 2  
 int iStars2_A = getReverseInt( starsUV.y );  
 int iStars2_B = getInt( starsUV.x - 1.0 ); 
   
 float fStarsNoise2 = integerNoise( iStars2_A + iStars2_B );  
        
   
 // * Iteration 3  
 int iStars3_A = getReverseInt( starsUV.y - 1.0 );
 int iStars3_B = getInt( starsUV.x );
   
 float fStarsNoise3 = integerNoise( iStars3_A + iStars3_B );  
             
   
 // * Iteration 4  
 int iStars4_A = getReverseInt( starsUV.y - 1.0 ); 
 int iStars4_B = getInt( starsUV.x - 1.0 ); 
   
 float fStarsNoise4 = integerNoise( iStars4_A + iStars4_B );  

The final outputs of all these 4 iterations are (follow itof instructions to find them):

Iteration 1 - r5.x,
Iteration 2 - r4.w,
Iteration 3 - r1.w,
Iteration 4 - r5.y

After the last itof (line 216) we have:

  217: frc r0.w, r0.w   
  218: add r0.w, -r0.w, l(1.000000)   
  219: mul r5.z, r0.w, r0.w   
  220: mul r0.w, r0.w, r5.z   
  221: mul r5.xz, r5.xxzx, l(0.000000001, 0.000000, 3.000000, 0.000000)   
  222: mad r0.w, r0.w, l(-2.000000), r5.z   
  223: frc r2.w, r2.w   
  224: add r2.w, -r2.w, l(1.000000)   
  225: mul r5.z, r2.w, r2.w   
  226: mul r2.w, r2.w, r5.z   
  227: mul r5.z, r5.z, l(3.000000)   
  228: mad r2.w, r2.w, l(-2.000000), r5.z   

These lines calculate values for S-curve for weights based on fractional part of UVs, just like in case of lightnings. So:

  float s_curve( float x )   
  {   
    float x2 = x * x;   
    float x3 = x2 * x;   
      
    // -2x^3 + 3x^2   
    return -2.0*x3 + 3.0*x2;   
  }  
   
 ...  
 
 // lines 217-222
 float weightX = 1.0 - frac( starsUV.x );  
 weightX = s_curve( weightX );  
   
 // lines 223-228
 float weightY = 1.0 - frac( starsUV.y );  
 weightY = s_curve( weightY );  

As you can expect, these factors serve to interpolate noise smoothly and generate final offset for sampling coordinates:

  229: mad r4.w, r4.w, l(0.000000001), -r5.x   
  230: mad r4.w, r0.w, r4.w, r5.x   
  float noise0 = lerp( fStarsNoise1, fStarsNoise2, weightX );  
   
  231: mad r5.x, r5.y, l(0.000000001), -r1.w   
  232: mad r0.w, r0.w, r5.x, r1.w   
  float noise1 = lerp( fStarsNoise3, fStarsNoise4, weightX );  
   
  233: add r0.w, -r4.w, r0.w   
  234: mad r0.w, r2.w, r0.w, r4.w   
  float offset = lerp( noise0, noise1, weightY );            
   
  235: mad r2.xyz, r0.wwww, l(0.000500, 0.000500, 0.000500, 0.000000), r2.xyzx   
  236: sample_indexable(texturecube)(float,float,float,float) r2.xyz, r2.xyzx, t0.xyzw, s0   
  float3 starsPerturbedDir = dirXYZ + offset * 0.0005;  
    
  float3 starsColorDisturbed = texNightStars.Sample( samplerAnisoWrap, starsPerturbedDir ).rgb;

Here is small visualization of calculated offset:

Once we have starsColorDisturbed, the hardest part is over. Phew!

The next step is to perform gamma correction on both starsColor and starsColorDisturbed and multiply them:

  starsColor = pow( starsColor, 2.2 );  
  starsColorDisturbed = pow( starsColorDisturbed, 2.2 );  
   
  float3 starsFinal = starsColor * starsColorDisturbed;  

Stars - the final touches

We have starsFinal in r1.xyz. What's happening at the end of processing stars is this:

  256: log r1.xyz, r1.xyzx  
  257: mul r1.xyz, r1.xyzx, l(2.500000, 2.500000, 2.500000, 0.000000)  
  258: exp r1.xyz, r1.xyzx  
  259: min r1.xyz, r1.xyzx, l(1.000000, 1.000000, 1.000000, 0.000000)  
  260: add r0.w, -cb0[9].w, l(1.000000)  
  261: mul r1.xyz, r0.wwww, r1.xyzx  
  262: mul r1.xyz, r1.xyzx, l(10.000000, 10.000000, 10.000000, 0.000000)  

This is much, much easier comparing to blinking and moving stars.
So we start with raising starsFinal to power of 2.5 - this allows to control density of stars. Pretty clever. Then, we make sure the maximum color of stars is float3(1, 1, 1).

cb0[9].w is used to control general visibility of stars. So in daytime expect this to be set to 1.0 (which yields in multiplying by zero) and 0.0 during nights.

At the end we boost visibility of stars by 10. And this is over! :)

Summary

In this post I presented some cool tricks I found while investigating sky shader from The Witcher 3. I hope you enjoyed it. Thanks for reading!

Take care,
M.

Reverse engineering the rendering of The Witcher 3, part 11 - lightnings

This post is a part of the series "Reverse engineering the rendering of The Witcher 3".


Welcome back!

In the 11th part of the series we will take a look how lightnings are rendered in The Witcher 3: Wild Hunt.

Following the distant rain shafts effect, lightnings are rendered slightly after, but still in forward pass. You can see them in action in the following video:

They last for a very short time; therefore, it will be the best to play this video with 0.25 speed.
You can see that they are not static images; their intensity slightly changes with time.

There are many similarities here with distant rain shafts in terms of rendernig nuances, like the same blending (additive blending) and depth (test enabled, no depth write) states.

Scene without lightning

Scene with lightning

In terms of geometry, lightnings in The Witcher 3 are tree-like meshes, this particular lightning is represented by the following one:

It's provided with UV coordinates and normal vectors. They will be useful in the vertex shader stage.

Vertex Shader

Let's take a look at the assembly of the vertex shader:

 vs_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer cb1[9], immediateIndexed  
    dcl_constantbuffer cb2[6], immediateIndexed  
    dcl_input v0.xyz  
    dcl_input v1.xy  
    dcl_input v2.xyz  
    dcl_input v4.xyzw  
    dcl_input v5.xyzw  
    dcl_input v6.xyzw  
    dcl_input v7.xyzw  
    dcl_output o0.xy  
    dcl_output o1.xyzw  
    dcl_output_siv o2.xyzw, position  
    dcl_temps 3  
   0: mov o0.xy, v1.xyxx  
   1: mov o1.xyzw, v7.xyzw  
   2: mul r0.xyzw, v5.xyzw, cb1[0].yyyy  
   3: mad r0.xyzw, v4.xyzw, cb1[0].xxxx, r0.xyzw  
   4: mad r0.xyzw, v6.xyzw, cb1[0].zzzz, r0.xyzw  
   5: mad r0.xyzw, cb1[0].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r0.xyzw  
   6: mov r1.w, l(1.000000)  
   7: mad r1.xyz, v0.xyzx, cb2[4].xyzx, cb2[5].xyzx  
   8: dp4 r2.x, r1.xyzw, v4.xyzw  
   9: dp4 r2.y, r1.xyzw, v5.xyzw  
  10: dp4 r2.z, r1.xyzw, v6.xyzw  
  11: add r2.xyz, r2.xyzx, -cb1[8].xyzx  
  12: dp3 r1.w, r2.xyzx, r2.xyzx  
  13: rsq r1.w, r1.w  
  14: div r1.w, l(1.000000, 1.000000, 1.000000, 1.000000), r1.w  
  15: mul r1.w, r1.w, l(0.000001)  
  16: mad r2.xyz, v2.xyzx, l(2.000000, 2.000000, 2.000000, 0.000000), l(-1.000000, -1.000000, -1.000000, 0.000000)  
  17: mad r1.xyz, r2.xyzx, r1.wwww, r1.xyzx  
  18: mov r1.w, l(1.000000)  
  19: dp4 o2.x, r1.xyzw, r0.xyzw  
  20: mul r0.xyzw, v5.xyzw, cb1[1].yyyy  
  21: mad r0.xyzw, v4.xyzw, cb1[1].xxxx, r0.xyzw  
  22: mad r0.xyzw, v6.xyzw, cb1[1].zzzz, r0.xyzw  
  23: mad r0.xyzw, cb1[1].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r0.xyzw  
  24: dp4 o2.y, r1.xyzw, r0.xyzw  
  25: mul r0.xyzw, v5.xyzw, cb1[2].yyyy  
  26: mad r0.xyzw, v4.xyzw, cb1[2].xxxx, r0.xyzw  
  27: mad r0.xyzw, v6.xyzw, cb1[2].zzzz, r0.xyzw  
  28: mad r0.xyzw, cb1[2].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r0.xyzw  
  29: dp4 o2.z, r1.xyzw, r0.xyzw  
  30: mul r0.xyzw, v5.xyzw, cb1[3].yyyy  
  31: mad r0.xyzw, v4.xyzw, cb1[3].xxxx, r0.xyzw  
  32: mad r0.xyzw, v6.xyzw, cb1[3].zzzz, r0.xyzw  
  33: mad r0.xyzw, cb1[3].wwww, l(0.000000, 0.000000, 0.000000, 1.000000), r0.xyzw  
  34: dp4 o2.w, r1.xyzw, r0.xyzw  
  35: ret  

There are many similarities here compared to the vertex shader from the one from distant rain shafts so I won't repeat myself. The one major difference I want to show are lines 11-18:

  11: add r2.xyz, r2.xyzx, -cb1[8].xyzx  
  12: dp3 r1.w, r2.xyzx, r2.xyzx  
  13: rsq r1.w, r1.w  
  14: div r1.w, l(1.000000, 1.000000, 1.000000, 1.000000), r1.w  
  15: mul r1.w, r1.w, l(0.000001)  
  16: mad r2.xyz, v2.xyzx, l(2.000000, 2.000000, 2.000000, 0.000000), l(-1.000000, -1.000000, -1.000000, 0.000000)  
  17: mad r1.xyz, r2.xyzx, r1.wwww, r1.xyzx  
  18: mov r1.w, l(1.000000)  
  19: dp4 o2.x, r1.xyzw, r0.xyzw  

To start, cb1[8].xyz is camera position while r2.xyz is world-space position, so line 11 calculates vector from camera to world position. Then, lines 12-15 compute length( worldPos - cameraPos) * 0.000001.

v2.xyz is normal vector from input geometry. Line 16 unpacks it from [0-1] range to [-1;1] one.
Then, the final world-space position is calculated:

finalWorldPos = worldPos + length( worldPos - cameraPos) * 0.000001 * normalVector


HLSL snippet for this operation would be something like this:

      ...  
      // final world-space position  
      float3 vNormal = Input.NormalW * 2.0 - 1.0;  
      float lencameratoworld = length( PositionL - g_cameraPos.xyz) * 0.000001;  
   
      PositionL += vNormal*lencameratoworld;  
   
      // SV_Posiiton   
      float4x4 matModelViewProjection = mul(g_viewProjMatrix, matInstanceWorld );   
      Output.PositionH = mul( float4(PositionL, 1.0), transpose(matModelViewProjection) );      
   
      return Output;   

Such operation causes slight "explosion" of the mesh (in the direction of normal vector). I did some simple experiments and replaced 0.000001 with a few different values, see the results:

0.000002

0.000005

0.00001

0.000025

Pixel Shader

Ok, we're done with the vertex shader, time to see the assembly of pixel shader!

 ps_5_0  
    dcl_globalFlags refactoringAllowed  
    dcl_constantbuffer cb0[1], immediateIndexed  
    dcl_constantbuffer cb2[3], immediateIndexed  
    dcl_constantbuffer cb4[5], immediateIndexed  
    dcl_input_ps linear v0.x  
    dcl_input_ps linear v1.w  
    dcl_output o0.xyzw  
    dcl_temps 1  
   0: mad r0.x, cb0[0].x, cb4[4].x, v0.x  
   1: add r0.y, r0.x, l(-1.000000)  
   2: round_ni r0.y, r0.y  
   3: ishr r0.z, r0.y, l(13)  
   4: xor r0.y, r0.y, r0.z  
   5: imul null, r0.z, r0.y, r0.y  
   6: imad r0.z, r0.z, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
   7: imad r0.y, r0.y, r0.z, l(146956042240.000000)  
   8: and r0.y, r0.y, l(0x7fffffff)  
   9: round_ni r0.z, r0.x  
  10: frc r0.x, r0.x  
  11: add r0.x, -r0.x, l(1.000000)  
  12: ishr r0.w, r0.z, l(13)  
  13: xor r0.z, r0.z, r0.w  
  14: imul null, r0.w, r0.z, r0.z  
  15: imad r0.w, r0.w, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
  16: imad r0.z, r0.z, r0.w, l(146956042240.000000)  
  17: and r0.z, r0.z, l(0x7fffffff)  
  18: itof r0.yz, r0.yyzy  
  19: mul r0.z, r0.z, l(0.000000001)  
  20: mad r0.y, r0.y, l(0.000000001), -r0.z  
  21: mul r0.w, r0.x, r0.x  
  22: mul r0.x, r0.x, r0.w  
  23: mul r0.w, r0.w, l(3.000000)  
  24: mad r0.x, r0.x, l(-2.000000), r0.w  
  25: mad r0.x, r0.x, r0.y, r0.z  
  26: add r0.y, -cb4[2].x, cb4[3].x  
  27: mad_sat r0.x, r0.x, r0.y, cb4[2].x  
  28: mul r0.x, r0.x, v1.w  
  29: mul r0.yzw, cb4[0].xxxx, cb4[1].xxyz  
  30: mul r0.xyzw, r0.xyzw, cb2[2].wxyz  
  31: mul o0.xyz, r0.xxxx, r0.yzwy  
  32: mov o0.w, r0.x  
  33: ret  

The good thing: this code is not that long.
The bad thing:

   3: ishr r0.z, r0.y, l(13)  
   4: xor r0.y, r0.y, r0.z  
   5: imul null, r0.z, r0.y, r0.y  
   6: imad r0.z, r0.z, l(0x0000ec4d), l(0.0000000000000000000000000000000000001)  
   7: imad r0.y, r0.y, r0.z, l(146956042240.000000)  
   8: and r0.y, r0.y, l(0x7fffffff)  

...what the heck is this???

To be honest, it's not the first time when I see that piece of cra... assembly in shaders of The Witcher 3. But when I found it for the first time, I was like: "oh crap, wtf is this?".

Indeed, you can find something like this in a few shaders of TW3. I won't go now through many adventures I've had with this one, but let me just say the answer is integer noise:

 // For more details see: http://libnoise.sourceforge.net/noisegen/  
 float integerNoise( int n )  
 {  
      n = (n >> 13) ^ n;  
      int nn = (n * (n * n * 60493 + 19990303) + 1376312589) & 0x7fffffff;  
      return ((float)nn / 1073741824.0);  
 }  

Phew. As you can see, it's invoked 2 times in the pixel shader. Following the guidance from its website gives us some useful tips how to implement smooth noise properly. I'll back to this in a minute.

Take a look at line 0, we perform animation here based on the following formula:

animation = elapsedTime * animationSpeed + TextureUV.x

These values, after being floored (round_ni instruction) are subsequent entry points to integer noise. Generally we calculate value of the noise for two integers and then calculate final, interpolated value between them (see libnoise's website for details).

Okay, this is integer noise while all previously mentioned values (also floored values) are floats!
Please notice that there are no ftoi instructions here. My guess is that programmers from CD Projekt Red used here asint HLSL intrinsic function which performs "reinterpret_cast" of floating-point values and treat it like integer pattern.

The interpolation weight for two values is calculated in lines 10-11

interpolationWeight = 1.0 - frac( animation );

Such approach allows to interpolate between values with time.
To make a smooth noise, this interpolant is passed to SCurve function:

 float s_curve( float x )  
 {  
   float x2 = x * x;  
   float x3 = x2 * x;  
     
   // -2x^3 + 3x^2  
   return -2.0*x3 + 3.0*x2;  
 }

Smoothstep function [libnoise.sourceforge.net]

This function is known as "smoothstep". But as you can see from the assembly, this is not  smoothstep instrinsic from HLSL. The intrinsic performs some clamps to make sure the values will be correct. Since we know that our interpolationWeight will always be in [0-1] range we can safely omit these checks.

Calculating the final value includes a few multiplications. Please take a look that the final output alpha can change, depending on noise value. This is handy, because it will affect opacity of the rendered lightning - just like in real life.

The final pixel shader:

 cbuffer cbPerFrame : register (b0)  
 {  
   float4 cb0_v0;  
   float4 cb0_v1;  
   float4 cb0_v2;  
   float4 cb0_v3;  
 }  
   
 cbuffer cbPerFrame : register (b2)  
 {  
   float4 cb2_v0;  
   float4 cb2_v1;  
   float4 cb2_v2;  
   float4 cb2_v3;  
 }  
   
 cbuffer cbPerFrame : register (b4)  
 {  
   float4 cb4_v0;  
   float4 cb4_v1;  
   float4 cb4_v2;  
   float4 cb4_v3;  
   float4 cb4_v4;  
 }  
   
 struct VS_OUTPUT  
 {  
   float2 Texcoords : Texcoord0;  
   float4 InstanceLODParams : INSTANCE_LOD_PARAMS;  
   float4 PositionH : SV_Position;  
 };  
   
 // Shaders in TW3 use integer noise.  
 // For more details see: http://libnoise.sourceforge.net/noisegen/  
 float integerNoise( int n )  
 {  
   n = (n >> 13) ^ n;  
   int nn = (n * (n * n * 60493 + 19990303) + 1376312589) & 0x7fffffff;  
   return ((float)nn / 1073741824.0);  
 }  
   
 float s_curve( float x )  
 {  
   float x2 = x * x;  
   float x3 = x2 * x;  
   
   // -2x^3 + 3x^2  
   return -2.0*x3 + 3.0*x2;  
 }  
   
 float4 Lightning_TW3_PS( in VS_OUTPUT Input ) : SV_Target
 {  
   // * Inputs  
   float elapsedTime = cb0_v0.x;  
   float animationSpeed = cb4_v4.x;  
   
   float minAmount = cb4_v2.x;  
   float maxAmount = cb4_v3.x;  
   
   float colorMultiplier = cb4_v0.x;  
   float3 colorFilter = cb4_v1.xyz;  
   float3 lightningColorRGB = cb2_v2.rgb;  
   
   
   // Animation using time and X texcoord  
   float animation = elapsedTime * animationSpeed + Input.Texcoords.x;  
   
   // Input parameters for Integer Noise.  
   // They are floored and please note there are using asint.  
   // That might be an optimization to avoid "ftoi" instructions.  
   int intX0 = asint( floor(animation) );  
   int intX1 = asint( floor(animation-1.0) );  
   
   float n0 = integerNoise( intX0 );  
   float n1 = integerNoise( intX1 );    
   
   // We interpolate "backwards" here.  
   float weight = 1.0 - frac(animation);  
   
   // Following the instructions from libnoise, we perform  
   // smooth interpolation here with cubic s-curve function.  
   float noise = lerp( n0, n1, s_curve(weight) );  
   
   // Make sure we are in [0.0 - 1.0] range.  
   float lightningAmount = saturate( lerp(minAmount, maxAmount, noise) );  
   lightningAmount *= Input.InstanceLODParams.w;    // 1.0  
   lightningAmount *= cb2_v2.w;             // 1.0  
   
   // Calculate final lightning color   
   float3 lightningColor = colorMultiplier * colorFilter;  
   lightningColor *= lighntingColorRGB;  
   
   float3 finalLightningColor = lightningColor * lightningAmount;  
   return float4( finalLightningColor, lightningAmount );  
 }  

Summary

In this post I described how lightnings are rendered in The Witcher 3.
I'm more than happy that output assembly from my shader is the same as the original one!

On the left - my shader, on the right - original assembly

I hope you enjoyed it! Thanks for reading.

Feel free to comment and take care,
M.