Using the Stencil Buffer for Sprite Occlusion

Back in the post on SpriteLightKit, we talked about the stencil buffer and we left things off with a homework assignment. I also did the homework assignment and will be sharing the result that I came up with.

First, let’s have a look at what the resulting shaders look like in action. We have one shader for the trees (occluders) and one for the player (occluded).

The full shaders are available below. At first glance they might seem really daunting but in reality they are super, duper simple. For some reason, Unity makes us use shaders for accessing the stencil buffer so because of that we have a lot more code than is really required (in “real life” stencil buffer access has nothing to do with a shader). Basically, all there is in these shaders below is a near copy of the standard sprite shader once for the occluder and twice for the occluded sprites and then our tiny stencil section on each. Lets ignore the sprite shader portion (since its so similar to the Unity default sprite shader) and have a look at the actual stencil portions.

First, the occluder sprites. Below is all of the code that actually matters for the stencil buffer. All it is doing is saying for every pixel we write to the color buffer lets replace the stencil buffer value with 4. So, in essence, anywhere there is an occluder pixel it will write 4 to the stencil buffer. Important side note: in the example the tree is not a solid sprite. It has lots of alpha = 0 portions (like most non-rectangle/square sprites) so in the shader we discard any pixels that are less than alpha 0.1. If you have a solid sprite that is not necessary.

1
2
3
4
5
6

Stencil
{
  Ref 4
  Comp Always
  Pass Replace
}

Occluded sprites need two passes (once for rendering the silhouette and one for rendering the non-occluded portation) so we will have two different stencil sections for them. The first pass is going to render wherever the stencil buffer value is not equal to 4. So, anywhere that there is not an occluder the first pass will render. The second pass is exactly the opposite: wherever the stencil buffer value is equal to 4 it will render. When it renders it multiplies the output by a dark color to make a silhouette. That’s all there is to it.

1
2
3
4
5
6
7
8
9
10
11
12
13

// first pass
Stencil
{
  Ref 4
  Comp NotEqual
}

// second pass
Stencil
{
  Ref 4
  Comp Equal
}

Below is the full shader to be used for any occluder sprites.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

Shader "Sprites/Occluder"
{
  Properties
  {
     [PerRendererData] _MainTex ( "Sprite Texture", 2D ) = "white" {}
      _Color ( "Tint", Color ) = ( 1, 1, 1, 1 )
     [MaterialToggle] PixelSnap ( "Pixel snap", Float ) = 0
      _AlphaCutoff ( "Alpha Cutoff", Range( 0.01, 1.0 ) ) = 0.1
  }


  SubShader
  {
      Tags
      {
          "Queue" = "Transparent"
          "IgnoreProjector" = "True"
          "RenderType" = "TransparentCutout"
          "PreviewType" = "Plane"
          "CanUseSpriteAtlas" = "True"
      }

      Cull Off
      Lighting Off
      ZWrite Off
      Blend One OneMinusSrcAlpha

      Pass
      {
          Stencil
          {
              Ref 4
              Comp Always
              Pass Replace
          }

      CGPROGRAM
          #pragma vertex vert
          #pragma fragment frag
          #pragma multi_compile _ PIXELSNAP_ON
          #include "UnityCG.cginc"

          struct appdata_t
          {
              float4 vertex   : POSITION;
              float4 color    : COLOR;
              float2 texcoord : TEXCOORD0;
          };

          struct v2f
          {
              float4 vertex   : SV_POSITION;
              fixed4 color    : COLOR;
              half2 texcoord  : TEXCOORD0;
          };

          fixed4 _Color;
          fixed _AlphaCutoff;

          v2f vert( appdata_t IN )
          {
              v2f OUT;
              OUT.vertex = mul( UNITY_MATRIX_MVP, IN.vertex );
              OUT.texcoord = IN.texcoord;
              OUT.color = IN.color * _Color;
              #ifdef PIXELSNAP_ON
              OUT.vertex = UnityPixelSnap( OUT.vertex );
              #endif

              return OUT;
          }

          sampler2D _MainTex;
          sampler2D _AlphaTex;


          fixed4 frag( v2f IN ) : SV_Target
          {
              fixed4 c = tex2D( _MainTex, IN.texcoord ) * IN.color;
              c.rgb *= c.a;

              // here we discard pixels below our _AlphaCutoff so the stencil buffer only gets written to
              // where there are actual pixels returned. If the occluders are all tight meshes (such as solid rectangles)
              // this is not necessary and a non-transparent shader would be a better fit.
              clip( c.a - _AlphaCutoff );

              return c;
          }
      ENDCG
      }
  }
}

Below is the full shader to be used for any occluded sprites.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

Shader "Sprites/Occluded"
{
  Properties
  {
     [PerRendererData] _MainTex ( "Sprite Texture", 2D ) = "white" {}
      _Color ( "Tint", Color ) = ( 1, 1, 1, 1 )
     [MaterialToggle] PixelSnap ( "Pixel snap", Float ) = 0
      _OccludedColor ( "Occluded Tint", Color ) = ( 0, 0, 0, 0.5 )
  }


CGINCLUDE

// shared structs and vert program used in both the vert and frag programs
struct appdata_t
{
  float4 vertex   : POSITION;
  float4 color    : COLOR;
  float2 texcoord : TEXCOORD0;
};

struct v2f
{
  float4 vertex   : SV_POSITION;
  fixed4 color    : COLOR;
  half2 texcoord  : TEXCOORD0;
};


fixed4 _Color;
sampler2D _MainTex;


v2f vert( appdata_t IN )
{
  v2f OUT;
  OUT.vertex = mul( UNITY_MATRIX_MVP, IN.vertex );
  OUT.texcoord = IN.texcoord;
  OUT.color = IN.color * _Color;
  #ifdef PIXELSNAP_ON
  OUT.vertex = UnityPixelSnap( OUT.vertex );
  #endif

  return OUT;
}

ENDCG



  SubShader
  {
      Tags
      {
          "Queue" = "Transparent"
          "IgnoreProjector" = "True"
          "RenderType" = "Transparent"
          "PreviewType" = "Plane"
          "CanUseSpriteAtlas" = "True"
      }

      Cull Off
      Lighting Off
      ZWrite Off
      Blend One OneMinusSrcAlpha

      Pass
      {
          Stencil
          {
              Ref 4
              Comp NotEqual
          }


      CGPROGRAM
          #pragma vertex vert
          #pragma fragment frag
          #pragma multi_compile _ PIXELSNAP_ON
          #include "UnityCG.cginc"


          fixed4 frag( v2f IN ) : SV_Target
          {
              fixed4 c = tex2D( _MainTex, IN.texcoord ) * IN.color;
              c.rgb *= c.a;
              return c;
          }
      ENDCG
      }


      // occluded pixel pass. Anything rendered here is behind an occluder
      Pass
      {
          Stencil
          {
              Ref 4
              Comp Equal
          }

      CGPROGRAM
          #pragma vertex vert
          #pragma fragment frag
          #pragma multi_compile _ PIXELSNAP_ON
          #include "UnityCG.cginc"

          fixed4 _OccludedColor;


          fixed4 frag( v2f IN ) : SV_Target
          {
              fixed4 c = tex2D( _MainTex, IN.texcoord );
              return _OccludedColor * c.a;
          }
      ENDCG
      }
  }
}