StencilBuffer in orthogonal mode - almost working?

I almost have stencils working in orthogonal mode. Here is the simpleInitGame:



   protected void simpleInitGame() {
      rootNode.setCullHint(CullHint.Never);
      rootNode.clearRenderState(StateType.Light);
      
      Node ortho = new Node("ortho");
      ortho.setRenderQueueMode(Renderer.QUEUE_ORTHO);
      
      StencilState ssa = display.getRenderer().createStencilState();
      ssa.setEnabled(true);
      ssa.setUseTwoSided(false);
      ssa.setStencilFunction(StencilFunction.Always);
      ssa.setStencilReference(0x1);
      ssa.setStencilMask(0x1);
      ssa.setStencilOpFail(StencilOperation.Replace);
      ssa.setStencilOpZFail(StencilOperation.Replace);
      ssa.setStencilOpZPass(StencilOperation.Replace);
      
      StencilState ssb = display.getRenderer().createStencilState();
      ssb.setEnabled(true);
      ssb.setUseTwoSided(false);
      ssb.setStencilFunction(StencilFunction.EqualTo);
      ssb.setStencilReference(0x1);
      ssb.setStencilMask(0x1);
      ssb.setStencilOpFail(StencilOperation.Keep);
      ssb.setStencilOpZFail(StencilOperation.Keep);
      ssb.setStencilOpZPass(StencilOperation.Keep);
      
      ColorMaskState cmsa = display.getRenderer().createColorMaskState();
      cmsa.setAll(false);
      
      Disk a = new Disk("disk", 10, 40, 100);
      a.setSolidColor(ColorRGBA.white);
      a.setLocalTranslation(400, 400, 0f);
      a.setRenderState(ssa);   
      a.setRenderState(cmsa);
      a.updateRenderState();      
      a.setZOrder(0);
      ortho.attachChild(a);      
      
      Quad b = new Quad("b", 200, 100);
      b.setRandomColors();
      b.setLocalTranslation(450, 410, 0f);
      b.setRenderState(ssb);
      b.setZOrder(1);      
      b.updateRenderState();
      ortho.attachChild(b);
      
      rootNode.attachChild(ortho);
      rootNode.updateRenderState();
      
      input = new InputHandler();
      MouseInput.get().setCursorVisible(true);
   }



Here's what it produces. Looks good until the disk is moved... the stencilbuffers aren't being cleared (yet).


So, now when I do clear the stencil buffer:


   protected void render( float interpolation ) {
      Renderer r = display.getRenderer();
      r.clearBuffers();
      r.clearStencilBuffer();
      GameTaskQueueManager.getManager().getQueue(GameTaskQueue.RENDER).execute();
      r.draw(rootNode);
      simpleRender();
   }



I get nothing... black.. void.

Any tips on debugging (is there a way of viewing the stencil buffer?) or any tips that might help me in understanding the nature of the problem?

Much appreciated!

Fixed it. Turned out to be a Z-order issue. Here's a working example of using the stencil buffer in orthogonal mode:



protected void simpleInitGame() {
      rootNode.setCullHint(CullHint.Never);
      rootNode.clearRenderState(StateType.Light);
            
      Node ortho = new Node("ortho"); // node holds all ORTHO/2D content
      ortho.setRenderQueueMode(Renderer.QUEUE_ORTHO);
      
      // when drawing the disk, set 1's in the stencil buffer
      StencilState diskStencilState = display.getRenderer().createStencilState();
      diskStencilState.setEnabled(true);
      diskStencilState.setUseTwoSided(false);
      diskStencilState.setStencilFunction(StencilFunction.Always);
      diskStencilState.setStencilReference(0x1);
      diskStencilState.setStencilMask(0x1);
      diskStencilState.setStencilOpFail(StencilOperation.Replace);
      diskStencilState.setStencilOpZFail(StencilOperation.Replace);
      diskStencilState.setStencilOpZPass(StencilOperation.Replace);

      // don't want the disk to appear on screen - only using it to mask
      ColorMaskState diskColorMaskState = display.getRenderer().createColorMaskState();
      diskColorMaskState.setAll(false);
      
      Disk disk = new Disk("disk", 10, 40, 100);
      disk.setSolidColor(ColorRGBA.white);
      disk.setLocalTranslation(400, 400, 0f);
      disk.setRenderState(diskStencilState);    // draw 1's int the stencil buffer
      disk.setRenderState(diskColorMaskState); // don't draw in the color buffer
      disk.updateRenderState();      
      disk.setZOrder(1);                    // this is very important! Draw disk first
      ortho.attachChild(disk);
      
      
      // when drawing the quad, only draw pixels where stencil buffer has 1's
      // and don't change the stencil buffer
      StencilState quadStencilState = display.getRenderer().createStencilState();
      quadStencilState.setEnabled(true);
      quadStencilState.setUseTwoSided(false);
      quadStencilState.setStencilFunction(StencilFunction.EqualTo);
      quadStencilState.setStencilReference(0x1);
      quadStencilState.setStencilMask(0x1);
      quadStencilState.setStencilOpFail(StencilOperation.Keep);
      quadStencilState.setStencilOpZFail(StencilOperation.Keep);
      quadStencilState.setStencilOpZPass(StencilOperation.Keep);

      Quad quad = new Quad("quad", 200, 100);
      quad.setRandomColors();
      quad.setLocalTranslation(450, 410, 0f);
      quad.setRenderState(quadStencilState);
      quad.setZOrder(0);          // very important! Draw quad after disk   
      quad.updateRenderState();
      ortho.attachChild(quad);
      
      rootNode.attachChild(ortho);
      rootNode.updateRenderState();
      
      input = new InputHandler();
      MouseInput.get().setCursorVisible(true);
   }

// make sure to clear the stencil buffer on each frame (if needed):

   protected void render( float interpolation ) {
      Renderer r = display.getRenderer();
      r.clearBuffers();
      r.clearStencilBuffer();
      GameTaskQueueManager.getManager().getQueue(GameTaskQueue.RENDER).execute();
      r.draw(rootNode);
      simpleRender();
   }


One more thing that can save you hours of frustration:



DisplaySystem.getDisplaySystem().setMinStencilBits(4);


good hint uzudil, i think the stencil bit must be set before the game is started / renderer is created