[SOLVED] Shader Artifacts with AA Turned On

Troubleshooting graphics
1

I have a shader using dFdx and dFdy to produce pseudo anti-aliasing. This is necessary because the shader draws curved edges which are not effected by hardware anti-aliasing methods. While this looks great when hardware anti-aliasing, settings.setSamples(1), is turned off, it produces artifacts when hardware anti-aliasing is turned on and I’m not sure why.

settings.setSamples(1):

settings.setSamples(8)

Any thoughts? I tried padding the mesh with extra space around the edges and just discarding those pixels, but that didn’t make any difference.

Fragment shader:

#ifdef USEAA
    #ifdef ISMOBILE
        #extension GL_OES_standard_derivatives:enable
    #endif
#endif

uniform vec4 m_brdr1;
uniform vec4 m_brdr2;
uniform vec4 m_brdr3;
uniform vec4 m_brdr4;

uniform float m_brdrstp1;
uniform float m_brdrstp2;
uniform float m_brdrstp3;
uniform float m_brdrstp4;

uniform float m_Radius;

uniform vec4 m_col1;
uniform vec4 m_col2;
uniform vec4 m_col3;
uniform vec4 m_col4;
uniform vec4 m_col5;
uniform vec4 m_col6;
uniform vec4 m_col7;
uniform vec4 m_col8;

uniform float m_stop1;
uniform float m_stop2;
uniform float m_stop3;
uniform float m_stop4;
uniform float m_stop5;
uniform float m_stop6;
uniform float m_stop7;
uniform float m_stop8;

uniform float m_Alpha;
uniform bool m_useAA;
uniform bool m_isCircle;

varying vec2 texCoord;
varying vec2 texCoord2;
varying vec2 borderCoord;

#ifdef USEAA
    float AA(in float r) {
        vec2 px = vec2(dFdx(texCoord.x), dFdx(texCoord.y));
        vec2 py = vec2(dFdy(texCoord.x), dFdy(texCoord.y));
        float fx = (2*texCoord.x)*px.x - px.y;
        float fy = (2*texCoord.y)*py.x - py.y;
        float sd = r/sqrt(fx*fx + fy*fy);
        
        return clamp(0.25 + sd, 0.0, 1.0);
    }
#endif

void fadeColor(inout vec4 color, in vec4 color2, in float percent) {
    color *= percent;
    color += color2 * (1.0 - percent);
}

vec4 panelColor(inout vec4 color) {
    color = m_col1;
    
    float perc = clamp((m_stop1 - texCoord2.y) / min(m_stop1 - m_stop2, -0.00001), 0.0, 1.0);
    fadeColor(color, m_col2, 1.0 - perc);
    
    #ifdef HAS_COL3
        perc = clamp((m_stop2 - texCoord2.y) / min(m_stop2 - m_stop3, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col3, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL4
        perc = clamp((m_stop3 - texCoord2.y) / min(m_stop3 - m_stop4, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col4, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL5
        perc = clamp((m_stop4 - texCoord2.y) / min(m_stop4 - m_stop5, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col5, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL6
        perc = clamp((m_stop5 - texCoord2.y) / min(m_stop5 - m_stop6, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col6, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL7
        perc = clamp((m_stop6 - texCoord2.y) / min(m_stop6 - m_stop7, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col7, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL8
        perc = clamp((m_stop7 - texCoord2.y) / min(m_stop7 - m_stop8, -0.00001), 0.0, 1.0);
        fadeColor(color, m_col8, 1.0 - perc);
    #endif
    
    return color;
}

vec4 radialPanelColor(inout vec4 color) {
    color = m_col1;
    float r = sqrt((texCoord2.x*texCoord2.x) + (texCoord2.y*texCoord2.y));
    
    float perc = clamp((r - m_stop1) / max(m_stop2 - m_stop1, 0.00001), 0.0, 1.0);
    fadeColor(color, m_col2, 1.0 - perc);
    
    #ifdef HAS_COL3
        perc = clamp((r - m_stop2) / max(m_stop3 - m_stop2, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col3, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL4
        perc = clamp((r - m_stop3) / max(m_stop4 - m_stop3, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col4, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL5
        perc = clamp((r - m_stop4) / max(m_stop5 - m_stop4, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col5, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL6
        perc = clamp((r - m_stop5) / max(m_stop6 - m_stop5, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col6, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL7
        perc = clamp((r - m_stop6) / max(m_stop7 - m_stop6, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col7, 1.0 - perc);
    #endif
    
    #ifdef HAS_COL8
        perc = clamp((r - m_stop7) / max(m_stop8 - m_stop7, 0.00001), 0.0, 1.0);
        fadeColor(color, m_col8, 1.0 - perc);
    #endif
    
    return color;
}

void border(inout vec4 color) {
    color = m_brdr1;
    #ifdef HAS_BRDR2
        float perc = clamp((m_brdrstp1 - borderCoord.y) / min(m_brdrstp1 - m_brdrstp2, -0.00001), 0.0, 1.0);
        fadeColor(color, m_brdr2, 1.0 - perc);
        
        #ifdef HAS_BRDR3
            perc = clamp((m_brdrstp2 - borderCoord.y) / min(m_brdrstp2 - m_brdrstp3, -0.00001), 0.0, 1.0);
            fadeColor(color, m_brdr3, 1.0 - perc);
        #endif

        #ifdef HAS_BRDR4
            perc = clamp((m_brdrstp3 - borderCoord.y) / min(m_brdrstp3 - m_brdrstp4, -0.00001), 0.0, 1.0);
            fadeColor(color, m_brdr4, 1.0 - perc);
        #endif
    #endif
}

void roundedBorder(inout vec4 color, inout float alpha) {
    float c = sqrt((texCoord.x*texCoord.x) + (texCoord.y*texCoord.y));
    float r = 1.0 - c;
    #ifdef USEAA
        alpha *= AA(r);
    #else
        alpha = step(0.0001, r);
    #endif

    float f = 1.0 - (c * (1.0 / max(m_Radius, 0.00001)));
    #ifdef USEAA
        f = AA(f);
    #else
        f = step(0.0001, f);
    #endif
    
    border(color);

    #ifdef HAS_COLFADE
        vec4 pCol = vec4(0);
        #ifdef IS_CIRCLE
            fadeColor(color, radialPanelColor(pCol), 1.0 - f);
        #else
            fadeColor(color, panelColor(pCol), 1.0 - f);
        #endif
    #else
        fadeColor(color, m_col1, 1.0 - f);
    #endif
}

void main() {
    float alpha = m_Alpha;
    vec4 col = vec4(0);
    
    if (texCoord.x < 2.0) {
        #ifdef HAS_BORDER
            #ifdef IS_ROUNDED
                roundedBorder(col, alpha);
            #else
                border(col);
            #endif
        #else
            #ifdef IS_ROUNDED
                float c = sqrt((texCoord.x*texCoord.x) + (texCoord.y*texCoord.y));
                float r = 1.0 - c;
                #ifdef USEAA
                    alpha *= AA(r);
                #else
                    alpha = step(0.0001, r);
                #endif
                #ifdef HAS_COLFADE
                    #ifdef IS_CIRCLE
                        radialPanelColor(col);
                    #else
                        panelColor(col);
                    #endif
                #else
                    col = m_col1;
                #endif
            #else
                alpha = 0.0;
            #endif
        #endif
    } else {
        #ifdef HAS_COLFADE
            #ifdef IS_CIRCLE
                radialPanelColor(col);
            #else
                panelColor(col);
            #endif
        #else
            col = m_col1;
        #endif
    }
    
    if (alpha <= 0.0) {
        discard;
    } else {
        gl_FragColor = vec4(col.rgb, alpha * col.a); 
    }
}
2

I’d just disable AA for GUI node - AA often do it’s “things” with images and icons as well. Not sure how to achieve it, though.

3

AA is a setting of the whole rendering context. You can’t just disable it for part of the scene.

4

It could be a driver issue, I haven’t tested on Windows, just Linux using MESA. I’m guessing it’s an issue having to do with pixel centers. I read that the shader might be run for a pixel that the geometry crosses, but does not fully cover, when multi-sampling is enabled. In such a case the coordinate passed to the shader will be outside of the normal bounds because the center of the pixel being called does not fall inside the geometry. Apparently you can use ‘centroid’ for this in order to obtain the center of the area covered by the geometry rather than the center of the pixel, but there are two caveats here. Centroids are not supported on mobile and, apparently, dFdx/dFdy are highly inaccurate when using centroids.

Of course, that may not be the issue as I figure padding the edges of the mesh with extra space and discarding those pixels would’ve taken care of that had it been the case.

My other option is to disable the pseudo anti-aliasing in the shader when drawing the straight edge. While this does solve the artifacts there’s a fairly noticeable switch between curved and straight edges.

5

Well, GUI node and all the non GUI stuff are not really parts of the same scene, are they? Considering they have separate cameras and viewports.

viewPort.attachScene(rootNode);
guiViewPort.attachScene(guiNode);

viewPort = renderManager.createMainView("Default", cam);
guiViewPort = renderManager.createPostView("Gui Default", guiCam);

Yes, I did dig it already when noticed how AA wrecked my icons on buttons but later got distracted by side-project on UE4 so never finished this research.

I’m not even that good to understand if this fact alone is enough to separate AA processing somehow or not. It may require A LOT of code rewrite and CPU/GPU overhead in the end.

6

No. But notice how I said “render context” and not “scene”. As in the “context within with EVERYTHING is rendered”. When OpenGL is setup for rendering, that’s when AA is setup… it’s down in the driver. You’d have to run two entirely separate OpenGL context’s to do what you are suggesting.

7

That’s sad. Thanks for info. Looks like I’ll stick with side-projects longer than expected - have to find a way around it or to make myself used to idea that we could only have decent UI or decent game graphics and not both at the same time due to AA issues.

To keep this post a bit useful - I use windows and still get hardware AA interfering with how my UI looks, making it “not so good” looking when AA is on. And not on internal Intel graphics card at it. Hopefully :chimpanzee_wink:

8

I don’t know how jME is doing the anti-aliasing, but could you toggle multisampling between viewport renders?

Render main views
glDisable(GL_MULTISAMPLE_ARB);
Render post views
glEnable(GL_MULTISAMPLE_ARB);

9

I’m not an expert here but I think it would require different frame buffer setup. All viewports (except off screen ones that setup their own) write to the main frame buffer.

I don’t think simply disabling it is going to work very well… but this is an area I don’t know much about. It’s just logical that the frame buffer would need to keep a lot more data for AA given how that works.

10

I don’t really know much about it either. The glDisable/Enable thing is just something I came across while researching solutions to my shader issue here.

11

Did you consider using FXAA instead of MSAA?
It’s much more flexible since it’s applied as post processing effect on the entire screen, and it’s also a lot faster. The output is a bit blurry though, but depending on the style of your game it might not be a big issue.

12

Probably not a great solution for this. I plan on releasing this to the community as part of a Lemur modification and I figure it’d be nice if options weren’t limited. At the same time I was curious to see how the FXAA filter effected this and I came across something interesting:

FilterPostProcessor fpp = new FilterPostProcessor(assetManager);
FXAAFilter fxaa = new FXAAFilter();
fpp.addFilter(fxaa);
this.getGuiViewPort().addProcessor(fpp);

When I use settings.setSamples(8) in conjunction with the above code it looks horrific, the background shaders don’t look bad, but the fonts look awful. Not just the TrueTypeFonts either, but the bitmap fonts used in the frame rate display look equally horrendous.

Now when I use settings.setSamples(1) in conjunction with the above code I get just a black screen, not even the frame rate display shows up. :poop:

13

Ah, i don’t know if you can attach filters to the guiViewPort.
You could try to render the gui on a framebuffer, it will be rendered without AA, and then apply either FXAA or your pseudo-aa to it, using a fullscreen quad and a material (or maybe another off-screen viewport with an fpp on it), and then you can show the final result on the guiViewPort.

14

Doing the same thing I did in my last post my results are a little different.

settings.setSamples(8):

fxaa8x.png798×448 85.4 KB

The screen starts out looking the same as it does with settings.setSamples(1), but once I resize it a bit, with Display.setResizable(true), the objects show up but with artifacting all over the place.

settings.setSamples(1):

fxaa1x.png798×448 4.05 KB

This is in windowed mode of course. Both look the same when I start up, but with settings.setSamples(8) the objects appear on the screen after I resize the window a tiny bit, if I resize too much they disappear again. With settings.setSamples(1), they never show up. Same results in fullscreen mode, but Linux doesn’t really have a true fullscreen mode, it’s just a window that takes up the whole screen.

15

I finally got around to testing the original issue on Windows and found that it is not a problem so it must be an issue with the open source MESA drivers on Linux.