JME 3.1 alpha crashes when using CartoonEdgeFilter and an InstanceNode

It is as the title says, my application crashes as soon as I have a InstanceNode and a CartoonEdgeFilter at the same time. Here’s a test case (which is simply the TestInstanceNode with an added CartoonEdgeFilter)

/*
 * Copyright (c) 2009-2012 jMonkeyEngine
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 * * Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 *
 * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
 *   may be used to endorse or promote products derived from this software
 *   without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package jme3test.scene.instancing;

import com.jme3.app.SimpleApplication;
import com.jme3.material.Material;
import com.jme3.math.ColorRGBA;
import com.jme3.math.FastMath;
import com.jme3.math.Quaternion;
import com.jme3.math.Vector3f;
import com.jme3.post.FilterPostProcessor;
import com.jme3.post.filters.CartoonEdgeFilter;
import com.jme3.scene.Geometry;
import com.jme3.scene.Mesh;
import com.jme3.scene.Spatial;
import com.jme3.scene.Node;
import com.jme3.scene.instancing.InstancedGeometry;
import com.jme3.scene.instancing.InstancedNode;
import com.jme3.scene.shape.Box;
import com.jme3.scene.shape.Sphere;
import com.jme3.system.AppSettings;

public class TestInstanceNode extends SimpleApplication  {

    private Mesh mesh1;
    private Mesh mesh2;
    private final Material[] materials = new Material[6];
    private Node instancedNode;
    private float time = 0;
    private boolean INSTANCING = true;
    
    public static void main(String[] args){
        TestInstanceNode app = new TestInstanceNode();
        AppSettings settings = new AppSettings(true);
        settings.setVSync(false);
        app.setSettings(settings);
        app.start();
    }

    private Geometry createInstance(float x, float z) {
        Mesh mesh; 
        if (FastMath.nextRandomInt(0, 1) == 1) mesh = mesh2;
        else mesh = mesh1;
        Geometry geometry = new Geometry("randomGeom", mesh);
        geometry.setMaterial(materials[FastMath.nextRandomInt(0, materials.length - 1)]);
        geometry.setLocalTranslation(x, 0, z);
        return geometry;
    }
    
    @Override
    public void simpleInitApp() {
        mesh1 = new Sphere(13, 13, 0.4f, true, false);
        mesh2 = new Box(0.4f, 0.4f, 0.4f);
        
        materials[0] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[0].setBoolean("UseInstancing", INSTANCING);
        materials[0].setColor("Color", ColorRGBA.Red);
        
        materials[1] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[1].setBoolean("UseInstancing", INSTANCING);
        materials[1].setColor("Color", ColorRGBA.Green);
        
        materials[2] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[2].setBoolean("UseInstancing", INSTANCING);
        materials[2].setColor("Color", ColorRGBA.Blue);
        
        materials[3] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[3].setBoolean("UseInstancing", INSTANCING);
        materials[3].setColor("Color", ColorRGBA.Cyan);
        
        materials[4] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[4].setBoolean("UseInstancing", INSTANCING);
        materials[4].setColor("Color", ColorRGBA.Magenta);
        
        materials[5] = new Material(assetManager, "Common/MatDefs/Misc/Unshaded.j3md");
        materials[5].setBoolean("UseInstancing", INSTANCING);
        materials[5].setColor("Color", ColorRGBA.Yellow);
       
        instancedNode = new InstancedNode("instanced_node");
        
        rootNode.attachChild(instancedNode);
        
        int extent = 30;
        
        for (int y = -extent; y < extent; y++) {
            for (int x = -extent; x < extent; x++) {
                Geometry instance = createInstance(x, y);
                
                float height = (smoothstep(0, 1, FastMath.nextRandomFloat()) * 2.5f) - 1.25f;
                instance.setUserData("height", height);
                instance.setUserData("dir", 1f);
                
                instancedNode.attachChild(instance);
            }
        }
        
        if (INSTANCING) {
            ((InstancedNode)instancedNode).instance();
        }
        
        //instancedNode = (InstancedNode) instancedNode.clone();
        //instancedNode.move(0, 5, 0);
        //rootNode.attachChild(instancedNode);
        
        cam.setLocation(new Vector3f(38.373516f, 6.689055f, 38.482082f));
        cam.setRotation(new Quaternion(-0.04004206f, 0.918326f, -0.096310444f, -0.38183528f));
        flyCam.setMoveSpeed(15);
        flyCam.setEnabled(false);
        
        FilterPostProcessor filterPostProcessor = new FilterPostProcessor(assetManager);
        CartoonEdgeFilter cartoonEdge = new CartoonEdgeFilter();
        filterPostProcessor.addFilter(cartoonEdge);
        viewPort.addProcessor(filterPostProcessor);
    }
    
    private float smoothstep(float edge0, float edge1, float x) {
        // Scale, bias and saturate x to 0..1 range
        x = FastMath.clamp((x - edge0) / (edge1 - edge0), 0.0f, 1.0f);
        // Evaluate polynomial
        return x * x * (3 - 2 * x);
    }
    
    
    @Override
    public void simpleUpdate(float tpf) {
        time += tpf;

        if (time > 1f) {
            time = 0f;
            
            for (Spatial instance : instancedNode.getChildren()) {
                if (!(instance instanceof InstancedGeometry)) {
                    Geometry geom = (Geometry) instance;
                    geom.setMaterial(materials[FastMath.nextRandomInt(0, materials.length - 1)]);

                    Mesh mesh; 
                    if (FastMath.nextRandomInt(0, 1) == 1) mesh = mesh2;
                    else mesh = mesh1;
                    geom.setMesh(mesh);
                }
            }
        }
        
        for (Spatial child : instancedNode.getChildren()) {
            if (!(child instanceof InstancedGeometry)) {
                float val = ((Float)child.getUserData("height")).floatValue();
                float dir = ((Float)child.getUserData("dir")).floatValue();

                val += (dir + ((FastMath.nextRandomFloat() * 0.5f) - 0.25f)) * tpf;

                if (val > 1f) {
                    val = 1f;
                    dir = -dir;
                } else if (val < 0f) {
                    val = 0f;
                    dir = -dir;
                }

                Vector3f translation = child.getLocalTranslation();
                translation.y = (smoothstep(0, 1, val) * 2.5f) - 1.25f;

                child.setUserData("height", val);
                child.setUserData("dir", dir);

                child.setLocalTranslation(translation);
            }
        }
    }
}

Here’s the output of the error:

WARNING: Bad compile of:
1	#version 110
2	#define VERTEX_SHADER 1
3	#define INSTANCING 1
4	// -- begin import Common/ShaderLib/Instancing.glsllib --
5	// Instancing GLSL library.
6	// 
7	// When the INSTANCING define is set in the shader, 
8	// all global matrices are replaced with "instanced" versions.
9	// One exception is g_NormalMatrix which becomes unusable,
10	// instead the function ApplyNormalTransform is used to transform
11	// the normal and tangent vectors into world view space.
12	
13	// The world matrix and normal transform quaternion need to be passed
14	// as vertex attributes "inWorldMatrix" and "inNormalRotationQuaternion"
15	// respectively. 
16	// The VertexBuffers for those two attributes 
17	// need to be configured into instanced mode (VertexBuffer.setInstanced(true)). 
18	//  - inWorldMatrix should have 12 * numInstances floats.
19	//  - inNormalRotationQuaternion should have 4 * numInstances.
20	// Thus, instancing data occupies 4 vertex attributes (16 / 4 = 4).
21	// 
22	// The GL_ARB_draw_instanced and GL_ARB_instanced_arrays extensions 
23	// are required (OGL 3.3).
24	
25	uniform mat4 g_WorldMatrix;
26	uniform mat4 g_ViewMatrix;
27	uniform mat4 g_ProjectionMatrix;
28	uniform mat4 g_WorldViewMatrix;
29	uniform mat4 g_WorldViewProjectionMatrix;
30	uniform mat4 g_ViewProjectionMatrix;
31	uniform mat3 g_NormalMatrix;
32	
33	#if defined INSTANCING
34	
35	// World Matrix + Normal Rotation Quaternion. 
36	// The World Matrix is the top 3 rows - 
37	//     since the bottom row is always 0,0,0,1 for this transform.
38	// The bottom row is the transpose of the inverse of WorldView Transform 
39	//     as a quaternion. i.e. g_NormalMatrix converted to a quaternion.
40	//
41	// Using a quaternion instead of a matrix here allows saving approximately
42	// 2 vertex attributes which now can be used for additional per-vertex data.
43	attribute mat4 inInstanceData;
44	
45	// Extract the world matrix out of the instance data, leaving out the 
46	// quaternion at the end.
47	mat4 worldMatrix = mat4(vec4(inInstanceData[0].xyz, 0.0), 
48	                            vec4(inInstanceData[1].xyz, 0.0), 
49	                            vec4(inInstanceData[2].xyz, 0.0), 
50	                            vec4(inInstanceData[3].xyz, 1.0));
51	
52	vec4 TransformWorld(vec4 position)
53	{
54	    return (worldMatrix * position);
55	}
56	
57	vec4 TransformWorldView(vec4 position)
58	{
59	    return g_ViewMatrix * TransformWorld(position);
60	}
61	
62	vec4 TransformWorldViewProjection(vec4 position)
63	{
64	    return g_ViewProjectionMatrix * TransformWorld(position);
65	}
66	
67	vec3 TransformNormal(vec3 vec)
68	{
69	    vec4 quat = vec4(inInstanceData[0].w, inInstanceData[1].w,
70	                     inInstanceData[2].w, inInstanceData[3].w);
71	
72	    vec3 worldNormal = vec + vec3(2.0) * cross(cross(vec, quat.xyz) + vec3(quat.w) * vec, quat.xyz);
73	    
74	    return (g_ViewMatrix * vec4(worldNormal, 0.0)).xyz;
75	}
76	
77	// Prevent user from using g_** matrices which will have invalid data in this case.
78	#define g_WorldMatrix               Use_the_instancing_functions_for_this
79	#define g_WorldViewMatrix           Use_the_instancing_functions_for_this
80	#define g_WorldViewProjectionMatrix Use_the_instancing_functions_for_this
81	#define g_NormalMatrix              Use_the_instancing_functions_for_this
82	
83	#else
84	
85	vec4 TransformWorld(vec4 position)
86	{
87	    return g_WorldMatrix * position;
88	}
89	
90	vec4 TransformWorldView(vec4 position)
91	{
92	    return g_WorldViewMatrix * position;
93	}
94	
95	vec4 TransformWorldViewProjection(vec4 position)
96	{
97	    return g_WorldViewProjectionMatrix * position;
98	}
99	
100	vec3 TransformNormal(vec3 normal) {
101		return g_NormalMatrix * normal;
102	}
103	
104	#endif
105	// -- end import Common/ShaderLib/Instancing.glsllib --
106	// -- begin import Common/ShaderLib/Skinning.glsllib --
107	#ifdef NUM_BONES
108	
109	#if NUM_BONES < 1 || NUM_BONES > 255
110	#error NUM_BONES must be between 1 and 255.
111	#endif
112	
113	#define NUM_WEIGHTS_PER_VERT 4
114	 
115	attribute vec4 inHWBoneWeight;
116	attribute vec4 inHWBoneIndex;
117	uniform mat4 m_BoneMatrices[NUM_BONES];
118	
119	void Skinning_Compute(inout vec4 position){
120	    if (inHWBoneWeight.x != 0.0) {
121	#if NUM_WEIGHTS_PER_VERT == 1
122	        position = m_BoneMatrices[int(inHWBoneIndex.x)] * position;
123	#else
124	        mat4 mat = mat4(0.0);
125	        mat += m_BoneMatrices[int(inHWBoneIndex.x)] * inHWBoneWeight.x;
126	        mat += m_BoneMatrices[int(inHWBoneIndex.y)] * inHWBoneWeight.y;
127	        mat += m_BoneMatrices[int(inHWBoneIndex.z)] * inHWBoneWeight.z;
128	        mat += m_BoneMatrices[int(inHWBoneIndex.w)] * inHWBoneWeight.w;
129	        position = mat * position;
130	#endif
131	    }
132	}
133	 
134	void Skinning_Compute(inout vec4 position, inout vec3 normal){
135	    if (inHWBoneWeight.x != 0.0) {
136	#if NUM_WEIGHTS_PER_VERT == 1
137	        position = m_BoneMatrices[int(inHWBoneIndex.x)] * position;
138	        normal = (mat3(m_BoneMatrices[int(inHWBoneIndex.x)][0].xyz,
139	                       m_BoneMatrices[int(inHWBoneIndex.x)][1].xyz,
140	                       m_BoneMatrices[int(inHWBoneIndex.x)][2].xyz) * normal);
141	#else
142	        mat4 mat = mat4(0.0);
143	        mat += m_BoneMatrices[int(inHWBoneIndex.x)] * inHWBoneWeight.x;
144	        mat += m_BoneMatrices[int(inHWBoneIndex.y)] * inHWBoneWeight.y;
145	        mat += m_BoneMatrices[int(inHWBoneIndex.z)] * inHWBoneWeight.z;
146	        mat += m_BoneMatrices[int(inHWBoneIndex.w)] * inHWBoneWeight.w;
147	        position = mat * position;
148	
149	        mat3 rotMat = mat3(mat[0].xyz, mat[1].xyz, mat[2].xyz);
150	        normal = rotMat * normal;
151	#endif
152	    }
153	}
154	 
155	void Skinning_Compute(inout vec4 position, inout vec3 tangent, inout vec3 normal){
156	    if (inHWBoneWeight.x != 0.0) {
157	#if NUM_WEIGHTS_PER_VERT == 1
158	        position = m_BoneMatrices[int(inHWBoneIndex.x)] * position;
159	        tangent = m_BoneMatrices[int(inHWBoneIndex.x)] * tangent;
160	        normal = (mat3(m_BoneMatrices[int(inHWBoneIndex.x)][0].xyz,
161	                       m_BoneMatrices[int(inHWBoneIndex.x)][1].xyz,
162	                       m_BoneMatrices[int(inHWBoneIndex.x)][2].xyz) * normal);
163	#else
164	        mat4 mat = mat4(0.0);
165	        mat += m_BoneMatrices[int(inHWBoneIndex.x)] * inHWBoneWeight.x;
166	        mat += m_BoneMatrices[int(inHWBoneIndex.y)] * inHWBoneWeight.y;
167	        mat += m_BoneMatrices[int(inHWBoneIndex.z)] * inHWBoneWeight.z;
168	        mat += m_BoneMatrices[int(inHWBoneIndex.w)] * inHWBoneWeight.w;
169	        position = mat * position;
170	
171	        mat3 rotMat = mat3(mat[0].xyz, mat[1].xyz, mat[2].xyz);
172	        tangent = rotMat * tangent;
173	        normal = rotMat * normal;
174	#endif
175	    }
176	}
177	
178	#endif
179	// -- end import Common/ShaderLib/Skinning.glsllib --
180	// These are included in the above now
181	//uniform mat4 g_WorldViewProjectionMatrix;
182	//uniform mat3 g_NormalMatrix;
183	
184	attribute vec3 inPosition;
185	attribute vec3 inNormal;
186	attribute vec4 inTexCoord;
187	
188	varying vec3 normal;
189	varying vec2 texCoord;
190	
191	void main(void)
192	{
193	   texCoord=inTexCoord.xy;
194	   vec4 modelSpacePos = vec4(inPosition, 1.0);
195	   vec3 modelSpaceNormals = inNormal;
196	   #ifdef NUM_BONES
197	       Skinning_Compute(modelSpacePos,modelSpaceNormals);
198	   #endif
199	   normal = normalize(g_NormalMatrix * modelSpaceNormals);
200	   gl_Position = g_WorldViewProjectionMatrix * modelSpacePos;
201	}

Oct 21, 2015 7:59:23 PM com.jme3.app.Application handleError
SEVERE: Uncaught exception thrown in Thread[jME3 Main,5,main]
com.jme3.renderer.RendererException: compile error in: ShaderSource[name=Common/MatDefs/SSAO/normal.vert, defines, type=Vertex, language=GLSL100]
ERROR: 0:199: 'Use_the_instancing_functions_for_this' : undeclared identifier 


	at com.jme3.renderer.opengl.GLRenderer.updateShaderSourceData(GLRenderer.java:1126)
	at com.jme3.renderer.opengl.GLRenderer.updateShaderData(GLRenderer.java:1153)
	at com.jme3.renderer.opengl.GLRenderer.setShader(GLRenderer.java:1217)
	at com.jme3.material.Material.render(Material.java:1215)
	at com.jme3.renderer.RenderManager.renderGeometry(RenderManager.java:552)
	at com.jme3.renderer.queue.RenderQueue.renderGeometryList(RenderQueue.java:266)
	at com.jme3.renderer.queue.RenderQueue.renderQueue(RenderQueue.java:305)
	at com.jme3.renderer.RenderManager.renderViewPortQueues(RenderManager.java:803)
	at com.jme3.post.filters.CartoonEdgeFilter.postQueue(CartoonEdgeFilter.java:81)
	at com.jme3.post.FilterPostProcessor.postQueue(FilterPostProcessor.java:229)
	at com.jme3.renderer.RenderManager.renderViewPort(RenderManager.java:1025)
	at com.jme3.renderer.RenderManager.render(RenderManager.java:1078)
	at com.jme3.app.SimpleApplication.update(SimpleApplication.java:260)
	at com.jme3.system.lwjgl.LwjglAbstractDisplay.runLoop(LwjglAbstractDisplay.java:152)
	at com.jme3.system.lwjgl.LwjglDisplay.runLoop(LwjglDisplay.java:192)
	at com.jme3.system.lwjgl.LwjglAbstractDisplay.run(LwjglAbstractDisplay.java:233)
	at java.lang.Thread.run(Thread.java:745)

Old thread I know but it should be fixed in master now.

3 Likes

Thanks, I just saw the commit for the issue on github. I’ll have to try it out later :smile: