[Solved] Fade out Shadows

I have a PointLight and a PointLightShadowRenderer to cast shadows.
I noticed that when I combine the light with an AmbientLight, I get shadows casted outside of the radius of the PointLight.
In an ideal world I would like the shadows to fade out the same way the light itself does (something like AbstractShadowRenderer.setShadowZExtend+setShadowZFadeLength, but with the light itself as reference and not the viewport camera).
My idea would be to make the shadow intensitiy proportionate to the distance of the light source and the shadow fragment (in PostShadow.frag). Would that work?
Unfortunately, I’m pretty new to shader programming. I’ve read a few tutorials. I’ve looked into the fragment shader code. But even if my idea would be correct, I’m currently not able to implement that.
Edit: I could also live with a hard border where the sphere of the light ends. Not ideal, but ok.

It’s very surprising for me, but I’ve figured it out. To be specific: I’ve figured out how to change the behaviour of AbstractShadowRenderer.setShadowZExtend+setShadowZFadeLength to be relative to the light source instead of being relative to the camera.
Below is a fully working code sample that replaces the according shader code at runtime (just use PointLightShadowRenderer2 instead of the “normal” PointLightShadowRenderer):

package com.jme3;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import com.jme3.asset.AssetKey;
import com.jme3.asset.AssetManager;
import com.jme3.material.Material;
import com.jme3.material.MaterialDef;
import com.jme3.material.TechniqueDef;
import com.jme3.renderer.RenderManager;
import com.jme3.renderer.ViewPort;
import com.jme3.shadow.PointLightShadowRenderer;

public class PointLightShadowRenderer2 extends PointLightShadowRenderer {
	private static final String vertShaderName1 = "Common/MatDefs/Shadow/PostShadow.vert", vertShaderName2 = "Custom/PostShadow.vert";
	
	protected PointLightShadowRenderer2() {}
	
	public PointLightShadowRenderer2(AssetManager assetManager, int shadowMapSize) {
		super(assetManager,shadowMapSize);
		var str = (String)assetManager.loadAsset(vertShaderName1);
		var lines = new ArrayList<>(Arrays.asList(str.split("\n"))); var found = false;
		for(int i = 0; i < lines.size(); i++) {
			if(!lines.get(i).strip().equals("#if defined(PSSM) || defined(FADE)")) continue;
			if(!lines.get(i+1).strip().equals("shadowPosition = gl_Position.z;")) continue;
			if(!lines.get(i+8).strip().equals("worldPos = TransformWorld(modelSpacePos);")) throw new Error();
			found = true;
			var a = lines.remove(i); var b = lines.remove(i); var c = lines.remove(i);
			b = "shadowPosition = distance(worldPos,vec4(m_LightPos,1.0));";
			lines.addAll(i+6,Arrays.asList(a,b,c));
			break;
		}
		if(!found) throw new Error();
		assetManager.addToCache(new AssetKey<>(vertShaderName2),String.join("\n",lines));
		for(var techs : techniques(postshadowMat).values()) {
			for(var tech : techs) {
				if(tech.getVertexShaderName().equals(vertShaderName2)) continue;
				if(!tech.getVertexShaderName().equals(vertShaderName1)) throw new Error();
				tech.setShaderFile(vertShaderName2,tech.getFragmentShaderName(),tech.getVertexShaderLanguage(),tech.getFragmentShaderLanguage());
			}
		}
	}
	
	@SuppressWarnings("unchecked")
	private Map<String,List<TechniqueDef>> techniques(Material mat) {
		try {
			var fld = MaterialDef.class.getDeclaredField("techniques"); fld.setAccessible(true);
			return (Map<String,List<TechniqueDef>>)fld.get(mat.getMaterialDef());
		} catch (Throwable t) { throw new Error(t); }
	}
	
	@Override
	public void initialize(RenderManager rm, ViewPort vp) {
		super.initialize(rm,vp); skipPostPass = true; needsfallBackMaterial = true; postTechniqueName = null;
	}
	
	@Override protected void initFrustumCam() {}
	
	@Override
	public void preFrame(float tpf) {
		if(light.getRadius() != getShadowZExtend()) {
			setShadowZExtend(light.getRadius()); setShadowZFadeLength(light.getRadius()/2);
		}
	}
}

3 Likes