Mesa depth stencil bug bypass

src/main/java/me/cortex/voxy/client/core/gl/Capabilities.java

@@ -4,6 +4,8 @@ import me.cortex.voxy.client.core.gl.shader.ShaderType;
 import org.lwjgl.opengl.GL;
 import org.lwjgl.opengl.GL20C;
 
+import static org.lwjgl.opengl.GL11.GL_VENDOR;
+import static org.lwjgl.opengl.GL11.glGetString;
 import static org.lwjgl.opengl.GL32.glGetInteger64;
 import static org.lwjgl.opengl.GL43C.GL_MAX_SHADER_STORAGE_BLOCK_SIZE;
 
@@ -14,6 +16,7 @@ public class Capabilities {
     public final boolean meshShaders;
     public final boolean INT64_t;
     public final long ssboMaxSize;
+    public final boolean isMesa;
     public Capabilities() {
         var cap = GL.getCapabilities();
         this.meshShaders = cap.GL_NV_mesh_shader && cap.GL_NV_representative_fragment_test;
@@ -29,6 +32,8 @@ public class Capabilities {
                 """);
 
         this.ssboMaxSize = glGetInteger64(GL_MAX_SHADER_STORAGE_BLOCK_SIZE);
+
+        this.isMesa = "Mesa".equalsIgnoreCase(glGetString(GL_VENDOR));
     }
 
     public static void init() {

src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java

@@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
 import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
 import it.unimi.dsi.fastutil.objects.ObjectSet;
+import me.cortex.voxy.client.core.gl.Capabilities;
 import me.cortex.voxy.client.core.rendering.util.RawDownloadStream;
 import me.cortex.voxy.client.core.rendering.util.UploadStream;
 import me.cortex.voxy.common.Logger;
@@ -261,6 +262,12 @@ public class ModelFactory {
 
         int checkMode = blockRenderLayer==RenderLayer.getSolid()?TextureUtils.WRITE_CHECK_STENCIL:TextureUtils.WRITE_CHECK_ALPHA;
 
+        if (Capabilities.INSTANCE.isMesa) {
+            //Mesa does not work with GL_DEPTH_STENCIL_TEXTURE_MODE GL_STENCIL_INDEX
+            // the sampler in the compute shader always reads zero even when stencil is guarenteed not to be zero
+            // (e.g. clearing with stencil 10)
+            checkMode = TextureUtils.WRITE_CHECK_ALPHA;
+        }
 
 
 

src/main/java/me/cortex/voxy/client/core/model/ModelTextureBakery.java

@@ -80,11 +80,12 @@ public class ModelTextureBakery {
         this.height = height;
         this.colourTex = new GlTexture().store(GL_RGBA8, 1, width, height).name("ModelBakeryColour");
         this.depthTex = new GlTexture().store(GL_DEPTH24_STENCIL8, 1, width, height).name("ModelBakeryDepth");
+        glTextureParameteri(this.depthTex.id, GL_DEPTH_STENCIL_TEXTURE_MODE, GL_STENCIL_INDEX);
         this.depthTexView = this.depthTex.createView();
+        glTextureParameteri(this.depthTex.id, GL_DEPTH_STENCIL_TEXTURE_MODE, GL_DEPTH_COMPONENT);
 
         this.framebuffer = new GlFramebuffer().bind(GL_COLOR_ATTACHMENT0, this.colourTex).bind(GL_DEPTH_STENCIL_ATTACHMENT, this.depthTex).verify().name("ModelFramebuffer");
 
-        glTextureParameteri(this.depthTex.id, GL_DEPTH_STENCIL_TEXTURE_MODE, GL_DEPTH_COMPONENT);
         glTextureParameteri(this.depthTexView.id, GL_DEPTH_STENCIL_TEXTURE_MODE, GL_STENCIL_INDEX);
 
         this.copyOutShader = Shader.make()
@@ -159,6 +160,7 @@ public class ModelTextureBakery {
 
         glClearColor(0,0,0,0);
         glClearDepth(1);
+        glClearStencil(0);
         glBindFramebuffer(GL_FRAMEBUFFER, this.framebuffer.id);
 
         glEnable(GL_STENCIL_TEST);
@@ -332,6 +334,8 @@ public class ModelTextureBakery {
         this.emitToStream(streamBuffer, streamOffset);
     }
 
+    //TODO: FIXME: Mesa is broken when trying to read from a sampler of GL_STENCIL_INDEX
+    // it seems to just ignore the value set in GL_DEPTH_STENCIL_TEXTURE_MODE
     private void emitToStream(int streamBuffer, int streamOffset) {
         if (streamOffset%4 != 0) {
             throw new IllegalArgumentException();
@@ -339,10 +343,13 @@ public class ModelTextureBakery {
         this.copyOutShader.bind();
         glActiveTexture(GL_TEXTURE0);
         GL11C.glBindTexture(GL11.GL_TEXTURE_2D, this.colourTex.id);
+        glBindSampler(0, 0);
         glActiveTexture(GL_TEXTURE1);
         glBindTexture(GL11.GL_TEXTURE_2D, this.depthTex.id);
+        glBindSampler(1, 0);
         glActiveTexture(GL_TEXTURE2);
         glBindTexture(GL11.GL_TEXTURE_2D, this.depthTexView.id);
+        glBindSampler(2, 0);
         glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, streamBuffer);
         glUniform1ui(4, streamOffset/4);