Wip, shader tweeks, manager tweeks, wip on cleaner

src/main/java/me/cortex/voxy/client/core/gl/shader/AutoBindingShader.java

@@ -0,0 +1,61 @@
+package me.cortex.voxy.client.core.gl.shader;
+
+import me.cortex.voxy.client.core.gl.GlBuffer;
+import me.cortex.voxy.common.util.Pair;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.lwjgl.opengl.GL30.glBindBufferBase;
+import static org.lwjgl.opengl.GL30.glBindBufferRange;
+import static org.lwjgl.opengl.GL31.GL_UNIFORM_BUFFER;
+import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BUFFER;
+
+
+//TODO: rewrite the entire shader builder system
+public class AutoBindingShader extends Shader {
+    private record BufferBinding(int target, int index, GlBuffer buffer, long offset, long size) {}
+
+    private final Map<String, String> defines;
+    private final List<BufferBinding> bindings = new ArrayList<>();
+
+    AutoBindingShader(Shader.Builder<AutoBindingShader> builder, int program) {
+        super(program);
+        this.defines = builder.defines;
+    }
+
+    public AutoBindingShader ssbo(int index, GlBuffer binding) {
+        return this.ssbo(index, binding, 0);
+    }
+
+    public AutoBindingShader ssbo(String define, GlBuffer binding) {
+        return this.ssbo(Integer.parseInt(this.defines.get(define)), binding, 0);
+    }
+
+    public AutoBindingShader ssbo(int index, GlBuffer buffer, long offset) {
+        this.bindings.add(new BufferBinding(GL_SHADER_STORAGE_BUFFER, index, buffer, offset, -1));
+        return this;
+    }
+
+    public AutoBindingShader ubo(int index, GlBuffer buffer) {
+        return this.ubo(index, buffer, 0);
+    }
+
+    public AutoBindingShader ubo(int index, GlBuffer buffer, long offset) {
+        this.bindings.add(new BufferBinding(GL_UNIFORM_BUFFER, index, buffer, offset, -1));
+        return this;
+    }
+
+    @Override
+    public void bind() {
+        super.bind();
+        for (var binding : this.bindings) {
+            if (binding.offset == 0 && binding.size == -1) {
+                glBindBufferBase(binding.target, binding.index, binding.buffer.id);
+            } else {
+                glBindBufferRange(binding.target, binding.index, binding.buffer.id, binding.offset, binding.size);
+            }
+        }
+    }
+}

src/main/java/me/cortex/voxy/client/core/gl/shader/Shader.java

@@ -11,25 +11,10 @@ import static org.lwjgl.opengl.GL20.glUseProgram;
 
 public class Shader extends TrackedObject {
     private final int id;
-    private Shader(int program) {
+    Shader(int program) {
         id = program;
     }
 
-    public static Builder make(IShaderProcessor... processors) {
-        List<IShaderProcessor> aa = new ArrayList<>(List.of(processors));
-        Collections.reverse(aa);
-        IShaderProcessor applicator = (type,source)->source;
-        for (IShaderProcessor processor : processors) {
-            IShaderProcessor finalApplicator = applicator;
-            applicator = (type, source) -> finalApplicator.process(type, processor.process(type, source));
-        }
-        return new Builder(applicator);
-    }
-
-    public static Builder make() {
-        return new Builder((aa,source)->source);
-    }
-
     public int id() {
         return this.id;
     }
@@ -43,43 +28,73 @@ public class Shader extends TrackedObject {
         glDeleteProgram(this.id);
     }
 
-    public static class Builder {
+
-        private final Map<String, String> defines = new HashMap<>();
+
+
+    public static Builder<Shader> make(IShaderProcessor... processor) {
+        return makeInternal((a,b)->new Shader(b), processor);
+    }
+
+    public static Builder<AutoBindingShader> makeAuto(IShaderProcessor... processor) {
+        return makeInternal(AutoBindingShader::new, processor);
+    }
+
+
+
+    static <T extends Shader> Builder<T> makeInternal(Builder.IShaderObjectConstructor<T> constructor, IShaderProcessor[] processors) {
+        List<IShaderProcessor> aa = new ArrayList<>(List.of(processors));
+        Collections.reverse(aa);
+        IShaderProcessor applicator = (type,source)->source;
+        for (IShaderProcessor processor : processors) {
+            IShaderProcessor finalApplicator = applicator;
+            applicator = (type, source) -> finalApplicator.process(type, processor.process(type, source));
+        }
+        return new Builder<>(constructor, applicator);
+    }
+
+    public static class Builder <T extends Shader> {
+        protected interface IShaderObjectConstructor <J extends Shader> {
+            J make(Builder<J> builder, int program);
+        }
+        final Map<String, String> defines = new HashMap<>();
         private final Map<ShaderType, String> sources = new HashMap<>();
         private final IShaderProcessor processor;
-        private Builder(IShaderProcessor processor) {
+        private final IShaderObjectConstructor<T> constructor;
+        private Builder(IShaderObjectConstructor<T> constructor, IShaderProcessor processor) {
+            this.constructor = constructor;
             this.processor = processor;
         }
 
-        public Builder define(String name) {
+        public Builder<T> define(String name) {
             this.defines.put(name, "");
             return this;
         }
 
         //Useful for inline setting (such as debug)
-        public Builder defineIf(String name, boolean condition) {
+        public Builder<T> defineIf(String name, boolean condition) {
             if (condition) {
                 this.defines.put(name, "");
             }
             return this;
         }
 
-        public Builder define(String name, int value) {
+        public Builder<T> define(String name, int value) {
             this.defines.put(name, Integer.toString(value));
             return this;
         }
 
-        public Builder add(ShaderType type, String id) {
+        public Builder<T> add(ShaderType type, String id) {
             this.addSource(type, ShaderLoader.parse(id));
             return this;
         }
 
-        public Builder addSource(ShaderType type, String source) {
+        public Builder<T> addSource(ShaderType type, String source) {
             this.sources.put(type, this.processor.process(type, source));
             return this;
         }
 
-        public Shader compile() {
+
+        private int compileToProgram() {
             int program = GL20C.glCreateProgram();
             int[] shaders = new int[this.sources.size()];
             {
@@ -107,9 +122,12 @@ public class Shader extends TrackedObject {
             }
             printProgramLinkLog(program);
             verifyProgramLinked(program);
-            return new Shader(program);
+            return program;
         }
 
+        public T compile() {
+            return this.constructor.make(this, this.compileToProgram());
+        }
 
         private static void printProgramLinkLog(int program) {
             String log = GL20C.glGetProgramInfoLog(program);
@@ -148,5 +166,4 @@ public class Shader extends TrackedObject {
             return shader;
         }
     }
-
 }

src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java

@@ -11,6 +11,7 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.gl.GlTexture;
 import me.cortex.voxy.client.core.rendering.util.RawDownloadStream;
 import me.cortex.voxy.client.core.rendering.util.UploadStream;
+import me.cortex.voxy.common.Logger;
 import me.cortex.voxy.common.world.other.Mapper;
 import net.minecraft.block.BlockState;
 import net.minecraft.block.FluidBlock;
@@ -311,7 +312,8 @@ public class ModelFactory {
 
             if (allFalse == allTrue) {//If only some sides where self culled then abort
                 cullsSame = false;
-                if (LOGGED_SELF_CULLING_WARNING.add(blockState)) System.err.println("Warning! blockstate: " + blockState + " only culled against its self some of the time");
+                if (LOGGED_SELF_CULLING_WARNING.add(blockState))
+                    Logger.info("Warning! blockstate: " + blockState + " only culled against its self some of the time");
             }
 
             if (allTrue) {

src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java

@@ -1,8 +1,16 @@
 package me.cortex.voxy.client.core.rendering.hierachical2;
 
+import it.unimi.dsi.fastutil.ints.IntArrayFIFOQueue;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
 import me.cortex.voxy.client.core.gl.GlBuffer;
+import me.cortex.voxy.client.core.gl.shader.AutoBindingShader;
 import me.cortex.voxy.client.core.gl.shader.Shader;
 import me.cortex.voxy.client.core.gl.shader.ShaderType;
+import me.cortex.voxy.client.core.rendering.util.UploadStream;
+import org.lwjgl.system.MemoryUtil;
+
+import static org.lwjgl.opengl.GL20.glUniform1i;
+import static org.lwjgl.opengl.GL43C.glDispatchCompute;
 
 //Uses compute shaders to compute the last 256 rendered section (64x64 workgroup size maybe)
 // done via warp level sort, then workgroup sort (shared memory), (/w sorting network)
@@ -14,6 +22,8 @@ public class NodeCleaner {
 
     private static final int OUTPUT_COUNT = 64;
 
+    private static final int BATCH_SET_SIZE = 2048;
+
     private final Shader sorter = Shader.make()
             .define("OUTPUT_SIZE", OUTPUT_COUNT)
             .define("VISIBILITY_BUFFER_BINDING", 1)
@@ -21,8 +31,17 @@ public class NodeCleaner {
             .add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/sort_visibility.comp")
             .compile();
 
+    private final AutoBindingShader batchClear = Shader.makeAuto()
+            .define("VISIBILITY_BUFFER_BINDING", 0)
+            .define("LIST_BUFFER_BINDING", 1)
+            .add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/batch_visibility_set.com")
+            .compile();
+
     private final GlBuffer visibilityBuffer;
     private final GlBuffer outputBuffer = new GlBuffer(OUTPUT_COUNT*4);
+    private final GlBuffer scratchBuffer = new GlBuffer(BATCH_SET_SIZE*4);//Scratch buffer for setting ids with
+
+    private final IntArrayFIFOQueue idsToClear = new IntArrayFIFOQueue();
 
     private final NodeManager2 nodeManager;
     int visibilityId = 0;
@@ -30,16 +49,42 @@ public class NodeCleaner {
 
     public NodeCleaner(NodeManager2 nodeManager) {
         this.nodeManager = nodeManager;
-        this.visibilityBuffer = new GlBuffer(nodeManager.maxNodeCount*4L);
+        this.visibilityBuffer = new GlBuffer(nodeManager.maxNodeCount*4L).zero();
+
+        this.batchClear
+                .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer)
+                .ssbo("LIST_BUFFER_BINDING", this.scratchBuffer);
+    }
+
+    public void clearId(int id) {
+        this.idsToClear.enqueue(id);
     }
 
     public void tick() {
+        this.clearIds();
+    }
 
+    private void clearIds() {
+        if (!this.idsToClear.isEmpty()) {
+            this.batchClear.bind();
+
+            while (!this.idsToClear.isEmpty()) {
+                int cnt = Math.min(this.idsToClear.size(), BATCH_SET_SIZE);
+                long ptr = UploadStream.INSTANCE.upload(this.scratchBuffer, 0, cnt * 4L);
+                for (int i = 0; i < cnt; i++) {
+                    MemoryUtil.memPutInt(ptr + cnt * 4, this.idsToClear.dequeueInt());
+                }
+                UploadStream.INSTANCE.commit();
+                glUniform1i(0, cnt);
+                glDispatchCompute((cnt+127)/128, 1, 1);
+            }
+        }
     }
 
     public void free() {
         this.sorter.free();
         this.visibilityBuffer.free();
         this.outputBuffer.free();
+        this.scratchBuffer.free();
     }
 }

src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java

@@ -326,8 +326,10 @@ public class NodeManager2 {
         if (parentNodeId == -1 || (parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_REQUEST) {
             throw new IllegalStateException("CRITICAL BAD STATE!!! finishRequest tried to finish for a node that no longer exists in the map or has become a request type somehow?!!?!!" + WorldEngine.pprintPos(request.getPosition()) + " " + parentNodeId);
         }
+        int parentNodeType = parentNodeId&NODE_TYPE_MSK;
+        parentNodeId &= NODE_ID_MSK;
 
-        if ((parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_LEAF) {
+        if (parentNodeType==NODE_TYPE_LEAF) {
             int msk = Byte.toUnsignedInt(request.getMsk());
             int base = this.nodeData.allocate(Integer.bitCount(msk));
             int offset = -1;
@@ -363,9 +365,20 @@ public class NodeManager2 {
             this.nodeData.setNodeRequest(parentNodeId, 0);//TODO: create a better null request
             this.activeNodeRequestCount--;
             this.nodeData.unmarkRequestInFlight(parentNodeId);
+
+            //Change it from a leaf to an inner node
+            {
+                int pid = this.activeSectionMap.remove(request.getPosition());
+                if (pid == -1 || (pid & NODE_TYPE_MSK) != NODE_TYPE_LEAF) {
+                    throw new IllegalStateException("Unexpected node mapping: " + pid);
+                }
+            }
+            //_this is why it hasnt been working, grrr, wasnt doing this_
+            this.activeSectionMap.put(request.getPosition(), NODE_TYPE_INNER|parentNodeId);//Set the type from leaf to inner node
+
             this.invalidateNode(parentNodeId);
-        } else if ((parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_INNER) {
+        } else if (parentNodeType==NODE_TYPE_INNER) {
-            System.err.println("TODO: FIXME FINISH: finishRequest NODE_TYPE_INNER");
+            Logger.error("TODO: FIXME FINISH: finishRequest NODE_TYPE_INNER");
         } else {
             throw new IllegalStateException();
         }