Continued work on geometry removal

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -7,6 +7,7 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
 import me.cortex.voxy.client.core.rendering.*;
 import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4;
+import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
 import me.cortex.voxy.client.core.rendering.post.PostProcessing;
 import me.cortex.voxy.client.core.rendering.util.DownloadStream;
 import me.cortex.voxy.client.core.util.IrisUtil;
@@ -35,6 +36,7 @@ import org.lwjgl.opengl.GL11;
 
 import java.io.File;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.lwjgl.opengl.GL30C.*;
 
@@ -91,6 +93,7 @@ public class VoxelCore {
         //this.testMeshingPerformance();
 
         //this.testDbPerformance();
+        //this.testFullMesh();
     }
 
     public void enqueueIngest(WorldChunk worldChunk) {
@@ -266,6 +269,10 @@ public class VoxelCore {
         return this.world;
     }
 
+
+
+
+
     private void verifyTopNodeChildren(int X, int Y, int Z) {
         for (int lvl = 0; lvl < 5; lvl++) {
             for (int y = (Y<<5)>>lvl; y < ((Y+1)<<5)>>lvl; y++) {
@@ -378,4 +385,70 @@ public class VoxelCore {
         System.out.println("Total "+delta+"ms " + ((double)delta/c) + "ms average" );
     }
 
+
+
+    private void testFullMesh() {
+        var modelService = new ModelBakerySubsystem(this.world.getMapper());
+        var completedCounter = new AtomicInteger();
+        var generationService = new RenderGenerationService(this.world, modelService, this.serviceThreadPool, a-> {completedCounter.incrementAndGet(); a.free();}, false);
+
+
+        var r = new Random(12345);
+        {
+            for (int i = 0; i < 10_000; i++) {
+                int x = (r.nextInt(256*2+2)-256)>>1;//-32
+                int z = (r.nextInt(256*2+2)-256)>>1;//-32
+                int y = r.nextInt(10)-2;
+                int lvl = 0;//r.nextInt(5);
+                long key = WorldEngine.getWorldSectionId(lvl, x>>lvl, y>>lvl, z>>lvl);
+                generationService.enqueueTask(key);
+            }
+            int i = 0;
+            while (true) {
+                modelService.tick();
+                if (i++%5000==0)
+                    System.out.println(completedCounter.get());
+                glFinish();
+                List<String> a = new ArrayList<>();
+                generationService.addDebugData(a);
+                if (a.getFirst().endsWith(" 0")) {
+                    break;
+                }
+            }
+        }
+
+        System.out.println("Running benchmark");
+        while (true)
+        {
+            completedCounter.set(0);
+            long start = System.currentTimeMillis();
+            int C = 200_000;
+            for (int i = 0; i < C; i++) {
+                int x = (r.nextInt(256 * 2 + 2) - 256) >> 1;//-32
+                int z = (r.nextInt(256 * 2 + 2) - 256) >> 1;//-32
+                int y = r.nextInt(10) - 2;
+                int lvl = 0;//r.nextInt(5);
+                long key = WorldEngine.getWorldSectionId(lvl, x >> lvl, y >> lvl, z >> lvl);
+                generationService.enqueueTask(key);
+            }
+            //int i = 0;
+            while (true) {
+                //if (i++%5000==0)
+                //    System.out.println(completedCounter.get());
+                modelService.tick();
+                glFinish();
+                List<String> a = new ArrayList<>();
+                generationService.addDebugData(a);
+                if (a.getFirst().endsWith(" 0")) {
+                    break;
+                }
+            }
+            long delta = (System.currentTimeMillis()-start);
+            System.out.println("Time "+delta+"ms count: " + completedCounter.get() + " avg per mesh: " + ((double)delta/completedCounter.get()));
+            if (false)
+                break;
+        }
+        generationService.shutdown();
+        modelService.shutdown();
+    }
 }

src/main/java/me/cortex/voxy/client/core/gl/GlBuffer.java

@@ -1,6 +1,7 @@
 package me.cortex.voxy.client.core.gl;
 
 import me.cortex.voxy.common.util.TrackedObject;
+import org.lwjgl.opengl.GL11;
 
 import static org.lwjgl.opengl.GL11.GL_UNSIGNED_BYTE;
 import static org.lwjgl.opengl.GL15.glDeleteBuffers;
@@ -46,6 +47,16 @@ public class GlBuffer extends TrackedObject {
         return this;
     }
 
+    public GlBuffer fill(int data) {
+        //Clear unpack values
+        //Fixed in mesa commit a5c3c452
+        glPixelStorei(GL11.GL_UNPACK_SKIP_ROWS, 0);
+        glPixelStorei(GL11.GL_UNPACK_SKIP_PIXELS, 0);
+
+        glClearNamedBufferData(this.id, GL_R8UI, GL_RED_INTEGER, GL_UNSIGNED_BYTE, new int[]{data});
+        return this;
+    }
+
     public static int getCount() {
         return COUNT;
     }

src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java

@@ -49,7 +49,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
 
         //Max sections: ~500k
         //Max geometry: 1 gb
-        this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<20, (1L<<31)-1024);
+        this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<20, (1L<<32)-1024);
 
         //Do something incredibly hacky, we dont need to keep the reference to this around, so just connect and discard
         var router = new SectionUpdateRouter();
@@ -149,7 +149,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
 
     private int q = -60;
     public void setup(Camera camera) {
-        final int W = 32;
+        final int W = 3;
         final int H = 2;
         boolean SIDED = false;
         for (int i = 0; i<64 && q<((W*2+1)*(W*2+1)*H)&&q++>=0;i++) {

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory4.java

@@ -233,7 +233,7 @@ public class RenderDataFactory4 {
                 this.blockMesher.finish();
             }
 
-            {
+            if (true) {
                 this.blockMesher.doAuxiliaryFaceOffset = false;
                 //Hacky generate section side faces (without check neighbor section)
                 for (int side = 0; side < 2; side++) {
@@ -411,7 +411,7 @@ public class RenderDataFactory4 {
         }
 
         //Generate the side faces, hackily, using 0 and 1 mesher
-        {
+        if (true) {
             var ma = this.xAxisMeshers[0];
             var mb = this.xAxisMeshers[31];
             ma.finish();

src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java

@@ -5,6 +5,7 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.gl.shader.AutoBindingShader;
 import me.cortex.voxy.client.core.gl.shader.Shader;
 import me.cortex.voxy.client.core.gl.shader.ShaderType;
+import me.cortex.voxy.client.core.rendering.util.DownloadStream;
 import me.cortex.voxy.client.core.rendering.util.UploadStream;
 import org.lwjgl.system.MemoryUtil;
 
@@ -26,7 +27,7 @@ public class NodeCleaner {
 
     private static final int BATCH_SET_SIZE = 2048;
 
-    private final Shader sorter = Shader.make()
+    private final AutoBindingShader sorter = Shader.makeAuto()
             .define("OUTPUT_SIZE", OUTPUT_COUNT)
             .define("VISIBILITY_BUFFER_BINDING", 1)
             .define("OUTPUT_BUFFER_BINDING", 2)
@@ -56,6 +57,10 @@ public class NodeCleaner {
         this.batchClear
                 .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer)
                 .ssbo("LIST_BUFFER_BINDING", this.scratchBuffer);
+
+        this.sorter
+                .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer)
+                .ssbo("OUTPUT_BUFFER_BINDING", this.outputBuffer);
     }
 
     public void clearId(int id) {
@@ -63,6 +68,7 @@ public class NodeCleaner {
     }
 
     public void tick() {
+        this.visibilityId++;
         this.clearIds();
 
         if (false) {
@@ -70,14 +76,19 @@ public class NodeCleaner {
 
             this.sorter.bind();
             //TODO: choose whether this is in nodeSpace or section/geometryId space
-            //glDispatchCompute(this.nodeManager.getCurrentMaxNodeId()/, 1, 1);
+            //this.nodeManager.getCurrentMaxNodeId()
+            glDispatchCompute((200_000+127)/128, 1, 1);
 
-            //DownloadStream.INSTANCE.download(this.outputBuffer, this::onDownload);
+            DownloadStream.INSTANCE.download(this.outputBuffer, this::onDownload);
         }
     }
 
     private void onDownload(long ptr, long size) {
-
+        StringBuilder b = new StringBuilder();
+        for (int i = 0; i < 64; i++) {
+            b.append(", ").append(MemoryUtil.memGetInt(ptr + 4 * i));
+        }
+        System.out.println(b);
     }
 
     private void clearIds() {

src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeStore.java

@@ -272,7 +272,7 @@ public final class NodeStore {
         MemoryUtil.memPutInt(ptr, w); ptr += 4;
     }
 
-    //public int getEndNodeId() {
+    public int getEndNodeId() {
-
+        return this.allocationSet.getMaxIndex();
-    //}
+    }
 }

src/main/java/me/cortex/voxy/common/util/HierarchicalBitSet.java

@@ -161,6 +161,10 @@ public class HierarchicalBitSet {
     }
 
 
+    public int getMaxIndex() {
+        throw new IllegalStateException();
+    }
+
 
     public static void main(String[] args)  {
         var h = new HierarchicalBitSet(1<<19);

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -40,7 +40,7 @@ public class WorldEngine {
         this.storage = storageBackend;
         this.mapper = new Mapper(this.storage);
         //4 cache size bits means that the section tracker has 16 separate maps that it uses
-        this.sectionTracker = new ActiveSectionTracker(3, this::unsafeLoadSection);
+        this.sectionTracker = new ActiveSectionTracker(4, this::unsafeLoadSection);
 
         this.savingService = new SectionSavingService(this, serviceThreadPool);
         this.ingestService  = new VoxelIngestService(this, serviceThreadPool);

src/main/resources/assets/voxy/shaders/lod/hierarchical/cleaner/sort_visibility.comp

@@ -4,7 +4,7 @@
 
 //#define OUTPUT_SIZE 128
 
-layout(local_size_x=32, local_size_y=8) in;
+layout(local_size_x=128, local_size_y=1) in;
 //256 workgroup
 
 
@@ -16,6 +16,7 @@ layout(binding = OUTPUT_BUFFER_BINDING, std430) restrict volatile buffer Minimum
     uint minVisIds[OUTPUT_SIZE];
 };
 
+//Returns the id of the max value
 uint atomicDerefMin(uint atId, uint id, uint value) {
     uint existingId = minVisIds[atId];
     while (true) {
@@ -26,9 +27,10 @@ uint atomicDerefMin(uint atId, uint id, uint value) {
         //Attempt to swap, since we know we are less than the existingId
         atomicCompSwap(minVisIds[atId], existingId, id);
         //Check if we did swap, else if we failed (or got reswapped else where) recheck
+        uint pExistingId = existingId;
         existingId = minVisIds[atId];
         if (existingId == id) {
-            return existingId;
+            return pExistingId;
         }
     }
 }
@@ -37,9 +39,6 @@ uint atomicDerefMin(uint atId, uint id, uint value) {
 void bubbleSort(uint start, uint id, uint value) {
     for (uint i = start; i < OUTPUT_SIZE; i++) {
         uint nextId = atomicDerefMin(i, id, value);
-        if (nextId == id) {
-            return;//Not inserted, so return
-        }
         //Else we need to bubble the value up
         id = nextId;
         value = visiblity[id];
@@ -47,5 +46,9 @@ void bubbleSort(uint start, uint id, uint value) {
 }
 
 void main() {
+    //if (gl_GlobalInvocationID.x <64) {
+    //    minVisIds[gl_GlobalInvocationID.x] = visiblity[gl_GlobalInvocationID.x];
+    //}
     //First do a min sort/set of min OUTPUT_SIZE values of the set
+    bubbleSort(0, gl_GlobalInvocationID.x, visiblity[gl_GlobalInvocationID.x]);
 }