Continue work on adding node cleaner support

src/main/java/me/cortex/voxy/client/core/rendering/ChunkBoundRenderer.java

@@ -195,6 +195,9 @@ public class ChunkBoundRenderer {
 
     private void ensureSize1() {
         if (this.chunk2idx.size() < this.idx2chunk.length) return;
+        //Commit any copies, ensures is synced to new buffer
+        UploadStream.INSTANCE.commit();
+
         int size = (int) (this.idx2chunk.length*1.5);
         Logger.info("Resizing chunk position buffer to: " + size);
         //Need to resize

src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java

@@ -143,7 +143,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, Q>, J extends Vi
             //Tick download stream
             DownloadStream.INSTANCE.tick();
 
-            this.nodeManager.tick(this.traversal.getNodeBuffer());
+            this.nodeManager.tick(this.traversal.getNodeBuffer(), this.nodeCleaner);
             //glFlush();
 
             this.nodeCleaner.tick(this.traversal.getNodeBuffer());//Probably do this here??

src/main/java/me/cortex/voxy/client/core/rendering/hierachical/AsyncNodeManager.java

@@ -75,6 +75,9 @@ public class AsyncNodeManager {
 
     //locals for during iteration
     private final IntOpenHashSet tlnIdChange = new IntOpenHashSet();//"Encoded" add/remove id, first bit indicates if its add or remove, 1 is add
+    //Top bit indicates clear or reset
+    private final IntOpenHashSet cleanerIdResetClear = new IntOpenHashSet();//Tells the cleaner if it needs to clear the id to 0, or reset the id to the current frame
+
     private boolean needsWaitForSync = false;
 
     public AsyncNodeManager(int maxNodeCount, ISectionWatcher watcher, IGeometryData geometryData) {
@@ -98,20 +101,23 @@ public class AsyncNodeManager {
 
         this.geometryManager = new BasicAsyncGeometryManager(((BasicSectionGeometryData)geometryData).getMaxSectionCount(), ((BasicSectionGeometryData)geometryData).getGeometryCapacity());
         this.manager = new NodeManager(maxNodeCount, this.geometryManager, watcher);
+        //Dont do the move... is just to much effort
         this.manager.setClear(new NodeManager.ICleaner() {
             @Override
             public void alloc(int id) {
-
+                AsyncNodeManager.this.cleanerIdResetClear.remove(id);//Remove clear
+                AsyncNodeManager.this.cleanerIdResetClear.add(id|(1<<31));//Add reset
             }
 
             @Override
             public void move(int from, int to) {
-
+                //noop (sorry :( will cause some perf loss/incorrect cleaning )
             }
 
             @Override
             public void free(int id) {
-
+                AsyncNodeManager.this.cleanerIdResetClear.remove(id|(1<<31));//Remove reset
+                AsyncNodeManager.this.cleanerIdResetClear.add(id);//Add clear
             }
         });
         this.manager.setTLNCallbacks(id->{
@@ -353,6 +359,7 @@ public class AsyncNodeManager {
             results.geometryUploads.putAll(this.geometryManager.getUploads());
             this.geometryManager.getUploads().clear();//Put in new data into sync set
             this.geometryManager.getHeapRemovals().clear();//We dont do removals on new data (as there is "none")
+            results.cleanerOperations.addAll(this.cleanerIdResetClear); this.cleanerIdResetClear.clear();
         } else {
             results = prev;
             // merge with the previous result set
@@ -368,6 +375,16 @@ public class AsyncNodeManager {
                 this.tlnIdChange.clear();
             }
 
+            if (!this.cleanerIdResetClear.isEmpty()) {//Merge top level node id changes
+                var iter = this.cleanerIdResetClear.intIterator();
+                while (iter.hasNext()) {
+                    int val = iter.nextInt();
+                    results.cleanerOperations.remove(val^(1<<31));//Remove opposite
+                    results.cleanerOperations.add(val);//Add this
+                }
+                this.cleanerIdResetClear.clear();
+            }
+
             if (!this.geometryManager.getHeapRemovals().isEmpty()) {//Remove and free all the removed geometry uploads
                 var rem = this.geometryManager.getHeapRemovals();
                 var iter = rem.intIterator();
@@ -440,7 +457,7 @@ public class AsyncNodeManager {
 
     private IntConsumer tlnAddCallback; private IntConsumer tlnRemoveCallback;
     //Render thread synchronization
-    public void tick(GlBuffer nodeBuffer) {//TODO: dont pass nodeBuffer here??, do something else thats better
+    public void tick(GlBuffer nodeBuffer, NodeCleaner cleaner) {//TODO: dont pass nodeBuffer here??, do something else thats better
         var results = (SyncResults)RESULT_HANDLE.getAndSet(this, null);//Acquire the results
         if (results == null) {//There are no new results to process, return
             return;
@@ -498,6 +515,12 @@ public class AsyncNodeManager {
             glMemoryBarrier(GL_UNIFORM_BARRIER_BIT|GL_SHADER_STORAGE_BARRIER_BIT);
         }
 
+        if (!results.cleanerOperations.isEmpty()) {
+            cleaner.updateIds(results.cleanerOperations);
+        }
+
+        this.currentMaxNodeId = results.currentMaxNodeId;
+
         //Insert the result set into the cache
         if (!RESULT_CACHE_1_HANDLE.compareAndSet(this, null, results)) {
             //Failed to insert into result set 1, insert it into result set 2
@@ -513,6 +536,11 @@ public class AsyncNodeManager {
         this.tlnRemoveCallback = remove;
     }
 
+    private int currentMaxNodeId = 0;
+    public int getCurrentMaxNodeId() {
+        return this.currentMaxNodeId;
+    }
+
     //==================================================================================================================
     //Incoming events
 
@@ -679,11 +707,15 @@ public class AsyncNodeManager {
         private MemoryBuffer scatterWriteBuffer = new MemoryBuffer(8192*2);
         private final Int2IntOpenHashMap scatterWriteLocationMap = new Int2IntOpenHashMap(1024);
 
+        //Cleaner operations
+        private final IntOpenHashSet cleanerOperations = new IntOpenHashSet();
+
         public SyncResults() {
             this.scatterWriteLocationMap.defaultReturnValue(-1);
         }
 
         public void reset() {
+            this.cleanerOperations.clear();
             this.scatterWriteLocationMap.clear();
             this.currentMaxNodeId = 0;
             this.tlnDelta.clear();

src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java

@@ -32,9 +32,6 @@ public class NodeCleaner {
     static final int OUTPUT_COUNT = 256;
 
 
-    private static final int BATCH_SET_SIZE = 2048;
-
-
     private final AutoBindingShader sorter = Shader.makeAuto(PrintfDebugUtil.PRINTF_processor)
             .define("WORK_SIZE", SORTING_WORKER_SIZE)
             .define("ELEMS_PER_THREAD", WORK_PER_THREAD)
@@ -63,10 +60,6 @@ public class NodeCleaner {
 
     final GlBuffer visibilityBuffer;
     private final GlBuffer outputBuffer = new GlBuffer(OUTPUT_COUNT*4+OUTPUT_COUNT*8);//Scratch + output
-    private final GlBuffer scratchBuffer = new GlBuffer(BATCH_SET_SIZE*4);//Scratch buffer for setting ids with
-
-    private final IntOpenHashSet allocIds = new IntOpenHashSet();
-    private final IntOpenHashSet freeIds = new IntOpenHashSet();
 
     private final AsyncNodeManager nodeManager;
     int visibilityId = 0;
@@ -78,8 +71,7 @@ public class NodeCleaner {
         this.visibilityBuffer.fill(-1);
 
         this.batchClear
-                .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer)
+                .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer);
-                .ssbo("LIST_BUFFER_BINDING", this.scratchBuffer);
 
         this.sorter
                 .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer)
@@ -111,10 +103,6 @@ public class NodeCleaner {
 
     public void tick(GlBuffer nodeDataBuffer) {
         this.visibilityId++;
-
-        this.setIds(this.allocIds, this.visibilityId);
-        this.setIds(this.freeIds, -1);
-
         if (this.shouldCleanGeometry()) {
             this.outputBuffer.fill(this.nodeManager.maxNodeCount - 2);//TODO: maybe dont set to zero??
 
@@ -124,7 +112,7 @@ public class NodeCleaner {
             //TODO: choose whether this is in nodeSpace or section/geometryId space
             //
             glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
-            //glDispatchCompute((this.nodeManager.getCurrentMaxNodeId() + (SORTING_WORKER_SIZE+WORK_PER_THREAD) - 1) / (SORTING_WORKER_SIZE+WORK_PER_THREAD), 1, 1);
+            glDispatchCompute((this.nodeManager.getCurrentMaxNodeId() + (SORTING_WORKER_SIZE+WORK_PER_THREAD) - 1) / (SORTING_WORKER_SIZE+WORK_PER_THREAD), 1, 1);
 
             this.resultTransformer.bind();
             glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 0, this.outputBuffer.id, 0, 4 * OUTPUT_COUNT);
@@ -151,22 +139,26 @@ public class NodeCleaner {
         return false;//return 3<((double)this.nodeManager.getGeometryManager().getUsedCapacity())/((double)this.nodeManager.getGeometryManager().getRemainingCapacity());
     }
 
-    private void setIds(IntOpenHashSet collection, int setTo) {
+    public void updateIds(IntOpenHashSet collection) {
         if (!collection.isEmpty()) {
-            this.batchClear.bind();
+            int count = collection.size();
+            long addr = UploadStream.INSTANCE.rawUploadAddress(count * 4 + 16);//TODO ensure alignment, create method todo alignment things
+            addr = (addr+15)&~15L;//Align to 16 bytes
+
+            long ptr = UploadStream.INSTANCE.getBaseAddress() + addr;
             var iter = collection.iterator();
             while (iter.hasNext()) {
-                int cnt = Math.min(collection.size(), BATCH_SET_SIZE);
+                MemoryUtil.memPutInt(ptr, iter.nextInt()); ptr+=4;
-                long ptr = UploadStream.INSTANCE.upload(this.scratchBuffer, 0, cnt * 4L);
-                for (int i = 0; i < cnt; i++) {
-                    MemoryUtil.memPutInt(ptr + i * 4, iter.nextInt());
-                    iter.remove();
-                }
-                UploadStream.INSTANCE.commit();
-                glUniform1ui(0, cnt);
-                glUniform1ui(1, setTo);
-                glDispatchCompute((cnt+127)/128, 1, 1);
             }
+            UploadStream.INSTANCE.commit();
+
+            this.batchClear.bind();
+            glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, UploadStream.INSTANCE.getRawBufferId(), addr, count*4L);
+            glUniform1ui(0, count);
+            glUniform1ui(1, this.visibilityId);
+            glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
+            glDispatchCompute((count+127)/128, 1, 1);
+            glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
         }
     }
 
@@ -174,7 +166,6 @@ public class NodeCleaner {
         this.sorter.free();
         this.visibilityBuffer.free();
         this.outputBuffer.free();
-        this.scratchBuffer.free();
         this.batchClear.free();
         this.resultTransformer.free();
     }

src/main/resources/assets/voxy/shaders/lod/hierarchical/cleaner/batch_visibility_set.comp

@@ -17,5 +17,6 @@ void main() {
     if (count <= id) {
         return;
     }
-    visiblity[ids[id]] = setTo;
+    uint pos = ids[id];
+    visiblity[pos&((1u<<31)-1)] = mix(setTo, uint(-1), (pos&(1u<<31)) == 0);
 }