Changed core

src/main/java/me/cortex/voxy/client/core/DistanceTracker.java

@@ -14,39 +14,38 @@ import net.minecraft.client.MinecraftClient;
 // make the rebuild range like +-5 chunks along each axis (that means at higher levels, should only need to rebuild like)
 // 4 sections or something
 public class DistanceTracker {
-    private final TransitionRing2D[] rings;
+    private final TransitionRing2D[] loDRings;
+    private final TransitionRing2D[] cacheLoadRings;
+    private final TransitionRing2D[] cacheUnloadRings;
     private final RenderTracker tracker;
-    private final int scale;
     private final int minYSection;
     private final int maxYSection;
 
-    public DistanceTracker(RenderTracker tracker, int rings, int scale) {
+    public DistanceTracker(RenderTracker tracker, int[] lodRingScales, int cacheLoadDistance, int cacheUnloadDistance) {
-        this.rings = new TransitionRing2D[rings];
+        this.loDRings = new TransitionRing2D[lodRingScales.length];
+        this.cacheLoadRings = new TransitionRing2D[lodRingScales.length];
+        this.cacheUnloadRings = new TransitionRing2D[lodRingScales.length];
         this.tracker = tracker;
-        this.scale = scale;
-
         this.minYSection = MinecraftClient.getInstance().world.getBottomSectionCoord()/2;
         this.maxYSection = MinecraftClient.getInstance().world.getTopSectionCoord()/2;
 
-        int radius = (MinecraftClient.getInstance().options.getViewDistance().getValue() / 2) - 4;
-        if (radius > 0 && false) {
-            this.rings[0] = new TransitionRing2D(5, radius, (x, z) -> {
-                for (int y = this.minYSection; y <= this.maxYSection; y++) {
-                    this.tracker.remLvl0(x, y, z);
-                }
-            }, (x, z) -> {
-                for (int y = this.minYSection; y <= this.maxYSection; y++) {
-                    this.tracker.addLvl0(x, y, z);
-                }
-            });
-        }
 
         //The rings 0+ start at 64 vanilla rd, no matter what the game is set at, that is if the game is set to 32 rd
         // there will still be 32 chunks untill the first lod drop
         // if the game is set to 16, then there will be 48 chunks until the drop
-        for (int i = 1; i < rings; i++) {
+        for (int i = 0; i < this.loDRings.length; i++) {
             int capRing = i;
-            this.rings[i] = new TransitionRing2D(5+i, scale, (x, z) -> this.dec(capRing, x, z), (x, z) -> this.inc(capRing, x, z));
+            this.loDRings[i] = new TransitionRing2D(6+i, lodRingScales[i], (x, z) -> this.dec(capRing+1, x, z), (x, z) -> this.inc(capRing+1, x, z));
+
+            //TODO:FIXME i think the radius is wrong and (lodRingScales[i]) needs to be (lodRingScales[i]<<1) since the transition ring (the thing above)
+            // acts on LoD level + 1
+            this.cacheLoadRings[i] = new TransitionRing2D(5+i, lodRingScales[i] + cacheLoadDistance, (x, z) -> {
+                //When entering a cache ring, trigger a mesh op and inject into cache
+
+            }, (x, z) -> {});
+            this.cacheUnloadRings[i] = new TransitionRing2D(5+i, lodRingScales[i] + cacheUnloadDistance, (x, z) -> {}, (x, z) -> {
+                //When exiting the cache unload ring, tell the cache to dump whatever mesh it has cached and not add any mesh from that position
+            });
         }
     }
 
@@ -69,10 +68,14 @@ public class DistanceTracker {
     //if the center suddenly changes (say more than 1<<(7+lodlvl) block) then invalidate the entire ring and recompute
     // the lod sections
     public void setCenter(int x, int y, int z) {
-        for (var ring : this.rings) {
+        for (var ring : this.cacheLoadRings) {
-            if (ring != null) {
             ring.update(x, z);
         }
+        for (var ring : this.loDRings) {
+            ring.update(x, z);
+        }
+        for (var ring : this.cacheUnloadRings) {
+            ring.update(x, z);
         }
     }
 
@@ -82,14 +85,18 @@ public class DistanceTracker {
         //Insert highest LOD level
         for (int ox = -SIZE; ox <= SIZE; ox++) {
             for (int oz = -SIZE; oz <= SIZE; oz++) {
-                this.inc(4, (x>>(5+this.rings.length-1)) + ox, (z>>(5+this.rings.length-1)) + oz);
+                this.inc(4, (x>>(5+this.loDRings.length)) + ox, (z>>(5+this.loDRings.length)) + oz);
             }
         }
 
 
-        for (int i = this.rings.length-1; 0 <= i; i--) {
+        for (var ring : this.cacheLoadRings) {
-            if (this.rings[i] != null) {
+            ring.fill(x, z);
-                this.rings[i].fill(x, z);
+        }
+
+        for (int i = this.loDRings.length-1; 0 <= i; i--) {
+            if (this.loDRings[i] != null) {
+                this.loDRings[i].fill(x, z);
             }
         }
     }

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -63,7 +63,8 @@ public class VoxelCore {
         System.out.println("Render tracker and generator initialized");
 
         //To get to chunk scale multiply the scale by 2, the scale is after how many chunks does the lods halve
-        this.distanceTracker = new DistanceTracker(this.renderTracker, 5, VoxyConfig.CONFIG.qualityScale);
+        int q = VoxyConfig.CONFIG.qualityScale;
+        this.distanceTracker = new DistanceTracker(this.renderTracker, new int[]{q,q,q,q}, 2, 4);
         System.out.println("Distance tracker initialized");
 
         this.postProcessing = new PostProcessing();

src/main/java/me/cortex/voxy/client/core/model/ModelManager.java

@@ -490,18 +490,31 @@ public class ModelManager {
         return res;
     }
 
-    //TODO:FIXME: if the model is not already in the cache for some reason it renders black, need to figure out why
+    //TODO:FIXME: DONT DO SPIN LOCKS :WAA:
     public long getModelMetadata(int blockId) {
-        int map = 0;
+        int map = this.idMappings[blockId];
-        while ((map = this.idMappings[blockId]) == -1) {
+        if (map == -1) {
-            Thread.onSpinWait();
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
             }
+            map = this.idMappings[blockId];
+        }
+        if (map == -1) {
+            throw new IllegalArgumentException("Id hasnt been computed yet: " + blockId);
+        }
+        return this.metadataCache[map];
+        //int map = 0;
+        //int i = 10;
+        //while ((map = this.idMappings[blockId]) == -1) {
+        //    Thread.onSpinWait();
+        //}
 
-        long meta = 0;
+        //long meta = 0;
-        while ((meta = this.metadataCache[map]) == 0) {
+        //while ((meta = this.metadataCache[map]) == 0) {
-            Thread.onSpinWait();
+        //    Thread.onSpinWait();
-        }
+        //}
-        return meta;
     }
 
     public int getModelId(int blockId) {

src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java

@@ -6,6 +6,7 @@ package me.cortex.voxy.client.core.rendering;
 import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.model.ModelManager;
 import me.cortex.voxy.client.core.rendering.building.BuiltSection;
+import me.cortex.voxy.client.core.rendering.util.DownloadStream;
 import me.cortex.voxy.client.core.rendering.util.UploadStream;
 import me.cortex.voxy.common.world.other.Mapper;
 import net.minecraft.client.MinecraftClient;
@@ -66,6 +67,7 @@ public abstract class AbstractFarWorldRenderer {
         // once per frame when using multi viewport mods
         //it shouldent matter if its called multiple times a frame however, as its synced with fences
         UploadStream.INSTANCE.tick();
+        DownloadStream.INSTANCE.tick();
 
         //Update the lightmap
         {

src/main/java/me/cortex/voxy/client/core/rendering/RenderTracker.java

@@ -35,7 +35,7 @@ public class RenderTracker {
     //Adds a lvl 0 section into the world renderer
     public void addLvl0(int x, int y, int z) {
         this.activeSections.put(WorldEngine.getWorldSectionId(0, x, y, z), O);
-        this.renderGen.enqueueTask(0, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(0, x, y, z, this::shouldStillBuild);
     }
 
     //Removes a lvl 0 section from the world renderer
@@ -61,7 +61,7 @@ public class RenderTracker {
         // concurrent hashmap or something, this is so that e.g. the build data position
         // can be updated
 
-        this.renderGen.enqueueTask(lvl, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl, x, y, z, this::shouldStillBuild);
 
         this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1))));
         this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1)+1)));
@@ -98,35 +98,35 @@ public class RenderTracker {
         this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl, x, y, z)));
         this.renderGen.removeTask(lvl, x, y, z);
 
-        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1), this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1, this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1), this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1, this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1), this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1, this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1), this::shouldStillBuild);
-        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+        this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1, this::shouldStillBuild);
+    }
+
+    //Enqueues a renderTask for a section to cache the result
+    public void addCache() {
 
     }
 
-    //Updates a sections direction mask (e.g. if the player goes past the axis, the chunk must be updated)
-    public void updateDirMask(int lvl, int x, int y, int z, int newMask) {
-
-    }
 
 
     //Called by the world engine when a section gets dirtied
     public void sectionUpdated(WorldSection section) {
-        if (this.activeSections.containsKey(section.getKey())) {
+        if (this.activeSections.containsKey(section.key)) {
             //TODO:FIXME: if the section gets updated, that means that its neighbors might need to be updated aswell
             // (due to block occlusion)
 
             //TODO: FIXME: REBUILDING THE ENTIRE NEIGHBORS when probably only the internal layout changed is NOT SMART
-            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z, this::shouldStillBuild);
-            this.renderGen.enqueueTask(section.lvl, section.x-1, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+            this.renderGen.enqueueTask(section.lvl, section.x-1, section.y, section.z, this::shouldStillBuild);
-            this.renderGen.enqueueTask(section.lvl, section.x+1, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+            this.renderGen.enqueueTask(section.lvl, section.x+1, section.y, section.z, this::shouldStillBuild);
-            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z-1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z-1, this::shouldStillBuild);
-            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z+1, this::shouldStillBuild, this::getBuildFlagsOrAbort);
+            this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z+1, this::shouldStillBuild);
         }
         //this.renderGen.enqueueTask(section);
     }
@@ -143,32 +143,6 @@ public class RenderTracker {
         }
     }
 
-    public int getBuildFlagsOrAbort(WorldSection section) {
-        var cam = MinecraftClient.getInstance().cameraEntity;
-        if (cam == null) {
-            return 0;
-        }
-        var holder = this.activeSections.get(section.getKey());
-        int buildMask = 0;
-        if (holder != null) {
-            if (section.z<(((int)cam.getPos().z)>>(5+section.lvl))+1) {
-                buildMask |= 1<< Direction.SOUTH.getId();
-            }
-            if (section.z>(((int)cam.getPos().z)>>(5+section.lvl))-1) {
-                buildMask |= 1<<Direction.NORTH.getId();
-            }
-            if (section.x<(((int)cam.getPos().x)>>(5+section.lvl))+1) {
-                buildMask |= 1<<Direction.EAST.getId();
-            }
-            if (section.x>(((int)cam.getPos().x)>>(5+section.lvl))-1) {
-                buildMask |= 1<<Direction.WEST.getId();
-            }
-            buildMask |= 1<<Direction.UP.getId();
-            buildMask |= ((1<<6)-1)^(1);
-        }
-        return buildMask;
-    }
-
     public boolean shouldStillBuild(int lvl, int x, int y, int z) {
         return this.activeSections.containsKey(WorldEngine.getWorldSectionId(lvl, x, y, z));
     }

src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java

@@ -0,0 +1,27 @@
+package me.cortex.voxy.client.core.rendering.building;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+//TODO: instead of storing duplicate render geometry between here and gpu memory
+// when a section is unloaded from the gpu, put it into a download stream and recover the BuiltSection
+// and put that into the cache, then remove the uploaded mesh from the cache
+public class BuiltSectionMeshCache {
+    private final ConcurrentHashMap<Long, BuiltSection> renderCache = new ConcurrentHashMap<>(1000,0.75f,10);
+
+    public BuiltSection getMesh(long key) {
+        return null;
+    }
+
+    //Returns true if the mesh was used, (this is so the parent method can free mesh object)
+    public boolean putMesh(BuiltSection mesh) {
+        return false;
+    }
+
+    public void clearMesh(long key) {
+    }
+
+
+    public void free() {
+
+    }
+}

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java

@@ -46,7 +46,7 @@ public class RenderDataFactory {
 
     //buildMask in the lower 6 bits contains the faces to build, the next 6 bits are whether the edge face builds against
     // its neigbor or not (0 if it does 1 if it doesnt (0 is default behavior))
-    public BuiltSection generateMesh(WorldSection section, int buildMask) {
+    public BuiltSection generateMesh(WorldSection section) {
         section.copyDataTo(this.sectionCache);
         this.translucentQuadCollector.clear();
         this.doubleSidedQuadCollector.clear();
@@ -75,7 +75,7 @@ public class RenderDataFactory {
         }
 
         if (quadCount == 0) {
-            return new BuiltSection(section.getKey());
+            return new BuiltSection(section.key);
         }
 
         var buff = new MemoryBuffer(quadCount*8L);
@@ -109,7 +109,7 @@ public class RenderDataFactory {
         aabb |= (this.maxY-this.minY)<<20;
         aabb |= (this.maxZ-this.minZ)<<25;
 
-        return new BuiltSection(section.getKey(), aabb, buff, offsets);
+        return new BuiltSection(section.key, aabb, buff, offsets);
     }
 
 

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java

@@ -14,7 +14,7 @@ import java.util.function.ToIntFunction;
 //TODO: Add a render cache
 public class RenderGenerationService {
     public interface TaskChecker {boolean check(int lvl, int x, int y, int z);}
-    private record BuildTask(Supplier<WorldSection> sectionSupplier, ToIntFunction<WorldSection> flagSupplier) {}
+    private record BuildTask(Supplier<WorldSection> sectionSupplier) {}
 
     private volatile boolean running = true;
     private final Thread[] workers;
@@ -25,6 +25,7 @@ public class RenderGenerationService {
     private final WorldEngine world;
     private final ModelManager modelManager;
     private final Consumer<BuiltSection> resultConsumer;
+    private final BuiltSectionMeshCache meshCache = new BuiltSectionMeshCache();
 
     public RenderGenerationService(WorldEngine world, ModelManager modelManager, int workers, Consumer<BuiltSection> consumer) {
         this.world = world;
@@ -39,8 +40,6 @@ public class RenderGenerationService {
         }
     }
 
-    private final ConcurrentHashMap<Long, BuiltSection> renderCache = new ConcurrentHashMap<>(1000,0.75f,10);
-
     //TODO: add a generated render data cache
     private void renderWorker() {
         //Thread local instance of the factory
@@ -57,24 +56,13 @@ public class RenderGenerationService {
                 continue;
             }
             section.assertNotFree();
-            int buildFlags = task.flagSupplier.applyAsInt(section);
+            var mesh = factory.generateMesh(section);
-            if (buildFlags != 0) {
-                var mesh = factory.generateMesh(section, buildFlags);
             section.release();
 
             this.resultConsumer.accept(mesh.clone());
-
+            if (!this.meshCache.putMesh(mesh)) {
-                if (false) {
-                    var prevCache = this.renderCache.put(mesh.position, mesh);
-                    if (prevCache != null) {
-                        prevCache.free();
-                    }
-                } else {
                 mesh.free();
             }
-            } else {
-                section.release();
-            }
         }
     }
 
@@ -94,14 +82,14 @@ public class RenderGenerationService {
     // like if its in the render queue and if we should abort building the render data
     //1 proposal fix is a Long2ObjectLinkedOpenHashMap<WorldSection> which means we can abort if needed,
     // also gets rid of dependency on a WorldSection (kinda)
-    public void enqueueTask(int lvl, int x, int y, int z, ToIntFunction<WorldSection> flagSupplier) {
+    public void enqueueTask(int lvl, int x, int y, int z) {
-        this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true, flagSupplier);
+        this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true);
     }
 
-    public void enqueueTask(int lvl, int x, int y, int z, TaskChecker checker, ToIntFunction<WorldSection> flagSupplier) {
+    public void enqueueTask(int lvl, int x, int y, int z, TaskChecker checker) {
         long ikey = WorldEngine.getWorldSectionId(lvl, x, y, z);
         {
-            var cache = this.renderCache.get(ikey);
+            var cache = this.meshCache.getMesh(ikey);
             if (cache != null) {
                 this.resultConsumer.accept(cache.clone());
                 return;
@@ -116,7 +104,7 @@ public class RenderGenerationService {
                     } else {
                         return null;
                     }
-                }, flagSupplier);
+                });
             });
         }
     }
@@ -159,6 +147,6 @@ public class RenderGenerationService {
         while (!this.taskQueue.isEmpty()) {
             this.taskQueue.removeFirst();
         }
-        this.renderCache.values().forEach(BuiltSection::free);
+        this.meshCache.free();
     }
 }

src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java

@@ -0,0 +1,127 @@
+package me.cortex.voxy.client.core.rendering.util;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongConsumer;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import me.cortex.voxy.client.core.gl.GlBuffer;
+import me.cortex.voxy.client.core.gl.GlFence;
+import me.cortex.voxy.client.core.gl.GlPersistentMappedBuffer;
+import me.cortex.voxy.client.core.util.AllocationArena;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+
+import static me.cortex.voxy.client.core.util.AllocationArena.SIZE_LIMIT;
+import static org.lwjgl.opengl.ARBDirectStateAccess.glCopyNamedBufferSubData;
+import static org.lwjgl.opengl.ARBDirectStateAccess.glFlushMappedNamedBufferRange;
+import static org.lwjgl.opengl.ARBMapBufferRange.*;
+import static org.lwjgl.opengl.GL11.glFinish;
+import static org.lwjgl.opengl.GL42.glMemoryBarrier;
+import static org.lwjgl.opengl.GL42C.GL_BUFFER_UPDATE_BARRIER_BIT;
+import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BARRIER_BIT;
+import static org.lwjgl.opengl.GL44.GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT;
+
+public class DownloadStream {
+    public interface DownloadResultConsumer {
+        void consume(long ptr, long size);
+    }
+
+    private final AllocationArena allocationArena = new AllocationArena();
+    private final GlPersistentMappedBuffer downloadBuffer;
+
+    private final Deque<DownloadFrame> frames = new ArrayDeque<>();
+    private final LongArrayList thisFrameAllocations = new LongArrayList();
+    private final Deque<DownloadData> downloadList = new ArrayDeque<>();
+    private final ArrayList<DownloadData> thisFrameDownloadList = new ArrayList<>();
+
+    public DownloadStream(long size) {
+        this.downloadBuffer = new GlPersistentMappedBuffer(size, GL_MAP_READ_BIT);
+        this.allocationArena.setLimit(size);
+    }
+
+    private long caddr = -1;
+    private long offset = 0;
+    public void download(GlBuffer buffer, long destOffset, long size, DownloadResultConsumer resultConsumer) {
+        if (size > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException();
+        }
+
+        long addr;
+        if (this.caddr == -1 || !this.allocationArena.expand(this.caddr, (int) size)) {
+            this.caddr = this.allocationArena.alloc((int) size);//TODO: replace with allocFromLargest
+            if (this.caddr == SIZE_LIMIT) {
+                this.commit();
+                int attempts = 10;
+                while (--attempts != 0 && this.caddr == SIZE_LIMIT) {
+                    glFinish();
+                    this.tick();
+                    this.caddr = this.allocationArena.alloc((int) size);
+                }
+                if (this.caddr == SIZE_LIMIT) {
+                    throw new IllegalStateException("Could not allocate memory segment big enough for upload even after force flush");
+                }
+            }
+            this.thisFrameAllocations.add(this.caddr);
+            this.offset = size;
+            addr = this.caddr;
+        } else {//Could expand the allocation so just update it
+            addr = this.caddr + this.offset;
+            this.offset += size;
+        }
+
+        if (this.caddr + size > this.downloadBuffer.size()) {
+            throw new IllegalStateException();
+        }
+
+        this.downloadList.add(new DownloadData(buffer, addr, destOffset, size, resultConsumer));
+    }
+
+
+    public void commit() {
+        //Copies all the data from target buffers into the download stream
+        for (var entry : this.downloadList) {
+            glCopyNamedBufferSubData(entry.target.id, this.downloadBuffer.id, entry.downloadOffset, entry.targetOffset, entry.size);
+        }
+        thisFrameDownloadList.addAll(this.downloadList);
+        this.downloadList.clear();
+
+        this.caddr = -1;
+        this.offset = 0;
+    }
+
+    public void tick() {
+        this.commit();
+        if (!this.thisFrameAllocations.isEmpty()) {
+            glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT | GL_BUFFER_UPDATE_BARRIER_BIT);
+            this.frames.add(new DownloadFrame(new GlFence(), new LongArrayList(this.thisFrameAllocations), new ArrayList<>(this.thisFrameDownloadList)));
+            this.thisFrameAllocations.clear();
+            this.thisFrameDownloadList.clear();
+        }
+
+        while (!this.frames.isEmpty()) {
+            //Since the ordering of frames is the ordering of the gl commands if we encounter an unsignaled fence
+            // all the other fences should also be unsignaled
+            if (!this.frames.peek().fence.signaled()) {
+                break;
+            }
+            //Release all the allocations from the frame
+            var frame = this.frames.pop();
+
+            //Apply all the callbacks
+            for (var data : frame.data) {
+                data.resultConsumer.consume(this.downloadBuffer.addr() + data.downloadOffset, data.size);
+            }
+
+            frame.allocations.forEach(this.allocationArena::free);
+            frame.fence.free();
+        }
+    }
+
+    private record DownloadFrame(GlFence fence, LongArrayList allocations, ArrayList<DownloadData> data) {}
+    private record DownloadData(GlBuffer target, long downloadOffset, long targetOffset, long size, DownloadResultConsumer resultConsumer) {}
+
+
+    // Global download stream
+    public static final DownloadStream INSTANCE = new DownloadStream(1<<25);//32 mb download buffer
+}

src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java

@@ -111,7 +111,7 @@ public class UploadStream {
             }
             //Release all the allocations from the frame
             var frame = this.frames.pop();
-            frame.allocations.forEach(allocationArena::free);
+            frame.allocations.forEach(this.allocationArena::free);
             frame.fence.free();
         }
     }

src/main/java/me/cortex/voxy/common/world/ActiveSectionTracker.java

@@ -13,17 +13,17 @@ public class ActiveSectionTracker {
     private final SectionLoader loader;
 
     @SuppressWarnings("unchecked")
-    public ActiveSectionTracker(int layers, SectionLoader loader) {
+    public ActiveSectionTracker(int cacheSizeBits, SectionLoader loader) {
         this.loader = loader;
-        this.loadedSectionCache = new Long2ObjectOpenHashMap[layers];
+        this.loadedSectionCache = new Long2ObjectOpenHashMap[1<<cacheSizeBits];
-        for (int i = 0; i < layers; i++) {
+        for (int i = 0; i < this.loadedSectionCache.length; i++) {
-            this.loadedSectionCache[i] = new Long2ObjectOpenHashMap<>(1<<(16-i));
+            this.loadedSectionCache[i] = new Long2ObjectOpenHashMap<>(1024);
         }
     }
 
     public WorldSection acquire(int lvl, int x, int y, int z, boolean nullOnEmpty) {
         long key = WorldEngine.getWorldSectionId(lvl, x, y, z);
-        var cache = this.loadedSectionCache[lvl];
+        var cache = this.loadedSectionCache[this.getCacheArrayIndex(key)];
         VolatileHolder<WorldSection> holder = null;
         boolean isLoader = false;
         synchronized (cache) {
@@ -69,16 +69,25 @@ public class ActiveSectionTracker {
     }
 
     void tryUnload(WorldSection section) {
-        var cache = this.loadedSectionCache[section.lvl];
+        var cache = this.loadedSectionCache[this.getCacheArrayIndex(section.key)];
         synchronized (cache) {
             if (section.trySetFreed()) {
-                if (cache.remove(section.getKey()).obj != section) {
+                if (cache.remove(section.key).obj != section) {
                     throw new IllegalStateException("Removed section not the same as the referenced section in the cache");
                 }
             }
         }
     }
 
+    private int getCacheArrayIndex(long pos) {
+        return (int) (mixStafford13(pos) & (this.loadedSectionCache.length-1));
+    }
+
+    public static long mixStafford13(long seed) {
+        seed = (seed ^ seed >>> 30) * -4658895280553007687L;
+        seed = (seed ^ seed >>> 27) * -7723592293110705685L;
+        return seed ^ seed >>> 31;
+    }
 
     public int[] getCacheCounts() {
         int[] res = new int[this.loadedSectionCache.length];

src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java

@@ -25,8 +25,8 @@ public class SaveLoadSystem {
         long[] lut = LUTVAL.toLongArray();
         ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512);
 
-        long hash = section.getKey()^(lut.length*1293481298141L);
+        long hash = section.key^(lut.length*1293481298141L);
-        raw.putLong(section.getKey());
+        raw.putLong(section.key);
         raw.putInt(lut.length);
         for (long id : lut) {
             raw.putLong(id);
@@ -74,8 +74,8 @@ public class SaveLoadSystem {
             hash ^= lut[i];
         }
 
-        if (section.getKey() != key) {
+        if (section.key != key) {
-            throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.getKey());
+            throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
         }
 
         for (int i = 0; i < section.data.length; i++) {

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -33,18 +33,19 @@ public class WorldEngine {
         this.maxMipLevels = maxMipLayers;
         this.storage = storageBackend;
         this.mapper = new Mapper(this.storage);
-        this.sectionTracker = new ActiveSectionTracker(maxMipLayers, this::unsafeLoadSection);
+        //4 cache size bits means that the section tracker has 16 separate maps that it uses
+        this.sectionTracker = new ActiveSectionTracker(4, this::unsafeLoadSection);
 
         this.savingService = new SectionSavingService(this, savingServiceWorkers, compressionLevel);
         this.ingestService  = new VoxelIngestService(this, ingestWorkers);
     }
 
     private int unsafeLoadSection(WorldSection into) {
-        var data = this.storage.getSectionData(into.getKey());
+        var data = this.storage.getSectionData(into.key);
         if (data != null) {
             try {
                 if (!SaveLoadSystem.deserialize(into, data)) {
-                    this.storage.deleteSectionData(into.getKey());
+                    this.storage.deleteSectionData(into.key);
                     //TODO: regenerate the section from children
                     Arrays.fill(into.data, Mapper.AIR);
                     System.err.println("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, setting to air");

src/main/java/me/cortex/voxy/common/world/WorldSection.java

@@ -12,6 +12,7 @@ public final class WorldSection {
     public final int x;
     public final int y;
     public final int z;
+    public final long key;
 
     long[] data;
     private final ActiveSectionTracker tracker;
@@ -25,6 +26,7 @@ public final class WorldSection {
         this.x = x;
         this.y = y;
         this.z = z;
+        this.key = WorldEngine.getWorldSectionId(lvl, x, y, z);
         this.tracker = tracker;
 
         this.data = new long[32*32*32];
@@ -78,10 +80,6 @@ public final class WorldSection {
         }
     }
 
-    public long getKey() {
-        return WorldEngine.getWorldSectionId(this.lvl, this.x, this.y, this.z);
-    }
-
     public static int getIndex(int x, int y, int z) {
         int M = (1<<5)-1;
         if (x<0||x>M||y<0||y>M||z<0||z>M) {

src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java

@@ -43,7 +43,7 @@ public class SectionSavingService {
             section.inSaveQueue.set(false);
 
             var saveData = SaveLoadSystem.serialize(section, this.compressionLevel);
-            this.world.storage.setSectionData(section.getKey(), saveData);
+            this.world.storage.setSectionData(section.key, saveData);
             MemoryUtil.memFree(saveData);
 
             section.release();

src/main/resources/assets/voxy/shaders/lod/gl46/quad_format.glsl

@@ -58,7 +58,7 @@ uint extractFace(ivec2 quad) {
 
 uint extractStateId(ivec2 quad) {
     //Eu32(quad, 20, 26);
-    return 1;
+    return Eu32v(quad, 6, 26)|(Eu32v(quad, 14, 32)<<6);
 }
 
 uint extractBiomeId(ivec2 quad) {