QQQQ

src/main/java/me/cortex/voxy/client/core/model/ModelTextureBakery.java

@@ -223,6 +223,8 @@ public class ModelTextureBakery {
 
         var bb = new BufferBuilder(this.allocator, VertexFormat.DrawMode.QUADS, VertexFormats.POSITION_TEXTURE_COLOR);
         if (!renderFluid) {
+            //TODO: need to do 2 variants for quads, one which have coloured, ones that dont, might be able to pull a spare bit
+            // at the end whether or not a pixel should be mixed with texture
             renderQuads(bb, state, model, new MatrixStack(), randomValue);
         } else {
             MinecraftClient.getInstance().getBlockRenderManager().renderFluid(BlockPos.ORIGIN, new BlockRenderView() {
@@ -328,8 +330,8 @@ public class ModelTextureBakery {
             var quads = model.getQuads(state, direction, new LocalRandom(randomValue));
             for (var quad : quads) {
                 //TODO: mark pixels that have
-                int meta = quad.hasColor()?1:0;
-                builder.quad(stack.peek(), quad, 255f/((meta>>16)&0xff), 255f/((meta>>8)&0xff), 255f/(meta&0xff), 1.0f, 0, 0);
+                int meta = 1;
+                builder.quad(stack.peek(), quad, ((meta>>16)&0xff)/255f, ((meta>>8)&0xff)/255f, (meta&0xff)/255f, 1.0f, 0, 0);
             }
         }
     }

src/main/java/me/cortex/voxy/client/core/rendering/GeometryCache.java

@@ -9,7 +9,11 @@ public class GeometryCache {
     private long currentSize;
     private final Long2ObjectLinkedOpenHashMap<BuiltSection> cache = new Long2ObjectLinkedOpenHashMap<>();
     public GeometryCache(long maxSize) {
-        this.maxCombinedSize = maxSize;
+        this.setMaxTotalSize(maxSize);
+    }
+
+    public void setMaxTotalSize(long size) {
+        this.maxCombinedSize = size;
     }
 
     //Puts the section into the cache

src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java

@@ -47,7 +47,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
 
         //Max sections: ~500k
         //Max geometry: 1 gb
-        this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<19, (1L<<31)-1024);
+        this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<19, (1L<<30)-1024);
 
         //Do something incredibly hacky, we dont need to keep the reference to this around, so just connect and discard
         var router = new SectionUpdateRouter();

src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/DebugRenderer.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.client.core.rendering.hierarchical;
+package me.cortex.voxy.client.core.rendering.hierachical2;
 
 import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.gl.shader.Shader;

src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java

@@ -1,4 +1,13 @@
 package me.cortex.voxy.client.core.rendering.hierachical2;
 
+//Composed of 2 (3) parts
+// a node cleaner
+// and a geometr/section cleaner
 public class NodeCleaner {
+
+
+    public void setNodeMemoryUsage() {
+        //Needs to bubble up information to all parents
+        // also needs to update a tick/last seen time for node removal
+    }
 }

src/main/java/me/cortex/voxy/client/core/rendering/section/AbstractSectionGeometryManager.java

@@ -3,6 +3,8 @@ package me.cortex.voxy.client.core.rendering.section;
 import me.cortex.voxy.client.core.rendering.building.BuiltSection;
 import me.jellysquid.mods.sodium.client.util.MathUtil;
 
+import java.util.function.Consumer;
+
 //Does not care about the position of the sections, multiple sections that have the same position can be uploaded
 // it is up to the traversal system to manage what sections exist in the geometry buffer
 // the system is basicly "dumb" as in it just follows orders
@@ -24,4 +26,6 @@ public abstract class AbstractSectionGeometryManager {
     public void tick() {}
 
     public void free() {}
+
+    public abstract void downloadAndRemove(int id, Consumer<BuiltSection> callback);
 }

src/main/java/me/cortex/voxy/client/core/rendering/section/BasicSectionGeometryManager.java

@@ -7,12 +7,16 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.rendering.building.BuiltSection;
 import me.cortex.voxy.client.core.rendering.hierachical2.HierarchicalOcclusionTraverser;
 import me.cortex.voxy.client.core.rendering.util.BufferArena;
+import me.cortex.voxy.client.core.rendering.util.DownloadStream;
 import me.cortex.voxy.client.core.rendering.util.UploadStream;
 import me.cortex.voxy.common.util.HierarchicalBitSet;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.text.Text;
 import org.lwjgl.system.MemoryUtil;
 
+import java.util.function.Consumer;
+
 public class BasicSectionGeometryManager extends AbstractSectionGeometryManager {
     private static final int SECTION_METADATA_SIZE = 32;
     private final GlBuffer sectionMetadataBuffer;
@@ -86,10 +90,11 @@ public class BasicSectionGeometryManager extends AbstractSectionGeometryManager
             throw new IllegalStateException("Unable to upload section geometry as geometry buffer is full");
         }
         //8 bytes per quad
-        return new SectionMeta(geometry.position, geometry.aabb, geometryPtr, (int) (geometry.geometryBuffer.size/8), geometry.offsets);
+        return new SectionMeta(geometry.position, geometry.aabb, geometryPtr, (int) (geometry.geometryBuffer.size/8), geometry.offsets, geometry.childExistence);
     }
 
-    private record SectionMeta(long position, int aabb, int geometryPtr, int itemCount, int[] offsets) {
+    //TODO: move child existence to and external thing to not get confused
+    private record SectionMeta(long position, int aabb, int geometryPtr, int itemCount, int[] offsets, byte childExistence) {
         public void writeMetadata(long ptr) {
             //Split the long into 2 ints to solve endian issues
             MemoryUtil.memPutInt(ptr, (int) (this.position>>32)); ptr += 4;
@@ -129,6 +134,20 @@ public class BasicSectionGeometryManager extends AbstractSectionGeometryManager
         this.geometry.free();
     }
 
+    @Override
+    public void downloadAndRemove(int id, Consumer<BuiltSection> callback) {
+        if (!this.allocationSet.free(id)) {
+            throw new IllegalStateException("Id was not already allocated. id: " + id);
+        }
+        var oldMetadata = this.sectionMetadata.set(id, null);
+        this.geometry.downloadRemove(oldMetadata.geometryPtr, buffer ->
+            callback.accept(new BuiltSection(oldMetadata.position, oldMetadata.childExistence, oldMetadata.aabb, buffer.copy(), oldMetadata.offsets))
+        );
+        this.geometry.free(oldMetadata.geometryPtr);
+        this.invalidatedSectionIds.add(id);
+    }
+
+
     int getSectionCount() {
         return this.allocationSet.getCount();
     }

src/main/java/me/cortex/voxy/client/core/rendering/util/BufferArena.java

@@ -7,6 +7,8 @@ import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;
 
+import java.util.function.Consumer;
+
 public class BufferArena {
     private final long size;
     private final int elementSize;
@@ -62,4 +64,10 @@ public class BufferArena {
     public long getUsedBytes() {
         return this.used*this.elementSize;
     }
+
+    public void downloadRemove(long allocation, Consumer<MemoryBuffer> consumer) {
+        int size = this.allocationMap.free(allocation);
+        this.used -= size;
+        DownloadStream.INSTANCE.download(this.buffer, allocation*this.elementSize, (long) size *this.elementSize, consumer);
+    }
 }

src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java

@@ -7,10 +7,12 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
 import me.cortex.voxy.client.core.gl.GlFence;
 import me.cortex.voxy.client.core.gl.GlPersistentMappedBuffer;
 import me.cortex.voxy.client.core.util.AllocationArena;
+import me.cortex.voxy.common.util.MemoryBuffer;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Deque;
+import java.util.function.Consumer;
 
 import static me.cortex.voxy.client.core.util.AllocationArena.SIZE_LIMIT;
 import static org.lwjgl.opengl.ARBDirectStateAccess.glCopyNamedBufferSubData;
@@ -49,14 +51,24 @@ public class DownloadStream {
         this.download(buffer, 0, buffer.size(), resultConsumer);
     }
 
-    public void download(GlBuffer buffer, long destOffset, long size, DownloadResultConsumer resultConsumer) {
+    public void download(GlBuffer buffer, Consumer<MemoryBuffer> resultConsumer) {
+        this.download(buffer, 0, buffer.size(), resultConsumer);
+    }
+
+    public void download(GlBuffer buffer, long downloadOffset, long size, Consumer<MemoryBuffer> consumer) {
+        this.download(buffer, downloadOffset, size, (ptr,size2)-> {
+            consumer.accept(MemoryBuffer.createUntrackedUnfreeableRawFrom(ptr, size));
+        });
+    }
+
+    public void download(GlBuffer buffer, long downloadOffset, long size, DownloadResultConsumer resultConsumer) {
         if (size > Integer.MAX_VALUE) {
             throw new IllegalArgumentException();
         }
         if (size <= 0) {
             throw new IllegalArgumentException();
         }
-        if (destOffset+size > buffer.size()) {
+        if (downloadOffset+size > buffer.size()) {
             throw new IllegalArgumentException();
         }
 
@@ -87,7 +99,7 @@ public class DownloadStream {
             throw new IllegalStateException();
         }
 
-        this.downloadList.add(new DownloadData(buffer, addr, destOffset, size, resultConsumer));
+        this.downloadList.add(new DownloadData(buffer, addr, downloadOffset, size, resultConsumer));
     }
 
 

src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java

@@ -5,10 +5,6 @@ import me.cortex.voxy.common.storage.config.CompressorConfig;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.world.SaveLoadSystem;
-import me.cortex.voxy.common.world.service.SectionSavingService;
-import org.lwjgl.system.MemoryUtil;
-
-import java.nio.ByteBuffer;
 
 import static org.lwjgl.util.zstd.Zstd.*;
 

src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java

@@ -6,7 +6,6 @@ import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
-import org.lwjgl.system.MemoryUtil;
 import redis.clients.jedis.JedisPool;
 
 import java.nio.ByteBuffer;

src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java

@@ -6,13 +6,9 @@ import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
-import org.lwjgl.system.MemoryUtil;
 import org.rocksdb.*;
 
-import java.io.File;
-import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;

src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java

@@ -5,15 +5,17 @@ import org.lwjgl.system.MemoryUtil;
 public class MemoryBuffer extends TrackedObject {
     public final long address;
     public final long size;
+    private final boolean freeable;
 
     public MemoryBuffer(long size) {
-        this.size = size;
-        this.address = MemoryUtil.nmemAlloc(size);
+        this(true, MemoryUtil.nmemAlloc(size), size, true);
     }
 
-    private MemoryBuffer(long address, long size) {
+    private MemoryBuffer(boolean track, long address, long size, boolean freeable) {
+        super(track);
         this.size = size;
         this.address = address;
+        this.freeable = freeable;
     }
 
     public void cpyTo(long dst) {
@@ -24,11 +26,15 @@ public class MemoryBuffer extends TrackedObject {
     @Override
     public void free() {
         super.free0();
+        if (this.freeable) {
             MemoryUtil.nmemFree(this.address);
+        } else {
+            throw new IllegalArgumentException("Tried to free unfreeable buffer");
+        }
     }
 
     public MemoryBuffer copy() {
-        var copy = new MemoryBuffer(this.size);
+        var copy = new MemoryBuffer(false, this.size, size, freeable);
         this.cpyTo(copy.address);
         return copy;
     }
@@ -41,10 +47,18 @@ public class MemoryBuffer extends TrackedObject {
         //Free the current object, but not the memory associated with it
         super.free0();
 
-        return new MemoryBuffer(this.address, size);
+        return new MemoryBuffer(true, this.address, size, this.freeable);
     }
 
 
     //TODO: create like Long(offset) -> value at offset
     // methods for get and set, that way can have a single unifed system to ensure memory access bounds
+
+
+    public static MemoryBuffer createUntrackedRawFrom(long address, long size) {
+        return new MemoryBuffer(false, address, size, true);
+    }
+    public static MemoryBuffer createUntrackedUnfreeableRawFrom(long address, long size) {
+        return new MemoryBuffer(false, address, size, false);
+    }
 }

src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java

@@ -1,16 +1,12 @@
 package me.cortex.voxy.common.world;
 
-import it.unimi.dsi.fastutil.longs.Long2ShortFunction;
 import it.unimi.dsi.fastutil.longs.Long2ShortOpenHashMap;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
 import me.cortex.voxy.common.world.other.Mapper;
 import me.cortex.voxy.commonImpl.VoxyCommon;
 import org.lwjgl.system.MemoryUtil;
 
-import java.nio.ByteBuffer;
-
 import static org.lwjgl.util.zstd.Zstd.*;
 
 public class SaveLoadSystem {

src/main/resources/assets/voxy/shaders/bakery/position_tex.fsh

@@ -4,6 +4,7 @@ layout(location=0) uniform sampler2D tex;
 in vec2 texCoord;
 in flat uint metadata;
 out vec4 colour;
+
 void main() {
-    colour = texture(tex, texCoord);//*(metadata&1);
+    colour = texture(tex, texCoord)*(metadata&1);
 }