Prep for async texture fetch

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -170,12 +170,12 @@ public class VoxelCore {
             System.out.println("Shutting down importer");
             try {this.importer.shutdown();this.importer = null;} catch (Exception e) {e.printStackTrace();}
         }
-        System.out.println("Shutting down voxel core");
+        System.out.println("Shutting down rendering");
-        try {this.world.shutdown();} catch (Exception e) {e.printStackTrace();}
-        System.out.println("World engine shut down");
         try {this.renderer.shutdown(); this.viewportSelector.free();} catch (Exception e) {e.printStackTrace();}
-        System.out.println("Renderer shut down");
+        System.out.println("Shutting down post processor");
         if (this.postProcessing!=null){try {this.postProcessing.shutdown();} catch (Exception e) {e.printStackTrace();}}
+        System.out.println("Shutting down world engine");
+        try {this.world.shutdown();} catch (Exception e) {e.printStackTrace();}
         System.out.println("Voxel core shut down");
     }
 

src/main/java/me/cortex/voxy/client/core/model/ModelBakerySubsystem.java

@@ -3,28 +3,27 @@ package me.cortex.voxy.client.core.model;
 
 import it.unimi.dsi.fastutil.ints.IntLinkedOpenHashSet;
 import me.cortex.voxy.common.world.other.Mapper;
-import net.minecraft.client.MinecraftClient;
-import org.lwjgl.glfw.GLFW;
-import org.lwjgl.opengl.GL;
-import org.lwjgl.opengl.GLCapabilities;
 
 import java.lang.invoke.VarHandle;
 import java.util.List;
-import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.concurrent.Semaphore;
-public class OnThreadModelBakerySystem {
 
-    private final ModelStore storage = new ModelStore(16);
+public class ModelBakerySubsystem {
+    //Redo to just make it request the block faces with the async texture download stream which
+    // basicly solves all the render stutter due to the baking
+
+
+    private final ModelStore storage = new ModelStore();
     public final ModelFactory factory;
     private final IntLinkedOpenHashSet blockIdQueue = new IntLinkedOpenHashSet();
 
-    public OnThreadModelBakerySystem(Mapper mapper) {
+    public ModelBakerySubsystem(Mapper mapper) {
-        this.factory = new ModelFactory(mapper);
+        this.factory = new ModelFactory(mapper, this.storage);
     }
 
     public void tick() {
         //There should be a method to access the frame time IIRC, if the user framecap is unlimited lock it to like 60 fps for computation
         int BUDGET = 20;//TODO: make this computed based on the remaining free time in a frame (and like div by 2 to reduce overhead) (with a min of 1)
+
         for (int i = 0; i < BUDGET; i++) {
             if (!this.blockIdQueue.isEmpty()) {
                 int blockId = -1;

src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java

@@ -109,18 +109,21 @@ public class ModelFactory {
     private final int[] idMappings;
     private final Object2IntOpenHashMap<ModelEntry> modelTexture2id = new Object2IntOpenHashMap<>();
 
-    private final Mapper mapper;
 
     private final List<Biome> biomes = new ArrayList<>();
     private final List<Pair<Integer, BlockState>> modelsRequiringBiomeColours = new ArrayList<>();
 
     private static final ObjectSet<BlockState> LOGGED_SELF_CULLING_WARNING = new ObjectOpenHashSet<>();
 
+    private final Mapper mapper;
+    private final ModelStore storage;
+
 
     //TODO: NOTE!!! is it worth even uploading as a 16x16 texture, since automatic lod selection... doing 8x8 textures might be perfectly ok!!!
     // this _quarters_ the memory requirements for the texture atlas!!! WHICH IS HUGE saving
-    public ModelFactory(Mapper mapper) {
+    public ModelFactory(Mapper mapper, ModelStore storage) {
         this.mapper = mapper;
+        this.storage = storage;
         this.bakery = new ModelTextureBakery(MODEL_TEXTURE_SIZE, MODEL_TEXTURE_SIZE);
 
         this.metadataCache = new long[1<<16];
@@ -150,6 +153,10 @@ public class ModelFactory {
 
     //TODO: what i need to do is seperate out fluid states from blockStates
 
+    //Processes the results of the baking, its a seperate function due to the flight
+    private void processBakingResult() {
+
+    }
 
     //TODO: so need a few things, per face sizes and offsets, the sizes should be computed from the pixels and find the minimum bounding pixel
     // while the depth is computed from the depth buffer data
@@ -161,6 +168,11 @@ public class ModelFactory {
 
         boolean isFluid = blockState.getBlock() instanceof FluidBlock;
         int modelId = -1;
+
+        //TODO: FIRST!! dispatch a face request the fluid state if it doesnt exist!!!
+        // THEN dispatch this block face request, the ordering should result in a gurentee that the fluid block state is
+        // computed before this block state
+
         var textureData = this.bakery.renderFaces(blockState, 123456, isFluid);
 
         int clientFluidStateId = -1;
@@ -216,8 +228,7 @@ public class ModelFactory {
 
 
 
-        final long uploadPtrConst = MemoryUtil.nmemAlloc(MODEL_SIZE);
+        long uploadPtr = UploadStream.INSTANCE.upload(this.storage.modelBuffer, (long) modelId * MODEL_SIZE, MODEL_SIZE);;
-        long uploadPtr = uploadPtrConst;
 
 
         //TODO: implement;
@@ -358,8 +369,6 @@ public class ModelFactory {
         //modelFlags |= blockRenderLayer == RenderLayer.getSolid()?0:1;// should discard alpha
         MemoryUtil.memPutInt(uploadPtr, modelFlags);
 
-        int[] biomeData = null;
-        int biomeIndex = -1;
         //Temporary override to always be non biome specific
         if (colourProvider == null) {
             MemoryUtil.memPutInt(uploadPtr + 4, -1);//Set the default to nothing so that its faster on the gpu
@@ -367,13 +376,14 @@ public class ModelFactory {
             MemoryUtil.memPutInt(uploadPtr + 4, captureColourConstant(colourProvider, blockState, DEFAULT_BIOME)|0xFF000000);
         } else if (!this.biomes.isEmpty()) {
             //Populate the list of biomes for the model state
-            biomeIndex = this.modelsRequiringBiomeColours.size() * this.biomes.size();
+            int biomeIndex = this.modelsRequiringBiomeColours.size() * this.biomes.size();
             MemoryUtil.memPutInt(uploadPtr + 4, biomeIndex);
             this.modelsRequiringBiomeColours.add(new Pair<>(modelId, blockState));
-            //long clrUploadPtr = UploadStream.INSTANCE.upload(this.modelColourBuffer, biomeIndex * 4L, 4L * this.biomes.size());
+            //NOTE: UploadStream.INSTANCE is called _after_ uploadPtr is finished being used, this is cause the upload pointer
-            biomeData = new int[this.biomes.size()];
+            // may be invalidated as soon as another upload stream is invoked
-            for (int biomeId = 0; biomeId < this.biomes.size(); biomeId++) {
+            long clrUploadPtr = UploadStream.INSTANCE.upload(this.storage.modelColourBuffer, biomeIndex * 4L, 4L * this.biomes.size());
-                biomeData[biomeId] = captureColourConstant(colourProvider, blockState, this.biomes.get(biomeId))|0xFF000000;
+            for (var biome : this.biomes) {
+                MemoryUtil.memPutInt(clrUploadPtr, captureColourConstant(colourProvider, blockState, biome)|0xFF000000); clrUploadPtr += 4;
             }
         }
 
@@ -383,21 +393,19 @@ public class ModelFactory {
         //TODO
 
 
-        var textureUpload = this.putTextures(modelId, textureData);
+        this.putTextures(modelId, textureData);
 
         //glGenerateTextureMipmap(this.textures.id);
 
         //Set the mapping at the very end
         this.idMappings[blockId] = modelId;
 
-
+        //Upload/commit stream
-        new NewModelBufferDelta(modelId, uploadPtrConst, biomeIndex, biomeData, textureUpload);
+        //TODO maybe dont do it for every uploaded block?? try to batch it
+        UploadStream.INSTANCE.commit();
     }
 
     public void addBiome(int id, Biome biome) {
-        throw new IllegalStateException("IMPLEMENT");
-
-        /*
         this.biomes.add(biome);
         if (this.biomes.size()-1 != id) {
             throw new IllegalStateException("Biome ordering not consistent with biome id for biome " + biome + " expected id: " + (this.biomes.size()-1) + " got id: " + id);
@@ -411,13 +419,12 @@ public class ModelFactory {
             }
             //Populate the list of biomes for the model state
             int biomeIndex = (i++) * this.biomes.size();
-            MemoryUtil.memPutInt(UploadStream.INSTANCE.upload(this.modelBuffer, (entry.getLeft()* MODEL_SIZE)+ 4*6 + 4, 4), biomeIndex);
+            MemoryUtil.memPutInt(UploadStream.INSTANCE.upload(this.storage.modelBuffer, (entry.getLeft()* MODEL_SIZE)+ 4*6 + 4, 4), biomeIndex);
-            long clrUploadPtr = UploadStream.INSTANCE.upload(this.modelColourBuffer, biomeIndex * 4L, 4L * this.biomes.size());
+            long clrUploadPtr = UploadStream.INSTANCE.upload(this.storage.modelColourBuffer, biomeIndex * 4L, 4L * this.biomes.size());
             for (var biomeE : this.biomes) {
                 MemoryUtil.memPutInt(clrUploadPtr, captureColourConstant(colourProvider, entry.getRight(), biomeE)|0xFF000000); clrUploadPtr += 4;
             }
         }
-         */
     }
 
 
@@ -567,15 +574,23 @@ public class ModelFactory {
         return this.metadataCache[clientId];
     }
 
-    private ModelTextureUpload putTextures(int id, ColourDepthTextureData[] textures) {
+    private void putTextures(int id, ColourDepthTextureData[] textures) {
-        int texIndex = 0;
+        int X = (id&0xFF) * MODEL_TEXTURE_SIZE*3;
-        int[][] texData = new int[6*4][];
+        int Y = ((id>>8)&0xFF) * MODEL_TEXTURE_SIZE*2;
-        for (int subTex = 0; subTex < 6; subTex++) {
 
+        for (int subTex = 0; subTex < 6; subTex++) {
+            int x = X + (subTex>>1)*MODEL_TEXTURE_SIZE;
+            int y = Y + (subTex&1)*MODEL_TEXTURE_SIZE;
+
+            GlStateManager._pixelStore(GlConst.GL_UNPACK_ROW_LENGTH, 0);
+            GlStateManager._pixelStore(GlConst.GL_UNPACK_SKIP_PIXELS, 0);
+            GlStateManager._pixelStore(GlConst.GL_UNPACK_SKIP_ROWS, 0);
+            GlStateManager._pixelStore(GlConst.GL_UNPACK_ALIGNMENT, 4);
             var current = textures[subTex].colour();
             var next = new int[current.length>>1];
-            for (int i = 0; i < 4; i++) {
+            final int layers = Integer.numberOfTrailingZeros(MODEL_TEXTURE_SIZE);
-                texData[texIndex++] = Arrays.copyOf(current, current.length);
+            for (int i = 0; i < layers; i++) {
+                glTextureSubImage2D(this.storage.textures.id, i, x>>i, y>>i, MODEL_TEXTURE_SIZE>>i, MODEL_TEXTURE_SIZE>>i, GL_RGBA, GL_UNSIGNED_BYTE, current);
 
                 int size = MODEL_TEXTURE_SIZE>>(i+1);
                 for (int pX = 0; pX < size; pX++) {
@@ -592,7 +607,6 @@ public class ModelFactory {
                 next = new int[current.length>>1];
             }
         }
-        return new ModelTextureUpload(id, texData);
     }
 
     public int getSamplerId() {

src/main/java/me/cortex/voxy/client/core/model/ModelStore.java

@@ -7,14 +7,14 @@ import static org.lwjgl.opengl.GL11.GL_RGBA8;
 
 public class ModelStore {
     public static final int MODEL_SIZE = 64;
-    private final GlBuffer modelBuffer;
+    final GlBuffer modelBuffer;
-    private final GlBuffer modelColourBuffer;
+    final GlBuffer modelColourBuffer;
-    private final GlTexture textures;
+    final GlTexture textures;
 
-    public ModelStore(int modelTextureSize) {
+    public ModelStore() {
         this.modelBuffer = new GlBuffer(MODEL_SIZE * (1<<16));
         this.modelColourBuffer = new GlBuffer(4 * (1<<16));
-        this.textures = new GlTexture().store(GL_RGBA8, 4, modelTextureSize*3*256,modelTextureSize*2*256);
+        this.textures = new GlTexture().store(GL_RGBA8, 4, ModelFactory.MODEL_TEXTURE_SIZE*3*256,ModelFactory.MODEL_TEXTURE_SIZE*2*256);
     }
 
 

src/main/java/me/cortex/voxy/client/core/model/ModelTextureUpload.java

@@ -1,30 +0,0 @@
-package me.cortex.voxy.client.core.model;
-
-import com.mojang.blaze3d.platform.GlConst;
-import com.mojang.blaze3d.platform.GlStateManager;
-
-import static me.cortex.voxy.client.core.model.ModelFactory.MODEL_TEXTURE_SIZE;
-import static org.lwjgl.opengl.ARBDirectStateAccess.glTextureSubImage2D;
-import static org.lwjgl.opengl.GL11.GL_RGBA;
-import static org.lwjgl.opengl.GL11.GL_UNSIGNED_BYTE;
-
-public record ModelTextureUpload(int id, int[][] data) {
-
-    public void upload(int textureId) {
-        GlStateManager._pixelStore(GlConst.GL_UNPACK_ROW_LENGTH, 0);
-        GlStateManager._pixelStore(GlConst.GL_UNPACK_SKIP_PIXELS, 0);
-        GlStateManager._pixelStore(GlConst.GL_UNPACK_SKIP_ROWS, 0);
-        GlStateManager._pixelStore(GlConst.GL_UNPACK_ALIGNMENT, 4);
-
-        int i = 0;
-        int X = (this.id&0xFF) * MODEL_TEXTURE_SIZE*3;
-        int Y = ((this.id>>8)&0xFF) * MODEL_TEXTURE_SIZE*2;
-        for (int face = 0; face < 6; face++) {
-            int x = X + (face>>1)*MODEL_TEXTURE_SIZE;
-            int y = Y + (face&1)*MODEL_TEXTURE_SIZE;
-            for (int mip = 0; mip < 4; mip++) {
-                glTextureSubImage2D(id, mip, x >> mip, y >> mip, MODEL_TEXTURE_SIZE >> mip, MODEL_TEXTURE_SIZE >> mip, GL_RGBA, GL_UNSIGNED_BYTE, this.data[i++]);
-            }
-        }
-    }
-}

src/main/java/me/cortex/voxy/client/core/model/NewModelBufferDelta.java

@@ -1,17 +0,0 @@
-package me.cortex.voxy.client.core.model;
-
-import org.lwjgl.system.MemoryUtil;
-
-public record NewModelBufferDelta(int modelClientId, long modelBufferChangesPtr, int biomeIndex, int[] biomeData, ModelTextureUpload textureUpload) {
-    public static NewModelBufferDelta empty(int modelClientId) {
-        return new NewModelBufferDelta(modelClientId, 0, -1, null, null);
-    }
-
-    public boolean isEmpty() {
-        return this.modelBufferChangesPtr == 0;
-    }
-
-    public void free() {
-        MemoryUtil.nmemFree(this.modelBufferChangesPtr);
-    }
-}

src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java

@@ -1,7 +1,7 @@
 package me.cortex.voxy.client.core.rendering;
 
 import me.cortex.voxy.client.config.VoxyConfig;
-import me.cortex.voxy.client.core.model.OnThreadModelBakerySystem;
+import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
 import me.cortex.voxy.client.core.rendering.building.BuiltSection;
 import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
 import me.cortex.voxy.common.world.WorldEngine;
@@ -10,11 +10,11 @@ import net.minecraft.client.render.Camera;
 import java.util.List;
 
 public class RenderService {
-    private final OnThreadModelBakerySystem modelService;
+    private final ModelBakerySubsystem modelService;
     private final RenderGenerationService renderGen;
 
     public RenderService(WorldEngine world) {
-        this.modelService = new OnThreadModelBakerySystem(world.getMapper());
+        this.modelService = new ModelBakerySubsystem(world.getMapper());
         this.renderGen = new RenderGenerationService(world, this.modelService, VoxyConfig.CONFIG.renderThreads, this::consumeRenderBuildResult, false);
         for(int x = -200; x<=200;x++) {
             for (int z = -200; z <= 200; z++) {
@@ -35,6 +35,12 @@ public class RenderService {
     }
 
     public void renderFarAwayOpaque(Viewport viewport) {
+        //Render previous geometry with the abstract renderer
+        //Execute the hieracial selector
+        // render delta sections
+
+        //Hieracial is not an abstract thing but
+        // the section renderer is as it might have different backends, but they all accept a buffer containing the section list
 
     }
 

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java

@@ -3,7 +3,7 @@ package me.cortex.voxy.client.core.rendering.building;
 import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
 import me.cortex.voxy.client.core.model.IdNotYetComputedException;
-import me.cortex.voxy.client.core.model.OnThreadModelBakerySystem;
+import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.common.world.WorldSection;
 import me.cortex.voxy.common.world.other.Mapper;
@@ -27,12 +27,12 @@ public class RenderGenerationService {
 
     private final Semaphore taskCounter = new Semaphore(0);
     private final WorldEngine world;
-    private final OnThreadModelBakerySystem modelBakery;
+    private final ModelBakerySubsystem modelBakery;
     private final Consumer<BuiltSection> resultConsumer;
     private final BuiltSectionMeshCache meshCache = new BuiltSectionMeshCache();
     private final boolean emitMeshlets;
 
-    public RenderGenerationService(WorldEngine world, OnThreadModelBakerySystem modelBakery, int workers, Consumer<BuiltSection> consumer, boolean emitMeshlets) {
+    public RenderGenerationService(WorldEngine world, ModelBakerySubsystem modelBakery, int workers, Consumer<BuiltSection> consumer, boolean emitMeshlets) {
         this.emitMeshlets = emitMeshlets;
         this.world = world;
         this.modelBakery = modelBakery;

src/main/java/me/cortex/voxy/client/core/rendering/util/RawDownloadStream.java

@@ -0,0 +1,82 @@
+package me.cortex.voxy.client.core.rendering.util;
+
+
+import me.cortex.voxy.client.core.gl.GlBuffer;
+import me.cortex.voxy.client.core.gl.GlFence;
+import me.cortex.voxy.client.core.gl.GlPersistentMappedBuffer;
+import me.cortex.voxy.client.core.util.AllocationArena;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+
+import static org.lwjgl.opengl.ARBMapBufferRange.GL_MAP_READ_BIT;
+import static org.lwjgl.opengl.GL11.glFinish;
+import static org.lwjgl.opengl.GL44.GL_MAP_COHERENT_BIT;
+
+//Special download stream which allows access to the download buffer directly
+public class RawDownloadStream {
+    //NOTE: after the callback returns the pointer is no longer valid for client use
+    public interface IDownloadCompletedCallback{void accept(long ptr);}
+    private record DownloadFragment(int allocation, IDownloadCompletedCallback callback){}
+    private record DownloadFrame(GlFence fence, DownloadFragment[] fragments) {}
+
+    private final GlPersistentMappedBuffer downloadBuffer;
+    private final AllocationArena allocationArena = new AllocationArena();
+    private final ArrayList<DownloadFragment> frameFragments = new ArrayList<>();
+    private final Deque<DownloadFrame> frames = new ArrayDeque<>();
+
+    public RawDownloadStream(int size) {
+        this.downloadBuffer = new GlPersistentMappedBuffer(size, GL_MAP_READ_BIT|GL_MAP_COHERENT_BIT);
+        this.allocationArena.setLimit(size);
+    }
+
+    public int download(int size, IDownloadCompletedCallback callback) {
+        int allocation = (int) this.allocationArena.alloc(size);
+        if (allocation == AllocationArena.SIZE_LIMIT) {
+            //Hit the download limit, attempt to free
+            glFinish();
+            this.tick();
+            allocation = (int) this.allocationArena.alloc(size);
+            if (allocation == AllocationArena.SIZE_LIMIT) {
+                throw new IllegalStateException("Unable free enough memory for raw download stream");
+            }
+        }
+        this.frameFragments.add(new DownloadFragment(allocation, callback));
+        return allocation;
+    }
+
+    //Creates a new "frame" for previously allocated downloads and enqueues a fence
+    // also invalidates all previous download pointers from this instance
+    public void submit() {
+        if (!this.frameFragments.isEmpty()) {
+            var fragments = this.frameFragments.toArray(new DownloadFragment[0]);
+            this.frameFragments.clear();
+            this.frames.add(new DownloadFrame(new GlFence(), fragments));
+        }
+    }
+
+    public void tick() {
+        while (!this.frames.isEmpty()) {
+            //If the first element is not signaled, none of the others will be signaled so break
+            if (!this.frames.peek().fence.signaled()) {
+                break;
+            }
+            var frame = this.frames.poll();
+            for (var fragment : frame.fragments) {
+                long addr = this.downloadBuffer.addr() + fragment.allocation;
+                fragment.callback.accept(addr);
+                this.allocationArena.free(fragment.allocation);
+            }
+            frame.fence.free();
+        }
+    }
+
+    public int getBufferId() {
+        return this.downloadBuffer.id;
+    }
+
+    public void free() {
+        this.downloadBuffer.free();
+    }
+}

src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java

@@ -48,11 +48,14 @@ public class UploadStream {
         if (this.caddr == -1 || !this.allocationArena.expand(this.caddr, (int) size)) {
             this.caddr = this.allocationArena.alloc((int) size);//TODO: replace with allocFromLargest
             if (this.caddr == SIZE_LIMIT) {
-                this.commit();
+                //Note! we dont commit here, we only try to flush existing memory copies, we dont commit
+                // since commit is an explicit op saying we are done any to push upload everything
+                //We dont commit since we dont want to invalidate existing upload pointers
+
                 int attempts = 10;
                 while (--attempts != 0 && this.caddr == SIZE_LIMIT) {
                     glFinish();
-                    this.tick();
+                    this.tick(false);
                     this.caddr = this.allocationArena.alloc((int) size);
                 }
                 if (this.caddr == SIZE_LIMIT) {
@@ -91,7 +94,13 @@ public class UploadStream {
     }
 
     public void tick() {
+        this.tick(true);
+    }
+    private void tick(boolean commit) {
+        if (commit) {
             this.commit();
+        }
+
         if (!this.thisFrameAllocations.isEmpty()) {
             this.frames.add(new UploadFrame(new GlFence(), new LongArrayList(this.thisFrameAllocations)));
             this.thisFrameAllocations.clear();