Fixed biome colours when loading new biomes, Started on save selector

src/main/java/me/cortex/voxy/client/Test.java

@@ -1,60 +0,0 @@
-package me.cortex.voxy.client;
-
-import me.cortex.voxy.common.storage.lmdb.LMDBInterface;
-import me.cortex.voxy.client.importers.WorldImporter;
-import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend;
-import org.lwjgl.system.MemoryUtil;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import static org.lwjgl.util.lmdb.LMDB.MDB_NOLOCK;
-import static org.lwjgl.util.lmdb.LMDB.MDB_NOSUBDIR;
-
-public class Test {
-    public static void main1(String[] args) {
-        var dbi = new LMDBInterface.Builder()
-                .setMaxDbs(1)
-                .open("testdbdir.db", MDB_NOLOCK | MDB_NOSUBDIR)
-                .fetch();
-        dbi.setMapSize(1<<29);
-        var db = dbi.createDb(null);
-        db.transaction(obj->{
-            var key = ByteBuffer.allocateDirect(4);
-            var val = ByteBuffer.allocateDirect(1);
-            for (int i = 0; i < 1<<20; i++) {
-                key.putInt(0, i);
-                obj.put(key, val, 0);
-            }
-            return 1;
-        });
-        db.close();
-        dbi.close();
-    }
-
-    public static void main2(String[] args) throws Exception {
-        var storage = new LMDBStorageBackend(new File("run/storagefile.db"));
-        for (int i = 0; i < 2; i++) {
-            new Thread(()->{
-                //storage.getSectionData(1143914312599863680L);
-                storage.setSectionData(1143914312599863680L, MemoryUtil.memAlloc(12345));
-            }).start();
-        }
-        //storage.getSectionData(1143914312599863680L);
-        //storage.setSectionData(1143914312599863612L, ByteBuffer.allocateDirect(12345));
-        //storage.setSectionData(1143914312599863680L, ByteBuffer.allocateDirect(12345));
-        //storage.close();
-
-        System.out.println(storage.getIdMappingsData());
-        storage.putIdMapping(1, ByteBuffer.allocateDirect(12));
-
-        Thread.sleep(1000);
-        storage.close();
-    }
-
-    public static void main(String[] args) {
-        //WorldEngine engine = new WorldEngine(new File("storagefile2.db"), 5);
-        WorldImporter importer = new WorldImporter(null, null);
-        //importer.importWorld(new File("run/saves/Drehmal 2.2 Apotheosis Beta - 1.0.0/region/"));
-    }
-}

src/main/java/me/cortex/voxy/client/Voxy.java

@@ -1,8 +1,21 @@
 package me.cortex.voxy.client;
 
+import me.cortex.voxy.client.config.VoxyConfig;
+import me.cortex.voxy.client.core.VoxelCore;
+import me.cortex.voxy.client.mixin.minecraft.MixinWorldRenderer;
 import me.cortex.voxy.client.terrain.WorldImportCommand;
+import me.cortex.voxy.common.storage.CompressionStorageAdaptor;
+import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor;
+import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.storage.ZSTDCompressor;
+import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend;
+import me.cortex.voxy.common.world.WorldEngine;
 import net.fabricmc.api.ClientModInitializer;
         import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback;
+import net.minecraft.client.render.WorldRenderer;
+import net.minecraft.client.world.ClientWorld;
+
+import java.io.File;
 
 public class Voxy implements ClientModInitializer {
     @Override
@@ -11,4 +24,12 @@ public class Voxy implements ClientModInitializer {
             dispatcher.register(WorldImportCommand.register());
         });
     }
+
+    public static VoxelCore createVoxelCore(ClientWorld world) {
+        StorageBackend storage = new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath));
+        //StorageBackend storage = new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath));
+        storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), storage);
+        var engine = new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
+        return new VoxelCore(engine);
+    }
 }

src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java

@@ -2,7 +2,7 @@ package me.cortex.voxy.client.config;
 
 import com.terraformersmc.modmenu.api.ConfigScreenFactory;
 import com.terraformersmc.modmenu.api.ModMenuApi;
-import me.cortex.voxy.client.IGetVoxelCore;
+import me.cortex.voxy.client.core.IGetVoxelCore;
 import me.shedaniel.clothconfig2.api.ConfigBuilder;
 import me.shedaniel.clothconfig2.api.ConfigCategory;
 import me.shedaniel.clothconfig2.api.ConfigEntryBuilder;

src/main/java/me/cortex/voxy/client/core/IGetVoxelCore.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.client;
+package me.cortex.voxy.client.core;
 
 import me.cortex.voxy.client.core.VoxelCore;
 

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -50,15 +50,14 @@ public class VoxelCore {
 
     //private final Thread shutdownThread = new Thread(this::shutdown);
 
-    public VoxelCore() {
+    public VoxelCore(WorldEngine engine) {
+        this.world = engine;
         System.out.println("Initializing voxy core");
 
         //Trigger the shared index buffer loading
         SharedIndexBuffer.INSTANCE.id();
         this.renderer = new Gl46FarWorldRenderer(VoxyConfig.CONFIG.geometryBufferSize, VoxyConfig.CONFIG.maxSections);
         System.out.println("Renderer initialized");
-        this.world = new WorldEngine(new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath))), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
-        System.out.println("World engine");
 
         this.renderTracker = new RenderTracker(this.world, this.renderer);
         this.renderGen = new RenderGenerationService(this.world, this.renderer.getModelManager(), VoxyConfig.CONFIG.renderThreads, this.renderTracker::processBuildResult);
@@ -74,7 +73,7 @@ public class VoxelCore {
 
         this.postProcessing = new PostProcessing();
 
-        this.world.getMapper().setCallbacks(this.renderer::addBlockState, a->{});
+        this.world.getMapper().setCallbacks(this.renderer::addBlockState, this.renderer::addBiome);
 
 
         ////Resave the db incase it failed a recovery

src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java

@@ -13,6 +13,10 @@ import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.render.Camera;
 import net.minecraft.client.render.Frustum;
 import net.minecraft.client.util.math.MatrixStack;
+import net.minecraft.registry.RegistryKeys;
+import net.minecraft.util.Identifier;
+import net.minecraft.world.biome.Biome;
+import net.minecraft.world.biome.BiomeKeys;
 import org.joml.FrustumIntersection;
 import org.joml.Matrix4f;
 import org.lwjgl.system.MemoryUtil;
@@ -48,6 +52,7 @@ public abstract class AbstractFarWorldRenderer {
     protected FrustumIntersection frustum;
 
     private final ConcurrentLinkedDeque<Mapper.StateEntry> blockStateUpdates = new ConcurrentLinkedDeque<>();
+    private final ConcurrentLinkedDeque<Mapper.BiomeEntry> biomeUpdates = new ConcurrentLinkedDeque<>();
     public AbstractFarWorldRenderer(int geometrySize, int maxSections) {
         this.uniformBuffer  = new GlBuffer(1024);
         this.lightDataBuffer  = new GlBuffer(256*4);//256 of uint
@@ -85,6 +90,20 @@ public abstract class AbstractFarWorldRenderer {
 
         //Upload any new geometry
         this.geometry.uploadResults();
+        {
+            boolean didHaveBiomeChange = false;
+
+            //Do any BiomeChanges
+            while (!this.biomeUpdates.isEmpty()) {
+                var update = this.biomeUpdates.pop();
+                var biomeReg = MinecraftClient.getInstance().world.getRegistryManager().get(RegistryKeys.BIOME);
+                this.models.addBiome(update.id, biomeReg.get(new Identifier(update.biome)));
+                didHaveBiomeChange = true;
+            }
+
+            if (didHaveBiomeChange) {
+                UploadStream.INSTANCE.commit();
+            }
 
             //Do any BlockChanges
             while (!this.blockStateUpdates.isEmpty()) {
@@ -93,6 +112,8 @@ public abstract class AbstractFarWorldRenderer {
             }
         }
 
+    }
+
     public abstract void renderFarAwayOpaque(Matrix4f projection, MatrixStack stack, double cx, double cy, double cz);
 
     public abstract void renderFarAwayTranslucent();
@@ -105,6 +126,10 @@ public abstract class AbstractFarWorldRenderer {
         this.blockStateUpdates.add(entry);
     }
 
+    public void addBiome(Mapper.BiomeEntry entry) {
+        this.biomeUpdates.add(entry);
+    }
+
     public void addDebugData(List<String> debug) {
         this.models.addDebugInfo(debug);
     }

src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkManager.java

@@ -1,11 +1,14 @@
 package me.cortex.voxy.client.mixin.minecraft;
 
-import me.cortex.voxy.client.IGetVoxelCore;
+import me.cortex.voxy.client.core.IGetVoxelCore;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.world.ClientChunkManager;
+import net.minecraft.client.world.ClientWorld;
 import net.minecraft.util.math.ChunkPos;
 import net.minecraft.world.chunk.WorldChunk;
+import org.spongepowered.asm.mixin.Final;
 import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
 import org.spongepowered.asm.mixin.injection.At;
 import org.spongepowered.asm.mixin.injection.Inject;
 import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
@@ -13,9 +16,11 @@ import org.spongepowered.asm.mixin.injection.callback.LocalCapture;
 
 @Mixin(ClientChunkManager.class)
 public class MixinClientChunkManager {
+    @Shadow @Final ClientWorld world;
+
     @Inject(require = 0, method = "unload", at = @At(value = "INVOKE", target = "Lnet/minecraft/client/world/ClientChunkManager$ClientChunkMap;compareAndSet(ILnet/minecraft/world/chunk/WorldChunk;Lnet/minecraft/world/chunk/WorldChunk;)Lnet/minecraft/world/chunk/WorldChunk;", shift = At.Shift.BEFORE), locals = LocalCapture.CAPTURE_FAILHARD)
     private void injectUnload(ChunkPos pos, CallbackInfo ci, int index, WorldChunk worldChunk) {
-        var core = ((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore();
+        var core = ((IGetVoxelCore)(world.worldRenderer)).getVoxelCore();
         if (core != null) {
             core.enqueueIngest(worldChunk);
         }

src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinDebugHud.java

@@ -1,6 +1,6 @@
 package me.cortex.voxy.client.mixin.minecraft;
 
-import me.cortex.voxy.client.IGetVoxelCore;
+import me.cortex.voxy.client.core.IGetVoxelCore;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.gui.hud.DebugHud;
 import org.spongepowered.asm.mixin.Mixin;

src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java

@@ -1,6 +1,7 @@
 package me.cortex.voxy.client.mixin.minecraft;
 
-import me.cortex.voxy.client.IGetVoxelCore;
+import me.cortex.voxy.client.Voxy;
+import me.cortex.voxy.client.core.IGetVoxelCore;
 import me.cortex.voxy.client.config.VoxyConfig;
 import me.cortex.voxy.client.core.VoxelCore;
 import net.minecraft.client.render.*;
@@ -42,6 +43,14 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore {
         }
     }
 
+    @Unique
+    public void populateCore() {
+        if (this.core != null) {
+            throw new IllegalStateException("Trying to create new core while a core already exists");
+        }
+        this.core = Voxy.createVoxelCore(this.world);
+    }
+
     public VoxelCore getVoxelCore() {
         return this.core;
     }
@@ -50,7 +59,10 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore {
     private void resetVoxelCore(CallbackInfo ci) {
         if (this.world != null && this.core != null) {
             this.core.shutdown();
-            this.core = new VoxelCore();
+            this.core = null;
+            if (VoxyConfig.CONFIG.enabled) {
+                this.populateCore();
+            }
         }
     }
 
@@ -69,7 +81,7 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore {
             this.core = null;
         }
         if (VoxyConfig.CONFIG.enabled) {
-            this.core = new VoxelCore();
+            this.populateCore();
         }
     }
 
@@ -80,7 +92,7 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore {
             this.core = null;
         }
         if (this.world != null && VoxyConfig.CONFIG.enabled) {
-            this.core = new VoxelCore();
+            this.populateCore();
         }
     }
 

src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java

@@ -0,0 +1,28 @@
+package me.cortex.voxy.client.saver;
+
+import me.cortex.voxy.client.config.VoxyConfig;
+import me.cortex.voxy.common.storage.CompressionStorageAdaptor;
+import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor;
+import me.cortex.voxy.common.storage.ZSTDCompressor;
+import me.cortex.voxy.common.world.WorldEngine;
+import net.minecraft.client.MinecraftClient;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+
+//Sets up a world engine with respect to the world the client is currently loaded into
+// this is a bit tricky as each world has its own config, e.g. storage configuration
+public class SaveSelectionSystem {
+
+    //The way this works is saves are segmented into base worlds, e.g. server ip, local save etc
+    // these are then segmented into subsaves for different worlds within the parent
+    public SaveSelectionSystem(List<Path> storagePaths) {
+
+    }
+
+    public WorldEngine createWorldEngine() {
+        var storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath)));
+        return new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
+    }
+}

src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java

@@ -3,7 +3,7 @@ package me.cortex.voxy.client.terrain;
 import com.mojang.brigadier.arguments.StringArgumentType;
 import com.mojang.brigadier.builder.LiteralArgumentBuilder;
 import com.mojang.brigadier.context.CommandContext;
-import me.cortex.voxy.client.IGetVoxelCore;
+import me.cortex.voxy.client.core.IGetVoxelCore;
 import me.cortex.voxy.client.importers.WorldImporter;
 import net.fabricmc.fabric.api.client.command.v2.ClientCommandManager;
 import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource;

src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java

@@ -0,0 +1,108 @@
+package me.cortex.voxy.common.storage.inmemory;
+
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectCollection;
+import me.cortex.voxy.common.storage.StorageBackend;
+import net.minecraft.util.math.random.RandomSeed;
+import org.apache.commons.lang3.stream.Streams;
+import org.lwjgl.system.MemoryUtil;
+
+import java.nio.ByteBuffer;
+import java.util.stream.Stream;
+
+public class MemoryStorageBackend extends StorageBackend {
+    private final Long2ObjectMap<ByteBuffer>[] maps;
+    private final Int2ObjectMap<ByteBuffer> idMappings = new Int2ObjectOpenHashMap<>();
+
+    public MemoryStorageBackend() {
+        this(4);
+    }
+
+    public MemoryStorageBackend(int slicesBitCount) {
+        this.maps = new Long2ObjectMap[1<<slicesBitCount];
+        for (int i = 0; i < this.maps.length; i++) {
+            this.maps[i] = new Long2ObjectOpenHashMap<>();
+        }
+    }
+
+    private Long2ObjectMap<ByteBuffer> getMap(long key) {
+        return this.maps[(int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&(this.maps.length-1))];
+    }
+
+    @Override
+    public ByteBuffer getSectionData(long key) {
+        var map = this.getMap(key);
+        synchronized (map) {
+            var data = map.get(key);
+            if (data != null) {
+                var cpy = MemoryUtil.memAlloc(data.remaining());
+                MemoryUtil.memCopy(data, cpy);
+                return cpy;
+            } else {
+                return null;
+            }
+        }
+    }
+
+    @Override
+    public void setSectionData(long key, ByteBuffer data) {
+        var map = this.getMap(key);
+        synchronized (map) {
+            var old = map.put(key, data);
+            if (old != null) {
+                MemoryUtil.memFree(old);
+            }
+        }
+    }
+
+    @Override
+    public void deleteSectionData(long key) {
+        var map = this.getMap(key);
+        synchronized (map) {
+            var data = map.remove(key);
+            if (data != null) {
+                MemoryUtil.memFree(data);
+            }
+        }
+    }
+
+    @Override
+    public void putIdMapping(int id, ByteBuffer data) {
+        synchronized (this.idMappings) {
+            var cpy = MemoryUtil.memAlloc(data.remaining());
+            MemoryUtil.memCopy(data, cpy);
+            var prev = this.idMappings.put(id, cpy);
+            if (prev != null) {
+                MemoryUtil.memFree(prev);
+            }
+        }
+    }
+
+    @Override
+    public Int2ObjectOpenHashMap<byte[]> getIdMappingsData() {
+        Int2ObjectOpenHashMap<byte[]> out = new Int2ObjectOpenHashMap<>();
+        synchronized (this.idMappings) {
+            for (var entry : this.idMappings.int2ObjectEntrySet()) {
+                var buf = new byte[entry.getValue().remaining()];
+                entry.getValue().get(buf);
+                entry.getValue().rewind();
+                out.put(entry.getIntKey(), buf);
+            }
+            return out;
+        }
+    }
+
+    @Override
+    public void flush() {
+
+    }
+
+    @Override
+    public void close() {
+        Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryUtil::memFree);
+        this.idMappings.values().forEach(MemoryUtil::memFree);
+    }
+}

src/main/resources/voxy.accesswidener

@@ -7,5 +7,6 @@ accessible field net/minecraft/client/color/block/BlockColors providers Lnet/min
 accessible field net/minecraft/client/render/GameRenderer zoomX F
 accessible field net/minecraft/client/render/GameRenderer zoomY F
 accessible field net/minecraft/client/render/GameRenderer zoom F
+accessible field net/minecraft/client/world/ClientWorld worldRenderer Lnet/minecraft/client/render/WorldRenderer;
 
 accessible method net/minecraft/client/render/GameRenderer getFov (Lnet/minecraft/client/render/Camera;FZ)D