From 861fa40c6cfb4f4fe10e980f2c9bb2a96c2b7a93 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:20:47 +1000 Subject: [PATCH] Fixed biome colours when loading new biomes, Started on save selector --- src/main/java/me/cortex/voxy/client/Test.java | 60 ---------- src/main/java/me/cortex/voxy/client/Voxy.java | 21 ++++ .../config/VoxyConfigScreenFactory.java | 2 +- .../voxy/client/{ => core}/IGetVoxelCore.java | 2 +- .../me/cortex/voxy/client/core/VoxelCore.java | 7 +- .../rendering/AbstractFarWorldRenderer.java | 33 +++++- .../minecraft/MixinClientChunkManager.java | 9 +- .../client/mixin/minecraft/MixinDebugHud.java | 2 +- .../mixin/minecraft/MixinWorldRenderer.java | 20 +++- .../client/saver/SaveSelectionSystem.java | 28 +++++ .../client/terrain/WorldImportCommand.java | 2 +- .../inmemory/MemoryStorageBackend.java | 108 ++++++++++++++++++ src/main/resources/voxy.accesswidener | 1 + 13 files changed, 217 insertions(+), 78 deletions(-) delete mode 100644 src/main/java/me/cortex/voxy/client/Test.java rename src/main/java/me/cortex/voxy/client/{ => core}/IGetVoxelCore.java (79%) create mode 100644 src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java create mode 100644 src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java diff --git a/src/main/java/me/cortex/voxy/client/Test.java b/src/main/java/me/cortex/voxy/client/Test.java deleted file mode 100644 index 7360d967..00000000 --- a/src/main/java/me/cortex/voxy/client/Test.java +++ /dev/null @@ -1,60 +0,0 @@ -package me.cortex.voxy.client; - -import me.cortex.voxy.common.storage.lmdb.LMDBInterface; -import me.cortex.voxy.client.importers.WorldImporter; -import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend; -import org.lwjgl.system.MemoryUtil; - -import java.io.File; -import java.nio.ByteBuffer; - -import static org.lwjgl.util.lmdb.LMDB.MDB_NOLOCK; -import static org.lwjgl.util.lmdb.LMDB.MDB_NOSUBDIR; - -public class Test { - public static void main1(String[] args) { - var dbi = new LMDBInterface.Builder() - .setMaxDbs(1) - .open("testdbdir.db", MDB_NOLOCK | MDB_NOSUBDIR) - .fetch(); - dbi.setMapSize(1<<29); - var db = dbi.createDb(null); - db.transaction(obj->{ - var key = ByteBuffer.allocateDirect(4); - var val = ByteBuffer.allocateDirect(1); - for (int i = 0; i < 1<<20; i++) { - key.putInt(0, i); - obj.put(key, val, 0); - } - return 1; - }); - db.close(); - dbi.close(); - } - - public static void main2(String[] args) throws Exception { - var storage = new LMDBStorageBackend(new File("run/storagefile.db")); - for (int i = 0; i < 2; i++) { - new Thread(()->{ - //storage.getSectionData(1143914312599863680L); - storage.setSectionData(1143914312599863680L, MemoryUtil.memAlloc(12345)); - }).start(); - } - //storage.getSectionData(1143914312599863680L); - //storage.setSectionData(1143914312599863612L, ByteBuffer.allocateDirect(12345)); - //storage.setSectionData(1143914312599863680L, ByteBuffer.allocateDirect(12345)); - //storage.close(); - - System.out.println(storage.getIdMappingsData()); - storage.putIdMapping(1, ByteBuffer.allocateDirect(12)); - - Thread.sleep(1000); - storage.close(); - } - - public static void main(String[] args) { - //WorldEngine engine = new WorldEngine(new File("storagefile2.db"), 5); - WorldImporter importer = new WorldImporter(null, null); - //importer.importWorld(new File("run/saves/Drehmal 2.2 Apotheosis Beta - 1.0.0/region/")); - } -} diff --git a/src/main/java/me/cortex/voxy/client/Voxy.java b/src/main/java/me/cortex/voxy/client/Voxy.java index 273cb004..f164b3bc 100644 --- a/src/main/java/me/cortex/voxy/client/Voxy.java +++ b/src/main/java/me/cortex/voxy/client/Voxy.java @@ -1,8 +1,21 @@ package me.cortex.voxy.client; +import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.client.core.VoxelCore; +import me.cortex.voxy.client.mixin.minecraft.MixinWorldRenderer; import me.cortex.voxy.client.terrain.WorldImportCommand; +import me.cortex.voxy.common.storage.CompressionStorageAdaptor; +import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor; +import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.ZSTDCompressor; +import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend; +import me.cortex.voxy.common.world.WorldEngine; import net.fabricmc.api.ClientModInitializer; import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback; +import net.minecraft.client.render.WorldRenderer; +import net.minecraft.client.world.ClientWorld; + +import java.io.File; public class Voxy implements ClientModInitializer { @Override @@ -11,4 +24,12 @@ public class Voxy implements ClientModInitializer { dispatcher.register(WorldImportCommand.register()); }); } + + public static VoxelCore createVoxelCore(ClientWorld world) { + StorageBackend storage = new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath)); + //StorageBackend storage = new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath)); + storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), storage); + var engine = new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); + return new VoxelCore(engine); + } } diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java b/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java index ce55ddc7..aef588ce 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java +++ b/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java @@ -2,7 +2,7 @@ package me.cortex.voxy.client.config; import com.terraformersmc.modmenu.api.ConfigScreenFactory; import com.terraformersmc.modmenu.api.ModMenuApi; -import me.cortex.voxy.client.IGetVoxelCore; +import me.cortex.voxy.client.core.IGetVoxelCore; import me.shedaniel.clothconfig2.api.ConfigBuilder; import me.shedaniel.clothconfig2.api.ConfigCategory; import me.shedaniel.clothconfig2.api.ConfigEntryBuilder; diff --git a/src/main/java/me/cortex/voxy/client/IGetVoxelCore.java b/src/main/java/me/cortex/voxy/client/core/IGetVoxelCore.java similarity index 79% rename from src/main/java/me/cortex/voxy/client/IGetVoxelCore.java rename to src/main/java/me/cortex/voxy/client/core/IGetVoxelCore.java index db3fbaaf..e0689363 100644 --- a/src/main/java/me/cortex/voxy/client/IGetVoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/IGetVoxelCore.java @@ -1,4 +1,4 @@ -package me.cortex.voxy.client; +package me.cortex.voxy.client.core; import me.cortex.voxy.client.core.VoxelCore; diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index d589e65a..e4bd5b64 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -50,15 +50,14 @@ public class VoxelCore { //private final Thread shutdownThread = new Thread(this::shutdown); - public VoxelCore() { + public VoxelCore(WorldEngine engine) { + this.world = engine; System.out.println("Initializing voxy core"); //Trigger the shared index buffer loading SharedIndexBuffer.INSTANCE.id(); this.renderer = new Gl46FarWorldRenderer(VoxyConfig.CONFIG.geometryBufferSize, VoxyConfig.CONFIG.maxSections); System.out.println("Renderer initialized"); - this.world = new WorldEngine(new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath))), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); - System.out.println("World engine"); this.renderTracker = new RenderTracker(this.world, this.renderer); this.renderGen = new RenderGenerationService(this.world, this.renderer.getModelManager(), VoxyConfig.CONFIG.renderThreads, this.renderTracker::processBuildResult); @@ -74,7 +73,7 @@ public class VoxelCore { this.postProcessing = new PostProcessing(); - this.world.getMapper().setCallbacks(this.renderer::addBlockState, a->{}); + this.world.getMapper().setCallbacks(this.renderer::addBlockState, this.renderer::addBiome); ////Resave the db incase it failed a recovery diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java b/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java index d094de11..905f5525 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java @@ -13,6 +13,10 @@ import net.minecraft.client.MinecraftClient; import net.minecraft.client.render.Camera; import net.minecraft.client.render.Frustum; import net.minecraft.client.util.math.MatrixStack; +import net.minecraft.registry.RegistryKeys; +import net.minecraft.util.Identifier; +import net.minecraft.world.biome.Biome; +import net.minecraft.world.biome.BiomeKeys; import org.joml.FrustumIntersection; import org.joml.Matrix4f; import org.lwjgl.system.MemoryUtil; @@ -48,6 +52,7 @@ public abstract class AbstractFarWorldRenderer { protected FrustumIntersection frustum; private final ConcurrentLinkedDeque blockStateUpdates = new ConcurrentLinkedDeque<>(); + private final ConcurrentLinkedDeque biomeUpdates = new ConcurrentLinkedDeque<>(); public AbstractFarWorldRenderer(int geometrySize, int maxSections) { this.uniformBuffer = new GlBuffer(1024); this.lightDataBuffer = new GlBuffer(256*4);//256 of uint @@ -85,12 +90,28 @@ public abstract class AbstractFarWorldRenderer { //Upload any new geometry this.geometry.uploadResults(); + { + boolean didHaveBiomeChange = false; - //Do any BlockChanges - while (!this.blockStateUpdates.isEmpty()) { - var update = this.blockStateUpdates.pop(); - this.models.addEntry(update.id, update.state); + //Do any BiomeChanges + while (!this.biomeUpdates.isEmpty()) { + var update = this.biomeUpdates.pop(); + var biomeReg = MinecraftClient.getInstance().world.getRegistryManager().get(RegistryKeys.BIOME); + this.models.addBiome(update.id, biomeReg.get(new Identifier(update.biome))); + didHaveBiomeChange = true; + } + + if (didHaveBiomeChange) { + UploadStream.INSTANCE.commit(); + } + + //Do any BlockChanges + while (!this.blockStateUpdates.isEmpty()) { + var update = this.blockStateUpdates.pop(); + this.models.addEntry(update.id, update.state); + } } + } public abstract void renderFarAwayOpaque(Matrix4f projection, MatrixStack stack, double cx, double cy, double cz); @@ -105,6 +126,10 @@ public abstract class AbstractFarWorldRenderer { this.blockStateUpdates.add(entry); } + public void addBiome(Mapper.BiomeEntry entry) { + this.biomeUpdates.add(entry); + } + public void addDebugData(List debug) { this.models.addDebugInfo(debug); } diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkManager.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkManager.java index 4e5c9e0c..70e7a585 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkManager.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkManager.java @@ -1,11 +1,14 @@ package me.cortex.voxy.client.mixin.minecraft; -import me.cortex.voxy.client.IGetVoxelCore; +import me.cortex.voxy.client.core.IGetVoxelCore; import net.minecraft.client.MinecraftClient; import net.minecraft.client.world.ClientChunkManager; +import net.minecraft.client.world.ClientWorld; import net.minecraft.util.math.ChunkPos; import net.minecraft.world.chunk.WorldChunk; +import org.spongepowered.asm.mixin.Final; import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.Shadow; import org.spongepowered.asm.mixin.injection.At; import org.spongepowered.asm.mixin.injection.Inject; import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; @@ -13,9 +16,11 @@ import org.spongepowered.asm.mixin.injection.callback.LocalCapture; @Mixin(ClientChunkManager.class) public class MixinClientChunkManager { + @Shadow @Final ClientWorld world; + @Inject(require = 0, method = "unload", at = @At(value = "INVOKE", target = "Lnet/minecraft/client/world/ClientChunkManager$ClientChunkMap;compareAndSet(ILnet/minecraft/world/chunk/WorldChunk;Lnet/minecraft/world/chunk/WorldChunk;)Lnet/minecraft/world/chunk/WorldChunk;", shift = At.Shift.BEFORE), locals = LocalCapture.CAPTURE_FAILHARD) private void injectUnload(ChunkPos pos, CallbackInfo ci, int index, WorldChunk worldChunk) { - var core = ((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore(); + var core = ((IGetVoxelCore)(world.worldRenderer)).getVoxelCore(); if (core != null) { core.enqueueIngest(worldChunk); } diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinDebugHud.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinDebugHud.java index ac803c3c..0a968da8 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinDebugHud.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinDebugHud.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.mixin.minecraft; -import me.cortex.voxy.client.IGetVoxelCore; +import me.cortex.voxy.client.core.IGetVoxelCore; import net.minecraft.client.MinecraftClient; import net.minecraft.client.gui.hud.DebugHud; import org.spongepowered.asm.mixin.Mixin; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java index 225de17c..6e9e4c97 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java @@ -1,6 +1,7 @@ package me.cortex.voxy.client.mixin.minecraft; -import me.cortex.voxy.client.IGetVoxelCore; +import me.cortex.voxy.client.Voxy; +import me.cortex.voxy.client.core.IGetVoxelCore; import me.cortex.voxy.client.config.VoxyConfig; import me.cortex.voxy.client.core.VoxelCore; import net.minecraft.client.render.*; @@ -42,6 +43,14 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore { } } + @Unique + public void populateCore() { + if (this.core != null) { + throw new IllegalStateException("Trying to create new core while a core already exists"); + } + this.core = Voxy.createVoxelCore(this.world); + } + public VoxelCore getVoxelCore() { return this.core; } @@ -50,7 +59,10 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore { private void resetVoxelCore(CallbackInfo ci) { if (this.world != null && this.core != null) { this.core.shutdown(); - this.core = new VoxelCore(); + this.core = null; + if (VoxyConfig.CONFIG.enabled) { + this.populateCore(); + } } } @@ -69,7 +81,7 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore { this.core = null; } if (VoxyConfig.CONFIG.enabled) { - this.core = new VoxelCore(); + this.populateCore(); } } @@ -80,7 +92,7 @@ public abstract class MixinWorldRenderer implements IGetVoxelCore { this.core = null; } if (this.world != null && VoxyConfig.CONFIG.enabled) { - this.core = new VoxelCore(); + this.populateCore(); } } diff --git a/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java b/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java new file mode 100644 index 00000000..86dd3443 --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java @@ -0,0 +1,28 @@ +package me.cortex.voxy.client.saver; + +import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.storage.CompressionStorageAdaptor; +import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor; +import me.cortex.voxy.common.storage.ZSTDCompressor; +import me.cortex.voxy.common.world.WorldEngine; +import net.minecraft.client.MinecraftClient; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; + +//Sets up a world engine with respect to the world the client is currently loaded into +// this is a bit tricky as each world has its own config, e.g. storage configuration +public class SaveSelectionSystem { + + //The way this works is saves are segmented into base worlds, e.g. server ip, local save etc + // these are then segmented into subsaves for different worlds within the parent + public SaveSelectionSystem(List storagePaths) { + + } + + public WorldEngine createWorldEngine() { + var storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath))); + return new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); + } +} diff --git a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java index 3f743dd3..68a5f96f 100644 --- a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java +++ b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java @@ -3,7 +3,7 @@ package me.cortex.voxy.client.terrain; import com.mojang.brigadier.arguments.StringArgumentType; import com.mojang.brigadier.builder.LiteralArgumentBuilder; import com.mojang.brigadier.context.CommandContext; -import me.cortex.voxy.client.IGetVoxelCore; +import me.cortex.voxy.client.core.IGetVoxelCore; import me.cortex.voxy.client.importers.WorldImporter; import net.fabricmc.fabric.api.client.command.v2.ClientCommandManager; import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource; diff --git a/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java new file mode 100644 index 00000000..098da75a --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java @@ -0,0 +1,108 @@ +package me.cortex.voxy.common.storage.inmemory; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectCollection; +import me.cortex.voxy.common.storage.StorageBackend; +import net.minecraft.util.math.random.RandomSeed; +import org.apache.commons.lang3.stream.Streams; +import org.lwjgl.system.MemoryUtil; + +import java.nio.ByteBuffer; +import java.util.stream.Stream; + +public class MemoryStorageBackend extends StorageBackend { + private final Long2ObjectMap[] maps; + private final Int2ObjectMap idMappings = new Int2ObjectOpenHashMap<>(); + + public MemoryStorageBackend() { + this(4); + } + + public MemoryStorageBackend(int slicesBitCount) { + this.maps = new Long2ObjectMap[1<(); + } + } + + private Long2ObjectMap getMap(long key) { + return this.maps[(int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&(this.maps.length-1))]; + } + + @Override + public ByteBuffer getSectionData(long key) { + var map = this.getMap(key); + synchronized (map) { + var data = map.get(key); + if (data != null) { + var cpy = MemoryUtil.memAlloc(data.remaining()); + MemoryUtil.memCopy(data, cpy); + return cpy; + } else { + return null; + } + } + } + + @Override + public void setSectionData(long key, ByteBuffer data) { + var map = this.getMap(key); + synchronized (map) { + var old = map.put(key, data); + if (old != null) { + MemoryUtil.memFree(old); + } + } + } + + @Override + public void deleteSectionData(long key) { + var map = this.getMap(key); + synchronized (map) { + var data = map.remove(key); + if (data != null) { + MemoryUtil.memFree(data); + } + } + } + + @Override + public void putIdMapping(int id, ByteBuffer data) { + synchronized (this.idMappings) { + var cpy = MemoryUtil.memAlloc(data.remaining()); + MemoryUtil.memCopy(data, cpy); + var prev = this.idMappings.put(id, cpy); + if (prev != null) { + MemoryUtil.memFree(prev); + } + } + } + + @Override + public Int2ObjectOpenHashMap getIdMappingsData() { + Int2ObjectOpenHashMap out = new Int2ObjectOpenHashMap<>(); + synchronized (this.idMappings) { + for (var entry : this.idMappings.int2ObjectEntrySet()) { + var buf = new byte[entry.getValue().remaining()]; + entry.getValue().get(buf); + entry.getValue().rewind(); + out.put(entry.getIntKey(), buf); + } + return out; + } + } + + @Override + public void flush() { + + } + + @Override + public void close() { + Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryUtil::memFree); + this.idMappings.values().forEach(MemoryUtil::memFree); + } +} diff --git a/src/main/resources/voxy.accesswidener b/src/main/resources/voxy.accesswidener index 791ced83..6e0d3862 100644 --- a/src/main/resources/voxy.accesswidener +++ b/src/main/resources/voxy.accesswidener @@ -7,5 +7,6 @@ accessible field net/minecraft/client/color/block/BlockColors providers Lnet/min accessible field net/minecraft/client/render/GameRenderer zoomX F accessible field net/minecraft/client/render/GameRenderer zoomY F accessible field net/minecraft/client/render/GameRenderer zoom F +accessible field net/minecraft/client/world/ClientWorld worldRenderer Lnet/minecraft/client/render/WorldRenderer; accessible method net/minecraft/client/render/GameRenderer getFov (Lnet/minecraft/client/render/Camera;FZ)D \ No newline at end of file