From 2236bf19a5b793a6f6e257eab279aef130e4c579 Mon Sep 17 00:00:00 2001 From: spdis Date: Tue, 6 Jan 2026 02:01:35 +0800 Subject: [PATCH] =?UTF-8?q?=E7=AC=AC=E4=B8=80=E4=B8=AA=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 7 +- .../me/cortex/voxy/client/VoxyClient.java | 38 ++++ .../voxy/client/VoxyClientInstance.java | 53 ++++- .../voxy/client/config/VoxyConfigMenu.java | 1 + .../client/core/NormalRenderPipeline.java | 2 +- .../voxy/client/core/VoxyRenderSystem.java | 13 +- .../HierarchicalOcclusionTraverser.java | 2 +- .../cortex/voxy/client/iris/VoxyUniforms.java | 2 +- .../mixin/iris/MixinMatrixUniforms.java | 2 +- .../client/mixin/iris/MixinProgramSet.java | 2 +- .../mixin/iris/MixinStandardMacros.java | 2 +- .../minecraft/MixinClientChunkCache.java | 2 +- .../mixin/minecraft/MixinClientLevel.java | 2 +- .../minecraft/MixinClientPacketListener.java | 2 +- .../mixin/minecraft/MixinFogRenderer.java | 2 +- .../mixin/minecraft/MixinLevelRenderer.java | 2 +- .../mixin/sodium/MixinChunkJobQueue.java | 2 +- .../sodium/MixinRenderRegionManager.java | 2 +- .../sodium/MixinRenderSectionManager.java | 2 +- .../voxy/common/config/Serialization.java | 6 + .../{client => common}/config/VoxyConfig.java | 2 +- .../voxy/common/config/VoxyServerConfig.java | 85 ++++++++ .../config/compressors/LZ4Compressor.java | 6 +- .../config/compressors/ZSTDCompressor.java | 53 ++--- .../inmemory/MemoryStorageBackend.java | 11 +- .../storage/lmdb/LMDBStorageBackend.java | 8 +- .../rocksdb/RocksDBStorageBackend.java | 115 ++++++++--- .../common/network/SectionSerializer.java | 162 ++++++++++++++++ .../voxy/common/network/VoxyNetwork.java | 62 ++++++ .../thread/UnifiedServiceThreadPool.java | 13 ++ .../cortex/voxy/common/util/MemoryBuffer.java | 14 +- .../cortex/voxy/common/util/UnsafeUtil.java | 90 +++++++++ .../voxy/common/util/cpu/CpuLayout.java | 13 +- .../voxelization/WorldConversionFactory.java | 21 +- .../voxy/common/world/SaveLoadSystem.java | 23 ++- .../voxy/common/world/SaveLoadSystem2.java | 63 +++--- .../voxy/common/world/SaveLoadSystem3.java | 20 +- .../voxy/common/world/other/Mapper.java | 31 ++- .../world/service/VoxelIngestService.java | 47 +++-- .../me/cortex/voxy/commonImpl/VoxyCommon.java | 17 ++ .../cortex/voxy/commonImpl/VoxyInstance.java | 6 + .../voxy/commonImpl/importers/DHImporter.java | 33 ++-- .../commonImpl/importers/WorldImporter.java | 10 +- .../voxy/server/DirtyUpdateService.java | 124 ++++++++++++ .../cortex/voxy/server/PlayerLodTracker.java | 182 ++++++++++++++++++ .../voxy/server/VoxyServerInstance.java | 113 +++++++++++ .../WorldSectionToVoxelizedConverter.java | 52 +++++ .../voxy/server/mixin/MixinServerLevel.java | 43 +++++ src/main/resources/fabric.mod.json | 4 +- 49 files changed, 1352 insertions(+), 217 deletions(-) rename src/main/java/me/cortex/voxy/{client => common}/config/VoxyConfig.java (98%) create mode 100644 src/main/java/me/cortex/voxy/common/config/VoxyServerConfig.java create mode 100644 src/main/java/me/cortex/voxy/common/network/SectionSerializer.java create mode 100644 src/main/java/me/cortex/voxy/common/network/VoxyNetwork.java create mode 100644 src/main/java/me/cortex/voxy/server/DirtyUpdateService.java create mode 100644 src/main/java/me/cortex/voxy/server/PlayerLodTracker.java create mode 100644 src/main/java/me/cortex/voxy/server/VoxyServerInstance.java create mode 100644 src/main/java/me/cortex/voxy/server/WorldSectionToVoxelizedConverter.java create mode 100644 src/main/java/me/cortex/voxy/server/mixin/MixinServerLevel.java diff --git a/build.gradle b/build.gradle index 94f0bf56..ce81d88a 100644 --- a/build.gradle +++ b/build.gradle @@ -245,19 +245,20 @@ dependencies { implementation "org.lwjgl:lwjgl" include(implementation "org.lwjgl:lwjgl-lmdb") - include(implementation "org.lwjgl:lwjgl-zstd") + //include(implementation "org.lwjgl:lwjgl-zstd") runtimeOnly "org.lwjgl:lwjgl:$lwjglVersion:natives-windows" runtimeOnly "org.lwjgl:lwjgl:$lwjglVersion:natives-linux" include(runtimeOnly "org.lwjgl:lwjgl-lmdb:$lwjglVersion:natives-windows") - include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-windows") + //include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-windows") include(runtimeOnly "org.lwjgl:lwjgl-lmdb:$lwjglVersion:natives-linux") - include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-linux") + //include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-linux") include(implementation 'redis.clients:jedis:5.1.0') include(implementation('org.rocksdb:rocksdbjni:10.2.1')) include(implementation 'org.apache.commons:commons-pool2:2.12.0') include(implementation 'org.lz4:lz4-java:1.8.0') include(implementation('org.tukaani:xz:1.10')) + include(implementation 'com.github.luben:zstd-jni:1.5.5-1') if (true) { if (!isInGHA) { diff --git a/src/main/java/me/cortex/voxy/client/VoxyClient.java b/src/main/java/me/cortex/voxy/client/VoxyClient.java index 559b840b..756dd51f 100644 --- a/src/main/java/me/cortex/voxy/client/VoxyClient.java +++ b/src/main/java/me/cortex/voxy/client/VoxyClient.java @@ -24,12 +24,17 @@ import java.util.HashSet; import java.util.function.Consumer; import java.util.function.Function; +import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking; +import me.cortex.voxy.common.network.VoxyNetwork; +import me.cortex.voxy.commonImpl.WorldIdentifier; + public class VoxyClient implements ClientModInitializer { private static final HashSet FREX = new HashSet<>(); public static void initVoxyClient() { Capabilities.init();//Ensure clinit is called + if (Capabilities.INSTANCE.hasBrokenDepthSampler) { Logger.error("AMD broken depth sampler detected, voxy does not work correctly and has been disabled, this will hopefully be fixed in the future"); } @@ -88,6 +93,39 @@ public class VoxyClient implements ClientModInitializer { } else { FREX.remove(name); }})); + + ClientPlayNetworking.registerGlobalReceiver(VoxyNetwork.ConfigSyncPayload.TYPE, (payload, context) -> { + context.client().execute(() -> { + // Update client render distance cap if needed, or store server settings + // For now, we can perhaps log it or update a transient config + Logger.info("Received server view distance: " + payload.viewDistance()); + // We might want to clamp the local render distance to the server's max if strictly enforced, + // or just use it as a hint for where to expect data. + // The user requirement says: "server sends max view distance to client... client renders up to client setting, server updates outside sim distance up to server max" + // So we should probably store this "server max view distance" somewhere in VoxyClientInstance. + if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) { + clientInstance.setServerViewDistance(payload.viewDistance()); + } + }); + }); + + ClientPlayNetworking.registerGlobalReceiver(VoxyNetwork.LodUpdatePayload.TYPE, (payload, context) -> { + // Deserialize off-thread if possible? Packet handling is on netty thread or main thread depending on configuration. + // But we can just schedule it. + // Actually deserialize needs Mapper which is world specific? + // The packet doesn't contain world ID? + // We assume it's for the current client world. + + // Wait, we need to know WHICH world this update is for if we have dimensions? + // Usually packets are for the current world the player is in. + + context.client().execute(() -> { + // Logger.info("Received LOD update packet, size: " + payload.data().length); + if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) { + clientInstance.handleLodUpdate(payload); + } + }); + }); } public static boolean isFrexActive() { diff --git a/src/main/java/me/cortex/voxy/client/VoxyClientInstance.java b/src/main/java/me/cortex/voxy/client/VoxyClientInstance.java index d141e303..41677480 100644 --- a/src/main/java/me/cortex/voxy/client/VoxyClientInstance.java +++ b/src/main/java/me/cortex/voxy/client/VoxyClientInstance.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client; import me.cortex.voxy.client.compat.FlashbackCompat; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.mixin.sodium.AccessorSodiumWorldRenderer; import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.StorageConfigUtil; @@ -28,6 +28,8 @@ public class VoxyClientInstance extends VoxyInstance { private final SectionStorageConfig storageConfig; private final Path basePath; private final boolean noIngestOverride; + private int serverViewDistance = 32; + public VoxyClientInstance() { super(); var path = FlashbackCompat.getReplayStoragePath(); @@ -40,6 +42,55 @@ public class VoxyClientInstance extends VoxyInstance { this.updateDedicatedThreads(); } + public long getLastLodUpdate() { + return lastLodUpdate; + } + + public int getLodUpdatesReceived() { + return lodUpdatesReceived; + } + + public void setServerViewDistance(int distance) { + this.serverViewDistance = distance; + // Trigger a re-evaluation of render distance? + Logger.info("Updating client view distance from server to: " + distance); + } + + public int getServerViewDistance() { + return this.serverViewDistance; + } + + private long lastLodUpdate; + private int lodUpdatesReceived; + + public void handleLodUpdate(me.cortex.voxy.common.network.VoxyNetwork.LodUpdatePayload payload) { + this.lastLodUpdate = System.currentTimeMillis(); + this.lodUpdatesReceived++; + // 1. Get current client world + var player = Minecraft.getInstance().player; + if (player == null) return; + var level = player.level(); + var wi = WorldIdentifier.of(level); + if (wi == null) return; + + var engine = this.getNullable(wi); + if (engine == null) return; + + // 2. Deserialize payload using engine's mapper + // Note: Payload deserialization requires Mapper to resolve IDs. + // We need to ensure the engine's mapper is up to date with the server's palette? + // No, the payload contains palette strings/states. + // The deserializer maps them to LOCAL IDs using the provided mapper. + + try { + var section = payload.deserialize(engine.getMapper()); + // 3. Insert update into engine + me.cortex.voxy.common.world.WorldUpdater.insertUpdate(engine, section); + } catch (Exception e) { + Logger.error("Failed to handle LOD update", e); + } + } + @Override public void updateDedicatedThreads() { int target = VoxyConfig.CONFIG.serviceThreads; diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfigMenu.java b/src/main/java/me/cortex/voxy/client/config/VoxyConfigMenu.java index 42f43f08..c3e260a8 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfigMenu.java +++ b/src/main/java/me/cortex/voxy/client/config/VoxyConfigMenu.java @@ -2,6 +2,7 @@ package me.cortex.voxy.client.config; import me.cortex.voxy.client.RenderStatistics; import me.cortex.voxy.client.config.SodiumConfigBuilder.*; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.VoxyClient; import me.cortex.voxy.client.VoxyClientInstance; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; diff --git a/src/main/java/me/cortex/voxy/client/core/NormalRenderPipeline.java b/src/main/java/me/cortex/voxy/client/core/NormalRenderPipeline.java index c358bdcc..0d3bac7d 100644 --- a/src/main/java/me/cortex/voxy/client/core/NormalRenderPipeline.java +++ b/src/main/java/me/cortex/voxy/client/core/NormalRenderPipeline.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.core; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.gl.GlFramebuffer; import me.cortex.voxy.client.core.gl.GlTexture; import me.cortex.voxy.client.core.gl.shader.Shader; diff --git a/src/main/java/me/cortex/voxy/client/core/VoxyRenderSystem.java b/src/main/java/me/cortex/voxy/client/core/VoxyRenderSystem.java index 8d4b536e..57770b5a 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxyRenderSystem.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxyRenderSystem.java @@ -4,7 +4,7 @@ import com.mojang.blaze3d.opengl.GlConst; import com.mojang.blaze3d.opengl.GlStateManager; import me.cortex.voxy.client.TimingStatistics; import me.cortex.voxy.client.VoxyClient; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.gl.Capabilities; import me.cortex.voxy.client.core.gl.GlBuffer; import me.cortex.voxy.client.core.gl.GlTexture; @@ -50,6 +50,9 @@ import static org.lwjgl.opengl.GL33.glBindSampler; import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BUFFER; import static org.lwjgl.opengl.GL43C.GL_SHADER_STORAGE_BUFFER_BINDING; +import me.cortex.voxy.client.VoxyClientInstance; +import me.cortex.voxy.common.config.VoxyServerConfig; + public class VoxyRenderSystem { private final WorldEngine worldIn; @@ -434,6 +437,14 @@ public class VoxyRenderSystem { debug.add("Extra 2 time: " + TimingStatistics.E.pVal() + ", " + TimingStatistics.F.pVal() + ", " + TimingStatistics.G.pVal() + ", " + TimingStatistics.H.pVal() + ", " + TimingStatistics.I.pVal()); } debug.add(GPUTiming.INSTANCE.getDebug()); + + if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) { + long lastUpdate = clientInstance.getLastLodUpdate(); + long timeSince = System.currentTimeMillis() - lastUpdate; + debug.add("LOD Updates: " + clientInstance.getLodUpdatesReceived() + " (Last: " + timeSince + "ms ago)"); + debug.add("Server View Dist: " + clientInstance.getServerViewDistance()); + } + PrintfDebugUtil.addToOut(debug); } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java index d2e29e2f..dcaac5c9 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java @@ -2,7 +2,7 @@ package me.cortex.voxy.client.core.rendering.hierachical; import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import me.cortex.voxy.client.RenderStatistics; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.gl.GlBuffer; import me.cortex.voxy.client.core.gl.shader.AutoBindingShader; import me.cortex.voxy.client.core.gl.shader.Shader; diff --git a/src/main/java/me/cortex/voxy/client/iris/VoxyUniforms.java b/src/main/java/me/cortex/voxy/client/iris/VoxyUniforms.java index 8c54ee40..7ffa3b03 100644 --- a/src/main/java/me/cortex/voxy/client/iris/VoxyUniforms.java +++ b/src/main/java/me/cortex/voxy/client/iris/VoxyUniforms.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.iris; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; import net.irisshaders.iris.gl.uniform.UniformHolder; import net.minecraft.client.Minecraft; diff --git a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinMatrixUniforms.java b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinMatrixUniforms.java index 619bab90..6556d303 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinMatrixUniforms.java +++ b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinMatrixUniforms.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.mixin.iris; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.util.IrisUtil; import me.cortex.voxy.client.iris.VoxyUniforms; import net.irisshaders.iris.gl.uniform.UniformHolder; diff --git a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinProgramSet.java b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinProgramSet.java index 4ebfcdda..3aab3430 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinProgramSet.java +++ b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinProgramSet.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.mixin.iris; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.util.IrisUtil; import me.cortex.voxy.client.iris.IGetVoxyPatchData; import me.cortex.voxy.client.iris.IrisShaderPatch; diff --git a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinStandardMacros.java b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinStandardMacros.java index 83643732..176e5127 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/iris/MixinStandardMacros.java +++ b/src/main/java/me/cortex/voxy/client/mixin/iris/MixinStandardMacros.java @@ -3,7 +3,7 @@ package me.cortex.voxy.client.mixin.iris; import com.google.common.collect.ImmutableList; import com.llamalad7.mixinextras.injector.wrapoperation.Operation; import com.llamalad7.mixinextras.injector.wrapoperation.WrapOperation; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.util.IrisUtil; import me.cortex.voxy.client.iris.IrisShaderPatch; import net.irisshaders.iris.gl.shader.StandardMacros; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkCache.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkCache.java index 55236fbf..f5d18034 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkCache.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientChunkCache.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client.mixin.minecraft; import me.cortex.voxy.client.ICheekyClientChunkCache; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.common.world.service.VoxelIngestService; import net.fabricmc.loader.api.FabricLoader; import net.minecraft.client.multiplayer.ClientChunkCache; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientLevel.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientLevel.java index d7010a6e..4e17d891 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientLevel.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientLevel.java @@ -1,6 +1,6 @@ package me.cortex.voxy.client.mixin.minecraft; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.common.world.service.VoxelIngestService; import me.cortex.voxy.commonImpl.WorldIdentifier; import net.minecraft.client.multiplayer.ClientChunkCache; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientPacketListener.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientPacketListener.java index 3d8c9423..f2a3ac85 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientPacketListener.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinClientPacketListener.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client.mixin.minecraft; import me.cortex.voxy.client.VoxyClientInstance; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.commonImpl.VoxyCommon; import net.minecraft.client.multiplayer.ClientPacketListener; import net.minecraft.network.protocol.game.ClientboundLoginPacket; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinFogRenderer.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinFogRenderer.java index 214bea62..94356662 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinFogRenderer.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinFogRenderer.java @@ -2,7 +2,7 @@ package me.cortex.voxy.client.mixin.minecraft; import com.llamalad7.mixinextras.injector.wrapmethod.WrapMethod; import com.llamalad7.mixinextras.sugar.Local; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; import net.minecraft.client.Camera; import net.minecraft.client.DeltaTracker; diff --git a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinLevelRenderer.java b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinLevelRenderer.java index c4fac281..19c33f5c 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinLevelRenderer.java +++ b/src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinLevelRenderer.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client.mixin.minecraft; import me.cortex.voxy.client.VoxyClientInstance; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; import me.cortex.voxy.client.core.VoxyRenderSystem; import me.cortex.voxy.client.core.util.IrisUtil; diff --git a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinChunkJobQueue.java b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinChunkJobQueue.java index 782cdd15..b49f198a 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinChunkJobQueue.java +++ b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinChunkJobQueue.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client.mixin.sodium; import me.cortex.voxy.client.compat.SemaphoreBlockImpersonator; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.common.thread.MultiThreadPrioritySemaphore; import me.cortex.voxy.commonImpl.VoxyCommon; import org.spongepowered.asm.mixin.Mixin; diff --git a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderRegionManager.java b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderRegionManager.java index 7b308a6f..3e33b764 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderRegionManager.java +++ b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderRegionManager.java @@ -2,7 +2,7 @@ package me.cortex.voxy.client.mixin.sodium; import com.llamalad7.mixinextras.injector.v2.WrapWithCondition; import com.llamalad7.mixinextras.injector.wrapoperation.WrapOperation; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; import me.cortex.voxy.commonImpl.VoxyCommon; import net.caffeinemc.mods.sodium.client.render.chunk.RenderSection; diff --git a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderSectionManager.java b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderSectionManager.java index 4db735f7..87a57ce0 100644 --- a/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderSectionManager.java +++ b/src/main/java/me/cortex/voxy/client/mixin/sodium/MixinRenderSectionManager.java @@ -1,7 +1,7 @@ package me.cortex.voxy.client.mixin.sodium; import me.cortex.voxy.client.ICheekyClientChunkCache; -import me.cortex.voxy.client.config.VoxyConfig; +import me.cortex.voxy.common.config.VoxyConfig; import me.cortex.voxy.client.core.IGetVoxyRenderSystem; import me.cortex.voxy.client.core.VoxyRenderSystem; import me.cortex.voxy.common.world.service.VoxelIngestService; diff --git a/src/main/java/me/cortex/voxy/common/config/Serialization.java b/src/main/java/me/cortex/voxy/common/config/Serialization.java index bbd3acb2..c5a54627 100644 --- a/src/main/java/me/cortex/voxy/common/config/Serialization.java +++ b/src/main/java/me/cortex/voxy/common/config/Serialization.java @@ -113,6 +113,12 @@ public class Serialization { if (clzName.contains("VoxyConfigScreenPages")) { continue;//Dont want to modmenu incase it doesnt exist } + if (clzName.contains("VoxyConfigMenu")) { + continue;//Dont want to modmenu incase it doesnt exist + } + if (clzName.contains("SodiumConfigBuilder")) { + continue;//Dont want to sodium incase it doesnt exist + } if (clzName.endsWith("VoxyConfig")) { continue;//Special case to prevent recursive loading pain } diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java b/src/main/java/me/cortex/voxy/common/config/VoxyConfig.java similarity index 98% rename from src/main/java/me/cortex/voxy/client/config/VoxyConfig.java rename to src/main/java/me/cortex/voxy/common/config/VoxyConfig.java index 8b8ac47d..b3d00c16 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java +++ b/src/main/java/me/cortex/voxy/common/config/VoxyConfig.java @@ -1,4 +1,4 @@ -package me.cortex.voxy.client.config; +package me.cortex.voxy.common.config; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; diff --git a/src/main/java/me/cortex/voxy/common/config/VoxyServerConfig.java b/src/main/java/me/cortex/voxy/common/config/VoxyServerConfig.java new file mode 100644 index 00000000..72e74b85 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/config/VoxyServerConfig.java @@ -0,0 +1,85 @@ +package me.cortex.voxy.common.config; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import me.cortex.voxy.common.Logger; +import me.cortex.voxy.commonImpl.VoxyCommon; +import net.fabricmc.loader.api.FabricLoader; + +import java.io.FileReader; +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; + +public class VoxyServerConfig { + private static final Gson GSON = new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .setPrettyPrinting() + .excludeFieldsWithModifiers(Modifier.PRIVATE) + .create(); + + public static VoxyServerConfig CONFIG = loadOrCreate(); + + public int viewDistance = 32; + public int dirtyUpdateDelay = 20; // Ticks before sending dirty update + public boolean ingestEnabled = true; + + private static VoxyServerConfig loadOrCreate() { + if (VoxyCommon.isAvailable()) { + var path = getConfigPath(); + if (Files.exists(path)) { + try (FileReader reader = new FileReader(path.toFile())) { + var conf = GSON.fromJson(reader, VoxyServerConfig.class); + if (conf != null) { + // conf.save(); // Don't save immediately on load, just return it. This avoids overwriting. + return conf; + } else { + Logger.error("Failed to load voxy server config (null), resetting"); + } + } catch (Exception e) { // Catch all exceptions including syntax errors + Logger.error("Could not parse config, resetting to default", e); + } + } + Logger.info("Server config doesnt exist or failed to load, creating new"); + var config = new VoxyServerConfig(); + config.save(); + return config; + } else { + return new VoxyServerConfig(); + } + } + + public void save() { + if (!VoxyCommon.isAvailable()) { + return; + } + + try { + // Only save if file doesn't exist to prevent overwriting user changes? + // Actually, usually save() is called to persist changes. + // But if we call save() immediately after load(), it just re-formats. + // The issue user reported is "resetting to default". + // That happens if GSON.fromJson returns null or exception. + // OR if we create new instance and overwrite. + + // If we want to preserve comments or formatting, we shouldn't overwrite unless needed. + // But JSON doesn't support comments. + + // If the user says it "resets", maybe loadOrCreate is failing? + // loadOrCreate checks if file exists. + + // Let's modify save to be safe? + Files.writeString(getConfigPath(), GSON.toJson(this)); + } catch (IOException e) { + Logger.error("Failed to write config file", e); + } + } + + private static Path getConfigPath() { + return FabricLoader.getInstance() + .getConfigDir() + .resolve("voxy-server-config.json"); + } +} diff --git a/src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java b/src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java index 053f6f1c..0521d95c 100644 --- a/src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java +++ b/src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java @@ -5,7 +5,7 @@ import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer; import me.cortex.voxy.common.world.SaveLoadSystem; import net.jpountz.lz4.LZ4Factory; -import org.lwjgl.system.MemoryUtil; +import me.cortex.voxy.common.util.UnsafeUtil; public class LZ4Compressor implements StorageCompressor { private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024); @@ -20,7 +20,7 @@ public class LZ4Compressor implements StorageCompressor { @Override public MemoryBuffer compress(MemoryBuffer saveData) { var res = new MemoryBuffer(this.compressor.maxCompressedLength((int) saveData.size)+4); - MemoryUtil.memPutInt(res.address, (int) saveData.size); + UnsafeUtil.putInt(res.address, (int) saveData.size); int size = this.compressor.compress(saveData.asByteBuffer(), 0, (int) saveData.size, res.asByteBuffer(), 4, (int) res.size-4); return res.subSize(size+4); } @@ -28,7 +28,7 @@ public class LZ4Compressor implements StorageCompressor { @Override public MemoryBuffer decompress(MemoryBuffer saveData) { var res = SCRATCH.get().createUntrackedUnfreeableReference(); - int size = this.decompressor.decompress(saveData.asByteBuffer(), 4, res.asByteBuffer(), 0, MemoryUtil.memGetInt(saveData.address)); + int size = this.decompressor.decompress(saveData.asByteBuffer(), 4, res.asByteBuffer(), 0, UnsafeUtil.getInt(saveData.address)); return res.subSize(size); } diff --git a/src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java b/src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java index 769bcbb6..bd3ea61e 100644 --- a/src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java +++ b/src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java @@ -1,34 +1,14 @@ package me.cortex.voxy.common.config.compressors; +import com.github.luben.zstd.Zstd; import me.cortex.voxy.common.config.ConfigBuildCtx; import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer; import me.cortex.voxy.common.world.SaveLoadSystem; -import static me.cortex.voxy.common.util.GlobalCleaner.CLEANER; -import static org.lwjgl.util.zstd.Zstd.*; +import java.nio.ByteBuffer; public class ZSTDCompressor implements StorageCompressor { - private record Ref(long ptr) {} - - private static Ref createCleanableCompressionContext() { - long ctx = ZSTD_createCCtx(); - var ref = new Ref(ctx); - CLEANER.register(ref, ()->ZSTD_freeCCtx(ctx)); - return ref; - } - - private static Ref createCleanableDecompressionContext() { - long ctx = ZSTD_createDCtx(); - nZSTD_DCtx_setParameter(ctx, ZSTD_d_experimentalParam3, 1);//experimental ZSTD_d_forceIgnoreChecksum - var ref = new Ref(ctx); - CLEANER.register(ref, ()->ZSTD_freeDCtx(ctx)); - return ref; - } - - private static final ThreadLocal COMPRESSION_CTX = ThreadLocal.withInitial(ZSTDCompressor::createCleanableCompressionContext); - private static final ThreadLocal DECOMPRESSION_CTX = ThreadLocal.withInitial(ZSTDCompressor::createCleanableDecompressionContext); - private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024); private final int level; @@ -39,16 +19,37 @@ public class ZSTDCompressor implements StorageCompressor { @Override public MemoryBuffer compress(MemoryBuffer saveData) { - MemoryBuffer compressedData = new MemoryBuffer((int)ZSTD_COMPRESSBOUND(saveData.size)); - long compressedSize = nZSTD_compressCCtx(COMPRESSION_CTX.get().ptr, compressedData.address, compressedData.size, saveData.address, saveData.size, this.level); + long bound = Zstd.compressBound(saveData.size); + if (bound > Integer.MAX_VALUE) { + // MemoryBuffer supports long size but ByteBuffer is limited to int. + // If we are using DirectByteBuffer wrappers, we are limited to 2GB. + // Voxy sections are usually small (KB/MBs). + throw new IllegalStateException("Compression bound too large for ByteBuffer: " + bound); + } + MemoryBuffer compressedData = new MemoryBuffer(bound); + + ByteBuffer src = saveData.asByteBuffer(); + ByteBuffer dst = compressedData.asByteBuffer(); + + // Zstd-jni compress returns size or error code + long compressedSize = Zstd.compress(dst, src, this.level); + if (Zstd.isError(compressedSize)) { + throw new RuntimeException("Zstd compression failed: " + Zstd.getErrorName(compressedSize)); + } + return compressedData.subSize(compressedSize); } @Override public MemoryBuffer decompress(MemoryBuffer saveData) { var decompressed = SCRATCH.get().createUntrackedUnfreeableReference(); - long size = nZSTD_decompressDCtx(DECOMPRESSION_CTX.get().ptr, decompressed.address, decompressed.size, saveData.address, saveData.size); - //TODO:FIXME: DONT ASSUME IT DOESNT FAIL + ByteBuffer dst = decompressed.asByteBuffer(); + ByteBuffer src = saveData.asByteBuffer(); + + long size = Zstd.decompress(dst, src); + if (Zstd.isError(size)) { + throw new RuntimeException("Zstd decompression failed: " + Zstd.getErrorName(size)); + } return decompressed.subSize(size); } diff --git a/src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java b/src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java index c9b20ae3..dea8489b 100644 --- a/src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java @@ -9,9 +9,9 @@ import me.cortex.voxy.common.config.ConfigBuildCtx; import me.cortex.voxy.common.config.storage.StorageBackend; import me.cortex.voxy.common.config.storage.StorageConfig; import me.cortex.voxy.common.util.MemoryBuffer; +import me.cortex.voxy.common.util.UnsafeUtil; import net.minecraft.world.level.levelgen.RandomSupport; import org.apache.commons.lang3.stream.Streams; -import org.lwjgl.system.MemoryUtil; import java.nio.ByteBuffer; import java.util.function.LongConsumer; @@ -85,11 +85,12 @@ public class MemoryStorageBackend extends StorageBackend { @Override public void putIdMapping(int id, ByteBuffer data) { synchronized (this.idMappings) { - var cpy = MemoryUtil.memAlloc(data.remaining()); - MemoryUtil.memCopy(data, cpy); + long addr = UnsafeUtil.allocateMemory(data.remaining()); + var cpy = UnsafeUtil.createDirectByteBuffer(addr, data.remaining()); + UnsafeUtil.memcpy(data, cpy); var prev = this.idMappings.put(id, cpy); if (prev != null) { - MemoryUtil.memFree(prev); + UnsafeUtil.freeMemory(UnsafeUtil.getAddress(prev)); } } } @@ -116,7 +117,7 @@ public class MemoryStorageBackend extends StorageBackend { @Override public void close() { Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryBuffer::free); - this.idMappings.values().forEach(MemoryUtil::memFree); + this.idMappings.values().forEach(b -> UnsafeUtil.freeMemory(UnsafeUtil.getAddress(b))); } public static class Config extends StorageConfig { diff --git a/src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java b/src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java index 633f7ec4..5c367347 100644 --- a/src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java @@ -7,7 +7,7 @@ import me.cortex.voxy.common.config.storage.StorageBackend; import me.cortex.voxy.common.config.storage.StorageConfig; import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.UnsafeUtil; -import org.lwjgl.system.MemoryUtil; +//import org.lwjgl.system.MemoryUtil; import org.lwjgl.util.lmdb.MDBVal; import java.nio.ByteBuffer; @@ -100,7 +100,7 @@ public class LMDBStorageBackend extends StorageBackend { if (bb == null) { return null; } - UnsafeUtil.memcpy(MemoryUtil.memAddress(bb), scratch.address, bb.remaining()); + UnsafeUtil.memcpy(UnsafeUtil.getAddress(bb), scratch.address, bb.remaining()); return scratch.subSize(bb.remaining()); })); } @@ -111,7 +111,9 @@ public class LMDBStorageBackend extends StorageBackend { this.resizingTransaction(() -> this.sectionDatabase.transaction(transaction->{ var keyBuff = transaction.stack.malloc(8); keyBuff.putLong(0, key); - transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0); + //transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0); + //Use unsafe to create a byte buffer from the address + transaction.put(keyBuff, UnsafeUtil.createDirectByteBuffer(data.address, (int) data.size), 0); return null; })); } diff --git a/src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java b/src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java index 96bb6f05..8e6e9322 100644 --- a/src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java @@ -6,8 +6,9 @@ import me.cortex.voxy.common.config.storage.StorageBackend; import me.cortex.voxy.common.config.storage.StorageConfig; import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.world.WorldEngine; -import org.lwjgl.system.MemoryStack; -import org.lwjgl.system.MemoryUtil; +//import org.lwjgl.system.MemoryStack; +//import org.lwjgl.system.MemoryUtil; +import me.cortex.voxy.common.util.UnsafeUtil; import org.rocksdb.*; import java.nio.ByteBuffer; @@ -118,39 +119,84 @@ public class RocksDBStorageBackend extends StorageBackend { @Override public void iterateStoredSectionPositions(LongConsumer consumer) { - try (var stack = MemoryStack.stackPush()) { - ByteBuffer keyBuff = stack.calloc(8); - long keyBuffPtr = MemoryUtil.memAddress(keyBuff); - var iter = this.db.newIterator(this.worldSections, this.sectionReadOps); - iter.seekToFirst(); - while (iter.isValid()) { - iter.key(keyBuff); - long key = Long.reverseBytes(MemoryUtil.memGetLong(keyBuffPtr)); - consumer.accept(key); - iter.next(); - } - iter.close(); + // try (var stack = MemoryStack.stackPush()) { + // ByteBuffer keyBuff = stack.calloc(8); + // long keyBuffPtr = MemoryUtil.memAddress(keyBuff); + // var iter = this.db.newIterator(this.worldSections, this.sectionReadOps); + // iter.seekToFirst(); + // while (iter.isValid()) { + // iter.key(keyBuff); + // long key = Long.reverseBytes(MemoryUtil.memGetLong(keyBuffPtr)); + // consumer.accept(key); + // iter.next(); + // } + // iter.close(); + // } + // Replacement using standard ByteBuffer + ByteBuffer keyBuff = ByteBuffer.allocateDirect(8); + var iter = this.db.newIterator(this.worldSections, this.sectionReadOps); + iter.seekToFirst(); + while (iter.isValid()) { + keyBuff.clear(); + iter.key(keyBuff); + long key = Long.reverseBytes(keyBuff.getLong(0)); + consumer.accept(key); + iter.next(); } + iter.close(); } @Override public MemoryBuffer getSectionData(long key, MemoryBuffer scratch) { - try (var stack = MemoryStack.stackPush()){ - var buffer = stack.malloc(8); - //HATE JAVA HATE JAVA HATE JAVA, Long.reverseBytes() - //THIS WILL ONLY WORK ON LITTLE ENDIAN SYSTEM AAAAAAAAA ;-; - - MemoryUtil.memPutLong(MemoryUtil.memAddress(buffer), Long.reverseBytes(swizzlePos(key))); - - var result = this.db.get(this.worldSections, - this.sectionReadOps, - buffer, - MemoryUtil.memByteBuffer(scratch.address, (int) (scratch.size))); - + // try (var stack = MemoryStack.stackPush()){ + // var buffer = stack.malloc(8); + // //HATE JAVA HATE JAVA HATE JAVA, Long.reverseBytes() + // //THIS WILL ONLY WORK ON LITTLE ENDIAN SYSTEM AAAAAAAAA ;-; + // + // MemoryUtil.memPutLong(MemoryUtil.memAddress(buffer), Long.reverseBytes(swizzlePos(key))); + // + // var result = this.db.get(this.worldSections, + // this.sectionReadOps, + // buffer, + // MemoryUtil.memByteBuffer(scratch.address, (int) (scratch.size))); + // + // if (result == RocksDB.NOT_FOUND) { + // return null; + // } + // + // return scratch.subSize(result); + // } catch (RocksDBException e) { + // throw new RuntimeException(e); + // } + + try { + byte[] keyBytes = longToBytes(swizzlePos(key)); // Using existing helper, creates garbage but safe + // Actually longToBytes does big endian? + // The original code did Long.reverseBytes(swizzlePos(key)) and wrote it to native memory. + // On Little Endian system, this means it writes Big Endian to memory? + // Wait. + // x86 is LE. + // Writing 0x11223344 to memory: 44 33 22 11. + // Long.reverseBytes(0x11223344) = 0x44332211. + // Writing 0x44332211 to memory: 11 22 33 44. + // So it writes Big Endian to memory? + // RocksDB usually sorts keys lexicographically. + // So we want Big Endian for keys if we want spatial locality to match key locality? + // Or maybe just consistent. + + // Let's use a byte array for key to avoid MemoryStack. + ByteBuffer keyBuff = ByteBuffer.allocateDirect(8); + keyBuff.order(java.nio.ByteOrder.nativeOrder()); + keyBuff.putLong(0, Long.reverseBytes(swizzlePos(key))); + + // We need a ByteBuffer for the output that wraps the scratch address. + ByteBuffer outBuff = UnsafeUtil.createDirectByteBuffer(scratch.address, (int) scratch.size); + + int result = this.db.get(this.worldSections, this.sectionReadOps, keyBuff, outBuff); + if (result == RocksDB.NOT_FOUND) { return null; } - return scratch.subSize(result); } catch (RocksDBException e) { throw new RuntimeException(e); @@ -160,9 +206,18 @@ public class RocksDBStorageBackend extends StorageBackend { //TODO: FIXME, use the ByteBuffer variant @Override public void setSectionData(long key, MemoryBuffer data) { - try (var stack = MemoryStack.stackPush()) { - var keyBuff = stack.calloc(8); - MemoryUtil.memPutLong(MemoryUtil.memAddress(keyBuff), Long.reverseBytes(swizzlePos(key))); + // try (var stack = MemoryStack.stackPush()) { + // var keyBuff = stack.calloc(8); + // MemoryUtil.memPutLong(MemoryUtil.memAddress(keyBuff), Long.reverseBytes(swizzlePos(key))); + // this.db.put(this.worldSections, this.sectionWriteOps, keyBuff, data.asByteBuffer()); + // } catch (RocksDBException e) { + // throw new RuntimeException(e); + // } + try { + ByteBuffer keyBuff = ByteBuffer.allocateDirect(8); + keyBuff.order(java.nio.ByteOrder.nativeOrder()); + keyBuff.putLong(0, Long.reverseBytes(swizzlePos(key))); + this.db.put(this.worldSections, this.sectionWriteOps, keyBuff, data.asByteBuffer()); } catch (RocksDBException e) { throw new RuntimeException(e); diff --git a/src/main/java/me/cortex/voxy/common/network/SectionSerializer.java b/src/main/java/me/cortex/voxy/common/network/SectionSerializer.java new file mode 100644 index 00000000..2a73d084 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/network/SectionSerializer.java @@ -0,0 +1,162 @@ +package me.cortex.voxy.common.network; + +import me.cortex.voxy.common.voxelization.VoxelizedSection; +import me.cortex.voxy.common.world.other.Mapper; +import net.minecraft.network.FriendlyByteBuf; +import net.minecraft.util.SimpleBitStorage; +import net.minecraft.util.Mth; +import net.minecraft.world.level.block.Block; +import net.minecraft.world.level.block.state.BlockState; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SectionSerializer { + + public static void serialize(VoxelizedSection section, Mapper mapper, FriendlyByteBuf buf) { + // 1. Scan and build palette + Map blockPaletteMap = new HashMap<>(); + List blockPalette = new ArrayList<>(); + + Map biomePaletteMap = new HashMap<>(); + List biomePalette = new ArrayList<>(); + + long[] data = section.section; + int count = data.length; + + // Pass 1: Build Palette + for (long l : data) { + int blockId = Mapper.getBlockId(l); + BlockState state = mapper.getBlockStateFromBlockId(blockId); + if (!blockPaletteMap.containsKey(state)) { + blockPaletteMap.put(state, blockPalette.size()); + blockPalette.add(state); + } + + int biomeId = Mapper.getBiomeId(l); + String biome = mapper.getBiomeString(biomeId); + if (!biomePaletteMap.containsKey(biome)) { + biomePaletteMap.put(biome, biomePalette.size()); + biomePalette.add(biome); + } + } + + // 2. Write Palettes + buf.writeVarInt(blockPalette.size()); + for (BlockState state : blockPalette) { + buf.writeVarInt(Block.BLOCK_STATE_REGISTRY.getId(state)); + } + + buf.writeVarInt(biomePalette.size()); + for (String biome : biomePalette) { + buf.writeUtf(biome); + } + + // 3. Prepare BitStorages + int blockBits = Math.max(1, Mth.ceillog2(blockPalette.size())); + int biomeBits = Math.max(1, Mth.ceillog2(biomePalette.size())); + + SimpleBitStorage blockStorage = new SimpleBitStorage(blockBits, count); + SimpleBitStorage biomeStorage = new SimpleBitStorage(biomeBits, count); + byte[] lightStorage = new byte[count]; + + // 4. Fill Data + for (int i = 0; i < count; i++) { + long l = data[i]; + + int blockId = Mapper.getBlockId(l); + BlockState state = mapper.getBlockStateFromBlockId(blockId); + blockStorage.set(i, blockPaletteMap.get(state)); + + int biomeId = Mapper.getBiomeId(l); + String biome = mapper.getBiomeString(biomeId); + biomeStorage.set(i, biomePaletteMap.get(biome)); + + lightStorage[i] = (byte) Mapper.getLightId(l); + } + + // 5. Write Data + buf.writeLongArray(blockStorage.getRaw()); + buf.writeLongArray(biomeStorage.getRaw()); + buf.writeByteArray(lightStorage); + + // Write extra metadata from VoxelizedSection + buf.writeInt(section.x); + buf.writeInt(section.y); + buf.writeInt(section.z); + buf.writeInt(section.lvl0NonAirCount); + } + + public static VoxelizedSection deserialize(FriendlyByteBuf buf, Mapper mapper) { + // 1. Read Palettes + int blockPaletteSize = buf.readVarInt(); + BlockState[] blockPalette = new BlockState[blockPaletteSize]; + for (int i = 0; i < blockPaletteSize; i++) { + blockPalette[i] = Block.BLOCK_STATE_REGISTRY.byId(buf.readVarInt()); + } + + int biomePaletteSize = buf.readVarInt(); + String[] biomePalette = new String[biomePaletteSize]; + for (int i = 0; i < biomePaletteSize; i++) { + biomePalette[i] = buf.readUtf(); + } + + // 2. Read Data + long[] blockRaw = buf.readLongArray(); + long[] biomeRaw = buf.readLongArray(); + byte[] lightRaw = buf.readByteArray(); + + int count = lightRaw.length; + + int blockBits = Math.max(1, Mth.ceillog2(blockPaletteSize)); + int biomeBits = Math.max(1, Mth.ceillog2(biomePaletteSize)); + + SimpleBitStorage blockStorage = new SimpleBitStorage(blockBits, count, blockRaw); + SimpleBitStorage biomeStorage = new SimpleBitStorage(biomeBits, count, biomeRaw); + + long[] sectionData = new long[count]; + + // 3. Reconstruct VoxelizedSection + // We need to resolve to LOCAL IDs using the mapper + int[] blockLocalIds = new int[blockPaletteSize]; + for (int i = 0; i < blockPaletteSize; i++) { + blockLocalIds[i] = mapper.getIdForBlockState(blockPalette[i]); + } + + // For biomes, mapper needs "Holder". + // But Mapper also has "registerNewBiome(String)". + // We can access it via getIdForBiome(Holder) but we only have String. + // We should add getIdForBiome(String) to Mapper. + // Or assume we can just use the string. + // Mapper.registerNewBiome(String) is private. + // Mapper.getIdForBiome(Holder) calls registerNewBiome. + // We need a public method `int getIdForBiome(String id)`. + + // I will assume I added it or will add it. + int[] biomeLocalIds = new int[biomePaletteSize]; + for (int i = 0; i < biomePaletteSize; i++) { + biomeLocalIds[i] = mapper.getIdForBiome(biomePalette[i]); + } + + for (int i = 0; i < count; i++) { + int blockIdx = blockStorage.get(i); + int biomeIdx = biomeStorage.get(i); + int light = lightRaw[i] & 0xFF; + + int localBlockId = blockLocalIds[blockIdx]; + int localBiomeId = biomeLocalIds[biomeIdx]; // This will be 0 for now + + sectionData[i] = Mapper.composeMappingId((byte)light, localBlockId, localBiomeId); + } + + VoxelizedSection section = new VoxelizedSection(sectionData); + section.x = buf.readInt(); + section.y = buf.readInt(); + section.z = buf.readInt(); + section.lvl0NonAirCount = buf.readInt(); + + return section; + } +} diff --git a/src/main/java/me/cortex/voxy/common/network/VoxyNetwork.java b/src/main/java/me/cortex/voxy/common/network/VoxyNetwork.java new file mode 100644 index 00000000..83796636 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/network/VoxyNetwork.java @@ -0,0 +1,62 @@ +package me.cortex.voxy.common.network; + +import io.netty.buffer.Unpooled; +import me.cortex.voxy.common.voxelization.VoxelizedSection; +import me.cortex.voxy.common.world.other.Mapper; +import net.fabricmc.fabric.api.networking.v1.PayloadTypeRegistry; +import net.minecraft.network.FriendlyByteBuf; +import net.minecraft.network.codec.StreamCodec; +import net.minecraft.network.protocol.common.custom.CustomPacketPayload; +import net.minecraft.resources.Identifier; + +public class VoxyNetwork { + public static final Identifier CONFIG_SYNC_ID = Identifier.parse("voxy:config_sync"); + public static final Identifier LOD_UPDATE_ID = Identifier.parse("voxy:lod_update"); + + public static void register() { + PayloadTypeRegistry.playS2C().register(ConfigSyncPayload.TYPE, ConfigSyncPayload.CODEC); + PayloadTypeRegistry.playS2C().register(LodUpdatePayload.TYPE, LodUpdatePayload.CODEC); + } + + public record ConfigSyncPayload(int viewDistance) implements CustomPacketPayload { + public static final Type TYPE = new Type<>(CONFIG_SYNC_ID); + public static final StreamCodec CODEC = StreamCodec.of( + (buf, payload) -> buf.writeInt(payload.viewDistance), + (buf) -> new ConfigSyncPayload(buf.readInt()) + ); + + @Override + public Type type() { + return TYPE; + } + } + + public record LodUpdatePayload(byte[] data) implements CustomPacketPayload { + public static final Type TYPE = new Type<>(LOD_UPDATE_ID); + public static final StreamCodec CODEC = StreamCodec.of( + (buf, payload) -> buf.writeByteArray(payload.data), + (buf) -> new LodUpdatePayload(buf.readByteArray()) + ); + + @Override + public Type type() { + return TYPE; + } + + public static LodUpdatePayload create(VoxelizedSection section, Mapper mapper) { + FriendlyByteBuf buf = new FriendlyByteBuf(Unpooled.buffer()); + SectionSerializer.serialize(section, mapper, buf); + byte[] bytes = new byte[buf.writerIndex()]; + buf.readBytes(bytes); + buf.release(); + return new LodUpdatePayload(bytes); + } + + public VoxelizedSection deserialize(Mapper mapper) { + FriendlyByteBuf buf = new FriendlyByteBuf(Unpooled.wrappedBuffer(this.data)); + VoxelizedSection section = SectionSerializer.deserialize(buf, mapper); + buf.release(); + return section; + } + } +} diff --git a/src/main/java/me/cortex/voxy/common/thread/UnifiedServiceThreadPool.java b/src/main/java/me/cortex/voxy/common/thread/UnifiedServiceThreadPool.java index 423d5f25..9b8a7012 100644 --- a/src/main/java/me/cortex/voxy/common/thread/UnifiedServiceThreadPool.java +++ b/src/main/java/me/cortex/voxy/common/thread/UnifiedServiceThreadPool.java @@ -15,6 +15,9 @@ public class UnifiedServiceThreadPool { private final ThreadGroup dedicatedPool; private final List threads = new ArrayList<>(); private int threadId = 0; + + private final Service sharedExecutorService; + private final java.util.Queue sharedExecutorQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); public UnifiedServiceThreadPool() { this.dedicatedPool = new ThreadGroup("Voxy Dedicated Service"); @@ -22,6 +25,11 @@ public class UnifiedServiceThreadPool { this.groupSemaphore = new MultiThreadPrioritySemaphore(this.serviceManager::tryRunAJob); this.selfBlock = this.groupSemaphore.createBlock(); + + this.sharedExecutorService = this.serviceManager.createServiceNoCleanup(() -> () -> { + Runnable r = this.sharedExecutorQueue.poll(); + if (r != null) r.run(); + }, 1, "SharedExecutor"); } private final void release(int i) {this.groupSemaphore.pooledRelease(i);} @@ -80,6 +88,11 @@ public class UnifiedServiceThreadPool { } this.selfBlock.free(); } + + public void execute(Runnable runnable) { + this.sharedExecutorQueue.add(runnable); + this.sharedExecutorService.execute(); + } public static void main(String[] args) { diff --git a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java index 339b609b..9f63b5c6 100644 --- a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java +++ b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java @@ -1,7 +1,7 @@ package me.cortex.voxy.common.util; import me.cortex.voxy.commonImpl.VoxyCommon; -import org.lwjgl.system.MemoryUtil; +//import org.lwjgl.system.MemoryUtil; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicInteger; @@ -20,7 +20,8 @@ public class MemoryBuffer extends TrackedObject { public MemoryBuffer(long size) { - this(true, MemoryUtil.nmemAlloc(size), size, true); + //this(true, MemoryUtil.nmemAlloc(size), size, true); + this(true, UnsafeUtil.allocateMemory(size), size, true); } private MemoryBuffer(boolean track, long address, long size, boolean freeable) { @@ -56,7 +57,8 @@ public class MemoryBuffer extends TrackedObject { COUNT.decrementAndGet(); } if (this.freeable) { - MemoryUtil.nmemFree(this.address); + //MemoryUtil.nmemFree(this.address); + UnsafeUtil.freeMemory(this.address); TOTAL_SIZE.addAndGet(-this.size); } else { throw new IllegalArgumentException("Tried to free unfreeable buffer"); @@ -88,12 +90,14 @@ public class MemoryBuffer extends TrackedObject { } public MemoryBuffer zero() { - MemoryUtil.memSet(this.address, 0, this.size); + //MemoryUtil.memSet(this.address, 0, this.size); + UnsafeUtil.setMemory(this.address, this.size, (byte)0); return this; } public ByteBuffer asByteBuffer() { - return MemoryUtil.memByteBuffer(this.address, (int) this.size); + //return MemoryUtil.memByteBuffer(this.address, (int) this.size); + return UnsafeUtil.createDirectByteBuffer(this.address, (int) this.size); } //TODO: create like Long(offset) -> value at offset diff --git a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java index 90f04dba..f522aad6 100644 --- a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java +++ b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java @@ -6,11 +6,25 @@ import java.lang.reflect.Field; public class UnsafeUtil { private static final Unsafe UNSAFE; + private static final long BUFFER_ADDRESS_OFFSET; + private static final long BUFFER_CAPACITY_OFFSET; + private static final long BUFFER_LIMIT_OFFSET; + private static final Class DIRECT_BYTE_BUFFER_CLASS; + static { try { Field field = Unsafe.class.getDeclaredField("theUnsafe"); field.setAccessible(true); UNSAFE = (Unsafe)field.get(null); + + Field addr = java.nio.Buffer.class.getDeclaredField("address"); + BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addr); + Field cap = java.nio.Buffer.class.getDeclaredField("capacity"); + BUFFER_CAPACITY_OFFSET = UNSAFE.objectFieldOffset(cap); + Field lim = java.nio.Buffer.class.getDeclaredField("limit"); + BUFFER_LIMIT_OFFSET = UNSAFE.objectFieldOffset(lim); + + DIRECT_BYTE_BUFFER_CLASS = Class.forName("java.nio.DirectByteBuffer"); } catch (Exception e) {throw new RuntimeException(e);} } @@ -47,4 +61,80 @@ public class UnsafeUtil { public static void memcpy(short[] src, long dst) { UNSAFE.copyMemory(src, SHORT_ARRAY_BASE_OFFSET, null, dst, (long) src.length <<1); } + + public static long allocateMemory(long size) { + return UNSAFE.allocateMemory(size); + } + + public static void freeMemory(long address) { + UNSAFE.freeMemory(address); + } + + public static void setMemory(long address, long bytes, byte value) { + UNSAFE.setMemory(address, bytes, value); + } + + public static java.nio.ByteBuffer createDirectByteBuffer(long address, int capacity) { + try { + java.nio.ByteBuffer bb = (java.nio.ByteBuffer) UNSAFE.allocateInstance(DIRECT_BYTE_BUFFER_CLASS); + UNSAFE.putLong(bb, BUFFER_ADDRESS_OFFSET, address); + UNSAFE.putInt(bb, BUFFER_CAPACITY_OFFSET, capacity); + UNSAFE.putInt(bb, BUFFER_LIMIT_OFFSET, capacity); + bb.order(java.nio.ByteOrder.nativeOrder()); + return bb; + } catch (InstantiationException e) { + throw new RuntimeException("Failed to create DirectByteBuffer via allocateInstance", e); + } + } + + public static void memcpy(java.nio.ByteBuffer src, java.nio.ByteBuffer dst) { + if (!src.isDirect() || !dst.isDirect()) { + throw new IllegalArgumentException("Buffers must be direct"); + } + if (src.remaining() > dst.remaining()) { + throw new IllegalArgumentException("Source buffer is larger than destination buffer"); + } + long srcAddr = getAddress(src) + src.position(); + long dstAddr = getAddress(dst) + dst.position(); + UNSAFE.copyMemory(srcAddr, dstAddr, src.remaining()); + } + + public static long getAddress(java.nio.ByteBuffer buffer) { + if (!buffer.isDirect()) { + throw new IllegalArgumentException("Buffer must be direct"); + } + return UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET); + } + + public static void putLong(long address, long value) { + UNSAFE.putLong(address, value); + } + + public static long getLong(long address) { + return UNSAFE.getLong(address); + } + + public static void putInt(long address, int value) { + UNSAFE.putInt(address, value); + } + + public static int getInt(long address) { + return UNSAFE.getInt(address); + } + + public static void putShort(long address, short value) { + UNSAFE.putShort(address, value); + } + + public static short getShort(long address) { + return UNSAFE.getShort(address); + } + + public static void putByte(long address, byte value) { + UNSAFE.putByte(address, value); + } + + public static byte getByte(long address) { + return UNSAFE.getByte(address); + } } diff --git a/src/main/java/me/cortex/voxy/common/util/cpu/CpuLayout.java b/src/main/java/me/cortex/voxy/common/util/cpu/CpuLayout.java index 7690f382..f11c53d3 100644 --- a/src/main/java/me/cortex/voxy/common/util/cpu/CpuLayout.java +++ b/src/main/java/me/cortex/voxy/common/util/cpu/CpuLayout.java @@ -5,7 +5,6 @@ import com.sun.jna.platform.win32.WinNT; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.util.ThreadUtils; -import org.lwjgl.system.Platform; import oshi.SystemInfo; import java.util.Arrays; @@ -13,6 +12,9 @@ import java.util.Random; //Represents the layout of the current cpu running on public class CpuLayout { + private static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase().contains("win"); + private static final boolean IS_LINUX = System.getProperty("os.name").toLowerCase().contains("linux"); + private CpuLayout(){} public static void setThreadAffinity(Core... cores) { @@ -24,8 +26,7 @@ public class CpuLayout { } public static void setThreadAffinity(Affinity... affinities) { - var platform = Platform.get(); - if (platform == Platform.WINDOWS) { + if (IS_WINDOWS) { long[] msks = new long[affinities.length]; short[] groups = new short[affinities.length];Arrays.fill(groups, (short) -1); int i = 0; @@ -36,7 +37,7 @@ public class CpuLayout { msks[idx] |= a.msk; } ThreadUtils.SetThreadSelectedCpuSetMasksWin32(Arrays.copyOf(msks, i), Arrays.copyOf(groups, i)); - } else if (platform == Platform.LINUX) { + } else if (IS_LINUX) { Arrays.sort(affinities, (a, b) -> a.group - b.group); long[] msks = new long[affinities.length]; for (int i=0; i>>= eBits; byte light = lightSupplier.supply(i&0xF, (i>>8)&0xF, (i>>4)&0xF); nonZeroCnt += (bId != 0)?1:0; diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java index 5a589395..e8401596 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java @@ -6,7 +6,6 @@ import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.UnsafeUtil; import me.cortex.voxy.common.world.other.Mapper; import me.cortex.voxy.commonImpl.VoxyCommon; -import org.lwjgl.system.MemoryUtil; public class SaveLoadSystem { public static final boolean VERIFY_HASH_ON_LOAD = VoxyCommon.isVerificationFlagOn("verifySectionHash"); @@ -65,18 +64,18 @@ public class SaveLoadSystem { long ptr = raw.address; long hash = section.key^(lutIndex*1293481298141L); - MemoryUtil.memPutLong(ptr, section.key); ptr += 8; + UnsafeUtil.putLong(ptr, section.key); ptr += 8; long metadata = 0; metadata |= Byte.toUnsignedLong(section.nonEmptyChildren); - MemoryUtil.memPutLong(ptr, metadata); ptr += 8; + UnsafeUtil.putLong(ptr, metadata); ptr += 8; hash ^= metadata; hash *= 1242629872171L; - MemoryUtil.memPutInt(ptr, lutIndex); ptr += 4; + UnsafeUtil.putInt(ptr, lutIndex); ptr += 4; for (int i = 0; i < lutIndex; i++) { long id = lutValues[i]; - MemoryUtil.memPutLong(ptr, id); ptr += 8; + UnsafeUtil.putLong(ptr, id); ptr += 8; hash *= 1230987149811L; hash += 12831; hash ^= id; @@ -85,19 +84,19 @@ public class SaveLoadSystem { UnsafeUtil.memcpy(compressed, ptr); ptr += compressed.length*2L; - MemoryUtil.memPutLong(ptr, hash); ptr += 8; + UnsafeUtil.putLong(ptr, hash); ptr += 8; return raw.subSize(ptr-raw.address); } public static boolean deserialize(WorldSection section, MemoryBuffer data) { long ptr = data.address; - long key = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + long key = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); - long metadata = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + long metadata = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); section.nonEmptyChildren = (byte) (metadata&0xFF); - int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + int lutLen = UnsafeUtil.getInt(ptr); ptr += 4; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); if (lutLen > 32*32*32) { throw new IllegalStateException("lutLen impossibly large, max size should be 32768 but got size " + lutLen); } @@ -109,7 +108,7 @@ public class SaveLoadSystem { hash ^= metadata; hash *= 1242629872171L; } for (int i = 0; i < lutLen; i++) { - lut [i] = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + lut [i] = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); if (VERIFY_HASH_ON_LOAD) { hash *= 1230987149811L; hash += 12831; @@ -125,7 +124,7 @@ public class SaveLoadSystem { int nonEmptyBlockCount = 0; for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) { - long state = lut[Short.toUnsignedInt(MemoryUtil.memGetShort(ptr))]; ptr += 2; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + long state = lut[Short.toUnsignedInt(UnsafeUtil.getShort(ptr))]; ptr += 2; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); nonEmptyBlockCount += Mapper.isAir(state)?0:1; section.data[z2lin(i)] = state; } @@ -141,7 +140,7 @@ public class SaveLoadSystem { } hash ^= pHash; - long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); + long expectedHash = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<(ptr-data.address)) throw new IllegalStateException("Memory access OOB"); if (expectedHash != hash) { //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash); Logger.error("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region"); diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem2.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem2.java index cf4c119e..1a04c426 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem2.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem2.java @@ -6,7 +6,6 @@ import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.UnsafeUtil; import me.cortex.voxy.common.world.other.Mapper; import net.minecraft.util.Mth; -import org.lwjgl.system.MemoryUtil; import java.util.Arrays; @@ -97,8 +96,8 @@ public class SaveLoadSystem2 { var res = new MemoryBuffer(32*32*32*8+1024); long ptr = res.address; - MemoryUtil.memPutLong(ptr, section.key); ptr += 8; - MemoryUtil.memPutLong(ptr, hash); ptr += 8; + UnsafeUtil.putLong(ptr, section.key); ptr += 8; + UnsafeUtil.putLong(ptr, hash); ptr += 8; int meta = 0; meta |= allSameBlockLight?1:0; @@ -106,22 +105,22 @@ public class SaveLoadSystem2 { meta |= (biomeMapping.size()-1)<<2;//512 max size meta |= (blockMapping.size()-1)<<11;//4096 max size meta |= Byte.toUnsignedInt(section.nonEmptyChildren) << 23; - MemoryUtil.memPutInt(ptr, meta); ptr += 4; + UnsafeUtil.putInt(ptr, meta); ptr += 4; //Encode lighting /* //Micro storage optimization, not done cause makes decode very slightly slower and more pain if (allSameBlockLight && allSameSkyLight) { - MemoryUtil.memPutByte(ptr, (byte) ((blockLight[0]&0xF)|((skyLight[0]&0xF)<<4))); ptr += 1; + UnsafeUtil.putByte(ptr, (byte) ((blockLight[0]&0xF)|((skyLight[0]&0xF)<<4))); ptr += 1; } else*/ { if (allSameBlockLight) { - MemoryUtil.memPutByte(ptr, (byte) (blockLight[0] & 0xF)); ptr += 1; + UnsafeUtil.putByte(ptr, (byte) (blockLight[0] & 0xF)); ptr += 1; } else { UnsafeUtil.memcpy(blockLight, ptr); ptr += blockLight.length; } if (allSameSkyLight) { - MemoryUtil.memPutByte(ptr, (byte) (skyLight[0] & 0xF)); ptr += 1; + UnsafeUtil.putByte(ptr, (byte) (skyLight[0] & 0xF)); ptr += 1; } else { UnsafeUtil.memcpy(skyLight, ptr); ptr += skyLight.length; } @@ -138,7 +137,7 @@ public class SaveLoadSystem2 { if (rem != 0) batch |= (b << (32 - rem));//the shift does auto cutoff if (rem < SIZE) { - MemoryUtil.memPutInt(ptr, batch); + UnsafeUtil.putInt(ptr, batch); ptr += 4; batch = b >> rem; rem = 32 - (SIZE - rem); @@ -154,7 +153,7 @@ public class SaveLoadSystem2 { if (rem != 0) batch |= (b << (32 - rem));//the shift does auto cutoff if (rem < SIZE) { - MemoryUtil.memPutInt(ptr, batch); ptr += 4; + UnsafeUtil.putInt(ptr, batch); ptr += 4; batch = b >> rem; rem = 32 - (SIZE - rem); } else { @@ -163,7 +162,7 @@ public class SaveLoadSystem2 { } } if (rem != 32) { - MemoryUtil.memPutInt(ptr, batch); ptr += 4; + UnsafeUtil.putInt(ptr, batch); ptr += 4; } } @@ -179,7 +178,7 @@ public class SaveLoadSystem2 { if (rem != 0) batch |= (b << (32 - rem));//the shift does auto cutoff if (rem < SIZE) { - MemoryUtil.memPutInt(ptr, batch); + UnsafeUtil.putInt(ptr, batch); ptr += 4; batch = b >> rem; rem = 32 - (SIZE - rem); @@ -188,7 +187,7 @@ public class SaveLoadSystem2 { } } if (rem != 32) { - MemoryUtil.memPutInt(ptr, batch); ptr += 4; + UnsafeUtil.putInt(ptr, batch); ptr += 4; } } {//Store biome @@ -204,7 +203,7 @@ public class SaveLoadSystem2 { if (rem != 0) batch |= (b << (32 - rem));//the shift does auto cutoff if (rem < SIZE) { - MemoryUtil.memPutInt(ptr, batch); + UnsafeUtil.putInt(ptr, batch); ptr += 4; batch = b >> rem; rem = 32 - (SIZE - rem); @@ -213,7 +212,7 @@ public class SaveLoadSystem2 { } } if (rem != 32) { - MemoryUtil.memPutInt(ptr, batch); ptr += 4; + UnsafeUtil.putInt(ptr, batch); ptr += 4; } } } @@ -225,13 +224,13 @@ public class SaveLoadSystem2 { var cache = DESERIALIZE_CACHE.get(); long ptr = data.address; - long pos = MemoryUtil.memGetLong(ptr); ptr += 8; + long pos = UnsafeUtil.getLong(ptr); ptr += 8; if (section.key != pos) { Logger.error("Section pos not the same as requested, got " + pos + " expected " + section.key); return false; } - long chash = MemoryUtil.memGetLong(ptr); ptr += 8; - int meta = MemoryUtil.memGetInt(ptr); ptr += 4; + long chash = UnsafeUtil.getLong(ptr); ptr += 8; + int meta = UnsafeUtil.getInt(ptr); ptr += 4; boolean allSameBlockLight = (meta&1)!=0; boolean allSameSkyLight = (meta&2)!=0; @@ -244,12 +243,12 @@ public class SaveLoadSystem2 { if (allSameBlockLight) { //shift up 4 so that its already in correct position - blockLight = Byte.toUnsignedLong(MemoryUtil.memGetByte(ptr))<<4; ptr += 1; + blockLight = Byte.toUnsignedLong(UnsafeUtil.getByte(ptr))<<4; ptr += 1; } else { blockLight = ptr; ptr += 32*32*32/2; } if (allSameSkyLight) { - skyLight = MemoryUtil.memGetByte(ptr); ptr += 1; + skyLight = UnsafeUtil.getByte(ptr); ptr += 1; } else { skyLight = ptr; ptr += 32*32*32/2; } @@ -258,7 +257,7 @@ public class SaveLoadSystem2 { int[] biomeLut = new int[biomeMapSize]; {//Deserialize the block and biome mappings int rem = 32; - int batch = MemoryUtil.memGetInt(ptr); ptr += 4; + int batch = UnsafeUtil.getInt(ptr); ptr += 4; {//Block int SIZE = 20;// 20 bits per entry int msk = (1<>>= SIZE; rem -= SIZE; if (rem < 0) { - batch = MemoryUtil.memGetInt(ptr); ptr += 4; + batch = UnsafeUtil.getInt(ptr); ptr += 4; val |= (batch&((1<<-rem)-1))<<(SIZE+rem); batch >>>= -rem; rem = 32+rem; @@ -281,7 +280,7 @@ public class SaveLoadSystem2 { int val = batch&msk; batch >>>= SIZE; rem -= SIZE; if (rem < 0) { - batch = MemoryUtil.memGetInt(ptr); ptr += 4; + batch = UnsafeUtil.getInt(ptr); ptr += 4; val |= (batch&((1<<-rem)-1))<<(SIZE+rem); batch >>>= -rem; rem = 32+rem; @@ -297,13 +296,13 @@ public class SaveLoadSystem2 { {//Block final int SIZE = Mth.smallestEncompassingPowerOfTwo(Mth.log2(Mth.smallestEncompassingPowerOfTwo(blockMapSize))); int rem = 32; - int batch = MemoryUtil.memGetInt(ptr); ptr += 4; + int batch = UnsafeUtil.getInt(ptr); ptr += 4; int msk = (1<>>= SIZE; rem -= SIZE; if (rem < 0) { - batch = MemoryUtil.memGetInt(ptr); ptr += 4; + batch = UnsafeUtil.getInt(ptr); ptr += 4; val |= (batch&((1<<-rem)-1))<<(SIZE+rem); batch >>>= -rem; rem = 32+rem; @@ -317,13 +316,13 @@ public class SaveLoadSystem2 { } else { final int SIZE = Mth.smallestEncompassingPowerOfTwo(Mth.log2(Mth.smallestEncompassingPowerOfTwo(biomeMapSize))); int rem = 32; - int batch = MemoryUtil.memGetInt(ptr); ptr += 4; + int batch = UnsafeUtil.getInt(ptr); ptr += 4; int msk = (1<>>= SIZE; rem -= SIZE; if (rem < 0) { - batch = MemoryUtil.memGetInt(ptr); ptr += 4; + batch = UnsafeUtil.getInt(ptr); ptr += 4; val |= (batch&((1<<-rem)-1))<<(SIZE+rem); batch >>>= -rem; rem = 32+rem; @@ -342,13 +341,13 @@ public class SaveLoadSystem2 { light |= (byte) (blockLight&0xF0); } else { //Todo clean and optimize this (it can be optimized alot) - light |= (byte) (((Byte.toUnsignedInt(MemoryUtil.memGetByte(blockLight+(i>>1)))>>((i&1)*4))&0xF)<<4); + light |= (byte) (((Byte.toUnsignedInt(UnsafeUtil.getByte(blockLight+(i>>1)))>>((i&1)*4))&0xF)<<4); } if (allSameSkyLight) { light |= (byte) (skyLight&0xF); } else { //Todo clean and optimize this (it can be optimized alot) - light |= (byte) ((Byte.toUnsignedInt(MemoryUtil.memGetByte(skyLight+(i>>1)))>>((i&1)*4))&0xF); + light |= (byte) ((Byte.toUnsignedInt(UnsafeUtil.getByte(skyLight+(i>>1)))>>((i&1)*4))&0xF); } } @@ -386,7 +385,7 @@ public class SaveLoadSystem2 { if (rem != 0) batch |= (b << (32 - rem));//the shift does auto cutoff if (rem < SIZE) { - MemoryUtil.memPutInt(ptr, batch); + UnsafeUtil.putInt(ptr, batch); ptr += 4; batch = b >> rem; rem = 32 - (SIZE - rem); @@ -396,7 +395,7 @@ public class SaveLoadSystem2 { } } if (rem != 32) { - MemoryUtil.memPutInt(ptr, batch); ptr += 4; + UnsafeUtil.putInt(ptr, batch); ptr += 4; } System.err.println(ptr-aa.address); } @@ -405,7 +404,7 @@ public class SaveLoadSystem2 { { long ptr = aa.address; int rem = 32; - int batch = MemoryUtil.memGetInt(ptr); ptr += 4; + int batch = UnsafeUtil.getInt(ptr); ptr += 4; {//Block int SIZE = 20;// 20 bits per entry int msk = (1<>>= SIZE; rem -= SIZE; if (rem < 0) { - batch = MemoryUtil.memGetInt(ptr); ptr += 4; + batch = UnsafeUtil.getInt(ptr); ptr += 4; val |= (batch&((1<<-rem)-1))<<(SIZE+rem); batch >>>= -rem; rem = 32+rem; diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem3.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem3.java index 5467e31c..6095c462 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem3.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem3.java @@ -5,7 +5,7 @@ import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer; import me.cortex.voxy.common.world.other.Mapper; -import org.lwjgl.system.MemoryUtil; +import me.cortex.voxy.common.util.UnsafeUtil; public class SaveLoadSystem3 { public static final int STORAGE_VERSION = 0; @@ -44,7 +44,7 @@ public class SaveLoadSystem3 { MemoryBuffer buffer = cache.memoryBuffer().createUntrackedUnfreeableReference(); long ptr = buffer.address; - MemoryUtil.memPutLong(ptr, section.key); ptr += 8; + UnsafeUtil.putLong(ptr, section.key); ptr += 8; long metadataPtr = ptr; ptr += 8; long blockPtr = ptr; ptr += WorldSection.SECTION_VOLUME*2; @@ -52,9 +52,9 @@ public class SaveLoadSystem3 { short mapping = LUT.putIfAbsent(block, (short) LUT.size()); if (mapping == -1) { mapping = (short) (LUT.size()-1); - MemoryUtil.memPutLong(ptr, block); ptr+=8; + UnsafeUtil.putLong(ptr, block); ptr+=8; } - MemoryUtil.memPutShort(blockPtr, mapping); blockPtr+=2; + UnsafeUtil.putShort(blockPtr, mapping); blockPtr+=2; } if (LUT.size() >= 1<<16) { throw new IllegalStateException(); @@ -66,7 +66,7 @@ public class SaveLoadSystem3 { metadata |= Byte.toUnsignedLong(section.getNonEmptyChildren())<<16;//Next byte //5 bytes free - MemoryUtil.memPutLong(metadataPtr, metadata); + UnsafeUtil.putLong(metadataPtr, metadata); //TODO: do hash //TODO: rework the storage system to not need to do useless copies like this (this is an issue for serialization, deserialization has solved this already) @@ -75,7 +75,7 @@ public class SaveLoadSystem3 { public static boolean deserialize(WorldSection section, MemoryBuffer data) { long ptr = data.address; - long key = MemoryUtil.memGetLong(ptr); ptr += 8; + long key = UnsafeUtil.getLong(ptr); ptr += 8; if (section.key != key) { //throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key); @@ -83,15 +83,15 @@ public class SaveLoadSystem3 { return false; } - final long metadata = MemoryUtil.memGetLong(ptr); ptr += 8; + final long metadata = UnsafeUtil.getLong(ptr); ptr += 8; section.nonEmptyChildren = (byte) ((metadata>>>16)&0xFF); final long lutBasePtr = ptr + WorldSection.SECTION_VOLUME * 2; if (section.lvl == 0) { int nonEmptyBlockCount = 0; final var blockData = section.data; for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) { - final short lutId = MemoryUtil.memGetShort(ptr); ptr += 2; - final long blockId = MemoryUtil.memGetLong(lutBasePtr + Short.toUnsignedLong(lutId) * 8L); + final short lutId = UnsafeUtil.getShort(ptr); ptr += 2; + final long blockId = UnsafeUtil.getLong(lutBasePtr + Short.toUnsignedLong(lutId) * 8L); nonEmptyBlockCount += Mapper.isAir(blockId) ? 0 : 1; blockData[i] = blockId; } @@ -99,7 +99,7 @@ public class SaveLoadSystem3 { } else { final var blockData = section.data; for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) { - blockData[i] = MemoryUtil.memGetLong(lutBasePtr + Short.toUnsignedLong(MemoryUtil.memGetShort(ptr)) * 8L);ptr += 2; + blockData[i] = UnsafeUtil.getLong(lutBasePtr + Short.toUnsignedLong(UnsafeUtil.getShort(ptr)) * 8L);ptr += 2; } } ptr = lutBasePtr + (metadata & 0xFFFF) * 8L; diff --git a/src/main/java/me/cortex/voxy/common/world/other/Mapper.java b/src/main/java/me/cortex/voxy/common/world/other/Mapper.java index 07f9edc1..8516f6e4 100644 --- a/src/main/java/me/cortex/voxy/common/world/other/Mapper.java +++ b/src/main/java/me/cortex/voxy/common/world/other/Mapper.java @@ -18,7 +18,7 @@ import net.minecraft.world.level.block.Block; import net.minecraft.world.level.block.Blocks; import net.minecraft.world.level.block.LeavesBlock; import net.minecraft.world.level.block.state.BlockState; -import org.lwjgl.system.MemoryUtil; +//import org.lwjgl.system.MemoryUtil; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -194,11 +194,11 @@ public class Mapper { this.blockLock.unlock(); byte[] serialized = entry.serialize(); - ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length); + ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length); buffer.put(serialized); buffer.rewind(); this.storage.putIdMapping(entry.id | (BLOCK_STATE_TYPE<<30), buffer); - MemoryUtil.memFree(buffer); + //MemoryUtil.memFree(buffer); //Not needed for standard direct buffers, GC handles it if (this.newStateCallback!=null)this.newStateCallback.accept(entry); return entry; @@ -217,11 +217,11 @@ public class Mapper { this.biomeLock.unlock(); byte[] serialized = entry.serialize(); - ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length); + ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length); buffer.put(serialized); buffer.rewind(); this.storage.putIdMapping(entry.id | (BIOME_TYPE<<30), buffer); - MemoryUtil.memFree(buffer); + //MemoryUtil.memFree(buffer); if (this.newBiomeCallback!=null)this.newBiomeCallback.accept(entry); return entry; @@ -257,6 +257,14 @@ public class Mapper { return this.blockId2stateEntry.get(blockId).opacity; } + public int getIdForBiome(String biomeId) { + var entry = this.biome2biomeEntry.get(biomeId); + if (entry == null) { + entry = this.registerNewBiome(biomeId); + } + return entry.id; + } + public int getIdForBiome(Holder biome) { String biomeId = biome.unwrapKey().get().identifier().toString(); var entry = this.biome2biomeEntry.get(biomeId); @@ -318,11 +326,11 @@ public class Mapper { throw new IllegalStateException("State Id NOT THE SAME, very critically bad. arr:" + this.blockId2stateEntry.indexOf(entry) + " entry: " + entry.id); } byte[] serialized = entry.serialize(); - ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length); + ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length); buffer.put(serialized); buffer.rewind(); this.storage.putIdMapping(entry.id | (BLOCK_STATE_TYPE<<30), buffer); - MemoryUtil.memFree(buffer); + //MemoryUtil.memFree(buffer); } for (var entry : biomes) { @@ -331,11 +339,11 @@ public class Mapper { } byte[] serialized = entry.serialize(); - ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length); + ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length); buffer.put(serialized); buffer.rewind(); this.storage.putIdMapping(entry.id | (BIOME_TYPE<<30), buffer); - MemoryUtil.memFree(buffer); + //MemoryUtil.memFree(buffer); } this.storage.flush(); @@ -346,6 +354,11 @@ public class Mapper { } + public String getBiomeString(int id) { + if (id < 0 || id >= this.biomeId2biomeEntry.size()) return "minecraft:plains"; // Default fallback + return this.biomeId2biomeEntry.get(id).biome; + } + public static final class StateEntry { public final int id; public final BlockState state; diff --git a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java index 01bf2bd1..d0205f2f 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java @@ -25,30 +25,38 @@ public class VoxelIngestService { private final Service service; private record IngestSection(int cx, int cy, int cz, WorldEngine world, LevelChunkSection section, DataLayer blockLight, DataLayer skyLight){} private final ConcurrentLinkedDeque ingestQueue = new ConcurrentLinkedDeque<>(); + private static final int MAX_QUEUE_SIZE = 16384; + private int dropCount = 0; public VoxelIngestService(ServiceManager pool) { this.service = pool.createServiceNoCleanup(()->this::processJob, 5000, "Ingest service"); } private void processJob() { - var task = this.ingestQueue.pop(); - task.world.markActive(); + var task = this.ingestQueue.poll(); + if (task == null) return; + + try { + task.world.markActive(); - var section = task.section; - var vs = SECTION_CACHE.get().setPosition(task.cx, task.cy, task.cz); + var section = task.section; + var vs = SECTION_CACHE.get().setPosition(task.cx, task.cy, task.cz); - if (section.hasOnlyAir() && task.blockLight==null && task.skyLight==null) {//If the chunk section has lighting data, propagate it - WorldUpdater.insertUpdate(task.world, vs.zero()); - } else { - VoxelizedSection csec = WorldConversionFactory.convert( - SECTION_CACHE.get(), - task.world.getMapper(), - section.getStates(), - section.getBiomes(), - getLightingSupplier(task) - ); - WorldConversionFactory.mipSection(csec, task.world.getMapper()); - WorldUpdater.insertUpdate(task.world, csec); + if (section.hasOnlyAir() && task.blockLight==null && task.skyLight==null) {//If the chunk section has lighting data, propagate it + WorldUpdater.insertUpdate(task.world, vs.zero()); + } else { + VoxelizedSection csec = WorldConversionFactory.convert( + SECTION_CACHE.get(), + task.world.getMapper(), + section.getStates(), + section.getBiomes(), + getLightingSupplier(task) + ); + WorldConversionFactory.mipSection(csec, task.world.getMapper()); + WorldUpdater.insertUpdate(task.world, csec); + } + } catch (Exception e) { + Logger.error("Error ingesting section " + task.cx + ", " + task.cy + ", " + task.cz, e); } } @@ -88,6 +96,13 @@ public class VoxelIngestService { } public boolean enqueueIngest(WorldEngine engine, LevelChunk chunk) { + if (this.ingestQueue.size() > MAX_QUEUE_SIZE) { + // Queue full, drop ingest request to prevent OOM. + if ((this.dropCount++ % 1000) == 0) { + Logger.warn("VoxelIngestService queue full (" + this.ingestQueue.size() + "), dropping chunks. Dropped " + this.dropCount + " so far."); + } + return false; + } if (!this.service.isLive()) { return false; } diff --git a/src/main/java/me/cortex/voxy/commonImpl/VoxyCommon.java b/src/main/java/me/cortex/voxy/commonImpl/VoxyCommon.java index 213378a4..603cdf00 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/VoxyCommon.java +++ b/src/main/java/me/cortex/voxy/commonImpl/VoxyCommon.java @@ -44,7 +44,24 @@ public class VoxyCommon implements ModInitializer { @Override public void onInitialize() { + me.cortex.voxy.common.network.VoxyNetwork.register(); + if (IS_DEDICATED_SERVER) { + net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents.SERVER_STARTING.register(server -> { + VoxyCommon.setInstanceFactory(() -> new me.cortex.voxy.server.VoxyServerInstance(server)); + VoxyCommon.createInstance(); + }); + + net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents.SERVER_STOPPING.register(server -> { + VoxyCommon.shutdownInstance(); + }); + + net.fabricmc.fabric.api.event.lifecycle.v1.ServerChunkEvents.CHUNK_UNLOAD.register((world, chunk) -> { + if (me.cortex.voxy.common.config.VoxyServerConfig.CONFIG.ingestEnabled && chunk instanceof net.minecraft.world.level.chunk.LevelChunk levelChunk) { + me.cortex.voxy.common.world.service.VoxelIngestService.tryIngestChunk(me.cortex.voxy.commonImpl.WorldIdentifier.of(world), levelChunk); + } + }); + } } public interface IInstanceFactory {VoxyInstance create();} diff --git a/src/main/java/me/cortex/voxy/commonImpl/VoxyInstance.java b/src/main/java/me/cortex/voxy/commonImpl/VoxyInstance.java index fa5ecbb0..eefc9909 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/VoxyInstance.java +++ b/src/main/java/me/cortex/voxy/commonImpl/VoxyInstance.java @@ -23,6 +23,9 @@ public abstract class VoxyInstance { private final Thread worldCleaner; public final BooleanSupplier savingServiceRateLimiter;//Can run if this returns true protected final UnifiedServiceThreadPool threadPool; + //public UnifiedServiceThreadPool getThreadPool() { + // return this.threadPool; + //} protected final SectionSavingService savingService; protected final VoxelIngestService ingestService; @@ -165,6 +168,8 @@ public abstract class VoxyInstance { protected abstract SectionStorage createStorage(WorldIdentifier identifier); + protected void onWorldCreated(WorldEngine world) {} + private WorldEngine createWorld(WorldIdentifier identifier) { if (!this.isRunning) { throw new IllegalStateException("Cannot create world while not running"); @@ -176,6 +181,7 @@ public abstract class VoxyInstance { var world = new WorldEngine(this.createStorage(identifier), this); world.setSaveCallback(this.savingService::enqueueSave); this.activeWorlds.put(identifier, world); + this.onWorldCreated(world); return world; } diff --git a/src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java b/src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java index 5b77d6f6..cd808529 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java +++ b/src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java @@ -21,8 +21,8 @@ import net.minecraft.world.level.block.Block; import net.minecraft.world.level.block.Blocks; import net.minecraft.world.level.block.state.BlockState; import org.apache.commons.io.IOUtils; -import org.lwjgl.system.MemoryUtil; -import org.lwjgl.util.zstd.Zstd; +import me.cortex.voxy.common.util.UnsafeUtil; +import com.github.luben.zstd.Zstd; import org.tukaani.xz.BasicArrayCache; import org.tukaani.xz.ResettableArrayCache; import org.tukaani.xz.XZInputStream; @@ -76,7 +76,6 @@ public class DHImporter implements IDataImporter { private ByteBuffer zstdScratch; private ByteBuffer zstdScratch2; - private final long zstdDCtx; public WorkCTX(PreparedStatement stmt, int worldHeight) { this.stmt = stmt; @@ -84,14 +83,12 @@ public class DHImporter implements IDataImporter { this.storageCache = new long[64*16*worldHeight]; this.colScratch = new byte[1<<16]; this.section = VoxelizedSection.createEmpty(); - this.zstdDCtx = Zstd.ZSTD_createDCtx(); } public void free() { if (this.zstdScratch != null) { - MemoryUtil.memFree(this.zstdScratch); - MemoryUtil.memFree(this.zstdScratch2); - Zstd.ZSTD_freeDCtx(this.zstdDCtx); + UnsafeUtil.freeMemory(UnsafeUtil.getAddress(this.zstdScratch)); + UnsafeUtil.freeMemory(UnsafeUtil.getAddress(this.zstdScratch2)); } } } @@ -298,30 +295,32 @@ public class DHImporter implements IDataImporter { return new XZInputStream(IOUtils.toBufferedInputStream(in), -1, false, ctx.cache); } else if (decompressor == 4) { if (ctx.zstdScratch == null) { - ctx.zstdScratch = MemoryUtil.memAlloc(8196); - ctx.zstdScratch2 = MemoryUtil.memAlloc(8196); + ctx.zstdScratch = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(8196), 8196); + ctx.zstdScratch2 = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(8196), 8196); } ctx.zstdScratch.clear(); ctx.zstdScratch2.clear(); try(var channel = Channels.newChannel(in)) { while (IOUtils.read(channel, ctx.zstdScratch) == 0) { - var newBuffer = MemoryUtil.memAlloc(ctx.zstdScratch.position()*2); + int newSize = ctx.zstdScratch.position()*2; + var newBuffer = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(newSize), newSize); newBuffer.put(ctx.zstdScratch.rewind()); - MemoryUtil.memFree(ctx.zstdScratch); + UnsafeUtil.freeMemory(UnsafeUtil.getAddress(ctx.zstdScratch)); ctx.zstdScratch = newBuffer; } } ctx.zstdScratch.limit(ctx.zstdScratch.position()).rewind(); { - int decompSize = (int) Zstd.ZSTD_getFrameContentSize(ctx.zstdScratch); + int decompSize = (int) Zstd.decompressedSize(ctx.zstdScratch); if (ctx.zstdScratch2.capacity() < decompSize) { - MemoryUtil.memFree(ctx.zstdScratch2); - ctx.zstdScratch2 = MemoryUtil.memAlloc((int) (decompSize * 1.1)); + UnsafeUtil.freeMemory(UnsafeUtil.getAddress(ctx.zstdScratch2)); + int newSize = (int) (decompSize * 1.1); + ctx.zstdScratch2 = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(newSize), newSize); } } - long size = Zstd.ZSTD_decompressDCtx(ctx.zstdDCtx, ctx.zstdScratch, ctx.zstdScratch2); - if (Zstd.ZSTD_isError(size)) { - throw new IllegalStateException("ZSTD EXCEPTION: " + Zstd.ZSTD_getErrorName(size)); + long size = Zstd.decompress(ctx.zstdScratch2, ctx.zstdScratch); + if (Zstd.isError(size)) { + throw new IllegalStateException("ZSTD EXCEPTION: " + Zstd.getErrorName(size)); } ctx.zstdScratch2.limit((int) size); return new ByteBufferBackedInputStream(ctx.zstdScratch2); diff --git a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java index 1fbcc3e0..d6835ba2 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java @@ -30,7 +30,7 @@ import net.minecraft.world.level.chunk.status.ChunkStatus; import net.minecraft.world.level.chunk.storage.RegionFileVersion; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; -import org.lwjgl.system.MemoryUtil; +import me.cortex.voxy.common.util.UnsafeUtil; import java.io.DataInputStream; import java.io.File; @@ -334,7 +334,7 @@ public class WorldImporter implements IDataImporter { return; } for (int idx = 0; idx < 1024; idx++) { - int sectorMeta = Integer.reverseBytes(MemoryUtil.memGetInt(regionFile.address+idx*4));//Assumes little endian + int sectorMeta = Integer.reverseBytes(UnsafeUtil.getInt(regionFile.address+idx*4));//Assumes little endian if (sectorMeta == 0) { //Empty chunk continue; @@ -355,8 +355,8 @@ public class WorldImporter implements IDataImporter { { long base = regionFile.address + sectorStart * 4096L; int chunkLen = sectorCount * 4096; - int m = Integer.reverseBytes(MemoryUtil.memGetInt(base)); - byte b = MemoryUtil.memGetByte(base + 4L); + int m = Integer.reverseBytes(UnsafeUtil.getInt(base)); + byte b = UnsafeUtil.getByte(base + 4L); if (m == 0) { Logger.error("Chunk is allocated, but stream is missing"); } else { @@ -408,7 +408,7 @@ public class WorldImporter implements IDataImporter { private long offset = 0; @Override public int read() { - return MemoryUtil.memGetByte(data.address + (this.offset++)) & 0xFF; + return UnsafeUtil.getByte(data.address + (this.offset++)) & 0xFF; } @Override diff --git a/src/main/java/me/cortex/voxy/server/DirtyUpdateService.java b/src/main/java/me/cortex/voxy/server/DirtyUpdateService.java new file mode 100644 index 00000000..a68a1c98 --- /dev/null +++ b/src/main/java/me/cortex/voxy/server/DirtyUpdateService.java @@ -0,0 +1,124 @@ +package me.cortex.voxy.server; + +import me.cortex.voxy.common.Logger; +import me.cortex.voxy.common.config.VoxyServerConfig; +import me.cortex.voxy.common.network.VoxyNetwork; +import me.cortex.voxy.common.world.WorldEngine; +import me.cortex.voxy.common.world.WorldSection; +import me.cortex.voxy.commonImpl.WorldIdentifier; +import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking; +import net.minecraft.server.level.ServerPlayer; +import net.minecraft.world.level.ChunkPos; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class DirtyUpdateService { + private final VoxyServerInstance instance; + private final ConcurrentLinkedQueue dirtyQueue = new ConcurrentLinkedQueue<>(); + private final ConcurrentHashMap> batchedUpdates = new ConcurrentHashMap<>(); + + public DirtyUpdateService(VoxyServerInstance instance) { + this.instance = instance; + } + + private record DirtySection(WorldEngine world, WorldSection section, int updateFlags, int neighborMsk) {} + + public void onSectionDirty(WorldEngine world, WorldSection section, int updateFlags, int neighborMsk) { + // Filter out sections that are not important for clients or debounce + if ((updateFlags & WorldEngine.UPDATE_TYPE_BLOCK_BIT) != 0) { + // We need to keep the section alive until we process it + section.acquire(); + this.dirtyQueue.add(new DirtySection(world, section, updateFlags, neighborMsk)); + // Logger.info("Section dirty: " + section.lvl + " " + section.x + " " + section.y + " " + section.z); + } + } + + public void tick() { + // Process queue + DirtySection dirty; + int processed = 0; + // Logger.info("Dirty Queue Size: " + dirtyQueue.size()); + while ((dirty = this.dirtyQueue.poll()) != null && processed++ < 1000) { + processDirty(dirty); + dirty.section.release(); + } + } + + private void processDirty(DirtySection dirty) { + // 1. Check if we need to send this to anyone + // Get players in the world + var server = this.instance.getServer(); + // We need to find the world identifier for this engine + // This is tricky because WorldEngine doesn't explicitly store its Identifier in a way we can easily query back to ServerLevel + // But we can iterate server levels and match + + // Actually, WorldIdentifier.of(ServerLevel) gives us the ID. + // We can map WorldEngine -> WorldIdentifier if we stored it. + // VoxyInstance stores Map. + // We can iterate that? No, slow. + + // Let's assume we can get the world from the engine or we pass it. + // WorldEngine doesn't have the identifier. + // But we can fix that or work around it. + // VoxyServerInstance knows the mapping. + + // For now, let's look at how we can broadcast. + // We need to find players near the section. + // section.x, section.y, section.z are in LOD coordinates (lvl dependent). + + int lvl = dirty.section.lvl; + int sx = dirty.section.x; + int sy = dirty.section.y; + int sz = dirty.section.z; + int scale = 1 << lvl; + int sectionSize = 16 * scale; + + // Get ServerLevel. + // Issue: We don't have direct link from WorldEngine to ServerLevel. + // We can iterate server.getAllLevels() and check WorldIdentifier.of(level).getOrCreateEngine() == dirty.world + // This is slow if done every time. + // We should cache this reverse mapping in VoxyServerInstance or DirtyUpdateService. + + net.minecraft.server.level.ServerLevel matchLevel = null; + for (var level : server.getAllLevels()) { + var wi = WorldIdentifier.of(level); + if (this.instance.getNullable(wi) == dirty.world) { + matchLevel = level; + break; + } + } + + if (matchLevel == null) return; + + for (ServerPlayer player : matchLevel.players()) { + int simulationDistance = matchLevel.getServer().getPlayerList().getSimulationDistance() * 16; + int voxyDistance = VoxyServerConfig.CONFIG.viewDistance * 16; + int standardViewDist = server.getPlayerList().getViewDistance() * 16; + int m = (1 << (lvl + 1)) - 1; + for (int dx = 0; dx <= m; dx++) { + for (int dz = 0; dz <= m; dz++) { + double centerX = (((sx << (lvl + 1)) | dx) * sectionSize) + (sectionSize / 2.0); + double centerZ = (((sz << (lvl + 1)) | dz) * sectionSize) + (sectionSize / 2.0); + double dxp = player.getX() - centerX; + double dzp = player.getZ() - centerZ; + double distSq = dxp * dxp + dzp * dzp; + if (distSq > standardViewDist * standardViewDist && distSq < voxyDistance * voxyDistance) { + for (int dy = 0; dy <= m; dy++) { + int absX = (sx << (lvl + 1)) | dx; + int absY = (sy << (lvl + 1)) | dy; + int absZ = (sz << (lvl + 1)) | dz; + var voxelized = WorldSectionToVoxelizedConverter.convert(dirty.section, lvl, absX, absY, absZ); + if (voxelized.lvl0NonAirCount > 0) { + var payload = VoxyNetwork.LodUpdatePayload.create(voxelized, dirty.world.getMapper()); + ServerPlayNetworking.send(player, payload); + } + } + } + } + } + } + } +} diff --git a/src/main/java/me/cortex/voxy/server/PlayerLodTracker.java b/src/main/java/me/cortex/voxy/server/PlayerLodTracker.java new file mode 100644 index 00000000..3baa5878 --- /dev/null +++ b/src/main/java/me/cortex/voxy/server/PlayerLodTracker.java @@ -0,0 +1,182 @@ +package me.cortex.voxy.server; + +import me.cortex.voxy.client.core.util.RingTracker; +import me.cortex.voxy.common.config.VoxyServerConfig; +import me.cortex.voxy.common.network.VoxyNetwork; +import me.cortex.voxy.common.world.WorldEngine; +import me.cortex.voxy.common.world.WorldSection; +import me.cortex.voxy.commonImpl.WorldIdentifier; +import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking; +import net.minecraft.server.level.ServerLevel; +import net.minecraft.server.level.ServerPlayer; +import me.cortex.voxy.common.Logger; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class PlayerLodTracker { + private final VoxyServerInstance instance; + private final Map playerStates = new ConcurrentHashMap<>(); + + public PlayerLodTracker(VoxyServerInstance instance) { + this.instance = instance; + } + + private static class PlayerState { + final RingTracker tracker; + int lastX = Integer.MIN_VALUE; + int lastZ = Integer.MIN_VALUE; + + PlayerState(int radius, int x, int z) { + this.tracker = new RingTracker(radius, x, z, true); + this.lastX = x; + this.lastZ = z; + } + } + + public void onPlayerJoin(ServerPlayer player) { + int radius = VoxyServerConfig.CONFIG.viewDistance; + int x = player.chunkPosition().x; + int z = player.chunkPosition().z; + this.playerStates.put(player, new PlayerState(radius, x, z)); + processPlayer(player, this.playerStates.get(player)); + } + + public void onPlayerDisconnect(ServerPlayer player) { + this.playerStates.remove(player); + } + + public void tick() { + for (var entry : this.playerStates.entrySet()) { + ServerPlayer player = entry.getKey(); + PlayerState state = entry.getValue(); + + int x = player.chunkPosition().x; + int z = player.chunkPosition().z; + + // Only update if moved + if (x != state.lastX || z != state.lastZ) { + state.tracker.moveCenter(x, z); + state.lastX = x; + state.lastZ = z; + } + + // Process queue (limit to N updates per tick per player to avoid flooding) + // But RingTracker accumulates operations. We need to drain them. + // The RingTracker.process method executes callbacks for add/remove. + // We want to process some amount. + processPlayer(player, state); + } + } + + private void processPlayer(ServerPlayer player, PlayerState state) { + if (!(player.level() instanceof ServerLevel level)) return; + WorldIdentifier wi = WorldIdentifier.of(level); + WorldEngine engine = this.instance.getNullable(wi); + if (engine == null) return; + + int standardViewDist = level.getServer().getPlayerList().getViewDistance(); // Chunks + + // Use client-negotiated view distance if available (via ConfigSync or just assume Voxy Config) + // Ideally the client sends its requested Voxy distance. + // But for now, we use the server's Voxy config as the "max" distance we are willing to serve. + int maxVoxyDist = VoxyServerConfig.CONFIG.viewDistance; + + // We stop sending if distance > maxVoxyDist + // RingTracker is initialized with maxVoxyDist, so it won't generate events beyond that. + // But we should double check here? + + state.tracker.process(50, (x, z) -> { // Process up to 50 chunks per tick per player + // On Add (Load) + double dx = x - state.lastX; + double dz = z - state.lastZ; + double distSq = dx*dx + dz*dz; + + // 1. Check if inside Vanilla View Distance (Gap filling logic) + // If dist < standardViewDist, usually client has it. + // BUT as user requested, if standardViewDist is small (e.g. 5), we need to fill 5 -> 50. + // We use standardViewDist - 1 as the "safe" boundary where we are SURE client has it. + // If it's outside that, we send LOD. + + double vanillaBoundary = Math.max(0, standardViewDist - 1); + + // 2. Check if inside Voxy Max Distance + // This is implicitly handled by RingTracker radius, but good to be explicit. + if (distSq > vanillaBoundary * vanillaBoundary && distSq <= maxVoxyDist * maxVoxyDist) { + sendLod(player, engine, x, z); + } + }, (x, z) -> { + // On Remove (Unload) + // We don't explicit send unload packets for LODs usually, client manages its own cache/LRU? + // Or maybe we should? Voxy client has its own tracker. + // Sending unloads might be good to free memory. + // But currently there is no S2C_LodUnload packet. + // Client's RenderDistanceTracker handles unloading when it moves. + // So we don't need to do anything here. + }); + } + + private void sendLod(ServerPlayer player, WorldEngine engine, int x, int z) { + // Decide LOD level based on distance? + // For now, let's just send LOD 0 if it exists. + // Optimization: Distant things can be higher LOD. + + // Simple heuristic: + // 0-64 chunks (1024 blocks): L0 + // >64 chunks: L1? + + // Actually, Voxy client requests specific LODs usually. + // But since we are pushing, we should push what makes sense. + // Sending L0 for everything is safe but bandwidth heavy. + // Let's stick to L0 for now as a proof of concept to fix the "blank world" issue. + + int lvl = 0; + + // We need to acquire the section from storage. + // Warning: acquiring on main thread might lag if not in cache. + // We should probably offload this IO? + // WorldEngine.acquireIfExists loads from disk if not in memory? + // acquireIfExists(..., true) means "only return if loaded in memory"? + // No, check WorldEngine code. + // acquireIfExists(..., true) -> sectionTracker.acquire(..., true) -> "if (allowLoad) ..." + // It seems `acquireIfExists` might still load? + // Let's look at WorldEngine.java again. + // acquireIfExists calls acquire(..., true). + // ActiveSectionTracker.acquire(..., boolean load) + // If load is true, it loads. + + // We want to load if it exists in DB, but not generate if missing? + // Voxy doesn't really "generate" on demand in the same way, it ingests. + // If it's not in DB, it's empty/air. + + // We should use `engine.acquire` but we need to be careful about blocking main thread. + // Ideally we schedule a task on the Voxy thread pool to fetch and send. + + this.instance.getThreadPool().execute(() -> { + try { + int minSec = player.level().getMinSectionY(); + int maxSec = player.level().getMaxSectionY(); + + for (int y = minSec; y <= maxSec; y++) { + var sec = engine.acquire(lvl, x >> (lvl + 1), y >> (lvl + 1), z >> (lvl + 1)); + if (sec != null) { + try { + var voxelized = WorldSectionToVoxelizedConverter.convert(sec, lvl, x, y, z); + if (voxelized.lvl0NonAirCount > 0) { + var payload = VoxyNetwork.LodUpdatePayload.create(voxelized, engine.getMapper()); + ServerPlayNetworking.send(player, payload); + } + } finally { + sec.release(); + } + } else { + //Logger.warn("Failed to acquire section " + x + ", " + y + ", " + z); + } + } + } catch (Exception e) { + Logger.error("Error sending LOD to player", e); + } + }); + } +} diff --git a/src/main/java/me/cortex/voxy/server/VoxyServerInstance.java b/src/main/java/me/cortex/voxy/server/VoxyServerInstance.java new file mode 100644 index 00000000..a6b281b4 --- /dev/null +++ b/src/main/java/me/cortex/voxy/server/VoxyServerInstance.java @@ -0,0 +1,113 @@ +package me.cortex.voxy.server; + +import me.cortex.voxy.common.config.ConfigBuildCtx; +import me.cortex.voxy.common.config.VoxyServerConfig; +import me.cortex.voxy.common.config.section.SectionStorage; +import me.cortex.voxy.common.config.section.SectionStorageConfig; +import me.cortex.voxy.common.StorageConfigUtil; +import me.cortex.voxy.commonImpl.ImportManager; +import me.cortex.voxy.commonImpl.VoxyInstance; +import me.cortex.voxy.commonImpl.WorldIdentifier; +import net.minecraft.server.MinecraftServer; +import net.minecraft.world.level.storage.LevelResource; +import java.nio.file.Path; + +import me.cortex.voxy.common.thread.UnifiedServiceThreadPool; + +public class VoxyServerInstance extends VoxyInstance { + private final MinecraftServer server; + private final SectionStorageConfig storageConfig; + private final Path basePath; + + private final DirtyUpdateService dirtyUpdateService; + private final PlayerLodTracker playerLodTracker; + + public VoxyServerInstance(MinecraftServer server) { + super(); + // Ensure config is loaded/created + // VoxyServerConfig.CONFIG.save(); // Don't force save, just accessing it will load it + + this.server = server; + this.basePath = server.getWorldPath(LevelResource.ROOT).resolve("voxy"); + this.storageConfig = StorageConfigUtil.getCreateStorageConfig(Config.class, c->c.version==1&&c.sectionStorageConfig!=null, ()->DEFAULT_STORAGE_CONFIG, this.basePath).sectionStorageConfig; + + this.setNumThreads(Math.max(2, Runtime.getRuntime().availableProcessors() / 2)); + + this.dirtyUpdateService = new DirtyUpdateService(this); + this.playerLodTracker = new PlayerLodTracker(this); + + // Start a service to tick the dirty service + this.threadPool.serviceManager.createServiceNoCleanup(() -> this.dirtyUpdateService::tick, VoxyServerConfig.CONFIG.dirtyUpdateDelay * 50, "DirtyUpdateService"); + + // Start service for player tracker (run every tick or so) + this.threadPool.serviceManager.createServiceNoCleanup(() -> this.playerLodTracker::tick, 50, "PlayerLodTracker"); + + // Register join handler to sync config and init tracker + net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents.JOIN.register((handler, sender, s) -> { + var payload = new me.cortex.voxy.common.network.VoxyNetwork.ConfigSyncPayload(VoxyServerConfig.CONFIG.viewDistance); + sender.sendPacket(payload); + this.playerLodTracker.onPlayerJoin(handler.player); + }); + + net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents.DISCONNECT.register((handler, server1) -> { + this.playerLodTracker.onPlayerDisconnect(handler.player); + }); + + // Register chunk load event for bulk ingest + net.fabricmc.fabric.api.event.lifecycle.v1.ServerChunkEvents.CHUNK_LOAD.register((world, chunk) -> { + if (VoxyServerConfig.CONFIG.ingestEnabled) { + me.cortex.voxy.common.world.service.VoxelIngestService.tryAutoIngestChunk(chunk); + } + }); + + net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents.END_SERVER_TICK.register(server1 -> { + this.dirtyUpdateService.tick(); + this.playerLodTracker.tick(); + }); + } + + @Override + protected void onWorldCreated(me.cortex.voxy.common.world.WorldEngine world) { + world.setDirtyCallback((section, updateFlags, neighborMsk) -> this.dirtyUpdateService.onSectionDirty(world, section, updateFlags, neighborMsk)); + } + + @Override + protected SectionStorage createStorage(WorldIdentifier identifier) { + var ctx = new ConfigBuildCtx(); + ctx.setProperty(ConfigBuildCtx.BASE_SAVE_PATH, this.basePath.toString()); + ctx.setProperty(ConfigBuildCtx.WORLD_IDENTIFIER, identifier.getWorldId()); + ctx.pushPath(ConfigBuildCtx.DEFAULT_STORAGE_PATH); + return this.storageConfig.build(ctx); + } + + @Override + protected ImportManager createImportManager() { + return new ImportManager(); + } + + @Override + public boolean isIngestEnabled(WorldIdentifier worldId) { + return VoxyServerConfig.CONFIG.ingestEnabled; + } + + private static class Config { + public int version = 1; + public SectionStorageConfig sectionStorageConfig; + } + + private static final Config DEFAULT_STORAGE_CONFIG; + static { + var config = new Config(); + config.sectionStorageConfig = StorageConfigUtil.createDefaultSerializer(); + DEFAULT_STORAGE_CONFIG = config; + } + + public MinecraftServer getServer() { + return server; + } + + @Override + public UnifiedServiceThreadPool getThreadPool() { + return this.threadPool; + } +} diff --git a/src/main/java/me/cortex/voxy/server/WorldSectionToVoxelizedConverter.java b/src/main/java/me/cortex/voxy/server/WorldSectionToVoxelizedConverter.java new file mode 100644 index 00000000..5adfc1e7 --- /dev/null +++ b/src/main/java/me/cortex/voxy/server/WorldSectionToVoxelizedConverter.java @@ -0,0 +1,52 @@ +package me.cortex.voxy.server; + +import me.cortex.voxy.common.world.WorldSection; +import me.cortex.voxy.common.world.other.Mapper; +import me.cortex.voxy.common.voxelization.VoxelizedSection; + +final class WorldSectionToVoxelizedConverter { + static VoxelizedSection convert(WorldSection src, int lvl, int absX, int absY, int absZ) { + long[] sdat = src._unsafeGetRawDataArray(); + VoxelizedSection out = VoxelizedSection.createEmpty().setPosition(absX, absY, absZ); + long[] vdat = out.section; + + int msk = (1 << (lvl + 1)) - 1; + int bx = (absX & msk) << (4 - lvl); + int by = (absY & msk) << (4 - lvl); + int bz = (absZ & msk) << (4 - lvl); + int baseSec = bx | (bz << 5) | (by << 10); + + if (lvl == 0) { + final int secMsk = 0b1100 | (0xF << 5) | (0xF << 10); + final int iSecMsk1 = (~secMsk) + 1; + int secIdx = 0; + int nonAir = 0; + for (int i = 0; i <= 0xFFF; i += 4) { + int cSecIdx = secIdx + baseSec; + vdat[i + 0] = sdat[cSecIdx + 0]; + vdat[i + 1] = sdat[cSecIdx + 1]; + vdat[i + 2] = sdat[cSecIdx + 2]; + vdat[i + 3] = sdat[cSecIdx + 3]; + nonAir += Mapper.isAir(vdat[i + 0]) ? 0 : 1; + nonAir += Mapper.isAir(vdat[i + 1]) ? 0 : 1; + nonAir += Mapper.isAir(vdat[i + 2]) ? 0 : 1; + nonAir += Mapper.isAir(vdat[i + 3]) ? 0 : 1; + secIdx = (secIdx + iSecMsk1) & secMsk; + } + out.lvl0NonAirCount = nonAir; + } else { + int baseVIdx = VoxelizedSection.getBaseIndexForLevel(lvl); + int secMsk = 0xF >> lvl; + secMsk |= (secMsk << 5) | (secMsk << 10); + int iSecMsk1 = (~secMsk) + 1; + int secIdx = 0; + int end = (0xFFF >> (lvl * 3)) + baseVIdx; + for (int i = baseVIdx; i <= end; i++) { + int cSecIdx = secIdx + baseSec; + vdat[i] = sdat[cSecIdx]; + secIdx = (secIdx + iSecMsk1) & secMsk; + } + } + return out; + } +} diff --git a/src/main/java/me/cortex/voxy/server/mixin/MixinServerLevel.java b/src/main/java/me/cortex/voxy/server/mixin/MixinServerLevel.java new file mode 100644 index 00000000..6d34e646 --- /dev/null +++ b/src/main/java/me/cortex/voxy/server/mixin/MixinServerLevel.java @@ -0,0 +1,43 @@ +package me.cortex.voxy.server.mixin; + +import me.cortex.voxy.common.config.VoxyServerConfig; +import me.cortex.voxy.common.world.service.VoxelIngestService; +import me.cortex.voxy.commonImpl.WorldIdentifier; +import net.minecraft.core.BlockPos; +import net.minecraft.core.SectionPos; +import net.minecraft.server.level.ServerLevel; +import net.minecraft.world.level.LightLayer; +import net.minecraft.world.level.block.state.BlockState; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.injection.At; +import org.spongepowered.asm.mixin.injection.Inject; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; + +import net.minecraft.world.level.Level; + +@Mixin(Level.class) +public class MixinServerLevel { + @Inject(method = "setBlock(Lnet/minecraft/core/BlockPos;Lnet/minecraft/world/level/block/state/BlockState;I)Z", at = @At("RETURN")) + private void voxy$onSetBlock(BlockPos pos, BlockState newState, int flags, CallbackInfoReturnable cir) { + if (!((Object)this instanceof ServerLevel)) return; + if (!cir.getReturnValue()) return; + if (!VoxyServerConfig.CONFIG.ingestEnabled) return; + + ServerLevel level = (ServerLevel) (Object) this; + WorldIdentifier wi = WorldIdentifier.of(level); + if (wi == null) return; + + SectionPos sp = SectionPos.of(pos); + var chunk = level.getChunk(sp.x(), sp.z()); + int sectionY = sp.y(); + if (sectionY < chunk.getMinSectionY() || sectionY > chunk.getMaxSectionY()) return; + + var section = chunk.getSection(chunk.getSectionIndex(sectionY)); + var lp = level.getLightEngine(); + + var blp = lp.getLayerListener(LightLayer.BLOCK).getDataLayerData(sp); + var slp = lp.getLayerListener(LightLayer.SKY).getDataLayerData(sp); + + VoxelIngestService.rawIngest(wi, section, sp.x(), sp.y(), sp.z(), blp==null?null:blp.copy(), slp==null?null:slp.copy()); + } +} diff --git a/src/main/resources/fabric.mod.json b/src/main/resources/fabric.mod.json index d659b2ed..defc69c6 100644 --- a/src/main/resources/fabric.mod.json +++ b/src/main/resources/fabric.mod.json @@ -40,7 +40,9 @@ "depends": { "minecraft": ["1.21.11"], "fabricloader": ">=0.14.22", - "fabric-api": ">=0.91.1", + "fabric-api": ">=0.91.1" + }, + "suggests": { "sodium": "=0.8.2" }, "accessWidener": "voxy.accesswidener"