From 6d683503cd282e1ee7283bf4650379312fbffec3 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:08:44 +1000 Subject: [PATCH] Improve ingest and import speed --- build.gradle | 2 + .../voxy/client/core/WorldImportWrapper.java | 17 ++- .../voxelization/ILightingSupplier.java | 4 +- .../voxelization/WorldConversionFactory.java | 129 ++++++++++++++---- .../voxy/common/world/WorldSection.java | 2 +- .../world/service/VoxelIngestService.java | 8 +- .../commonImpl/importers/WorldImporter.java | 3 +- 7 files changed, 124 insertions(+), 41 deletions(-) diff --git a/build.gradle b/build.gradle index 7053b1a2..d5ca9c26 100644 --- a/build.gradle +++ b/build.gradle @@ -77,6 +77,8 @@ dependencies { modRuntimeOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric" modCompileOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric" + modImplementation("maven.modrinth:lithium:mc1.21.4-0.14.7-fabric") + //modRuntimeOnly "maven.modrinth:nvidium:0.2.6-beta" modCompileOnly "maven.modrinth:nvidium:0.2.8-beta" diff --git a/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java b/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java index 80610974..194f7662 100644 --- a/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java +++ b/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java @@ -63,12 +63,17 @@ public class WorldImportWrapper { var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false); MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar); long start = System.currentTimeMillis(); - factory.create(this.importer, (a, b)-> - MinecraftClient.getInstance().executeSync(()-> { - Taskbar.INSTANCE.setProgress(a, Math.max(1,b)); - bossBar.setPercent(((float) a)/((float) Math.max(1,b))); - bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks")); - }), + long[] ticker = new long[1]; + factory.create(this.importer, (a, b)-> { + if (System.currentTimeMillis() - ticker[0] > 50) { + ticker[0] = System.currentTimeMillis(); + MinecraftClient.getInstance().executeSync(() -> { + Taskbar.INSTANCE.setProgress(a, Math.max(1, b)); + bossBar.setPercent(((float) a) / ((float) Math.max(1, b))); + bossBar.setName(Text.of("Voxy import: " + a + "/" + b + " chunks")); + }); + } + }, chunkCount -> { MinecraftClient.getInstance().executeSync(()-> { MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID); diff --git a/src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java b/src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java index b02badf6..b3f2a8e1 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java @@ -1,7 +1,5 @@ package me.cortex.voxy.common.voxelization; -import net.minecraft.block.BlockState; - public interface ILightingSupplier { - byte supply(int x, int y, int z, BlockState state); + byte supply(int x, int y, int z); } diff --git a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java index faf86c90..ac029028 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java @@ -1,33 +1,124 @@ package me.cortex.voxy.common.voxelization; import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap; +import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.util.Pair; import me.cortex.voxy.common.world.other.Mipper; import me.cortex.voxy.common.world.other.Mapper; +import net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette; import net.minecraft.block.BlockState; import net.minecraft.registry.entry.RegistryEntry; import net.minecraft.world.biome.Biome; -import net.minecraft.world.chunk.PalettedContainer; -import net.minecraft.world.chunk.ReadableContainer; +import net.minecraft.world.chunk.*; import java.util.WeakHashMap; public class WorldConversionFactory { + private static final class Cache { + private final int[] biomeCache = new int[4*4*4]; + private final WeakHashMap> localMapping = new WeakHashMap<>(); + private int[] paletteCache = new int[1024]; + private Reference2IntOpenHashMap getLocalMapping(Mapper mapper) { + return this.localMapping.computeIfAbsent(mapper, (a_)->new Reference2IntOpenHashMap<>()); + } + private int[] getPaletteCache(int size) { + if (this.paletteCache.length < size) { + this.paletteCache = new int[size]; + } + return this.paletteCache; + } + } + //TODO: create a mapping for world/mapper -> local mapping - private static final ThreadLocal>>> THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new int[4*4*4], new WeakHashMap<>())); + private static final ThreadLocal THREAD_LOCAL = ThreadLocal.withInitial(Cache::new); + + private static void setupLocalPalette(Palette vp,Reference2IntOpenHashMap blockCache, Mapper mapper, int[] pc) { + { + if (vp instanceof ArrayPalette) { + for (int i = 0; i < vp.getSize(); i++) { + var state = vp.get(i); + int blockId = -1; + if (state != null) { + blockId = blockCache.getOrDefault(state, -1); + if (blockId == -1) { + blockId = mapper.getIdForBlockState(state); + blockCache.put(state, blockId); + } + } + pc[i] = blockId; + } + } else if (vp instanceof LithiumHashPalette) { + for (int i = 0; i < vp.getSize(); i++) { + BlockState state = null; + int blockId = -1; + try { state = vp.get(i); } catch (Exception e) {} + if (state != null) { + blockId = blockCache.getOrDefault(state, -1); + if (blockId == -1) { + blockId = mapper.getIdForBlockState(state); + blockCache.put(state, blockId); + } + } + pc[i] = blockId; + } + } else { + if (vp instanceof BiMapPalette pal) { + //var map = pal.map; + //TODO: heavily optimize this by reading the map directly + + for (int i = 0; i < vp.getSize(); i++) { + BlockState state = null; + int blockId = -1; + try { state = vp.get(i); } catch (Exception e) {} + if (state != null) { + blockId = blockCache.getOrDefault(state, -1); + if (blockId == -1) { + blockId = mapper.getIdForBlockState(state); + blockCache.put(state, blockId); + } + } + pc[i] = blockId; + } + + } else if (vp instanceof SingularPalette) { + int blockId = -1; + var state = vp.get(0); + if (state != null) { + blockId = blockCache.getOrDefault(state, -1); + if (blockId == -1) { + blockId = mapper.getIdForBlockState(state); + blockCache.put(state, blockId); + } + } + pc[0] = blockId; + } else { + Logger.error("Unknown palette type: " + vp); + } + } + } + } public static VoxelizedSection convert(VoxelizedSection section, Mapper stateMapper, PalettedContainer blockContainer, ReadableContainer> biomeContainer, ILightingSupplier lightSupplier) { - var threadLocal = THREAD_LOCAL.get(); - var blockCache = threadLocal.right().computeIfAbsent(stateMapper, (mapper)->new Reference2IntOpenHashMap<>()); - var biomes = threadLocal.left(); + + //Cheat by creating a local pallet then read the data directly + + + var cache = THREAD_LOCAL.get(); + var blockCache = cache.getLocalMapping(stateMapper); + + var biomes = cache.biomeCache; var data = section.section; - int blockId = -1; - BlockState block = null; + var vp = blockContainer.data.palette; + var pc = cache.getPaletteCache(vp.getSize()); + + setupLocalPalette(vp, blockCache, stateMapper, pc); + + { int i = 0; for (int y = 0; y < 4; y++) { @@ -41,29 +132,15 @@ public class WorldConversionFactory { var bDat = blockContainer.data; var bStor = bDat.storage; - var bPall = bDat.palette; int i = 0; for (int y = 0; y < 16; y++) { for (int z = 0; z < 16; z++) { for (int x = 0; x < 16; x++) { - var state = bPall.get(bStor.get(i++)); + int bId = pc[bStor.get(i++)]; - byte light = lightSupplier.supply(x,y,z,state); - if (!(state.isAir() && (light==0))) { - if (block != state) { - if (state.isAir()) { - block = state; - blockId = 0; - } else { - blockId = blockCache.getOrDefault(state, -1); - if (blockId == -1) { - blockId = stateMapper.getIdForBlockState(state); - blockCache.put(state, blockId); - } - block = state; - } - } - data[G(x, y, z)] = Mapper.composeMappingId(light, blockId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]); + byte light = lightSupplier.supply(x,y,z); + if (!(bId==0 && (light==0))) { + data[G(x, y, z)] = Mapper.composeMappingId(light, bId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]); } else { data[G(x, y, z)] = Mapper.AIR; } diff --git a/src/main/java/me/cortex/voxy/common/world/WorldSection.java b/src/main/java/me/cortex/voxy/common/world/WorldSection.java index a2537fcb..cb9c7802 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldSection.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldSection.java @@ -33,7 +33,7 @@ public final class WorldSection { //TODO: should make it dynamically adjust the size allowance based on memory pressure/WorldSection allocation rate (e.g. is it doing a world import) - private static final int ARRAY_REUSE_CACHE_SIZE = 200;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes + private static final int ARRAY_REUSE_CACHE_SIZE = 300;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes //TODO: maybe just swap this to a ConcurrentLinkedDeque private static final Deque ARRAY_REUSE_CACHE = new ArrayDeque<>(1024); diff --git a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java index 6d2a765b..02f02dd1 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java @@ -41,16 +41,16 @@ public class VoxelIngestService { this.world.getMapper(), section.getBlockStateContainer(), section.getBiomeContainer(), - (x, y, z, state) -> { + (x, y, z) -> { if (lighting == null || ((lighting.first() != null && lighting.first().isUninitialized())&&(lighting.second()!=null&&lighting.second().isUninitialized()))) { return (byte) 0; } else { //Lighting is hell int block = lighting.first()!=null?Math.min(15,lighting.first().get(x, y, z)):0; int sky = lighting.second()!=null?Math.min(15,lighting.second().get(x, y, z)):0; - if (block { + (bx, by, bz) -> { int block = 0; int sky = 0; if (blockLight != null) {