Improve ingest and import speed

build.gradle

@@ -77,6 +77,8 @@ dependencies {
     modRuntimeOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric"
     modCompileOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric"
 
+    modImplementation("maven.modrinth:lithium:mc1.21.4-0.14.7-fabric")
+
     //modRuntimeOnly "maven.modrinth:nvidium:0.2.6-beta"
     modCompileOnly "maven.modrinth:nvidium:0.2.8-beta"
 

src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java

@@ -63,12 +63,17 @@ public class WorldImportWrapper {
         var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false);
         MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar);
         long start = System.currentTimeMillis();
-        factory.create(this.importer, (a, b)->
+        long[] ticker = new long[1];
-                        MinecraftClient.getInstance().executeSync(()-> {
+        factory.create(this.importer, (a, b)-> {
-                            Taskbar.INSTANCE.setProgress(a, Math.max(1,b));
+                    if (System.currentTimeMillis() - ticker[0] > 50) {
-                            bossBar.setPercent(((float) a)/((float) Math.max(1,b)));
+                        ticker[0] = System.currentTimeMillis();
-                            bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks"));
+                        MinecraftClient.getInstance().executeSync(() -> {
-                        }),
+                            Taskbar.INSTANCE.setProgress(a, Math.max(1, b));
+                            bossBar.setPercent(((float) a) / ((float) Math.max(1, b)));
+                            bossBar.setName(Text.of("Voxy import: " + a + "/" + b + " chunks"));
+                        });
+                    }
+                },
                 chunkCount -> {
                     MinecraftClient.getInstance().executeSync(()-> {
                         MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID);

src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java

@@ -1,7 +1,5 @@
 package me.cortex.voxy.common.voxelization;
 
-import net.minecraft.block.BlockState;
-
 public interface ILightingSupplier {
-    byte supply(int x, int y, int z, BlockState state);
+    byte supply(int x, int y, int z);
 }

src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java

@@ -1,33 +1,124 @@
 package me.cortex.voxy.common.voxelization;
 
 import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap;
+import me.cortex.voxy.common.Logger;
 import me.cortex.voxy.common.util.Pair;
 import me.cortex.voxy.common.world.other.Mipper;
 import me.cortex.voxy.common.world.other.Mapper;
+import net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette;
 import net.minecraft.block.BlockState;
 import net.minecraft.registry.entry.RegistryEntry;
 import net.minecraft.world.biome.Biome;
-import net.minecraft.world.chunk.PalettedContainer;
+import net.minecraft.world.chunk.*;
-import net.minecraft.world.chunk.ReadableContainer;
 
 import java.util.WeakHashMap;
 
 public class WorldConversionFactory {
+    private static final class Cache {
+        private final int[] biomeCache = new int[4*4*4];
+        private final WeakHashMap<Mapper, Reference2IntOpenHashMap<BlockState>> localMapping = new WeakHashMap<>();
+        private int[] paletteCache = new int[1024];
+        private Reference2IntOpenHashMap<BlockState> getLocalMapping(Mapper mapper) {
+            return this.localMapping.computeIfAbsent(mapper, (a_)->new Reference2IntOpenHashMap<>());
+        }
+        private int[] getPaletteCache(int size) {
+            if (this.paletteCache.length < size) {
+                this.paletteCache = new int[size];
+            }
+            return this.paletteCache;
+        }
+    }
+
     //TODO: create a mapping for world/mapper -> local mapping
-    private static final ThreadLocal<Pair<int[], WeakHashMap<Mapper, Reference2IntOpenHashMap<BlockState>>>> THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new int[4*4*4], new WeakHashMap<>()));
+    private static final ThreadLocal<Cache> THREAD_LOCAL = ThreadLocal.withInitial(Cache::new);
+
+    private static void setupLocalPalette(Palette<BlockState> vp,Reference2IntOpenHashMap<BlockState> blockCache, Mapper mapper, int[] pc) {
+        {
+            if (vp instanceof ArrayPalette<BlockState>) {
+                for (int i = 0; i < vp.getSize(); i++) {
+                    var state = vp.get(i);
+                    int blockId = -1;
+                    if (state != null) {
+                        blockId = blockCache.getOrDefault(state, -1);
+                        if (blockId == -1) {
+                            blockId = mapper.getIdForBlockState(state);
+                            blockCache.put(state, blockId);
+                        }
+                    }
+                    pc[i] = blockId;
+                }
+            } else if (vp instanceof LithiumHashPalette<BlockState>) {
+                for (int i = 0; i < vp.getSize(); i++) {
+                    BlockState state = null;
+                    int blockId = -1;
+                    try { state = vp.get(i); } catch (Exception e) {}
+                    if (state != null) {
+                        blockId = blockCache.getOrDefault(state, -1);
+                        if (blockId == -1) {
+                            blockId = mapper.getIdForBlockState(state);
+                            blockCache.put(state, blockId);
+                        }
+                    }
+                    pc[i] = blockId;
+                }
+            } else {
+                if (vp instanceof BiMapPalette<BlockState> pal) {
+                    //var map = pal.map;
+                    //TODO: heavily optimize this by reading the map directly
+
+                    for (int i = 0; i < vp.getSize(); i++) {
+                        BlockState state = null;
+                        int blockId = -1;
+                        try { state = vp.get(i); } catch (Exception e) {}
+                        if (state != null) {
+                            blockId = blockCache.getOrDefault(state, -1);
+                            if (blockId == -1) {
+                                blockId = mapper.getIdForBlockState(state);
+                                blockCache.put(state, blockId);
+                            }
+                        }
+                        pc[i] = blockId;
+                    }
+
+                } else if (vp instanceof SingularPalette<BlockState>) {
+                    int blockId = -1;
+                    var state = vp.get(0);
+                    if (state != null) {
+                        blockId = blockCache.getOrDefault(state, -1);
+                        if (blockId == -1) {
+                            blockId = mapper.getIdForBlockState(state);
+                            blockCache.put(state, blockId);
+                        }
+                    }
+                    pc[0] = blockId;
+                } else {
+                    Logger.error("Unknown palette type: " + vp);
+                }
+            }
+        }
+    }
 
     public static VoxelizedSection convert(VoxelizedSection section,
                                            Mapper stateMapper,
                                            PalettedContainer<BlockState> blockContainer,
                                            ReadableContainer<RegistryEntry<Biome>> biomeContainer,
                                            ILightingSupplier lightSupplier) {
-        var threadLocal = THREAD_LOCAL.get();
+
-        var blockCache = threadLocal.right().computeIfAbsent(stateMapper, (mapper)->new Reference2IntOpenHashMap<>());
+        //Cheat by creating a local pallet then read the data directly
-        var biomes = threadLocal.left();
+
+
+        var cache = THREAD_LOCAL.get();
+        var blockCache = cache.getLocalMapping(stateMapper);
+
+        var biomes = cache.biomeCache;
         var data = section.section;
 
-        int blockId = -1;
+        var vp = blockContainer.data.palette;
-        BlockState block = null;
+        var pc = cache.getPaletteCache(vp.getSize());
+
+        setupLocalPalette(vp, blockCache, stateMapper, pc);
+
+
         {
             int i = 0;
             for (int y = 0; y < 4; y++) {
@@ -41,29 +132,15 @@ public class WorldConversionFactory {
 
         var bDat = blockContainer.data;
         var bStor = bDat.storage;
-        var bPall = bDat.palette;
         int i = 0;
         for (int y = 0; y < 16; y++) {
             for (int z = 0; z < 16; z++) {
                 for (int x = 0; x < 16; x++) {
-                    var state = bPall.get(bStor.get(i++));
+                    int bId = pc[bStor.get(i++)];
 
-                    byte light = lightSupplier.supply(x,y,z,state);
+                    byte light = lightSupplier.supply(x,y,z);
-                    if (!(state.isAir() && (light==0))) {
+                    if (!(bId==0 && (light==0))) {
-                        if (block != state) {
+                        data[G(x, y, z)] = Mapper.composeMappingId(light, bId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]);
-                            if (state.isAir()) {
-                                block = state;
-                                blockId = 0;
-                            } else {
-                                blockId = blockCache.getOrDefault(state, -1);
-                                if (blockId == -1) {
-                                    blockId = stateMapper.getIdForBlockState(state);
-                                    blockCache.put(state, blockId);
-                                }
-                                block = state;
-                            }
-                        }
-                        data[G(x, y, z)] = Mapper.composeMappingId(light, blockId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]);
                     } else {
                         data[G(x, y, z)] = Mapper.AIR;
                     }

src/main/java/me/cortex/voxy/common/world/WorldSection.java

@@ -33,7 +33,7 @@ public final class WorldSection {
 
 
     //TODO: should make it dynamically adjust the size allowance based on memory pressure/WorldSection allocation rate (e.g. is it doing a world import)
-    private static final int ARRAY_REUSE_CACHE_SIZE = 200;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes
+    private static final int ARRAY_REUSE_CACHE_SIZE = 300;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes
     //TODO: maybe just swap this to a ConcurrentLinkedDeque
     private static final Deque<long[]> ARRAY_REUSE_CACHE = new ArrayDeque<>(1024);
 

src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java

@@ -41,16 +41,16 @@ public class VoxelIngestService {
                         this.world.getMapper(),
                         section.getBlockStateContainer(),
                         section.getBiomeContainer(),
-                        (x, y, z, state) -> {
+                        (x, y, z) -> {
                             if (lighting == null || ((lighting.first() != null && lighting.first().isUninitialized())&&(lighting.second()!=null&&lighting.second().isUninitialized()))) {
                                 return (byte) 0;
                             } else {
                                 //Lighting is hell
                                 int block = lighting.first()!=null?Math.min(15,lighting.first().get(x, y, z)):0;
                                 int sky = lighting.second()!=null?Math.min(15,lighting.second().get(x, y, z)):0;
-                                if (block<state.getLuminance()) {
+                                //if (block<state.getLuminance()) {
-                                    block = state.getLuminance();
+                                //    block = state.getLuminance();
-                                }
+                                //}
                                 return (byte) (sky|(block<<4));
                             }
                         }

src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java

@@ -430,12 +430,13 @@ public class WorldImporter {
         }
         var blockStates = blockStatesRes.getPartialOrThrow();
         var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(this.defaultBiomeProvider);
+
         VoxelizedSection csec = WorldConversionFactory.convert(
                 SECTION_CACHE.get().setPosition(x, y, z),
                 this.world.getMapper(),
                 blockStates,
                 biomes,
-                (bx, by, bz, state) -> {
+                (bx, by, bz) -> {
                     int block = 0;
                     int sky = 0;
                     if (blockLight != null) {