slight rework on how world unload works, tweek rocksdb, move to WorldUpdater

src/main/java/me/cortex/voxy/client/VoxyClientInstance.java

@@ -2,12 +2,16 @@ package me.cortex.voxy.client;
 
 import me.cortex.voxy.client.config.VoxyConfig;
 import me.cortex.voxy.client.saver.ContextSelectionSystem;
+import me.cortex.voxy.common.util.Pair;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.commonImpl.IVoxyWorld;
 import me.cortex.voxy.commonImpl.ImportManager;
 import me.cortex.voxy.commonImpl.VoxyInstance;
 import net.minecraft.client.world.ClientWorld;
 
+import java.util.Random;
+import java.util.concurrent.ConcurrentLinkedDeque;
+
 public class VoxyClientInstance extends VoxyInstance {
     private static final ContextSelectionSystem SELECTOR = new ContextSelectionSystem();
 
@@ -25,6 +29,7 @@ public class VoxyClientInstance extends VoxyInstance {
         if (vworld == null) {
             vworld = this.createWorld(SELECTOR.getBestSelectionOrCreate(world).createSectionStorageBackend());
             ((IVoxyWorld)world).setWorldEngine(vworld);
+            //testDbPerformance2(vworld);
         } else {
             if (!this.activeWorlds.contains(vworld)) {
                 throw new IllegalStateException("World referenced does not exist in instance");
@@ -32,4 +37,64 @@ public class VoxyClientInstance extends VoxyInstance {
         }
         return vworld;
     }
+
+
+
+    private static void testDbPerformance(WorldEngine engine) {
+        Random r = new Random(123456);
+        r.nextLong();
+        long start = System.currentTimeMillis();
+        int c = 0;
+        long tA = 0;
+        long tR = 0;
+        for (int i = 0; i < 1_000_000; i++) {
+            if (i == 20_000) {
+                c = 0;
+                start = System.currentTimeMillis();
+            }
+            c++;
+            int x = (r.nextInt(256*2+2)-256);//-32
+            int z = (r.nextInt(256*2+2)-256);//-32
+            int y = r.nextInt(2)-1;
+            int lvl = 0;//r.nextInt(5);
+            long t = System.nanoTime();
+            var sec = engine.acquire(WorldEngine.getWorldSectionId(lvl, x>>lvl, y>>lvl, z>>lvl));
+            tA += System.nanoTime()-t;
+            t = System.nanoTime();
+            sec.release();
+            tR += System.nanoTime()-t;
+        }
+        long delta = System.currentTimeMillis() - start;
+        System.out.println("Total "+delta+"ms " + ((double)delta/c) + "ms average tA: " + tA + " tR: " + tR);
+    }
+    private static void testDbPerformance2(WorldEngine engine) {
+        Random r = new Random(123456);
+        r.nextLong();
+        ConcurrentLinkedDeque<Long> queue = new ConcurrentLinkedDeque<>();
+        var ser = engine.instanceIn.getThreadPool().createServiceNoCleanup("aa", 1, ()-> () ->{
+            var sec = engine.acquire(queue.poll());
+            sec.release();
+        });
+        int priming = 1_000_000;
+        for (int i = 0; i < 2_000_000+priming; i++) {
+            int x = (r.nextInt(256*2+2)-256)>>2;//-32
+            int z = (r.nextInt(256*2+2)-256)>>2;//-32
+            int y = r.nextInt(2)-1;
+            int lvl = 0;//r.nextInt(5);
+            queue.add(WorldEngine.getWorldSectionId(lvl, x>>lvl, y>>lvl, z>>lvl));
+        }
+        for (int i = 0; i < priming; i++) {
+            ser.execute();
+        }
+        ser.blockTillEmpty();
+        int c = queue.size();
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < c; i++) {
+            ser.execute();
+        }
+        ser.blockTillEmpty();
+        long delta = System.currentTimeMillis() - start;
+        ser.shutdown();
+        System.out.println("Total "+delta+"ms " + ((double)delta/c) + "ms average total, avg wrt threads: " + (((double)delta/c)*engine.instanceIn.getThreadPool().getThreadCount()) + "ms");
+    }
 }

src/main/java/me/cortex/voxy/client/VoxyCommands.java

@@ -63,12 +63,8 @@ public class VoxyCommands {
             ((IGetVoxyRenderSystem)wr).shutdownRenderer();
         }
         var w = ((IVoxyWorld)MinecraftClient.getInstance().world);
-        if (w != null) {
+        if (w != null) w.shutdownEngine();
-            if (w.getWorldEngine() != null) {
+
-                instance.stopWorld(w.getWorldEngine());
-            }
-            w.setWorldEngine(null);
-        }
         VoxyCommon.shutdownInstance();
         VoxyCommon.createInstance();
         if (wr!=null) {

src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java

@@ -46,13 +46,7 @@ public class VoxyConfigScreenFactory implements ModMenuApi {
             }
             //Shutdown world
             if (world != null && ON_SAVE_RELOAD_ALL) {
-                //This is a hack inserted for the client world thing
+                world.shutdownEngine();
-                //TODO: FIXME: MAKE BETTER
-                var engine = world.getWorldEngine();
-                if (engine != null) {
-                    VoxyCommon.getInstance().stopWorld(engine);
-                }
-                world.setWorldEngine(null);
             }
             //Shutdown instance
             if (ON_SAVE_RELOAD_ALL) {

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -136,25 +136,4 @@ public class VoxelCore {
 
 
 
-
-    private void testDbPerformance() {
-        Random r = new Random(123456);
-        r.nextLong();
-        long start = System.currentTimeMillis();
-        int c = 0;
-        for (int i = 0; i < 500_000; i++) {
-            if (i == 20_000) {
-                c = 0;
-                start = System.currentTimeMillis();
-            }
-            c++;
-            int x = (r.nextInt(256*2+2)-256)>>1;//-32
-            int z = (r.nextInt(256*2+2)-256)>>1;//-32
-            int y = 0;
-            int lvl = 0;//r.nextInt(5);
-            this.world.acquire(WorldEngine.getWorldSectionId(lvl, x>>lvl, y>>lvl, z>>lvl)).release();
-        }
-        long delta = System.currentTimeMillis() - start;
-        System.out.println("Total "+delta+"ms " + ((double)delta/c) + "ms average" );
-    }
 }

src/main/java/me/cortex/voxy/client/mixin/minecraft/MixinWorldRenderer.java

@@ -52,11 +52,7 @@ public abstract class MixinWorldRenderer implements IGetVoxyRenderSystem {
             this.shutdownRenderer();
 
             if (this.world != null) {
-                var engine = ((IVoxyWorld)this.world).getWorldEngine();
+                ((IVoxyWorld)this.world).shutdownEngine();
-                if (engine != null) {
-                    VoxyCommon.getInstance().stopWorld(engine);
-                }
-                ((IVoxyWorld)this.world).setWorldEngine(null);
             }
         }
     }

src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java

@@ -50,7 +50,9 @@ public class RocksDBStorageBackend extends StorageBackend {
         }
          */
 
-        final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().optimizeUniversalStyleCompaction();
+        final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()
+                .optimizeUniversalStyleCompaction()
+                .optimizeForPointLookup(128);
 
         final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(
             new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts),

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -26,7 +26,7 @@ public class WorldEngine {
     private final ActiveSectionTracker sectionTracker;
     private ISectionChangeCallback dirtyCallback;
     private ISectionSaveCallback saveCallback;
-    private volatile boolean isLive = true;
+    volatile boolean isLive = true;
 
     public void setDirtyCallback(ISectionChangeCallback callback) {
         this.dirtyCallback = callback;
@@ -117,89 +117,6 @@ public class WorldEngine {
         }
     }
 
-
-    //TODO: move this to auxilery class  so that it can take into account larger than 4 mip levels
-    //Executes an update to the world and automatically updates all the parent mip layers up to level 4 (e.g. where 1 chunk section is 1 block big)
-
-    //NOTE: THIS RUNS ON THE THREAD IT WAS EXECUTED ON, when this method exits, the calling method may assume that VoxelizedSection is no longer needed
-    public void insertUpdate(VoxelizedSection section) {//TODO: add a bitset of levels to update and if it should force update
-        if (!this.isLive) throw new IllegalStateException("World is not live");
-        boolean shouldCheckEmptiness = false;
-        WorldSection previousSection = null;
-
-        for (int lvl = 0; lvl < MAX_LOD_LAYER+1; lvl++) {
-            var worldSection = this.acquire(lvl, section.x >> (lvl + 1), section.y >> (lvl + 1), section.z >> (lvl + 1));
-
-            int emptinessStateChange = 0;
-            //Propagate the child existence state of the previous iteration to this section
-            if (lvl != 0 && shouldCheckEmptiness) {
-                emptinessStateChange = worldSection.updateEmptyChildState(previousSection);
-                //We kept the previous section acquired, so we need to release it
-                previousSection.release();
-                previousSection = null;
-            }
-
-
-            int msk = (1<<(lvl+1))-1;
-            int bx = (section.x&msk)<<(4-lvl);
-            int by = (section.y&msk)<<(4-lvl);
-            int bz = (section.z&msk)<<(4-lvl);
-
-            int nonAirCountDelta = 0;
-            boolean didStateChange = false;
-
-
-            {//Do a bunch of funny math
-                int baseVIdx = VoxelizedSection.getBaseIndexForLevel(lvl);
-                int baseSec = bx | (bz << 5) | (by << 10);
-                int secMsk = 0xF >> lvl;
-                secMsk |= (secMsk << 5) | (secMsk << 10);
-                var secD = worldSection.data;
-                for (int i = 0; i <= 0xFFF >> (lvl * 3); i++) {
-                    int secIdx = Integer.expand(i, secMsk)+baseSec;
-                    long newId = section.section[baseVIdx+i];
-                    long oldId = secD[secIdx]; secD[secIdx] = newId;
-                    nonAirCountDelta += Mapper.isAir(oldId) == Mapper.isAir(newId) ? 0 : (Mapper.isAir(newId) ? -1 : 1);
-                    didStateChange |= newId != oldId;
-                }
-            }
-
-            if (nonAirCountDelta != 0) {
-                worldSection.addNonEmptyBlockCount(nonAirCountDelta);
-                if (lvl == 0) {
-                    emptinessStateChange = worldSection.updateLvl0State() ? 2 : 0;
-                }
-            }
-
-            if (didStateChange||(emptinessStateChange!=0)) {
-                this.markDirty(worldSection, (didStateChange?UPDATE_TYPE_BLOCK_BIT:0)|(emptinessStateChange!=0?UPDATE_TYPE_CHILD_EXISTENCE_BIT:0));
-            }
-
-            //Need to release the section after using it
-            if (didStateChange||(emptinessStateChange==2)) {
-                if (emptinessStateChange==2) {
-                    //Major state emptiness change, bubble up
-                    shouldCheckEmptiness = true;
-                    //Dont release the section, it will be released on the next loop
-                    previousSection = worldSection;
-                } else {
-                    //Propagate up without state change
-                    shouldCheckEmptiness = false;
-                    previousSection = null;
-                    worldSection.release();
-                }
-            } else {
-                //If nothing changed just need to release, dont need to update parent mips
-                worldSection.release();
-                break;
-            }
-        }
-
-        if (previousSection != null) {
-            previousSection.release();
-        }
-    }
-
     public void addDebugData(List<String> debug) {
         debug.add("ACC/SCC: " + this.sectionTracker.getLoadedCacheCount()+"/"+this.sectionTracker.getSecondaryCacheSize());//Active cache count, Secondary cache counts
     }

src/main/java/me/cortex/voxy/common/world/WorldUpdater.java

@@ -0,0 +1,90 @@
+package me.cortex.voxy.common.world;
+
+import me.cortex.voxy.common.voxelization.VoxelizedSection;
+import me.cortex.voxy.common.world.other.Mapper;
+
+import static me.cortex.voxy.common.world.WorldEngine.*;
+
+public class WorldUpdater {
+    //TODO: move this to auxilery class  so that it can take into account larger than 4 mip levels
+    //Executes an update to the world and automatically updates all the parent mip layers up to level 4 (e.g. where 1 chunk section is 1 block big)
+
+    //NOTE: THIS RUNS ON THE THREAD IT WAS EXECUTED ON, when this method exits, the calling method may assume that VoxelizedSection is no longer needed
+    public static void insertUpdate(WorldEngine into, VoxelizedSection section) {//TODO: add a bitset of levels to update and if it should force update
+        if (!into.isLive) throw new IllegalStateException("World is not live");
+        boolean shouldCheckEmptiness = false;
+        WorldSection previousSection = null;
+
+        for (int lvl = 0; lvl < MAX_LOD_LAYER+1; lvl++) {
+            var worldSection = into.acquire(lvl, section.x >> (lvl + 1), section.y >> (lvl + 1), section.z >> (lvl + 1));
+
+            int emptinessStateChange = 0;
+            //Propagate the child existence state of the previous iteration to this section
+            if (lvl != 0 && shouldCheckEmptiness) {
+                emptinessStateChange = worldSection.updateEmptyChildState(previousSection);
+                //We kept the previous section acquired, so we need to release it
+                previousSection.release();
+                previousSection = null;
+            }
+
+
+            int msk = (1<<(lvl+1))-1;
+            int bx = (section.x&msk)<<(4-lvl);
+            int by = (section.y&msk)<<(4-lvl);
+            int bz = (section.z&msk)<<(4-lvl);
+
+            int nonAirCountDelta = 0;
+            boolean didStateChange = false;
+
+
+            {//Do a bunch of funny math
+                int baseVIdx = VoxelizedSection.getBaseIndexForLevel(lvl);
+                int baseSec = bx | (bz << 5) | (by << 10);
+                int secMsk = 0xF >> lvl;
+                secMsk |= (secMsk << 5) | (secMsk << 10);
+                var secD = worldSection.data;
+                for (int i = 0; i <= 0xFFF >> (lvl * 3); i++) {
+                    int secIdx = Integer.expand(i, secMsk)+baseSec;
+                    long newId = section.section[baseVIdx+i];
+                    long oldId = secD[secIdx]; secD[secIdx] = newId;
+                    nonAirCountDelta += Mapper.isAir(oldId) == Mapper.isAir(newId) ? 0 : (Mapper.isAir(newId) ? -1 : 1);
+                    didStateChange |= newId != oldId;
+                }
+            }
+
+            if (nonAirCountDelta != 0) {
+                worldSection.addNonEmptyBlockCount(nonAirCountDelta);
+                if (lvl == 0) {
+                    emptinessStateChange = worldSection.updateLvl0State() ? 2 : 0;
+                }
+            }
+
+            if (didStateChange||(emptinessStateChange!=0)) {
+                into.markDirty(worldSection, (didStateChange?UPDATE_TYPE_BLOCK_BIT:0)|(emptinessStateChange!=0?UPDATE_TYPE_CHILD_EXISTENCE_BIT:0));
+            }
+
+            //Need to release the section after using it
+            if (didStateChange||(emptinessStateChange==2)) {
+                if (emptinessStateChange==2) {
+                    //Major state emptiness change, bubble up
+                    shouldCheckEmptiness = true;
+                    //Dont release the section, it will be released on the next loop
+                    previousSection = worldSection;
+                } else {
+                    //Propagate up without state change
+                    shouldCheckEmptiness = false;
+                    previousSection = null;
+                    worldSection.release();
+                }
+            } else {
+                //If nothing changed just need to release, dont need to update parent mips
+                worldSection.release();
+                break;
+            }
+        }
+
+        if (previousSection != null) {
+            previousSection.release();
+        }
+    }
+}

src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java

@@ -7,12 +7,14 @@ import me.cortex.voxy.common.voxelization.WorldConversionFactory;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.common.thread.ServiceSlice;
 import me.cortex.voxy.common.thread.ServiceThreadPool;
+import me.cortex.voxy.common.world.WorldUpdater;
 import me.cortex.voxy.commonImpl.IVoxyWorld;
 import net.minecraft.util.math.ChunkSectionPos;
 import net.minecraft.world.LightType;
 import net.minecraft.world.chunk.ChunkNibbleArray;
 import net.minecraft.world.chunk.ChunkSection;
 import net.minecraft.world.chunk.WorldChunk;
+import org.jetbrains.annotations.NotNull;
 
 import java.util.concurrent.ConcurrentLinkedDeque;
 
@@ -32,46 +34,51 @@ public class VoxelIngestService {
         var vs = SECTION_CACHE.get().setPosition(task.cx, task.cy, task.cz);
 
         if (section.isEmpty() && task.blockLight==null && task.skyLight==null) {//If the chunk section has lighting data, propagate it
-            task.world.insertUpdate(vs.zero());
+            WorldUpdater.insertUpdate(task.world, vs.zero());
         } else {
-            ILightingSupplier supplier = (x,y,z) -> (byte) 0;
-            var sla = task.skyLight;
-            var bla = task.blockLight;
-            boolean sl = sla != null && !sla.isUninitialized();
-            boolean bl = bla != null && !bla.isUninitialized();
-            if (sl || bl) {
-                if (sl && bl) {
-                    supplier = (x,y,z)-> {
-                        int block = Math.min(15,bla.get(x, y, z));
-                        int sky = Math.min(15,sla.get(x, y, z));
-                        return (byte) (sky|(block<<4));
-                    };
-                } else if (bl) {
-                    supplier = (x,y,z)-> {
-                        int block = Math.min(15,bla.get(x, y, z));
-                        int sky = 0;
-                        return (byte) (sky|(block<<4));
-                    };
-                } else {
-                    supplier = (x,y,z)-> {
-                        int block = 0;
-                        int sky = Math.min(15,sla.get(x, y, z));
-                        return (byte) (sky|(block<<4));
-                    };
-                }
-            }
             VoxelizedSection csec = WorldConversionFactory.convert(
                     SECTION_CACHE.get(),
                     task.world.getMapper(),
                     section.getBlockStateContainer(),
                     section.getBiomeContainer(),
-                    supplier
+                    getLightingSupplier(task)
             );
             WorldConversionFactory.mipSection(csec, task.world.getMapper());
-            task.world.insertUpdate(csec);
+            WorldUpdater.insertUpdate(task.world, csec);
         }
     }
 
+    @NotNull
+    private static ILightingSupplier getLightingSupplier(IngestSection task) {
+        ILightingSupplier supplier = (x,y,z) -> (byte) 0;
+        var sla = task.skyLight;
+        var bla = task.blockLight;
+        boolean sl = sla != null && !sla.isUninitialized();
+        boolean bl = bla != null && !bla.isUninitialized();
+        if (sl || bl) {
+            if (sl && bl) {
+                supplier = (x,y,z)-> {
+                    int block = Math.min(15,bla.get(x, y, z));
+                    int sky = Math.min(15,sla.get(x, y, z));
+                    return (byte) (sky|(block<<4));
+                };
+            } else if (bl) {
+                supplier = (x,y,z)-> {
+                    int block = Math.min(15,bla.get(x, y, z));
+                    int sky = 0;
+                    return (byte) (sky|(block<<4));
+                };
+            } else {
+                supplier = (x,y,z)-> {
+                    int block = 0;
+                    int sky = Math.min(15,sla.get(x, y, z));
+                    return (byte) (sky|(block<<4));
+                };
+            }
+        }
+        return supplier;
+    }
+
     private static boolean shouldIngestSection(ChunkSection section, int cx, int cy, int cz) {
         return true;
     }

src/main/java/me/cortex/voxy/commonImpl/IVoxyWorld.java

@@ -5,4 +5,5 @@ import me.cortex.voxy.common.world.WorldEngine;
 public interface IVoxyWorld {
     WorldEngine getWorldEngine();
     void setWorldEngine(WorldEngine engine);
+    void shutdownEngine();
 }

src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java

@@ -7,6 +7,7 @@ import me.cortex.voxy.common.util.Pair;
 import me.cortex.voxy.common.voxelization.VoxelizedSection;
 import me.cortex.voxy.common.voxelization.WorldConversionFactory;
 import me.cortex.voxy.common.world.WorldEngine;
+import me.cortex.voxy.common.world.WorldUpdater;
 import me.cortex.voxy.common.world.other.Mapper;
 import me.cortex.voxy.common.world.service.SectionSavingService;
 import net.minecraft.block.Block;
@@ -231,6 +232,9 @@ public class DHImporter implements IDataImporter {
                             Logger.warn("Could not find block state with data", encEntry.substring(b));
                         }
                     }
+                    if (block  == Blocks.AIR) {
+                        Logger.warn("Could not find block entry with id:", bId);
+                    }
                     blockId = this.engine.getMapper().getIdForBlockState(state);
                 }
             }
@@ -295,6 +299,7 @@ public class DHImporter implements IDataImporter {
                     }
                 }
             }
+
             if ((x+1)%16==0) {
                 for (int sz = 0; sz < 4; sz++) {
                     for (int sy = 0; sy < this.worldHeightSections; sy++) {
@@ -302,7 +307,7 @@ public class DHImporter implements IDataImporter {
                         WorldConversionFactory.mipSection(section, this.engine.getMapper());
 
                         section.setPosition(X*4+(x>>4), sy+(this.bottomOfWorld>>4), (Z*4)+sz);
-                        this.engine.insertUpdate(section);
+                        WorldUpdater.insertUpdate(this.engine, section);
                     }
 
                     int count = this.processedChunks.incrementAndGet();

src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java

@@ -9,6 +9,7 @@ import me.cortex.voxy.common.voxelization.WorldConversionFactory;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.common.thread.ServiceSlice;
 import me.cortex.voxy.common.thread.ServiceThreadPool;
+import me.cortex.voxy.common.world.WorldUpdater;
 import me.cortex.voxy.common.world.service.SectionSavingService;
 import net.minecraft.block.Block;
 import net.minecraft.block.BlockState;
@@ -472,7 +473,6 @@ public class WorldImporter implements IDataImporter {
         );
 
         WorldConversionFactory.mipSection(csec, this.world.getMapper());
-
+        WorldUpdater.insertUpdate(this.world, csec);
-        this.world.insertUpdate(csec);
     }
 }

src/main/java/me/cortex/voxy/commonImpl/mixin/minecraft/MixinWorld.java

@@ -25,4 +25,12 @@ public class MixinWorld implements IVoxyWorld {
         }
         this.voxyWorld = engine;
     }
+
+    @Override
+    public void shutdownEngine() {
+        if (this.voxyWorld != null && this.voxyWorld.instanceIn != null) {
+            this.voxyWorld.instanceIn.stopWorld(this.voxyWorld);
+            this.setWorldEngine(null);
+        }
+    }
 }