Refactor storage to include a section backend

src/main/java/me/cortex/voxy/client/Voxy.java

@@ -3,19 +3,9 @@ package me.cortex.voxy.client;
 import me.cortex.voxy.client.core.VoxelCore;
 import me.cortex.voxy.client.saver.ContextSelectionSystem;
 import me.cortex.voxy.client.terrain.WorldImportCommand;
-import me.cortex.voxy.common.Logger;
-import me.cortex.voxy.common.config.Serialization;
-import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
-import me.cortex.voxy.common.storage.config.StorageConfig;
-import net.caffeinemc.mods.sodium.client.compatibility.environment.OsUtils;
 import net.fabricmc.api.ClientModInitializer;
         import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback;
-import net.fabricmc.loader.api.FabricLoader;
-import net.fabricmc.loader.api.ModContainer;
 import net.minecraft.client.world.ClientWorld;
-import org.apache.commons.lang3.SystemUtils;
-
-import java.util.Arrays;
 
 public class Voxy implements ClientModInitializer {
     @Override

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -4,12 +4,15 @@ import com.mojang.blaze3d.systems.RenderSystem;
 import me.cortex.voxy.client.config.VoxyConfig;
 import me.cortex.voxy.client.core.gl.Capabilities;
 import me.cortex.voxy.client.core.gl.GlBuffer;
+import me.cortex.voxy.client.core.model.ColourDepthTextureData;
 import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
+import me.cortex.voxy.client.core.model.ModelTextureBakery;
 import me.cortex.voxy.client.core.rendering.*;
 import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4;
 import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
 import me.cortex.voxy.client.core.rendering.post.PostProcessing;
 import me.cortex.voxy.client.core.rendering.util.DownloadStream;
+import me.cortex.voxy.client.core.rendering.util.RawDownloadStream;
 import me.cortex.voxy.client.core.util.IrisUtil;
 import me.cortex.voxy.client.saver.ContextSelectionSystem;
 import me.cortex.voxy.client.taskbar.Taskbar;
@@ -21,6 +24,7 @@ import me.cortex.voxy.common.thread.ServiceThreadPool;
 import me.cortex.voxy.common.world.WorldSection;
 import me.cortex.voxy.common.world.other.Mapper;
 import me.cortex.voxy.commonImpl.VoxyCommon;
+import net.minecraft.block.Blocks;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.client.gui.hud.ClientBossBar;
 import net.minecraft.client.render.Camera;
@@ -33,6 +37,7 @@ import net.minecraft.world.World;
 import net.minecraft.world.chunk.WorldChunk;
 import org.joml.Matrix4f;
 import org.lwjgl.opengl.GL11;
+import org.lwjgl.system.MemoryUtil;
 
 import java.io.File;
 import java.util.*;
@@ -127,7 +132,19 @@ public class VoxelCore {
         ).mulLocal(makeProjectionMatrix(16, 16*3000));
     }
 
+    //private static final ModelTextureBakery mtb = new ModelTextureBakery(16, 16);
+    //private static final RawDownloadStream downstream = new RawDownloadStream(1<<20);
     public void renderOpaque(MatrixStack matrices, double cameraX, double cameraY, double cameraZ) {
+        /*
+        int allocation = downstream.download(2*4*6*16*16, ptr->{
+        });
+        mtb.renderFacesToStream(Blocks.WHITE_STAINED_GLASS.getDefaultState(), 123456, false, downstream.getBufferId(), allocation);
+        downstream.submit();
+        downstream.tick();
+         */
+        //if (true) return;
+
+
         if (IrisUtil.irisShadowActive()) {
             return;
         }
@@ -187,8 +204,11 @@ public class VoxelCore {
 
         this.postProcessing.renderPost(projection, RenderSystem.getProjectionMatrix(), boundFB);
 
+
+
     }
 
+
     public void addDebugInfo(List<String> debug) {
         debug.add("");
         debug.add("");

src/main/java/me/cortex/voxy/client/saver/ContextSelectionSystem.java

@@ -1,13 +1,15 @@
 package me.cortex.voxy.client.saver;
 
 import me.cortex.voxy.client.config.VoxyConfig;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.Logger;
-import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
+import me.cortex.voxy.common.config.section.SectionStorageConfig;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.section.SectionSerializationStorage;
+import me.cortex.voxy.common.config.section.SectionStorage;
+import me.cortex.voxy.common.config.compressors.ZSTDCompressor;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
 import me.cortex.voxy.common.config.Serialization;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.other.CompressionStorageAdaptor;
-import me.cortex.voxy.common.storage.other.CompressionStorageAdaptor;
+import me.cortex.voxy.common.config.storage.rocksdb.RocksDBStorageBackend;
-import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.common.thread.ServiceThreadPool;
 import net.minecraft.client.MinecraftClient;
@@ -26,7 +28,7 @@ public class ContextSelectionSystem {
     public static class WorldConfig {
         public int minYOverride = Integer.MAX_VALUE;
         public int maxYOverride = Integer.MIN_VALUE;
-        public StorageConfig storageConfig;
+        public SectionStorageConfig sectionStorageConfig;
     }
     public static final String DEFAULT_STORAGE_CONFIG;
     static {
@@ -42,7 +44,10 @@ public class ContextSelectionSystem {
         compression.delegate = baseDB;
         compression.compressor = compressor;
 
-        config.storageConfig = compression;
+        var serializer = new SectionSerializationStorage.Config();
+        serializer.storage = compression;
+        config.sectionStorageConfig = serializer;
+
         DEFAULT_STORAGE_CONFIG = Serialization.GSON.toJson(config);
 
         if (Serialization.GSON.fromJson(DEFAULT_STORAGE_CONFIG, WorldConfig.class) == null) {
@@ -71,10 +76,12 @@ public class ContextSelectionSystem {
                     if (this.config == null) {
                         throw new IllegalStateException("Config deserialization null, reverting to default");
                     }
+                    if (this.config.sectionStorageConfig == null) {
+                        throw new IllegalStateException("Config section storage null, reverting to default");
+                    }
                     return;
                 } catch (Exception e) {
-                    System.err.println("Failed to load the storage configuration file, resetting it to default");
+                    Logger.error("Failed to load the storage configuration file, resetting it to default, this will probably break your save if you used a custom storage config", e);
-                    e.printStackTrace();
                 }
             }
 
@@ -89,16 +96,16 @@ public class ContextSelectionSystem {
             }
         }
 
-        public StorageBackend createStorageBackend() {
+        public SectionStorage createSectionStorageBackend() {
             var ctx = new ConfigBuildCtx();
             ctx.setProperty(ConfigBuildCtx.BASE_SAVE_PATH, this.selectionFolder.toString());
             ctx.setProperty(ConfigBuildCtx.WORLD_IDENTIFIER, this.worldId);
             ctx.pushPath(ConfigBuildCtx.DEFAULT_STORAGE_PATH);
-            return this.config.storageConfig.build(ctx);
+            return this.config.sectionStorageConfig.build(ctx);
         }
 
         public WorldEngine createEngine(ServiceThreadPool serviceThreadPool) {
-            return new WorldEngine(this.createStorageBackend(), serviceThreadPool, VoxyConfig.CONFIG.secondaryLruCacheSize);
+            return new WorldEngine(this.createSectionStorageBackend(), serviceThreadPool, VoxyConfig.CONFIG.secondaryLruCacheSize);
         }
 
         //Saves the config for the world selection or something, need to figure out how to make it work with dimensional configs maybe?

src/main/java/me/cortex/voxy/common/config/ConfigBuildCtx.java

@@ -1,8 +1,7 @@
-package me.cortex.voxy.common.storage.config;
+package me.cortex.voxy.common.config;
 
 import java.io.File;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Stack;

src/main/java/me/cortex/voxy/common/config/IMappingStorage.java

@@ -0,0 +1,12 @@
+package me.cortex.voxy.common.config;
+
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+
+import java.nio.ByteBuffer;
+
+public interface IMappingStorage {
+    void putIdMapping(int id, ByteBuffer data);
+    Int2ObjectOpenHashMap<byte[]> getIdMappingsData();
+    void flush();
+    void close();
+}

src/main/java/me/cortex/voxy/common/config/Serialization.java

@@ -4,8 +4,6 @@ import com.google.gson.*;
 import com.google.gson.reflect.TypeToken;
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
-import me.cortex.voxy.common.storage.config.StorageConfig;
 import net.fabricmc.loader.api.FabricLoader;
 
 import java.io.BufferedReader;

src/main/java/me/cortex/voxy/common/config/compressors/CompressorConfig.java

@@ -1,7 +1,7 @@
-package me.cortex.voxy.common.storage.config;
+package me.cortex.voxy.common.config.compressors;
 
+import me.cortex.voxy.common.config.ConfigBuildCtx;
 import me.cortex.voxy.common.config.Serialization;
-import me.cortex.voxy.common.storage.StorageCompressor;
 
 public abstract class CompressorConfig {
     static {

src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java

@@ -1,18 +1,11 @@
-package me.cortex.voxy.common.storage.compressors;
+package me.cortex.voxy.common.config.compressors;
 
-import me.cortex.voxy.common.storage.StorageCompressor;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.util.MemoryBuffer;
-import me.cortex.voxy.common.util.Pair;
 import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
-import me.cortex.voxy.common.util.UnsafeUtil;
 import me.cortex.voxy.common.world.SaveLoadSystem;
 import net.jpountz.lz4.LZ4Factory;
 import org.lwjgl.system.MemoryUtil;
-import org.tukaani.xz.*;
-
-import java.io.IOException;
 
 public class LZ4Compressor implements StorageCompressor {
     private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);

src/main/java/me/cortex/voxy/common/config/compressors/LZMACompressor.java

@@ -1,21 +1,16 @@
-package me.cortex.voxy.common.storage.compressors;
+package me.cortex.voxy.common.config.compressors;
 
-import me.cortex.voxy.common.storage.StorageCompressor;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.Pair;
 import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
 import me.cortex.voxy.common.world.SaveLoadSystem;
-import org.jetbrains.annotations.NotNull;
 import org.lwjgl.system.MemoryUtil;
 import org.tukaani.xz.*;
-import org.tukaani.xz.lzma.LZMAEncoder;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.ByteBuffer;
 
 public class LZMACompressor implements StorageCompressor {
     private static final ThreadLocal<Pair<byte[], ResettableArrayCache>> CACHE_THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new byte[SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE], new ResettableArrayCache(new ArrayCache())));

src/main/java/me/cortex/voxy/common/config/compressors/StorageCompressor.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.common.storage;
+package me.cortex.voxy.common.config.compressors;
 
 import me.cortex.voxy.common.util.MemoryBuffer;
 

src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java

@@ -1,8 +1,6 @@
-package me.cortex.voxy.common.storage.compressors;
+package me.cortex.voxy.common.config.compressors;
 
-import me.cortex.voxy.common.storage.StorageCompressor;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
 import me.cortex.voxy.common.world.SaveLoadSystem;

src/main/java/me/cortex/voxy/common/config/section/SectionSerializationStorage.java

@@ -0,0 +1,83 @@
+package me.cortex.voxy.common.config.section;
+
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import me.cortex.voxy.common.Logger;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageConfig;
+import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
+import me.cortex.voxy.common.world.SaveLoadSystem;
+import me.cortex.voxy.common.world.WorldSection;
+import me.cortex.voxy.common.world.other.Mapper;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+public class SectionSerializationStorage extends SectionStorage {
+    private final StorageBackend backend;
+    public SectionSerializationStorage(StorageBackend storageBackend) {
+        this.backend = storageBackend;
+    }
+
+    private static final ThreadLocalMemoryBuffer MEMORY_CACHE = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
+
+    public int loadSection(WorldSection into) {
+        var data = this.backend.getSectionData(into.key, MEMORY_CACHE.get().createUntrackedUnfreeableReference());
+        if (data != null) {
+            if (!SaveLoadSystem.deserialize(into, data)) {
+                this.backend.deleteSectionData(into.key);
+                //TODO: regenerate the section from children
+                Arrays.fill(into._unsafeGetRawDataArray(), Mapper.AIR);
+                Logger.error("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, removing");
+                return -1;
+            } else {
+                return 0;
+            }
+        } else {
+            //TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished
+            // the response should be put into the local db so that future data can just use that
+            // the server can also send arbitrary updates to the client for arbitrary lods
+            return 1;
+        }
+    }
+
+    @Override
+    public void saveSection(WorldSection section) {
+        var saveData = SaveLoadSystem.serialize(section);
+        this.backend.setSectionData(section.key, saveData);
+        saveData.free();
+    }
+
+    @Override
+    public void putIdMapping(int id, ByteBuffer data) {
+        this.backend.putIdMapping(id, data);
+    }
+
+    @Override
+    public Int2ObjectOpenHashMap<byte[]> getIdMappingsData() {
+        return this.backend.getIdMappingsData();
+    }
+
+    @Override
+    public void flush() {
+        this.backend.flush();
+    }
+
+    @Override
+    public void close() {
+        this.backend.close();
+    }
+
+    public static class Config extends SectionStorageConfig {
+        public StorageConfig storage;
+
+        @Override
+        public SectionStorage build(ConfigBuildCtx ctx) {
+            return new SectionSerializationStorage(this.storage.build(ctx));
+        }
+
+        public static String getConfigTypeName() {
+            return "Serializer";
+        }
+    }
+}

src/main/java/me/cortex/voxy/common/config/section/SectionStorage.java

@@ -0,0 +1,10 @@
+package me.cortex.voxy.common.config.section;
+
+import me.cortex.voxy.common.config.IMappingStorage;
+import me.cortex.voxy.common.world.WorldSection;
+
+public abstract class SectionStorage implements IMappingStorage {
+    public abstract int loadSection(WorldSection into);
+
+    public abstract void saveSection(WorldSection section);
+}

src/main/java/me/cortex/voxy/common/config/section/SectionStorageConfig.java

@@ -0,0 +1,12 @@
+package me.cortex.voxy.common.config.section;
+
+import me.cortex.voxy.common.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.Serialization;
+
+public abstract class SectionStorageConfig {
+    static {
+        Serialization.CONFIG_TYPES.add(SectionStorageConfig.class);
+    }
+
+    public abstract SectionStorage build(ConfigBuildCtx ctx);
+}

src/main/java/me/cortex/voxy/common/config/storage/StorageBackend.java

@@ -1,6 +1,7 @@
-package me.cortex.voxy.common.storage;
+package me.cortex.voxy.common.config.storage;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import me.cortex.voxy.common.config.IMappingStorage;
 import me.cortex.voxy.common.util.MemoryBuffer;
 
 import java.nio.ByteBuffer;
@@ -8,7 +9,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.function.LongConsumer;
 
-public abstract class StorageBackend {
+public abstract class StorageBackend implements IMappingStorage {
     public abstract void iterateStoredSectionPositions(LongConsumer consumer);
 
     //Implementation may use the scratch buffer as the return value, it MUST NOT free the scratch buffer
@@ -18,10 +19,6 @@ public abstract class StorageBackend {
 
     public abstract void deleteSectionData(long key);
 
-    public abstract void putIdMapping(int id, ByteBuffer data);
-
-    public abstract Int2ObjectOpenHashMap<byte[]> getIdMappingsData();
-
     public abstract void flush();
 
     public abstract void close();

src/main/java/me/cortex/voxy/common/config/storage/StorageConfig.java

@@ -1,7 +1,7 @@
-package me.cortex.voxy.common.storage.config;
+package me.cortex.voxy.common.config.storage;
 
+import me.cortex.voxy.common.config.ConfigBuildCtx;
 import me.cortex.voxy.common.config.Serialization;
-import me.cortex.voxy.common.storage.StorageBackend;
 
 import java.util.ArrayList;
 import java.util.List;

src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java

@@ -1,13 +1,13 @@
-package me.cortex.voxy.common.storage.inmemory;
+package me.cortex.voxy.common.config.storage.inmemory;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
 import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.objects.ObjectCollection;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import net.minecraft.util.math.random.RandomSeed;
 import org.apache.commons.lang3.stream.Streams;

src/main/java/me/cortex/voxy/common/config/storage/lmdb/Cursor.java

@@ -1,8 +1,8 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 import org.lwjgl.util.lmdb.MDBVal;
 
-import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
+import static me.cortex.voxy.common.config.storage.lmdb.LMDBInterface.E;
 import static org.lwjgl.util.lmdb.LMDB.*;
 
 public class Cursor implements AutoCloseable {

src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBInterface.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 import org.lwjgl.PointerBuffer;
 import org.lwjgl.system.MemoryStack;

src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java

@@ -1,10 +1,10 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import me.cortex.voxy.common.Logger;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;

src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionCallback.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 import org.lwjgl.system.MemoryStack;
 

src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionWrappedCallback.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 public interface TransactionWrappedCallback<T> {
     T exec(TransactionWrapper wrapper);

src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionWrapper.java

@@ -1,4 +1,4 @@
-package me.cortex.voxy.common.storage.lmdb;
+package me.cortex.voxy.common.config.storage.lmdb;
 
 import org.lwjgl.PointerBuffer;
 import org.lwjgl.system.MemoryStack;
@@ -6,7 +6,7 @@ import org.lwjgl.util.lmdb.MDBVal;
 
 import java.nio.ByteBuffer;
 
-import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
+import static me.cortex.voxy.common.config.storage.lmdb.LMDBInterface.E;
 import static org.lwjgl.system.MemoryStack.stackPush;
 import static org.lwjgl.util.lmdb.LMDB.*;
 

src/main/java/me/cortex/voxy/common/config/storage/other/BasicPathInsertionConfig.java

@@ -1,7 +1,7 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
 
 //Very simple config that adds a path to the config builder
 public class BasicPathInsertionConfig extends DelegateStorageConfig {

src/main/java/me/cortex/voxy/common/config/storage/other/CompressionStorageAdaptor.java

@@ -1,16 +1,10 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
-import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.compressors.StorageCompressor;
-import me.cortex.voxy.common.storage.StorageCompressor;
+import me.cortex.voxy.common.config.compressors.CompressorConfig;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
-import org.lwjgl.system.MemoryUtil;
-
-import java.nio.ByteBuffer;
-import java.util.List;
 
 //Compresses the section data
 public class CompressionStorageAdaptor extends DelegatingStorageAdaptor {

src/main/java/me/cortex/voxy/common/config/storage/other/ConditionalStorageBackendConfig.java

@@ -1,8 +1,8 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import org.apache.commons.lang3.NotImplementedException;
 
 //A conditional storage backend depending on build time config, this enables conditional backends depending on the

src/main/java/me/cortex/voxy/common/config/storage/other/DelegateStorageConfig.java

@@ -1,6 +1,6 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 
 import java.util.List;
 

src/main/java/me/cortex/voxy/common/config/storage/other/DelegatingStorageAdaptor.java

@@ -1,13 +1,8 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.StorageCompressor;
-import me.cortex.voxy.common.storage.config.CompressorConfig;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
-import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
 import java.util.List;

src/main/java/me/cortex/voxy/common/config/storage/other/FragmentedStorageBackendAdaptor.java

@@ -1,11 +1,11 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import net.minecraft.util.math.random.RandomSeed;
 

src/main/java/me/cortex/voxy/common/config/storage/other/ReadonlyCachingLayer.java

@@ -1,11 +1,10 @@
-package me.cortex.voxy.common.storage.other;
+package me.cortex.voxy.common.config.storage.other;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
-import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
 import java.util.List;

src/main/java/me/cortex/voxy/common/config/storage/redis/RedisStorageBackend.java

@@ -1,9 +1,9 @@
-package me.cortex.voxy.common.storage.redis;
+package me.cortex.voxy.common.config.storage.redis;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
 import redis.clients.jedis.JedisPool;

src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java

@@ -1,12 +1,11 @@
-package me.cortex.voxy.common.storage.rocksdb;
+package me.cortex.voxy.common.config.storage.rocksdb;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.storage.StorageBackend;
-import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.config.ConfigBuildCtx;
-import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.config.storage.StorageConfig;
 import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.util.UnsafeUtil;
-import me.cortex.voxy.common.world.SaveLoadSystem;
 import org.lwjgl.system.MemoryStack;
 import org.lwjgl.system.MemoryUtil;
 import org.rocksdb.*;

src/main/java/me/cortex/voxy/common/storage/config/GsonTypeAdaptor.java

@@ -1,13 +0,0 @@
-package me.cortex.voxy.common.storage.config;
-
-import com.google.gson.Gson;
-import com.google.gson.TypeAdapter;
-import com.google.gson.TypeAdapterFactory;
-import com.google.gson.reflect.TypeToken;
-
-public class GsonTypeAdaptor<T> implements TypeAdapterFactory {
-    @Override
-    public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
-        return null;
-    }
-}

src/main/java/me/cortex/voxy/common/util/ThreadLocalMemoryBuffer.java

@@ -1,12 +1,7 @@
 package me.cortex.voxy.common.util;
 
-import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
-
 import java.lang.ref.Cleaner;
 
-import static org.lwjgl.util.zstd.Zstd.ZSTD_createCCtx;
-import static org.lwjgl.util.zstd.Zstd.ZSTD_freeCCtx;
-
 public class ThreadLocalMemoryBuffer {
     private static final Cleaner CLEANER = Cleaner.create();
     private static MemoryBuffer createMemoryBuffer(long size) {

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -1,12 +1,12 @@
 package me.cortex.voxy.common.world;
 
 import me.cortex.voxy.common.Logger;
+import me.cortex.voxy.common.config.section.SectionStorage;
 import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
 import me.cortex.voxy.common.voxelization.VoxelizedSection;
 import me.cortex.voxy.common.world.other.Mapper;
 import me.cortex.voxy.common.world.service.SectionSavingService;
 import me.cortex.voxy.common.world.service.VoxelIngestService;
-import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.thread.ServiceThreadPool;
 
 import java.util.Arrays;
@@ -24,7 +24,7 @@ public class WorldEngine {
     public interface ISectionChangeCallback {void accept(WorldSection section, int updateFlags);}
 
 
-    public final StorageBackend storage;
+    public final SectionStorage storage;
     private final Mapper mapper;
     private final ActiveSectionTracker sectionTracker;
     public final VoxelIngestService ingestService;
@@ -39,43 +39,21 @@ public class WorldEngine {
     public Mapper getMapper() {return this.mapper;}
 
 
-    public WorldEngine(StorageBackend storageBackend, ServiceThreadPool serviceThreadPool, int cacheCount) {
+    public WorldEngine(SectionStorage storage, ServiceThreadPool serviceThreadPool, int cacheCount) {
-        this(storageBackend, serviceThreadPool, MAX_LOD_LAYERS, cacheCount);
+        this(storage, serviceThreadPool, MAX_LOD_LAYERS, cacheCount);
     }
 
-    private WorldEngine(StorageBackend storageBackend, ServiceThreadPool serviceThreadPool, int maxMipLayers, int cacheCount) {
+    private WorldEngine(SectionStorage storage, ServiceThreadPool serviceThreadPool, int maxMipLayers, int cacheCount) {
         this.maxMipLevels = maxMipLayers;
-        this.storage = storageBackend;
+        this.storage = storage;
         this.mapper = new Mapper(this.storage);
         //4 cache size bits means that the section tracker has 16 separate maps that it uses
-        this.sectionTracker = new ActiveSectionTracker(4, this::unsafeLoadSection, cacheCount);
+        this.sectionTracker = new ActiveSectionTracker(4, storage::loadSection, cacheCount);
 
         this.savingService = new SectionSavingService(this, serviceThreadPool);
         this.ingestService  = new VoxelIngestService(this, serviceThreadPool);
     }
 
-
-    private static final ThreadLocalMemoryBuffer MEMORY_CACHE = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
-    private int unsafeLoadSection(WorldSection into) {
-        var data = this.storage.getSectionData(into.key, MEMORY_CACHE.get().createUntrackedUnfreeableReference());
-        if (data != null) {
-            if (!SaveLoadSystem.deserialize(into, data)) {
-                this.storage.deleteSectionData(into.key);
-                //TODO: regenerate the section from children
-                Arrays.fill(into.data, Mapper.AIR);
-                Logger.error("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, removing");
-                return -1;
-            } else {
-                return 0;
-            }
-        } else {
-            //TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished
-            // the response should be put into the local db so that future data can just use that
-            // the server can also send arbitrary updates to the client for arbitrary lods
-            return 1;
-        }
-    }
-
     public WorldSection acquireIfExists(int lvl, int x, int y, int z) {
         return this.sectionTracker.acquire(lvl, x, y, z, true);
     }

src/main/java/me/cortex/voxy/common/world/WorldSection.java

@@ -80,6 +80,10 @@ public final class WorldSection {
         ATOMIC_STATE_HANDLE.set(this, 1);
     }
 
+    public long[] _unsafeGetRawDataArray() {
+        return this.data;
+    }
+
     @Override
     public int hashCode() {
         return ((x*1235641+y)*8127451+z)*918267913+lvl;
@@ -120,7 +124,15 @@ public final class WorldSection {
             }
         }
         if ((state>>1)==0) {
+            if (this.tracker != null) {
                 this.tracker.tryUnload(this);
+            } else {
+                //This should _ONLY_ ever happen when its an untracked section
+                // If it is, try release it
+                if (this.trySetFreed()) {
+                    this._releaseArray();
+                }
+            }
         }
         return state>>1;
     }
@@ -239,4 +251,8 @@ public final class WorldSection {
         } while (!NON_EMPTY_CHILD_HANDLE.compareAndSet(this, prev, next));
         return prev != next;
     }
+
+    public static WorldSection _createRawUntrackedUnsafeSection(int lvl, int x, int y, int z) {
+        return new WorldSection(lvl, x, y, z, null);
+    }
 }

src/main/java/me/cortex/voxy/common/world/other/Mapper.java

@@ -1,11 +1,11 @@
 package me.cortex.voxy.common.world.other;
 
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
-import me.cortex.voxy.common.storage.StorageBackend;
+import me.cortex.voxy.common.config.IMappingStorage;
+import me.cortex.voxy.common.config.section.SectionStorage;
 import net.minecraft.block.Block;
 import net.minecraft.block.BlockState;
 import net.minecraft.block.Blocks;
-import net.minecraft.block.RedstoneWireBlock;
 import net.minecraft.nbt.NbtCompound;
 import net.minecraft.nbt.NbtIo;
 import net.minecraft.nbt.NbtOps;
@@ -30,7 +30,7 @@ public class Mapper {
     private static final int BLOCK_STATE_TYPE = 1;
     private static final int BIOME_TYPE = 2;
 
-    private final StorageBackend storage;
+    private final IMappingStorage storage;
     public static final long UNKNOWN_MAPPING = -1;
     public static final long AIR = 0;
 
@@ -41,7 +41,7 @@ public class Mapper {
 
     private Consumer<StateEntry> newStateCallback;
     private Consumer<BiomeEntry> newBiomeCallback;
-    public Mapper(StorageBackend storage) {
+    public Mapper(IMappingStorage storage) {
         this.storage = storage;
         //Insert air since its a special entry (index 0)
         var airEntry = new StateEntry(0, Blocks.AIR.getDefaultState());

src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java

@@ -1,5 +1,6 @@
 package me.cortex.voxy.common.world.service;
 
+import me.cortex.voxy.common.Logger;
 import me.cortex.voxy.common.world.SaveLoadSystem;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.common.world.WorldSection;
@@ -28,12 +29,11 @@ public class SectionSavingService {
         section.assertNotFree();
         try {
             section.inSaveQueue.set(false);
-            var saveData = SaveLoadSystem.serialize(section);
+            this.world.storage.saveSection(section);
-            this.world.storage.setSectionData(section.key, saveData);
-            saveData.free();
         } catch (Exception e) {
-            e.printStackTrace();
+            String err = "Voxy saver had an exception while executing please check logs and report error";
-            MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error"), true));
+            Logger.error(err, e);
+            MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal(err), true));
         }
         section.release();
     }