Swap storage backend type from ByteBuffer to MemoryBuffer

src/main/java/me/cortex/voxy/common/storage/StorageBackend.java

@@ -1,6 +1,7 @@
 package me.cortex.voxy.common.storage;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import me.cortex.voxy.common.util.MemoryBuffer;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -10,9 +11,9 @@ import java.util.function.LongConsumer;
 public abstract class StorageBackend {
     public abstract void iterateStoredSectionPositions(LongConsumer consumer);
 
-    public abstract ByteBuffer getSectionData(long key);
+    public abstract MemoryBuffer getSectionData(long key);
 
-    public abstract void setSectionData(long key, ByteBuffer data);
+    public abstract void setSectionData(long key, MemoryBuffer data);
 
     public abstract void deleteSectionData(long key);
 

src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java

@@ -1,11 +1,13 @@
 package me.cortex.voxy.common.storage;
 
+import me.cortex.voxy.common.util.MemoryBuffer;
+
 import java.nio.ByteBuffer;
 
 public interface StorageCompressor {
-    ByteBuffer compress(ByteBuffer saveData);
+    MemoryBuffer compress(MemoryBuffer saveData);
 
-    ByteBuffer decompress(ByteBuffer saveData);
+    MemoryBuffer decompress(MemoryBuffer saveData);
 
     void close();
 }

src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java

@@ -3,6 +3,9 @@ package me.cortex.voxy.common.storage.compressors;
 import me.cortex.voxy.common.storage.StorageCompressor;
 import me.cortex.voxy.common.storage.config.CompressorConfig;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.world.SaveLoadSystem;
+import me.cortex.voxy.common.world.service.SectionSavingService;
 import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
@@ -26,10 +29,10 @@ public class ZSTDCompressor implements StorageCompressor {
     }
 
     @Override
-    public ByteBuffer decompress(ByteBuffer saveData) {
+    public MemoryBuffer decompress(MemoryBuffer saveData) {
-        var decompressed = MemoryUtil.memAlloc(32*32*32*8*2);
+        var decompressed = new MemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE);
-        long size = ZSTD_decompress(decompressed, saveData);
+        //TODO: mark the size of the decompressed data to verify its length later
-        decompressed.limit((int) size);
+        long size = nZSTD_decompress(decompressed.address, decompressed.size, saveData.address, saveData.size);
         return decompressed;
     }
 

src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java

@@ -8,6 +8,7 @@ import it.unimi.dsi.fastutil.objects.ObjectCollection;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import net.minecraft.util.math.random.RandomSeed;
 import org.apache.commons.lang3.stream.Streams;
 import org.lwjgl.system.MemoryUtil;
@@ -16,14 +17,14 @@ import java.nio.ByteBuffer;
 import java.util.function.LongConsumer;
 
 public class MemoryStorageBackend extends StorageBackend {
-    private final Long2ObjectMap<ByteBuffer>[] maps;
+    private final Long2ObjectMap<MemoryBuffer>[] maps;
     private final Int2ObjectMap<ByteBuffer> idMappings = new Int2ObjectOpenHashMap<>();
 
     public MemoryStorageBackend() {
         this(4);
     }
 
-    private Long2ObjectMap<ByteBuffer> getMap(long key) {
+    private Long2ObjectMap<MemoryBuffer> getMap(long key) {
         return this.maps[(int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&(this.maps.length-1))];
     }
 
@@ -44,14 +45,12 @@ public class MemoryStorageBackend extends StorageBackend {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         var map = this.getMap(key);
         synchronized (map) {
             var data = map.get(key);
             if (data != null) {
-                var cpy = MemoryUtil.memAlloc(data.remaining());
+                return data.copy();
-                MemoryUtil.memCopy(data, cpy);
-                return cpy;
             } else {
                 return null;
             }
@@ -59,14 +58,13 @@ public class MemoryStorageBackend extends StorageBackend {
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         var map = this.getMap(key);
         synchronized (map) {
-            var cpy = MemoryUtil.memAlloc(data.remaining());
+            var cpy = data.copy();
-            MemoryUtil.memCopy(data, cpy);
             var old = map.put(key, cpy);
             if (old != null) {
-                MemoryUtil.memFree(old);
+                old.free();
             }
         }
     }
@@ -77,7 +75,7 @@ public class MemoryStorageBackend extends StorageBackend {
         synchronized (map) {
             var data = map.remove(key);
             if (data != null) {
-                MemoryUtil.memFree(data);
+                data.free();
             }
         }
     }
@@ -115,7 +113,7 @@ public class MemoryStorageBackend extends StorageBackend {
 
     @Override
     public void close() {
-        Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryUtil::memFree);
+        Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryBuffer::free);
         this.idMappings.values().forEach(MemoryUtil::memFree);
     }
 

src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;
 import org.lwjgl.util.lmdb.MDBVal;
 
@@ -89,7 +91,7 @@ public class LMDBStorageBackend extends StorageBackend {
     }
 
     //TODO: make batch get and updates
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         return this.synchronizedTransaction(() -> this.sectionDatabase.transaction(MDB_RDONLY, transaction->{
             var buff = transaction.stack.malloc(8);
             buff.putLong(0, key);
@@ -97,19 +99,19 @@ public class LMDBStorageBackend extends StorageBackend {
             if (bb == null) {
                 return null;
             }
-            var copy = MemoryUtil.memAlloc(bb.remaining());
+            var copy = new MemoryBuffer(bb.remaining());
-            MemoryUtil.memCopy(bb, copy);
+            UnsafeUtil.memcpy(MemoryUtil.memAddress(bb), copy.address, copy.size);
             return copy;
         }));
     }
 
     //TODO: pad data to like some alignemnt so that when the section gets saved or updated
     // it can use the same allocation
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         this.resizingTransaction(() -> this.sectionDatabase.transaction(transaction->{
             var keyBuff = transaction.stack.malloc(8);
             keyBuff.putLong(0, key);
-            transaction.put(keyBuff, data, 0);
+            transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0);
             return null;
         }));
     }

src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java

@@ -6,6 +6,7 @@ import me.cortex.voxy.common.storage.StorageCompressor;
 import me.cortex.voxy.common.storage.config.CompressorConfig;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
@@ -20,21 +21,21 @@ public class CompressionStorageAdaptor extends DelegatingStorageAdaptor {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         var data = this.delegate.getSectionData(key);
         if (data == null) {
             return null;
         }
         var decompressed = this.compressor.decompress(data);
-        MemoryUtil.memFree(data);
+        data.free();
         return decompressed;
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         var cdata = this.compressor.compress(data);
         this.delegate.setSectionData(key, cdata);
-        MemoryUtil.memFree(cdata);
+        cdata.free();
     }
 
     @Override

src/main/java/me/cortex/voxy/common/storage/other/DelegatingStorageAdaptor.java

@@ -6,6 +6,7 @@ import me.cortex.voxy.common.storage.StorageCompressor;
 import me.cortex.voxy.common.storage.config.CompressorConfig;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
@@ -22,12 +23,12 @@ public class DelegatingStorageAdaptor extends StorageBackend {
     public void iterateStoredSectionPositions(LongConsumer consumer) {this.delegate.iterateStoredSectionPositions(consumer);}
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         return this.delegate.getSectionData(key);
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         this.delegate.setSectionData(key, data);
     }
 

src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java

@@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import net.minecraft.util.math.random.RandomSeed;
 
 import java.nio.ByteBuffer;
@@ -40,12 +41,12 @@ public class FragmentedStorageBackendAdaptor extends StorageBackend {
     // multiple layers of spliced storage backends can be stacked
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         return this.backends[this.getSegmentId(key)].getSectionData(key);
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         this.backends[this.getSegmentId(key)].setSectionData(key, data);
     }
 

src/main/java/me/cortex/voxy/common/storage/other/ReadonlyCachingLayer.java

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -19,7 +21,7 @@ public class ReadonlyCachingLayer extends StorageBackend {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         var result = this.cache.getSectionData(key);
         if (result != null) {
             return result;
@@ -37,7 +39,7 @@ public class ReadonlyCachingLayer extends StorageBackend {
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         this.cache.setSectionData(key, data);
     }
 

src/main/java/me/cortex/voxy/common/storage/other/TranslocatingStorageAdaptor.java

@@ -3,6 +3,7 @@ package me.cortex.voxy.common.storage.other;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
 import me.cortex.voxy.common.world.WorldEngine;
 
 import java.nio.ByteBuffer;
@@ -47,7 +48,7 @@ public class TranslocatingStorageAdaptor extends DelegatingStorageAdaptor {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         for (var transform : this.transforms) {
             long tpos = transform.transformIfInBox(key);
             if (tpos != -1) {
@@ -72,7 +73,7 @@ public class TranslocatingStorageAdaptor extends DelegatingStorageAdaptor {
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         //Dont save data if its a transformed position
         for (var transform : this.transforms) {
             long tpos = transform.transformIfInBox(key);

src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;
 import redis.clients.jedis.JedisPool;
 
@@ -36,7 +38,7 @@ public class RedisStorageBackend extends StorageBackend {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         try (var jedis = this.pool.getResource()) {
             if (this.user != null) {
                 jedis.auth(this.user, this.password);
@@ -47,23 +49,21 @@ public class RedisStorageBackend extends StorageBackend {
                 return null;
             }
             //Need to copy to native memory
-            var buffer = MemoryUtil.memAlloc(result.length);
+            var buffer = new MemoryBuffer(result.length);
-            buffer.put(result);
+            UnsafeUtil.memcpy(result, buffer.address);
-            buffer.rewind();
             return buffer;
         }
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         try (var jedis = this.pool.getResource()) {
             if (this.user != null) {
                 jedis.auth(this.user, this.password);
             }
 
-            var buffer = new byte[data.remaining()];
+            var buffer = new byte[(int) data.size];
-            data.get(buffer);
+            UnsafeUtil.memcpy(data.address, buffer);
-            data.rewind();
             jedis.hset(WORLD, longToBytes(key), buffer);
         }
     }

src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import me.cortex.voxy.common.storage.StorageBackend;
 import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
 import me.cortex.voxy.common.storage.config.StorageConfig;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;
 import org.rocksdb.*;
 
@@ -87,16 +89,15 @@ public class RocksDBStorageBackend extends StorageBackend {
     }
 
     @Override
-    public ByteBuffer getSectionData(long key) {
+    public MemoryBuffer getSectionData(long key) {
         try {
             var result = this.db.get(this.worldSections, longToBytes(key));
             if (result == null) {
                 return null;
             }
             //Need to copy to native memory
-            var buffer = MemoryUtil.memAlloc(result.length);
+            var buffer = new MemoryBuffer(result.length);
-            buffer.put(result);
+            UnsafeUtil.memcpy(result, buffer.address);
-            buffer.rewind();
             return buffer;
         } catch (RocksDBException e) {
             throw new RuntimeException(e);
@@ -104,11 +105,10 @@ public class RocksDBStorageBackend extends StorageBackend {
     }
 
     @Override
-    public void setSectionData(long key, ByteBuffer data) {
+    public void setSectionData(long key, MemoryBuffer data) {
         try {
-            var buffer = new byte[data.remaining()];
+            var buffer = new byte[(int) data.size];
-            data.get(buffer);
+            UnsafeUtil.memcpy(data.address, buffer);
-            data.rewind();
             this.db.put(this.worldSections, longToBytes(key), buffer);
         } catch (RocksDBException e) {
             throw new RuntimeException(e);

src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java

@@ -14,8 +14,23 @@ public class UnsafeUtil {
         } catch (Exception e) {throw new RuntimeException(e);}
     }
 
+    private static final long BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
+
     public static void memcpy(long src, long dst, long length) {
         UNSAFE.copyMemory(src, dst, length);
     }
 
+
+
+    //Copy the entire length of src to the dst memory where dst is a byte array (source length from dst)
+    public static void memcpy(long src, byte[] dst) {
+        UNSAFE.copyMemory(0, src, dst, BYTE_ARRAY_BASE_OFFSET, dst.length);
+    }
+
+    //Copy the entire length of src to the dst memory where src is a byte array (source length from src)
+    public static void memcpy(byte[] src, long dst) {
+        UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length);
+    }
+
+
 }

src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java

@@ -2,6 +2,8 @@ package me.cortex.voxy.common.world;
 
 import it.unimi.dsi.fastutil.longs.Long2ShortOpenHashMap;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.util.UnsafeUtil;
 import org.lwjgl.system.MemoryUtil;
 
 import java.nio.ByteBuffer;
@@ -9,6 +11,7 @@ import java.nio.ByteBuffer;
 import static org.lwjgl.util.zstd.Zstd.*;
 
 public class SaveLoadSystem {
+    public static final int BIGGEST_SERIALIZED_SECTION_SIZE = 32 * 32 * 32 * 8 * 2;
 
     public static int lin2z(int i) {
         int x = i&0x1F;
@@ -25,7 +28,7 @@ public class SaveLoadSystem {
     }
 
     //TODO: Cache like long2short and the short and other data to stop allocs
-    public static ByteBuffer serialize(WorldSection section) {
+    public static MemoryBuffer serialize(WorldSection section) {
         var data = section.copyData();
         var compressed = new short[data.length];
         Long2ShortOpenHashMap LUT = new Long2ShortOpenHashMap(data.length);
@@ -62,21 +65,22 @@ public class SaveLoadSystem {
         }
 
         raw.putLong(hash);
-
+        //The amount of memory copies are not ideal
-        raw.limit(raw.position());
+        var out = new MemoryBuffer(raw.position());
-        raw.rewind();
+        UnsafeUtil.memcpy(MemoryUtil.memAddress(raw), out.address, out.size);
-
+        MemoryUtil.memFree(raw);
-        return raw;
+        return out;
     }
 
-    public static boolean deserialize(WorldSection section, ByteBuffer data, boolean ignoreMismatchPosition) {
+    public static boolean deserialize(WorldSection section, MemoryBuffer data, boolean ignoreMismatchPosition) {
+        long ptr = data.address;
         long hash = 0;
-        long key = data.getLong();
+        long key = MemoryUtil.memGetLong(ptr); ptr += 8;
-        int lutLen = data.getInt();
+        int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4;
         long[] lut = new long[lutLen];
         hash = key^(lut.length*1293481298141L);
         for (int i = 0; i < lutLen; i++) {
-            lut[i] = data.getLong();
+            lut[i] = MemoryUtil.memGetLong(ptr); ptr += 8;
             hash *= 1230987149811L;
             hash += 12831;
             hash ^= lut[i];
@@ -89,7 +93,7 @@ public class SaveLoadSystem {
         }
 
         for (int i = 0; i < section.data.length; i++) {
-            section.data[z2lin(i)] = lut[data.getShort()];
+            section.data[z2lin(i)] = lut[MemoryUtil.memGetShort(ptr)]; ptr += 2;
         }
 
         long pHash = 99;
@@ -101,18 +105,13 @@ public class SaveLoadSystem {
         }
         hash ^= pHash;
 
-        long expectedHash = data.getLong();
+        long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8;
         if (expectedHash != hash) {
             //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);
             System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region");
             return false;
         }
 
-        if (data.hasRemaining()) {
-            //throw new IllegalStateException("Decompressed section had excess data");
-            System.err.println("Decompressed section had excess data removing region");
-            return false;
-        }
         return true;
     }
 }

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -53,7 +53,7 @@ public class WorldEngine {
                     return 0;
                 }
             } finally {
-                MemoryUtil.memFree(data);
+                data.free();
             }
         } else {
             //TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished

src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java

@@ -31,7 +31,7 @@ public class SectionSavingService {
             section.inSaveQueue.set(false);
             var saveData = SaveLoadSystem.serialize(section);
             this.world.storage.setSectionData(section.key, saveData);
-            MemoryUtil.memFree(saveData);
+            saveData.free();
         } catch (Exception e) {
             e.printStackTrace();
             MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error")));