spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Swap storage backend type from ByteBuffer to MemoryBuffer

Browse Source
This commit is contained in:
mcrcortex
2024-08-06 23:41:31 +10:00
parent 082143ed15
commit ee5bca7db5
16 changed files with 98 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/me/cortex/voxy/common/storage/StorageBackend.java
View File

@@ -1,6 +1,7 @@
package me.cortex.voxy.common.storage;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.util.MemoryBuffer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -10,9 +11,9 @@ import java.util.function.LongConsumer;
public abstract class StorageBackend {
public abstract void iterateStoredSectionPositions(LongConsumer consumer);
public abstract ByteBuffer getSectionData(long key);
public abstract MemoryBuffer getSectionData(long key);
public abstract void setSectionData(long key, ByteBuffer data);
public abstract void setSectionData(long key, MemoryBuffer data);
public abstract void deleteSectionData(long key);

6
src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java
View File

@@ -1,11 +1,13 @@
package me.cortex.voxy.common.storage;
import me.cortex.voxy.common.util.MemoryBuffer;
import java.nio.ByteBuffer;
public interface StorageCompressor {
ByteBuffer compress(ByteBuffer saveData);
MemoryBuffer compress(MemoryBuffer saveData);
ByteBuffer decompress(ByteBuffer saveData);
MemoryBuffer decompress(MemoryBuffer saveData);
void close();
}

11
src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java
View File

@@ -3,6 +3,9 @@ package me.cortex.voxy.common.storage.compressors;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;
import me.cortex.voxy.common.world.service.SectionSavingService;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
@@ -26,10 +29,10 @@ public class ZSTDCompressor implements StorageCompressor {
}
@Override
public ByteBuffer decompress(ByteBuffer saveData) {
var decompressed = MemoryUtil.memAlloc(32*32*32*8*2);
long size = ZSTD_decompress(decompressed, saveData);
decompressed.limit((int) size);
public MemoryBuffer decompress(MemoryBuffer saveData) {
var decompressed = new MemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE);
//TODO: mark the size of the decompressed data to verify its length later
long size = nZSTD_decompress(decompressed.address, decompressed.size, saveData.address, saveData.size);
return decompressed;
}

22
src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java
View File

@@ -8,6 +8,7 @@ import it.unimi.dsi.fastutil.objects.ObjectCollection;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import net.minecraft.util.math.random.RandomSeed;
import org.apache.commons.lang3.stream.Streams;
import org.lwjgl.system.MemoryUtil;
@@ -16,14 +17,14 @@ import java.nio.ByteBuffer;
import java.util.function.LongConsumer;
public class MemoryStorageBackend extends StorageBackend {
private final Long2ObjectMap<ByteBuffer>[] maps;
private final Long2ObjectMap<MemoryBuffer>[] maps;
private final Int2ObjectMap<ByteBuffer> idMappings = new Int2ObjectOpenHashMap<>();
public MemoryStorageBackend() {
this(4);
}
private Long2ObjectMap<ByteBuffer> getMap(long key) {
private Long2ObjectMap<MemoryBuffer> getMap(long key) {
return this.maps[(int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&(this.maps.length-1))];
}
@@ -44,14 +45,12 @@ public class MemoryStorageBackend extends StorageBackend {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
var map = this.getMap(key);
synchronized (map) {
var data = map.get(key);
if (data != null) {
var cpy = MemoryUtil.memAlloc(data.remaining());
MemoryUtil.memCopy(data, cpy);
return cpy;
return data.copy();
} else {
return null;
}
@@ -59,14 +58,13 @@ public class MemoryStorageBackend extends StorageBackend {
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
var map = this.getMap(key);
synchronized (map) {
var cpy = MemoryUtil.memAlloc(data.remaining());
MemoryUtil.memCopy(data, cpy);
var cpy = data.copy();
var old = map.put(key, cpy);
if (old != null) {
MemoryUtil.memFree(old);
old.free();
}
}
}
@@ -77,7 +75,7 @@ public class MemoryStorageBackend extends StorageBackend {
synchronized (map) {
var data = map.remove(key);
if (data != null) {
MemoryUtil.memFree(data);
data.free();
}
}
}
@@ -115,7 +113,7 @@ public class MemoryStorageBackend extends StorageBackend {
@Override
public void close() {
Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryUtil::memFree);
Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryBuffer::free);
this.idMappings.values().forEach(MemoryUtil::memFree);
}

12
src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java
View File

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;
import org.lwjgl.util.lmdb.MDBVal;
@@ -89,7 +91,7 @@ public class LMDBStorageBackend extends StorageBackend {
}
//TODO: make batch get and updates
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
return this.synchronizedTransaction(() -> this.sectionDatabase.transaction(MDB_RDONLY, transaction->{
var buff = transaction.stack.malloc(8);
buff.putLong(0, key);
@@ -97,19 +99,19 @@ public class LMDBStorageBackend extends StorageBackend {
if (bb == null) {
return null;
}
var copy = MemoryUtil.memAlloc(bb.remaining());
MemoryUtil.memCopy(bb, copy);
var copy = new MemoryBuffer(bb.remaining());
UnsafeUtil.memcpy(MemoryUtil.memAddress(bb), copy.address, copy.size);
return copy;
}));
}
//TODO: pad data to like some alignemnt so that when the section gets saved or updated
// it can use the same allocation
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
this.resizingTransaction(() -> this.sectionDatabase.transaction(transaction->{
var keyBuff = transaction.stack.malloc(8);
keyBuff.putLong(0, key);
transaction.put(keyBuff, data, 0);
transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0);
return null;
}));
}

9
src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java
View File

@@ -6,6 +6,7 @@ import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
@@ -20,21 +21,21 @@ public class CompressionStorageAdaptor extends DelegatingStorageAdaptor {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
var data = this.delegate.getSectionData(key);
if (data == null) {
return null;
}
var decompressed = this.compressor.decompress(data);
MemoryUtil.memFree(data);
data.free();
return decompressed;
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
var cdata = this.compressor.compress(data);
this.delegate.setSectionData(key, cdata);
MemoryUtil.memFree(cdata);
cdata.free();
}
@Override

5
src/main/java/me/cortex/voxy/common/storage/other/DelegatingStorageAdaptor.java
View File

@@ -6,6 +6,7 @@ import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
@@ -22,12 +23,12 @@ public class DelegatingStorageAdaptor extends StorageBackend {
public void iterateStoredSectionPositions(LongConsumer consumer) {this.delegate.iterateStoredSectionPositions(consumer);}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
return this.delegate.getSectionData(key);
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
this.delegate.setSectionData(key, data);
}

5
src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java
View File

@@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import net.minecraft.util.math.random.RandomSeed;
import java.nio.ByteBuffer;
@@ -40,12 +41,12 @@ public class FragmentedStorageBackendAdaptor extends StorageBackend {
// multiple layers of spliced storage backends can be stacked
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
return this.backends[this.getSegmentId(key)].getSectionData(key);
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
this.backends[this.getSegmentId(key)].setSectionData(key, data);
}

6
src/main/java/me/cortex/voxy/common/storage/other/ReadonlyCachingLayer.java
View File

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.List;
@@ -19,7 +21,7 @@ public class ReadonlyCachingLayer extends StorageBackend {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
var result = this.cache.getSectionData(key);
if (result != null) {
return result;
@@ -37,7 +39,7 @@ public class ReadonlyCachingLayer extends StorageBackend {
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
this.cache.setSectionData(key, data);
}

5
src/main/java/me/cortex/voxy/common/storage/other/TranslocatingStorageAdaptor.java
View File

@@ -3,6 +3,7 @@ package me.cortex.voxy.common.storage.other;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.world.WorldEngine;
import java.nio.ByteBuffer;
@@ -47,7 +48,7 @@ public class TranslocatingStorageAdaptor extends DelegatingStorageAdaptor {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
for (var transform : this.transforms) {
long tpos = transform.transformIfInBox(key);
if (tpos != -1) {
@@ -72,7 +73,7 @@ public class TranslocatingStorageAdaptor extends DelegatingStorageAdaptor {
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
//Dont save data if its a transformed position
for (var transform : this.transforms) {
long tpos = transform.transformIfInBox(key);

16
src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java
View File

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;
import redis.clients.jedis.JedisPool;
@@ -36,7 +38,7 @@ public class RedisStorageBackend extends StorageBackend {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
try (var jedis = this.pool.getResource()) {
if (this.user != null) {
jedis.auth(this.user, this.password);
@@ -47,23 +49,21 @@ public class RedisStorageBackend extends StorageBackend {
return null;
}
//Need to copy to native memory
var buffer = MemoryUtil.memAlloc(result.length);
buffer.put(result);
buffer.rewind();
var buffer = new MemoryBuffer(result.length);
UnsafeUtil.memcpy(result, buffer.address);
return buffer;
}
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
try (var jedis = this.pool.getResource()) {
if (this.user != null) {
jedis.auth(this.user, this.password);
}
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
var buffer = new byte[(int) data.size];
UnsafeUtil.memcpy(data.address, buffer);
jedis.hset(WORLD, longToBytes(key), buffer);
}
}

16
src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java
View File

@@ -4,6 +4,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;
import org.rocksdb.*;
@@ -87,16 +89,15 @@ public class RocksDBStorageBackend extends StorageBackend {
}
@Override
public ByteBuffer getSectionData(long key) {
public MemoryBuffer getSectionData(long key) {
try {
var result = this.db.get(this.worldSections, longToBytes(key));
if (result == null) {
return null;
}
//Need to copy to native memory
var buffer = MemoryUtil.memAlloc(result.length);
buffer.put(result);
buffer.rewind();
var buffer = new MemoryBuffer(result.length);
UnsafeUtil.memcpy(result, buffer.address);
return buffer;
} catch (RocksDBException e) {
throw new RuntimeException(e);
@@ -104,11 +105,10 @@ public class RocksDBStorageBackend extends StorageBackend {
}
@Override
public void setSectionData(long key, ByteBuffer data) {
public void setSectionData(long key, MemoryBuffer data) {
try {
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
var buffer = new byte[(int) data.size];
UnsafeUtil.memcpy(data.address, buffer);
this.db.put(this.worldSections, longToBytes(key), buffer);
} catch (RocksDBException e) {
throw new RuntimeException(e);

15
src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java
View File

@@ -14,8 +14,23 @@ public class UnsafeUtil {
} catch (Exception e) {throw new RuntimeException(e);}
}
private static final long BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
public static void memcpy(long src, long dst, long length) {
UNSAFE.copyMemory(src, dst, length);
}
//Copy the entire length of src to the dst memory where dst is a byte array (source length from dst)
public static void memcpy(long src, byte[] dst) {
UNSAFE.copyMemory(0, src, dst, BYTE_ARRAY_BASE_OFFSET, dst.length);
}
//Copy the entire length of src to the dst memory where src is a byte array (source length from src)
public static void memcpy(byte[] src, long dst) {
UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length);
}
}

33
src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java
View File

@@ -2,6 +2,8 @@ package me.cortex.voxy.common.world;
import it.unimi.dsi.fastutil.longs.Long2ShortOpenHashMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
@@ -9,6 +11,7 @@ import java.nio.ByteBuffer;
import static org.lwjgl.util.zstd.Zstd.*;
public class SaveLoadSystem {
public static final int BIGGEST_SERIALIZED_SECTION_SIZE = 32 * 32 * 32 * 8 * 2;
public static int lin2z(int i) {
int x = i&0x1F;
@@ -25,7 +28,7 @@ public class SaveLoadSystem {
}
//TODO: Cache like long2short and the short and other data to stop allocs
public static ByteBuffer serialize(WorldSection section) {
public static MemoryBuffer serialize(WorldSection section) {
var data = section.copyData();
var compressed = new short[data.length];
Long2ShortOpenHashMap LUT = new Long2ShortOpenHashMap(data.length);
@@ -62,21 +65,22 @@ public class SaveLoadSystem {
}
raw.putLong(hash);
raw.limit(raw.position());
raw.rewind();
return raw;
//The amount of memory copies are not ideal
var out = new MemoryBuffer(raw.position());
UnsafeUtil.memcpy(MemoryUtil.memAddress(raw), out.address, out.size);
MemoryUtil.memFree(raw);
return out;
}
public static boolean deserialize(WorldSection section, ByteBuffer data, boolean ignoreMismatchPosition) {
public static boolean deserialize(WorldSection section, MemoryBuffer data, boolean ignoreMismatchPosition) {
long ptr = data.address;
long hash = 0;
long key = data.getLong();
int lutLen = data.getInt();
long key = MemoryUtil.memGetLong(ptr); ptr += 8;
int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4;
long[] lut = new long[lutLen];
hash = key^(lut.length*1293481298141L);
for (int i = 0; i < lutLen; i++) {
lut[i] = data.getLong();
lut[i] = MemoryUtil.memGetLong(ptr); ptr += 8;
hash *= 1230987149811L;
hash += 12831;
hash ^= lut[i];
@@ -89,7 +93,7 @@ public class SaveLoadSystem {
}
for (int i = 0; i < section.data.length; i++) {
section.data[z2lin(i)] = lut[data.getShort()];
section.data[z2lin(i)] = lut[MemoryUtil.memGetShort(ptr)]; ptr += 2;
}
long pHash = 99;
@@ -101,18 +105,13 @@ public class SaveLoadSystem {
}
hash ^= pHash;
long expectedHash = data.getLong();
long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8;
if (expectedHash != hash) {
//throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);
System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region");
return false;
}
if (data.hasRemaining()) {
//throw new IllegalStateException("Decompressed section had excess data");
System.err.println("Decompressed section had excess data removing region");
return false;
}
return true;
}
}

2
src/main/java/me/cortex/voxy/common/world/WorldEngine.java
View File

@@ -53,7 +53,7 @@ public class WorldEngine {
return 0;
}
} finally {
MemoryUtil.memFree(data);
data.free();
}
} else {
//TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished

2
src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java
View File

@@ -31,7 +31,7 @@ public class SectionSavingService {
section.inSaveQueue.set(false);
var saveData = SaveLoadSystem.serialize(section);
this.world.storage.setSectionData(section.key, saveData);
MemoryUtil.memFree(saveData);
saveData.free();
} catch (Exception e) {
e.printStackTrace();
MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error")));