diff --git a/build.gradle b/build.gradle index f37da568..c2b85a40 100644 --- a/build.gradle +++ b/build.gradle @@ -199,6 +199,8 @@ dependencies { include(implementation 'redis.clients:jedis:5.1.0') include(implementation('org.rocksdb:rocksdbjni:8.10.0')) include(implementation 'org.apache.commons:commons-pool2:2.12.0') + //include(implementation 'org.tukaani:xz:1.10') + include(implementation 'org.lz4:lz4-java:1.8.0') //implementation 'org.rocksdb:rocksdbjni:8.10.0' //implementation 'redis.clients:jedis:5.1.0' } diff --git a/src/main/java/me/cortex/voxy/common/storage/compressors/LZ4Compressor.java b/src/main/java/me/cortex/voxy/common/storage/compressors/LZ4Compressor.java new file mode 100644 index 00000000..aa730215 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/compressors/LZ4Compressor.java @@ -0,0 +1,57 @@ +package me.cortex.voxy.common.storage.compressors; + +import me.cortex.voxy.common.storage.StorageCompressor; +import me.cortex.voxy.common.storage.config.CompressorConfig; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.util.MemoryBuffer; +import me.cortex.voxy.common.util.Pair; +import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer; +import me.cortex.voxy.common.util.UnsafeUtil; +import me.cortex.voxy.common.world.SaveLoadSystem; +import net.jpountz.lz4.LZ4Factory; +import org.lwjgl.system.MemoryUtil; +import org.tukaani.xz.*; + +import java.io.IOException; + +public class LZ4Compressor implements StorageCompressor { + private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024); + + private final net.jpountz.lz4.LZ4Compressor compressor; + private final net.jpountz.lz4.LZ4FastDecompressor decompressor; + public LZ4Compressor() { + this.decompressor = LZ4Factory.nativeInstance().fastDecompressor(); + this.compressor = LZ4Factory.nativeInstance().fastCompressor(); + } + + @Override + public MemoryBuffer compress(MemoryBuffer saveData) { + var res = new MemoryBuffer(this.compressor.maxCompressedLength((int) saveData.size)+4); + MemoryUtil.memPutInt(res.address, (int) saveData.size); + int size = this.compressor.compress(saveData.asByteBuffer(), 0, (int) saveData.size, res.asByteBuffer(), 4, (int) res.size-4); + return res.subSize(size+4); + } + + @Override + public MemoryBuffer decompress(MemoryBuffer saveData) { + var res = SCRATCH.get().createUntrackedUnfreeableReference(); + int size = this.decompressor.decompress(saveData.asByteBuffer(), 4, res.asByteBuffer(), 0, MemoryUtil.memGetInt(saveData.address)); + return res.subSize(size); + } + + @Override + public void close() { + } + + public static class Config extends CompressorConfig { + + @Override + public StorageCompressor build(ConfigBuildCtx ctx) { + return new LZ4Compressor(); + } + + public static String getConfigTypeName() { + return "LZ4"; + } + } +} diff --git a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java index 4a236bcd..62b5f22d 100644 --- a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java +++ b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java @@ -2,6 +2,7 @@ package me.cortex.voxy.common.util; import org.lwjgl.system.MemoryUtil; +import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -77,6 +78,9 @@ public class MemoryBuffer extends TrackedObject { return new MemoryBuffer(this.tracked, this.address, size, this.freeable); } + public ByteBuffer asByteBuffer() { + return MemoryUtil.memByteBuffer(this.address, (int) this.size); + } //TODO: create like Long(offset) -> value at offset // methods for get and set, that way can have a single unifed system to ensure memory access bounds @@ -100,4 +104,5 @@ public class MemoryBuffer extends TrackedObject { public MemoryBuffer createUntrackedUnfreeableReference() { return new MemoryBuffer(false, this.address, this.size, false); } + } diff --git a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java index d71392be..86bf0c5c 100644 --- a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java +++ b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java @@ -28,6 +28,10 @@ public class UnsafeUtil { UNSAFE.copyMemory(null, src, dst, BYTE_ARRAY_BASE_OFFSET, dst.length); } + public static void memcpy(long src, int length, byte[] dst) { + UNSAFE.copyMemory(null, src, dst, BYTE_ARRAY_BASE_OFFSET, length); + } + //Copy the entire length of src to the dst memory where src is a byte array (source length from src) public static void memcpy(byte[] src, long dst) { UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length);