From 72ef321bfbf7dd3ca427a27fc606d09fa551d52d Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Fri, 31 Jan 2025 04:29:58 +1000 Subject: [PATCH] Compile only lzma (its terrible so not included) --- build.gradle | 2 +- .../storage/compressors/LZMACompressor.java | 117 ++++++++++++++++++ .../cortex/voxy/common/util/UnsafeUtil.java | 8 ++ 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 src/main/java/me/cortex/voxy/common/storage/compressors/LZMACompressor.java diff --git a/build.gradle b/build.gradle index c2b85a40..7053b1a2 100644 --- a/build.gradle +++ b/build.gradle @@ -199,7 +199,7 @@ dependencies { include(implementation 'redis.clients:jedis:5.1.0') include(implementation('org.rocksdb:rocksdbjni:8.10.0')) include(implementation 'org.apache.commons:commons-pool2:2.12.0') - //include(implementation 'org.tukaani:xz:1.10') + compileOnly('org.tukaani:xz:1.10') include(implementation 'org.lz4:lz4-java:1.8.0') //implementation 'org.rocksdb:rocksdbjni:8.10.0' //implementation 'redis.clients:jedis:5.1.0' diff --git a/src/main/java/me/cortex/voxy/common/storage/compressors/LZMACompressor.java b/src/main/java/me/cortex/voxy/common/storage/compressors/LZMACompressor.java new file mode 100644 index 00000000..b53b40ef --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/compressors/LZMACompressor.java @@ -0,0 +1,117 @@ +package me.cortex.voxy.common.storage.compressors; + +import me.cortex.voxy.common.storage.StorageCompressor; +import me.cortex.voxy.common.storage.config.CompressorConfig; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.util.MemoryBuffer; +import me.cortex.voxy.common.util.Pair; +import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer; +import me.cortex.voxy.common.util.UnsafeUtil; +import me.cortex.voxy.common.world.SaveLoadSystem; +import org.jetbrains.annotations.NotNull; +import org.lwjgl.system.MemoryUtil; +import org.tukaani.xz.*; +import org.tukaani.xz.lzma.LZMAEncoder; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +public class LZMACompressor implements StorageCompressor { + private static final ThreadLocal> CACHE_THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new byte[SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE], new ResettableArrayCache(new ArrayCache()))); + private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024); + + public LZMACompressor(int compressionLevel) { + + } + + @Override + public MemoryBuffer compress(MemoryBuffer saveData) { + MemoryBuffer res = new MemoryBuffer(saveData.size+1024); + try { + var cache = CACHE_THREAD_LOCAL.get(); + var xzCache = cache.right(); + xzCache.reset(); + var out = new FinishableOutputStream() { + private final long ptr = res.address+4; + private long size = 0; + @Override + public void write(int b) throws IOException { + MemoryUtil.memPutByte(this.ptr+this.size++, (byte) b); + } + + @Override + public void write(byte[] b) throws IOException { + UnsafeUtil.memcpy(b, this.ptr+this.size); this.size+=b.length; + } + }; + var bCache = cache.left(); + UnsafeUtil.memcpy(saveData.address, (int) saveData.size, bCache); + var stream = new XZOutputStream(out, new LZMA2Options(3), XZ.CHECK_NONE, xzCache); + stream.write(bCache); + stream.close(); + MemoryUtil.memPutInt(res.address, (int) saveData.size); + return res.subSize(out.size+4); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public MemoryBuffer decompress(MemoryBuffer saveData) { + var ret = SCRATCH.get().createUntrackedUnfreeableReference(); + + try { + var cache = CACHE_THREAD_LOCAL.get(); + var xzCache = cache.right(); + xzCache.reset(); + var bCache = cache.left(); + var stream = new XZInputStream(new InputStream() { + private long ptr = saveData.address+4; + + @Override + public int read() { + return MemoryUtil.memGetByte(this.ptr++) & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) { + len = Math.min(len, this.available()); + UnsafeUtil.memcpy(this.ptr, len, b, off); this.ptr+=len; + return len; + } + + @Override + public int available() { + return (int) (saveData.size-(this.ptr-saveData.address)); + } + }, -1, false, xzCache); + + stream.read(bCache, 0, MemoryUtil.memGetInt(saveData.address)); + UnsafeUtil.memcpy(bCache, MemoryUtil.memGetInt(saveData.address), ret.address); + + stream.close(); + return ret.subSize(MemoryUtil.memGetInt(saveData.address)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + + } + + public static class Config extends CompressorConfig { + public int compressionLevel; + + @Override + public StorageCompressor build(ConfigBuildCtx ctx) { + return new LZMACompressor(this.compressionLevel); + } + + public static String getConfigTypeName() { + return "LZMA2"; + } + } +} diff --git a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java index 86bf0c5c..0490aab3 100644 --- a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java +++ b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java @@ -32,10 +32,18 @@ public class UnsafeUtil { UNSAFE.copyMemory(null, src, dst, BYTE_ARRAY_BASE_OFFSET, length); } + public static void memcpy(long src, int length, byte[] dst, int offset) { + UNSAFE.copyMemory(null, src, dst, BYTE_ARRAY_BASE_OFFSET+offset, length); + } + //Copy the entire length of src to the dst memory where src is a byte array (source length from src) public static void memcpy(byte[] src, long dst) { UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length); } + + public static void memcpy(byte[] src, int len, long dst) { + UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, len); + } public static void memcpy(short[] src, long dst) { UNSAFE.copyMemory(src, SHORT_ARRAY_BASE_OFFSET, null, dst, (long) src.length <<1); }