From 6eed71656d76208cdada4c391d0eb8c955e3a4a9 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Wed, 7 Aug 2024 00:06:02 +1000 Subject: [PATCH] Optimizations aswell as the option to disable section verification --- .../storage/compressors/ZSTDCompressor.java | 13 ++-- .../cortex/voxy/common/util/MemoryBuffer.java | 16 +++++ .../voxy/common/util/TrackedObject.java | 12 ++-- .../cortex/voxy/common/util/UnsafeUtil.java | 4 ++ .../voxy/common/world/SaveLoadSystem.java | 68 ++++++++++--------- .../cortex/voxy/common/world/WorldEngine.java | 2 +- 6 files changed, 69 insertions(+), 46 deletions(-) diff --git a/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java b/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java index ce5695c5..0b0fe01e 100644 --- a/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java +++ b/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java @@ -20,20 +20,17 @@ public class ZSTDCompressor implements StorageCompressor { } @Override - public ByteBuffer compress(ByteBuffer saveData) { - ByteBuffer compressedData = MemoryUtil.memAlloc((int)ZSTD_COMPRESSBOUND(saveData.remaining())); - long compressedSize = ZSTD_compress(compressedData, saveData, this.level); - compressedData.limit((int) compressedSize); - compressedData.rewind(); - return compressedData; + public MemoryBuffer compress(MemoryBuffer saveData) { + MemoryBuffer compressedData = new MemoryBuffer((int)ZSTD_COMPRESSBOUND(saveData.size)); + long compressedSize = nZSTD_compress(compressedData.address, compressedData.size, saveData.address, saveData.size, this.level); + return compressedData.subSize(compressedSize); } @Override public MemoryBuffer decompress(MemoryBuffer saveData) { var decompressed = new MemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE); - //TODO: mark the size of the decompressed data to verify its length later long size = nZSTD_decompress(decompressed.address, decompressed.size, saveData.address, saveData.size); - return decompressed; + return decompressed.subSize(size); } @Override diff --git a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java index 6fe7fdde..eb6da7c9 100644 --- a/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java +++ b/src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java @@ -11,6 +11,11 @@ public class MemoryBuffer extends TrackedObject { this.address = MemoryUtil.nmemAlloc(size); } + private MemoryBuffer(long address, long size) { + this.size = size; + this.address = address; + } + public void cpyTo(long dst) { super.assertNotFreed(); UnsafeUtil.memcpy(this.address, dst, this.size); @@ -27,4 +32,15 @@ public class MemoryBuffer extends TrackedObject { this.cpyTo(copy.address); return copy; } + + //Creates a new MemoryBuffer, defunking this buffer and sets the size to be a subsize of the current size + public MemoryBuffer subSize(long size) { + if (size > this.size) { + throw new IllegalArgumentException("Requested size larger than current size"); + } + //Free the current object, but not the memory associated with it + super.free0(); + + return new MemoryBuffer(this.address, size); + } } diff --git a/src/main/java/me/cortex/voxy/common/util/TrackedObject.java b/src/main/java/me/cortex/voxy/common/util/TrackedObject.java index 943fe235..52d4a7df 100644 --- a/src/main/java/me/cortex/voxy/common/util/TrackedObject.java +++ b/src/main/java/me/cortex/voxy/common/util/TrackedObject.java @@ -8,8 +8,12 @@ public abstract class TrackedObject { public static final boolean TRACK_OBJECT_ALLOCATION_STACKS = System.getProperty("voxy.trackObjectAllocationStacks", "false").equals("true"); private final Ref ref; - public TrackedObject() { - this.ref = register(this); + protected TrackedObject() { + this(true); + } + + protected TrackedObject(boolean shouldTrack) { + this.ref = register(shouldTrack, this); } protected void free0() { @@ -44,10 +48,10 @@ public abstract class TrackedObject { cleaner = null; } } - public static Ref register(Object obj) { + public static Ref register(boolean track, Object obj) { boolean[] freed = new boolean[1]; Cleaner.Cleanable cleanable = null; - if (TRACK_OBJECT_ALLOCATIONS) { + if (TRACK_OBJECT_ALLOCATIONS && track) { String clazz = obj.getClass().getName(); Throwable trace; if (TRACK_OBJECT_ALLOCATION_STACKS) { diff --git a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java index a31f4786..a0c21875 100644 --- a/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java +++ b/src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java @@ -15,6 +15,7 @@ public class UnsafeUtil { } private static final long BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + private static final long SHORT_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(short[].class); public static void memcpy(long src, long dst, long length) { UNSAFE.copyMemory(src, dst, length); @@ -31,6 +32,9 @@ public class UnsafeUtil { public static void memcpy(byte[] src, long dst) { UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length); } + public static void memcpy(short[] src, long dst) { + UNSAFE.copyMemory(src, SHORT_ARRAY_BASE_OFFSET, null, dst, (long) src.length <<1); + } } diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java index d56eb17a..b83d6f25 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java @@ -11,6 +11,7 @@ import java.nio.ByteBuffer; import static org.lwjgl.util.zstd.Zstd.*; public class SaveLoadSystem { + public static final boolean VERIFY_HASH_ON_LOAD = System.getProperty("voxy.verifySectionOnLoad", "true").equals("true"); public static final int BIGGEST_SERIALIZED_SECTION_SIZE = 32 * 32 * 32 * 8 * 2; public static int lin2z(int i) { @@ -47,46 +48,46 @@ public class SaveLoadSystem { pHash ^= block; } long[] lut = LUTVAL.toLongArray(); - ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512); + MemoryBuffer raw = new MemoryBuffer(compressed.length*2L+lut.length*8L+512); + long ptr = raw.address; long hash = section.key^(lut.length*1293481298141L); - raw.putLong(section.key); - raw.putInt(lut.length); + MemoryUtil.memPutLong(ptr, section.key); ptr += 8; + MemoryUtil.memPutInt(ptr, lut.length); ptr += 8; for (long id : lut) { - raw.putLong(id); + MemoryUtil.memPutLong(ptr, id); ptr += 8; hash *= 1230987149811L; hash += 12831; hash ^= id; } hash ^= pHash; - for (short block : compressed) { - raw.putShort(block); - } + UnsafeUtil.memcpy(compressed, ptr); ptr += compressed.length*2L; - raw.putLong(hash); - //The amount of memory copies are not ideal - var out = new MemoryBuffer(raw.position()); - UnsafeUtil.memcpy(MemoryUtil.memAddress(raw), out.address, out.size); - MemoryUtil.memFree(raw); - return out; + MemoryUtil.memPutLong(ptr, hash); ptr += 8; + + return raw.subSize(ptr-raw.address); } - public static boolean deserialize(WorldSection section, MemoryBuffer data, boolean ignoreMismatchPosition) { + public static boolean deserialize(WorldSection section, MemoryBuffer data) { long ptr = data.address; long hash = 0; long key = MemoryUtil.memGetLong(ptr); ptr += 8; int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4; long[] lut = new long[lutLen]; - hash = key^(lut.length*1293481298141L); + if (VERIFY_HASH_ON_LOAD) { + hash = key ^ (lut.length * 1293481298141L); + } for (int i = 0; i < lutLen; i++) { lut[i] = MemoryUtil.memGetLong(ptr); ptr += 8; - hash *= 1230987149811L; - hash += 12831; - hash ^= lut[i]; + if (VERIFY_HASH_ON_LOAD) { + hash *= 1230987149811L; + hash += 12831; + hash ^= lut[i]; + } } - if ((!ignoreMismatchPosition) && section.key != key) { + if (section.key != key) { //throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key); System.err.println("Decompressed section not the same as requested. got: " + key + " expected: " + section.key); return false; @@ -96,22 +97,23 @@ public class SaveLoadSystem { section.data[z2lin(i)] = lut[MemoryUtil.memGetShort(ptr)]; ptr += 2; } - long pHash = 99; - for (long block : section.data) { - pHash *= 127817112311121L; - pHash ^= pHash>>31; - pHash += 9918322711L; - pHash ^= block; - } - hash ^= pHash; + if (VERIFY_HASH_ON_LOAD) { + long pHash = 99; + for (long block : section.data) { + pHash *= 127817112311121L; + pHash ^= pHash >> 31; + pHash += 9918322711L; + pHash ^= block; + } + hash ^= pHash; - long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8; - if (expectedHash != hash) { - //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash); - System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region"); - return false; + long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8; + if (expectedHash != hash) { + //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash); + System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region"); + return false; + } } - return true; } } diff --git a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java index f3f14b47..1cebc696 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java @@ -43,7 +43,7 @@ public class WorldEngine { var data = this.storage.getSectionData(into.key); if (data != null) { try { - if (!SaveLoadSystem.deserialize(into, data, true)) { + if (!SaveLoadSystem.deserialize(into, data)) { this.storage.deleteSectionData(into.key); //TODO: regenerate the section from children Arrays.fill(into.data, Mapper.AIR);