spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimizations aswell as the option to disable section verification

Browse Source
This commit is contained in:
mcrcortex
2024-08-07 00:06:02 +10:00
parent ee5bca7db5
commit 6eed71656d
6 changed files with 69 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java
View File

@@ -20,20 +20,17 @@ public class ZSTDCompressor implements StorageCompressor {
}
@Override
public ByteBuffer compress(ByteBuffer saveData) {
ByteBuffer compressedData = MemoryUtil.memAlloc((int)ZSTD_COMPRESSBOUND(saveData.remaining()));
long compressedSize = ZSTD_compress(compressedData, saveData, this.level);
compressedData.limit((int) compressedSize);
compressedData.rewind();
return compressedData;
public MemoryBuffer compress(MemoryBuffer saveData) {
MemoryBuffer compressedData = new MemoryBuffer((int)ZSTD_COMPRESSBOUND(saveData.size));
long compressedSize = nZSTD_compress(compressedData.address, compressedData.size, saveData.address, saveData.size, this.level);
return compressedData.subSize(compressedSize);
}
@Override
public MemoryBuffer decompress(MemoryBuffer saveData) {
var decompressed = new MemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE);
//TODO: mark the size of the decompressed data to verify its length later
long size = nZSTD_decompress(decompressed.address, decompressed.size, saveData.address, saveData.size);
return decompressed;
return decompressed.subSize(size);
}
@Override

16
src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java
View File

@@ -11,6 +11,11 @@ public class MemoryBuffer extends TrackedObject {
this.address = MemoryUtil.nmemAlloc(size);
}
private MemoryBuffer(long address, long size) {
this.size = size;
this.address = address;
}
public void cpyTo(long dst) {
super.assertNotFreed();
UnsafeUtil.memcpy(this.address, dst, this.size);
@@ -27,4 +32,15 @@ public class MemoryBuffer extends TrackedObject {
this.cpyTo(copy.address);
return copy;
}
//Creates a new MemoryBuffer, defunking this buffer and sets the size to be a subsize of the current size
public MemoryBuffer subSize(long size) {
if (size > this.size) {
throw new IllegalArgumentException("Requested size larger than current size");
}
//Free the current object, but not the memory associated with it
super.free0();
return new MemoryBuffer(this.address, size);
}
}

12
src/main/java/me/cortex/voxy/common/util/TrackedObject.java
View File

@@ -8,8 +8,12 @@ public abstract class TrackedObject {
public static final boolean TRACK_OBJECT_ALLOCATION_STACKS = System.getProperty("voxy.trackObjectAllocationStacks", "false").equals("true");
private final Ref ref;
public TrackedObject() {
this.ref = register(this);
protected TrackedObject() {
this(true);
}
protected TrackedObject(boolean shouldTrack) {
this.ref = register(shouldTrack, this);
}
protected void free0() {
@@ -44,10 +48,10 @@ public abstract class TrackedObject {
cleaner = null;
}
}
public static Ref register(Object obj) {
public static Ref register(boolean track, Object obj) {
boolean[] freed = new boolean[1];
Cleaner.Cleanable cleanable = null;
if (TRACK_OBJECT_ALLOCATIONS) {
if (TRACK_OBJECT_ALLOCATIONS && track) {
String clazz = obj.getClass().getName();
Throwable trace;
if (TRACK_OBJECT_ALLOCATION_STACKS) {

4
src/main/java/me/cortex/voxy/common/util/UnsafeUtil.java
View File

@@ -15,6 +15,7 @@ public class UnsafeUtil {
}
private static final long BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
private static final long SHORT_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(short[].class);
public static void memcpy(long src, long dst, long length) {
UNSAFE.copyMemory(src, dst, length);
@@ -31,6 +32,9 @@ public class UnsafeUtil {
public static void memcpy(byte[] src, long dst) {
UNSAFE.copyMemory(src, BYTE_ARRAY_BASE_OFFSET, null, dst, src.length);
}
public static void memcpy(short[] src, long dst) {
UNSAFE.copyMemory(src, SHORT_ARRAY_BASE_OFFSET, null, dst, (long) src.length <<1);
}
}

68
src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java
View File

@@ -11,6 +11,7 @@ import java.nio.ByteBuffer;
import static org.lwjgl.util.zstd.Zstd.*;
public class SaveLoadSystem {
public static final boolean VERIFY_HASH_ON_LOAD = System.getProperty("voxy.verifySectionOnLoad", "true").equals("true");
public static final int BIGGEST_SERIALIZED_SECTION_SIZE = 32 * 32 * 32 * 8 * 2;
public static int lin2z(int i) {
@@ -47,46 +48,46 @@ public class SaveLoadSystem {
pHash ^= block;
}
long[] lut = LUTVAL.toLongArray();
ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512);
MemoryBuffer raw = new MemoryBuffer(compressed.length*2L+lut.length*8L+512);
long ptr = raw.address;
long hash = section.key^(lut.length*1293481298141L);
raw.putLong(section.key);
raw.putInt(lut.length);
MemoryUtil.memPutLong(ptr, section.key); ptr += 8;
MemoryUtil.memPutInt(ptr, lut.length); ptr += 8;
for (long id : lut) {
raw.putLong(id);
MemoryUtil.memPutLong(ptr, id); ptr += 8;
hash *= 1230987149811L;
hash += 12831;
hash ^= id;
}
hash ^= pHash;
for (short block : compressed) {
raw.putShort(block);
}
UnsafeUtil.memcpy(compressed, ptr); ptr += compressed.length*2L;
raw.putLong(hash);
//The amount of memory copies are not ideal
var out = new MemoryBuffer(raw.position());
UnsafeUtil.memcpy(MemoryUtil.memAddress(raw), out.address, out.size);
MemoryUtil.memFree(raw);
return out;
MemoryUtil.memPutLong(ptr, hash); ptr += 8;
return raw.subSize(ptr-raw.address);
}
public static boolean deserialize(WorldSection section, MemoryBuffer data, boolean ignoreMismatchPosition) {
public static boolean deserialize(WorldSection section, MemoryBuffer data) {
long ptr = data.address;
long hash = 0;
long key = MemoryUtil.memGetLong(ptr); ptr += 8;
int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4;
long[] lut = new long[lutLen];
hash = key^(lut.length*1293481298141L);
if (VERIFY_HASH_ON_LOAD) {
hash = key ^ (lut.length * 1293481298141L);
}
for (int i = 0; i < lutLen; i++) {
lut[i] = MemoryUtil.memGetLong(ptr); ptr += 8;
hash *= 1230987149811L;
hash += 12831;
hash ^= lut[i];
if (VERIFY_HASH_ON_LOAD) {
hash *= 1230987149811L;
hash += 12831;
hash ^= lut[i];
}
}
if ((!ignoreMismatchPosition) && section.key != key) {
if (section.key != key) {
//throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
System.err.println("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
return false;
@@ -96,22 +97,23 @@ public class SaveLoadSystem {
section.data[z2lin(i)] = lut[MemoryUtil.memGetShort(ptr)]; ptr += 2;
}
long pHash = 99;
for (long block : section.data) {
pHash *= 127817112311121L;
pHash ^= pHash>>31;
pHash += 9918322711L;
pHash ^= block;
}
hash ^= pHash;
if (VERIFY_HASH_ON_LOAD) {
long pHash = 99;
for (long block : section.data) {
pHash *= 127817112311121L;
pHash ^= pHash >> 31;
pHash += 9918322711L;
pHash ^= block;
}
hash ^= pHash;
long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8;
if (expectedHash != hash) {
//throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);
System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region");
return false;
long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8;
if (expectedHash != hash) {
//throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);
System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region");
return false;
}
}
return true;
}
}

2
src/main/java/me/cortex/voxy/common/world/WorldEngine.java
View File

@@ -43,7 +43,7 @@ public class WorldEngine {
var data = this.storage.getSectionData(into.key);
if (data != null) {
try {
if (!SaveLoadSystem.deserialize(into, data, true)) {
if (!SaveLoadSystem.deserialize(into, data)) {
this.storage.deleteSectionData(into.key);
//TODO: regenerate the section from children
Arrays.fill(into.data, Mapper.AIR);