diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index 491f9683..d9962f86 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -6,6 +6,8 @@ import me.cortex.voxy.client.core.rendering.*; import me.cortex.voxy.client.core.rendering.building.RenderGenerationService; import me.cortex.voxy.client.core.rendering.post.PostProcessing; import me.cortex.voxy.client.core.util.DebugUtil; +import me.cortex.voxy.common.storage.CompressionStorageAdaptor; +import me.cortex.voxy.common.storage.ZSTDCompressor; import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend; import me.cortex.voxy.common.world.WorldEngine; import me.cortex.voxy.client.importers.WorldImporter; @@ -55,7 +57,7 @@ public class VoxelCore { SharedIndexBuffer.INSTANCE.id(); this.renderer = new Gl46FarWorldRenderer(VoxyConfig.CONFIG.geometryBufferSize, VoxyConfig.CONFIG.maxSections); System.out.println("Renderer initialized"); - this.world = new WorldEngine(new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath)), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, VoxyConfig.CONFIG.savingCompressionLevel, 5);//"storagefile.db"//"ethoslab.db" + this.world = new WorldEngine(new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath))), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); System.out.println("World engine"); this.renderTracker = new RenderTracker(this.world, this.renderer); diff --git a/src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java new file mode 100644 index 00000000..7b95954f --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java @@ -0,0 +1,68 @@ +package me.cortex.voxy.common.storage; + +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend; +import net.minecraft.util.math.random.RandomSeed; +import org.lwjgl.system.MemoryUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Arrays; + +//Compresses the section data +public class CompressionStorageAdaptor extends StorageBackend { + private final StorageCompressor compressor; + private final StorageBackend child; + public CompressionStorageAdaptor(StorageCompressor compressor, StorageBackend child) { + this.compressor = compressor; + this.child = child; + } + + @Override + public ByteBuffer getSectionData(long key) { + var data = this.child.getSectionData(key); + if (data == null) { + return null; + } + var decompressed = this.compressor.decompress(data); + MemoryUtil.memFree(data); + return decompressed; + } + + @Override + public void setSectionData(long key, ByteBuffer data) { + var cdata = this.compressor.compress(data); + this.child.setSectionData(key, cdata); + MemoryUtil.memFree(cdata); + } + + @Override + public void deleteSectionData(long key) { + this.child.deleteSectionData(key); + } + + @Override + public void putIdMapping(int id, ByteBuffer data) { + this.child.putIdMapping(id, data); + } + + @Override + public Int2ObjectOpenHashMap getIdMappingsData() { + return this.child.getIdMappingsData(); + } + + @Override + public void flush() { + this.child.flush(); + } + + @Override + public void close() { + this.compressor.close(); + this.child.close(); + } +} diff --git a/src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java b/src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java new file mode 100644 index 00000000..5acdf003 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java @@ -0,0 +1,11 @@ +package me.cortex.voxy.common.storage; + +import java.nio.ByteBuffer; + +public interface StorageCompressor { + ByteBuffer compress(ByteBuffer saveData); + + ByteBuffer decompress(ByteBuffer saveData); + + void close(); +} diff --git a/src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java b/src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java new file mode 100644 index 00000000..7027834d --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java @@ -0,0 +1,37 @@ +package me.cortex.voxy.common.storage; + +import org.lwjgl.system.MemoryUtil; + +import java.nio.ByteBuffer; + +import static org.lwjgl.util.zstd.Zstd.*; + +public class ZSTDCompressor implements StorageCompressor { + private final int level; + + public ZSTDCompressor(int level) { + this.level = level; + } + + @Override + public ByteBuffer compress(ByteBuffer saveData) { + ByteBuffer compressedData = MemoryUtil.memAlloc((int)ZSTD_COMPRESSBOUND(saveData.remaining())); + long compressedSize = ZSTD_compress(compressedData, saveData, this.level); + compressedData.limit((int) compressedSize); + compressedData.rewind(); + return compressedData; + } + + @Override + public ByteBuffer decompress(ByteBuffer saveData) { + var decompressed = MemoryUtil.memAlloc(32*32*32*8*2); + long size = ZSTD_decompress(decompressed, saveData); + decompressed.limit((int) size); + return decompressed; + } + + @Override + public void close() { + + } +} diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java index 41a0aaf0..769c4119 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java @@ -9,7 +9,8 @@ import java.nio.ByteBuffer; import static org.lwjgl.util.zstd.Zstd.*; public class SaveLoadSystem { - public static ByteBuffer serialize(WorldSection section, int compressionLevel) { + + public static ByteBuffer serialize(WorldSection section) { var data = section.copyData(); var compressed = new Short[data.length]; Long2ShortOpenHashMap LUT = new Long2ShortOpenHashMap(); @@ -47,28 +48,19 @@ public class SaveLoadSystem { raw.limit(raw.position()); raw.rewind(); - ByteBuffer compressedData = MemoryUtil.memAlloc((int)ZSTD_COMPRESSBOUND(raw.remaining())); - long compressedSize = ZSTD_compress(compressedData, raw, compressionLevel); - compressedData.limit((int) compressedSize); - compressedData.rewind(); - MemoryUtil.memFree(raw); //Compress into a key + data pallet format - return compressedData; + return raw; } public static boolean deserialize(WorldSection section, ByteBuffer data) { - var decompressed = MemoryUtil.memAlloc(32*32*32*4*2); - long size = ZSTD_decompress(decompressed, data); - decompressed.limit((int) size); - long hash = 0; - long key = decompressed.getLong(); - int lutLen = decompressed.getInt(); + long key = data.getLong(); + int lutLen = data.getInt(); long[] lut = new long[lutLen]; hash = key^(lut.length*1293481298141L); for (int i = 0; i < lutLen; i++) { - lut[i] = decompressed.getLong(); + lut[i] = data.getLong(); hash *= 1230987149811L; hash += 12831; hash ^= lut[i]; @@ -79,27 +71,25 @@ public class SaveLoadSystem { } for (int i = 0; i < section.data.length; i++) { - short lutId = decompressed.getShort(); + short lutId = data.getShort(); section.data[i] = lut[lutId]; hash *= 1230987149811L; hash += 12831; hash ^= (lutId*1827631L) ^ section.data[i]; } - long expectedHash = decompressed.getLong(); + long expectedHash = data.getLong(); if (expectedHash != hash) { //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash); System.err.println("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region"); return false; } - if (decompressed.hasRemaining()) { + if (data.hasRemaining()) { //throw new IllegalStateException("Decompressed section had excess data"); System.err.println("Decompressed section had excess data removing region"); return false; } - MemoryUtil.memFree(decompressed); - return true; } } diff --git a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java index 750b6ea0..f22b749c 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java @@ -1,5 +1,6 @@ package me.cortex.voxy.common.world; +import me.cortex.voxy.common.storage.StorageCompressor; import me.cortex.voxy.common.voxelization.VoxelizedSection; import me.cortex.voxy.common.world.other.Mapper; import me.cortex.voxy.common.world.service.SectionSavingService; @@ -13,7 +14,6 @@ import java.util.function.Consumer; //Use an LMDB backend to store the world, use a local inmemory cache for lod sections // automatically manages and invalidates sections of the world as needed public class WorldEngine { - public final StorageBackend storage; private final Mapper mapper; private final ActiveSectionTracker sectionTracker; @@ -29,14 +29,14 @@ public class WorldEngine { public Mapper getMapper() {return this.mapper;} - public WorldEngine(StorageBackend storageBackend, int ingestWorkers, int savingServiceWorkers, int compressionLevel, int maxMipLayers) { + public WorldEngine(StorageBackend storageBackend, int ingestWorkers, int savingServiceWorkers, int maxMipLayers) { this.maxMipLevels = maxMipLayers; this.storage = storageBackend; this.mapper = new Mapper(this.storage); //4 cache size bits means that the section tracker has 16 separate maps that it uses this.sectionTracker = new ActiveSectionTracker(3, this::unsafeLoadSection); - this.savingService = new SectionSavingService(this, savingServiceWorkers, compressionLevel); + this.savingService = new SectionSavingService(this, savingServiceWorkers); this.ingestService = new VoxelIngestService(this, ingestWorkers); } diff --git a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java index 037e4e1e..0ba7847b 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java @@ -1,5 +1,6 @@ package me.cortex.voxy.common.world.service; +import me.cortex.voxy.common.storage.StorageCompressor; import me.cortex.voxy.common.world.SaveLoadSystem; import me.cortex.voxy.common.world.WorldEngine; import me.cortex.voxy.common.world.WorldSection; @@ -15,13 +16,13 @@ public class SectionSavingService { private volatile boolean running = true; private final Thread[] workers; - private final int compressionLevel; private final ConcurrentLinkedDeque saveQueue = new ConcurrentLinkedDeque<>(); private final Semaphore saveCounter = new Semaphore(0); private final WorldEngine world; - public SectionSavingService(WorldEngine worldEngine, int workers, int compressionLevel) { + + public SectionSavingService(WorldEngine worldEngine, int workers) { this.workers = new Thread[workers]; for (int i = 0; i < workers; i++) { var worker = new Thread(this::saveWorker); @@ -30,7 +31,6 @@ public class SectionSavingService { worker.start(); this.workers[i] = worker; } - this.compressionLevel = compressionLevel; this.world = worldEngine; } @@ -42,7 +42,7 @@ public class SectionSavingService { section.assertNotFree(); section.inSaveQueue.set(false); - var saveData = SaveLoadSystem.serialize(section, this.compressionLevel); + var saveData = SaveLoadSystem.serialize(section); this.world.storage.setSectionData(section.key, saveData); MemoryUtil.memFree(saveData);