From 5b5939855f2957556825b655fac0c0703dceba04 Mon Sep 17 00:00:00 2001 From: mcrcortex <{ID}+{username}@users.noreply.github.com> Date: Mon, 15 Jul 2024 12:24:50 +1000 Subject: [PATCH] World tinkering --- .../me/cortex/voxy/client/core/VoxelCore.java | 10 ++--- .../building/RenderGenerationService.java | 2 +- .../voxy/client/importers/WorldImporter.java | 15 ++++--- .../rocksdb/RocksDBStorageBackend.java | 4 ++ .../common/voxelization/VoxelizedSection.java | 16 ++++--- .../voxelization/WorldConversionFactory.java | 10 ++--- .../voxy/common/world/SaveLoadSystem.java | 43 ++++++++++++++----- .../cortex/voxy/common/world/WorldEngine.java | 10 +++-- .../world/service/SectionSavingService.java | 2 +- .../world/service/ServiceThreadPool.java | 2 +- .../world/service/VoxelIngestService.java | 11 +++-- 11 files changed, 78 insertions(+), 47 deletions(-) diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index f955f745..46ef2c3d 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -234,16 +234,16 @@ public class VoxelCore { //this.world.getMapper().forceResaveStates(); if (this.importer != null) { System.out.println("Shutting down importer"); - try {this.importer.shutdown();this.importer = null;} catch (Exception e) {System.err.println(e);} + try {this.importer.shutdown();this.importer = null;} catch (Exception e) {e.printStackTrace();} } System.out.println("Shutting down voxel core"); - try {this.renderGen.shutdown();} catch (Exception e) {System.err.println(e);} + try {this.renderGen.shutdown();} catch (Exception e) {e.printStackTrace();} System.out.println("Render gen shut down"); - try {this.world.shutdown();} catch (Exception e) {System.err.println(e);} + try {this.world.shutdown();} catch (Exception e) {e.printStackTrace();} System.out.println("World engine shut down"); - try {this.renderer.shutdown(); this.viewportSelector.free(); this.modelManager.free();} catch (Exception e) {System.err.println(e);} + try {this.renderer.shutdown(); this.viewportSelector.free(); this.modelManager.free();} catch (Exception e) {e.printStackTrace();} System.out.println("Renderer shut down"); - if (this.postProcessing!=null){try {this.postProcessing.shutdown();} catch (Exception e) {System.err.println(e);}} + if (this.postProcessing!=null){try {this.postProcessing.shutdown();} catch (Exception e) {e.printStackTrace();}} System.out.println("Voxel core shut down"); } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java index 8b1e986b..e05f1169 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java @@ -87,7 +87,7 @@ public class RenderGenerationService { } } } catch (Exception e) { - System.err.println(e); + e.printStackTrace(); MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy render service had an exception while executing please check logs and report error"))); } } diff --git a/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java b/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java index aeea3ff2..50d70777 100644 --- a/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java @@ -237,6 +237,8 @@ public class WorldImporter { } + + private static final ThreadLocal SECTION_CACHE = ThreadLocal.withInitial(VoxelizedSection::createEmpty); private static final Codec> BLOCK_STATE_CODEC = PalettedContainer.createPalettedContainerCodec(Block.STATE_IDS, BlockState.CODEC, PalettedContainer.PaletteProvider.BLOCK_STATE, Blocks.AIR.getDefaultState()); private void importSectionNBT(int x, int y, int z, NbtCompound section) { if (section.getCompound("block_states").isEmpty()) { @@ -260,9 +262,15 @@ public class WorldImporter { skyLight = null; } - var blockStates = BLOCK_STATE_CODEC.parse(NbtOps.INSTANCE, section.getCompound("block_states")).result().get(); + var blockStatesRes = BLOCK_STATE_CODEC.parse(NbtOps.INSTANCE, section.getCompound("block_states")); + if (!blockStatesRes.hasResultOrPartial()) { + //TODO: if its only partial, it means should try to upgrade the nbt format with datafixerupper probably + return; + } + var blockStates = blockStatesRes.getPartialOrThrow(); var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(this.defaultBiomeProvider); VoxelizedSection csec = WorldConversionFactory.convert( + SECTION_CACHE.get().setPosition(x, y, z), this.world.getMapper(), blockStates, biomes, @@ -277,10 +285,7 @@ public class WorldImporter { } sky = 15-sky; return (byte) (sky|(block<<4)); - }, - x, - y, - z + } ); WorldConversionFactory.mipSection(csec, this.world.getMapper()); diff --git a/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java index 303cd463..443ab8c7 100644 --- a/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java @@ -24,6 +24,7 @@ public class RocksDBStorageBackend extends StorageBackend { private final List closeList = new ArrayList<>(); public RocksDBStorageBackend(String path) { + /* var lockPath = new File(path).toPath().resolve("LOCK"); if (Files.exists(lockPath)) { System.err.println("WARNING, deleting rocksdb LOCK file"); @@ -44,6 +45,7 @@ public class RocksDBStorageBackend extends StorageBackend { throw new RuntimeException("Unable to delete rocksdb lock file"); } } + */ final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().optimizeUniversalStyleCompaction(); @@ -71,6 +73,8 @@ public class RocksDBStorageBackend extends StorageBackend { this.worldSections = handles.get(1); this.idMappings = handles.get(2); + + this.db.flushWal(true); } catch (RocksDBException e) { throw new RuntimeException(e); } diff --git a/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java b/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java index ff01d470..6b9db0a4 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java @@ -5,15 +5,19 @@ import me.cortex.voxy.common.world.other.Mapper; //16x16x16 block section public class VoxelizedSection { - public final int x; - public final int y; - public final int z; + public int x; + public int y; + public int z; final long[] section; - public VoxelizedSection(long[] section, int x, int y, int z) { + public VoxelizedSection(long[] section) { this.section = section; + } + + public VoxelizedSection setPosition(int x, int y, int z) { this.x = x; this.y = y; this.z = z; + return this; } private static int getIdx(int x, int y, int z, int shiftBy, int size) { @@ -32,7 +36,7 @@ public class VoxelizedSection { return this.section[getIdx(x, y, z, 0, 4-lvl) + offset]; } - public static VoxelizedSection createEmpty(int x, int y, int z) { - return new VoxelizedSection(new long[16*16*16 + 8*8*8 + 4*4*4 + 2*2*2 + 1], x, y, z); + public static VoxelizedSection createEmpty() { + return new VoxelizedSection(new long[16*16*16 + 8*8*8 + 4*4*4 + 2*2*2 + 1]); } } diff --git a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java index 6eb373c5..5286fc09 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java @@ -13,16 +13,12 @@ public class WorldConversionFactory { private static final ThreadLocal> BLOCK_CACHE = ThreadLocal.withInitial(Reference2IntOpenHashMap::new); //TODO: add a local mapper cache since it should be smaller and faster - public static VoxelizedSection convert(Mapper stateMapper, + public static VoxelizedSection convert(VoxelizedSection section, + Mapper stateMapper, PalettedContainer blockContainer, ReadableContainer> biomeContainer, - ILightingSupplier lightSupplier, - int sx, - int sy, - int sz) { + ILightingSupplier lightSupplier) { var blockCache = BLOCK_CACHE.get(); - - var section = VoxelizedSection.createEmpty(sx, sy, sz); var data = section.section; int blockId = -1; diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java index 1df68b0a..e59e3767 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java @@ -10,19 +10,38 @@ import static org.lwjgl.util.zstd.Zstd.*; public class SaveLoadSystem { + public static int lin2z(int i) { + int x = i&0x1F; + int y = (i>>10)&0x1F; + int z = (i>>5)&0x1F; + return Integer.expand(x,0b1001001001001)|Integer.expand(y,0b10010010010010)|Integer.expand(z,0b100100100100100); + } + + public static int z2lin(int i) { + int x = Integer.compress(i, 0b1001001001001); + int y = Integer.compress(i, 0b10010010010010); + int z = Integer.compress(i, 0b100100100100100); + return x|(y<<10)|(z<<5); + } + //TODO: Cache like long2short and the short and other data to stop allocs public static ByteBuffer serialize(WorldSection section) { var data = section.copyData(); var compressed = new short[data.length]; Long2ShortOpenHashMap LUT = new Long2ShortOpenHashMap(data.length); LongArrayList LUTVAL = new LongArrayList(); + long pHash = 99; for (int i = 0; i < data.length; i++) { long block = data[i]; short mapping = LUT.computeIfAbsent(block, id->{ LUTVAL.add(id); return (short)(LUTVAL.size()-1); }); - compressed[i] = mapping; + compressed[lin2z(i)] = mapping; + pHash *= 127817112311121L; + pHash ^= pHash>>31; + pHash += 9918322711L; + pHash ^= block; } long[] lut = LUTVAL.toLongArray(); ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512); @@ -36,13 +55,10 @@ public class SaveLoadSystem { hash += 12831; hash ^= id; } + hash ^= pHash; - for (int i = 0; i < compressed.length; i++) { - short block = compressed[i]; + for (short block : compressed) { raw.putShort(block); - hash *= 1230987149811L; - hash += 12831; - hash ^= (block*1827631L) ^ data[i]; } raw.putLong(hash); @@ -73,13 +89,18 @@ public class SaveLoadSystem { } for (int i = 0; i < section.data.length; i++) { - short lutId = data.getShort(); - section.data[i] = lut[lutId]; - hash *= 1230987149811L; - hash += 12831; - hash ^= (lutId*1827631L) ^ section.data[i]; + section.data[z2lin(i)] = lut[data.getShort()]; } + long pHash = 99; + for (long block : section.data) { + pHash *= 127817112311121L; + pHash ^= pHash>>31; + pHash += 9918322711L; + pHash ^= block; + } + hash ^= pHash; + long expectedHash = data.getLong(); if (expectedHash != hash) { //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash); diff --git a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java index 5687eef0..c9d0f99b 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java @@ -109,6 +109,8 @@ public class WorldEngine { //TODO: move this to auxilery class so that it can take into account larger than 4 mip levels //Executes an update to the world and automatically updates all the parent mip layers up to level 4 (e.g. where 1 chunk section is 1 block big) + + //NOTE: THIS RUNS ON THE THREAD IT WAS EXECUTED ON, when this method exits, the calling method may assume that VoxelizedSection is no longer needed public void insertUpdate(VoxelizedSection section) {//TODO: add a bitset of levels to update and if it should force update //The >>1 is cause the world sections size is 32x32x32 vs the 16x16x16 of the voxelized section for (int lvl = 0; lvl < this.maxMipLevels; lvl++) { @@ -146,10 +148,10 @@ public class WorldEngine { } public void shutdown() { - try {this.storage.flush();} catch (Exception e) {System.err.println(e);} + try {this.storage.flush();} catch (Exception e) {e.printStackTrace();} //Shutdown in this order to preserve as much data as possible - try {this.ingestService.shutdown();} catch (Exception e) {System.err.println(e);} - try {this.savingService.shutdown();} catch (Exception e) {System.err.println(e);} - try {this.storage.close();} catch (Exception e) {System.err.println(e);} + try {this.ingestService.shutdown();} catch (Exception e) {e.printStackTrace();} + try {this.savingService.shutdown();} catch (Exception e) {e.printStackTrace();} + try {this.storage.close();} catch (Exception e) {e.printStackTrace();} } } diff --git a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java index e40b5c87..1415e920 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java @@ -48,7 +48,7 @@ public class SectionSavingService { this.world.storage.setSectionData(section.key, saveData); MemoryUtil.memFree(saveData); } catch (Exception e) { - System.err.println(e); + e.printStackTrace(); MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error"))); } section.release(); diff --git a/src/main/java/me/cortex/voxy/common/world/service/ServiceThreadPool.java b/src/main/java/me/cortex/voxy/common/world/service/ServiceThreadPool.java index a90dd762..66cf38e3 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/ServiceThreadPool.java +++ b/src/main/java/me/cortex/voxy/common/world/service/ServiceThreadPool.java @@ -42,7 +42,7 @@ public class ServiceThreadPool { try { job.run(); } catch (Exception e) { - System.err.println(e); + e.printStackTrace(); MinecraftClient.getInstance().executeSync(()-> MinecraftClient.getInstance().player.sendMessage( Text.literal( diff --git a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java index 8f8aa947..de918d2c 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java @@ -51,9 +51,11 @@ public class VoxelIngestService { i++; var lighting = this.captureLightMap.remove(ChunkSectionPos.from(chunk.getPos(), i).asLong()); if (section.isEmpty()) { - this.world.insertUpdate(VoxelizedSection.createEmpty(chunk.getPos().x, i, chunk.getPos().z)); + //TODO: add local cache so that it doesnt constantly create new sections + this.world.insertUpdate(VoxelizedSection.createEmpty().setPosition(chunk.getPos().x, i, chunk.getPos().z)); } else { VoxelizedSection csec = WorldConversionFactory.convert( + VoxelizedSection.createEmpty().setPosition(chunk.getPos().x, i, chunk.getPos().z), this.world.getMapper(), section.getBlockStateContainer(), section.getBiomeContainer(), @@ -70,17 +72,14 @@ public class VoxelIngestService { sky = 15-sky;//This is cause sky light is inverted which saves memory when saving empty sections return (byte) (sky|(block<<4)); } - }, - chunk.getPos().x, - i, - chunk.getPos().z + } ); WorldConversionFactory.mipSection(csec, this.world.getMapper()); this.world.insertUpdate(csec); } } } catch (Exception e) { - System.err.println(e); + e.printStackTrace(); MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy ingester had an exception while executing please check logs and report error"))); } }