World tinkering

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -234,16 +234,16 @@ public class VoxelCore {
         //this.world.getMapper().forceResaveStates();
         if (this.importer != null) {
             System.out.println("Shutting down importer");
-            try {this.importer.shutdown();this.importer = null;} catch (Exception e) {System.err.println(e);}
+            try {this.importer.shutdown();this.importer = null;} catch (Exception e) {e.printStackTrace();}
         }
         System.out.println("Shutting down voxel core");
-        try {this.renderGen.shutdown();} catch (Exception e) {System.err.println(e);}
+        try {this.renderGen.shutdown();} catch (Exception e) {e.printStackTrace();}
         System.out.println("Render gen shut down");
-        try {this.world.shutdown();} catch (Exception e) {System.err.println(e);}
+        try {this.world.shutdown();} catch (Exception e) {e.printStackTrace();}
         System.out.println("World engine shut down");
-        try {this.renderer.shutdown(); this.viewportSelector.free(); this.modelManager.free();} catch (Exception e) {System.err.println(e);}
+        try {this.renderer.shutdown(); this.viewportSelector.free(); this.modelManager.free();} catch (Exception e) {e.printStackTrace();}
         System.out.println("Renderer shut down");
-        if (this.postProcessing!=null){try {this.postProcessing.shutdown();} catch (Exception e) {System.err.println(e);}}
+        if (this.postProcessing!=null){try {this.postProcessing.shutdown();} catch (Exception e) {e.printStackTrace();}}
         System.out.println("Voxel core shut down");
     }
 

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java

@@ -87,7 +87,7 @@ public class RenderGenerationService {
                     }
                 }
             } catch (Exception e) {
-                System.err.println(e);
+                e.printStackTrace();
                 MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy render service had an exception while executing please check logs and report error")));
             }
         }

src/main/java/me/cortex/voxy/client/importers/WorldImporter.java

@@ -237,6 +237,8 @@ public class WorldImporter {
     }
 
 
+
+    private static final ThreadLocal<VoxelizedSection> SECTION_CACHE = ThreadLocal.withInitial(VoxelizedSection::createEmpty);
     private static final Codec<PalettedContainer<BlockState>> BLOCK_STATE_CODEC = PalettedContainer.createPalettedContainerCodec(Block.STATE_IDS, BlockState.CODEC, PalettedContainer.PaletteProvider.BLOCK_STATE, Blocks.AIR.getDefaultState());
     private void importSectionNBT(int x, int y, int z, NbtCompound section) {
         if (section.getCompound("block_states").isEmpty()) {
@@ -260,9 +262,15 @@ public class WorldImporter {
             skyLight = null;
         }
 
-        var blockStates = BLOCK_STATE_CODEC.parse(NbtOps.INSTANCE, section.getCompound("block_states")).result().get();
+        var blockStatesRes = BLOCK_STATE_CODEC.parse(NbtOps.INSTANCE, section.getCompound("block_states"));
+        if (!blockStatesRes.hasResultOrPartial()) {
+            //TODO: if its only partial, it means should try to upgrade the nbt format with datafixerupper probably
+            return;
+        }
+        var blockStates = blockStatesRes.getPartialOrThrow();
         var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(this.defaultBiomeProvider);
         VoxelizedSection csec = WorldConversionFactory.convert(
+                SECTION_CACHE.get().setPosition(x, y, z),
                 this.world.getMapper(),
                 blockStates,
                 biomes,
@@ -277,10 +285,7 @@ public class WorldImporter {
                     }
                     sky = 15-sky;
                     return (byte) (sky|(block<<4));
-                },
+                }
-                x,
-                y,
-                z
         );
 
         WorldConversionFactory.mipSection(csec, this.world.getMapper());

src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java

@@ -24,6 +24,7 @@ public class RocksDBStorageBackend extends StorageBackend {
     private final List<AbstractImmutableNativeReference> closeList = new ArrayList<>();
 
     public RocksDBStorageBackend(String path) {
+        /*
         var lockPath = new File(path).toPath().resolve("LOCK");
         if (Files.exists(lockPath)) {
             System.err.println("WARNING, deleting rocksdb LOCK file");
@@ -44,6 +45,7 @@ public class RocksDBStorageBackend extends StorageBackend {
                 throw new RuntimeException("Unable to delete rocksdb lock file");
             }
         }
+         */
 
         final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().optimizeUniversalStyleCompaction();
 
@@ -71,6 +73,8 @@ public class RocksDBStorageBackend extends StorageBackend {
 
             this.worldSections = handles.get(1);
             this.idMappings = handles.get(2);
+
+            this.db.flushWal(true);
         } catch (RocksDBException e) {
             throw new RuntimeException(e);
         }

src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java

@@ -5,15 +5,19 @@ import me.cortex.voxy.common.world.other.Mapper;
 
 //16x16x16 block section
 public class VoxelizedSection {
-    public final int x;
+    public int x;
-    public final int y;
+    public int y;
-    public final int z;
+    public int z;
     final long[] section;
-    public VoxelizedSection(long[] section, int x, int y, int z) {
+    public VoxelizedSection(long[] section) {
         this.section = section;
+    }
+
+    public VoxelizedSection setPosition(int x, int y, int z) {
         this.x = x;
         this.y = y;
         this.z = z;
+        return this;
     }
 
     private static int getIdx(int x, int y, int z, int shiftBy, int size) {
@@ -32,7 +36,7 @@ public class VoxelizedSection {
         return this.section[getIdx(x, y, z, 0, 4-lvl) + offset];
     }
 
-    public static VoxelizedSection createEmpty(int x, int y, int z) {
+    public static VoxelizedSection createEmpty() {
-        return new VoxelizedSection(new long[16*16*16 + 8*8*8 + 4*4*4 + 2*2*2 + 1], x, y, z);
+        return new VoxelizedSection(new long[16*16*16 + 8*8*8 + 4*4*4 + 2*2*2 + 1]);
     }
 }

src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java

@@ -13,16 +13,12 @@ public class WorldConversionFactory {
     private static final ThreadLocal<Reference2IntOpenHashMap<BlockState>> BLOCK_CACHE = ThreadLocal.withInitial(Reference2IntOpenHashMap::new);
 
     //TODO: add a local mapper cache since it should be smaller and faster
-    public static VoxelizedSection convert(Mapper stateMapper,
+    public static VoxelizedSection convert(VoxelizedSection section,
+                                           Mapper stateMapper,
                                            PalettedContainer<BlockState> blockContainer,
                                            ReadableContainer<RegistryEntry<Biome>> biomeContainer,
-                                           ILightingSupplier lightSupplier,
+                                           ILightingSupplier lightSupplier) {
-                                           int sx,
-                                           int sy,
-                                           int sz) {
         var blockCache = BLOCK_CACHE.get();
-
-        var section = VoxelizedSection.createEmpty(sx, sy, sz);
         var data = section.section;
 
         int blockId = -1;

src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java

@@ -10,19 +10,38 @@ import static org.lwjgl.util.zstd.Zstd.*;
 
 public class SaveLoadSystem {
 
+    public static int lin2z(int i) {
+        int x = i&0x1F;
+        int y = (i>>10)&0x1F;
+        int z = (i>>5)&0x1F;
+        return Integer.expand(x,0b1001001001001)|Integer.expand(y,0b10010010010010)|Integer.expand(z,0b100100100100100);
+    }
+
+    public static int z2lin(int i) {
+        int x = Integer.compress(i, 0b1001001001001);
+        int y = Integer.compress(i, 0b10010010010010);
+        int z = Integer.compress(i, 0b100100100100100);
+        return x|(y<<10)|(z<<5);
+    }
+
     //TODO: Cache like long2short and the short and other data to stop allocs
     public static ByteBuffer serialize(WorldSection section) {
         var data = section.copyData();
         var compressed = new short[data.length];
         Long2ShortOpenHashMap LUT = new Long2ShortOpenHashMap(data.length);
         LongArrayList LUTVAL = new LongArrayList();
+        long pHash = 99;
         for (int i = 0; i < data.length; i++) {
             long block = data[i];
             short mapping = LUT.computeIfAbsent(block, id->{
                 LUTVAL.add(id);
                 return (short)(LUTVAL.size()-1);
             });
-            compressed[i] = mapping;
+            compressed[lin2z(i)] = mapping;
+            pHash *= 127817112311121L;
+            pHash ^= pHash>>31;
+            pHash += 9918322711L;
+            pHash ^= block;
         }
         long[] lut = LUTVAL.toLongArray();
         ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512);
@@ -36,13 +55,10 @@ public class SaveLoadSystem {
             hash += 12831;
             hash ^= id;
         }
+        hash ^= pHash;
 
-        for (int i = 0; i < compressed.length; i++) {
+        for (short block : compressed) {
-            short block = compressed[i];
             raw.putShort(block);
-            hash *= 1230987149811L;
-            hash += 12831;
-            hash ^= (block*1827631L) ^ data[i];
         }
 
         raw.putLong(hash);
@@ -73,13 +89,18 @@ public class SaveLoadSystem {
         }
 
         for (int i = 0; i < section.data.length; i++) {
-            short lutId = data.getShort();
+            section.data[z2lin(i)] = lut[data.getShort()];
-            section.data[i] = lut[lutId];
-            hash *= 1230987149811L;
-            hash += 12831;
-            hash ^= (lutId*1827631L) ^ section.data[i];
         }
 
+        long pHash = 99;
+        for (long block : section.data) {
+            pHash *= 127817112311121L;
+            pHash ^= pHash>>31;
+            pHash += 9918322711L;
+            pHash ^= block;
+        }
+        hash ^= pHash;
+
         long expectedHash = data.getLong();
         if (expectedHash != hash) {
             //throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);

src/main/java/me/cortex/voxy/common/world/WorldEngine.java

@@ -109,6 +109,8 @@ public class WorldEngine {
 
     //TODO: move this to auxilery class  so that it can take into account larger than 4 mip levels
     //Executes an update to the world and automatically updates all the parent mip layers up to level 4 (e.g. where 1 chunk section is 1 block big)
+
+    //NOTE: THIS RUNS ON THE THREAD IT WAS EXECUTED ON, when this method exits, the calling method may assume that VoxelizedSection is no longer needed
     public void insertUpdate(VoxelizedSection section) {//TODO: add a bitset of levels to update and if it should force update
         //The >>1 is cause the world sections size is 32x32x32 vs the 16x16x16 of the voxelized section
         for (int lvl = 0; lvl < this.maxMipLevels; lvl++) {
@@ -146,10 +148,10 @@ public class WorldEngine {
     }
 
     public void shutdown() {
-        try {this.storage.flush();} catch (Exception e) {System.err.println(e);}
+        try {this.storage.flush();} catch (Exception e) {e.printStackTrace();}
         //Shutdown in this order to preserve as much data as possible
-        try {this.ingestService.shutdown();} catch (Exception e) {System.err.println(e);}
+        try {this.ingestService.shutdown();} catch (Exception e) {e.printStackTrace();}
-        try {this.savingService.shutdown();} catch (Exception e) {System.err.println(e);}
+        try {this.savingService.shutdown();} catch (Exception e) {e.printStackTrace();}
-        try {this.storage.close();} catch (Exception e) {System.err.println(e);}
+        try {this.storage.close();} catch (Exception e) {e.printStackTrace();}
     }
 }

src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java

@@ -48,7 +48,7 @@ public class SectionSavingService {
                 this.world.storage.setSectionData(section.key, saveData);
                 MemoryUtil.memFree(saveData);
             } catch (Exception e) {
-                System.err.println(e);
+                e.printStackTrace();
                 MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error")));
             }
             section.release();

src/main/java/me/cortex/voxy/common/world/service/ServiceThreadPool.java

@@ -42,7 +42,7 @@ public class ServiceThreadPool {
             try {
                 job.run();
             } catch (Exception e) {
-                System.err.println(e);
+                e.printStackTrace();
                 MinecraftClient.getInstance().executeSync(()->
                         MinecraftClient.getInstance().player.sendMessage(
                                 Text.literal(

src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java

@@ -51,9 +51,11 @@ public class VoxelIngestService {
                     i++;
                     var lighting = this.captureLightMap.remove(ChunkSectionPos.from(chunk.getPos(), i).asLong());
                     if (section.isEmpty()) {
-                        this.world.insertUpdate(VoxelizedSection.createEmpty(chunk.getPos().x, i, chunk.getPos().z));
+                        //TODO: add local cache so that it doesnt constantly create new sections
+                        this.world.insertUpdate(VoxelizedSection.createEmpty().setPosition(chunk.getPos().x, i, chunk.getPos().z));
                     } else {
                         VoxelizedSection csec = WorldConversionFactory.convert(
+                                VoxelizedSection.createEmpty().setPosition(chunk.getPos().x, i, chunk.getPos().z),
                                 this.world.getMapper(),
                                 section.getBlockStateContainer(),
                                 section.getBiomeContainer(),
@@ -70,17 +72,14 @@ public class VoxelIngestService {
                                         sky = 15-sky;//This is cause sky light is inverted which saves memory when saving empty sections
                                         return (byte) (sky|(block<<4));
                                     }
-                                },
+                                }
-                                chunk.getPos().x,
-                                i,
-                                chunk.getPos().z
                         );
                         WorldConversionFactory.mipSection(csec, this.world.getMapper());
                         this.world.insertUpdate(csec);
                     }
                 }
             } catch (Exception e) {
-                System.err.println(e);
+                e.printStackTrace();
                 MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy ingester had an exception while executing please check logs and report error")));
             }
         }