From 30967459821d0feb6d096242129074045f55b08b Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Mon, 12 Feb 2024 22:14:54 +1000 Subject: [PATCH] World seperation, path config, faster importing/ingesting, fixed render error when client model id hadnt been computed, version bump --- gradle.properties | 2 +- .../cortex/voxy/client/config/VoxyConfig.java | 2 - .../config/VoxyConfigScreenFactory.java | 24 +-- .../core/model/IdNotYetComputedException.java | 7 + .../voxy/client/core/model/ModelManager.java | 4 +- .../building/RenderGenerationService.java | 21 ++- .../voxy/client/importers/WorldImporter.java | 58 ++++++- .../client/terrain/WorldImportCommand.java | 7 +- .../other/BasicPathInsertionConfig.java | 21 +++ .../other/CompressionStorageAdaptor.java | 8 +- .../storage/other/DelegateStorageConfig.java | 14 ++ .../common/voxelization/VoxelizedSection.java | 37 +---- .../voxelization/WorldConversionFactory.java | 151 +++++++++++------- .../cortex/voxy/common/world/WorldEngine.java | 2 +- .../world/service/VoxelIngestService.java | 5 +- 15 files changed, 232 insertions(+), 131 deletions(-) create mode 100644 src/main/java/me/cortex/voxy/client/core/model/IdNotYetComputedException.java create mode 100644 src/main/java/me/cortex/voxy/common/storage/other/BasicPathInsertionConfig.java create mode 100644 src/main/java/me/cortex/voxy/common/storage/other/DelegateStorageConfig.java diff --git a/gradle.properties b/gradle.properties index 1b7922a5..20488469 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ yarn_mappings=1.20.4+build.1 loader_version=0.15.0 # Mod Properties -mod_version = 0.0.6-alpha +mod_version = 0.0.7-alpha maven_group = me.cortex archives_base_name = voxy diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java b/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java index 7c46a846..26d5b094 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java +++ b/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java @@ -26,8 +26,6 @@ public class VoxyConfig { public int ingestThreads = 5; public int savingThreads = 10; public int renderThreads = 5; - public int savingCompressionLevel = 7; - public String storagePath = "voxy_db"; public static VoxyConfig loadOrCreate() { diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java b/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java index 42ca0453..c3c54df2 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java +++ b/src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java @@ -39,7 +39,7 @@ public class VoxyConfigScreenFactory implements ModMenuApi { VoxyConfig.CONFIG.save(); }); - return ClothConfigDemo.getConfigBuilderWithDemo().build(); + return builder.build();//ClothConfigDemo.getConfigBuilderWithDemo().build(); } private static void addGeneralCategory(ConfigBuilder builder, VoxyConfig config) { @@ -84,11 +84,11 @@ public class VoxyConfigScreenFactory implements ModMenuApi { .setDefaultValue(DEFAULT.maxSections) .build()); - category.addEntry(entryBuilder.startIntSlider(Text.translatable("voxy.config.general.compression"), config.savingCompressionLevel, 1, 21) - .setTooltip(Text.translatable("voxy.config.general.compression.tooltip")) - .setSaveConsumer(val -> config.savingCompressionLevel = val) - .setDefaultValue(DEFAULT.savingCompressionLevel) - .build()); + //category.addEntry(entryBuilder.startIntSlider(Text.translatable("voxy.config.general.compression"), config.savingCompressionLevel, 1, 21) + // .setTooltip(Text.translatable("voxy.config.general.compression.tooltip")) + // .setSaveConsumer(val -> config.savingCompressionLevel = val) + // .setDefaultValue(DEFAULT.savingCompressionLevel) + // .build()); } private static void addThreadsCategory(ConfigBuilder builder, VoxyConfig config) { @@ -118,12 +118,12 @@ public class VoxyConfigScreenFactory implements ModMenuApi { ConfigCategory category = builder.getOrCreateCategory(Text.translatable("voxy.config.storage")); ConfigEntryBuilder entryBuilder = builder.entryBuilder(); - //Temporary until i figure out how to do more complex multi layer configuration for storage - category.addEntry(entryBuilder.startStrField(Text.translatable("voxy.config.storage.path"), config.storagePath) - .setTooltip(Text.translatable("voxy.config.storage.path.tooltip")) - .setSaveConsumer(val -> config.storagePath = val) - .setDefaultValue(DEFAULT.storagePath) - .build()); + ////Temporary until i figure out how to do more complex multi layer configuration for storage + //category.addEntry(entryBuilder.startStrField(Text.translatable("voxy.config.storage.path"), config.storagePath) + // .setTooltip(Text.translatable("voxy.config.storage.path.tooltip")) + // .setSaveConsumer(val -> config.storagePath = val) + // .setDefaultValue(DEFAULT.storagePath) + // .build()); } } diff --git a/src/main/java/me/cortex/voxy/client/core/model/IdNotYetComputedException.java b/src/main/java/me/cortex/voxy/client/core/model/IdNotYetComputedException.java new file mode 100644 index 00000000..2a87971f --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/core/model/IdNotYetComputedException.java @@ -0,0 +1,7 @@ +package me.cortex.voxy.client.core.model; + +public class IdNotYetComputedException extends RuntimeException { + public IdNotYetComputedException(int id) { + super("Id not yet computed: " + id); + } +} diff --git a/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java b/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java index 8372e56f..6d7687b7 100644 --- a/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java +++ b/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java @@ -523,7 +523,7 @@ public class ModelManager { map = this.idMappings[blockId]; } if (map == -1) { - throw new IllegalArgumentException("Id hasnt been computed yet: " + blockId); + throw new IdNotYetComputedException(blockId); } return this.metadataCache[map]; //int map = 0; @@ -541,7 +541,7 @@ public class ModelManager { public int getModelId(int blockId) { int map = this.idMappings[blockId]; if (map == -1) { - throw new IllegalArgumentException("Id hasnt been computed yet: " + blockId); + throw new IdNotYetComputedException(blockId); } return map; } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java index 0d866cb6..edb914c5 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java @@ -1,6 +1,7 @@ package me.cortex.voxy.client.core.rendering.building; import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; +import me.cortex.voxy.client.core.model.IdNotYetComputedException; import me.cortex.voxy.client.core.model.ModelManager; import me.cortex.voxy.common.world.WorldEngine; import me.cortex.voxy.common.world.WorldSection; @@ -56,12 +57,22 @@ public class RenderGenerationService { continue; } section.assertNotFree(); - var mesh = factory.generateMesh(section); + BuiltSection mesh = null; + try { + mesh = factory.generateMesh(section); + } catch (IdNotYetComputedException e) { + //We need to reinsert the build task into the queue + System.err.println("Render task failed to complete due to un-computed client id"); + synchronized (this.taskQueue) { + this.taskQueue.computeIfAbsent(section.key, key->{this.taskCounter.release(); return task;}); + } + } section.release(); - - this.resultConsumer.accept(mesh.clone()); - if (!this.meshCache.putMesh(mesh)) { - mesh.free(); + if (mesh != null) { + this.resultConsumer.accept(mesh.clone()); + if (!this.meshCache.putMesh(mesh)) { + mesh.free(); + } } } } diff --git a/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java b/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java index 6592fbfc..d87b92c4 100644 --- a/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxy/client/importers/WorldImporter.java @@ -5,12 +5,15 @@ import me.cortex.voxy.client.core.util.ByteBufferBackedInputStream; import me.cortex.voxy.common.voxelization.VoxelizedSection; import me.cortex.voxy.common.voxelization.WorldConversionFactory; import me.cortex.voxy.common.world.WorldEngine; +import me.cortex.voxy.common.world.other.Mipper; import net.minecraft.block.Block; import net.minecraft.block.BlockState; import net.minecraft.block.Blocks; import net.minecraft.nbt.*; +import net.minecraft.network.PacketByteBuf; import net.minecraft.registry.RegistryKeys; import net.minecraft.registry.entry.RegistryEntry; +import net.minecraft.util.collection.IndexedIterable; import net.minecraft.world.World; import net.minecraft.world.biome.Biome; import net.minecraft.world.biome.BiomeKeys; @@ -28,14 +31,16 @@ import java.nio.file.StandardOpenOption; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; public class WorldImporter { public record ImportUpdate(){} private final WorldEngine world; private final World mcWorld; - private final RegistryEntry defaultBiome; + private final ReadableContainer> defaultBiomeProvider; private final Codec>> biomeCodec; private final AtomicInteger totalRegions = new AtomicInteger(); private final AtomicInteger regionsProcessed = new AtomicInteger(); @@ -46,7 +51,49 @@ public class WorldImporter { this.mcWorld = mcWorld; var biomeRegistry = mcWorld.getRegistryManager().get(RegistryKeys.BIOME); - this.defaultBiome = biomeRegistry.entryOf(BiomeKeys.PLAINS); + var defaultBiome = biomeRegistry.entryOf(BiomeKeys.PLAINS); + this.defaultBiomeProvider = new ReadableContainer>() { + @Override + public RegistryEntry get(int x, int y, int z) { + return defaultBiome; + } + + @Override + public void forEachValue(Consumer> action) { + + } + + @Override + public void writePacket(PacketByteBuf buf) { + + } + + @Override + public int getPacketSize() { + return 0; + } + + @Override + public boolean hasAny(Predicate> predicate) { + return false; + } + + @Override + public void count(PalettedContainer.Counter> counter) { + + } + + @Override + public PalettedContainer> slice() { + return null; + } + + @Override + public Serialized> serialize(IndexedIterable> idList, PalettedContainer.PaletteProvider paletteProvider) { + return null; + } + }; + this.biomeCodec = PalettedContainer.createReadableContainerCodec(biomeRegistry.getIndexedEntries(), biomeRegistry.createEntryCodec(), PalettedContainer.PaletteProvider.BIOME, biomeRegistry.entryOf(BiomeKeys.PLAINS)); } @@ -209,7 +256,7 @@ public class WorldImporter { } var blockStates = BLOCK_STATE_CODEC.parse(NbtOps.INSTANCE, section.getCompound("block_states")).result().get(); - var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(null); + var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(this.defaultBiomeProvider); VoxelizedSection csec = WorldConversionFactory.convert( this.world.getMapper(), blockStates, @@ -228,10 +275,11 @@ public class WorldImporter { }, x, y, - z, - this.defaultBiome + z ); + WorldConversionFactory.mipSection(csec, this.world.getMapper()); + this.world.insertUpdate(csec); while (this.world.savingService.getTaskCount() > 4000) { try { diff --git a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java index 68a5f96f..f562037e 100644 --- a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java +++ b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java @@ -14,14 +14,17 @@ import java.io.File; public class WorldImportCommand { public static LiteralArgumentBuilder register() { - return ClientCommandManager.literal("voxy").then(ClientCommandManager.literal("import").then(ClientCommandManager.literal("world").then(ClientCommandManager.argument("world_name", StringArgumentType.string()).executes(WorldImportCommand::importWorld)))); + return ClientCommandManager.literal("voxy").then( + ClientCommandManager.literal("import") + .then(ClientCommandManager.literal("world") + .then(ClientCommandManager.argument("world_name", StringArgumentType.string()).executes(WorldImportCommand::importWorld)))); } public static WorldImporter importerInstance; private static int importWorld(CommandContext ctx) { var instance = MinecraftClient.getInstance(); - var file = new File(ctx.getArgument("world_name", String.class)); + var file = new File("saves").toPath().resolve(ctx.getArgument("world_name", String.class)).resolve("region").toFile(); importerInstance = ((IGetVoxelCore)instance.worldRenderer).getVoxelCore().createWorldImporter(MinecraftClient.getInstance().player.clientWorld, file); return 0; } diff --git a/src/main/java/me/cortex/voxy/common/storage/other/BasicPathInsertionConfig.java b/src/main/java/me/cortex/voxy/common/storage/other/BasicPathInsertionConfig.java new file mode 100644 index 00000000..43b76b0f --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/other/BasicPathInsertionConfig.java @@ -0,0 +1,21 @@ +package me.cortex.voxy.common.storage.other; + +import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; + +//Very simple config that adds a path to the config builder +public class BasicPathInsertionConfig extends DelegateStorageConfig { + public String path = ""; + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + ctx.pushPath(this.path); + var storage = this.delegate.build(ctx); + ctx.popPath(); + return storage; + } + + public static String getConfigTypeName() { + return "BasicPathConfig"; + } +} diff --git a/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java index 3b0a2736..304c0d4d 100644 --- a/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java +++ b/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java @@ -43,20 +43,14 @@ public class CompressionStorageAdaptor extends DelegatingStorageAdaptor { super.close(); } - public static class Config extends StorageConfig { + public static class Config extends DelegateStorageConfig { public CompressorConfig compressor; - public StorageConfig delegate; @Override public StorageBackend build(ConfigBuildCtx ctx) { return new CompressionStorageAdaptor(this.compressor.build(ctx), this.delegate.build(ctx)); } - @Override - public List getChildStorageConfigs() { - return List.of(this.delegate); - } - public static String getConfigTypeName() { return "CompressionAdaptor"; } diff --git a/src/main/java/me/cortex/voxy/common/storage/other/DelegateStorageConfig.java b/src/main/java/me/cortex/voxy/common/storage/other/DelegateStorageConfig.java new file mode 100644 index 00000000..899704e3 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/other/DelegateStorageConfig.java @@ -0,0 +1,14 @@ +package me.cortex.voxy.common.storage.other; + +import me.cortex.voxy.common.storage.config.StorageConfig; + +import java.util.List; + +public abstract class DelegateStorageConfig extends StorageConfig { + public StorageConfig delegate; + + @Override + public List getChildStorageConfigs() { + return List.of(this.delegate); + } +} diff --git a/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java b/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java index 20fe5e90..ff01d470 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/VoxelizedSection.java @@ -8,13 +8,9 @@ public class VoxelizedSection { public final int x; public final int y; public final int z; - private final long[] section; - private final long populationMsk; - private final long[][] subSections; - public VoxelizedSection(long[] section, long populationMsk, long[][] subSections, int x, int y, int z) { + final long[] section; + public VoxelizedSection(long[] section, int x, int y, int z) { this.section = section; - this.populationMsk = populationMsk; - this.subSections = subSections; this.x = x; this.y = y; this.z = z; @@ -29,31 +25,14 @@ public class VoxelizedSection { } public long get(int lvl, int x, int y, int z) { - if (lvl < 2) { - int subIdx = getIdx(x,y,z,2-lvl,2); - var subSec = this.subSections[subIdx]; - if (subSec == null) { - return Mapper.AIR; - } - - if (lvl == 0) { - return subSec[getIdx(x,y,z,0,2)]; - } else if (lvl == 1) { - return subSec[4*4*4+getIdx(x,y,z,0,1)]; - } - } else { - if (lvl == 2) { - return section[getIdx(x,y,z,0,2)]; - } else if (lvl == 3) { - return section[4*4*4+getIdx(x,y,z,0,1)]; - } else if (lvl == 4) { - return section[4*4*4+2*2*2]; - } - } - return Mapper.UNKNOWN_MAPPING; + int offset = lvl==1?(1<<12):0; + offset |= lvl==2?(1<<12)|(1<<9):0; + offset |= lvl==3?(1<<12)|(1<<9)|(1<<6):0; + offset |= lvl==4?(1<<12)|(1<<9)|(1<<6)|(1<<3):0; + return this.section[getIdx(x, y, z, 0, 4-lvl) + offset]; } public static VoxelizedSection createEmpty(int x, int y, int z) { - return new VoxelizedSection(new long[4*4*4+2*2*2+1], 0, new long[4*4*4][], x, y, z); + return new VoxelizedSection(new long[16*16*16 + 8*8*8 + 4*4*4 + 2*2*2 + 1], x, y, z); } } diff --git a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java index eb977c76..a545c3dd 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java @@ -1,5 +1,6 @@ package me.cortex.voxy.common.voxelization; +import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap; import me.cortex.voxy.common.world.other.Mipper; import me.cortex.voxy.common.world.other.Mapper; import net.minecraft.block.BlockState; @@ -9,14 +10,7 @@ import net.minecraft.world.chunk.PalettedContainer; import net.minecraft.world.chunk.ReadableContainer; public class WorldConversionFactory { - - private static int I(int x, int y, int z) { - return (y<<4)|(z<<2)|x; - } - - private static int J(int x, int y, int z) { - return ((y<<2)|(z<<1)|x) + 4*4*4; - } + private static final ThreadLocal> BLOCK_CACHE = ThreadLocal.withInitial(Reference2IntOpenHashMap::new); //TODO: add a local mapper cache since it should be smaller and faster public static VoxelizedSection convert(Mapper stateMapper, @@ -25,20 +19,20 @@ public class WorldConversionFactory { ILightingSupplier lightSupplier, int sx, int sy, - int sz, - RegistryEntry defaultBiome) { - long[] section = new long[4*4*4+2*2*2+1];//Mipping - long[][] subSections = new long[4*4*4][]; - long[] current = new long[4*4*4+2*2*2]; - long msk = 0; + int sz) { + var blockCache = BLOCK_CACHE.get(); + + var section = VoxelizedSection.createEmpty(sx, sy, sz); + var data = section.section; + + int blockId = -1; + BlockState block = null; + for (int oy = 0; oy < 4; oy++) { for (int oz = 0; oz < 4; oz++) { for (int ox = 0; ox < 4; ox++) { - RegistryEntry biome = defaultBiome; - if (biomeContainer != null) { - biome = biomeContainer.get(ox, oy, oz); - } - int nonAir = 0; + int biomeId = stateMapper.getIdForBiome(biomeContainer.get(ox, oy, oz)); + for (int iy = 0; iy < 4; iy++) { for (int iz = 0; iz < 4; iz++) { for (int ix = 0; ix < 4; ix++) { @@ -47,61 +41,94 @@ public class WorldConversionFactory { int z = (oz<<2)|iz; var state = blockContainer.get(x, y, z); byte light = lightSupplier.supply(x,y,z,state); - if (!(state.isAir() && (light==0))) {//TODO:FIXME:optimize this in such a way that having skylight access/no skylight means that an entire section is created, WHICH IS VERY BAD FOR PERFORMANCE!!!! - nonAir++; - current[I(ix, iy, iz)] = stateMapper.getBaseId(light, state, biome); - } - } - } - } - if (nonAir != 0) { - {//Generate mipping - //Mip L1 - int i = 0; - for (int y = 0; y < 4; y += 2) { - for (int z = 0; z < 4; z += 2) { - for (int x = 0; x < 4; x += 2) { - current[4 * 4 * 4 + i++] = Mipper.mip( - current[I(x, y, z)], current[I(x+1, y, z)], current[I(x, y, z+1)], current[I(x+1, y, z+1)], - current[I(x, y+1, z)], current[I(x+1, y+1, z)], current[I(x, y+1, z+1)], current[I(x+1, y+1, z+1)], - stateMapper); + if (!(state.isAir() && (light==0))) { + if (block != state) { + if (state.isAir()) { + block = state; + blockId = 0; + } else { + blockId = blockCache.computeIfAbsent(state, stateMapper::getIdForBlockState); + block = state; + } } + data[G(x, y, z)] = Mapper.composeMappingId(light, blockId, biomeId); } } - //Mip L2 - section[I(ox, oy, oz)] = Mipper.mip( - current[J(0,0,0)], current[J(1,0,0)], current[J(0,0,1)], current[J(1,0,1)], - current[J(0,1,0)], current[J(1,1,0)], current[J(0,1,1)], current[J(1,1,1)], - stateMapper); } - - //Update existence mask - msk |= 1L<>1 is cause the world sections size is 32x32x32 vs the 16x16x16 of the voxelized section for (int lvl = 0; lvl < this.maxMipLevels; lvl++) { var worldSection = this.acquire(lvl, section.x >> (lvl + 1), section.y >> (lvl + 1), section.z >> (lvl + 1)); diff --git a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java index 90cfee05..fda8c10c 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java @@ -69,10 +69,9 @@ public class VoxelIngestService { }, chunk.getPos().x, i, - chunk.getPos().z, - null + chunk.getPos().z ); - + WorldConversionFactory.mipSection(csec, this.world.getMapper()); this.world.insertUpdate(csec); } }