From 6450069d8a150f11126b93974a2dcca0f21807b4 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Fri, 9 Aug 2024 00:09:32 +1000 Subject: [PATCH] Move to SectionUpdate as a data wrapper --- .../client/core/rendering/RenderService.java | 13 ++-- .../core/rendering/building/BuiltSection.java | 6 +- .../building/BuiltSectionMeshCache.java | 69 ------------------ .../rendering/building/RenderDataFactory.java | 2 +- .../building/RenderGenerationService.java | 72 ++----------------- .../SectionPositionUpdateFilterer.java | 2 +- .../rendering/building/SectionUpdate.java | 6 ++ .../hierachical2/HierarchicalNodeManager.java | 27 ++----- .../voxelization/WorldConversionFactory.java | 2 - .../cortex/voxy/common/world/WorldEngine.java | 22 +++--- 10 files changed, 48 insertions(+), 173 deletions(-) delete mode 100644 src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java create mode 100644 src/main/java/me/cortex/voxy/client/core/rendering/building/SectionUpdate.java diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java b/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java index 0d0132e6..75e695c2 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java @@ -5,6 +5,7 @@ import me.cortex.voxy.client.core.model.ModelStore; import me.cortex.voxy.client.core.rendering.building.BuiltSection; import me.cortex.voxy.client.core.rendering.building.RenderGenerationService; import me.cortex.voxy.client.core.rendering.building.SectionPositionUpdateFilterer; +import me.cortex.voxy.client.core.rendering.building.SectionUpdate; import me.cortex.voxy.client.core.rendering.hierachical2.HierarchicalNodeManager; import me.cortex.voxy.client.core.rendering.hierachical2.HierarchicalOcclusionTraverser; import me.cortex.voxy.client.core.rendering.section.AbstractSectionRenderer; @@ -37,7 +38,7 @@ public class RenderService, J extends Vi private final ModelBakerySubsystem modelService; private final RenderGenerationService renderGen; - private final ConcurrentLinkedDeque sectionBuildResultQueue = new ConcurrentLinkedDeque<>(); + private final ConcurrentLinkedDeque sectionUpdateQueue = new ConcurrentLinkedDeque<>(); public RenderService(WorldEngine world, ServiceThreadPool serviceThreadPool) { this.modelService = new ModelBakerySubsystem(world.getMapper()); @@ -52,7 +53,7 @@ public class RenderService, J extends Vi this.nodeManager = new HierarchicalNodeManager(1<<21, this.sectionRenderer.getGeometryManager(), positionFilterForwarder); this.viewportSelector = new ViewportSelector<>(this.sectionRenderer::createViewport); - this.renderGen = new RenderGenerationService(world, this.modelService, serviceThreadPool, this.sectionBuildResultQueue::add, this.sectionRenderer.getGeometryManager() instanceof IUsesMeshlets); + this.renderGen = new RenderGenerationService(world, this.modelService, serviceThreadPool, this.sectionUpdateQueue::add, this.sectionRenderer.getGeometryManager() instanceof IUsesMeshlets); positionFilterForwarder.setCallback(this.renderGen::enqueueTask); this.traversal = new HierarchicalOcclusionTraverser(this.nodeManager, 512); @@ -92,8 +93,8 @@ public class RenderService, J extends Vi if (true /* firstInvocationThisFrame */) { DownloadStream.INSTANCE.tick(); //Process the build results here (this is done atomically/on the render thread) - while (!this.sectionBuildResultQueue.isEmpty()) { - this.nodeManager.processBuildResult(this.sectionBuildResultQueue.poll()); + while (!this.sectionUpdateQueue.isEmpty()) { + this.nodeManager.processBuildResult(this.sectionUpdateQueue.poll()); } } UploadStream.INSTANCE.tick(); @@ -123,8 +124,8 @@ public class RenderService, J extends Vi this.sectionRenderer.free(); this.traversal.free(); //Release all the unprocessed built geometry - this.sectionBuildResultQueue.forEach(BuiltSection::free); - this.sectionBuildResultQueue.clear(); + this.sectionUpdateQueue.forEach(update -> {if(update.geometry()!=null)update.geometry().free();}); + this.sectionUpdateQueue.clear(); } public Viewport getViewport() { diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSection.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSection.java index 10001cb0..64d3367c 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSection.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSection.java @@ -12,10 +12,14 @@ public final class BuiltSection { public final MemoryBuffer geometryBuffer; public final int[] offsets; - public BuiltSection(long position) { + private BuiltSection(long position) { this(position, -1, null, null); } + public static BuiltSection empty(long position) { + return new BuiltSection(position); + } + public BuiltSection(long position, int aabb, MemoryBuffer geometryBuffer, int[] offsets) { this.position = position; this.aabb = aabb; diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java deleted file mode 100644 index c0887e0f..00000000 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java +++ /dev/null @@ -1,69 +0,0 @@ -package me.cortex.voxy.client.core.rendering.building; - -import java.util.concurrent.ConcurrentHashMap; - -//TODO: Have a second level disk cache - -//TODO: instead of storing duplicate render geometry between here and gpu memory -// when a section is unloaded from the gpu, put it into a download stream and recover the BuiltSection -// and put that into the cache, then remove the uploaded mesh from the cache -public class BuiltSectionMeshCache { - private static final BuiltSection HOLDER = new BuiltSection(-1); - private final ConcurrentHashMap renderCache = new ConcurrentHashMap<>(1000,0.75f,10); - - public BuiltSection getMesh(long key) { - BuiltSection[] res = new BuiltSection[1]; - this.renderCache.computeIfPresent(key, (a, value) -> { - if (value == HOLDER) { - return value; - } - res[0] = value.clone(); - return value; - }); - return res[0]; - } - - //Returns true if the mesh was used, (this is so the parent method can free mesh object) - public boolean putMesh(BuiltSection mesh) { - var mesh2 = this.renderCache.computeIfPresent(mesh.position, (id, value) -> { - if (value != HOLDER) { - value.free(); - } - return mesh; - }); - return mesh2 == mesh; - } - - public void clearMesh(long key) { - this.renderCache.computeIfPresent(key, (a,val)->{ - if (val != HOLDER) { - val.free(); - } - return HOLDER; - }); - } - - public void markCache(long key) { - this.renderCache.putIfAbsent(key, HOLDER); - } - - public void unmarkCache(long key) { - var mesh = this.renderCache.remove(key); - if (mesh != null && mesh != HOLDER) { - mesh.free(); - } - } - - - public void free() { - for (var mesh : this.renderCache.values()) { - if (mesh != HOLDER) { - mesh.free(); - } - } - } - - public int getCount() { - return this.renderCache.size(); - } -} diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java index ae9201f1..e80cab54 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java @@ -103,7 +103,7 @@ public class RenderDataFactory { } if (bufferSize == 0) { - return new BuiltSection(section.key); + return BuiltSection.empty(section.key); } //TODO: generate the meshlets here diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java index a9776135..b92b1bcd 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java @@ -23,14 +23,13 @@ public class RenderGenerationService { private final WorldEngine world; private final ModelBakerySubsystem modelBakery; - private final Consumer resultConsumer; - private final BuiltSectionMeshCache meshCache = new BuiltSectionMeshCache(); + private final Consumer resultConsumer; private final boolean emitMeshlets; private final ServiceSlice threads; - public RenderGenerationService(WorldEngine world, ModelBakerySubsystem modelBakery, ServiceThreadPool serviceThreadPool, Consumer consumer, boolean emitMeshlets) { + public RenderGenerationService(WorldEngine world, ModelBakerySubsystem modelBakery, ServiceThreadPool serviceThreadPool, Consumer consumer, boolean emitMeshlets) { this.emitMeshlets = emitMeshlets; this.world = world; this.modelBakery = modelBakery; @@ -68,9 +67,10 @@ public class RenderGenerationService { synchronized (this.taskQueue) { task = this.taskQueue.removeFirst(); } + long time = System.nanoTime(); var section = task.sectionSupplier.get(); if (section == null) { - this.resultConsumer.accept(new BuiltSection(task.position)); + this.resultConsumer.accept(new SectionUpdate(task.position, time, BuiltSection.empty(task.position), (byte) 0)); return; } section.assertNotFree(); @@ -103,37 +103,12 @@ public class RenderGenerationService { } } - //TODO: if the section was _not_ built, maybe dont release it, or release it with the hint + byte childMask = section.getNonEmptyChildren(); section.release(); - if (mesh != null) { - //TODO: if the mesh is null, need to clear the cache at that point - this.resultConsumer.accept(mesh.clone()); - if (!this.meshCache.putMesh(mesh)) { - mesh.free(); - } - } + //Time is the time at the start of the update + this.resultConsumer.accept(new SectionUpdate(section.key, time, mesh, childMask)); } - public int getMeshCacheCount() { - return this.meshCache.getCount(); - } - - //TODO: Add a priority system, higher detail sections must always be updated before lower detail - // e.g. priorities NONE->lvl0 and lvl1 -> lvl0 over lvl0 -> lvl1 - - - //TODO: make it pass either a world section, _or_ coodinates so that the render thread has to do the loading of the sections - // not the calling method - - //TODO: maybe make it so that it pulls from the world to stop the inital loading absolutly butt spamming the queue - // and thus running out of memory - - //TODO: REDO THIS ENTIRE THING - // render tasks should not be bound to a WorldSection, instead it should be bound to either a WorldSection or - // an LoD position, the issue is that if we bound to a LoD position we loose all the info of the WorldSection - // like if its in the render queue and if we should abort building the render data - //1 proposal fix is a Long2ObjectLinkedOpenHashMap which means we can abort if needed, - // also gets rid of dependency on a WorldSection (kinda) public void enqueueTask(int lvl, int x, int y, int z) { this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true); } @@ -147,13 +122,6 @@ public class RenderGenerationService { } public void enqueueTask(long ikey, TaskChecker checker) { - { - var cache = this.meshCache.getMesh(ikey); - if (cache != null) { - this.resultConsumer.accept(cache); - return; - } - } synchronized (this.taskQueue) { this.taskQueue.computeIfAbsent(ikey, key->{ this.threads.execute(); @@ -168,31 +136,6 @@ public class RenderGenerationService { } } - //Tells the render cache that the mesh at the specified position should be cached - public void markCache(int lvl, int x, int y, int z) { - this.meshCache.markCache(WorldEngine.getWorldSectionId(lvl, x, y, z)); - } - - //Tells the render cache that the mesh at the specified position should not be cached/any previous cache result, freed - public void unmarkCache(int lvl, int x, int y, int z) { - this.meshCache.unmarkCache(WorldEngine.getWorldSectionId(lvl, x, y, z)); - } - - //Resets a chunks cache mesh - public void clearCache(int lvl, int x, int y, int z) { - this.meshCache.clearMesh(WorldEngine.getWorldSectionId(lvl, x, y, z)); - } - - /* - public void removeTask(int lvl, int x, int y, int z) { - synchronized (this.taskQueue) { - if (this.taskQueue.remove(WorldEngine.getWorldSectionId(lvl, x, y, z)) != null) { - this.taskCounter.acquireUninterruptibly(); - } - } - } - */ - public int getTaskCount() { return this.threads.getJobCount(); } @@ -204,7 +147,6 @@ public class RenderGenerationService { while (!this.taskQueue.isEmpty()) { this.taskQueue.removeFirst(); } - this.meshCache.free(); } public void addDebugData(List debug) { diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionPositionUpdateFilterer.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionPositionUpdateFilterer.java index 6504faf9..5fe09b6b 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionPositionUpdateFilterer.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionPositionUpdateFilterer.java @@ -52,7 +52,7 @@ public class SectionPositionUpdateFilterer { } } - public void maybeForward(WorldSection section) { + public void maybeForward(WorldSection section, int updateType) { this.maybeForward(section.key); } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionUpdate.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionUpdate.java new file mode 100644 index 00000000..b5208a58 --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/SectionUpdate.java @@ -0,0 +1,6 @@ +package me.cortex.voxy.client.core.rendering.building; + +import org.jetbrains.annotations.Nullable; + +public record SectionUpdate(long position, long buildTime, @Nullable BuiltSection geometry, byte childExistence) { +} diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/HierarchicalNodeManager.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/HierarchicalNodeManager.java index 8257f415..55cfdde0 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/HierarchicalNodeManager.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/HierarchicalNodeManager.java @@ -4,6 +4,7 @@ package me.cortex.voxy.client.core.rendering.hierachical2; import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; import me.cortex.voxy.client.core.rendering.building.BuiltSection; import me.cortex.voxy.client.core.rendering.building.SectionPositionUpdateFilterer; +import me.cortex.voxy.client.core.rendering.building.SectionUpdate; import me.cortex.voxy.client.core.rendering.section.AbstractSectionGeometryManager; import me.cortex.voxy.client.core.util.ExpandingObjectAllocationList; import me.cortex.voxy.common.world.WorldEngine; @@ -31,24 +32,6 @@ public class HierarchicalNodeManager { this.maxNodeCount = maxNodeCount; this.nodeData = new NodeStore(maxNodeCount); this.geometryManager = geometryManager; - - - - new Thread(()->{ - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - for(int x = -50; x<=50;x++) { - for (int z = -50; z <= 50; z++) { - for (int y = -3; y <= 3; y++) { - updateFilterer.watch(0,x,y,z); - updateFilterer.unwatch(0,x,y,z); - } - } - } - }).start(); } @@ -101,7 +84,11 @@ public class HierarchicalNodeManager { } } - public void processBuildResult(BuiltSection section) { + public void processBuildResult(SectionUpdate section) { + if (section.geometry() != null) { + section.geometry().free(); + } + /* if (!section.isEmpty()) { this.geometryManager.uploadSection(section); } else { @@ -120,7 +107,7 @@ public class HierarchicalNodeManager { // however could result in a reallocation if it needs to mark a child position as being possibly visible } - } + }*/ } private static long makeChildPos(long basePos, int addin) { diff --git a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java index 5286fc09..76e88317 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java @@ -12,7 +12,6 @@ import net.minecraft.world.chunk.ReadableContainer; public class WorldConversionFactory { private static final ThreadLocal> BLOCK_CACHE = ThreadLocal.withInitial(Reference2IntOpenHashMap::new); - //TODO: add a local mapper cache since it should be smaller and faster public static VoxelizedSection convert(VoxelizedSection section, Mapper stateMapper, PalettedContainer blockContainer, @@ -76,7 +75,6 @@ public class WorldConversionFactory { return ((y<<2)|(z<<1)|x) + 4*4*4 + 8*8*8 + 16*16*16; } - //TODO: Instead of this mip section as we are updating the data in the world public static void mipSection(VoxelizedSection section, Mapper mapper) { var data = section.section; diff --git a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java index a2f28751..105e266d 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java @@ -1,6 +1,5 @@ package me.cortex.voxy.common.world; -import me.cortex.voxy.client.Voxy; import me.cortex.voxy.common.voxelization.VoxelizedSection; import me.cortex.voxy.common.world.other.Mapper; import me.cortex.voxy.common.world.service.SectionSavingService; @@ -14,16 +13,22 @@ import java.util.function.Consumer; //Use an LMDB backend to store the world, use a local inmemory cache for lod sections // automatically manages and invalidates sections of the world as needed public class WorldEngine { + public static final int UPDATE_TYPE_BLOCK_BIT = 1; + public static final int UPDATE_TYPE_CHILD_EXISTENCE_BIT = 2; + public static final int UPDATE_FLAGS = UPDATE_TYPE_BLOCK_BIT | UPDATE_TYPE_CHILD_EXISTENCE_BIT; + public interface ISectionChangeCallback {void accept(WorldSection section, int updateFlags);} + + public final StorageBackend storage; private final Mapper mapper; private final ActiveSectionTracker sectionTracker; public final VoxelIngestService ingestService; public final SectionSavingService savingService; - private Consumer dirtyCallback; + private ISectionChangeCallback dirtyCallback; private final int maxMipLevels; - public void setDirtyCallback(Consumer tracker) { - this.dirtyCallback = tracker; + public void setDirtyCallback(ISectionChangeCallback callback) { + this.dirtyCallback = callback; } public Mapper getMapper() {return this.mapper;} @@ -100,12 +105,13 @@ public class WorldEngine { //Marks a section as dirty, enqueuing it for saving and or render data rebuilding public void markDirty(WorldSection section) { + this.markDirty(section, UPDATE_FLAGS); + } + + public void markDirty(WorldSection section, int changeState) { if (this.dirtyCallback != null) { - this.dirtyCallback.accept(section); + this.dirtyCallback.accept(section, changeState); } - //TODO: add an option for having synced saving, that is when call enqueueSave, that will instead, instantly - // save to the db, this can be useful for just reducing the amount of thread pools in total - // might have some issues with threading if the same section is saved from multiple threads? this.savingService.enqueueSave(section); }