From 44c66d5c26648820394194a591a2ac8e866ae430 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Tue, 21 Jan 2025 05:35:22 +1000 Subject: [PATCH] Wip --- .../client/core/rendering/RenderService.java | 3 +- .../HierarchicalOcclusionTraverser.java | 12 +- .../rendering/hierachical/NodeCleaner.java | 7 +- .../rendering/hierachical/NodeManager.java | 112 ++++++++++++++++-- .../core/rendering/hierachical/NodeStore.java | 3 + .../FragmentedStorageBackendAdaptor.java | 2 +- .../voxelization/WorldConversionFactory.java | 1 + .../voxy/common/world/WorldSection.java | 1 + .../commonImpl/importers/WorldImporter.java | 4 + 9 files changed, 123 insertions(+), 22 deletions(-) diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java b/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java index fdbb2fb7..4ef7c7ac 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java @@ -151,8 +151,9 @@ public class RenderService, J extends Vi public void setup(Camera camera) { final int W = 32; final int H = 2; + boolean SIDED = false; for (int i = 0; i<64 && q<((W*2+1)*(W*2+1)*H)&&q++>=0;i++) { - this.nodeManager.insertTopLevelNode(WorldEngine.getWorldSectionId(4, (q%(W*2+1))-W, ((q/(W*2+1))/(W*2+1))-1, ((q/(W*2+1))%(W*2+1))-W)); + this.nodeManager.insertTopLevelNode(WorldEngine.getWorldSectionId(4, (q%(W*2+1))-(SIDED?0:W), ((q/(W*2+1))/(W*2+1))-1, ((q/(W*2+1))%(W*2+1))-(SIDED?0:W))); } if (q==((W*2+1)*(W*2+1)*H)) { q++; diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java index b398d0fc..0840607f 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/HierarchicalOcclusionTraverser.java @@ -41,9 +41,6 @@ public class HierarchicalOcclusionTraverser { - private final GlBuffer renderTrackingBuffer; - - private final GlBuffer queueMetaBuffer = new GlBuffer(4*4*5).zero(); private final GlBuffer scratchQueueA = new GlBuffer(100_000*4).zero(); private final GlBuffer scratchQueueB = new GlBuffer(100_000*4).zero(); @@ -95,8 +92,6 @@ public class HierarchicalOcclusionTraverser { this.requestBuffer = new GlBuffer(REQUEST_QUEUE_SIZE*8L+8).zero(); this.nodeBuffer = new GlBuffer(nodeManager.maxNodeCount*16L).zero(); - this.renderTrackingBuffer = new GlBuffer(nodeManager.maxNodeCount*4L).zero(); - glSamplerParameteri(this.hizSampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST); glSamplerParameteri(this.hizSampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR); @@ -133,8 +128,8 @@ public class HierarchicalOcclusionTraverser { MemoryUtil.memPutFloat(ptr, (float) (screenspaceAreaDecreasingSize) /(viewport.width*viewport.height)); ptr += 4; - //FrameId for timing info - MemoryUtil.memPutInt(ptr, viewport.frameId); ptr += 4; + //VisibilityId + MemoryUtil.memPutInt(ptr, this.nodeCleaner.visibilityId); ptr += 4; /* //Very funny and cool thing that is possible @@ -153,7 +148,7 @@ public class HierarchicalOcclusionTraverser { glBindBufferBase(GL_SHADER_STORAGE_BUFFER, RENDER_QUEUE_BINDING, this.renderList.id); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, NODE_DATA_BINDING, this.nodeBuffer.id); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, NODE_QUEUE_META_BINDING, this.queueMetaBuffer.id); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, RENDER_TRACKER_BINDING, this.renderTrackingBuffer.id); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, RENDER_TRACKER_BINDING, this.nodeCleaner.visibilityBuffer.id); glBindBuffer(GL_DISPATCH_INDIRECT_BUFFER, this.queueMetaBuffer.id); //Bind the hiz buffer @@ -307,7 +302,6 @@ public class HierarchicalOcclusionTraverser { this.queueMetaBuffer.free(); this.scratchQueueA.free(); this.scratchQueueB.free(); - this.renderTrackingBuffer.free(); glDeleteSamplers(this.hizSampler); } } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java index 0274348a..10eecafb 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeCleaner.java @@ -16,6 +16,9 @@ import static org.lwjgl.opengl.GL43C.glDispatchCompute; // then use bubble sort (/w fast path going to middle or 2 subdivisions deep) the bubble it up // can do incremental sorting pass aswell, so only scan and sort a rolling sector of sections // (over a few frames to not cause lag, maybe) + + +//TODO : USE THIS IN HierarchicalOcclusionTraverser instead of other shit public class NodeCleaner { //TODO: use batch_visibility_set to clear visibility data when nodes are removed!! (TODO: nodeManager will need to forward info to this) @@ -36,7 +39,7 @@ public class NodeCleaner { .add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/batch_visibility_set.comp") .compile(); - private final GlBuffer visibilityBuffer; + final GlBuffer visibilityBuffer; private final GlBuffer outputBuffer = new GlBuffer(OUTPUT_COUNT*4); private final GlBuffer scratchBuffer = new GlBuffer(BATCH_SET_SIZE*4);//Scratch buffer for setting ids with @@ -67,7 +70,7 @@ public class NodeCleaner { this.sorter.bind(); //TODO: choose whether this is in nodeSpace or section/geometryId space - //glDispatchCompute(, 1, 1); + //glDispatchCompute(this.nodeManager.getCurrentMaxNodeId()/, 1, 1); //DownloadStream.INSTANCE.download(this.outputBuffer, this::onDownload); } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeManager.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeManager.java index 4080367a..4279156d 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeManager.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeManager.java @@ -275,6 +275,7 @@ public class NodeManager { if (this.activeSectionMap.remove(cPos) == -1) {//TODO: verify the removed section is a request type of child and the request id matches this throw new IllegalStateException("Child pos was in a request but not in active section map"); } + if (!this.updateRouter.unwatch(cPos, WorldEngine.UPDATE_FLAGS)) { throw new IllegalStateException("Child pos was not being watched"); } @@ -391,6 +392,10 @@ public class NodeManager { } } } + //TODO: FIXME: This isnt right, as we need to remove node + geometry if it was in a request aswell as a child? + // BUT dont think thats possible? + + rem ^= reqRem; //If the request is satisfied, submit the result if (request.isSatisfied()) { @@ -399,13 +404,73 @@ public class NodeManager { } if (rem != 0) { + //"TODO" //There are child node entries that need removing + // TODO: this should be ok to do before request is satisfied Logger.error("UNFINISHED OPERATION TODO: FIXME"); + for (int i = 0; i < 8; i++) { + if ((rem & (1 << i)) == 0) continue; + long cPos = makeChildPos(pos, i); + + this.recurseRemoveNode(cPos); + //TOdo: update the child existance afak + } //TODO:FIXME:FINISH:CRITICAL } } } + //Recursivly fully removes all nodes and children + private void recurseRemoveNode(long pos) { + //NOTE: this also removes from the section map + int nodeId = this.activeSectionMap.remove(pos); + if (nodeId == -1) { + throw new IllegalStateException("Cannot remove pos that doesnt exist"); + } + int type = nodeId&NODE_TYPE_MSK; + nodeId &= NODE_ID_MSK; + if (type == NODE_TYPE_INNER || type == NODE_TYPE_LEAF) { + if (!this.nodeData.nodeExists(nodeId)) { + throw new IllegalStateException("Node exists in section map but not in nodeData"); + } + + + byte childExistence = this.nodeData.getNodeChildExistence(nodeId); + if (this.nodeData.isNodeRequestInFlight(nodeId)) { + //If there is an inflight request, the request and all associated data + int reqId = this.nodeData.getNodeRequest(nodeId); + //TODO: Dont assume this can only be a child request + + var req = this.childRequests.get(reqId); + childExistence ^= req.getMsk(); + + + this.childRequests.release(reqId);//Release the request + } + + + //Need to recurse into childExistence that exist, this is xor between a request mask if there is and the + // childRequest + // this is only valid if this node is an inner node + + + Logger.error("UNFINISHED OPERATION TODO: FIXME2"); + + //Free geometry and related memory for this node + + + //Unwatch geometry + if (!this.updateRouter.unwatch(pos, WorldEngine.UPDATE_FLAGS)) { + throw new IllegalStateException("Pos was not being watched"); + } + + } else { + + Logger.error("UNFINISHED OPERATION TODO: FIXME3"); + //NOTE: There are request type singles and request type child!!!! + } + } + //================================================================================================================== private void finishRequest(SingleNodeRequest request) { @@ -504,7 +569,7 @@ public class NodeManager { int reqMsk = Byte.toUnsignedInt(request.getMsk()); if ((byte) (existingChildMsk|reqMsk) != this.nodeData.getNodeChildExistence(parentNodeId)) { //System.out.println(Integer.toBinaryString(Byte.toUnsignedInt(this.nodeData.getNodeChildExistence(parentNodeId))));System.out.println(Integer.toBinaryString(existingChildMsk));System.out.println(Integer.toBinaryString(reqMsk)); - throw new IllegalStateException("node data existence state does not match pointer mask"); + throw new IllegalStateException("node data existence state does not match pointer mask"); } @@ -604,12 +669,6 @@ public class NodeManager { throw new IllegalStateException("Unknown node type: " + nodeType); } - if (this.nodeData.isNodeRequestInFlight(nodeId)) { - Logger.warn("Tried processing a node that already has a request in flight: " + nodeId + " pos: " + WorldEngine.pprintPos(pos) + " ignoring"); - return; - } - - //TODO: ADJUST AND FIX THIS TO MAKE IT REMOVE THE LAST THING IN QUEUE OR SOMETHING //if (this.activeNodeRequestCount > 100 && WorldEngine.getLevel(pos) < 2) { //Logger.info("Many active requests, declining request at " + WorldEngine.pprintPos(pos)); @@ -618,16 +677,47 @@ public class NodeManager { //} - this.nodeData.markRequestInFlight(nodeId); + + + + //TODO: + // Make it so that if a request is not in flight it has an invalid/null request entry + + // NOTE: inner nodes /w request should check they have geometry independenently of being inflight + + + + + + + + + //TODO: FIXTHIS: https://discord.com/channels/973046939375505408/973046939375505411/1328785093812031489 + // this causes things to go bad, when racing the gpu, i.e. this becomes an inner node that has geometry and there is now a request for it + // in this case we should not mark the node as inflight as it casuse very bad things to happen + // we should only mark inflight when there is actually a request if (nodeType == NODE_TYPE_LEAF) { + //Check if the node is already in-flight, if it is, dont do any processing + if (this.nodeData.isNodeRequestInFlight(nodeId)) { + Logger.warn("Tried processing a node that already has a request in flight: " + nodeId + " pos: " + WorldEngine.pprintPos(pos) + " ignoring"); + return; + } + + //Mark node as having an inflight request + this.nodeData.markRequestInFlight(nodeId); + //The hard one of processRequest, spin up a new request for the node this.makeLeafChildRequest(nodeId); } else {//nodeType == NODE_TYPE_INNER + //Dont mark node as having an inflight request + //TODO: assert that the node isnt already being watched for geometry, if it is, just spit out a warning? and ignore Logger.error("TODO FINISH THIS"); + // THis shouldent result in markRequestInFlight afak + if (!this.updateRouter.watch(pos, WorldEngine.UPDATE_TYPE_BLOCK_BIT)) { - //FIXME: i think this can occur accidently? when removing nodes or something creating leaf nodes + //FIXME: think this can occur accidently? when removing nodes or something creating leaf nodes // or other, the node might be wanted to be watched by gpu, but cpu already started watching it a few frames ago Logger.warn("Node: " + nodeId + " at pos: " + WorldEngine.pprintPos(pos) + " got update request, but geometry was already being watched"); } @@ -723,4 +813,8 @@ public class NodeManager { public void addDebug(List debug) { debug.add("NC/IF: " + this.activeSectionMap.size() + "/" + (this.singleRequests.count() + this.childRequests.count())); } + + //public int getCurrentMaxNodeId() { + // return this.nodeData.getEndNodeId(); + //} } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeStore.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeStore.java index 9650a9c7..8f0d62d6 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeStore.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical/NodeStore.java @@ -272,4 +272,7 @@ public final class NodeStore { MemoryUtil.memPutInt(ptr, w); ptr += 4; } + //public int getEndNodeId() { + + //} } diff --git a/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java index 8f3534df..45806e9b 100644 --- a/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java +++ b/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java @@ -22,7 +22,7 @@ public class FragmentedStorageBackendAdaptor extends StorageBackend { public FragmentedStorageBackendAdaptor(StorageBackend... backends) { this.backends = backends; int len = backends.length; - if ((len&(len-1)) != (len-1)) { + if ((len&(len-1)) != 0) { throw new IllegalArgumentException("Backend count not a power of 2"); } } diff --git a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java index a0627b65..3759536a 100644 --- a/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java +++ b/src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java @@ -10,6 +10,7 @@ import net.minecraft.world.chunk.PalettedContainer; import net.minecraft.world.chunk.ReadableContainer; public class WorldConversionFactory { + //TODO: create a mapping for world/mapper -> local mapping private static final ThreadLocal> BLOCK_CACHE = ThreadLocal.withInitial(Reference2IntOpenHashMap::new); public static VoxelizedSection convert(VoxelizedSection section, diff --git a/src/main/java/me/cortex/voxy/common/world/WorldSection.java b/src/main/java/me/cortex/voxy/common/world/WorldSection.java index 41e52ebf..289cf3f5 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldSection.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldSection.java @@ -55,6 +55,7 @@ public final class WorldSection { public final AtomicBoolean inSaveQueue = new AtomicBoolean(); //When the first bit is set it means its loaded + @SuppressWarnings("all") private volatile int atomicState = 1; WorldSection(int lvl, int x, int y, int z, ActiveSectionTracker tracker) { diff --git a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java index 1e9d583d..bb8bc4da 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java @@ -135,6 +135,9 @@ public class WorldImporter { this.worker = new Thread(() -> { this.isRunning = true; var files = directory.listFiles(); + if (files == null) { + onCompletion.accept(0); + } Arrays.sort(files, File::compareTo); this.estimatedTotalChunks.addAndGet(files.length*1024); for (var file : files) { @@ -159,6 +162,7 @@ public class WorldImporter { Thread.onSpinWait(); } if (!this.isRunning) { + onCompletion.accept(this.totalChunks.get()); return; } }