From 0069855c5fa4612ea1c65b2ab3b73c141b256905 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Sat, 3 Feb 2024 21:43:44 +1000 Subject: [PATCH] Changed core --- .../voxy/client/core/DistanceTracker.java | 63 +++++---- .../me/cortex/voxy/client/core/VoxelCore.java | 3 +- .../voxy/client/core/model/ModelManager.java | 31 +++-- .../rendering/AbstractFarWorldRenderer.java | 2 + .../client/core/rendering/RenderTracker.java | 66 +++------ .../building/BuiltSectionMeshCache.java | 27 ++++ .../rendering/building/RenderDataFactory.java | 6 +- .../building/RenderGenerationService.java | 38 ++---- .../core/rendering/util/DownloadStream.java | 127 ++++++++++++++++++ .../core/rendering/util/UploadStream.java | 2 +- .../common/world/ActiveSectionTracker.java | 23 +++- .../voxy/common/world/SaveLoadSystem.java | 8 +- .../cortex/voxy/common/world/WorldEngine.java | 7 +- .../voxy/common/world/WorldSection.java | 6 +- .../world/service/SectionSavingService.java | 2 +- .../voxy/shaders/lod/gl46/quad_format.glsl | 2 +- 16 files changed, 280 insertions(+), 133 deletions(-) create mode 100644 src/main/java/me/cortex/voxy/client/core/rendering/building/BuiltSectionMeshCache.java create mode 100644 src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java diff --git a/src/main/java/me/cortex/voxy/client/core/DistanceTracker.java b/src/main/java/me/cortex/voxy/client/core/DistanceTracker.java index bb447436..5621a963 100644 --- a/src/main/java/me/cortex/voxy/client/core/DistanceTracker.java +++ b/src/main/java/me/cortex/voxy/client/core/DistanceTracker.java @@ -14,39 +14,38 @@ import net.minecraft.client.MinecraftClient; // make the rebuild range like +-5 chunks along each axis (that means at higher levels, should only need to rebuild like) // 4 sections or something public class DistanceTracker { - private final TransitionRing2D[] rings; + private final TransitionRing2D[] loDRings; + private final TransitionRing2D[] cacheLoadRings; + private final TransitionRing2D[] cacheUnloadRings; private final RenderTracker tracker; - private final int scale; private final int minYSection; private final int maxYSection; - public DistanceTracker(RenderTracker tracker, int rings, int scale) { - this.rings = new TransitionRing2D[rings]; + public DistanceTracker(RenderTracker tracker, int[] lodRingScales, int cacheLoadDistance, int cacheUnloadDistance) { + this.loDRings = new TransitionRing2D[lodRingScales.length]; + this.cacheLoadRings = new TransitionRing2D[lodRingScales.length]; + this.cacheUnloadRings = new TransitionRing2D[lodRingScales.length]; this.tracker = tracker; - this.scale = scale; - this.minYSection = MinecraftClient.getInstance().world.getBottomSectionCoord()/2; this.maxYSection = MinecraftClient.getInstance().world.getTopSectionCoord()/2; - int radius = (MinecraftClient.getInstance().options.getViewDistance().getValue() / 2) - 4; - if (radius > 0 && false) { - this.rings[0] = new TransitionRing2D(5, radius, (x, z) -> { - for (int y = this.minYSection; y <= this.maxYSection; y++) { - this.tracker.remLvl0(x, y, z); - } - }, (x, z) -> { - for (int y = this.minYSection; y <= this.maxYSection; y++) { - this.tracker.addLvl0(x, y, z); - } - }); - } //The rings 0+ start at 64 vanilla rd, no matter what the game is set at, that is if the game is set to 32 rd // there will still be 32 chunks untill the first lod drop // if the game is set to 16, then there will be 48 chunks until the drop - for (int i = 1; i < rings; i++) { + for (int i = 0; i < this.loDRings.length; i++) { int capRing = i; - this.rings[i] = new TransitionRing2D(5+i, scale, (x, z) -> this.dec(capRing, x, z), (x, z) -> this.inc(capRing, x, z)); + this.loDRings[i] = new TransitionRing2D(6+i, lodRingScales[i], (x, z) -> this.dec(capRing+1, x, z), (x, z) -> this.inc(capRing+1, x, z)); + + //TODO:FIXME i think the radius is wrong and (lodRingScales[i]) needs to be (lodRingScales[i]<<1) since the transition ring (the thing above) + // acts on LoD level + 1 + this.cacheLoadRings[i] = new TransitionRing2D(5+i, lodRingScales[i] + cacheLoadDistance, (x, z) -> { + //When entering a cache ring, trigger a mesh op and inject into cache + + }, (x, z) -> {}); + this.cacheUnloadRings[i] = new TransitionRing2D(5+i, lodRingScales[i] + cacheUnloadDistance, (x, z) -> {}, (x, z) -> { + //When exiting the cache unload ring, tell the cache to dump whatever mesh it has cached and not add any mesh from that position + }); } } @@ -69,10 +68,14 @@ public class DistanceTracker { //if the center suddenly changes (say more than 1<<(7+lodlvl) block) then invalidate the entire ring and recompute // the lod sections public void setCenter(int x, int y, int z) { - for (var ring : this.rings) { - if (ring != null) { - ring.update(x, z); - } + for (var ring : this.cacheLoadRings) { + ring.update(x, z); + } + for (var ring : this.loDRings) { + ring.update(x, z); + } + for (var ring : this.cacheUnloadRings) { + ring.update(x, z); } } @@ -82,14 +85,18 @@ public class DistanceTracker { //Insert highest LOD level for (int ox = -SIZE; ox <= SIZE; ox++) { for (int oz = -SIZE; oz <= SIZE; oz++) { - this.inc(4, (x>>(5+this.rings.length-1)) + ox, (z>>(5+this.rings.length-1)) + oz); + this.inc(4, (x>>(5+this.loDRings.length)) + ox, (z>>(5+this.loDRings.length)) + oz); } } - for (int i = this.rings.length-1; 0 <= i; i--) { - if (this.rings[i] != null) { - this.rings[i].fill(x, z); + for (var ring : this.cacheLoadRings) { + ring.fill(x, z); + } + + for (int i = this.loDRings.length-1; 0 <= i; i--) { + if (this.loDRings[i] != null) { + this.loDRings[i].fill(x, z); } } } diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index 9f0b460a..6a01aae0 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -63,7 +63,8 @@ public class VoxelCore { System.out.println("Render tracker and generator initialized"); //To get to chunk scale multiply the scale by 2, the scale is after how many chunks does the lods halve - this.distanceTracker = new DistanceTracker(this.renderTracker, 5, VoxyConfig.CONFIG.qualityScale); + int q = VoxyConfig.CONFIG.qualityScale; + this.distanceTracker = new DistanceTracker(this.renderTracker, new int[]{q,q,q,q}, 2, 4); System.out.println("Distance tracker initialized"); this.postProcessing = new PostProcessing(); diff --git a/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java b/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java index b8613372..61c7ac96 100644 --- a/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java +++ b/src/main/java/me/cortex/voxy/client/core/model/ModelManager.java @@ -490,18 +490,31 @@ public class ModelManager { return res; } - //TODO:FIXME: if the model is not already in the cache for some reason it renders black, need to figure out why + //TODO:FIXME: DONT DO SPIN LOCKS :WAA: public long getModelMetadata(int blockId) { - int map = 0; - while ((map = this.idMappings[blockId]) == -1) { - Thread.onSpinWait(); + int map = this.idMappings[blockId]; + if (map == -1) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + map = this.idMappings[blockId]; } + if (map == -1) { + throw new IllegalArgumentException("Id hasnt been computed yet: " + blockId); + } + return this.metadataCache[map]; + //int map = 0; + //int i = 10; + //while ((map = this.idMappings[blockId]) == -1) { + // Thread.onSpinWait(); + //} - long meta = 0; - while ((meta = this.metadataCache[map]) == 0) { - Thread.onSpinWait(); - } - return meta; + //long meta = 0; + //while ((meta = this.metadataCache[map]) == 0) { + // Thread.onSpinWait(); + //} } public int getModelId(int blockId) { diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java b/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java index 5da5becf..5caba1c2 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/AbstractFarWorldRenderer.java @@ -6,6 +6,7 @@ package me.cortex.voxy.client.core.rendering; import me.cortex.voxy.client.core.gl.GlBuffer; import me.cortex.voxy.client.core.model.ModelManager; import me.cortex.voxy.client.core.rendering.building.BuiltSection; +import me.cortex.voxy.client.core.rendering.util.DownloadStream; import me.cortex.voxy.client.core.rendering.util.UploadStream; import me.cortex.voxy.common.world.other.Mapper; import net.minecraft.client.MinecraftClient; @@ -66,6 +67,7 @@ public abstract class AbstractFarWorldRenderer { // once per frame when using multi viewport mods //it shouldent matter if its called multiple times a frame however, as its synced with fences UploadStream.INSTANCE.tick(); + DownloadStream.INSTANCE.tick(); //Update the lightmap { diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/RenderTracker.java b/src/main/java/me/cortex/voxy/client/core/rendering/RenderTracker.java index 140ccf9c..94e347f5 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/RenderTracker.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/RenderTracker.java @@ -35,7 +35,7 @@ public class RenderTracker { //Adds a lvl 0 section into the world renderer public void addLvl0(int x, int y, int z) { this.activeSections.put(WorldEngine.getWorldSectionId(0, x, y, z), O); - this.renderGen.enqueueTask(0, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(0, x, y, z, this::shouldStillBuild); } //Removes a lvl 0 section from the world renderer @@ -61,7 +61,7 @@ public class RenderTracker { // concurrent hashmap or something, this is so that e.g. the build data position // can be updated - this.renderGen.enqueueTask(lvl, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl, x, y, z, this::shouldStillBuild); this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1)))); this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1)+1))); @@ -98,35 +98,35 @@ public class RenderTracker { this.renderer.enqueueResult(new BuiltSection(WorldEngine.getWorldSectionId(lvl, x, y, z))); this.renderGen.removeTask(lvl, x, y, z); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1), this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1, this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1), this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1, this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1), this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1, this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1), this::shouldStillBuild); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1, this::shouldStillBuild); + } + + //Enqueues a renderTask for a section to cache the result + public void addCache() { } - //Updates a sections direction mask (e.g. if the player goes past the axis, the chunk must be updated) - public void updateDirMask(int lvl, int x, int y, int z, int newMask) { - - } //Called by the world engine when a section gets dirtied public void sectionUpdated(WorldSection section) { - if (this.activeSections.containsKey(section.getKey())) { + if (this.activeSections.containsKey(section.key)) { //TODO:FIXME: if the section gets updated, that means that its neighbors might need to be updated aswell // (due to block occlusion) //TODO: FIXME: REBUILDING THE ENTIRE NEIGHBORS when probably only the internal layout changed is NOT SMART - this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(section.lvl, section.x-1, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(section.lvl, section.x+1, section.y, section.z, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z-1, this::shouldStillBuild, this::getBuildFlagsOrAbort); - this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z, this::shouldStillBuild); + this.renderGen.enqueueTask(section.lvl, section.x-1, section.y, section.z, this::shouldStillBuild); + this.renderGen.enqueueTask(section.lvl, section.x+1, section.y, section.z, this::shouldStillBuild); + this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z-1, this::shouldStillBuild); + this.renderGen.enqueueTask(section.lvl, section.x, section.y, section.z+1, this::shouldStillBuild); } //this.renderGen.enqueueTask(section); } @@ -143,32 +143,6 @@ public class RenderTracker { } } - public int getBuildFlagsOrAbort(WorldSection section) { - var cam = MinecraftClient.getInstance().cameraEntity; - if (cam == null) { - return 0; - } - var holder = this.activeSections.get(section.getKey()); - int buildMask = 0; - if (holder != null) { - if (section.z<(((int)cam.getPos().z)>>(5+section.lvl))+1) { - buildMask |= 1<< Direction.SOUTH.getId(); - } - if (section.z>(((int)cam.getPos().z)>>(5+section.lvl))-1) { - buildMask |= 1<>(5+section.lvl))+1) { - buildMask |= 1<(((int)cam.getPos().x)>>(5+section.lvl))-1) { - buildMask |= 1< renderCache = new ConcurrentHashMap<>(1000,0.75f,10); + + public BuiltSection getMesh(long key) { + return null; + } + + //Returns true if the mesh was used, (this is so the parent method can free mesh object) + public boolean putMesh(BuiltSection mesh) { + return false; + } + + public void clearMesh(long key) { + } + + + public void free() { + + } +} diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java index 96304e57..cbc49cd4 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java @@ -46,7 +46,7 @@ public class RenderDataFactory { //buildMask in the lower 6 bits contains the faces to build, the next 6 bits are whether the edge face builds against // its neigbor or not (0 if it does 1 if it doesnt (0 is default behavior)) - public BuiltSection generateMesh(WorldSection section, int buildMask) { + public BuiltSection generateMesh(WorldSection section) { section.copyDataTo(this.sectionCache); this.translucentQuadCollector.clear(); this.doubleSidedQuadCollector.clear(); @@ -75,7 +75,7 @@ public class RenderDataFactory { } if (quadCount == 0) { - return new BuiltSection(section.getKey()); + return new BuiltSection(section.key); } var buff = new MemoryBuffer(quadCount*8L); @@ -109,7 +109,7 @@ public class RenderDataFactory { aabb |= (this.maxY-this.minY)<<20; aabb |= (this.maxZ-this.minZ)<<25; - return new BuiltSection(section.getKey(), aabb, buff, offsets); + return new BuiltSection(section.key, aabb, buff, offsets); } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java index 5128efab..9aa34d7f 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java @@ -14,7 +14,7 @@ import java.util.function.ToIntFunction; //TODO: Add a render cache public class RenderGenerationService { public interface TaskChecker {boolean check(int lvl, int x, int y, int z);} - private record BuildTask(Supplier sectionSupplier, ToIntFunction flagSupplier) {} + private record BuildTask(Supplier sectionSupplier) {} private volatile boolean running = true; private final Thread[] workers; @@ -25,6 +25,7 @@ public class RenderGenerationService { private final WorldEngine world; private final ModelManager modelManager; private final Consumer resultConsumer; + private final BuiltSectionMeshCache meshCache = new BuiltSectionMeshCache(); public RenderGenerationService(WorldEngine world, ModelManager modelManager, int workers, Consumer consumer) { this.world = world; @@ -39,8 +40,6 @@ public class RenderGenerationService { } } - private final ConcurrentHashMap renderCache = new ConcurrentHashMap<>(1000,0.75f,10); - //TODO: add a generated render data cache private void renderWorker() { //Thread local instance of the factory @@ -57,23 +56,12 @@ public class RenderGenerationService { continue; } section.assertNotFree(); - int buildFlags = task.flagSupplier.applyAsInt(section); - if (buildFlags != 0) { - var mesh = factory.generateMesh(section, buildFlags); - section.release(); + var mesh = factory.generateMesh(section); + section.release(); - this.resultConsumer.accept(mesh.clone()); - - if (false) { - var prevCache = this.renderCache.put(mesh.position, mesh); - if (prevCache != null) { - prevCache.free(); - } - } else { - mesh.free(); - } - } else { - section.release(); + this.resultConsumer.accept(mesh.clone()); + if (!this.meshCache.putMesh(mesh)) { + mesh.free(); } } } @@ -94,14 +82,14 @@ public class RenderGenerationService { // like if its in the render queue and if we should abort building the render data //1 proposal fix is a Long2ObjectLinkedOpenHashMap which means we can abort if needed, // also gets rid of dependency on a WorldSection (kinda) - public void enqueueTask(int lvl, int x, int y, int z, ToIntFunction flagSupplier) { - this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true, flagSupplier); + public void enqueueTask(int lvl, int x, int y, int z) { + this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true); } - public void enqueueTask(int lvl, int x, int y, int z, TaskChecker checker, ToIntFunction flagSupplier) { + public void enqueueTask(int lvl, int x, int y, int z, TaskChecker checker) { long ikey = WorldEngine.getWorldSectionId(lvl, x, y, z); { - var cache = this.renderCache.get(ikey); + var cache = this.meshCache.getMesh(ikey); if (cache != null) { this.resultConsumer.accept(cache.clone()); return; @@ -116,7 +104,7 @@ public class RenderGenerationService { } else { return null; } - }, flagSupplier); + }); }); } } @@ -159,6 +147,6 @@ public class RenderGenerationService { while (!this.taskQueue.isEmpty()) { this.taskQueue.removeFirst(); } - this.renderCache.values().forEach(BuiltSection::free); + this.meshCache.free(); } } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java b/src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java new file mode 100644 index 00000000..9565f212 --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/core/rendering/util/DownloadStream.java @@ -0,0 +1,127 @@ +package me.cortex.voxy.client.core.rendering.util; + +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongConsumer; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import me.cortex.voxy.client.core.gl.GlBuffer; +import me.cortex.voxy.client.core.gl.GlFence; +import me.cortex.voxy.client.core.gl.GlPersistentMappedBuffer; +import me.cortex.voxy.client.core.util.AllocationArena; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; + +import static me.cortex.voxy.client.core.util.AllocationArena.SIZE_LIMIT; +import static org.lwjgl.opengl.ARBDirectStateAccess.glCopyNamedBufferSubData; +import static org.lwjgl.opengl.ARBDirectStateAccess.glFlushMappedNamedBufferRange; +import static org.lwjgl.opengl.ARBMapBufferRange.*; +import static org.lwjgl.opengl.GL11.glFinish; +import static org.lwjgl.opengl.GL42.glMemoryBarrier; +import static org.lwjgl.opengl.GL42C.GL_BUFFER_UPDATE_BARRIER_BIT; +import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BARRIER_BIT; +import static org.lwjgl.opengl.GL44.GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT; + +public class DownloadStream { + public interface DownloadResultConsumer { + void consume(long ptr, long size); + } + + private final AllocationArena allocationArena = new AllocationArena(); + private final GlPersistentMappedBuffer downloadBuffer; + + private final Deque frames = new ArrayDeque<>(); + private final LongArrayList thisFrameAllocations = new LongArrayList(); + private final Deque downloadList = new ArrayDeque<>(); + private final ArrayList thisFrameDownloadList = new ArrayList<>(); + + public DownloadStream(long size) { + this.downloadBuffer = new GlPersistentMappedBuffer(size, GL_MAP_READ_BIT); + this.allocationArena.setLimit(size); + } + + private long caddr = -1; + private long offset = 0; + public void download(GlBuffer buffer, long destOffset, long size, DownloadResultConsumer resultConsumer) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException(); + } + + long addr; + if (this.caddr == -1 || !this.allocationArena.expand(this.caddr, (int) size)) { + this.caddr = this.allocationArena.alloc((int) size);//TODO: replace with allocFromLargest + if (this.caddr == SIZE_LIMIT) { + this.commit(); + int attempts = 10; + while (--attempts != 0 && this.caddr == SIZE_LIMIT) { + glFinish(); + this.tick(); + this.caddr = this.allocationArena.alloc((int) size); + } + if (this.caddr == SIZE_LIMIT) { + throw new IllegalStateException("Could not allocate memory segment big enough for upload even after force flush"); + } + } + this.thisFrameAllocations.add(this.caddr); + this.offset = size; + addr = this.caddr; + } else {//Could expand the allocation so just update it + addr = this.caddr + this.offset; + this.offset += size; + } + + if (this.caddr + size > this.downloadBuffer.size()) { + throw new IllegalStateException(); + } + + this.downloadList.add(new DownloadData(buffer, addr, destOffset, size, resultConsumer)); + } + + + public void commit() { + //Copies all the data from target buffers into the download stream + for (var entry : this.downloadList) { + glCopyNamedBufferSubData(entry.target.id, this.downloadBuffer.id, entry.downloadOffset, entry.targetOffset, entry.size); + } + thisFrameDownloadList.addAll(this.downloadList); + this.downloadList.clear(); + + this.caddr = -1; + this.offset = 0; + } + + public void tick() { + this.commit(); + if (!this.thisFrameAllocations.isEmpty()) { + glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT | GL_BUFFER_UPDATE_BARRIER_BIT); + this.frames.add(new DownloadFrame(new GlFence(), new LongArrayList(this.thisFrameAllocations), new ArrayList<>(this.thisFrameDownloadList))); + this.thisFrameAllocations.clear(); + this.thisFrameDownloadList.clear(); + } + + while (!this.frames.isEmpty()) { + //Since the ordering of frames is the ordering of the gl commands if we encounter an unsignaled fence + // all the other fences should also be unsignaled + if (!this.frames.peek().fence.signaled()) { + break; + } + //Release all the allocations from the frame + var frame = this.frames.pop(); + + //Apply all the callbacks + for (var data : frame.data) { + data.resultConsumer.consume(this.downloadBuffer.addr() + data.downloadOffset, data.size); + } + + frame.allocations.forEach(this.allocationArena::free); + frame.fence.free(); + } + } + + private record DownloadFrame(GlFence fence, LongArrayList allocations, ArrayList data) {} + private record DownloadData(GlBuffer target, long downloadOffset, long targetOffset, long size, DownloadResultConsumer resultConsumer) {} + + + // Global download stream + public static final DownloadStream INSTANCE = new DownloadStream(1<<25);//32 mb download buffer +} diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java b/src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java index e7e2c2c0..ed037c93 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/util/UploadStream.java @@ -111,7 +111,7 @@ public class UploadStream { } //Release all the allocations from the frame var frame = this.frames.pop(); - frame.allocations.forEach(allocationArena::free); + frame.allocations.forEach(this.allocationArena::free); frame.fence.free(); } } diff --git a/src/main/java/me/cortex/voxy/common/world/ActiveSectionTracker.java b/src/main/java/me/cortex/voxy/common/world/ActiveSectionTracker.java index a54d3781..7d190962 100644 --- a/src/main/java/me/cortex/voxy/common/world/ActiveSectionTracker.java +++ b/src/main/java/me/cortex/voxy/common/world/ActiveSectionTracker.java @@ -13,17 +13,17 @@ public class ActiveSectionTracker { private final SectionLoader loader; @SuppressWarnings("unchecked") - public ActiveSectionTracker(int layers, SectionLoader loader) { + public ActiveSectionTracker(int cacheSizeBits, SectionLoader loader) { this.loader = loader; - this.loadedSectionCache = new Long2ObjectOpenHashMap[layers]; - for (int i = 0; i < layers; i++) { - this.loadedSectionCache[i] = new Long2ObjectOpenHashMap<>(1<<(16-i)); + this.loadedSectionCache = new Long2ObjectOpenHashMap[1<(1024); } } public WorldSection acquire(int lvl, int x, int y, int z, boolean nullOnEmpty) { long key = WorldEngine.getWorldSectionId(lvl, x, y, z); - var cache = this.loadedSectionCache[lvl]; + var cache = this.loadedSectionCache[this.getCacheArrayIndex(key)]; VolatileHolder holder = null; boolean isLoader = false; synchronized (cache) { @@ -69,16 +69,25 @@ public class ActiveSectionTracker { } void tryUnload(WorldSection section) { - var cache = this.loadedSectionCache[section.lvl]; + var cache = this.loadedSectionCache[this.getCacheArrayIndex(section.key)]; synchronized (cache) { if (section.trySetFreed()) { - if (cache.remove(section.getKey()).obj != section) { + if (cache.remove(section.key).obj != section) { throw new IllegalStateException("Removed section not the same as the referenced section in the cache"); } } } } + private int getCacheArrayIndex(long pos) { + return (int) (mixStafford13(pos) & (this.loadedSectionCache.length-1)); + } + + public static long mixStafford13(long seed) { + seed = (seed ^ seed >>> 30) * -4658895280553007687L; + seed = (seed ^ seed >>> 27) * -7723592293110705685L; + return seed ^ seed >>> 31; + } public int[] getCacheCounts() { int[] res = new int[this.loadedSectionCache.length]; diff --git a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java index d74dba13..41a0aaf0 100644 --- a/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java @@ -25,8 +25,8 @@ public class SaveLoadSystem { long[] lut = LUTVAL.toLongArray(); ByteBuffer raw = MemoryUtil.memAlloc(compressed.length*2+lut.length*8+512); - long hash = section.getKey()^(lut.length*1293481298141L); - raw.putLong(section.getKey()); + long hash = section.key^(lut.length*1293481298141L); + raw.putLong(section.key); raw.putInt(lut.length); for (long id : lut) { raw.putLong(id); @@ -74,8 +74,8 @@ public class SaveLoadSystem { hash ^= lut[i]; } - if (section.getKey() != key) { - throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.getKey()); + if (section.key != key) { + throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key); } for (int i = 0; i < section.data.length; i++) { diff --git a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java index a75a7fe4..6acf766d 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldEngine.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldEngine.java @@ -33,18 +33,19 @@ public class WorldEngine { this.maxMipLevels = maxMipLayers; this.storage = storageBackend; this.mapper = new Mapper(this.storage); - this.sectionTracker = new ActiveSectionTracker(maxMipLayers, this::unsafeLoadSection); + //4 cache size bits means that the section tracker has 16 separate maps that it uses + this.sectionTracker = new ActiveSectionTracker(4, this::unsafeLoadSection); this.savingService = new SectionSavingService(this, savingServiceWorkers, compressionLevel); this.ingestService = new VoxelIngestService(this, ingestWorkers); } private int unsafeLoadSection(WorldSection into) { - var data = this.storage.getSectionData(into.getKey()); + var data = this.storage.getSectionData(into.key); if (data != null) { try { if (!SaveLoadSystem.deserialize(into, data)) { - this.storage.deleteSectionData(into.getKey()); + this.storage.deleteSectionData(into.key); //TODO: regenerate the section from children Arrays.fill(into.data, Mapper.AIR); System.err.println("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, setting to air"); diff --git a/src/main/java/me/cortex/voxy/common/world/WorldSection.java b/src/main/java/me/cortex/voxy/common/world/WorldSection.java index 1fd3aaf5..add1fd7c 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldSection.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldSection.java @@ -12,6 +12,7 @@ public final class WorldSection { public final int x; public final int y; public final int z; + public final long key; long[] data; private final ActiveSectionTracker tracker; @@ -25,6 +26,7 @@ public final class WorldSection { this.x = x; this.y = y; this.z = z; + this.key = WorldEngine.getWorldSectionId(lvl, x, y, z); this.tracker = tracker; this.data = new long[32*32*32]; @@ -78,10 +80,6 @@ public final class WorldSection { } } - public long getKey() { - return WorldEngine.getWorldSectionId(this.lvl, this.x, this.y, this.z); - } - public static int getIndex(int x, int y, int z) { int M = (1<<5)-1; if (x<0||x>M||y<0||y>M||z<0||z>M) { diff --git a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java index 71a1f14e..037e4e1e 100644 --- a/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java +++ b/src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java @@ -43,7 +43,7 @@ public class SectionSavingService { section.inSaveQueue.set(false); var saveData = SaveLoadSystem.serialize(section, this.compressionLevel); - this.world.storage.setSectionData(section.getKey(), saveData); + this.world.storage.setSectionData(section.key, saveData); MemoryUtil.memFree(saveData); section.release(); diff --git a/src/main/resources/assets/voxy/shaders/lod/gl46/quad_format.glsl b/src/main/resources/assets/voxy/shaders/lod/gl46/quad_format.glsl index 51978518..8183f7d7 100644 --- a/src/main/resources/assets/voxy/shaders/lod/gl46/quad_format.glsl +++ b/src/main/resources/assets/voxy/shaders/lod/gl46/quad_format.glsl @@ -58,7 +58,7 @@ uint extractFace(ivec2 quad) { uint extractStateId(ivec2 quad) { //Eu32(quad, 20, 26); - return 1; + return Eu32v(quad, 6, 26)|(Eu32v(quad, 14, 32)<<6); } uint extractBiomeId(ivec2 quad) {