diff --git a/src/main/java/me/cortex/voxelmon/core/VoxelCore.java b/src/main/java/me/cortex/voxelmon/core/VoxelCore.java index 0a291e71..cfed5d5d 100644 --- a/src/main/java/me/cortex/voxelmon/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxelmon/core/VoxelCore.java @@ -60,10 +60,10 @@ public class VoxelCore { //Trigger the shared index buffer loading SharedIndexBuffer.INSTANCE.id(); this.renderer = new Gl46FarWorldRenderer(); - this.world = new WorldEngine(new File("storagefile.db"), 5, 5);//"storagefile.db"//"ethoslab.db" + this.world = new WorldEngine(new File("storagefile.db"), 20, 5);//"storagefile.db"//"ethoslab.db" this.renderTracker = new RenderTracker(this.world, this.renderer); - this.renderGen = new RenderGenerationService(this.world, this.renderTracker,4); + this.renderGen = new RenderGenerationService(this.world,4, this.renderTracker::processBuildResult); this.world.setRenderTracker(this.renderTracker); this.renderTracker.setRenderGen(this.renderGen); @@ -179,8 +179,8 @@ public class VoxelCore { public void shutdown() { try {this.renderGen.shutdown();} catch (Exception e) {System.err.println(e);} + try {this.world.shutdown();} catch (Exception e) {System.err.println(e);} try {this.renderer.shutdown();} catch (Exception e) {System.err.println(e);} try {this.postProcessing.shutdown();} catch (Exception e) {System.err.println(e);} - try {this.world.shutdown();} catch (Exception e) {System.err.println(e);} } } diff --git a/src/main/java/me/cortex/voxelmon/core/rendering/RenderTracker.java b/src/main/java/me/cortex/voxelmon/core/rendering/RenderTracker.java index 289f2bb4..0e5ac7f2 100644 --- a/src/main/java/me/cortex/voxelmon/core/rendering/RenderTracker.java +++ b/src/main/java/me/cortex/voxelmon/core/rendering/RenderTracker.java @@ -28,6 +28,7 @@ public class RenderTracker { public void setRenderGen(RenderGenerationService renderGen) { this.renderGen = renderGen; } + public RenderTracker(WorldEngine world, AbstractFarWorldRenderer renderer) { this.world = world; this.renderer = renderer; @@ -76,14 +77,15 @@ public class RenderTracker { //Adds a lvl 0 section into the world renderer public void addLvl0(int x, int y, int z) { - this.renderGen.enqueueTask(0, x, y, z); this.activeSections.put(WorldEngine.getWorldSectionId(0, x, y, z), O); + this.renderGen.enqueueTask(0, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort); } //Removes a lvl 0 section from the world renderer public void remLvl0(int x, int y, int z) { this.activeSections.remove(WorldEngine.getWorldSectionId(0, x, y, z)); this.renderer.enqueueResult(new BuiltSectionGeometry(WorldEngine.getWorldSectionId(0, x, y, z), null, null)); + this.renderGen.removeTask(0, x, y, z); } //Increases from lvl-1 to lvl at the coordinates (which are in lvl space) @@ -102,7 +104,7 @@ public class RenderTracker { // concurrent hashmap or something, this is so that e.g. the build data position // can be updated - this.renderGen.enqueueTask(lvl, x, y, z); + this.renderGen.enqueueTask(lvl, x, y, z, this::shouldStillBuild, this::getBuildFlagsOrAbort); this.renderer.enqueueResult(new BuiltSectionGeometry(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1)), null, null)); this.renderer.enqueueResult(new BuiltSectionGeometry(WorldEngine.getWorldSectionId(lvl-1, (x<<1), (y<<1), (z<<1)+1), null, null)); @@ -114,8 +116,14 @@ public class RenderTracker { this.renderer.enqueueResult(new BuiltSectionGeometry(WorldEngine.getWorldSectionId(lvl-1, (x<<1)+1, (y<<1)+1, (z<<1)+1), null, null)); - - + this.renderGen.removeTask(lvl-1, (x<<1), (y<<1), (z<<1)); + this.renderGen.removeTask(lvl-1, (x<<1), (y<<1), (z<<1)+1); + this.renderGen.removeTask(lvl-1, (x<<1), (y<<1)+1, (z<<1)); + this.renderGen.removeTask(lvl-1, (x<<1), (y<<1)+1, (z<<1)+1); + this.renderGen.removeTask(lvl-1, (x<<1)+1, (y<<1), (z<<1)); + this.renderGen.removeTask(lvl-1, (x<<1)+1, (y<<1), (z<<1)+1); + this.renderGen.removeTask(lvl-1, (x<<1)+1, (y<<1)+1, (z<<1)); + this.renderGen.removeTask(lvl-1, (x<<1)+1, (y<<1)+1, (z<<1)+1); } //Decreases from lvl to lvl-1 at the coordinates (which are in lvl space) @@ -131,15 +139,16 @@ public class RenderTracker { this.activeSections.remove(WorldEngine.getWorldSectionId(lvl, x, y, z)); this.renderer.enqueueResult(new BuiltSectionGeometry(lvl, x, y, z, null, null)); + this.renderGen.removeTask(lvl, x, y, z); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)); - this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)); - this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1), (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1), (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1), this::shouldStillBuild, this::getBuildFlagsOrAbort); + this.renderGen.enqueueTask(lvl - 1, (x<<1)+1, (y<<1)+1, (z<<1)+1, this::shouldStillBuild, this::getBuildFlagsOrAbort); } @@ -187,7 +196,7 @@ public class RenderTracker { buildMask |= 1< sectionSupplier) {} + public interface TaskChecker {boolean check(int lvl, int x, int y, int z);} + private record BuildTask(Supplier sectionSupplier, ToIntFunction flagSupplier) {} private volatile boolean running = true; private final Thread[] workers; - private final ConcurrentLinkedDeque taskQueue = new ConcurrentLinkedDeque<>(); + private final Long2ObjectLinkedOpenHashMap taskQueue = new Long2ObjectLinkedOpenHashMap<>(); private final Semaphore taskCounter = new Semaphore(0); - private final WorldEngine world; - private final RenderTracker tracker; + private final Consumer resultConsumer; - public RenderGenerationService(WorldEngine world, RenderTracker tracker, int workers) { + public RenderGenerationService(WorldEngine world, int workers, Consumer consumer) { this.world = world; - this.tracker = tracker; - + this.resultConsumer = consumer; this.workers = new Thread[workers]; for (int i = 0; i < workers; i++) { this.workers[i] = new Thread(this::renderWorker); @@ -41,6 +41,7 @@ public class RenderGenerationService { } } + private final ConcurrentHashMap renderCache = new ConcurrentHashMap<>(1000,0.75f,10); //TODO: add a generated render data cache private void renderWorker() { @@ -49,15 +50,23 @@ public class RenderGenerationService { while (this.running) { this.taskCounter.acquireUninterruptibly(); if (!this.running) break; - var task = this.taskQueue.pop(); + BuildTask task; + synchronized (this.taskQueue) { + task = this.taskQueue.removeFirst(); + } var section = task.sectionSupplier.get(); if (section == null) { continue; } section.assertNotFree(); - int buildFlags = this.tracker.getBuildFlagsOrAbort(section); + int buildFlags = task.flagSupplier.applyAsInt(section); if (buildFlags != 0) { - this.tracker.processBuildResult(factory.generateMesh(section, buildFlags)); + var mesh = factory.generateMesh(section, buildFlags); + this.resultConsumer.accept(mesh.clone()); + var prevCache = this.renderCache.put(mesh.position, mesh); + if (prevCache != null) { + prevCache.free(); + } } section.release(); } @@ -79,26 +88,39 @@ public class RenderGenerationService { // like if its in the render queue and if we should abort building the render data //1 proposal fix is a Long2ObjectLinkedOpenHashMap which means we can abort if needed, // also gets rid of dependency on a WorldSection (kinda) - public void enqueueTask(int lvl, int x, int y, int z) { - this.taskQueue.add(new BuildTask(()->{ - if (this.tracker.shouldStillBuild(lvl, x, y, z)) { - return this.world.acquire(lvl, x, y, z); - } else { - return null; - } - })); - this.taskCounter.release(); + public void enqueueTask(int lvl, int x, int y, int z, ToIntFunction flagSupplier) { + this.enqueueTask(lvl, x, y, z, (l,x1,y1,z1)->true, flagSupplier); } - public void enqueueTask(WorldSection section) { - //TODO: fixme! buildMask could have changed - //if (!section.inRenderQueue.getAndSet(true)) { - // //TODO: add a boolean for needsRenderUpdate that can be set to false if the section is no longer needed - // // to be rendered, e.g. LoD level changed so that lod is no longer being rendered - // section.acquire(); - // this.taskQueue.add(new BuildTask(()->section)); - // this.taskCounter.release(); - //} + public void enqueueTask(int lvl, int x, int y, int z, TaskChecker checker, ToIntFunction flagSupplier) { + long ikey = WorldEngine.getWorldSectionId(lvl, x, y, z); + { + var cache = this.renderCache.get(ikey); + if (cache != null) { + this.resultConsumer.accept(cache.clone()); + return; + } + } + synchronized (this.taskQueue) { + this.taskQueue.computeIfAbsent(ikey, key->{ + this.taskCounter.release(); + return new BuildTask(()->{ + if (checker.check(lvl, x, y, z)) { + return this.world.acquire(lvl, x, y, z); + } else { + return null; + } + }, flagSupplier); + }); + } + } + + public void removeTask(int lvl, int x, int y, int z) { + synchronized (this.taskQueue) { + if (this.taskQueue.remove(WorldEngine.getWorldSectionId(lvl, x, y, z)) != null) { + this.taskCounter.acquireUninterruptibly(); + } + } } public int getTaskCount() { @@ -129,7 +151,8 @@ public class RenderGenerationService { //Cleanup any remaining data while (!this.taskQueue.isEmpty()) { - this.taskQueue.pop(); + this.taskQueue.removeFirst(); } + this.renderCache.values().forEach(BuiltSectionGeometry::free); } } diff --git a/src/main/java/me/cortex/voxelmon/core/util/MemoryBuffer.java b/src/main/java/me/cortex/voxelmon/core/util/MemoryBuffer.java index f4f1ae95..14cb76e2 100644 --- a/src/main/java/me/cortex/voxelmon/core/util/MemoryBuffer.java +++ b/src/main/java/me/cortex/voxelmon/core/util/MemoryBuffer.java @@ -21,4 +21,10 @@ public class MemoryBuffer extends TrackedObject { super.free0(); MemoryUtil.nmemFree(this.address); } + + public MemoryBuffer copy() { + var copy = new MemoryBuffer(this.size); + this.cpyTo(copy.address); + return copy; + } } diff --git a/src/main/java/me/cortex/voxelmon/core/world/SaveLoadSystem.java b/src/main/java/me/cortex/voxelmon/core/world/SaveLoadSystem.java index 942409ad..f81b14c2 100644 --- a/src/main/java/me/cortex/voxelmon/core/world/SaveLoadSystem.java +++ b/src/main/java/me/cortex/voxelmon/core/world/SaveLoadSystem.java @@ -56,7 +56,7 @@ public class SaveLoadSystem { raw.limit(raw.position()); raw.rewind(); ByteBuffer compressedData = MemoryUtil.memAlloc((int)ZSTD_COMPRESSBOUND(raw.remaining())); - long compressedSize = ZSTD_compress(compressedData, raw, 19); + long compressedSize = ZSTD_compress(compressedData, raw, 15); byte[] out = new byte[(int) compressedSize]; compressedData.limit((int) compressedSize); compressedData.get(out); diff --git a/src/main/java/me/cortex/voxelmon/core/world/service/SectionSavingService.java b/src/main/java/me/cortex/voxelmon/core/world/service/SectionSavingService.java index 4225a602..634467a4 100644 --- a/src/main/java/me/cortex/voxelmon/core/world/service/SectionSavingService.java +++ b/src/main/java/me/cortex/voxelmon/core/world/service/SectionSavingService.java @@ -69,18 +69,28 @@ public class SectionSavingService { public void shutdown() { boolean anyAlive = false; + boolean allAlive = true; for (var worker : this.workers) { anyAlive |= worker.isAlive(); + allAlive &= worker.isAlive(); } if (!anyAlive) { System.err.println("Section saving workers already dead on shutdown! this is very very bad, check log for errors from this thread"); return; } + if (!allAlive) { + System.err.println("Some section saving works have died, please check log and report errors."); + } + + int i = 0; //Wait for all the saving to finish while (this.saveCounter.availablePermits() != 0) { - Thread.onSpinWait(); + try {Thread.sleep(500);} catch (InterruptedException e) {break;} + if (i++%10 == 0) { + System.out.println("Section saving shutdown has " + this.saveCounter.availablePermits() + " tasks remaining"); + } } //Shutdown this.running = false; diff --git a/src/main/java/me/cortex/voxelmon/importers/WorldImporter.java b/src/main/java/me/cortex/voxelmon/importers/WorldImporter.java index 0167e365..5c172071 100644 --- a/src/main/java/me/cortex/voxelmon/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxelmon/importers/WorldImporter.java @@ -72,6 +72,7 @@ public class WorldImporter { } private void importRegionFile(Path file, int x, int z) throws IOException { + //if (true) return; try (var fileStream = FileChannel.open(file, StandardOpenOption.READ)) { var sectorsSavesBB = MemoryUtil.memAlloc(8192); if (fileStream.read(sectorsSavesBB, 0) != 8192) {