spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added taskbar progress on windows when importing worlds

Browse Source 
Fixed memory leak in RenderDataFactory4
Moved shader debug options
Inital work on visibility sorting and node cleanup
Added optional cleanup system to services.
Track (and added to f3) GlBuffer and MemoryBuffer count and sizes.

More work on finishing node manager (its still broken when importing for some reason, requiring reload)
This commit is contained in:
mcrcortex
2024-12-13 12:47:59 +10:00
parent 8a3d7a9da5
commit 029a7df71a
26 changed files with 475 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/me/cortex/voxy/client/Voxy.java
View File

@@ -7,20 +7,17 @@ import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.Serialization;
import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
import me.cortex.voxy.common.storage.config.StorageConfig;
import net.caffeinemc.mods.sodium.client.compatibility.environment.OsUtils;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback;
import net.fabricmc.loader.api.FabricLoader;
import net.fabricmc.loader.api.ModContainer;
import net.minecraft.client.world.ClientWorld;
import org.apache.commons.lang3.SystemUtils;
import java.util.Arrays;
public class Voxy implements ClientModInitializer {
public static final boolean SHADER_DEBUG;
static {
SHADER_DEBUG = System.getProperty("voxy.shaderDebug", "false").equals("true");
}
@Override
public void onInitializeClient() {
ClientCommandRegistrationCallback.EVENT.register((dispatcher, registryAccess) -> {

2
src/main/java/me/cortex/voxy/client/config/VoxyConfigScreenFactory.java
View File

@@ -69,7 +69,7 @@ public class VoxyConfigScreenFactory implements ModMenuApi {
.setDefaultValue(DEFAULT.ingestEnabled)
.build());
category.addEntry(entryBuilder.startIntSlider(Text.translatable("voxy.config.general.subDivisionSize"), config.subDivisionSize, 32, 256)
category.addEntry(entryBuilder.startIntSlider(Text.translatable("voxy.config.general.subDivisionSize"), config.subDivisionSize, 40, 256)
.setTooltip(Text.translatable("voxy.config.general.subDivisionSize.tooltip"))
.setSaveConsumer(val -> config.subDivisionSize = val)
.setDefaultValue(DEFAULT.subDivisionSize)

29
src/main/java/me/cortex/voxy/client/core/VoxelCore.java
View File

@@ -1,19 +1,18 @@
package me.cortex.voxy.client.core;
import com.mojang.blaze3d.systems.RenderSystem;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import me.cortex.voxy.client.Voxy;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.client.core.model.IdNotYetComputedException;
import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
import me.cortex.voxy.client.core.rendering.*;
import me.cortex.voxy.client.core.rendering.building.RenderDataFactory;
import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4;
import me.cortex.voxy.client.core.rendering.post.PostProcessing;
import me.cortex.voxy.client.core.rendering.util.DownloadStream;
import me.cortex.voxy.client.core.util.IrisUtil;
import me.cortex.voxy.client.saver.ContextSelectionSystem;
import me.cortex.voxy.client.taskbar.Taskbar;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.client.importers.WorldImporter;
import me.cortex.voxy.common.thread.ServiceThreadPool;
@@ -67,6 +66,8 @@ public class VoxelCore {
private final ServiceThreadPool serviceThreadPool;
private WorldImporter importer;
private UUID importerBossBarUUID;
public VoxelCore(ContextSelectionSystem.Selection worldSelection) {
var cfg = worldSelection.getConfig();
this.serviceThreadPool = new ServiceThreadPool(VoxyConfig.CONFIG.serviceThreads);
@@ -167,6 +168,8 @@ public class VoxelCore {
debug.add("");
debug.add("");
debug.add("Voxy Core: " + VoxyCommon.MOD_VERSION);
debug.add("MemoryBuffer, Count/Size (mb): " + MemoryBuffer.getCount() + "/" + (MemoryBuffer.getTotalSize()/1_000_000));
debug.add("GlBuffer, Count/Size (mb): " + GlBuffer.getCount() + "/" + (GlBuffer.getTotalSize()/1_000_000));
/*
debug.add("Ingest service tasks: " + this.world.ingestService.getTaskCount());
debug.add("Saving service tasks: " + this.world.savingService.getTaskCount());
@@ -205,6 +208,11 @@ public class VoxelCore {
Logger.info("Shutting down service thread pool");
this.serviceThreadPool.shutdown();
Logger.info("Voxel core shut down");
//Remove bossbar
if (this.importerBossBarUUID != null) {
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID);
Taskbar.INSTANCE.setIsNone();
}
}
public boolean createWorldImporter(World mcWorld, File worldPath) {
@@ -215,19 +223,26 @@ public class VoxelCore {
return false;
}
var bossBar = new ClientBossBar(MathHelper.randomUuid(), Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false);
Taskbar.INSTANCE.setProgress(0,10000);
Taskbar.INSTANCE.setIsProgression();
this.importerBossBarUUID = MathHelper.randomUuid();
var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false);
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar);
this.importer.importWorldAsyncStart(worldPath, (a,b)->
MinecraftClient.getInstance().executeSync(()-> {
Taskbar.INSTANCE.setProgress(a, b);
bossBar.setPercent(((float) a)/((float) b));
bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks"));
}),
()-> {
MinecraftClient.getInstance().executeSync(()-> {
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(bossBar.getUuid());
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID);
this.importerBossBarUUID = null;
String msg = "Voxy world import finished";
MinecraftClient.getInstance().inGameHud.getChatHud().addMessage(Text.literal(msg));
System.err.println(msg);
Logger.info(msg);
Taskbar.INSTANCE.setIsNone();
});
});
return true;

18
src/main/java/me/cortex/voxy/client/core/gl/GlBuffer.java
View File

@@ -11,6 +11,9 @@ public class GlBuffer extends TrackedObject {
public final int id;
private final long size;
private static int COUNT;
private static long TOTAL_SIZE;
public GlBuffer(long size) {
this(size, 0);
}
@@ -19,12 +22,19 @@ public class GlBuffer extends TrackedObject {
this.id = glCreateBuffers();
this.size = size;
glNamedBufferStorage(this.id, size, flags);
this.zero();
COUNT++;
TOTAL_SIZE += size;
}
@Override
public void free() {
this.free0();
glDeleteBuffers(this.id);
COUNT--;
TOTAL_SIZE -= this.size;
}
public long size() {
@@ -35,4 +45,12 @@ public class GlBuffer extends TrackedObject {
nglClearNamedBufferData(this.id, GL_R8UI, GL_RED_INTEGER, GL_UNSIGNED_BYTE, 0);
return this;
}
public static int getCount() {
return COUNT;
}
public static long getTotalSize() {
return TOTAL_SIZE;
}
}

2
src/main/java/me/cortex/voxy/client/core/rendering/PrintfDebugUtil.java
View File

@@ -9,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
public final class PrintfDebugUtil {
public static final boolean ENABLE_PRINTF_DEBUGGING = System.getProperty("voxy.enableShaderDebugPrintf", "false").equals("true") || Voxy.SHADER_DEBUG;
public static final boolean ENABLE_PRINTF_DEBUGGING = System.getProperty("voxy.enableShaderDebugPrintf", "false").equals("true");
private static final List<String> printfQueue2 = new ArrayList<>();
private static final List<String> printfQueue = new ArrayList<>();

17
src/main/java/me/cortex/voxy/client/core/rendering/RenderService.java
View File

@@ -7,6 +7,7 @@ import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
import me.cortex.voxy.client.core.rendering.building.SectionUpdateRouter;
import me.cortex.voxy.client.core.rendering.hierachical2.HierarchicalNodeManager;
import me.cortex.voxy.client.core.rendering.hierachical2.HierarchicalOcclusionTraverser;
import me.cortex.voxy.client.core.rendering.hierachical2.NodeCleaner;
import me.cortex.voxy.client.core.rendering.hierachical2.NodeManager2;
import me.cortex.voxy.client.core.rendering.section.AbstractSectionRenderer;
import me.cortex.voxy.client.core.rendering.section.IUsesMeshlets;
@@ -35,6 +36,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
private final AbstractSectionRenderer<J, ?> sectionRenderer;
private final NodeManager2 nodeManager;
private final NodeCleaner nodeCleaner;
private final HierarchicalOcclusionTraverser traversal;
private final ModelBakerySubsystem modelService;
private final RenderGenerationService renderGen;
@@ -47,12 +49,13 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
//Max sections: ~500k
//Max geometry: 1 gb
this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<19, (1L<<31)-1024);
this.sectionRenderer = (T) createSectionRenderer(this.modelService.getStore(),1<<20, (1L<<31)-1024);
//Do something incredibly hacky, we dont need to keep the reference to this around, so just connect and discard
var router = new SectionUpdateRouter();
this.nodeManager = new NodeManager2(1<<21, this.sectionRenderer.getGeometryManager(), router);
this.nodeCleaner = new NodeCleaner(this.nodeManager);
this.sectionUpdateQueue = new MessageQueue<>(section -> {
byte childExistence = section.getNonEmptyChildren();
@@ -69,13 +72,15 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
this.sectionUpdateQueue.push(section);
});
this.traversal = new HierarchicalOcclusionTraverser(this.nodeManager);
this.traversal = new HierarchicalOcclusionTraverser(this.nodeManager, this.nodeCleaner);
world.setDirtyCallback(router::forward);
Arrays.stream(world.getMapper().getBiomeEntries()).forEach(this.modelService::addBiome);
world.getMapper().setBiomeCallback(this.modelService::addBiome);
//this.nodeManager.insertTopLevelNode(WorldEngine.getWorldSectionId(0, 0,0,0));
//this.nodeManager.insertTopLevelNode(WorldEngine.getWorldSectionId(4, 0,0,0));
@@ -89,7 +94,8 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
}
}*/
final int H_WIDTH = 10;
final int H_WIDTH = 20;
for (int x = -H_WIDTH; x <= H_WIDTH; x++) {
for (int y = -1; y <= 0; y++) {
for (int z = -H_WIDTH; z <= H_WIDTH; z++) {
@@ -97,6 +103,8 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
}
}
}
//this.nodeManager.insertTopLevelNode(WorldEngine.getWorldSectionId(4, 0,0,0));
}
public void setup(Camera camera) {
@@ -129,6 +137,8 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
if (true /* firstInvocationThisFrame */) {
DownloadStream.INSTANCE.tick();
this.nodeCleaner.tick();//Probably do this here??
this.sectionUpdateQueue.consume();
this.geometryUpdateQueue.consume();
if (this.nodeManager.writeChanges(this.traversal.getNodeBuffer())) {//TODO: maybe move the node buffer out of the traversal class
@@ -167,6 +177,7 @@ public class RenderService<T extends AbstractSectionRenderer<J, ?>, J extends Vi
this.viewportSelector.free();
this.sectionRenderer.free();
this.traversal.free();
this.nodeCleaner.free();
//Release all the unprocessed built geometry
this.geometryUpdateQueue.clear(BuiltSection::free);
}

44
src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory4.java
View File

@@ -43,6 +43,7 @@ public class RenderDataFactory4 {
private int quadCount = 0;
//Wont work for double sided quads
private final class Mesher extends ScanMesher2D {
public int auxiliaryPosition = 0;
@@ -123,48 +124,30 @@ public class RenderDataFactory4 {
private void prepareSectionData() {
final var sectionData = this.sectionData;
int opaque = 0;
int neighborAcquireMsk = 0;
for (int i = 0; i < 32*32*32;) {
long block = sectionData[i + 32*32*32];//Get the block mapping
long block = sectionData[i + 32 * 32 * 32];//Get the block mapping
int modelId = this.modelMan.getModelId(Mapper.getBlockId(block));
long modelMetadata = this.modelMan.getModelMetadataFromClientId(modelId);
sectionData[i*2] = modelId|((long) (Mapper.getLightId(block)) <<16)|(((long) (Mapper.getBiomeId(block)))<<24);
sectionData[i*2+1] = modelMetadata;
sectionData[i * 2] = modelId | ((long) (Mapper.getLightId(block)) << 16) | (((long) (Mapper.getBiomeId(block))) << 24);
sectionData[i * 2 + 1] = modelMetadata;
boolean isFullyOpaque = ModelQueries.isFullyOpaque(modelMetadata);
opaque |= (isFullyOpaque ? 1 : 0) << (i&31);
opaque |= (isFullyOpaque ? 1 : 0) << (i & 31);
//TODO: here also do bitmask of what neighboring sections are needed to compute (may be getting rid of this in future)
//Do increment here
i++;
if ((i&31)==0) {
this.opaqueMasks[(i>>5)-1] = opaque;
if ((i & 31) == 0) {
this.opaqueMasks[(i >> 5) - 1] = opaque;
opaque = 0;
}
}
/*
for (int i = 0; i < 32*32*32; i++) {
long block = sectionData[i + 32*32*32];//Get the block mapping
int modelId = this.modelMan.getModelId(Mapper.getBlockId(block));
long modelMetadata = this.modelMan.getModelMetadataFromClientId(modelId);
sectionData[i*2] = modelId|((long) (Mapper.getLightId(block)) <<16)|(((long) (Mapper.getBiomeId(block)))<<24);
//sectionData[i*2+1] = modelMetadata;
boolean isFullyOpaque = ModelQueries.isFullyOpaque(modelMetadata);
int msk = 1 << (i&31);
opaque &= ~msk; opaque |= isFullyOpaque?msk:0;
//TODO: here also do bitmask of what neighboring sections are needed to compute (may be getting rid of this in future)
if ((i&31)==31) this.opaqueMasks[i>>5] = opaque;
}
*/
}
@@ -384,7 +367,8 @@ public class RenderDataFactory4 {
this.world.acquire(section.lvl, section.x, section.y+1, section.z).release();
this.world.acquire(section.lvl, section.x, section.y-1, section.z).release();
this.world.acquire(section.lvl, section.x, section.y, section.z+1).release();
this.world.acquire(section.lvl, section.x, section.y, section.z-1).release();*/
this.world.acquire(section.lvl, section.x, section.y, section.z-1).release();
*/
//Prepare everything
this.prepareSectionData();
@@ -471,9 +455,9 @@ public class RenderDataFactory4 {
*/
}
public void free() {
this.directionalQuadBuffer.free();
}

5
src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java
View File

@@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import me.cortex.voxy.client.core.model.IdNotYetComputedException;
import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.common.world.other.Mapper;
@@ -46,9 +47,9 @@ public class RenderGenerationService {
this.threads = serviceThreadPool.createService("Section mesh generation service", 100, ()->{
//Thread local instance of the factory
var factory = new RenderDataFactory4(this.world, this.modelBakery.factory, this.emitMeshlets);
return () -> {
return new Pair<>(() -> {
this.processJob(factory);
};
}, factory::free);
});
}

20
src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/HierarchicalOcclusionTraverser.java
View File

@@ -6,12 +6,10 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.gl.shader.Shader;
import me.cortex.voxy.client.core.gl.shader.ShaderType;
import me.cortex.voxy.client.core.rendering.PrintfDebugUtil;
import me.cortex.voxy.client.core.rendering.hierarchical.NodeManager;
import me.cortex.voxy.client.core.rendering.util.HiZBuffer;
import me.cortex.voxy.client.core.rendering.Viewport;
import me.cortex.voxy.client.core.rendering.util.DownloadStream;
import me.cortex.voxy.client.core.rendering.util.UploadStream;
import me.cortex.voxy.common.Logger;
import net.minecraft.util.math.MathHelper;
import org.joml.Matrix4f;
import org.joml.Vector3f;
@@ -29,9 +27,12 @@ import static org.lwjgl.opengl.GL45.*;
// TODO: swap to persistent gpu threads instead of dispatching MAX_ITERATIONS of compute layers
public class HierarchicalOcclusionTraverser {
public static final boolean HIERARCHICAL_SHADER_DEBUG = System.getProperty("voxy.hierarchicalShaderDebug", "false").equals("true");
public static final int REQUEST_QUEUE_SIZE = 50;
private final NodeManager2 nodeManager;
private final NodeCleaner nodeCleaner;
private final GlBuffer requestBuffer;
@@ -60,7 +61,7 @@ public class HierarchicalOcclusionTraverser {
private final int hizSampler = glGenSamplers();
private final Shader traversal = Shader.make(PRINTF_processor)
.defineIf("DEBUG", Voxy.SHADER_DEBUG)
.defineIf("DEBUG", HIERARCHICAL_SHADER_DEBUG)
.define("MAX_ITERATIONS", MAX_ITERATIONS)
.define("LOCAL_SIZE_BITS", LOCAL_WORK_SIZE_BITS)
.define("REQUEST_QUEUE_SIZE", REQUEST_QUEUE_SIZE)
@@ -81,7 +82,8 @@ public class HierarchicalOcclusionTraverser {
.compile();
public HierarchicalOcclusionTraverser(NodeManager2 nodeManager) {
public HierarchicalOcclusionTraverser(NodeManager2 nodeManager, NodeCleaner nodeCleaner) {
this.nodeCleaner = nodeCleaner;
this.nodeManager = nodeManager;
this.requestBuffer = new GlBuffer(REQUEST_QUEUE_SIZE*8L+8).zero();
this.nodeBuffer = new GlBuffer(nodeManager.maxNodeCount*16L).zero();
@@ -117,6 +119,16 @@ public class HierarchicalOcclusionTraverser {
MemoryUtil.memPutInt(ptr, (int) (this.renderList.size()/4-1)); ptr += 4;
/*
//Very funny and cool thing that is possible
if (MinecraftClient.getInstance().getCurrentFps() < 30) {
VoxyConfig.CONFIG.subDivisionSize = Math.min(VoxyConfig.CONFIG.subDivisionSize + 5, 256);
}
if (60 < MinecraftClient.getInstance().getCurrentFps()) {
VoxyConfig.CONFIG.subDivisionSize = Math.max(VoxyConfig.CONFIG.subDivisionSize - 1, 32);
}*/
final float screenspaceAreaDecreasingSize = VoxyConfig.CONFIG.subDivisionSize*VoxyConfig.CONFIG.subDivisionSize;
//Screen space size for descending
MemoryUtil.memPutFloat(ptr, (float) (screenspaceAreaDecreasingSize) /(viewport.width*viewport.height)); ptr += 4;

45
src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java
View File

@@ -1,16 +1,45 @@
package me.cortex.voxy.client.core.rendering.hierachical2;
//Composed of 2 (3) parts
// a node cleaner
// and a geometr/section cleaner
import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.gl.shader.Shader;
import me.cortex.voxy.client.core.gl.shader.ShaderType;
//Uses compute shaders to compute the last 256 rendered section (64x64 workgroup size maybe)
// done via warp level sort, then workgroup sort (shared memory), (/w sorting network)
// then use bubble sort (/w fast path going to middle or 2 subdivisions deep) the bubble it up
// can do incremental sorting pass aswell, so only scan and sort a rolling sector of sections
// (over a few frames to not cause lag, maybe)
public class NodeCleaner {
//TODO: use batch_visibility_set to clear visibility data when nodes are removed!! (TODO: nodeManager will need to forward info to this)
private static final int OUTPUT_COUNT = 64;
private final Shader sorter = Shader.make()
.define("OUTPUT_SIZE", OUTPUT_COUNT)
.define("VISIBILITY_BUFFER_BINDING", 1)
.define("OUTPUT_BUFFER_BINDING", 2)
.add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/sort_visibility.comp")
.compile();
private final GlBuffer visibilityBuffer;
private final GlBuffer outputBuffer = new GlBuffer(OUTPUT_COUNT*4);
private final NodeManager2 nodeManager;
int visibilityId = 0;
//Clears memory via invoking node manager, clear section or delete node
// these should take hint parameters on if when removing section data, to store it in section cache or leave
public NodeCleaner(NodeManager2 nodeManager) {
this.nodeManager = nodeManager;
this.visibilityBuffer = new GlBuffer(nodeManager.maxNodeCount*4L);
}
public void setNodeMemoryUsage() {
//Needs to bubble up information to all parents
// also needs to update a tick/last seen time for node removal
public void tick() {
}
public void free() {
this.sorter.free();
this.visibilityBuffer.free();
this.outputBuffer.free();
}
}

65
src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java
View File

@@ -158,6 +158,15 @@ public class NodeManager2 {
throw new IllegalStateException();
}
} else if ((nodeId&NODE_TYPE_MSK)==NODE_TYPE_INNER || (nodeId&NODE_TYPE_MSK)==NODE_TYPE_LEAF) {
/*
//More verification
if (sectionResult.childExistence != this.nodeData.getNodeChildExistence(nodeId)) {
Logger.error("Child existance verification mismatch. expected: " + this.nodeData.getNodeChildExistence(nodeId) + " got: " + sectionResult.childExistence);
if (this.nodeData.isNodeRequestInFlight(nodeId)) {
Logger.error("AAAAAAAAAA");
}
}*/
// Just doing a geometry update
if (this.updateNodeGeometry(nodeId&NODE_ID_MSK, sectionResult) != 0) {
this.invalidateNode(nodeId&NODE_ID_MSK);
@@ -235,10 +244,62 @@ public class NodeManager2 {
}
} else if ((nodeId&NODE_TYPE_MSK)==NODE_TYPE_INNER) {
//Very complex and painful operation
Logger.error("UNFINISHED OPERATION TODO: FIXME");
} else if ((nodeId&NODE_TYPE_MSK)==NODE_TYPE_LEAF) {
//We might be leaf but we still might be inflight
if (this.nodeData.isNodeRequestInFlight(nodeId&NODE_ID_MSK)) {
// Logger.error("UNFINISHED OPERATION TODO: FIXME: painful operation, needs to account for both adding and removing, need to do the same with inner node, but also create requests, or cleanup children");
int requestId = this.nodeData.getNodeRequest(nodeId);
var request = this.childRequests.get(requestId);// TODO: do not assume request is childRequest (it will probably always be)
if (request.getPosition() != pos) throw new IllegalStateException("Request not in pos");
{//Update the request
byte oldMsk = request.getMsk();
byte change = (byte) (oldMsk ^ childExistence);
{//Remove children and free/release associated meshes
byte rem = (byte) (change&oldMsk);
for (int i = 0; i < 8; i++) {
if ((rem&(1<<i))==0) continue;
int meshId = request.removeAndUnRequire(i);
if (meshId != NULL_GEOMETRY_ID && meshId != EMPTY_GEOMETRY_ID) {
this.geometryManager.removeSection(meshId);
}
//Remove child from being watched and activeSections
long cPos = makeChildPos(pos, i);
if (this.activeSectionMap.remove(cPos) == -1) {//TODO: verify the removed section is a request type of child and the request id matches this
throw new IllegalStateException("Child pos was in a request but not in active section map");
}
if (!this.updateRouter.unwatch(cPos, WorldEngine.UPDATE_FLAGS)) {
throw new IllegalStateException("Child pos was not being watched");
}
}
}
{//Add new children to the request
byte rem = (byte) (change&childExistence);
for (int i = 0; i < 8; i++) {
if ((rem&(1<<i))==0) continue;
//Add child to request
request.addChildRequirement(i);
//Add child to active tracker and put in updateRouter
long cPos = makeChildPos(pos, i);
if (this.activeSectionMap.put(cPos, requestId|NODE_TYPE_REQUEST|REQUEST_TYPE_CHILD) != -1) {
throw new IllegalStateException("Child pos was already in active section tracker but was part of a request");
}
if (!this.updateRouter.watch(cPos, WorldEngine.UPDATE_FLAGS)) {
throw new IllegalStateException("Child pos update router issue");
}
}
}
}
}
//Just need to update the child node data, nothing else
this.nodeData.setNodeChildExistence(nodeId&NODE_ID_MSK, childExistence);
//Need to resubmit to gpu
this.invalidateNode(nodeId&NODE_ID_MSK);
}
}
@@ -347,7 +408,7 @@ public class NodeManager2 {
} else {//nodeType == NODE_TYPE_INNER
//TODO: assert that the node isnt already being watched for geometry, if it is, just spit out a warning? and ignore
Logger.error("TODO FINISH THIS");
if (!this.updateRouter.watch(pos, WorldEngine.UPDATE_TYPE_BLOCK_BIT)) {
//FIXME: i think this can occur accidently? when removing nodes or something creating leaf nodes
// or other, the node might be wanted to be watched by gpu, but cpu already started watching it a few frames ago

2
src/main/java/me/cortex/voxy/client/core/rendering/section/BasicSectionGeometryManager.java
View File

@@ -54,7 +54,7 @@ public class BasicSectionGeometryManager extends AbstractSectionGeometryManager
throw new IllegalStateException("Tried adding section when section count is already at capacity");
}
if (newId > this.sectionMetadata.size()) {
throw new IllegalStateException();
throw new IllegalStateException("Size exceeds limits: " + newId + ", " + this.sectionMetadata.size() + ", " + this.allocationSet.getCount());
}
var newMeta = createMeta(sectionData);

7
src/main/java/me/cortex/voxy/client/core/rendering/section/MDICSectionRenderer.java
View File

@@ -125,7 +125,12 @@ public class MDICSectionRenderer extends AbstractSectionRenderer<MDICViewport, B
this.uploadUniformBuffer(viewport);
//TODO compute the draw calls
//TODO Move this to after culling has occured instead of here, since here the geometry might have changed and
// can cause explosions, while if do after culling, its after geometry changes
// well the thing is here it technicnally should be before geometry changes anyway tho??
// so here should actually be fine aswell???
// but doing it before enables computing temporal draw commands aswell to fix temporal coherance
// yea thats true, should move it probably
{
this.commandGenShader.bind();
glBindBufferBase(GL_UNIFORM_BUFFER, 0, this.uniform.id);

5
src/main/java/me/cortex/voxy/client/core/rendering/util/BufferArena.java
View File

@@ -5,11 +5,14 @@ import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.util.AllocationArena;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.commonImpl.VoxyCommon;
import org.lwjgl.system.MemoryUtil;
import java.util.function.Consumer;
public class BufferArena {
private static final boolean CHECK_SSBO_MAX_SIZE_CHECK = VoxyCommon.isVerificationFlagOn("checkSSBOMaxSize");
private final long size;
private final int elementSize;
private final GlBuffer buffer;
@@ -20,7 +23,7 @@ public class BufferArena {
if (capacity%elementSize != 0) {
throw new IllegalArgumentException("Capacity not a multiple of element size");
}
if (capacity > Capabilities.INSTANCE.ssboMaxSize) {
if (CHECK_SSBO_MAX_SIZE_CHECK && capacity > Capabilities.INSTANCE.ssboMaxSize) {
throw new IllegalArgumentException("Buffer is bigger than max ssbo size (requested " + capacity + " but has max of " + Capabilities.INSTANCE.ssboMaxSize+")");
}
this.size = capacity;

13
src/main/java/me/cortex/voxy/client/importers/WorldImporter.java
View File

@@ -2,6 +2,7 @@ package me.cortex.voxy.client.importers;
import com.mojang.serialization.Codec;
import me.cortex.voxy.client.core.util.ByteBufferBackedInputStream;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.voxelization.VoxelizedSection;
import me.cortex.voxy.common.voxelization.WorldConversionFactory;
import me.cortex.voxy.common.world.WorldEngine;
@@ -54,7 +55,7 @@ public class WorldImporter {
private volatile boolean isRunning;
public WorldImporter(WorldEngine worldEngine, World mcWorld, ServiceThreadPool servicePool) {
this.world = worldEngine;
this.threadPool = servicePool.createService("World importer", 1, ()-> ()->jobQueue.poll().run(), ()->this.world.savingService.getTaskCount() < 4000);
this.threadPool = servicePool.createServiceNoCleanup("World importer", 1, ()->()->jobQueue.poll().run(), ()->this.world.savingService.getTaskCount() < 4000);
var biomeRegistry = mcWorld.getRegistryManager().getOrThrow(RegistryKeys.BIOME);
var defaultBiome = biomeRegistry.getOrThrow(BiomeKeys.PLAINS);
@@ -227,7 +228,7 @@ public class WorldImporter {
System.err.println("Error decompressing chunk data");
} else {
var nbt = NbtIo.readCompound(decompressedData);
this.importChunkNBT(nbt);
this.importChunkNBT(nbt, x, z);
}
}
} catch (Exception e) {
@@ -261,20 +262,26 @@ public class WorldImporter {
}
}
private void importChunkNBT(NbtCompound chunk) {
private void importChunkNBT(NbtCompound chunk, int regionX, int regionZ) {
if (!chunk.contains("Status")) {
//Its not real so decrement the chunk
this.totalChunks.decrementAndGet();
return;
}
//Dont process non full chunk sections
if (ChunkStatus.byId(chunk.getString("Status")) != ChunkStatus.FULL) {
this.totalChunks.decrementAndGet();
return;
}
try {
int x = chunk.getInt("xPos");
int z = chunk.getInt("zPos");
if (x>>5 != regionX || z>>5 != regionZ) {
Logger.error("Chunk position is not located in correct region, expected: (" + regionX + ", " + regionZ+"), got: " + "(" + (x>>5) + ", " + (z>>5)+"), importing anyway");
}
for (var sectionE : chunk.getList("sections", NbtElement.COMPOUND_TYPE)) {
var section = (NbtCompound) sectionE;
int y = section.getInt("Y");

49
src/main/java/me/cortex/voxy/client/taskbar/Taskbar.java Normal file
View File

@@ -0,0 +1,49 @@
package me.cortex.voxy.client.taskbar;
import me.cortex.voxy.common.Logger;
import net.minecraft.client.MinecraftClient;
import org.apache.commons.lang3.SystemUtils;
public abstract class Taskbar {
public interface ITaskbar {
void setProgress(long count, long outOf);
void setIsNone();
void setIsProgression();
void setIsPaused();
void setIsError();
}
public static class NoopTaskbar implements ITaskbar {
private NoopTaskbar() {}
@Override
public void setIsNone() {}
@Override
public void setProgress(long count, long outOf) {}
@Override
public void setIsPaused() {}
@Override
public void setIsProgression() {}
@Override
public void setIsError() {}
}
public static final ITaskbar INSTANCE = createInterface();
private static ITaskbar createInterface() {
if (SystemUtils.IS_OS_WINDOWS) {
try {
return new WindowsTaskbar(MinecraftClient.getInstance().getWindow().getHandle());
} catch (Exception e) {
Logger.error("Unable to create windows taskbar interface", e);
return new NoopTaskbar();
}
} else {
return new NoopTaskbar();
}
}
}

68
src/main/java/me/cortex/voxy/client/taskbar/WindowsTaskbar.java Normal file
View File

@@ -0,0 +1,68 @@
package me.cortex.voxy.client.taskbar;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.*;
import com.sun.jna.platform.win32.COM.COMInvoker;
import com.sun.jna.ptr.PointerByReference;
import me.cortex.voxy.common.util.TrackedObject;
import org.lwjgl.glfw.GLFWNativeWin32;
public class WindowsTaskbar extends COMInvoker implements Taskbar.ITaskbar {
private final WinDef.HWND hwnd;
WindowsTaskbar(long windowId) {
var itaskbar3res = new PointerByReference();
if (W32Errors.FAILED(Ole32.INSTANCE.CoCreateInstance(new Guid.GUID("56FDF344-FD6D-11d0-958A-006097C9A090"),
null,
WTypes.CLSCTX_SERVER,
new Guid.GUID("EA1AFB91-9E28-4B86-90E9-9E9F8A5EEFAF"),
itaskbar3res))) {
throw new IllegalStateException("Failed to create ITaskbar3");
}
this.setPointer(itaskbar3res.getValue());
this.hwnd = new WinDef.HWND(new Pointer(GLFWNativeWin32.glfwGetWin32Window(windowId)));
this.invokeNative(3); // HrInit
}
private void invokeNative(int ventry, Object... objects) {
Object[] args = new Object[objects.length+1];
args[0] = this.getPointer();
System.arraycopy(objects, 0, args, 1, objects.length);
if (W32Errors.FAILED((WinNT.HRESULT) this._invokeNativeObject(ventry, args, WinNT.HRESULT.class))) {
throw new IllegalStateException("Failed to invoke vtable: " + ventry);
}
}
public void close() {
this.invokeNative(10, this.hwnd, 0); // SetProgressState TBPF_NOPROGRESS (0x00000000)
this.invokeNative(2); // Release
this.setPointer(null);
}
@Override
public void setIsNone() {
this.invokeNative(10, this.hwnd, 0); // SetProgressState TBPF_NOPROGRESS (0x00000000)
}
@Override
public void setProgress(long count, long outOf) {
this.invokeNative(9, this.hwnd, count, outOf); // SetProgressValue
}
@Override
public void setIsPaused() {
this.invokeNative(10, this.hwnd, 8); // SetProgressState TBPF_PAUSED (0x00000008)
}
@Override
public void setIsProgression() {
this.invokeNative(10, this.hwnd, 2); // SetProgressState TBPF_NORMAL (0x00000002)
}
@Override
public void setIsError() {
this.invokeNative(10, this.hwnd, 2); // SetProgressState TBPF_ERROR (0x00000004)
}
}

8
src/main/java/me/cortex/voxy/common/thread/QueuedServiceSlice.java
View File

@@ -1,5 +1,6 @@
package me.cortex.voxy.common.thread;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.util.TrackedObject;
import me.cortex.voxy.common.world.WorldSection;
import net.minecraft.client.MinecraftClient;
@@ -12,15 +13,16 @@ import java.util.function.BooleanSupplier;
import java.util.function.Consumer;
import java.util.function.Supplier;
public class QueuedServiceSlice <T> extends ServiceSlice {
public class QueuedServiceSlice<T> extends ServiceSlice {
private final ConcurrentLinkedDeque<T> queue = new ConcurrentLinkedDeque<>();
QueuedServiceSlice(ServiceThreadPool threadPool, Supplier<Consumer<T>> workerGenerator, String name, int weightPerJob, BooleanSupplier condition) {
QueuedServiceSlice(ServiceThreadPool threadPool, Supplier<Pair<Consumer<T>, Runnable>> workerGenerator, String name, int weightPerJob, BooleanSupplier condition) {
super(threadPool, null, name, weightPerJob, condition);
//Fuck off java with the this bullshit before super constructor, fucking bullshit
super.setWorkerGenerator(() -> {
var work = workerGenerator.get();
return () -> work.accept(this.queue.pop());
var consumer = work.left();
return new Pair<>(() -> consumer.accept(this.queue.pop()), work.right());
});
}

25
src/main/java/me/cortex/voxy/common/thread/ServiceSlice.java
View File

@@ -1,9 +1,11 @@
package me.cortex.voxy.common.thread;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.util.TrackedObject;
import net.minecraft.client.MinecraftClient;
import net.minecraft.text.Text;
import java.util.Arrays;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BooleanSupplier;
@@ -14,23 +16,25 @@ public class ServiceSlice extends TrackedObject {
final int weightPerJob;
volatile boolean alive = true;
private final ServiceThreadPool threadPool;
private Supplier<Runnable> workerGenerator;
private Supplier<Pair<Runnable, Runnable>> workerGenerator;
final Semaphore jobCount = new Semaphore(0);
private final Runnable[] runningCtxs;
private final Runnable[] cleanupCtxs;
private final AtomicInteger activeCount = new AtomicInteger();
private final AtomicInteger jobCount2 = new AtomicInteger();
private final BooleanSupplier condition;
ServiceSlice(ServiceThreadPool threadPool, Supplier<Runnable> workerGenerator, String name, int weightPerJob, BooleanSupplier condition) {
ServiceSlice(ServiceThreadPool threadPool, Supplier<Pair<Runnable, Runnable>> workerGenerator, String name, int weightPerJob, BooleanSupplier condition) {
this.threadPool = threadPool;
this.condition = condition;
this.runningCtxs = new Runnable[threadPool.getThreadCount()];
this.cleanupCtxs = new Runnable[threadPool.getThreadCount()];
this.name = name;
this.weightPerJob = weightPerJob;
this.setWorkerGenerator(workerGenerator);
}
protected void setWorkerGenerator(Supplier<Runnable> workerGenerator) {
protected void setWorkerGenerator(Supplier<Pair<Runnable, Runnable>> workerGenerator) {
this.workerGenerator = workerGenerator;
}
@@ -62,7 +66,9 @@ public class ServiceSlice extends TrackedObject {
//If the running context is null, create and set it
var ctx = this.runningCtxs[threadIndex];
if (ctx == null) {
ctx = this.workerGenerator.get();
var pair = this.workerGenerator.get();
ctx = pair.left();
this.cleanupCtxs[threadIndex] = pair.right();//Set cleanup
this.runningCtxs[threadIndex] = ctx;
}
@@ -106,9 +112,20 @@ public class ServiceSlice extends TrackedObject {
//Tell parent to remove
this.threadPool.removeService(this);
this.runCleanup();
super.free0();
}
private void runCleanup() {
for (var runnable : this.cleanupCtxs) {
if (runnable != null) {
runnable.run();
}
}
Arrays.fill(this.cleanupCtxs, null);
}
@Override
public void free() {
this.shutdown();

13
src/main/java/me/cortex/voxy/common/thread/ServiceThreadPool.java
View File

@@ -1,6 +1,7 @@
package me.cortex.voxy.common.thread;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.Pair;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -35,11 +36,19 @@ public class ServiceThreadPool {
}
}
public synchronized ServiceSlice createService(String name, int weight, Supplier<Runnable> workGenerator) {
public ServiceSlice createServiceNoCleanup(String name, int weight, Supplier<Runnable> workGenerator) {
return this.createService(name, weight, ()->new Pair<>(workGenerator.get(), null));
}
public ServiceSlice createServiceNoCleanup(String name, int weight, Supplier<Runnable> workGenerator, BooleanSupplier executionCondition) {
return this.createService(name, weight, ()->new Pair<>(workGenerator.get(), null), executionCondition);
}
public synchronized ServiceSlice createService(String name, int weight, Supplier<Pair<Runnable, Runnable>> workGenerator) {
return this.createService(name, weight, workGenerator, ()->true);
}
public synchronized ServiceSlice createService(String name, int weight, Supplier<Runnable> workGenerator, BooleanSupplier executionCondition) {
public synchronized ServiceSlice createService(String name, int weight, Supplier<Pair<Runnable, Runnable>> workGenerator, BooleanSupplier executionCondition) {
var service = new ServiceSlice(this, workGenerator, name, weight, executionCondition);
this.insertService(service);
return service;

30
src/main/java/me/cortex/voxy/common/util/MemoryBuffer.java
View File

@@ -2,11 +2,18 @@ package me.cortex.voxy.common.util;
import org.lwjgl.system.MemoryUtil;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class MemoryBuffer extends TrackedObject {
public final long address;
public final long size;
private final boolean freeable;
private static final AtomicInteger COUNT = new AtomicInteger(0);
private static final AtomicLong TOTAL_SIZE = new AtomicLong(0);
public MemoryBuffer(long size) {
this(true, MemoryUtil.nmemAlloc(size), size, true);
}
@@ -16,6 +23,11 @@ public class MemoryBuffer extends TrackedObject {
this.size = size;
this.address = address;
this.freeable = freeable;
COUNT.incrementAndGet();
if (freeable) {
TOTAL_SIZE.addAndGet(size);
}
}
public void cpyTo(long dst) {
@@ -26,8 +38,11 @@ public class MemoryBuffer extends TrackedObject {
@Override
public void free() {
super.free0();
COUNT.decrementAndGet();
if (this.freeable) {
MemoryUtil.nmemFree(this.address);
TOTAL_SIZE.addAndGet(-this.size);
} else {
throw new IllegalArgumentException("Tried to free unfreeable buffer");
}
@@ -44,8 +59,13 @@ public class MemoryBuffer extends TrackedObject {
if (size > this.size) {
throw new IllegalArgumentException("Requested size larger than current size");
}
//Free the current object, but not the memory associated with it
super.free0();
this.free0();
COUNT.decrementAndGet();
if (this.freeable) {
TOTAL_SIZE.addAndGet(-this.size);
}
return new MemoryBuffer(true, this.address, size, this.freeable);
}
@@ -61,4 +81,12 @@ public class MemoryBuffer extends TrackedObject {
public static MemoryBuffer createUntrackedUnfreeableRawFrom(long address, long size) {
return new MemoryBuffer(false, address, size, false);
}
public static int getCount() {
return COUNT.get();
}
public static long getTotalSize() {
return TOTAL_SIZE.get();
}
}

6
src/main/java/me/cortex/voxy/common/util/Pair.java Normal file
View File

@@ -0,0 +1,6 @@
package me.cortex.voxy.common.util;
public record Pair<A, B>(A left, B right) {
}

2
src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java
View File

@@ -20,7 +20,7 @@ public class SectionSavingService {
public SectionSavingService(WorldEngine worldEngine, ServiceThreadPool threadPool) {
this.world = worldEngine;
this.threads = threadPool.createService("Section saving service", 100, () -> this::processJob);
this.threads = threadPool.createServiceNoCleanup("Section saving service", 100, () -> this::processJob);
}
private void processJob() {

2
src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java
View File

@@ -24,7 +24,7 @@ public class VoxelIngestService {
private final WorldEngine world;
public VoxelIngestService(WorldEngine world, ServiceThreadPool pool) {
this.world = world;
this.threads = pool.createService("Ingest service", 100, ()-> this::processJob);
this.threads = pool.createServiceNoCleanup("Ingest service", 100, ()-> this::processJob);
}
private void processJob() {

21
src/main/resources/assets/voxy/shaders/lod/hierarchical/cleaner/batch_visibility_set.comp Normal file
View File

@@ -0,0 +1,21 @@
#version 460 core
//
layout(local_size_x=128) in;
layout(binding = VISIBILITY_BUFFER_BINDING, std430) restrict writeonly buffer VisibilityDataBuffer {
uint[] visiblity;
};
layout(binding = LIST_BUFFER_BINDING, std430) restrict readonly buffer SetListBuffer {
uint[] ids;
};
layout(location=0) uniform uint count;
#define SET_TO uint(-1)
void main() {
uint id = gl_InvocationID;//It might be this or gl_GlobalInvocationID.x
if (count <= id) {
return;
}
visiblity[ids[id]] = SET_TO;
}

51
src/main/resources/assets/voxy/shaders/lod/hierarchical/cleaner/sort_visibility.comp Normal file
View File

@@ -0,0 +1,51 @@
#version 460 core
//Uses intrinsics and other operations to perform very fast sorting
// we dont need a propper sort, only a fuzzy sort, as in we only need to top 128 entries, but those can be unsorted
//#define OUTPUT_SIZE 128
layout(local_size_x=32, local_size_y=8) in;
//256 workgroup
layout(binding = VISIBILITY_BUFFER_BINDING, std430) restrict readonly buffer VisibilityDataBuffer {
uint[] visiblity;
};
layout(binding = OUTPUT_BUFFER_BINDING, std430) restrict volatile buffer MinimumVisibilityBuffer {//TODO: might need to be volatile
uint minVisIds[OUTPUT_SIZE];
};
uint atomicDerefMin(uint atId, uint id, uint value) {
uint existingId = minVisIds[atId];
while (true) {
//Check if the value is less than the dereferenced value, if its not, return our own id
if (visiblity[existingId] <= value) {
return id;
}
//Attempt to swap, since we know we are less than the existingId
atomicCompSwap(minVisIds[atId], existingId, id);
//Check if we did swap, else if we failed (or got reswapped else where) recheck
existingId = minVisIds[atId];
if (existingId == id) {
return existingId;
}
}
}
//TODO: optimize
void bubbleSort(uint start, uint id, uint value) {
for (uint i = start; i < OUTPUT_SIZE; i++) {
uint nextId = atomicDerefMin(i, id, value);
if (nextId == id) {
return;//Not inserted, so return
}
//Else we need to bubble the value up
id = nextId;
value = visiblity[id];
}
}
void main() {
//First do a min sort/set of min OUTPUT_SIZE values of the set
}