From fdeed5c257fbe567ed8f62ca9f7584581cb49599 Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Sat, 14 Dec 2024 13:42:14 +1000 Subject: [PATCH] Wip, shader tweeks, manager tweeks, wip on cleaner --- .../core/gl/shader/AutoBindingShader.java | 61 ++++++++++++++++ .../voxy/client/core/gl/shader/Shader.java | 71 ++++++++++++------- .../voxy/client/core/model/ModelFactory.java | 4 +- .../rendering/hierachical2/NodeCleaner.java | 47 +++++++++++- .../rendering/hierachical2/NodeManager2.java | 19 ++++- 5 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 src/main/java/me/cortex/voxy/client/core/gl/shader/AutoBindingShader.java diff --git a/src/main/java/me/cortex/voxy/client/core/gl/shader/AutoBindingShader.java b/src/main/java/me/cortex/voxy/client/core/gl/shader/AutoBindingShader.java new file mode 100644 index 00000000..058117b8 --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/core/gl/shader/AutoBindingShader.java @@ -0,0 +1,61 @@ +package me.cortex.voxy.client.core.gl.shader; + +import me.cortex.voxy.client.core.gl.GlBuffer; +import me.cortex.voxy.common.util.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.lwjgl.opengl.GL30.glBindBufferBase; +import static org.lwjgl.opengl.GL30.glBindBufferRange; +import static org.lwjgl.opengl.GL31.GL_UNIFORM_BUFFER; +import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BUFFER; + + +//TODO: rewrite the entire shader builder system +public class AutoBindingShader extends Shader { + private record BufferBinding(int target, int index, GlBuffer buffer, long offset, long size) {} + + private final Map defines; + private final List bindings = new ArrayList<>(); + + AutoBindingShader(Shader.Builder builder, int program) { + super(program); + this.defines = builder.defines; + } + + public AutoBindingShader ssbo(int index, GlBuffer binding) { + return this.ssbo(index, binding, 0); + } + + public AutoBindingShader ssbo(String define, GlBuffer binding) { + return this.ssbo(Integer.parseInt(this.defines.get(define)), binding, 0); + } + + public AutoBindingShader ssbo(int index, GlBuffer buffer, long offset) { + this.bindings.add(new BufferBinding(GL_SHADER_STORAGE_BUFFER, index, buffer, offset, -1)); + return this; + } + + public AutoBindingShader ubo(int index, GlBuffer buffer) { + return this.ubo(index, buffer, 0); + } + + public AutoBindingShader ubo(int index, GlBuffer buffer, long offset) { + this.bindings.add(new BufferBinding(GL_UNIFORM_BUFFER, index, buffer, offset, -1)); + return this; + } + + @Override + public void bind() { + super.bind(); + for (var binding : this.bindings) { + if (binding.offset == 0 && binding.size == -1) { + glBindBufferBase(binding.target, binding.index, binding.buffer.id); + } else { + glBindBufferRange(binding.target, binding.index, binding.buffer.id, binding.offset, binding.size); + } + } + } +} diff --git a/src/main/java/me/cortex/voxy/client/core/gl/shader/Shader.java b/src/main/java/me/cortex/voxy/client/core/gl/shader/Shader.java index 64da2c33..f2eac42e 100644 --- a/src/main/java/me/cortex/voxy/client/core/gl/shader/Shader.java +++ b/src/main/java/me/cortex/voxy/client/core/gl/shader/Shader.java @@ -11,25 +11,10 @@ import static org.lwjgl.opengl.GL20.glUseProgram; public class Shader extends TrackedObject { private final int id; - private Shader(int program) { + Shader(int program) { id = program; } - public static Builder make(IShaderProcessor... processors) { - List aa = new ArrayList<>(List.of(processors)); - Collections.reverse(aa); - IShaderProcessor applicator = (type,source)->source; - for (IShaderProcessor processor : processors) { - IShaderProcessor finalApplicator = applicator; - applicator = (type, source) -> finalApplicator.process(type, processor.process(type, source)); - } - return new Builder(applicator); - } - - public static Builder make() { - return new Builder((aa,source)->source); - } - public int id() { return this.id; } @@ -43,43 +28,73 @@ public class Shader extends TrackedObject { glDeleteProgram(this.id); } - public static class Builder { - private final Map defines = new HashMap<>(); + + + + public static Builder make(IShaderProcessor... processor) { + return makeInternal((a,b)->new Shader(b), processor); + } + + public static Builder makeAuto(IShaderProcessor... processor) { + return makeInternal(AutoBindingShader::new, processor); + } + + + + static Builder makeInternal(Builder.IShaderObjectConstructor constructor, IShaderProcessor[] processors) { + List aa = new ArrayList<>(List.of(processors)); + Collections.reverse(aa); + IShaderProcessor applicator = (type,source)->source; + for (IShaderProcessor processor : processors) { + IShaderProcessor finalApplicator = applicator; + applicator = (type, source) -> finalApplicator.process(type, processor.process(type, source)); + } + return new Builder<>(constructor, applicator); + } + + public static class Builder { + protected interface IShaderObjectConstructor { + J make(Builder builder, int program); + } + final Map defines = new HashMap<>(); private final Map sources = new HashMap<>(); private final IShaderProcessor processor; - private Builder(IShaderProcessor processor) { + private final IShaderObjectConstructor constructor; + private Builder(IShaderObjectConstructor constructor, IShaderProcessor processor) { + this.constructor = constructor; this.processor = processor; } - public Builder define(String name) { + public Builder define(String name) { this.defines.put(name, ""); return this; } //Useful for inline setting (such as debug) - public Builder defineIf(String name, boolean condition) { + public Builder defineIf(String name, boolean condition) { if (condition) { this.defines.put(name, ""); } return this; } - public Builder define(String name, int value) { + public Builder define(String name, int value) { this.defines.put(name, Integer.toString(value)); return this; } - public Builder add(ShaderType type, String id) { + public Builder add(ShaderType type, String id) { this.addSource(type, ShaderLoader.parse(id)); return this; } - public Builder addSource(ShaderType type, String source) { + public Builder addSource(ShaderType type, String source) { this.sources.put(type, this.processor.process(type, source)); return this; } - public Shader compile() { + + private int compileToProgram() { int program = GL20C.glCreateProgram(); int[] shaders = new int[this.sources.size()]; { @@ -107,9 +122,12 @@ public class Shader extends TrackedObject { } printProgramLinkLog(program); verifyProgramLinked(program); - return new Shader(program); + return program; } + public T compile() { + return this.constructor.make(this, this.compileToProgram()); + } private static void printProgramLinkLog(int program) { String log = GL20C.glGetProgramInfoLog(program); @@ -148,5 +166,4 @@ public class Shader extends TrackedObject { return shader; } } - } diff --git a/src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java b/src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java index 607453fb..4bc079cd 100644 --- a/src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java +++ b/src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java @@ -11,6 +11,7 @@ import me.cortex.voxy.client.core.gl.GlBuffer; import me.cortex.voxy.client.core.gl.GlTexture; import me.cortex.voxy.client.core.rendering.util.RawDownloadStream; import me.cortex.voxy.client.core.rendering.util.UploadStream; +import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.world.other.Mapper; import net.minecraft.block.BlockState; import net.minecraft.block.FluidBlock; @@ -311,7 +312,8 @@ public class ModelFactory { if (allFalse == allTrue) {//If only some sides where self culled then abort cullsSame = false; - if (LOGGED_SELF_CULLING_WARNING.add(blockState)) System.err.println("Warning! blockstate: " + blockState + " only culled against its self some of the time"); + if (LOGGED_SELF_CULLING_WARNING.add(blockState)) + Logger.info("Warning! blockstate: " + blockState + " only culled against its self some of the time"); } if (allTrue) { diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java index 5df5d8fb..28baf8fd 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeCleaner.java @@ -1,8 +1,16 @@ package me.cortex.voxy.client.core.rendering.hierachical2; +import it.unimi.dsi.fastutil.ints.IntArrayFIFOQueue; +import it.unimi.dsi.fastutil.ints.IntArrayList; import me.cortex.voxy.client.core.gl.GlBuffer; +import me.cortex.voxy.client.core.gl.shader.AutoBindingShader; import me.cortex.voxy.client.core.gl.shader.Shader; import me.cortex.voxy.client.core.gl.shader.ShaderType; +import me.cortex.voxy.client.core.rendering.util.UploadStream; +import org.lwjgl.system.MemoryUtil; + +import static org.lwjgl.opengl.GL20.glUniform1i; +import static org.lwjgl.opengl.GL43C.glDispatchCompute; //Uses compute shaders to compute the last 256 rendered section (64x64 workgroup size maybe) // done via warp level sort, then workgroup sort (shared memory), (/w sorting network) @@ -14,6 +22,8 @@ public class NodeCleaner { private static final int OUTPUT_COUNT = 64; + private static final int BATCH_SET_SIZE = 2048; + private final Shader sorter = Shader.make() .define("OUTPUT_SIZE", OUTPUT_COUNT) .define("VISIBILITY_BUFFER_BINDING", 1) @@ -21,8 +31,17 @@ public class NodeCleaner { .add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/sort_visibility.comp") .compile(); + private final AutoBindingShader batchClear = Shader.makeAuto() + .define("VISIBILITY_BUFFER_BINDING", 0) + .define("LIST_BUFFER_BINDING", 1) + .add(ShaderType.COMPUTE, "voxy:lod/hierarchical/cleaner/batch_visibility_set.com") + .compile(); + private final GlBuffer visibilityBuffer; private final GlBuffer outputBuffer = new GlBuffer(OUTPUT_COUNT*4); + private final GlBuffer scratchBuffer = new GlBuffer(BATCH_SET_SIZE*4);//Scratch buffer for setting ids with + + private final IntArrayFIFOQueue idsToClear = new IntArrayFIFOQueue(); private final NodeManager2 nodeManager; int visibilityId = 0; @@ -30,16 +49,42 @@ public class NodeCleaner { public NodeCleaner(NodeManager2 nodeManager) { this.nodeManager = nodeManager; - this.visibilityBuffer = new GlBuffer(nodeManager.maxNodeCount*4L); + this.visibilityBuffer = new GlBuffer(nodeManager.maxNodeCount*4L).zero(); + + this.batchClear + .ssbo("VISIBILITY_BUFFER_BINDING", this.visibilityBuffer) + .ssbo("LIST_BUFFER_BINDING", this.scratchBuffer); + } + + public void clearId(int id) { + this.idsToClear.enqueue(id); } public void tick() { + this.clearIds(); + } + private void clearIds() { + if (!this.idsToClear.isEmpty()) { + this.batchClear.bind(); + + while (!this.idsToClear.isEmpty()) { + int cnt = Math.min(this.idsToClear.size(), BATCH_SET_SIZE); + long ptr = UploadStream.INSTANCE.upload(this.scratchBuffer, 0, cnt * 4L); + for (int i = 0; i < cnt; i++) { + MemoryUtil.memPutInt(ptr + cnt * 4, this.idsToClear.dequeueInt()); + } + UploadStream.INSTANCE.commit(); + glUniform1i(0, cnt); + glDispatchCompute((cnt+127)/128, 1, 1); + } + } } public void free() { this.sorter.free(); this.visibilityBuffer.free(); this.outputBuffer.free(); + this.scratchBuffer.free(); } } diff --git a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java index 1e818170..075f8367 100644 --- a/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java +++ b/src/main/java/me/cortex/voxy/client/core/rendering/hierachical2/NodeManager2.java @@ -326,8 +326,10 @@ public class NodeManager2 { if (parentNodeId == -1 || (parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_REQUEST) { throw new IllegalStateException("CRITICAL BAD STATE!!! finishRequest tried to finish for a node that no longer exists in the map or has become a request type somehow?!!?!!" + WorldEngine.pprintPos(request.getPosition()) + " " + parentNodeId); } + int parentNodeType = parentNodeId&NODE_TYPE_MSK; + parentNodeId &= NODE_ID_MSK; - if ((parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_LEAF) { + if (parentNodeType==NODE_TYPE_LEAF) { int msk = Byte.toUnsignedInt(request.getMsk()); int base = this.nodeData.allocate(Integer.bitCount(msk)); int offset = -1; @@ -363,9 +365,20 @@ public class NodeManager2 { this.nodeData.setNodeRequest(parentNodeId, 0);//TODO: create a better null request this.activeNodeRequestCount--; this.nodeData.unmarkRequestInFlight(parentNodeId); + + //Change it from a leaf to an inner node + { + int pid = this.activeSectionMap.remove(request.getPosition()); + if (pid == -1 || (pid & NODE_TYPE_MSK) != NODE_TYPE_LEAF) { + throw new IllegalStateException("Unexpected node mapping: " + pid); + } + } + //_this is why it hasnt been working, grrr, wasnt doing this_ + this.activeSectionMap.put(request.getPosition(), NODE_TYPE_INNER|parentNodeId);//Set the type from leaf to inner node + this.invalidateNode(parentNodeId); - } else if ((parentNodeId&NODE_TYPE_MSK)==NODE_TYPE_INNER) { - System.err.println("TODO: FIXME FINISH: finishRequest NODE_TYPE_INNER"); + } else if (parentNodeType==NODE_TYPE_INNER) { + Logger.error("TODO: FIXME FINISH: finishRequest NODE_TYPE_INNER"); } else { throw new IllegalStateException(); }