Much changes, wip

src/main/java/me/cortex/voxy/client/core/VoxelCore.java

@@ -8,6 +8,7 @@ import me.cortex.voxy.client.core.model.IdNotYetComputedException;
 import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
 import me.cortex.voxy.client.core.rendering.*;
 import me.cortex.voxy.client.core.rendering.building.RenderDataFactory;
+import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4;
 import me.cortex.voxy.client.core.rendering.post.PostProcessing;
 import me.cortex.voxy.client.core.rendering.util.DownloadStream;
 import me.cortex.voxy.client.core.util.IrisUtil;
@@ -85,8 +86,7 @@ public class VoxelCore {
 
         //this.verifyTopNodeChildren(0,0,0);
 
-        this.testMeshingPerformance();
+        //this.testMeshingPerformance();
-
     }
 
 
@@ -152,8 +152,8 @@ public class VoxelCore {
 
         this.renderer.renderFarAwayOpaque(viewport);
 
-        //Compute the SSAO of the rendered terrain
+        //Compute the SSAO of the rendered terrain, TODO: fix it breaking depth
-        this.postProcessing.computeSSAO(projection, matrices);
+        //this.postProcessing.computeSSAO(projection, matrices);
 
         //We can render the translucent directly after as it is the furthest translucent objects
         this.renderer.renderFarAwayTranslucent(viewport);
@@ -274,7 +274,7 @@ public class VoxelCore {
 
     private void testMeshingPerformance() {
         var modelService = new ModelBakerySubsystem(this.world.getMapper());
-        RenderDataFactory factory = new RenderDataFactory(this.world, modelService.factory, false);
+        var factory = new RenderDataFactory4(this.world, modelService.factory, false);
 
         List<WorldSection> sections = new ArrayList<>();
 
@@ -321,6 +321,7 @@ public class VoxelCore {
                 }
                 long delta = System.currentTimeMillis() - start;
                 System.out.println("Iteration: " + (iteration++) + " took " + delta + "ms, for an average of " + ((float)delta/sections.size()) + "ms per section");
+                //System.out.println("Quad count: " + factory.quadCount);
             }
         }
 

src/main/java/me/cortex/voxy/client/core/model/ModelFactory.java

@@ -330,6 +330,8 @@ public class ModelFactory {
 
         metadata |= cullsSame?32:0;
 
+        boolean fullyOpaque = true;
+
         //TODO: add a bunch of control config options for overriding/setting options of metadata for each face of each type
         for (int face = 5; face != -1; face--) {//In reverse order to make indexing into the metadata long easier
             long faceUploadPtr = uploadPtr + 4L * face;//Each face gets 4 bytes worth of data
@@ -339,6 +341,8 @@ public class ModelFactory {
                 metadata |= 0xFF;//Mark the face as non-existent
                 //Set to -1 as safepoint
                 MemoryUtil.memPutInt(faceUploadPtr, -1);
+
+                fullyOpaque = false;
                 continue;
             }
             var faceSize = TextureUtils.computeBounds(textureData[face], checkMode);
@@ -360,6 +364,7 @@ public class ModelFactory {
                 occludesFace &= ((float)writeCount)/(MODEL_TEXTURE_SIZE * MODEL_TEXTURE_SIZE) > 0.9;// only occlude if the face covers more than 90% of the face
             }
             metadata |= occludesFace?1:0;
+            fullyOpaque &= occludesFace;
 
 
 
@@ -398,6 +403,7 @@ public class ModelFactory {
 
             MemoryUtil.memPutInt(faceUploadPtr, faceModelData);
         }
+        metadata |= fullyOpaque?(1L<<(48+6)):0;
         this.metadataCache[modelId] = metadata;
 
         uploadPtr += 4*6;

src/main/java/me/cortex/voxy/client/core/model/ModelQueries.java

@@ -41,4 +41,8 @@ public abstract class ModelQueries {
     public static boolean cullsSame(long metadata) {
         return ((metadata>>(8*6))&32) != 0;
     }
+
+    public static boolean isFullyOpaque(long metadata) {
+        return ((metadata>>(8*6))&64) != 0;
+    }
 }

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory.java

@@ -18,10 +18,10 @@ public class RenderDataFactory {
     private final WorldEngine world;
     private final ModelFactory modelMan;
 
-    private final Mesher2D negativeMesher = new Mesher2D(5, 15);
+    private final Mesher2D negativeMesher = new Mesher2D();
-    private final Mesher2D positiveMesher = new Mesher2D(5, 15);
+    private final Mesher2D positiveMesher = new Mesher2D();
-    private final Mesher2D negativeFluidMesher = new Mesher2D(5, 15);
+    private final Mesher2D negativeFluidMesher = new Mesher2D();
-    private final Mesher2D positiveFluidMesher = new Mesher2D(5, 15);
+    private final Mesher2D positiveFluidMesher = new Mesher2D();
 
     private final long[] sectionCache = new long[32*32*32];
     private final long[] connectedSectionCache = new long[32*32*32];
@@ -501,6 +501,7 @@ public class RenderDataFactory {
         otherFlags |= ModelQueries.isTranslucent(metadata)?1L<<33:0;
         otherFlags |= ModelQueries.isDoubleSided(metadata)?1L<<34:0;
         mesher.put(a, b, ((long)clientModelId) | (((long) Mapper.getLightId(ModelQueries.faceUsesSelfLighting(metadata, face)?self:facingState))<<16) | ((((long) Mapper.getBiomeId(self))<<24) * (ModelQueries.isBiomeColoured(metadata)?1:0)) | otherFlags);
+        //mesher.put(a, b, ((long)clientModelId) | (((long) 0)<<16) | (0) | otherFlags);
         return true;
     }
 
@@ -510,8 +511,9 @@ public class RenderDataFactory {
         int count = mesher.process();
         var array = mesher.getArray();
         for (int i = 0; i < count; i++) {
-            int quad = array[i];
+            int quad = array[i*3];
-            long data = mesher.getDataFromQuad(quad);
+            long data = Integer.toUnsignedLong(array[i*3+1]);
+            data |= ((long) array[i*3+2])<<32;
             long encodedQuad = Integer.toUnsignedLong(QuadEncoder.encodePosition(face, otherAxis, quad)) | ((data&0xFFFF)<<26) | (((data>>16)&0xFF)<<55) | (((data>>24)&0x1FF)<<46);
 
 

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderDataFactory4.java

@@ -0,0 +1,477 @@
+package me.cortex.voxy.client.core.rendering.building;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import me.cortex.voxy.client.core.Capabilities;
+import me.cortex.voxy.client.core.model.ModelFactory;
+import me.cortex.voxy.client.core.model.ModelQueries;
+import me.cortex.voxy.client.core.util.Mesher2D;
+import me.cortex.voxy.client.core.util.ScanMesher2D;
+import me.cortex.voxy.common.util.MemoryBuffer;
+import me.cortex.voxy.common.world.WorldEngine;
+import me.cortex.voxy.common.world.WorldSection;
+import me.cortex.voxy.common.world.other.Mapper;
+import org.lwjgl.system.MemoryUtil;
+
+import java.util.Arrays;
+
+
+public class RenderDataFactory4 {
+    private final WorldEngine world;
+    private final ModelFactory modelMan;
+
+    //private final long[] sectionData = new long[32*32*32*2];
+    private final long[] sectionData = new long[32*32*32*2];
+
+    private final int[] opaqueMasks = new int[32*32];
+
+
+    //TODO: emit directly to memory buffer instead of long arrays
+    private final LongArrayList[] directionalQuadCollectors = new LongArrayList[]{new LongArrayList(), new LongArrayList(), new LongArrayList(), new LongArrayList(), new LongArrayList(), new LongArrayList()};
+
+
+    private int minX;
+    private int minY;
+    private int minZ;
+    private int maxX;
+    private int maxY;
+    private int maxZ;
+
+    private int quadCount = 0;
+
+    //Wont work for double sided quads
+    private final class Mesher extends ScanMesher2D {
+        public int auxiliaryPosition = 0;
+        public int axis = 0;
+
+        //Note x, z are in top right
+        @Override
+        protected void emitQuad(int x, int z, int length, int width, long data) {
+            RenderDataFactory4.this.quadCount++;
+
+            x -= length-1;
+            z -= width-1;
+
+            //Lower 26 bits can be auxiliary data since that is where quad position information goes;
+            int auxData = (int) (data&((1<<26)-1));
+            data &= ~(data&((1<<26)-1));
+
+            final int axis = this.axis;
+            int face = (auxData&1)|(axis<<1);
+            int encodedPosition = (auxData&1)|(axis<<1);
+
+            //Shift up if is negative axis
+            int auxPos = this.auxiliaryPosition;
+            auxPos += (~auxData)&1;
+
+            encodedPosition |= ((width - 1) << 7) | ((length - 1) << 3);
+
+            encodedPosition |= x << (axis==2?16:21);
+            encodedPosition |= z << (axis==1?16:11);
+            encodedPosition |= auxPos << (axis==0?16:(axis==1?11:21));
+
+            long quad = data | encodedPosition;
+
+
+            RenderDataFactory4.this.directionalQuadCollectors[face].add(quad);
+        }
+    }
+
+
+    private final Mesher blockMesher = new Mesher();
+
+    public RenderDataFactory4(WorldEngine world, ModelFactory modelManager, boolean emitMeshlets) {
+        this.world = world;
+        this.modelMan = modelManager;
+    }
+
+
+    //TODO: MAKE a render cache that caches each WorldSection directional face generation, cause then can just pull that directly
+    // instead of needing to regen the entire thing
+
+
+    //Ok so the idea for fluid rendering is to make it use a seperate mesher and use a different code path for it
+    // since fluid states are explicitly overlays over the base block
+    // can do funny stuff like double rendering
+
+    private static final boolean USE_UINT64 = Capabilities.INSTANCE.INT64_t;
+    public static final int QUADS_PER_MESHLET = 14;
+    private static void writePos(long ptr, long pos) {
+        if (USE_UINT64) {
+            MemoryUtil.memPutLong(ptr, pos);
+        } else {
+            MemoryUtil.memPutInt(ptr, (int) (pos>>32));
+            MemoryUtil.memPutInt(ptr + 4, (int)pos);
+        }
+    }
+
+    private void prepareSectionData() {
+        final var sectionData = this.sectionData;
+        int opaque = 0;
+        for (int i = 0; i < 32*32*32;) {
+            long block = sectionData[i + 32*32*32];//Get the block mapping
+
+            int modelId = this.modelMan.getModelId(Mapper.getBlockId(block));
+            long modelMetadata = this.modelMan.getModelMetadataFromClientId(modelId);
+
+            sectionData[i*2] = modelId|((long) (Mapper.getLightId(block)) <<16)|(((long) (Mapper.getBiomeId(block)))<<24);
+            sectionData[i*2+1] = modelMetadata;
+
+            boolean isFullyOpaque = ModelQueries.isFullyOpaque(modelMetadata);
+            opaque |= (isFullyOpaque ? 1 : 0) << (i&31);
+
+            //TODO: here also do bitmask of what neighboring sections are needed to compute (may be getting rid of this in future)
+
+            //Do increment here
+            i++;
+
+            if ((i&31)==0) {
+                this.opaqueMasks[(i>>5)-1] = opaque;
+                opaque = 0;
+            }
+        }
+    }
+
+    private void generateYFaces() {
+        this.blockMesher.axis = 0;// Y axis
+        for (int y = 0; y < 31; y++) {
+            this.blockMesher.auxiliaryPosition = y;
+            for (int z = 0; z < 32; z++) {//TODO: need to do the faces that border sections
+                int current = this.opaqueMasks[(y+0)*32+z];
+                int next    = this.opaqueMasks[(y+1)*32+z];
+
+                int msk     = current ^ next;
+                if (msk == 0) {
+                    this.blockMesher.skip(32);
+                    continue;
+                }
+
+                //TODO: For boarder sections, should NOT EMIT neighbors faces
+                int faceForwardMsk = msk&current;
+
+
+                int cIdx = -1;
+                while (msk!=0) {
+                    int index = Integer.numberOfTrailingZeros(msk);//Is also the x-axis index
+                    int delta = index - cIdx - 1; cIdx = index; //index--;
+                    if (delta != 0) this.blockMesher.skip(delta);
+                    msk &= ~Integer.lowestOneBit(msk);
+
+                    int facingForward = ((faceForwardMsk>>index)&1);
+
+                    {
+                        int idx = index + (z * 32) + (y * 32 * 32);
+                        //TODO: swap this out for something not getting the next entry
+                        long A = this.sectionData[idx * 2];
+                        long B = this.sectionData[(idx + 32*32) * 2];
+
+                        //Flip data with respect to facing direction
+                        long selfModel = facingForward==1?A:B;
+                        long nextModel = facingForward==1?B:A;
+
+                        //Example thing thats just wrong but as example
+                        this.blockMesher.putNext((long) facingForward | ((selfModel&0xFFFF)<<26) | (0xFFL<<55));
+                    }
+                }
+                this.blockMesher.endRow();
+            }
+            this.blockMesher.finish();
+        }
+    }
+
+
+    private void generateZFaces() {
+        this.blockMesher.axis = 1;// Z axis
+        for (int z = 0; z < 31; z++) {
+            this.blockMesher.auxiliaryPosition = z;
+            for (int y = 0; y < 32; y++) {//TODO: need to do the faces that border sections
+                int current = this.opaqueMasks[y*32+z];
+                int next    = this.opaqueMasks[y*32+z+1];
+
+                int msk     = current ^ next;
+                if (msk == 0) {
+                    this.blockMesher.skip(32);
+                    continue;
+                }
+
+                //TODO: For boarder sections, should NOT EMIT neighbors faces
+                int faceForwardMsk = msk&current;
+
+
+                int cIdx = -1;
+                while (msk!=0) {
+                    int index = Integer.numberOfTrailingZeros(msk);//Is also the x-axis index
+                    int delta = index - cIdx - 1; cIdx = index; //index--;
+                    if (delta != 0) this.blockMesher.skip(delta);
+                    msk &= ~Integer.lowestOneBit(msk);
+
+                    int facingForward = ((faceForwardMsk>>index)&1);
+
+                    {
+                        int idx = index + (z * 32) + (y * 32 * 32);
+                        //TODO: swap this out for something not getting the next entry
+                        long A = this.sectionData[idx * 2];
+                        long B = this.sectionData[(idx + 32) * 2];
+
+                        //Flip data with respect to facing direction
+                        long selfModel = facingForward==1?A:B;
+                        long nextModel = facingForward==1?B:A;
+
+                        //Example thing thats just wrong but as example
+                        this.blockMesher.putNext((long) facingForward | ((selfModel&0xFFFF)<<26) | (0xFFL<<55));
+                    }
+                }
+                this.blockMesher.endRow();
+            }
+            this.blockMesher.finish();
+        }
+    }
+
+    private void generateXFaces() {
+        //TODO: actually fking accelerate this
+
+        this.blockMesher.axis = 2;// X axis
+        for (int x = 0; x < 31; x++) {//TODO: need to do the faces that border sections
+            this.blockMesher.auxiliaryPosition = x;
+            for (int z = 0; z < 32; z++) {
+                for (int y = 0; y < 32; y++) {
+                    int idx = x+z*32+y*32*32;
+                    long self = this.sectionData[idx*2];
+                    long next = this.sectionData[(idx+1)*2];
+
+                    boolean so = ModelQueries.isFullyOpaque(this.sectionData[idx*2+1]);
+                    boolean no = ModelQueries.isFullyOpaque(this.sectionData[(idx+1)*2+1]);
+                    if (so^no) {//Not culled
+                        //Flip data with respect to facing direction
+                        long selfModel = so?self:next;
+                        long nextModel = so?next:selfModel;
+
+                        //Example thing thats just wrong but as example
+                        this.blockMesher.putNext((long) (so?1L:0L) | ((selfModel&0xFFFF)<<26) | (0xFFL<<55));
+                    } else {
+                        this.blockMesher.putNext(0);
+                    }
+                }
+                this.blockMesher.endRow();
+            }
+            this.blockMesher.finish();
+        }
+    }
+
+    /*
+    private static long createQuad() {
+        ((long)clientModelId) | (((long) Mapper.getLightId(ModelQueries.faceUsesSelfLighting(metadata, face)?self:facingState))<<16) | ((((long) Mapper.getBiomeId(self))<<24) * (ModelQueries.isBiomeColoured(metadata)?1:0)) | otherFlags
+
+
+            long data = Integer.toUnsignedLong(array[i*3+1]);
+            data |= ((long) array[i*3+2])<<32;
+            long encodedQuad = Integer.toUnsignedLong(QuadEncoder.encodePosition(face, otherAxis, quad)) | ((data&0xFFFF)<<26) | (((data>>16)&0xFF)<<55) | (((data>>24)&0x1FF)<<46);
+    }*/
+
+    //section is already acquired and gets released by the parent
+    public BuiltSection generateMesh(WorldSection section) {
+        //Copy section data to end of array so that can mutate array while reading safely
+        section.copyDataTo(this.sectionData, 32*32*32);
+
+        this.quadCount = 0;
+
+        this.minX = Integer.MAX_VALUE;
+        this.minY = Integer.MAX_VALUE;
+        this.minZ = Integer.MAX_VALUE;
+        this.maxX = Integer.MIN_VALUE;
+        this.maxY = Integer.MIN_VALUE;
+        this.maxZ = Integer.MIN_VALUE;
+
+        for (var i : this.directionalQuadCollectors) {
+            i.clear();
+        }
+        /*
+        this.world.acquire(section.lvl, section.x+1, section.y, section.z).release();
+        this.world.acquire(section.lvl, section.x-1, section.y, section.z).release();
+        this.world.acquire(section.lvl, section.x, section.y+1, section.z).release();
+        this.world.acquire(section.lvl, section.x, section.y-1, section.z).release();
+        this.world.acquire(section.lvl, section.x, section.y, section.z+1).release();
+        this.world.acquire(section.lvl, section.x, section.y, section.z-1).release();*/
+
+        //Prepare everything
+        this.prepareSectionData();
+
+        this.generateYFaces();
+
+        this.generateZFaces();
+
+        this.generateXFaces();
+
+        //TODO:NOTE! when doing face culling of translucent blocks,
+        // if the connecting type of the translucent block is the same AND the face is full, discard it
+        // this stops e.g. multiple layers of glass (and ocean) from having 3000 layers of quads etc
+
+        if (this.quadCount == 0) {
+            return BuiltSection.empty(section.key);
+        }
+
+        //TODO: FIXME AND OPTIMIZE, get rid of the stupid quad collector bullshit
+
+        int[] offsets = new int[8];
+        var buff = new MemoryBuffer(this.quadCount * 8L);
+        long ptr = buff.address;
+        int coff = 0;
+
+        long size = 0;
+        for (int face = 0; face < 6; face++) {
+            offsets[face + 2] = coff;
+            final LongArrayList faceArray = this.directionalQuadCollectors[face];
+            size = faceArray.size();
+            for (int i = 0; i < size; i++) {
+                long data = faceArray.getLong(i);
+                MemoryUtil.memPutLong(ptr + ((coff++) * 8L), data);
+            }
+        }
+
+
+        int aabb = 0;
+        aabb |= 0;
+        aabb |= 0<<5;
+        aabb |= 0<<10;
+        aabb |= (31)<<15;
+        aabb |= (31)<<20;
+        aabb |= (31)<<25;
+
+        return new BuiltSection(section.key, section.getNonEmptyChildren(), aabb, buff, offsets);
+
+        /*
+            buff = new MemoryBuffer(bufferSize * 8L);
+            long ptr = buff.address;
+            int coff = 0;
+
+            //Ordering is: translucent, double sided quads, directional quads
+            offsets[0] = coff;
+            int size = this.translucentQuadCollector.size();
+            LongArrayList arrayList = this.translucentQuadCollector;
+            for (int i = 0; i < size; i++) {
+                long data = arrayList.getLong(i);
+                MemoryUtil.memPutLong(ptr + ((coff++) * 8L), data);
+            }
+
+            offsets[1] = coff;
+            size = this.doubleSidedQuadCollector.size();
+            arrayList = this.doubleSidedQuadCollector;
+            for (int i = 0; i < size; i++) {
+                long data = arrayList.getLong(i);
+                MemoryUtil.memPutLong(ptr + ((coff++) * 8L), data);
+            }
+
+            for (int face = 0; face < 6; face++) {
+                offsets[face + 2] = coff;
+                final LongArrayList faceArray = this.directionalQuadCollectors[face];
+                size = faceArray.size();
+                for (int i = 0; i < size; i++) {
+                    long data = faceArray.getLong(i);
+                    MemoryUtil.memPutLong(ptr + ((coff++) * 8L), data);
+                }
+            }
+
+        int aabb = 0;
+        aabb |= this.minX;
+        aabb |= this.minY<<5;
+        aabb |= this.minZ<<10;
+        aabb |= (this.maxX-this.minX)<<15;
+        aabb |= (this.maxY-this.minY)<<20;
+        aabb |= (this.maxZ-this.minZ)<<25;
+
+        return new BuiltSection(section.key, section.getNonEmptyChildren(), aabb, buff, offsets);
+         */
+    }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    //Returns true if a face was placed
+    private boolean putFluidFaceIfCan(Mesher2D mesher, int face, int opposingFace, long self, long metadata, int selfClientModelId, int selfBlockId, long facingState, long facingMetadata, int facingClientModelId, int a, int b) {
+        int selfFluidClientId = this.modelMan.getFluidClientStateId(selfClientModelId);
+        long selfFluidMetadata = this.modelMan.getModelMetadataFromClientId(selfFluidClientId);
+
+        int facingFluidClientId = -1;
+        if (ModelQueries.containsFluid(facingMetadata)) {
+            facingFluidClientId = this.modelMan.getFluidClientStateId(facingClientModelId);
+        }
+
+        //If both of the states are the same, then dont render the fluid face
+        if (selfFluidClientId == facingFluidClientId) {
+            return false;
+        }
+
+        if (facingFluidClientId != -1) {
+            //TODO: OPTIMIZE
+            if (this.world.getMapper().getBlockStateFromBlockId(selfBlockId).getBlock() == this.world.getMapper().getBlockStateFromBlockId(Mapper.getBlockId(facingState)).getBlock()) {
+               return false;
+            }
+        }
+
+
+        if (ModelQueries.faceOccludes(facingMetadata, opposingFace)) {
+            return false;
+        }
+
+        //if the model has a fluid state but is not a liquid need to see if the solid state had a face rendered and that face is occluding, if so, dont render the fluid state face
+        if ((!ModelQueries.isFluid(metadata)) && ModelQueries.faceOccludes(metadata, face)) {
+            return false;
+        }
+
+
+
+        //TODO:FIXME SOMEHOW THIS IS CRITICAL!!!!!!!!!!!!!!!!!!
+        // so there is one more issue need to be fixed, if water is layered ontop of eachother, the side faces depend on the water state ontop
+        // this has been hackfixed in the model texture bakery but a proper solution that doesnt explode the sides of the water textures needs to be done
+        // the issue is that the fluid rendering depends on the up state aswell not just the face state which is really really painful to account for
+        // e.g the sides of a full water is 8 high or something, not the full block height, this results in a gap between water layers
+
+
+
+        long otherFlags = 0;
+        otherFlags |= ModelQueries.isTranslucent(selfFluidMetadata)?1L<<33:0;
+        otherFlags |= ModelQueries.isDoubleSided(selfFluidMetadata)?1L<<34:0;
+        mesher.put(a, b, ((long)selfFluidClientId) | (((long) Mapper.getLightId(ModelQueries.faceUsesSelfLighting(selfFluidMetadata, face)?self:facingState))<<16) | ((((long) Mapper.getBiomeId(self))<<24) * (ModelQueries.isBiomeColoured(selfFluidMetadata)?1:0)) | otherFlags);
+        return true;
+    }
+
+    //Returns true if a face was placed
+    private boolean putFaceIfCan(Mesher2D mesher, int face, int opposingFace, long metadata, int clientModelId, int selfBiome, long facingModelId, long facingMetadata, int selfLight, int facingLight, int a, int b) {
+        if (ModelQueries.cullsSame(metadata) && clientModelId == facingModelId) {
+            //If we are facing a block, and we are both the same state, dont render that face
+            return false;
+        }
+
+        //If face can be occluded and is occluded from the facing block, then dont render the face
+        if (ModelQueries.faceCanBeOccluded(metadata, face) && ModelQueries.faceOccludes(facingMetadata, opposingFace)) {
+            return false;
+        }
+
+        long otherFlags = 0;
+        otherFlags |= ModelQueries.isTranslucent(metadata)?1L<<33:0;
+        otherFlags |= ModelQueries.isDoubleSided(metadata)?1L<<34:0;
+        mesher.put(a, b, ((long)clientModelId) | (((long) Mapper.getLightId(ModelQueries.faceUsesSelfLighting(metadata, face)?selfLight:facingLight))<<16) | (ModelQueries.isBiomeColoured(metadata)?(((long) Mapper.getBiomeId(selfBiome))<<24):0) | otherFlags);
+        return true;
+    }
+
+    private static int getMeshletHoldingCount(int quads, int quadsPerMeshlet, int meshletSize) {
+        return ((quads+(quadsPerMeshlet-1))/quadsPerMeshlet)*meshletSize;
+    }
+
+    public static int alignUp(int n, int alignment) {
+        return (n + alignment - 1) & -alignment;
+    }
+}

src/main/java/me/cortex/voxy/client/core/rendering/building/RenderGenerationService.java

@@ -43,7 +43,7 @@ public class RenderGenerationService {
 
         this.threads = serviceThreadPool.createService("Section mesh generation service", 100, ()->{
             //Thread local instance of the factory
-            var factory = new RenderDataFactory(this.world, this.modelBakery.factory, this.emitMeshlets);
+            var factory = new RenderDataFactory4(this.world, this.modelBakery.factory, this.emitMeshlets);
             return () -> {
                 this.processJob(factory);
             };
@@ -72,14 +72,11 @@ public class RenderGenerationService {
     }
 
     //TODO: add a generated render data cache
-    private void processJob(RenderDataFactory factory) {
+    private void processJob(RenderDataFactory4 factory) {
         BuildTask task;
         synchronized (this.taskQueue) {
-            if (Math.random() < 0.5) {
-                task = this.taskQueue.removeLast();
-            } else {
             task = this.taskQueue.removeFirst();
-            }
+            //task = (Math.random() < 0.1)?this.taskQueue.removeLast():this.taskQueue.removeFirst();
         }
         //long time = BuiltSection.getTime();
         var section = this.acquireSection(task.position);
@@ -96,8 +93,9 @@ public class RenderGenerationService {
                 this.modelBakery.requestBlockBake(e.id);
             }
             if (task.hasDoneModelRequest) {
+
                 try {
-                    Thread.sleep(10);
+                    Thread.sleep(1);
                 } catch (InterruptedException ex) {
                     throw new RuntimeException(ex);
                 }

src/main/java/me/cortex/voxy/client/core/util/Mesher2D.java

@@ -7,37 +7,29 @@ import java.util.Random;
 //TODO: redo this so that it works as you are inserting data into it maybe? since it should be much faster??
 
 public final class Mesher2D {
-    private final int size;
+    private static final int MAX_MERGED_SIZE = 15;//16
-    private final int maxSize;
+
+    private static final int SIZE_BITS = 5;
+    private static final int MSK = (1<<SIZE_BITS) -1;
+
     private final long[] data;
     private final long[] setset;
     private int[] quadCache;
     private boolean isEmpty = true;
     private int setsMsk = 0;
-    public Mesher2D(int sizeBits, int maxSize) {
+    public Mesher2D() {
-        if (sizeBits > 5) {
+        this.data = new long[1<<(SIZE_BITS<<1)];
-            throw new IllegalStateException("Due to the addition of the setsMsk, size greter than 32 is not supported atm");
+        this.setset = new long[(1<<(SIZE_BITS<<1))>>6];
-        }
-
-        this.size = sizeBits;
-        this.maxSize = maxSize;
-        this.data = new long[1<<(sizeBits<<1)];
-        this.setset = new long[(1<<(sizeBits<<1))>>6];
         this.quadCache = new int[128];
     }
 
-    private int getIdx(int x, int z) {
+    private static int getIdx(int x, int z) {
-        int M = (1<<this.size)-1;
+        return ((z&MSK)<<SIZE_BITS)|(x&MSK);
-        /*
-        if (false&&(x>M || z>M)) {
-            throw new IllegalStateException();
-        }*/
-        return ((z&M)<<this.size)|(x&M);
     }
 
     public Mesher2D put(int x, int z, long data) {
         this.isEmpty = false;
-        int idx = this.getIdx(x, z);
+        int idx = getIdx(x, z);
         this.data[idx] = data;
         this.setset[idx>>6] |= 1L<<(idx&0b111111);
         this.setsMsk |= 1<<(idx>>6);
@@ -66,7 +58,7 @@ public final class Mesher2D {
     }
 
     private boolean canMerge(int x, int z, long match) {
-        int id = this.getIdx(x, z);
+        int id = getIdx(x, z);
         return (this.setset[id>>6]&(1L<<(id&0b111111))) != 0 && this.data[id] == match;
     }
 
@@ -89,6 +81,7 @@ public final class Mesher2D {
 
         int[] quads = this.quadCache;
         int idxCount = 0;
+        int counter = 0;
 
         //TODO: add different strategies/ways to mesh
         int posId = this.data[0] == 0?this.nextSetBit(0):0;
@@ -96,18 +89,17 @@ public final class Mesher2D {
             int idx = posId;
             long data = this.data[idx];
 
-            int M = (1<<this.size)-1;
+            int x = idx&MSK;
-            int x = idx&M;
+            int z = (idx>>>SIZE_BITS)&MSK;
-            int z = (idx>>>this.size)&M;
 
-            boolean ex = x != ((1<<this.size)-1);
+            boolean ex = x != MSK;
-            boolean ez = z != ((1<<this.size)-1);
+            boolean ez = z != MSK;
             int endX = x;
             int endZ = z;
             while (ex || ez) {
                 //Expand in the x direction
                 if (ex) {
-                    if (endX - x >= this.maxSize || endX >= (1 << this.size) - 1) {
+                    if (endX - x >= MAX_MERGED_SIZE || endX >= MSK) {
                         ex = false;
                     }
                 }
@@ -123,7 +115,7 @@ public final class Mesher2D {
                     endX++;
                 }
                 if (ez) {
-                    if (endZ - z >= this.maxSize || endZ >= (1<<this.size)-1) {
+                    if (endZ - z >= SIZE_BITS || endZ >= MSK) {
                         ez = false;
                     }
                 }
@@ -143,7 +135,7 @@ public final class Mesher2D {
             //Mark the sections as meshed
             for (int mx = x; mx <= endX; mx++) {
                 for (int mz = z; mz <= endZ; mz++) {
-                    int cid = this.getIdx(mx, mz);
+                    int cid = getIdx(mx, mz);
                     this.setset[cid>>6] &= ~(1L<<(cid&0b111111));
                 }
             }
@@ -151,19 +143,24 @@ public final class Mesher2D {
             int encodedQuad = encodeQuad(x, z, endX - x + 1, endZ - z + 1);
 
             {
-                int pIdx = idxCount++;
+                counter++;
-                if (pIdx == quads.length) {
+                int pIdx = idxCount;
-                    var newArray = new int[quads.length + 64];
+                idxCount += 3;
+                if (quads.length <= idxCount+3) {
+                    var newArray = new int[quads.length + 64*3];
                     System.arraycopy(quads, 0, newArray, 0, quads.length);
                     quads = newArray;
                 }
                 quads[pIdx] = encodedQuad;
+                quads[pIdx+1] = (int) data;
+                quads[pIdx+2] = (int) (data>>32);
+
             }
             posId = this.nextSetBit(posId);
         }
 
         this.quadCache = quads;
-        return idxCount;
+        return counter;
     }
 
     public int[] getArray() {
@@ -179,16 +176,8 @@ public final class Mesher2D {
         }
     }
 
-    public long getDataFromQuad(int quad) {
-        return this.getData(getX(quad), getZ(quad));
-    }
-
-    public long getData(int x, int z) {
-        return this.data[this.getIdx(x, z)];
-    }
-
     public static void main3(String[] args) {
-        var mesh = new Mesher2D(5,15);
+        var mesh = new Mesher2D();
         mesh.put(30,30, 123);
         mesh.put(31,30, 123);
         mesh.put(30,31, 123);
@@ -198,9 +187,9 @@ public final class Mesher2D {
         System.err.println(count);
     }
 
-    public static void main(String[] args) {
+    public static void main2(String[] args) {
         var r = new Random(123451);
-        var mesh = new Mesher2D(5,15);
+        var mesh = new Mesher2D();
         /*
         for (int j = 0; j < 512; j++) {
             mesh.put(r.nextInt(32), r.nextInt(32), r.nextInt(10));
@@ -235,12 +224,25 @@ public final class Mesher2D {
         //}
     }
 
-    public static void main2(String[] args) {
+    public static void main(String[] args) {
         var r = new Random(123451);
         int a = 0;
+
+        //Prime code
+        for (int i = 0; i < 100000; i++) {
+            var mesh = new Mesher2D();
+            for (int j = 0; j < 512; j++) {
+                mesh.put(r.nextInt(32), r.nextInt(32), r.nextInt(100));
+            }
+            var result = mesh.process();
+            a += result;
+        }
+
+
         long total = 0;
-        for (int i = 0; i < 200000; i++) {
+        int COUNT = 200000;
-            var mesh = new Mesher2D(5,16);
+        for (int i = 0; i < COUNT; i++) {
+            var mesh = new Mesher2D();
             for (int j = 0; j < 512; j++) {
                 mesh.put(r.nextInt(32), r.nextInt(32), r.nextInt(100));
             }
@@ -249,9 +251,8 @@ public final class Mesher2D {
             total += System.nanoTime() - s;
             a += result;
         }
-        System.out.println(total/(1e+6));
+
-        System.out.println((double) (total/(1e+6))/200000);
+        System.out.println(((double) total/COUNT)*(1e-6));
-        //mesh.put(0,0,1);
     }
 }
 

src/main/java/me/cortex/voxy/client/core/util/ScanMesher2D.java

@@ -3,6 +3,10 @@ package me.cortex.voxy.client.core.util;
 import java.util.Random;
 
 public abstract class ScanMesher2D {
+
+    private static final int MAX_SIZE = 16;
+
+
     // is much faster if implemented inline into parent
     private final long[] rowData = new long[32];
     private final int[] rowLength = new int[32];//How long down does a row entry go
@@ -24,7 +28,7 @@ public abstract class ScanMesher2D {
             //If the previous data is not zero, that means it was not merge-able, so emit it at the pos
             if (this.currentData!=0) {
                 if ((this.rowBitset&(1<<31))!=0) {
-                    emitQuad(31, (this.currentIndex-1)>>5, this.rowLength[31], this.rowDepth[31], this.rowData[31]);
+                    emitQuad(31, ((this.currentIndex-1)>>5)-1, this.rowLength[31], this.rowDepth[31], this.rowData[31]);
                 }
                 this.rowBitset |= 1<<31;
                 this.rowLength[31] = this.currentSum;
@@ -38,13 +42,10 @@ public abstract class ScanMesher2D {
         }
 
         //If we are different from previous (this can never happen if previous is index 0)
-        if (data != this.currentData) {
+        if (data != this.currentData || this.currentSum == MAX_SIZE) {
             //write out previous data if its a non sentinel, it is guarenteed to not have a row bit set
             if (this.currentData != 0) {
                 int prev = idx-1;//We need to write in the previous entry
-                if ((this.rowBitset&(1<<prev))!=0) {
-                    throw new IllegalStateException();
-                }
                 this.rowDepth[prev] = 1;
                 this.rowLength[prev] = this.currentSum;
                 this.rowData[prev] = this.currentData;
@@ -58,16 +59,23 @@ public abstract class ScanMesher2D {
 
 
         boolean isSet = (this.rowBitset&(1<<idx))!=0;
+        boolean causedByDepthMax = false;
         //Greadily merge with previous row if possible
         if (this.currentData != 0 &&//Ignore sentinel empty
                 isSet &&
                 this.rowLength[idx] == this.currentSum &&
                 this.rowData[idx] == this.currentData) {//Can merge with previous row
-            this.rowDepth[idx]++;
+            int depth = ++this.rowDepth[idx];
             this.currentSum = 0;//Clear sum since we went down
             this.currentData = 0;//Zero is sentinel value for absent
-        } else if (isSet) {
+            if (depth != MAX_SIZE) {
-            this.emitQuad(idx&31, (this.currentIndex-1)>>5, this.rowLength[idx], this.rowDepth[idx], this.rowData[idx]);
+                return;
+            }
+            causedByDepthMax = true;
+        }
+
+        if (isSet) {
+            this.emitQuad(idx&31, ((this.currentIndex-1)>>5)-(causedByDepthMax?0:1), this.rowLength[idx], this.rowDepth[idx], this.rowData[idx]);
             this.rowBitset &= ~(1<<idx);
         }
     }
@@ -81,7 +89,7 @@ public abstract class ScanMesher2D {
                 rowSet &= ~Integer.lowestOneBit(rowSet);
 
                 //Emit the quad, dont need to clear the data since it not existing in the bitmask is implicit no data
-                this.emitQuad(index, this.currentIndex>>5, this.rowLength[index], this.rowDepth[index], this.rowData[index]);
+                this.emitQuad(index, ((this.currentIndex-1)>>5)-1, this.rowLength[index], this.rowDepth[index], this.rowData[index]);
             }
             this.rowBitset &= ~msk;
         }
@@ -90,36 +98,80 @@ public abstract class ScanMesher2D {
     //Note it is illegal for count to cause `this.currentIndex&31` to wrap and continue
     public final void skip(int count) {
         if (count == 0) return;
-        //TODO: replace with much better method
+        //TODO: replace with much better method, TODO: check this is right!!
         this.putNext(0);
-        this.emitRanged(((1<<(count-1))-1)<<this.currentIndex);
+        this.emitRanged(((1<<(count-1))-1)<<(this.currentIndex&31));
-        this.currentIndex += count;
+        this.currentIndex += count-1;
+        /*
+        for (int i = 0; i < count; i++) {
+            this.putNext(0);
+        }*/
     }
 
     public final void resetScanlineRowIndex() {
+        this.currentSum = 0;
+        this.currentData = 0;
         this.currentIndex = 0;
     }
 
     public final void endRow() {
         if ((this.currentIndex&31)!=0) {
-            this.skip(31-(this.currentIndex&31));
+            this.skip(32-(this.currentIndex&31));
         }
     }
 
     public final void finish() {
+        /*
         if ((this.currentIndex&31)!=0) {
-            this.skip(31-(this.currentIndex&31));
+            this.skip(32-(this.currentIndex&31));
         } else {
             this.putNext(0);
             this.currentIndex--;//HACK to reset currentIndex&31 to 0
         }
+        this.currentIndex++;
+        for (int i = 0; i < 32; i++) {
+            this.putNext(0);
+        }*/
+
+        //TODO: check this is correct
+        this.skip(32-(this.currentIndex&31));
         this.emitRanged(-1);
+
         this.resetScanlineRowIndex();
     }
 
     protected abstract void emitQuad(int x, int z, int length, int width, long data);
 
-    public static void main(String[] args) {
+
+    public static void main9(String[] args) {
+
+        int[] qc = new int[3];
+        var mesher = new ScanMesher2D(){
+            @Override
+            protected void emitQuad(int x, int z, int length, int width, long data) {
+                qc[0]++;
+                if (data != qc[0]) {
+                    throw new IllegalStateException();
+                }
+                if (length*width != 1) {
+                    if (data != qc[0])
+                        throw new IllegalStateException();
+                }
+                if (x!=(((qc[0])&0x1f)))
+                    throw new IllegalStateException();
+                if (z!=((qc[0])>>5))
+                    throw new IllegalStateException();
+            }
+        };
+
+        mesher.putNext(0);
+        int i = 1;
+        while (true) {
+            mesher.putNext(i++);
+        }
+    }
+
+    public static void main5(String[] args) {
         var r = new Random(0);
         long[] data = new long[32*32];
         float DENSITY = 0.5f;
@@ -183,7 +235,7 @@ public abstract class ScanMesher2D {
 
 
     }
-    public static void main3(String[] args) {
+    public static void main4(String[] args) {
         var r = new Random(0);
         int[] qc = new int[2];
         var mesher = new ScanMesher2D(){
@@ -196,12 +248,12 @@ public abstract class ScanMesher2D {
 
         var mesh2 = new Mesher2D();
 
-        float DENSITY = 0.5f;
+        float DENSITY = 0.75f;
-        int RANGE = 50;
+        int RANGE = 25;
         int total = 0;
         while (true) {
-            DENSITY = r.nextFloat();
+            //DENSITY = r.nextFloat();
-            RANGE = r.nextInt(500)+1;
+            //RANGE = r.nextInt(500)+1;
             qc[0] = 0; qc[1] = 0;
             int c = 0;
             for (int i = 0; i < 32*32; i++) {
@@ -214,12 +266,12 @@ public abstract class ScanMesher2D {
             }
             mesher.finish();
             if (c != qc[1]) {
-                System.out.println(c+", " + qc[1]);
+                System.out.println("ERROR: "+c+", " + qc[1]);
             }
             int count = mesh2.process();
             int delta = count - qc[0];
             total += delta;
-            System.out.println(total);
+            //System.out.println(total);
             //System.out.println(c+", new: " + qc[0] + " old: " + count);
         }
     }
@@ -265,4 +317,108 @@ public abstract class ScanMesher2D {
             j++;
         }
     }
+
+    public static void main6(String[] args) {
+        var r = new Random(0);
+        float DENSITY = 0.90f;
+        int RANGE = 3;
+        while (true) {
+            long[] data = new long[32*32];
+            for (int i = 0; i < data.length; i++) {
+                data[i] =  r.nextFloat()<DENSITY?(r.nextInt(RANGE)+1):0;
+            }
+            long[] out = new long[32*32];
+            var mesher = new ScanMesher2D(){
+
+                @Override
+                protected void emitQuad(int x, int z, int length, int width, long data) {
+                    if (data == 0) {
+                        throw new IllegalStateException();
+                    }
+                    if (z<0||x<0||x>31||z>31) {
+                        throw new IllegalStateException();
+                    }
+                    if (length<1||width<1||length>16||width>16) {
+                        throw new IllegalStateException();
+                    }
+                    x -= length-1;
+                    z -= width-1;
+                    if (z<0||x<0||x>31||z>31) {
+                        throw new IllegalStateException();
+                    }
+                    for (int X = x; X < x+length; X++) {
+                        for (int Z = z; Z < z+width; Z++) {
+                            int idx = Z*32+X;
+                            if (out[idx] != 0) {
+                                throw new IllegalStateException();
+                            }
+                            out[idx] = data;
+                        }
+                    }
+                }
+            };
+
+            for (long a : data) {
+                mesher.putNext(a);
+            }
+            mesher.finish();
+
+            for (int i = 0; i < 32*32; i++) {
+                if (data[i] != out[i]) {
+                    System.out.println("ERROR");
+                }
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        long[] data = new long[32*32];
+
+        for (int x = 0; x < 20; x++) {
+            for (int z = 0; z < 20; z++) {
+                data[z*32+x] = 1;
+            }
+        }
+        long[] out = new long[32*32];
+        var mesher = new ScanMesher2D(){
+
+            @Override
+            protected void emitQuad(int x, int z, int length, int width, long data) {
+                if (data == 0) {
+                    throw new IllegalStateException();
+                }
+                if (z<0||x<0||x>31||z>31) {
+                    throw new IllegalStateException();
+                }
+                if (length<1||width<1||length>16||width>16) {
+                    throw new IllegalStateException();
+                }
+                x -= length-1;
+                z -= width-1;
+                if (z<0||x<0||x>31||z>31) {
+                    throw new IllegalStateException();
+                }
+                for (int X = x; X < x+length; X++) {
+                    for (int Z = z; Z < z+width; Z++) {
+                        int idx = Z*32+X;
+                        if (out[idx] != 0) {
+                            throw new IllegalStateException();
+                        }
+                        out[idx] = data;
+                    }
+                }
+            }
+        };
+
+        for (long a : data) {
+            mesher.putNext(a);
+        }
+        mesher.finish();
+
+        for (int i = 0; i < 32*32; i++) {
+            if (data[i] != out[i]) {
+                System.out.println("ERROR");
+            }
+        }
+    }
 }

src/main/java/me/cortex/voxy/common/util/HierarchicalBitSet.java

@@ -78,6 +78,8 @@ public class HierarchicalBitSet {
         return 0;
     }
 
+
+    //TODO: FIXME: THIS IS SLOW AS SHIT
     public int allocateNextConsecutiveCounted(int count) {
         if (this.A==-1) {
             return -1;

src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java

@@ -110,7 +110,7 @@ public class SaveLoadSystem {
             hash ^= metadata; hash *= 1242629872171L;
         }
         for (int i = 0; i < lutLen; i++) {
-            lut[i] = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
+            lut [i] = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
             if (VERIFY_HASH_ON_LOAD) {
                 hash *= 1230987149811L;
                 hash += 12831;

src/main/java/me/cortex/voxy/common/world/WorldSection.java

@@ -150,7 +150,7 @@ public final class WorldSection {
     }
 
     public static int getIndex(int x, int y, int z) {
-        int M = (1<<5)-1;
+        final int M = (1<<5)-1;
         if (VERIFY_WORLD_SECTION_EXECUTION) {
             if (x < 0 || x > M || y < 0 || y > M || z < 0 || z > M) {
                 throw new IllegalArgumentException("Out of bounds: " + x + ", " + y + ", " + z);
@@ -173,9 +173,13 @@ public final class WorldSection {
     }
 
     public void copyDataTo(long[] cache) {
+        copyDataTo(cache, 0);
+    }
+
+    public void copyDataTo(long[] cache, int dstOffset) {
         this.assertNotFree();
-        if (cache.length != this.data.length) throw new IllegalArgumentException();
+        if ((cache.length-dstOffset) < this.data.length) throw new IllegalArgumentException();
-        System.arraycopy(this.data, 0, cache, 0, this.data.length);
+        System.arraycopy(this.data, 0, cache, dstOffset, this.data.length);
     }
 
     public static int getChildIndex(int x, int y, int z) {