spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tinkering

Browse Source
This commit is contained in:
mcrcortex
2025-03-03 10:44:22 +10:00
parent d475b114ee
commit a6ed760304
2 changed files with 408 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/me/cortex/voxy/common/config/section/SectionSerializationStorage.java
View File

@@ -7,11 +7,13 @@ import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;
import me.cortex.voxy.common.world.SaveLoadSystem2;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.common.world.other.Mapper;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
public class SectionSerializationStorage extends SectionStorage {
private final StorageBackend backend;
@@ -41,6 +43,7 @@ public class SectionSerializationStorage extends SectionStorage {
}
}
@Override
public void saveSection(WorldSection section) {
var saveData = SaveLoadSystem.serialize(section);

409
src/main/java/me/cortex/voxy/common/world/SaveLoadSystem2.java
View File

@@ -1,16 +1,20 @@
package me.cortex.voxy.common.world;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ShortOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ShortOpenHashMap;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.commonImpl.VoxyCommon;
import net.minecraft.util.math.MathHelper;
import org.lwjgl.system.MemoryUtil;
import java.util.Arrays;
public class SaveLoadSystem2 {
public static final boolean VERIFY_HASH_ON_LOAD = VoxyCommon.isVerificationFlagOn("verifySectionHash");
public static final boolean VERIFY_MEMORY_ACCESS = VoxyCommon.isVerificationFlagOn("verifyMemoryAccess");
public static final int BIGGEST_SERIALIZED_SECTION_SIZE = 32 * 32 * 32 * 8 * 2 + 8;
public static int lin2z(int i) {//y,z,x
int x = i&0x1F;
@@ -45,13 +49,408 @@ public class SaveLoadSystem2 {
// if doing bitpacking + pallet is larger than just emitting raw entries, do that
//Header includes position (long), (maybe time?), version storage type/version, child existence, air block count?
//
return null;
long[] data = new long[WorldSection.SECTION_VOLUME];
section.copyDataTo(data);
boolean allSameBlockLight = true;
boolean allSameSkyLight = true;
byte[] blockLight = new byte[32*32*32/2];
byte[] skyLight = new byte[32*32*32/2];
short[] blocks = new short[32*32*32];
short[] biome = new short[32*32*32];
Int2IntOpenHashMap blockMapping = new Int2IntOpenHashMap();blockMapping.defaultReturnValue(-1);
Int2IntOpenHashMap biomeMapping = new Int2IntOpenHashMap();biomeMapping.defaultReturnValue(-1);
int[] blockLutVals = new int[32*32*32];
int[] biomeLutVals = new int[32*32*32];
long hash = 12345;
for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) {
long state = data[lin2z(i)];//Sample from Z curve
hash ^= state*1892671+19827911;
hash *= 198729111; hash ^= hash >>> 32;
byte bl = (byte) ((Mapper.getLightId(state)>>4)&0xF);
blockLight[i>>1] |= (byte) (bl<<((i&1)*4));
byte sl = (byte) (Mapper.getLightId(state)&0xF);
skyLight[i>>1] |= (byte) (sl<<((i&1)*4));
if (i!=0) {
allSameBlockLight &= (blockLight[0]&0xF) == bl;
allSameSkyLight &= (skyLight[0]&0xF) == sl;
}
{
int bid = blockMapping.putIfAbsent(Mapper.getBlockId(state), blockMapping.size());
if (bid == -1) {
bid = blockMapping.size() - 1;
blockLutVals[bid] = Mapper.getBlockId(state);
}
blocks[i] = (short) bid;
}
{
int bid = biomeMapping.putIfAbsent(Mapper.getBiomeId(state), biomeMapping.size());
if (bid == -1) {
bid = biomeMapping.size() - 1;
biomeLutVals[bid] = Mapper.getBiomeId(state);
}
biome[i] = (short) bid;
}
}
var res = new MemoryBuffer(32*32*32*8+1024);
long ptr = res.address;
MemoryUtil.memPutLong(ptr, section.key); ptr += 8;
MemoryUtil.memPutLong(ptr, hash); ptr += 8;
int meta = 0;
meta |= allSameBlockLight?1:0;
meta |= allSameSkyLight?2:0;
meta |= (biomeMapping.size()-1)<<2;//512 max size
meta |= (blockMapping.size()-1)<<11;//4096 max size
meta |= Byte.toUnsignedInt(section.nonEmptyChildren) << 23;
MemoryUtil.memPutInt(ptr, meta); ptr += 4;
//Encode lighting
/*
//Micro storage optimization, not done cause makes decode very slightly slower and more pain
if (allSameBlockLight && allSameSkyLight) {
MemoryUtil.memPutByte(ptr, (byte) ((blockLight[0]&0xF)|((skyLight[0]&0xF)<<4))); ptr += 1;
} else*/
{
if (allSameBlockLight) {
MemoryUtil.memPutByte(ptr, (byte) (blockLight[0] & 0xF)); ptr += 1;
} else {
UnsafeUtil.memcpy(blockLight, ptr); ptr += blockLight.length;
}
if (allSameSkyLight) {
MemoryUtil.memPutByte(ptr, (byte) (skyLight[0] & 0xF)); ptr += 1;
} else {
UnsafeUtil.memcpy(skyLight, ptr); ptr += skyLight.length;
}
}
//Compact encoding of block and biome mappinigs
{
int rem = 32;
int batch = 0;
{//Block
int SIZE = 20;// 20 bits per entry
for (int i = 0; i < blockMapping.size(); i++) {
int b = blockLutVals[i];
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
rem -= SIZE;
}
}
}
{//Biome
int SIZE = 9;//9 bits per entry
for (int i = 0; i < biomeMapping.size(); i++) {
int b = biomeLutVals[i];
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
rem -= SIZE;
}
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
}
}
//TODO: see if tight bitpacking is better or if bitpacking with pow2 pack size is better
{//Store blocks
final int SIZE = MathHelper.smallestEncompassingPowerOfTwo(MathHelper.floorLog2(MathHelper.smallestEncompassingPowerOfTwo(blockMapping.size())));
int rem = 32;
int batch = 0;
for (int b : blocks) {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
rem -= SIZE;
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
}
}
{//Store biome
if (biomeMapping.size() == 1) {
//If its only a single mapping, dont put anything
} else {
final int SIZE = MathHelper.smallestEncompassingPowerOfTwo(MathHelper.floorLog2(MathHelper.smallestEncompassingPowerOfTwo(biomeMapping.size())));
int rem = 32;
int batch = 0;
for (int b : biome) {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
rem -= SIZE;
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
}
}
}
return res.subSize(ptr - res.address);
}
public static boolean deserialize(WorldSection section, MemoryBuffer data) {
var cache = DESERIALIZE_CACHE.get();
long ptr = data.address;
long pos = MemoryUtil.memGetLong(ptr); ptr += 8;
if (section.key != pos) {
Logger.error("Section pos not the same as requested, got " + pos + " expected " + section.key);
return false;
}
long chash = MemoryUtil.memGetLong(ptr); ptr += 8;
int meta = MemoryUtil.memGetInt(ptr); ptr += 4;
boolean allSameBlockLight = (meta&1)!=0;
boolean allSameSkyLight = (meta&2)!=0;
int biomeMapSize = ((meta>>2)&0x1FF)+1;
int blockMapSize = ((meta>>11)&((1<<12)-1))+1;
section._unsafeSetNonEmptyChildren((byte) ((meta>>23)&0xFF));
long blockLight;
long skyLight;
if (allSameBlockLight) {
//shift up 4 so that its already in correct position
blockLight = Byte.toUnsignedLong(MemoryUtil.memGetByte(ptr))<<4; ptr += 1;
} else {
blockLight = ptr; ptr += 32*32*32/2;
}
if (allSameSkyLight) {
skyLight = MemoryUtil.memGetByte(ptr); ptr += 1;
} else {
skyLight = ptr; ptr += 32*32*32/2;
}
int[] blockLut = new int[blockMapSize];
int[] biomeLut = new int[biomeMapSize];
{//Deserialize the block and biome mappings
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
{//Block
int SIZE = 20;// 20 bits per entry
int msk = (1<<SIZE)-1;
for (int i = 0; i < blockMapSize; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
}
blockLut[i] = val;
}
}
{//Biome
int SIZE = 9;// 9 bits per entry
int msk = (1<<SIZE)-1;
for (int i = 0; i < biomeMapSize; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
}
biomeLut[i] = val;
}
}
}
//unpack block and biome
short[] blocks = new short[32*32*32];
short[] biomes = new short[32*32*32];
{//Block
final int SIZE = MathHelper.smallestEncompassingPowerOfTwo(MathHelper.floorLog2(MathHelper.smallestEncompassingPowerOfTwo(blockMapSize)));
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int msk = (1<<SIZE)-1;
for (int i = 0; i < blocks.length; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
}
blocks[i] = (short) val;
}
}
{//Biome
if (biomeMapSize == 1) {
Arrays.fill(biomes, (short) 0);
} else {
final int SIZE = MathHelper.smallestEncompassingPowerOfTwo(MathHelper.floorLog2(MathHelper.smallestEncompassingPowerOfTwo(biomeMapSize)));
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int msk = (1<<SIZE)-1;
for (int i = 0; i < biomes.length; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
}
biomes[i] = (short) val;
}
}
}
//Reconstruct everything
long hash = 12345;
for (int i = 0; i < 32*32*32; i++) {
byte light = 0;
{
if (allSameBlockLight) {
light |= (byte) (blockLight&0xF0);
} else {
//Todo clean and optimize this (it can be optimized alot)
light |= (byte) (((Byte.toUnsignedInt(MemoryUtil.memGetByte(blockLight+(i>>1)))>>((i&1)*4))&0xF)<<4);
}
if (allSameSkyLight) {
light |= (byte) (skyLight&0xF);
} else {
//Todo clean and optimize this (it can be optimized alot)
light |= (byte) ((Byte.toUnsignedInt(MemoryUtil.memGetByte(skyLight+(i>>1)))>>((i&1)*4))&0xF);
}
}
int block = blockLut[blocks[i]];
int biome = biomeLut[biomes[i]];
long state = Mapper.composeMappingId(light, block, biome);
hash ^= state*1892671+19827911;
hash *= 198729111; hash ^= hash >>> 32;
section.data[lin2z(i)] = state;
}
if (chash != hash) {
Logger.error("Hash does not match what is expected, got: " + hash + " expected: " + chash);
return false;
}
return true;
}
public static void main2(String[] args) {
var aa = new MemoryBuffer(502400);
int blockMapSize = 100000;
{
long ptr = aa.address;
int rem = 32;
int batch = 0;
{//Block
int SIZE = 20;// 20 bits per entry
for (int i = 0; i < blockMapSize; i++) {
int b = i;
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
rem -= SIZE;
}
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
}
System.err.println(ptr-aa.address);
}
{
long ptr = aa.address;
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
{//Block
int SIZE = 20;// 20 bits per entry
int msk = (1<<SIZE)-1;
for (int i = 0; i < blockMapSize; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
}
//System.out.println(val);
if (val != i) {
throw new IllegalStateException();
}
}
}
System.err.println(ptr-aa.address);
}
}
public static void main(String[] args) {
var test = WorldSection._createRawUntrackedUnsafeSection(0,1,2,3);
test._unsafeSetNonEmptyChildren((byte) 0b10110011);
for (int i = 0; i < 32*32*32; i++) {
test.data[i] = Mapper.composeMappingId((byte) (i%256), 12+(i%1666), i%300);
}
var res = serialize(test);
var test2 = WorldSection._createRawUntrackedUnsafeSection(test.lvl, test.x, test.y, test.z);
System.out.println(deserialize(test2, res));
int a = 0;
for (int i = 0; i < 32*32*32; i++) {
if (test.data[i] != test2.data[i]) {
throw new IllegalStateException();
}
}
}
}