第一个版本
Some checks failed
check-does-build / build (push) Failing after 13m6s

This commit is contained in:
2026-01-06 02:01:35 +08:00
parent c5e447125b
commit 2236bf19a5
49 changed files with 1352 additions and 217 deletions

View File

@@ -245,19 +245,20 @@ dependencies {
implementation "org.lwjgl:lwjgl"
include(implementation "org.lwjgl:lwjgl-lmdb")
include(implementation "org.lwjgl:lwjgl-zstd")
//include(implementation "org.lwjgl:lwjgl-zstd")
runtimeOnly "org.lwjgl:lwjgl:$lwjglVersion:natives-windows"
runtimeOnly "org.lwjgl:lwjgl:$lwjglVersion:natives-linux"
include(runtimeOnly "org.lwjgl:lwjgl-lmdb:$lwjglVersion:natives-windows")
include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-windows")
//include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-windows")
include(runtimeOnly "org.lwjgl:lwjgl-lmdb:$lwjglVersion:natives-linux")
include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-linux")
//include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-linux")
include(implementation 'redis.clients:jedis:5.1.0')
include(implementation('org.rocksdb:rocksdbjni:10.2.1'))
include(implementation 'org.apache.commons:commons-pool2:2.12.0')
include(implementation 'org.lz4:lz4-java:1.8.0')
include(implementation('org.tukaani:xz:1.10'))
include(implementation 'com.github.luben:zstd-jni:1.5.5-1')
if (true) {
if (!isInGHA) {

View File

@@ -24,12 +24,17 @@ import java.util.HashSet;
import java.util.function.Consumer;
import java.util.function.Function;
import net.fabricmc.fabric.api.client.networking.v1.ClientPlayNetworking;
import me.cortex.voxy.common.network.VoxyNetwork;
import me.cortex.voxy.commonImpl.WorldIdentifier;
public class VoxyClient implements ClientModInitializer {
private static final HashSet<String> FREX = new HashSet<>();
public static void initVoxyClient() {
Capabilities.init();//Ensure clinit is called
if (Capabilities.INSTANCE.hasBrokenDepthSampler) {
Logger.error("AMD broken depth sampler detected, voxy does not work correctly and has been disabled, this will hopefully be fixed in the future");
}
@@ -88,6 +93,39 @@ public class VoxyClient implements ClientModInitializer {
} else {
FREX.remove(name);
}}));
ClientPlayNetworking.registerGlobalReceiver(VoxyNetwork.ConfigSyncPayload.TYPE, (payload, context) -> {
context.client().execute(() -> {
// Update client render distance cap if needed, or store server settings
// For now, we can perhaps log it or update a transient config
Logger.info("Received server view distance: " + payload.viewDistance());
// We might want to clamp the local render distance to the server's max if strictly enforced,
// or just use it as a hint for where to expect data.
// The user requirement says: "server sends max view distance to client... client renders up to client setting, server updates outside sim distance up to server max"
// So we should probably store this "server max view distance" somewhere in VoxyClientInstance.
if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) {
clientInstance.setServerViewDistance(payload.viewDistance());
}
});
});
ClientPlayNetworking.registerGlobalReceiver(VoxyNetwork.LodUpdatePayload.TYPE, (payload, context) -> {
// Deserialize off-thread if possible? Packet handling is on netty thread or main thread depending on configuration.
// But we can just schedule it.
// Actually deserialize needs Mapper which is world specific?
// The packet doesn't contain world ID?
// We assume it's for the current client world.
// Wait, we need to know WHICH world this update is for if we have dimensions?
// Usually packets are for the current world the player is in.
context.client().execute(() -> {
// Logger.info("Received LOD update packet, size: " + payload.data().length);
if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) {
clientInstance.handleLodUpdate(payload);
}
});
});
}
public static boolean isFrexActive() {

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client;
import me.cortex.voxy.client.compat.FlashbackCompat;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.mixin.sodium.AccessorSodiumWorldRenderer;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.StorageConfigUtil;
@@ -28,6 +28,8 @@ public class VoxyClientInstance extends VoxyInstance {
private final SectionStorageConfig storageConfig;
private final Path basePath;
private final boolean noIngestOverride;
private int serverViewDistance = 32;
public VoxyClientInstance() {
super();
var path = FlashbackCompat.getReplayStoragePath();
@@ -40,6 +42,55 @@ public class VoxyClientInstance extends VoxyInstance {
this.updateDedicatedThreads();
}
public long getLastLodUpdate() {
return lastLodUpdate;
}
public int getLodUpdatesReceived() {
return lodUpdatesReceived;
}
public void setServerViewDistance(int distance) {
this.serverViewDistance = distance;
// Trigger a re-evaluation of render distance?
Logger.info("Updating client view distance from server to: " + distance);
}
public int getServerViewDistance() {
return this.serverViewDistance;
}
private long lastLodUpdate;
private int lodUpdatesReceived;
public void handleLodUpdate(me.cortex.voxy.common.network.VoxyNetwork.LodUpdatePayload payload) {
this.lastLodUpdate = System.currentTimeMillis();
this.lodUpdatesReceived++;
// 1. Get current client world
var player = Minecraft.getInstance().player;
if (player == null) return;
var level = player.level();
var wi = WorldIdentifier.of(level);
if (wi == null) return;
var engine = this.getNullable(wi);
if (engine == null) return;
// 2. Deserialize payload using engine's mapper
// Note: Payload deserialization requires Mapper to resolve IDs.
// We need to ensure the engine's mapper is up to date with the server's palette?
// No, the payload contains palette strings/states.
// The deserializer maps them to LOCAL IDs using the provided mapper.
try {
var section = payload.deserialize(engine.getMapper());
// 3. Insert update into engine
me.cortex.voxy.common.world.WorldUpdater.insertUpdate(engine, section);
} catch (Exception e) {
Logger.error("Failed to handle LOD update", e);
}
}
@Override
public void updateDedicatedThreads() {
int target = VoxyConfig.CONFIG.serviceThreads;

View File

@@ -2,6 +2,7 @@ package me.cortex.voxy.client.config;
import me.cortex.voxy.client.RenderStatistics;
import me.cortex.voxy.client.config.SodiumConfigBuilder.*;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.VoxyClient;
import me.cortex.voxy.client.VoxyClientInstance;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;

View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.client.core;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.gl.GlFramebuffer;
import me.cortex.voxy.client.core.gl.GlTexture;
import me.cortex.voxy.client.core.gl.shader.Shader;

View File

@@ -4,7 +4,7 @@ import com.mojang.blaze3d.opengl.GlConst;
import com.mojang.blaze3d.opengl.GlStateManager;
import me.cortex.voxy.client.TimingStatistics;
import me.cortex.voxy.client.VoxyClient;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.gl.Capabilities;
import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.gl.GlTexture;
@@ -50,6 +50,9 @@ import static org.lwjgl.opengl.GL33.glBindSampler;
import static org.lwjgl.opengl.GL43.GL_SHADER_STORAGE_BUFFER;
import static org.lwjgl.opengl.GL43C.GL_SHADER_STORAGE_BUFFER_BINDING;
import me.cortex.voxy.client.VoxyClientInstance;
import me.cortex.voxy.common.config.VoxyServerConfig;
public class VoxyRenderSystem {
private final WorldEngine worldIn;
@@ -434,6 +437,14 @@ public class VoxyRenderSystem {
debug.add("Extra 2 time: " + TimingStatistics.E.pVal() + ", " + TimingStatistics.F.pVal() + ", " + TimingStatistics.G.pVal() + ", " + TimingStatistics.H.pVal() + ", " + TimingStatistics.I.pVal());
}
debug.add(GPUTiming.INSTANCE.getDebug());
if (VoxyCommon.getInstance() instanceof VoxyClientInstance clientInstance) {
long lastUpdate = clientInstance.getLastLodUpdate();
long timeSince = System.currentTimeMillis() - lastUpdate;
debug.add("LOD Updates: " + clientInstance.getLodUpdatesReceived() + " (Last: " + timeSince + "ms ago)");
debug.add("Server View Dist: " + clientInstance.getServerViewDistance());
}
PrintfDebugUtil.addToOut(debug);
}

View File

@@ -2,7 +2,7 @@ package me.cortex.voxy.client.core.rendering.hierachical;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
import me.cortex.voxy.client.RenderStatistics;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.gl.shader.AutoBindingShader;
import me.cortex.voxy.client.core.gl.shader.Shader;

View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.client.iris;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;
import net.irisshaders.iris.gl.uniform.UniformHolder;
import net.minecraft.client.Minecraft;

View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.client.mixin.iris;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.util.IrisUtil;
import me.cortex.voxy.client.iris.VoxyUniforms;
import net.irisshaders.iris.gl.uniform.UniformHolder;

View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.client.mixin.iris;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.util.IrisUtil;
import me.cortex.voxy.client.iris.IGetVoxyPatchData;
import me.cortex.voxy.client.iris.IrisShaderPatch;

View File

@@ -3,7 +3,7 @@ package me.cortex.voxy.client.mixin.iris;
import com.google.common.collect.ImmutableList;
import com.llamalad7.mixinextras.injector.wrapoperation.Operation;
import com.llamalad7.mixinextras.injector.wrapoperation.WrapOperation;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.util.IrisUtil;
import me.cortex.voxy.client.iris.IrisShaderPatch;
import net.irisshaders.iris.gl.shader.StandardMacros;

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client.mixin.minecraft;
import me.cortex.voxy.client.ICheekyClientChunkCache;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.common.world.service.VoxelIngestService;
import net.fabricmc.loader.api.FabricLoader;
import net.minecraft.client.multiplayer.ClientChunkCache;

View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.client.mixin.minecraft;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.common.world.service.VoxelIngestService;
import me.cortex.voxy.commonImpl.WorldIdentifier;
import net.minecraft.client.multiplayer.ClientChunkCache;

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client.mixin.minecraft;
import me.cortex.voxy.client.VoxyClientInstance;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.commonImpl.VoxyCommon;
import net.minecraft.client.multiplayer.ClientPacketListener;
import net.minecraft.network.protocol.game.ClientboundLoginPacket;

View File

@@ -2,7 +2,7 @@ package me.cortex.voxy.client.mixin.minecraft;
import com.llamalad7.mixinextras.injector.wrapmethod.WrapMethod;
import com.llamalad7.mixinextras.sugar.Local;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;
import net.minecraft.client.Camera;
import net.minecraft.client.DeltaTracker;

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client.mixin.minecraft;
import me.cortex.voxy.client.VoxyClientInstance;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;
import me.cortex.voxy.client.core.VoxyRenderSystem;
import me.cortex.voxy.client.core.util.IrisUtil;

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client.mixin.sodium;
import me.cortex.voxy.client.compat.SemaphoreBlockImpersonator;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.common.thread.MultiThreadPrioritySemaphore;
import me.cortex.voxy.commonImpl.VoxyCommon;
import org.spongepowered.asm.mixin.Mixin;

View File

@@ -2,7 +2,7 @@ package me.cortex.voxy.client.mixin.sodium;
import com.llamalad7.mixinextras.injector.v2.WrapWithCondition;
import com.llamalad7.mixinextras.injector.wrapoperation.WrapOperation;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;
import me.cortex.voxy.commonImpl.VoxyCommon;
import net.caffeinemc.mods.sodium.client.render.chunk.RenderSection;

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.client.mixin.sodium;
import me.cortex.voxy.client.ICheekyClientChunkCache;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.config.VoxyConfig;
import me.cortex.voxy.client.core.IGetVoxyRenderSystem;
import me.cortex.voxy.client.core.VoxyRenderSystem;
import me.cortex.voxy.common.world.service.VoxelIngestService;

View File

@@ -113,6 +113,12 @@ public class Serialization {
if (clzName.contains("VoxyConfigScreenPages")) {
continue;//Dont want to modmenu incase it doesnt exist
}
if (clzName.contains("VoxyConfigMenu")) {
continue;//Dont want to modmenu incase it doesnt exist
}
if (clzName.contains("SodiumConfigBuilder")) {
continue;//Dont want to sodium incase it doesnt exist
}
if (clzName.endsWith("VoxyConfig")) {
continue;//Special case to prevent recursive loading pain
}

View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.client.config;
package me.cortex.voxy.common.config;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;

View File

@@ -0,0 +1,85 @@
package me.cortex.voxy.common.config;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.commonImpl.VoxyCommon;
import net.fabricmc.loader.api.FabricLoader;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Modifier;
import java.nio.file.Files;
import java.nio.file.Path;
public class VoxyServerConfig {
private static final Gson GSON = new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.setPrettyPrinting()
.excludeFieldsWithModifiers(Modifier.PRIVATE)
.create();
public static VoxyServerConfig CONFIG = loadOrCreate();
public int viewDistance = 32;
public int dirtyUpdateDelay = 20; // Ticks before sending dirty update
public boolean ingestEnabled = true;
private static VoxyServerConfig loadOrCreate() {
if (VoxyCommon.isAvailable()) {
var path = getConfigPath();
if (Files.exists(path)) {
try (FileReader reader = new FileReader(path.toFile())) {
var conf = GSON.fromJson(reader, VoxyServerConfig.class);
if (conf != null) {
// conf.save(); // Don't save immediately on load, just return it. This avoids overwriting.
return conf;
} else {
Logger.error("Failed to load voxy server config (null), resetting");
}
} catch (Exception e) { // Catch all exceptions including syntax errors
Logger.error("Could not parse config, resetting to default", e);
}
}
Logger.info("Server config doesnt exist or failed to load, creating new");
var config = new VoxyServerConfig();
config.save();
return config;
} else {
return new VoxyServerConfig();
}
}
public void save() {
if (!VoxyCommon.isAvailable()) {
return;
}
try {
// Only save if file doesn't exist to prevent overwriting user changes?
// Actually, usually save() is called to persist changes.
// But if we call save() immediately after load(), it just re-formats.
// The issue user reported is "resetting to default".
// That happens if GSON.fromJson returns null or exception.
// OR if we create new instance and overwrite.
// If we want to preserve comments or formatting, we shouldn't overwrite unless needed.
// But JSON doesn't support comments.
// If the user says it "resets", maybe loadOrCreate is failing?
// loadOrCreate checks if file exists.
// Let's modify save to be safe?
Files.writeString(getConfigPath(), GSON.toJson(this));
} catch (IOException e) {
Logger.error("Failed to write config file", e);
}
}
private static Path getConfigPath() {
return FabricLoader.getInstance()
.getConfigDir()
.resolve("voxy-server-config.json");
}
}

View File

@@ -5,7 +5,7 @@ import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;
import net.jpountz.lz4.LZ4Factory;
import org.lwjgl.system.MemoryUtil;
import me.cortex.voxy.common.util.UnsafeUtil;
public class LZ4Compressor implements StorageCompressor {
private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
@@ -20,7 +20,7 @@ public class LZ4Compressor implements StorageCompressor {
@Override
public MemoryBuffer compress(MemoryBuffer saveData) {
var res = new MemoryBuffer(this.compressor.maxCompressedLength((int) saveData.size)+4);
MemoryUtil.memPutInt(res.address, (int) saveData.size);
UnsafeUtil.putInt(res.address, (int) saveData.size);
int size = this.compressor.compress(saveData.asByteBuffer(), 0, (int) saveData.size, res.asByteBuffer(), 4, (int) res.size-4);
return res.subSize(size+4);
}
@@ -28,7 +28,7 @@ public class LZ4Compressor implements StorageCompressor {
@Override
public MemoryBuffer decompress(MemoryBuffer saveData) {
var res = SCRATCH.get().createUntrackedUnfreeableReference();
int size = this.decompressor.decompress(saveData.asByteBuffer(), 4, res.asByteBuffer(), 0, MemoryUtil.memGetInt(saveData.address));
int size = this.decompressor.decompress(saveData.asByteBuffer(), 4, res.asByteBuffer(), 0, UnsafeUtil.getInt(saveData.address));
return res.subSize(size);
}

View File

@@ -1,34 +1,14 @@
package me.cortex.voxy.common.config.compressors;
import com.github.luben.zstd.Zstd;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;
import static me.cortex.voxy.common.util.GlobalCleaner.CLEANER;
import static org.lwjgl.util.zstd.Zstd.*;
import java.nio.ByteBuffer;
public class ZSTDCompressor implements StorageCompressor {
private record Ref(long ptr) {}
private static Ref createCleanableCompressionContext() {
long ctx = ZSTD_createCCtx();
var ref = new Ref(ctx);
CLEANER.register(ref, ()->ZSTD_freeCCtx(ctx));
return ref;
}
private static Ref createCleanableDecompressionContext() {
long ctx = ZSTD_createDCtx();
nZSTD_DCtx_setParameter(ctx, ZSTD_d_experimentalParam3, 1);//experimental ZSTD_d_forceIgnoreChecksum
var ref = new Ref(ctx);
CLEANER.register(ref, ()->ZSTD_freeDCtx(ctx));
return ref;
}
private static final ThreadLocal<Ref> COMPRESSION_CTX = ThreadLocal.withInitial(ZSTDCompressor::createCleanableCompressionContext);
private static final ThreadLocal<Ref> DECOMPRESSION_CTX = ThreadLocal.withInitial(ZSTDCompressor::createCleanableDecompressionContext);
private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
private final int level;
@@ -39,16 +19,37 @@ public class ZSTDCompressor implements StorageCompressor {
@Override
public MemoryBuffer compress(MemoryBuffer saveData) {
MemoryBuffer compressedData = new MemoryBuffer((int)ZSTD_COMPRESSBOUND(saveData.size));
long compressedSize = nZSTD_compressCCtx(COMPRESSION_CTX.get().ptr, compressedData.address, compressedData.size, saveData.address, saveData.size, this.level);
long bound = Zstd.compressBound(saveData.size);
if (bound > Integer.MAX_VALUE) {
// MemoryBuffer supports long size but ByteBuffer is limited to int.
// If we are using DirectByteBuffer wrappers, we are limited to 2GB.
// Voxy sections are usually small (KB/MBs).
throw new IllegalStateException("Compression bound too large for ByteBuffer: " + bound);
}
MemoryBuffer compressedData = new MemoryBuffer(bound);
ByteBuffer src = saveData.asByteBuffer();
ByteBuffer dst = compressedData.asByteBuffer();
// Zstd-jni compress returns size or error code
long compressedSize = Zstd.compress(dst, src, this.level);
if (Zstd.isError(compressedSize)) {
throw new RuntimeException("Zstd compression failed: " + Zstd.getErrorName(compressedSize));
}
return compressedData.subSize(compressedSize);
}
@Override
public MemoryBuffer decompress(MemoryBuffer saveData) {
var decompressed = SCRATCH.get().createUntrackedUnfreeableReference();
long size = nZSTD_decompressDCtx(DECOMPRESSION_CTX.get().ptr, decompressed.address, decompressed.size, saveData.address, saveData.size);
//TODO:FIXME: DONT ASSUME IT DOESNT FAIL
ByteBuffer dst = decompressed.asByteBuffer();
ByteBuffer src = saveData.asByteBuffer();
long size = Zstd.decompress(dst, src);
if (Zstd.isError(size)) {
throw new RuntimeException("Zstd decompression failed: " + Zstd.getErrorName(size));
}
return decompressed.subSize(size);
}

View File

@@ -9,9 +9,9 @@ import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import net.minecraft.world.level.levelgen.RandomSupport;
import org.apache.commons.lang3.stream.Streams;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.function.LongConsumer;
@@ -85,11 +85,12 @@ public class MemoryStorageBackend extends StorageBackend {
@Override
public void putIdMapping(int id, ByteBuffer data) {
synchronized (this.idMappings) {
var cpy = MemoryUtil.memAlloc(data.remaining());
MemoryUtil.memCopy(data, cpy);
long addr = UnsafeUtil.allocateMemory(data.remaining());
var cpy = UnsafeUtil.createDirectByteBuffer(addr, data.remaining());
UnsafeUtil.memcpy(data, cpy);
var prev = this.idMappings.put(id, cpy);
if (prev != null) {
MemoryUtil.memFree(prev);
UnsafeUtil.freeMemory(UnsafeUtil.getAddress(prev));
}
}
}
@@ -116,7 +117,7 @@ public class MemoryStorageBackend extends StorageBackend {
@Override
public void close() {
Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryBuffer::free);
this.idMappings.values().forEach(MemoryUtil::memFree);
this.idMappings.values().forEach(b -> UnsafeUtil.freeMemory(UnsafeUtil.getAddress(b)));
}
public static class Config extends StorageConfig {

View File

@@ -7,7 +7,7 @@ import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;
//import org.lwjgl.system.MemoryUtil;
import org.lwjgl.util.lmdb.MDBVal;
import java.nio.ByteBuffer;
@@ -100,7 +100,7 @@ public class LMDBStorageBackend extends StorageBackend {
if (bb == null) {
return null;
}
UnsafeUtil.memcpy(MemoryUtil.memAddress(bb), scratch.address, bb.remaining());
UnsafeUtil.memcpy(UnsafeUtil.getAddress(bb), scratch.address, bb.remaining());
return scratch.subSize(bb.remaining());
}));
}
@@ -111,7 +111,9 @@ public class LMDBStorageBackend extends StorageBackend {
this.resizingTransaction(() -> this.sectionDatabase.transaction(transaction->{
var keyBuff = transaction.stack.malloc(8);
keyBuff.putLong(0, key);
transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0);
//transaction.put(keyBuff, MemoryUtil.memByteBuffer(data.address, (int) data.size), 0);
//Use unsafe to create a byte buffer from the address
transaction.put(keyBuff, UnsafeUtil.createDirectByteBuffer(data.address, (int) data.size), 0);
return null;
}));
}

View File

@@ -6,8 +6,9 @@ import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.world.WorldEngine;
import org.lwjgl.system.MemoryStack;
import org.lwjgl.system.MemoryUtil;
//import org.lwjgl.system.MemoryStack;
//import org.lwjgl.system.MemoryUtil;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.rocksdb.*;
import java.nio.ByteBuffer;
@@ -118,39 +119,84 @@ public class RocksDBStorageBackend extends StorageBackend {
@Override
public void iterateStoredSectionPositions(LongConsumer consumer) {
try (var stack = MemoryStack.stackPush()) {
ByteBuffer keyBuff = stack.calloc(8);
long keyBuffPtr = MemoryUtil.memAddress(keyBuff);
// try (var stack = MemoryStack.stackPush()) {
// ByteBuffer keyBuff = stack.calloc(8);
// long keyBuffPtr = MemoryUtil.memAddress(keyBuff);
// var iter = this.db.newIterator(this.worldSections, this.sectionReadOps);
// iter.seekToFirst();
// while (iter.isValid()) {
// iter.key(keyBuff);
// long key = Long.reverseBytes(MemoryUtil.memGetLong(keyBuffPtr));
// consumer.accept(key);
// iter.next();
// }
// iter.close();
// }
// Replacement using standard ByteBuffer
ByteBuffer keyBuff = ByteBuffer.allocateDirect(8);
var iter = this.db.newIterator(this.worldSections, this.sectionReadOps);
iter.seekToFirst();
while (iter.isValid()) {
keyBuff.clear();
iter.key(keyBuff);
long key = Long.reverseBytes(MemoryUtil.memGetLong(keyBuffPtr));
long key = Long.reverseBytes(keyBuff.getLong(0));
consumer.accept(key);
iter.next();
}
iter.close();
}
}
@Override
public MemoryBuffer getSectionData(long key, MemoryBuffer scratch) {
try (var stack = MemoryStack.stackPush()){
var buffer = stack.malloc(8);
//HATE JAVA HATE JAVA HATE JAVA, Long.reverseBytes()
//THIS WILL ONLY WORK ON LITTLE ENDIAN SYSTEM AAAAAAAAA ;-;
// try (var stack = MemoryStack.stackPush()){
// var buffer = stack.malloc(8);
// //HATE JAVA HATE JAVA HATE JAVA, Long.reverseBytes()
// //THIS WILL ONLY WORK ON LITTLE ENDIAN SYSTEM AAAAAAAAA ;-;
//
// MemoryUtil.memPutLong(MemoryUtil.memAddress(buffer), Long.reverseBytes(swizzlePos(key)));
//
// var result = this.db.get(this.worldSections,
// this.sectionReadOps,
// buffer,
// MemoryUtil.memByteBuffer(scratch.address, (int) (scratch.size)));
//
// if (result == RocksDB.NOT_FOUND) {
// return null;
// }
//
// return scratch.subSize(result);
// } catch (RocksDBException e) {
// throw new RuntimeException(e);
// }
MemoryUtil.memPutLong(MemoryUtil.memAddress(buffer), Long.reverseBytes(swizzlePos(key)));
try {
byte[] keyBytes = longToBytes(swizzlePos(key)); // Using existing helper, creates garbage but safe
// Actually longToBytes does big endian?
// The original code did Long.reverseBytes(swizzlePos(key)) and wrote it to native memory.
// On Little Endian system, this means it writes Big Endian to memory?
// Wait.
// x86 is LE.
// Writing 0x11223344 to memory: 44 33 22 11.
// Long.reverseBytes(0x11223344) = 0x44332211.
// Writing 0x44332211 to memory: 11 22 33 44.
// So it writes Big Endian to memory?
// RocksDB usually sorts keys lexicographically.
// So we want Big Endian for keys if we want spatial locality to match key locality?
// Or maybe just consistent.
var result = this.db.get(this.worldSections,
this.sectionReadOps,
buffer,
MemoryUtil.memByteBuffer(scratch.address, (int) (scratch.size)));
// Let's use a byte array for key to avoid MemoryStack.
ByteBuffer keyBuff = ByteBuffer.allocateDirect(8);
keyBuff.order(java.nio.ByteOrder.nativeOrder());
keyBuff.putLong(0, Long.reverseBytes(swizzlePos(key)));
// We need a ByteBuffer for the output that wraps the scratch address.
ByteBuffer outBuff = UnsafeUtil.createDirectByteBuffer(scratch.address, (int) scratch.size);
int result = this.db.get(this.worldSections, this.sectionReadOps, keyBuff, outBuff);
if (result == RocksDB.NOT_FOUND) {
return null;
}
return scratch.subSize(result);
} catch (RocksDBException e) {
throw new RuntimeException(e);
@@ -160,9 +206,18 @@ public class RocksDBStorageBackend extends StorageBackend {
//TODO: FIXME, use the ByteBuffer variant
@Override
public void setSectionData(long key, MemoryBuffer data) {
try (var stack = MemoryStack.stackPush()) {
var keyBuff = stack.calloc(8);
MemoryUtil.memPutLong(MemoryUtil.memAddress(keyBuff), Long.reverseBytes(swizzlePos(key)));
// try (var stack = MemoryStack.stackPush()) {
// var keyBuff = stack.calloc(8);
// MemoryUtil.memPutLong(MemoryUtil.memAddress(keyBuff), Long.reverseBytes(swizzlePos(key)));
// this.db.put(this.worldSections, this.sectionWriteOps, keyBuff, data.asByteBuffer());
// } catch (RocksDBException e) {
// throw new RuntimeException(e);
// }
try {
ByteBuffer keyBuff = ByteBuffer.allocateDirect(8);
keyBuff.order(java.nio.ByteOrder.nativeOrder());
keyBuff.putLong(0, Long.reverseBytes(swizzlePos(key)));
this.db.put(this.worldSections, this.sectionWriteOps, keyBuff, data.asByteBuffer());
} catch (RocksDBException e) {
throw new RuntimeException(e);

View File

@@ -0,0 +1,162 @@
package me.cortex.voxy.common.network;
import me.cortex.voxy.common.voxelization.VoxelizedSection;
import me.cortex.voxy.common.world.other.Mapper;
import net.minecraft.network.FriendlyByteBuf;
import net.minecraft.util.SimpleBitStorage;
import net.minecraft.util.Mth;
import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.state.BlockState;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class SectionSerializer {
public static void serialize(VoxelizedSection section, Mapper mapper, FriendlyByteBuf buf) {
// 1. Scan and build palette
Map<BlockState, Integer> blockPaletteMap = new HashMap<>();
List<BlockState> blockPalette = new ArrayList<>();
Map<String, Integer> biomePaletteMap = new HashMap<>();
List<String> biomePalette = new ArrayList<>();
long[] data = section.section;
int count = data.length;
// Pass 1: Build Palette
for (long l : data) {
int blockId = Mapper.getBlockId(l);
BlockState state = mapper.getBlockStateFromBlockId(blockId);
if (!blockPaletteMap.containsKey(state)) {
blockPaletteMap.put(state, blockPalette.size());
blockPalette.add(state);
}
int biomeId = Mapper.getBiomeId(l);
String biome = mapper.getBiomeString(biomeId);
if (!biomePaletteMap.containsKey(biome)) {
biomePaletteMap.put(biome, biomePalette.size());
biomePalette.add(biome);
}
}
// 2. Write Palettes
buf.writeVarInt(blockPalette.size());
for (BlockState state : blockPalette) {
buf.writeVarInt(Block.BLOCK_STATE_REGISTRY.getId(state));
}
buf.writeVarInt(biomePalette.size());
for (String biome : biomePalette) {
buf.writeUtf(biome);
}
// 3. Prepare BitStorages
int blockBits = Math.max(1, Mth.ceillog2(blockPalette.size()));
int biomeBits = Math.max(1, Mth.ceillog2(biomePalette.size()));
SimpleBitStorage blockStorage = new SimpleBitStorage(blockBits, count);
SimpleBitStorage biomeStorage = new SimpleBitStorage(biomeBits, count);
byte[] lightStorage = new byte[count];
// 4. Fill Data
for (int i = 0; i < count; i++) {
long l = data[i];
int blockId = Mapper.getBlockId(l);
BlockState state = mapper.getBlockStateFromBlockId(blockId);
blockStorage.set(i, blockPaletteMap.get(state));
int biomeId = Mapper.getBiomeId(l);
String biome = mapper.getBiomeString(biomeId);
biomeStorage.set(i, biomePaletteMap.get(biome));
lightStorage[i] = (byte) Mapper.getLightId(l);
}
// 5. Write Data
buf.writeLongArray(blockStorage.getRaw());
buf.writeLongArray(biomeStorage.getRaw());
buf.writeByteArray(lightStorage);
// Write extra metadata from VoxelizedSection
buf.writeInt(section.x);
buf.writeInt(section.y);
buf.writeInt(section.z);
buf.writeInt(section.lvl0NonAirCount);
}
public static VoxelizedSection deserialize(FriendlyByteBuf buf, Mapper mapper) {
// 1. Read Palettes
int blockPaletteSize = buf.readVarInt();
BlockState[] blockPalette = new BlockState[blockPaletteSize];
for (int i = 0; i < blockPaletteSize; i++) {
blockPalette[i] = Block.BLOCK_STATE_REGISTRY.byId(buf.readVarInt());
}
int biomePaletteSize = buf.readVarInt();
String[] biomePalette = new String[biomePaletteSize];
for (int i = 0; i < biomePaletteSize; i++) {
biomePalette[i] = buf.readUtf();
}
// 2. Read Data
long[] blockRaw = buf.readLongArray();
long[] biomeRaw = buf.readLongArray();
byte[] lightRaw = buf.readByteArray();
int count = lightRaw.length;
int blockBits = Math.max(1, Mth.ceillog2(blockPaletteSize));
int biomeBits = Math.max(1, Mth.ceillog2(biomePaletteSize));
SimpleBitStorage blockStorage = new SimpleBitStorage(blockBits, count, blockRaw);
SimpleBitStorage biomeStorage = new SimpleBitStorage(biomeBits, count, biomeRaw);
long[] sectionData = new long[count];
// 3. Reconstruct VoxelizedSection
// We need to resolve to LOCAL IDs using the mapper
int[] blockLocalIds = new int[blockPaletteSize];
for (int i = 0; i < blockPaletteSize; i++) {
blockLocalIds[i] = mapper.getIdForBlockState(blockPalette[i]);
}
// For biomes, mapper needs "Holder<Biome>".
// But Mapper also has "registerNewBiome(String)".
// We can access it via getIdForBiome(Holder) but we only have String.
// We should add getIdForBiome(String) to Mapper.
// Or assume we can just use the string.
// Mapper.registerNewBiome(String) is private.
// Mapper.getIdForBiome(Holder) calls registerNewBiome.
// We need a public method `int getIdForBiome(String id)`.
// I will assume I added it or will add it.
int[] biomeLocalIds = new int[biomePaletteSize];
for (int i = 0; i < biomePaletteSize; i++) {
biomeLocalIds[i] = mapper.getIdForBiome(biomePalette[i]);
}
for (int i = 0; i < count; i++) {
int blockIdx = blockStorage.get(i);
int biomeIdx = biomeStorage.get(i);
int light = lightRaw[i] & 0xFF;
int localBlockId = blockLocalIds[blockIdx];
int localBiomeId = biomeLocalIds[biomeIdx]; // This will be 0 for now
sectionData[i] = Mapper.composeMappingId((byte)light, localBlockId, localBiomeId);
}
VoxelizedSection section = new VoxelizedSection(sectionData);
section.x = buf.readInt();
section.y = buf.readInt();
section.z = buf.readInt();
section.lvl0NonAirCount = buf.readInt();
return section;
}
}

View File

@@ -0,0 +1,62 @@
package me.cortex.voxy.common.network;
import io.netty.buffer.Unpooled;
import me.cortex.voxy.common.voxelization.VoxelizedSection;
import me.cortex.voxy.common.world.other.Mapper;
import net.fabricmc.fabric.api.networking.v1.PayloadTypeRegistry;
import net.minecraft.network.FriendlyByteBuf;
import net.minecraft.network.codec.StreamCodec;
import net.minecraft.network.protocol.common.custom.CustomPacketPayload;
import net.minecraft.resources.Identifier;
public class VoxyNetwork {
public static final Identifier CONFIG_SYNC_ID = Identifier.parse("voxy:config_sync");
public static final Identifier LOD_UPDATE_ID = Identifier.parse("voxy:lod_update");
public static void register() {
PayloadTypeRegistry.playS2C().register(ConfigSyncPayload.TYPE, ConfigSyncPayload.CODEC);
PayloadTypeRegistry.playS2C().register(LodUpdatePayload.TYPE, LodUpdatePayload.CODEC);
}
public record ConfigSyncPayload(int viewDistance) implements CustomPacketPayload {
public static final Type<ConfigSyncPayload> TYPE = new Type<>(CONFIG_SYNC_ID);
public static final StreamCodec<FriendlyByteBuf, ConfigSyncPayload> CODEC = StreamCodec.of(
(buf, payload) -> buf.writeInt(payload.viewDistance),
(buf) -> new ConfigSyncPayload(buf.readInt())
);
@Override
public Type<? extends CustomPacketPayload> type() {
return TYPE;
}
}
public record LodUpdatePayload(byte[] data) implements CustomPacketPayload {
public static final Type<LodUpdatePayload> TYPE = new Type<>(LOD_UPDATE_ID);
public static final StreamCodec<FriendlyByteBuf, LodUpdatePayload> CODEC = StreamCodec.of(
(buf, payload) -> buf.writeByteArray(payload.data),
(buf) -> new LodUpdatePayload(buf.readByteArray())
);
@Override
public Type<? extends CustomPacketPayload> type() {
return TYPE;
}
public static LodUpdatePayload create(VoxelizedSection section, Mapper mapper) {
FriendlyByteBuf buf = new FriendlyByteBuf(Unpooled.buffer());
SectionSerializer.serialize(section, mapper, buf);
byte[] bytes = new byte[buf.writerIndex()];
buf.readBytes(bytes);
buf.release();
return new LodUpdatePayload(bytes);
}
public VoxelizedSection deserialize(Mapper mapper) {
FriendlyByteBuf buf = new FriendlyByteBuf(Unpooled.wrappedBuffer(this.data));
VoxelizedSection section = SectionSerializer.deserialize(buf, mapper);
buf.release();
return section;
}
}
}

View File

@@ -16,12 +16,20 @@ public class UnifiedServiceThreadPool {
private final List<Thread> threads = new ArrayList<>();
private int threadId = 0;
private final Service sharedExecutorService;
private final java.util.Queue<Runnable> sharedExecutorQueue = new java.util.concurrent.ConcurrentLinkedQueue<>();
public UnifiedServiceThreadPool() {
this.dedicatedPool = new ThreadGroup("Voxy Dedicated Service");
this.serviceManager = new ServiceManager(this::release);
this.groupSemaphore = new MultiThreadPrioritySemaphore(this.serviceManager::tryRunAJob);
this.selfBlock = this.groupSemaphore.createBlock();
this.sharedExecutorService = this.serviceManager.createServiceNoCleanup(() -> () -> {
Runnable r = this.sharedExecutorQueue.poll();
if (r != null) r.run();
}, 1, "SharedExecutor");
}
private final void release(int i) {this.groupSemaphore.pooledRelease(i);}
@@ -81,6 +89,11 @@ public class UnifiedServiceThreadPool {
this.selfBlock.free();
}
public void execute(Runnable runnable) {
this.sharedExecutorQueue.add(runnable);
this.sharedExecutorService.execute();
}
public static void main(String[] args) {
var ustp = new UnifiedServiceThreadPool();

View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.util;
import me.cortex.voxy.commonImpl.VoxyCommon;
import org.lwjgl.system.MemoryUtil;
//import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
@@ -20,7 +20,8 @@ public class MemoryBuffer extends TrackedObject {
public MemoryBuffer(long size) {
this(true, MemoryUtil.nmemAlloc(size), size, true);
//this(true, MemoryUtil.nmemAlloc(size), size, true);
this(true, UnsafeUtil.allocateMemory(size), size, true);
}
private MemoryBuffer(boolean track, long address, long size, boolean freeable) {
@@ -56,7 +57,8 @@ public class MemoryBuffer extends TrackedObject {
COUNT.decrementAndGet();
}
if (this.freeable) {
MemoryUtil.nmemFree(this.address);
//MemoryUtil.nmemFree(this.address);
UnsafeUtil.freeMemory(this.address);
TOTAL_SIZE.addAndGet(-this.size);
} else {
throw new IllegalArgumentException("Tried to free unfreeable buffer");
@@ -88,12 +90,14 @@ public class MemoryBuffer extends TrackedObject {
}
public MemoryBuffer zero() {
MemoryUtil.memSet(this.address, 0, this.size);
//MemoryUtil.memSet(this.address, 0, this.size);
UnsafeUtil.setMemory(this.address, this.size, (byte)0);
return this;
}
public ByteBuffer asByteBuffer() {
return MemoryUtil.memByteBuffer(this.address, (int) this.size);
//return MemoryUtil.memByteBuffer(this.address, (int) this.size);
return UnsafeUtil.createDirectByteBuffer(this.address, (int) this.size);
}
//TODO: create like Long(offset) -> value at offset

View File

@@ -6,11 +6,25 @@ import java.lang.reflect.Field;
public class UnsafeUtil {
private static final Unsafe UNSAFE;
private static final long BUFFER_ADDRESS_OFFSET;
private static final long BUFFER_CAPACITY_OFFSET;
private static final long BUFFER_LIMIT_OFFSET;
private static final Class<?> DIRECT_BYTE_BUFFER_CLASS;
static {
try {
Field field = Unsafe.class.getDeclaredField("theUnsafe");
field.setAccessible(true);
UNSAFE = (Unsafe)field.get(null);
Field addr = java.nio.Buffer.class.getDeclaredField("address");
BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addr);
Field cap = java.nio.Buffer.class.getDeclaredField("capacity");
BUFFER_CAPACITY_OFFSET = UNSAFE.objectFieldOffset(cap);
Field lim = java.nio.Buffer.class.getDeclaredField("limit");
BUFFER_LIMIT_OFFSET = UNSAFE.objectFieldOffset(lim);
DIRECT_BYTE_BUFFER_CLASS = Class.forName("java.nio.DirectByteBuffer");
} catch (Exception e) {throw new RuntimeException(e);}
}
@@ -47,4 +61,80 @@ public class UnsafeUtil {
public static void memcpy(short[] src, long dst) {
UNSAFE.copyMemory(src, SHORT_ARRAY_BASE_OFFSET, null, dst, (long) src.length <<1);
}
public static long allocateMemory(long size) {
return UNSAFE.allocateMemory(size);
}
public static void freeMemory(long address) {
UNSAFE.freeMemory(address);
}
public static void setMemory(long address, long bytes, byte value) {
UNSAFE.setMemory(address, bytes, value);
}
public static java.nio.ByteBuffer createDirectByteBuffer(long address, int capacity) {
try {
java.nio.ByteBuffer bb = (java.nio.ByteBuffer) UNSAFE.allocateInstance(DIRECT_BYTE_BUFFER_CLASS);
UNSAFE.putLong(bb, BUFFER_ADDRESS_OFFSET, address);
UNSAFE.putInt(bb, BUFFER_CAPACITY_OFFSET, capacity);
UNSAFE.putInt(bb, BUFFER_LIMIT_OFFSET, capacity);
bb.order(java.nio.ByteOrder.nativeOrder());
return bb;
} catch (InstantiationException e) {
throw new RuntimeException("Failed to create DirectByteBuffer via allocateInstance", e);
}
}
public static void memcpy(java.nio.ByteBuffer src, java.nio.ByteBuffer dst) {
if (!src.isDirect() || !dst.isDirect()) {
throw new IllegalArgumentException("Buffers must be direct");
}
if (src.remaining() > dst.remaining()) {
throw new IllegalArgumentException("Source buffer is larger than destination buffer");
}
long srcAddr = getAddress(src) + src.position();
long dstAddr = getAddress(dst) + dst.position();
UNSAFE.copyMemory(srcAddr, dstAddr, src.remaining());
}
public static long getAddress(java.nio.ByteBuffer buffer) {
if (!buffer.isDirect()) {
throw new IllegalArgumentException("Buffer must be direct");
}
return UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET);
}
public static void putLong(long address, long value) {
UNSAFE.putLong(address, value);
}
public static long getLong(long address) {
return UNSAFE.getLong(address);
}
public static void putInt(long address, int value) {
UNSAFE.putInt(address, value);
}
public static int getInt(long address) {
return UNSAFE.getInt(address);
}
public static void putShort(long address, short value) {
UNSAFE.putShort(address, value);
}
public static short getShort(long address) {
return UNSAFE.getShort(address);
}
public static void putByte(long address, byte value) {
UNSAFE.putByte(address, value);
}
public static byte getByte(long address) {
return UNSAFE.getByte(address);
}
}

View File

@@ -5,7 +5,6 @@ import com.sun.jna.platform.win32.WinNT;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.ThreadUtils;
import org.lwjgl.system.Platform;
import oshi.SystemInfo;
import java.util.Arrays;
@@ -13,6 +12,9 @@ import java.util.Random;
//Represents the layout of the current cpu running on
public class CpuLayout {
private static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase().contains("win");
private static final boolean IS_LINUX = System.getProperty("os.name").toLowerCase().contains("linux");
private CpuLayout(){}
public static void setThreadAffinity(Core... cores) {
@@ -24,8 +26,7 @@ public class CpuLayout {
}
public static void setThreadAffinity(Affinity... affinities) {
var platform = Platform.get();
if (platform == Platform.WINDOWS) {
if (IS_WINDOWS) {
long[] msks = new long[affinities.length];
short[] groups = new short[affinities.length];Arrays.fill(groups, (short) -1);
int i = 0;
@@ -36,7 +37,7 @@ public class CpuLayout {
msks[idx] |= a.msk;
}
ThreadUtils.SetThreadSelectedCpuSetMasksWin32(Arrays.copyOf(msks, i), Arrays.copyOf(groups, i));
} else if (platform == Platform.LINUX) {
} else if (IS_LINUX) {
Arrays.sort(affinities, (a, b) -> a.group - b.group);
long[] msks = new long[affinities.length];
for (int i=0; i<affinities.length; i++) {
@@ -128,9 +129,9 @@ public class CpuLayout {
public static final Core[] CORES;
static {
if (Platform.get() == Platform.WINDOWS) {
if (IS_WINDOWS) {
CORES = generateCoreLayoutWindows();
} else if (Platform.get() == Platform.LINUX) {
} else if (IS_LINUX) {
CORES = generateCoreLayoutLinux();
} else {
CORES = null;

View File

@@ -153,27 +153,14 @@ public class WorldConversionFactory {
int nonZeroCnt = 0;
if (blockContainer.data.storage instanceof SimpleBitStorage bStor) {
var bDat = bStor.getRaw();
int iterPerLong = (64 / bStor.getBits()) - 1;
int MSK = (1 << bStor.getBits()) - 1;
int eBits = bStor.getBits();
long sample = 0;
int c = 0;
int dec = 0;
for (int i = 0; i <= 0xFFF; i++) {
if (dec-- == 0) {
sample = bDat[c++];
dec = iterPerLong;
}
for (int i = 0; i < 4096; i++) {
int paletteIndex = bStor.get(i);
int bId;
if (bps == null) {
bId = pc[Math.min((int) (sample & MSK), pcc)];
bId = pc[paletteIndex];
} else {
bId = stateMapper.getIdForBlockState(bps.valueFor((int) (sample&MSK)));
bId = stateMapper.getIdForBlockState(bps.valueFor(paletteIndex));
}
sample >>>= eBits;
byte light = lightSupplier.supply(i&0xF, (i>>8)&0xF, (i>>4)&0xF);
nonZeroCnt += (bId != 0)?1:0;

View File

@@ -6,7 +6,6 @@ import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.commonImpl.VoxyCommon;
import org.lwjgl.system.MemoryUtil;
public class SaveLoadSystem {
public static final boolean VERIFY_HASH_ON_LOAD = VoxyCommon.isVerificationFlagOn("verifySectionHash");
@@ -65,18 +64,18 @@ public class SaveLoadSystem {
long ptr = raw.address;
long hash = section.key^(lutIndex*1293481298141L);
MemoryUtil.memPutLong(ptr, section.key); ptr += 8;
UnsafeUtil.putLong(ptr, section.key); ptr += 8;
long metadata = 0;
metadata |= Byte.toUnsignedLong(section.nonEmptyChildren);
MemoryUtil.memPutLong(ptr, metadata); ptr += 8;
UnsafeUtil.putLong(ptr, metadata); ptr += 8;
hash ^= metadata; hash *= 1242629872171L;
MemoryUtil.memPutInt(ptr, lutIndex); ptr += 4;
UnsafeUtil.putInt(ptr, lutIndex); ptr += 4;
for (int i = 0; i < lutIndex; i++) {
long id = lutValues[i];
MemoryUtil.memPutLong(ptr, id); ptr += 8;
UnsafeUtil.putLong(ptr, id); ptr += 8;
hash *= 1230987149811L;
hash += 12831;
hash ^= id;
@@ -85,19 +84,19 @@ public class SaveLoadSystem {
UnsafeUtil.memcpy(compressed, ptr); ptr += compressed.length*2L;
MemoryUtil.memPutLong(ptr, hash); ptr += 8;
UnsafeUtil.putLong(ptr, hash); ptr += 8;
return raw.subSize(ptr-raw.address);
}
public static boolean deserialize(WorldSection section, MemoryBuffer data) {
long ptr = data.address;
long key = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
long key = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
long metadata = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
long metadata = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
section.nonEmptyChildren = (byte) (metadata&0xFF);
int lutLen = MemoryUtil.memGetInt(ptr); ptr += 4; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
int lutLen = UnsafeUtil.getInt(ptr); ptr += 4; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
if (lutLen > 32*32*32) {
throw new IllegalStateException("lutLen impossibly large, max size should be 32768 but got size " + lutLen);
}
@@ -109,7 +108,7 @@ public class SaveLoadSystem {
hash ^= metadata; hash *= 1242629872171L;
}
for (int i = 0; i < lutLen; i++) {
lut [i] = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
lut [i] = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
if (VERIFY_HASH_ON_LOAD) {
hash *= 1230987149811L;
hash += 12831;
@@ -125,7 +124,7 @@ public class SaveLoadSystem {
int nonEmptyBlockCount = 0;
for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) {
long state = lut[Short.toUnsignedInt(MemoryUtil.memGetShort(ptr))]; ptr += 2; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
long state = lut[Short.toUnsignedInt(UnsafeUtil.getShort(ptr))]; ptr += 2; if (VERIFY_MEMORY_ACCESS && data.size<=(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
nonEmptyBlockCount += Mapper.isAir(state)?0:1;
section.data[z2lin(i)] = state;
}
@@ -141,7 +140,7 @@ public class SaveLoadSystem {
}
hash ^= pHash;
long expectedHash = MemoryUtil.memGetLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
long expectedHash = UnsafeUtil.getLong(ptr); ptr += 8; if (VERIFY_MEMORY_ACCESS && data.size<(ptr-data.address)) throw new IllegalStateException("Memory access OOB");
if (expectedHash != hash) {
//throw new IllegalStateException("Hash mismatch got: " + hash + " expected: " + expectedHash);
Logger.error("Hash mismatch got: " + hash + " expected: " + expectedHash + " removing region");

View File

@@ -6,7 +6,6 @@ import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.other.Mapper;
import net.minecraft.util.Mth;
import org.lwjgl.system.MemoryUtil;
import java.util.Arrays;
@@ -97,8 +96,8 @@ public class SaveLoadSystem2 {
var res = new MemoryBuffer(32*32*32*8+1024);
long ptr = res.address;
MemoryUtil.memPutLong(ptr, section.key); ptr += 8;
MemoryUtil.memPutLong(ptr, hash); ptr += 8;
UnsafeUtil.putLong(ptr, section.key); ptr += 8;
UnsafeUtil.putLong(ptr, hash); ptr += 8;
int meta = 0;
meta |= allSameBlockLight?1:0;
@@ -106,22 +105,22 @@ public class SaveLoadSystem2 {
meta |= (biomeMapping.size()-1)<<2;//512 max size
meta |= (blockMapping.size()-1)<<11;//4096 max size
meta |= Byte.toUnsignedInt(section.nonEmptyChildren) << 23;
MemoryUtil.memPutInt(ptr, meta); ptr += 4;
UnsafeUtil.putInt(ptr, meta); ptr += 4;
//Encode lighting
/*
//Micro storage optimization, not done cause makes decode very slightly slower and more pain
if (allSameBlockLight && allSameSkyLight) {
MemoryUtil.memPutByte(ptr, (byte) ((blockLight[0]&0xF)|((skyLight[0]&0xF)<<4))); ptr += 1;
UnsafeUtil.putByte(ptr, (byte) ((blockLight[0]&0xF)|((skyLight[0]&0xF)<<4))); ptr += 1;
} else*/
{
if (allSameBlockLight) {
MemoryUtil.memPutByte(ptr, (byte) (blockLight[0] & 0xF)); ptr += 1;
UnsafeUtil.putByte(ptr, (byte) (blockLight[0] & 0xF)); ptr += 1;
} else {
UnsafeUtil.memcpy(blockLight, ptr); ptr += blockLight.length;
}
if (allSameSkyLight) {
MemoryUtil.memPutByte(ptr, (byte) (skyLight[0] & 0xF)); ptr += 1;
UnsafeUtil.putByte(ptr, (byte) (skyLight[0] & 0xF)); ptr += 1;
} else {
UnsafeUtil.memcpy(skyLight, ptr); ptr += skyLight.length;
}
@@ -138,7 +137,7 @@ public class SaveLoadSystem2 {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
UnsafeUtil.putInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
@@ -154,7 +153,7 @@ public class SaveLoadSystem2 {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
UnsafeUtil.putInt(ptr, batch); ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
} else {
@@ -163,7 +162,7 @@ public class SaveLoadSystem2 {
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
UnsafeUtil.putInt(ptr, batch); ptr += 4;
}
}
@@ -179,7 +178,7 @@ public class SaveLoadSystem2 {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
UnsafeUtil.putInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
@@ -188,7 +187,7 @@ public class SaveLoadSystem2 {
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
UnsafeUtil.putInt(ptr, batch); ptr += 4;
}
}
{//Store biome
@@ -204,7 +203,7 @@ public class SaveLoadSystem2 {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
UnsafeUtil.putInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
@@ -213,7 +212,7 @@ public class SaveLoadSystem2 {
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
UnsafeUtil.putInt(ptr, batch); ptr += 4;
}
}
}
@@ -225,13 +224,13 @@ public class SaveLoadSystem2 {
var cache = DESERIALIZE_CACHE.get();
long ptr = data.address;
long pos = MemoryUtil.memGetLong(ptr); ptr += 8;
long pos = UnsafeUtil.getLong(ptr); ptr += 8;
if (section.key != pos) {
Logger.error("Section pos not the same as requested, got " + pos + " expected " + section.key);
return false;
}
long chash = MemoryUtil.memGetLong(ptr); ptr += 8;
int meta = MemoryUtil.memGetInt(ptr); ptr += 4;
long chash = UnsafeUtil.getLong(ptr); ptr += 8;
int meta = UnsafeUtil.getInt(ptr); ptr += 4;
boolean allSameBlockLight = (meta&1)!=0;
boolean allSameSkyLight = (meta&2)!=0;
@@ -244,12 +243,12 @@ public class SaveLoadSystem2 {
if (allSameBlockLight) {
//shift up 4 so that its already in correct position
blockLight = Byte.toUnsignedLong(MemoryUtil.memGetByte(ptr))<<4; ptr += 1;
blockLight = Byte.toUnsignedLong(UnsafeUtil.getByte(ptr))<<4; ptr += 1;
} else {
blockLight = ptr; ptr += 32*32*32/2;
}
if (allSameSkyLight) {
skyLight = MemoryUtil.memGetByte(ptr); ptr += 1;
skyLight = UnsafeUtil.getByte(ptr); ptr += 1;
} else {
skyLight = ptr; ptr += 32*32*32/2;
}
@@ -258,7 +257,7 @@ public class SaveLoadSystem2 {
int[] biomeLut = new int[biomeMapSize];
{//Deserialize the block and biome mappings
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int batch = UnsafeUtil.getInt(ptr); ptr += 4;
{//Block
int SIZE = 20;// 20 bits per entry
int msk = (1<<SIZE)-1;
@@ -266,7 +265,7 @@ public class SaveLoadSystem2 {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
batch = UnsafeUtil.getInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
@@ -281,7 +280,7 @@ public class SaveLoadSystem2 {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
batch = UnsafeUtil.getInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
@@ -297,13 +296,13 @@ public class SaveLoadSystem2 {
{//Block
final int SIZE = Mth.smallestEncompassingPowerOfTwo(Mth.log2(Mth.smallestEncompassingPowerOfTwo(blockMapSize)));
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int batch = UnsafeUtil.getInt(ptr); ptr += 4;
int msk = (1<<SIZE)-1;
for (int i = 0; i < blocks.length; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
batch = UnsafeUtil.getInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
@@ -317,13 +316,13 @@ public class SaveLoadSystem2 {
} else {
final int SIZE = Mth.smallestEncompassingPowerOfTwo(Mth.log2(Mth.smallestEncompassingPowerOfTwo(biomeMapSize)));
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int batch = UnsafeUtil.getInt(ptr); ptr += 4;
int msk = (1<<SIZE)-1;
for (int i = 0; i < biomes.length; i++) {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
batch = UnsafeUtil.getInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;
@@ -342,13 +341,13 @@ public class SaveLoadSystem2 {
light |= (byte) (blockLight&0xF0);
} else {
//Todo clean and optimize this (it can be optimized alot)
light |= (byte) (((Byte.toUnsignedInt(MemoryUtil.memGetByte(blockLight+(i>>1)))>>((i&1)*4))&0xF)<<4);
light |= (byte) (((Byte.toUnsignedInt(UnsafeUtil.getByte(blockLight+(i>>1)))>>((i&1)*4))&0xF)<<4);
}
if (allSameSkyLight) {
light |= (byte) (skyLight&0xF);
} else {
//Todo clean and optimize this (it can be optimized alot)
light |= (byte) ((Byte.toUnsignedInt(MemoryUtil.memGetByte(skyLight+(i>>1)))>>((i&1)*4))&0xF);
light |= (byte) ((Byte.toUnsignedInt(UnsafeUtil.getByte(skyLight+(i>>1)))>>((i&1)*4))&0xF);
}
}
@@ -386,7 +385,7 @@ public class SaveLoadSystem2 {
if (rem != 0)
batch |= (b << (32 - rem));//the shift does auto cutoff
if (rem < SIZE) {
MemoryUtil.memPutInt(ptr, batch);
UnsafeUtil.putInt(ptr, batch);
ptr += 4;
batch = b >> rem;
rem = 32 - (SIZE - rem);
@@ -396,7 +395,7 @@ public class SaveLoadSystem2 {
}
}
if (rem != 32) {
MemoryUtil.memPutInt(ptr, batch); ptr += 4;
UnsafeUtil.putInt(ptr, batch); ptr += 4;
}
System.err.println(ptr-aa.address);
}
@@ -405,7 +404,7 @@ public class SaveLoadSystem2 {
{
long ptr = aa.address;
int rem = 32;
int batch = MemoryUtil.memGetInt(ptr); ptr += 4;
int batch = UnsafeUtil.getInt(ptr); ptr += 4;
{//Block
int SIZE = 20;// 20 bits per entry
int msk = (1<<SIZE)-1;
@@ -413,7 +412,7 @@ public class SaveLoadSystem2 {
int val = batch&msk;
batch >>>= SIZE; rem -= SIZE;
if (rem < 0) {
batch = MemoryUtil.memGetInt(ptr); ptr += 4;
batch = UnsafeUtil.getInt(ptr); ptr += 4;
val |= (batch&((1<<-rem)-1))<<(SIZE+rem);
batch >>>= -rem;
rem = 32+rem;

View File

@@ -5,7 +5,7 @@ import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.other.Mapper;
import org.lwjgl.system.MemoryUtil;
import me.cortex.voxy.common.util.UnsafeUtil;
public class SaveLoadSystem3 {
public static final int STORAGE_VERSION = 0;
@@ -44,7 +44,7 @@ public class SaveLoadSystem3 {
MemoryBuffer buffer = cache.memoryBuffer().createUntrackedUnfreeableReference();
long ptr = buffer.address;
MemoryUtil.memPutLong(ptr, section.key); ptr += 8;
UnsafeUtil.putLong(ptr, section.key); ptr += 8;
long metadataPtr = ptr; ptr += 8;
long blockPtr = ptr; ptr += WorldSection.SECTION_VOLUME*2;
@@ -52,9 +52,9 @@ public class SaveLoadSystem3 {
short mapping = LUT.putIfAbsent(block, (short) LUT.size());
if (mapping == -1) {
mapping = (short) (LUT.size()-1);
MemoryUtil.memPutLong(ptr, block); ptr+=8;
UnsafeUtil.putLong(ptr, block); ptr+=8;
}
MemoryUtil.memPutShort(blockPtr, mapping); blockPtr+=2;
UnsafeUtil.putShort(blockPtr, mapping); blockPtr+=2;
}
if (LUT.size() >= 1<<16) {
throw new IllegalStateException();
@@ -66,7 +66,7 @@ public class SaveLoadSystem3 {
metadata |= Byte.toUnsignedLong(section.getNonEmptyChildren())<<16;//Next byte
//5 bytes free
MemoryUtil.memPutLong(metadataPtr, metadata);
UnsafeUtil.putLong(metadataPtr, metadata);
//TODO: do hash
//TODO: rework the storage system to not need to do useless copies like this (this is an issue for serialization, deserialization has solved this already)
@@ -75,7 +75,7 @@ public class SaveLoadSystem3 {
public static boolean deserialize(WorldSection section, MemoryBuffer data) {
long ptr = data.address;
long key = MemoryUtil.memGetLong(ptr); ptr += 8;
long key = UnsafeUtil.getLong(ptr); ptr += 8;
if (section.key != key) {
//throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
@@ -83,15 +83,15 @@ public class SaveLoadSystem3 {
return false;
}
final long metadata = MemoryUtil.memGetLong(ptr); ptr += 8;
final long metadata = UnsafeUtil.getLong(ptr); ptr += 8;
section.nonEmptyChildren = (byte) ((metadata>>>16)&0xFF);
final long lutBasePtr = ptr + WorldSection.SECTION_VOLUME * 2;
if (section.lvl == 0) {
int nonEmptyBlockCount = 0;
final var blockData = section.data;
for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) {
final short lutId = MemoryUtil.memGetShort(ptr); ptr += 2;
final long blockId = MemoryUtil.memGetLong(lutBasePtr + Short.toUnsignedLong(lutId) * 8L);
final short lutId = UnsafeUtil.getShort(ptr); ptr += 2;
final long blockId = UnsafeUtil.getLong(lutBasePtr + Short.toUnsignedLong(lutId) * 8L);
nonEmptyBlockCount += Mapper.isAir(blockId) ? 0 : 1;
blockData[i] = blockId;
}
@@ -99,7 +99,7 @@ public class SaveLoadSystem3 {
} else {
final var blockData = section.data;
for (int i = 0; i < WorldSection.SECTION_VOLUME; i++) {
blockData[i] = MemoryUtil.memGetLong(lutBasePtr + Short.toUnsignedLong(MemoryUtil.memGetShort(ptr)) * 8L);ptr += 2;
blockData[i] = UnsafeUtil.getLong(lutBasePtr + Short.toUnsignedLong(UnsafeUtil.getShort(ptr)) * 8L);ptr += 2;
}
}
ptr = lutBasePtr + (metadata & 0xFFFF) * 8L;

View File

@@ -18,7 +18,7 @@ import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.Blocks;
import net.minecraft.world.level.block.LeavesBlock;
import net.minecraft.world.level.block.state.BlockState;
import org.lwjgl.system.MemoryUtil;
//import org.lwjgl.system.MemoryUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -194,11 +194,11 @@ public class Mapper {
this.blockLock.unlock();
byte[] serialized = entry.serialize();
ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length);
ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length);
buffer.put(serialized);
buffer.rewind();
this.storage.putIdMapping(entry.id | (BLOCK_STATE_TYPE<<30), buffer);
MemoryUtil.memFree(buffer);
//MemoryUtil.memFree(buffer); //Not needed for standard direct buffers, GC handles it
if (this.newStateCallback!=null)this.newStateCallback.accept(entry);
return entry;
@@ -217,11 +217,11 @@ public class Mapper {
this.biomeLock.unlock();
byte[] serialized = entry.serialize();
ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length);
ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length);
buffer.put(serialized);
buffer.rewind();
this.storage.putIdMapping(entry.id | (BIOME_TYPE<<30), buffer);
MemoryUtil.memFree(buffer);
//MemoryUtil.memFree(buffer);
if (this.newBiomeCallback!=null)this.newBiomeCallback.accept(entry);
return entry;
@@ -257,6 +257,14 @@ public class Mapper {
return this.blockId2stateEntry.get(blockId).opacity;
}
public int getIdForBiome(String biomeId) {
var entry = this.biome2biomeEntry.get(biomeId);
if (entry == null) {
entry = this.registerNewBiome(biomeId);
}
return entry.id;
}
public int getIdForBiome(Holder<Biome> biome) {
String biomeId = biome.unwrapKey().get().identifier().toString();
var entry = this.biome2biomeEntry.get(biomeId);
@@ -318,11 +326,11 @@ public class Mapper {
throw new IllegalStateException("State Id NOT THE SAME, very critically bad. arr:" + this.blockId2stateEntry.indexOf(entry) + " entry: " + entry.id);
}
byte[] serialized = entry.serialize();
ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length);
ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length);
buffer.put(serialized);
buffer.rewind();
this.storage.putIdMapping(entry.id | (BLOCK_STATE_TYPE<<30), buffer);
MemoryUtil.memFree(buffer);
//MemoryUtil.memFree(buffer);
}
for (var entry : biomes) {
@@ -331,11 +339,11 @@ public class Mapper {
}
byte[] serialized = entry.serialize();
ByteBuffer buffer = MemoryUtil.memAlloc(serialized.length);
ByteBuffer buffer = ByteBuffer.allocateDirect(serialized.length);
buffer.put(serialized);
buffer.rewind();
this.storage.putIdMapping(entry.id | (BIOME_TYPE<<30), buffer);
MemoryUtil.memFree(buffer);
//MemoryUtil.memFree(buffer);
}
this.storage.flush();
@@ -346,6 +354,11 @@ public class Mapper {
}
public String getBiomeString(int id) {
if (id < 0 || id >= this.biomeId2biomeEntry.size()) return "minecraft:plains"; // Default fallback
return this.biomeId2biomeEntry.get(id).biome;
}
public static final class StateEntry {
public final int id;
public final BlockState state;

View File

@@ -25,13 +25,18 @@ public class VoxelIngestService {
private final Service service;
private record IngestSection(int cx, int cy, int cz, WorldEngine world, LevelChunkSection section, DataLayer blockLight, DataLayer skyLight){}
private final ConcurrentLinkedDeque<IngestSection> ingestQueue = new ConcurrentLinkedDeque<>();
private static final int MAX_QUEUE_SIZE = 16384;
private int dropCount = 0;
public VoxelIngestService(ServiceManager pool) {
this.service = pool.createServiceNoCleanup(()->this::processJob, 5000, "Ingest service");
}
private void processJob() {
var task = this.ingestQueue.pop();
var task = this.ingestQueue.poll();
if (task == null) return;
try {
task.world.markActive();
var section = task.section;
@@ -50,6 +55,9 @@ public class VoxelIngestService {
WorldConversionFactory.mipSection(csec, task.world.getMapper());
WorldUpdater.insertUpdate(task.world, csec);
}
} catch (Exception e) {
Logger.error("Error ingesting section " + task.cx + ", " + task.cy + ", " + task.cz, e);
}
}
@NotNull
@@ -88,6 +96,13 @@ public class VoxelIngestService {
}
public boolean enqueueIngest(WorldEngine engine, LevelChunk chunk) {
if (this.ingestQueue.size() > MAX_QUEUE_SIZE) {
// Queue full, drop ingest request to prevent OOM.
if ((this.dropCount++ % 1000) == 0) {
Logger.warn("VoxelIngestService queue full (" + this.ingestQueue.size() + "), dropping chunks. Dropped " + this.dropCount + " so far.");
}
return false;
}
if (!this.service.isLive()) {
return false;
}

View File

@@ -44,7 +44,24 @@ public class VoxyCommon implements ModInitializer {
@Override
public void onInitialize() {
me.cortex.voxy.common.network.VoxyNetwork.register();
if (IS_DEDICATED_SERVER) {
net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents.SERVER_STARTING.register(server -> {
VoxyCommon.setInstanceFactory(() -> new me.cortex.voxy.server.VoxyServerInstance(server));
VoxyCommon.createInstance();
});
net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents.SERVER_STOPPING.register(server -> {
VoxyCommon.shutdownInstance();
});
net.fabricmc.fabric.api.event.lifecycle.v1.ServerChunkEvents.CHUNK_UNLOAD.register((world, chunk) -> {
if (me.cortex.voxy.common.config.VoxyServerConfig.CONFIG.ingestEnabled && chunk instanceof net.minecraft.world.level.chunk.LevelChunk levelChunk) {
me.cortex.voxy.common.world.service.VoxelIngestService.tryIngestChunk(me.cortex.voxy.commonImpl.WorldIdentifier.of(world), levelChunk);
}
});
}
}
public interface IInstanceFactory {VoxyInstance create();}

View File

@@ -23,6 +23,9 @@ public abstract class VoxyInstance {
private final Thread worldCleaner;
public final BooleanSupplier savingServiceRateLimiter;//Can run if this returns true
protected final UnifiedServiceThreadPool threadPool;
//public UnifiedServiceThreadPool getThreadPool() {
// return this.threadPool;
//}
protected final SectionSavingService savingService;
protected final VoxelIngestService ingestService;
@@ -165,6 +168,8 @@ public abstract class VoxyInstance {
protected abstract SectionStorage createStorage(WorldIdentifier identifier);
protected void onWorldCreated(WorldEngine world) {}
private WorldEngine createWorld(WorldIdentifier identifier) {
if (!this.isRunning) {
throw new IllegalStateException("Cannot create world while not running");
@@ -176,6 +181,7 @@ public abstract class VoxyInstance {
var world = new WorldEngine(this.createStorage(identifier), this);
world.setSaveCallback(this.savingService::enqueueSave);
this.activeWorlds.put(identifier, world);
this.onWorldCreated(world);
return world;
}

View File

@@ -21,8 +21,8 @@ import net.minecraft.world.level.block.Block;
import net.minecraft.world.level.block.Blocks;
import net.minecraft.world.level.block.state.BlockState;
import org.apache.commons.io.IOUtils;
import org.lwjgl.system.MemoryUtil;
import org.lwjgl.util.zstd.Zstd;
import me.cortex.voxy.common.util.UnsafeUtil;
import com.github.luben.zstd.Zstd;
import org.tukaani.xz.BasicArrayCache;
import org.tukaani.xz.ResettableArrayCache;
import org.tukaani.xz.XZInputStream;
@@ -76,7 +76,6 @@ public class DHImporter implements IDataImporter {
private ByteBuffer zstdScratch;
private ByteBuffer zstdScratch2;
private final long zstdDCtx;
public WorkCTX(PreparedStatement stmt, int worldHeight) {
this.stmt = stmt;
@@ -84,14 +83,12 @@ public class DHImporter implements IDataImporter {
this.storageCache = new long[64*16*worldHeight];
this.colScratch = new byte[1<<16];
this.section = VoxelizedSection.createEmpty();
this.zstdDCtx = Zstd.ZSTD_createDCtx();
}
public void free() {
if (this.zstdScratch != null) {
MemoryUtil.memFree(this.zstdScratch);
MemoryUtil.memFree(this.zstdScratch2);
Zstd.ZSTD_freeDCtx(this.zstdDCtx);
UnsafeUtil.freeMemory(UnsafeUtil.getAddress(this.zstdScratch));
UnsafeUtil.freeMemory(UnsafeUtil.getAddress(this.zstdScratch2));
}
}
}
@@ -298,30 +295,32 @@ public class DHImporter implements IDataImporter {
return new XZInputStream(IOUtils.toBufferedInputStream(in), -1, false, ctx.cache);
} else if (decompressor == 4) {
if (ctx.zstdScratch == null) {
ctx.zstdScratch = MemoryUtil.memAlloc(8196);
ctx.zstdScratch2 = MemoryUtil.memAlloc(8196);
ctx.zstdScratch = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(8196), 8196);
ctx.zstdScratch2 = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(8196), 8196);
}
ctx.zstdScratch.clear();
ctx.zstdScratch2.clear();
try(var channel = Channels.newChannel(in)) {
while (IOUtils.read(channel, ctx.zstdScratch) == 0) {
var newBuffer = MemoryUtil.memAlloc(ctx.zstdScratch.position()*2);
int newSize = ctx.zstdScratch.position()*2;
var newBuffer = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(newSize), newSize);
newBuffer.put(ctx.zstdScratch.rewind());
MemoryUtil.memFree(ctx.zstdScratch);
UnsafeUtil.freeMemory(UnsafeUtil.getAddress(ctx.zstdScratch));
ctx.zstdScratch = newBuffer;
}
}
ctx.zstdScratch.limit(ctx.zstdScratch.position()).rewind();
{
int decompSize = (int) Zstd.ZSTD_getFrameContentSize(ctx.zstdScratch);
int decompSize = (int) Zstd.decompressedSize(ctx.zstdScratch);
if (ctx.zstdScratch2.capacity() < decompSize) {
MemoryUtil.memFree(ctx.zstdScratch2);
ctx.zstdScratch2 = MemoryUtil.memAlloc((int) (decompSize * 1.1));
UnsafeUtil.freeMemory(UnsafeUtil.getAddress(ctx.zstdScratch2));
int newSize = (int) (decompSize * 1.1);
ctx.zstdScratch2 = UnsafeUtil.createDirectByteBuffer(UnsafeUtil.allocateMemory(newSize), newSize);
}
}
long size = Zstd.ZSTD_decompressDCtx(ctx.zstdDCtx, ctx.zstdScratch, ctx.zstdScratch2);
if (Zstd.ZSTD_isError(size)) {
throw new IllegalStateException("ZSTD EXCEPTION: " + Zstd.ZSTD_getErrorName(size));
long size = Zstd.decompress(ctx.zstdScratch2, ctx.zstdScratch);
if (Zstd.isError(size)) {
throw new IllegalStateException("ZSTD EXCEPTION: " + Zstd.getErrorName(size));
}
ctx.zstdScratch2.limit((int) size);
return new ByteBufferBackedInputStream(ctx.zstdScratch2);

View File

@@ -30,7 +30,7 @@ import net.minecraft.world.level.chunk.status.ChunkStatus;
import net.minecraft.world.level.chunk.storage.RegionFileVersion;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.lwjgl.system.MemoryUtil;
import me.cortex.voxy.common.util.UnsafeUtil;
import java.io.DataInputStream;
import java.io.File;
@@ -334,7 +334,7 @@ public class WorldImporter implements IDataImporter {
return;
}
for (int idx = 0; idx < 1024; idx++) {
int sectorMeta = Integer.reverseBytes(MemoryUtil.memGetInt(regionFile.address+idx*4));//Assumes little endian
int sectorMeta = Integer.reverseBytes(UnsafeUtil.getInt(regionFile.address+idx*4));//Assumes little endian
if (sectorMeta == 0) {
//Empty chunk
continue;
@@ -355,8 +355,8 @@ public class WorldImporter implements IDataImporter {
{
long base = regionFile.address + sectorStart * 4096L;
int chunkLen = sectorCount * 4096;
int m = Integer.reverseBytes(MemoryUtil.memGetInt(base));
byte b = MemoryUtil.memGetByte(base + 4L);
int m = Integer.reverseBytes(UnsafeUtil.getInt(base));
byte b = UnsafeUtil.getByte(base + 4L);
if (m == 0) {
Logger.error("Chunk is allocated, but stream is missing");
} else {
@@ -408,7 +408,7 @@ public class WorldImporter implements IDataImporter {
private long offset = 0;
@Override
public int read() {
return MemoryUtil.memGetByte(data.address + (this.offset++)) & 0xFF;
return UnsafeUtil.getByte(data.address + (this.offset++)) & 0xFF;
}
@Override

View File

@@ -0,0 +1,124 @@
package me.cortex.voxy.server;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.VoxyServerConfig;
import me.cortex.voxy.common.network.VoxyNetwork;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.commonImpl.WorldIdentifier;
import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking;
import net.minecraft.server.level.ServerPlayer;
import net.minecraft.world.level.ChunkPos;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
public class DirtyUpdateService {
private final VoxyServerInstance instance;
private final ConcurrentLinkedQueue<DirtySection> dirtyQueue = new ConcurrentLinkedQueue<>();
private final ConcurrentHashMap<WorldIdentifier, List<DirtySection>> batchedUpdates = new ConcurrentHashMap<>();
public DirtyUpdateService(VoxyServerInstance instance) {
this.instance = instance;
}
private record DirtySection(WorldEngine world, WorldSection section, int updateFlags, int neighborMsk) {}
public void onSectionDirty(WorldEngine world, WorldSection section, int updateFlags, int neighborMsk) {
// Filter out sections that are not important for clients or debounce
if ((updateFlags & WorldEngine.UPDATE_TYPE_BLOCK_BIT) != 0) {
// We need to keep the section alive until we process it
section.acquire();
this.dirtyQueue.add(new DirtySection(world, section, updateFlags, neighborMsk));
// Logger.info("Section dirty: " + section.lvl + " " + section.x + " " + section.y + " " + section.z);
}
}
public void tick() {
// Process queue
DirtySection dirty;
int processed = 0;
// Logger.info("Dirty Queue Size: " + dirtyQueue.size());
while ((dirty = this.dirtyQueue.poll()) != null && processed++ < 1000) {
processDirty(dirty);
dirty.section.release();
}
}
private void processDirty(DirtySection dirty) {
// 1. Check if we need to send this to anyone
// Get players in the world
var server = this.instance.getServer();
// We need to find the world identifier for this engine
// This is tricky because WorldEngine doesn't explicitly store its Identifier in a way we can easily query back to ServerLevel
// But we can iterate server levels and match
// Actually, WorldIdentifier.of(ServerLevel) gives us the ID.
// We can map WorldEngine -> WorldIdentifier if we stored it.
// VoxyInstance stores Map<WorldIdentifier, WorldEngine>.
// We can iterate that? No, slow.
// Let's assume we can get the world from the engine or we pass it.
// WorldEngine doesn't have the identifier.
// But we can fix that or work around it.
// VoxyServerInstance knows the mapping.
// For now, let's look at how we can broadcast.
// We need to find players near the section.
// section.x, section.y, section.z are in LOD coordinates (lvl dependent).
int lvl = dirty.section.lvl;
int sx = dirty.section.x;
int sy = dirty.section.y;
int sz = dirty.section.z;
int scale = 1 << lvl;
int sectionSize = 16 * scale;
// Get ServerLevel.
// Issue: We don't have direct link from WorldEngine to ServerLevel.
// We can iterate server.getAllLevels() and check WorldIdentifier.of(level).getOrCreateEngine() == dirty.world
// This is slow if done every time.
// We should cache this reverse mapping in VoxyServerInstance or DirtyUpdateService.
net.minecraft.server.level.ServerLevel matchLevel = null;
for (var level : server.getAllLevels()) {
var wi = WorldIdentifier.of(level);
if (this.instance.getNullable(wi) == dirty.world) {
matchLevel = level;
break;
}
}
if (matchLevel == null) return;
for (ServerPlayer player : matchLevel.players()) {
int simulationDistance = matchLevel.getServer().getPlayerList().getSimulationDistance() * 16;
int voxyDistance = VoxyServerConfig.CONFIG.viewDistance * 16;
int standardViewDist = server.getPlayerList().getViewDistance() * 16;
int m = (1 << (lvl + 1)) - 1;
for (int dx = 0; dx <= m; dx++) {
for (int dz = 0; dz <= m; dz++) {
double centerX = (((sx << (lvl + 1)) | dx) * sectionSize) + (sectionSize / 2.0);
double centerZ = (((sz << (lvl + 1)) | dz) * sectionSize) + (sectionSize / 2.0);
double dxp = player.getX() - centerX;
double dzp = player.getZ() - centerZ;
double distSq = dxp * dxp + dzp * dzp;
if (distSq > standardViewDist * standardViewDist && distSq < voxyDistance * voxyDistance) {
for (int dy = 0; dy <= m; dy++) {
int absX = (sx << (lvl + 1)) | dx;
int absY = (sy << (lvl + 1)) | dy;
int absZ = (sz << (lvl + 1)) | dz;
var voxelized = WorldSectionToVoxelizedConverter.convert(dirty.section, lvl, absX, absY, absZ);
if (voxelized.lvl0NonAirCount > 0) {
var payload = VoxyNetwork.LodUpdatePayload.create(voxelized, dirty.world.getMapper());
ServerPlayNetworking.send(player, payload);
}
}
}
}
}
}
}
}

View File

@@ -0,0 +1,182 @@
package me.cortex.voxy.server;
import me.cortex.voxy.client.core.util.RingTracker;
import me.cortex.voxy.common.config.VoxyServerConfig;
import me.cortex.voxy.common.network.VoxyNetwork;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.commonImpl.WorldIdentifier;
import net.fabricmc.fabric.api.networking.v1.ServerPlayNetworking;
import net.minecraft.server.level.ServerLevel;
import net.minecraft.server.level.ServerPlayer;
import me.cortex.voxy.common.Logger;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
public class PlayerLodTracker {
private final VoxyServerInstance instance;
private final Map<ServerPlayer, PlayerState> playerStates = new ConcurrentHashMap<>();
public PlayerLodTracker(VoxyServerInstance instance) {
this.instance = instance;
}
private static class PlayerState {
final RingTracker tracker;
int lastX = Integer.MIN_VALUE;
int lastZ = Integer.MIN_VALUE;
PlayerState(int radius, int x, int z) {
this.tracker = new RingTracker(radius, x, z, true);
this.lastX = x;
this.lastZ = z;
}
}
public void onPlayerJoin(ServerPlayer player) {
int radius = VoxyServerConfig.CONFIG.viewDistance;
int x = player.chunkPosition().x;
int z = player.chunkPosition().z;
this.playerStates.put(player, new PlayerState(radius, x, z));
processPlayer(player, this.playerStates.get(player));
}
public void onPlayerDisconnect(ServerPlayer player) {
this.playerStates.remove(player);
}
public void tick() {
for (var entry : this.playerStates.entrySet()) {
ServerPlayer player = entry.getKey();
PlayerState state = entry.getValue();
int x = player.chunkPosition().x;
int z = player.chunkPosition().z;
// Only update if moved
if (x != state.lastX || z != state.lastZ) {
state.tracker.moveCenter(x, z);
state.lastX = x;
state.lastZ = z;
}
// Process queue (limit to N updates per tick per player to avoid flooding)
// But RingTracker accumulates operations. We need to drain them.
// The RingTracker.process method executes callbacks for add/remove.
// We want to process some amount.
processPlayer(player, state);
}
}
private void processPlayer(ServerPlayer player, PlayerState state) {
if (!(player.level() instanceof ServerLevel level)) return;
WorldIdentifier wi = WorldIdentifier.of(level);
WorldEngine engine = this.instance.getNullable(wi);
if (engine == null) return;
int standardViewDist = level.getServer().getPlayerList().getViewDistance(); // Chunks
// Use client-negotiated view distance if available (via ConfigSync or just assume Voxy Config)
// Ideally the client sends its requested Voxy distance.
// But for now, we use the server's Voxy config as the "max" distance we are willing to serve.
int maxVoxyDist = VoxyServerConfig.CONFIG.viewDistance;
// We stop sending if distance > maxVoxyDist
// RingTracker is initialized with maxVoxyDist, so it won't generate events beyond that.
// But we should double check here?
state.tracker.process(50, (x, z) -> { // Process up to 50 chunks per tick per player
// On Add (Load)
double dx = x - state.lastX;
double dz = z - state.lastZ;
double distSq = dx*dx + dz*dz;
// 1. Check if inside Vanilla View Distance (Gap filling logic)
// If dist < standardViewDist, usually client has it.
// BUT as user requested, if standardViewDist is small (e.g. 5), we need to fill 5 -> 50.
// We use standardViewDist - 1 as the "safe" boundary where we are SURE client has it.
// If it's outside that, we send LOD.
double vanillaBoundary = Math.max(0, standardViewDist - 1);
// 2. Check if inside Voxy Max Distance
// This is implicitly handled by RingTracker radius, but good to be explicit.
if (distSq > vanillaBoundary * vanillaBoundary && distSq <= maxVoxyDist * maxVoxyDist) {
sendLod(player, engine, x, z);
}
}, (x, z) -> {
// On Remove (Unload)
// We don't explicit send unload packets for LODs usually, client manages its own cache/LRU?
// Or maybe we should? Voxy client has its own tracker.
// Sending unloads might be good to free memory.
// But currently there is no S2C_LodUnload packet.
// Client's RenderDistanceTracker handles unloading when it moves.
// So we don't need to do anything here.
});
}
private void sendLod(ServerPlayer player, WorldEngine engine, int x, int z) {
// Decide LOD level based on distance?
// For now, let's just send LOD 0 if it exists.
// Optimization: Distant things can be higher LOD.
// Simple heuristic:
// 0-64 chunks (1024 blocks): L0
// >64 chunks: L1?
// Actually, Voxy client requests specific LODs usually.
// But since we are pushing, we should push what makes sense.
// Sending L0 for everything is safe but bandwidth heavy.
// Let's stick to L0 for now as a proof of concept to fix the "blank world" issue.
int lvl = 0;
// We need to acquire the section from storage.
// Warning: acquiring on main thread might lag if not in cache.
// We should probably offload this IO?
// WorldEngine.acquireIfExists loads from disk if not in memory?
// acquireIfExists(..., true) means "only return if loaded in memory"?
// No, check WorldEngine code.
// acquireIfExists(..., true) -> sectionTracker.acquire(..., true) -> "if (allowLoad) ..."
// It seems `acquireIfExists` might still load?
// Let's look at WorldEngine.java again.
// acquireIfExists calls acquire(..., true).
// ActiveSectionTracker.acquire(..., boolean load)
// If load is true, it loads.
// We want to load if it exists in DB, but not generate if missing?
// Voxy doesn't really "generate" on demand in the same way, it ingests.
// If it's not in DB, it's empty/air.
// We should use `engine.acquire` but we need to be careful about blocking main thread.
// Ideally we schedule a task on the Voxy thread pool to fetch and send.
this.instance.getThreadPool().execute(() -> {
try {
int minSec = player.level().getMinSectionY();
int maxSec = player.level().getMaxSectionY();
for (int y = minSec; y <= maxSec; y++) {
var sec = engine.acquire(lvl, x >> (lvl + 1), y >> (lvl + 1), z >> (lvl + 1));
if (sec != null) {
try {
var voxelized = WorldSectionToVoxelizedConverter.convert(sec, lvl, x, y, z);
if (voxelized.lvl0NonAirCount > 0) {
var payload = VoxyNetwork.LodUpdatePayload.create(voxelized, engine.getMapper());
ServerPlayNetworking.send(player, payload);
}
} finally {
sec.release();
}
} else {
//Logger.warn("Failed to acquire section " + x + ", " + y + ", " + z);
}
}
} catch (Exception e) {
Logger.error("Error sending LOD to player", e);
}
});
}
}

View File

@@ -0,0 +1,113 @@
package me.cortex.voxy.server;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.VoxyServerConfig;
import me.cortex.voxy.common.config.section.SectionStorage;
import me.cortex.voxy.common.config.section.SectionStorageConfig;
import me.cortex.voxy.common.StorageConfigUtil;
import me.cortex.voxy.commonImpl.ImportManager;
import me.cortex.voxy.commonImpl.VoxyInstance;
import me.cortex.voxy.commonImpl.WorldIdentifier;
import net.minecraft.server.MinecraftServer;
import net.minecraft.world.level.storage.LevelResource;
import java.nio.file.Path;
import me.cortex.voxy.common.thread.UnifiedServiceThreadPool;
public class VoxyServerInstance extends VoxyInstance {
private final MinecraftServer server;
private final SectionStorageConfig storageConfig;
private final Path basePath;
private final DirtyUpdateService dirtyUpdateService;
private final PlayerLodTracker playerLodTracker;
public VoxyServerInstance(MinecraftServer server) {
super();
// Ensure config is loaded/created
// VoxyServerConfig.CONFIG.save(); // Don't force save, just accessing it will load it
this.server = server;
this.basePath = server.getWorldPath(LevelResource.ROOT).resolve("voxy");
this.storageConfig = StorageConfigUtil.getCreateStorageConfig(Config.class, c->c.version==1&&c.sectionStorageConfig!=null, ()->DEFAULT_STORAGE_CONFIG, this.basePath).sectionStorageConfig;
this.setNumThreads(Math.max(2, Runtime.getRuntime().availableProcessors() / 2));
this.dirtyUpdateService = new DirtyUpdateService(this);
this.playerLodTracker = new PlayerLodTracker(this);
// Start a service to tick the dirty service
this.threadPool.serviceManager.createServiceNoCleanup(() -> this.dirtyUpdateService::tick, VoxyServerConfig.CONFIG.dirtyUpdateDelay * 50, "DirtyUpdateService");
// Start service for player tracker (run every tick or so)
this.threadPool.serviceManager.createServiceNoCleanup(() -> this.playerLodTracker::tick, 50, "PlayerLodTracker");
// Register join handler to sync config and init tracker
net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents.JOIN.register((handler, sender, s) -> {
var payload = new me.cortex.voxy.common.network.VoxyNetwork.ConfigSyncPayload(VoxyServerConfig.CONFIG.viewDistance);
sender.sendPacket(payload);
this.playerLodTracker.onPlayerJoin(handler.player);
});
net.fabricmc.fabric.api.networking.v1.ServerPlayConnectionEvents.DISCONNECT.register((handler, server1) -> {
this.playerLodTracker.onPlayerDisconnect(handler.player);
});
// Register chunk load event for bulk ingest
net.fabricmc.fabric.api.event.lifecycle.v1.ServerChunkEvents.CHUNK_LOAD.register((world, chunk) -> {
if (VoxyServerConfig.CONFIG.ingestEnabled) {
me.cortex.voxy.common.world.service.VoxelIngestService.tryAutoIngestChunk(chunk);
}
});
net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents.END_SERVER_TICK.register(server1 -> {
this.dirtyUpdateService.tick();
this.playerLodTracker.tick();
});
}
@Override
protected void onWorldCreated(me.cortex.voxy.common.world.WorldEngine world) {
world.setDirtyCallback((section, updateFlags, neighborMsk) -> this.dirtyUpdateService.onSectionDirty(world, section, updateFlags, neighborMsk));
}
@Override
protected SectionStorage createStorage(WorldIdentifier identifier) {
var ctx = new ConfigBuildCtx();
ctx.setProperty(ConfigBuildCtx.BASE_SAVE_PATH, this.basePath.toString());
ctx.setProperty(ConfigBuildCtx.WORLD_IDENTIFIER, identifier.getWorldId());
ctx.pushPath(ConfigBuildCtx.DEFAULT_STORAGE_PATH);
return this.storageConfig.build(ctx);
}
@Override
protected ImportManager createImportManager() {
return new ImportManager();
}
@Override
public boolean isIngestEnabled(WorldIdentifier worldId) {
return VoxyServerConfig.CONFIG.ingestEnabled;
}
private static class Config {
public int version = 1;
public SectionStorageConfig sectionStorageConfig;
}
private static final Config DEFAULT_STORAGE_CONFIG;
static {
var config = new Config();
config.sectionStorageConfig = StorageConfigUtil.createDefaultSerializer();
DEFAULT_STORAGE_CONFIG = config;
}
public MinecraftServer getServer() {
return server;
}
@Override
public UnifiedServiceThreadPool getThreadPool() {
return this.threadPool;
}
}

View File

@@ -0,0 +1,52 @@
package me.cortex.voxy.server;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.common.voxelization.VoxelizedSection;
final class WorldSectionToVoxelizedConverter {
static VoxelizedSection convert(WorldSection src, int lvl, int absX, int absY, int absZ) {
long[] sdat = src._unsafeGetRawDataArray();
VoxelizedSection out = VoxelizedSection.createEmpty().setPosition(absX, absY, absZ);
long[] vdat = out.section;
int msk = (1 << (lvl + 1)) - 1;
int bx = (absX & msk) << (4 - lvl);
int by = (absY & msk) << (4 - lvl);
int bz = (absZ & msk) << (4 - lvl);
int baseSec = bx | (bz << 5) | (by << 10);
if (lvl == 0) {
final int secMsk = 0b1100 | (0xF << 5) | (0xF << 10);
final int iSecMsk1 = (~secMsk) + 1;
int secIdx = 0;
int nonAir = 0;
for (int i = 0; i <= 0xFFF; i += 4) {
int cSecIdx = secIdx + baseSec;
vdat[i + 0] = sdat[cSecIdx + 0];
vdat[i + 1] = sdat[cSecIdx + 1];
vdat[i + 2] = sdat[cSecIdx + 2];
vdat[i + 3] = sdat[cSecIdx + 3];
nonAir += Mapper.isAir(vdat[i + 0]) ? 0 : 1;
nonAir += Mapper.isAir(vdat[i + 1]) ? 0 : 1;
nonAir += Mapper.isAir(vdat[i + 2]) ? 0 : 1;
nonAir += Mapper.isAir(vdat[i + 3]) ? 0 : 1;
secIdx = (secIdx + iSecMsk1) & secMsk;
}
out.lvl0NonAirCount = nonAir;
} else {
int baseVIdx = VoxelizedSection.getBaseIndexForLevel(lvl);
int secMsk = 0xF >> lvl;
secMsk |= (secMsk << 5) | (secMsk << 10);
int iSecMsk1 = (~secMsk) + 1;
int secIdx = 0;
int end = (0xFFF >> (lvl * 3)) + baseVIdx;
for (int i = baseVIdx; i <= end; i++) {
int cSecIdx = secIdx + baseSec;
vdat[i] = sdat[cSecIdx];
secIdx = (secIdx + iSecMsk1) & secMsk;
}
}
return out;
}
}

View File

@@ -0,0 +1,43 @@
package me.cortex.voxy.server.mixin;
import me.cortex.voxy.common.config.VoxyServerConfig;
import me.cortex.voxy.common.world.service.VoxelIngestService;
import me.cortex.voxy.commonImpl.WorldIdentifier;
import net.minecraft.core.BlockPos;
import net.minecraft.core.SectionPos;
import net.minecraft.server.level.ServerLevel;
import net.minecraft.world.level.LightLayer;
import net.minecraft.world.level.block.state.BlockState;
import org.spongepowered.asm.mixin.Mixin;
import org.spongepowered.asm.mixin.injection.At;
import org.spongepowered.asm.mixin.injection.Inject;
import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
import net.minecraft.world.level.Level;
@Mixin(Level.class)
public class MixinServerLevel {
@Inject(method = "setBlock(Lnet/minecraft/core/BlockPos;Lnet/minecraft/world/level/block/state/BlockState;I)Z", at = @At("RETURN"))
private void voxy$onSetBlock(BlockPos pos, BlockState newState, int flags, CallbackInfoReturnable<Boolean> cir) {
if (!((Object)this instanceof ServerLevel)) return;
if (!cir.getReturnValue()) return;
if (!VoxyServerConfig.CONFIG.ingestEnabled) return;
ServerLevel level = (ServerLevel) (Object) this;
WorldIdentifier wi = WorldIdentifier.of(level);
if (wi == null) return;
SectionPos sp = SectionPos.of(pos);
var chunk = level.getChunk(sp.x(), sp.z());
int sectionY = sp.y();
if (sectionY < chunk.getMinSectionY() || sectionY > chunk.getMaxSectionY()) return;
var section = chunk.getSection(chunk.getSectionIndex(sectionY));
var lp = level.getLightEngine();
var blp = lp.getLayerListener(LightLayer.BLOCK).getDataLayerData(sp);
var slp = lp.getLayerListener(LightLayer.SKY).getDataLayerData(sp);
VoxelIngestService.rawIngest(wi, section, sp.x(), sp.y(), sp.z(), blp==null?null:blp.copy(), slp==null?null:slp.copy());
}
}

View File

@@ -40,7 +40,9 @@
"depends": {
"minecraft": ["1.21.11"],
"fabricloader": ">=0.14.22",
"fabric-api": ">=0.91.1",
"fabric-api": ">=0.91.1"
},
"suggests": {
"sodium": "=0.8.2"
},
"accessWidener": "voxy.accesswidener"