spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor storage to include a section backend

Browse Source
This commit is contained in:
mcrcortex
2025-02-05 11:15:44 +10:00
parent db43e26b8a
commit cf0afbb545
38 changed files with 252 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/me/cortex/voxy/client/Voxy.java
View File

@@ -3,19 +3,9 @@ package me.cortex.voxy.client;
import me.cortex.voxy.client.core.VoxelCore;
import me.cortex.voxy.client.saver.ContextSelectionSystem;
import me.cortex.voxy.client.terrain.WorldImportCommand;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.Serialization;
import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
import me.cortex.voxy.common.storage.config.StorageConfig;
import net.caffeinemc.mods.sodium.client.compatibility.environment.OsUtils;
import net.fabricmc.api.ClientModInitializer;
import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback;
import net.fabricmc.loader.api.FabricLoader;
import net.fabricmc.loader.api.ModContainer;
import net.minecraft.client.world.ClientWorld;
import org.apache.commons.lang3.SystemUtils;
import java.util.Arrays;
public class Voxy implements ClientModInitializer {
@Override

20
src/main/java/me/cortex/voxy/client/core/VoxelCore.java
View File

@@ -4,12 +4,15 @@ import com.mojang.blaze3d.systems.RenderSystem;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.client.core.gl.Capabilities;
import me.cortex.voxy.client.core.gl.GlBuffer;
import me.cortex.voxy.client.core.model.ColourDepthTextureData;
import me.cortex.voxy.client.core.model.ModelBakerySubsystem;
import me.cortex.voxy.client.core.model.ModelTextureBakery;
import me.cortex.voxy.client.core.rendering.*;
import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4;
import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
import me.cortex.voxy.client.core.rendering.post.PostProcessing;
import me.cortex.voxy.client.core.rendering.util.DownloadStream;
import me.cortex.voxy.client.core.rendering.util.RawDownloadStream;
import me.cortex.voxy.client.core.util.IrisUtil;
import me.cortex.voxy.client.saver.ContextSelectionSystem;
import me.cortex.voxy.client.taskbar.Taskbar;
@@ -21,6 +24,7 @@ import me.cortex.voxy.common.thread.ServiceThreadPool;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.commonImpl.VoxyCommon;
import net.minecraft.block.Blocks;
import net.minecraft.client.MinecraftClient;
import net.minecraft.client.gui.hud.ClientBossBar;
import net.minecraft.client.render.Camera;
@@ -33,6 +37,7 @@ import net.minecraft.world.World;
import net.minecraft.world.chunk.WorldChunk;
import org.joml.Matrix4f;
import org.lwjgl.opengl.GL11;
import org.lwjgl.system.MemoryUtil;
import java.io.File;
import java.util.*;
@@ -127,7 +132,19 @@ public class VoxelCore {
).mulLocal(makeProjectionMatrix(16, 16*3000));
}
//private static final ModelTextureBakery mtb = new ModelTextureBakery(16, 16);
//private static final RawDownloadStream downstream = new RawDownloadStream(1<<20);
public void renderOpaque(MatrixStack matrices, double cameraX, double cameraY, double cameraZ) {
/*
int allocation = downstream.download(2*4*6*16*16, ptr->{
});
mtb.renderFacesToStream(Blocks.WHITE_STAINED_GLASS.getDefaultState(), 123456, false, downstream.getBufferId(), allocation);
downstream.submit();
downstream.tick();
*/
//if (true) return;
if (IrisUtil.irisShadowActive()) {
return;
}
@@ -187,8 +204,11 @@ public class VoxelCore {
this.postProcessing.renderPost(projection, RenderSystem.getProjectionMatrix(), boundFB);
}
public void addDebugInfo(List<String> debug) {
debug.add("");
debug.add("");

33
src/main/java/me/cortex/voxy/client/saver/ContextSelectionSystem.java
View File

@@ -1,13 +1,15 @@
package me.cortex.voxy.client.saver;
import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.section.SectionStorageConfig;
import me.cortex.voxy.common.config.section.SectionSerializationStorage;
import me.cortex.voxy.common.config.section.SectionStorage;
import me.cortex.voxy.common.config.compressors.ZSTDCompressor;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.Serialization;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.storage.other.CompressionStorageAdaptor;
import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend;
import me.cortex.voxy.common.config.storage.other.CompressionStorageAdaptor;
import me.cortex.voxy.common.config.storage.rocksdb.RocksDBStorageBackend;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.common.thread.ServiceThreadPool;
import net.minecraft.client.MinecraftClient;
@@ -26,7 +28,7 @@ public class ContextSelectionSystem {
public static class WorldConfig {
public int minYOverride = Integer.MAX_VALUE;
public int maxYOverride = Integer.MIN_VALUE;
public StorageConfig storageConfig;
public SectionStorageConfig sectionStorageConfig;
}
public static final String DEFAULT_STORAGE_CONFIG;
static {
@@ -42,7 +44,10 @@ public class ContextSelectionSystem {
compression.delegate = baseDB;
compression.compressor = compressor;
config.storageConfig = compression;
var serializer = new SectionSerializationStorage.Config();
serializer.storage = compression;
config.sectionStorageConfig = serializer;
DEFAULT_STORAGE_CONFIG = Serialization.GSON.toJson(config);
if (Serialization.GSON.fromJson(DEFAULT_STORAGE_CONFIG, WorldConfig.class) == null) {
@@ -71,10 +76,12 @@ public class ContextSelectionSystem {
if (this.config == null) {
throw new IllegalStateException("Config deserialization null, reverting to default");
}
if (this.config.sectionStorageConfig == null) {
throw new IllegalStateException("Config section storage null, reverting to default");
}
return;
} catch (Exception e) {
System.err.println("Failed to load the storage configuration file, resetting it to default");
e.printStackTrace();
Logger.error("Failed to load the storage configuration file, resetting it to default, this will probably break your save if you used a custom storage config", e);
}
}
@@ -89,16 +96,16 @@ public class ContextSelectionSystem {
}
}
public StorageBackend createStorageBackend() {
public SectionStorage createSectionStorageBackend() {
var ctx = new ConfigBuildCtx();
ctx.setProperty(ConfigBuildCtx.BASE_SAVE_PATH, this.selectionFolder.toString());
ctx.setProperty(ConfigBuildCtx.WORLD_IDENTIFIER, this.worldId);
ctx.pushPath(ConfigBuildCtx.DEFAULT_STORAGE_PATH);
return this.config.storageConfig.build(ctx);
return this.config.sectionStorageConfig.build(ctx);
}
public WorldEngine createEngine(ServiceThreadPool serviceThreadPool) {
return new WorldEngine(this.createStorageBackend(), serviceThreadPool, VoxyConfig.CONFIG.secondaryLruCacheSize);
return new WorldEngine(this.createSectionStorageBackend(), serviceThreadPool, VoxyConfig.CONFIG.secondaryLruCacheSize);
}
//Saves the config for the world selection or something, need to figure out how to make it work with dimensional configs maybe?

3
src/main/java/me/cortex/voxy/common/storage/config/ConfigBuildCtx.java → src/main/java/me/cortex/voxy/common/config/ConfigBuildCtx.java
View File

@@ -1,8 +1,7 @@
package me.cortex.voxy.common.storage.config;
package me.cortex.voxy.common.config;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;

12
src/main/java/me/cortex/voxy/common/config/IMappingStorage.java Normal file
View File

@@ -0,0 +1,12 @@
package me.cortex.voxy.common.config;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import java.nio.ByteBuffer;
public interface IMappingStorage {
void putIdMapping(int id, ByteBuffer data);
Int2ObjectOpenHashMap<byte[]> getIdMappingsData();
void flush();
void close();
}

2
src/main/java/me/cortex/voxy/common/config/Serialization.java
View File

@@ -4,8 +4,6 @@ import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.StorageConfig;
import net.fabricmc.loader.api.FabricLoader;
import java.io.BufferedReader;

4
src/main/java/me/cortex/voxy/common/storage/config/CompressorConfig.java → src/main/java/me/cortex/voxy/common/config/compressors/CompressorConfig.java
View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.storage.config;
package me.cortex.voxy.common.config.compressors;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.Serialization;
import me.cortex.voxy.common.storage.StorageCompressor;
public abstract class CompressorConfig {
static {

11
src/main/java/me/cortex/voxy/common/storage/compressors/LZ4Compressor.java → src/main/java/me/cortex/voxy/common/config/compressors/LZ4Compressor.java
View File

@@ -1,18 +1,11 @@
package me.cortex.voxy.common.storage.compressors;
package me.cortex.voxy.common.config.compressors;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.SaveLoadSystem;
import net.jpountz.lz4.LZ4Factory;
import org.lwjgl.system.MemoryUtil;
import org.tukaani.xz.*;
import java.io.IOException;
public class LZ4Compressor implements StorageCompressor {
private static final ThreadLocalMemoryBuffer SCRATCH = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);

9
src/main/java/me/cortex/voxy/common/storage/compressors/LZMACompressor.java → src/main/java/me/cortex/voxy/common/config/compressors/LZMACompressor.java
View File

@@ -1,21 +1,16 @@
package me.cortex.voxy.common.storage.compressors;
package me.cortex.voxy.common.config.compressors;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.SaveLoadSystem;
import org.jetbrains.annotations.NotNull;
import org.lwjgl.system.MemoryUtil;
import org.tukaani.xz.*;
import org.tukaani.xz.lzma.LZMAEncoder;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
public class LZMACompressor implements StorageCompressor {
private static final ThreadLocal<Pair<byte[], ResettableArrayCache>> CACHE_THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new byte[SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE], new ResettableArrayCache(new ArrayCache())));

2
src/main/java/me/cortex/voxy/common/storage/StorageCompressor.java → src/main/java/me/cortex/voxy/common/config/compressors/StorageCompressor.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.storage;
package me.cortex.voxy.common.config.compressors;
import me.cortex.voxy.common.util.MemoryBuffer;

6
src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java → src/main/java/me/cortex/voxy/common/config/compressors/ZSTDCompressor.java
View File

@@ -1,8 +1,6 @@
package me.cortex.voxy.common.storage.compressors;
package me.cortex.voxy.common.config.compressors;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;

83
src/main/java/me/cortex/voxy/common/config/section/SectionSerializationStorage.java Normal file
View File

@@ -0,0 +1,83 @@
package me.cortex.voxy.common.config.section;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.world.SaveLoadSystem;
import me.cortex.voxy.common.world.WorldSection;
import me.cortex.voxy.common.world.other.Mapper;
import java.nio.ByteBuffer;
import java.util.Arrays;
public class SectionSerializationStorage extends SectionStorage {
private final StorageBackend backend;
public SectionSerializationStorage(StorageBackend storageBackend) {
this.backend = storageBackend;
}
private static final ThreadLocalMemoryBuffer MEMORY_CACHE = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
public int loadSection(WorldSection into) {
var data = this.backend.getSectionData(into.key, MEMORY_CACHE.get().createUntrackedUnfreeableReference());
if (data != null) {
if (!SaveLoadSystem.deserialize(into, data)) {
this.backend.deleteSectionData(into.key);
//TODO: regenerate the section from children
Arrays.fill(into._unsafeGetRawDataArray(), Mapper.AIR);
Logger.error("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, removing");
return -1;
} else {
return 0;
}
} else {
//TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished
// the response should be put into the local db so that future data can just use that
// the server can also send arbitrary updates to the client for arbitrary lods
return 1;
}
}
@Override
public void saveSection(WorldSection section) {
var saveData = SaveLoadSystem.serialize(section);
this.backend.setSectionData(section.key, saveData);
saveData.free();
}
@Override
public void putIdMapping(int id, ByteBuffer data) {
this.backend.putIdMapping(id, data);
}
@Override
public Int2ObjectOpenHashMap<byte[]> getIdMappingsData() {
return this.backend.getIdMappingsData();
}
@Override
public void flush() {
this.backend.flush();
}
@Override
public void close() {
this.backend.close();
}
public static class Config extends SectionStorageConfig {
public StorageConfig storage;
@Override
public SectionStorage build(ConfigBuildCtx ctx) {
return new SectionSerializationStorage(this.storage.build(ctx));
}
public static String getConfigTypeName() {
return "Serializer";
}
}
}

10
src/main/java/me/cortex/voxy/common/config/section/SectionStorage.java Normal file
View File

@@ -0,0 +1,10 @@
package me.cortex.voxy.common.config.section;
import me.cortex.voxy.common.config.IMappingStorage;
import me.cortex.voxy.common.world.WorldSection;
public abstract class SectionStorage implements IMappingStorage {
public abstract int loadSection(WorldSection into);
public abstract void saveSection(WorldSection section);
}

12
src/main/java/me/cortex/voxy/common/config/section/SectionStorageConfig.java Normal file
View File

@@ -0,0 +1,12 @@
package me.cortex.voxy.common.config.section;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.Serialization;
public abstract class SectionStorageConfig {
static {
Serialization.CONFIG_TYPES.add(SectionStorageConfig.class);
}
public abstract SectionStorage build(ConfigBuildCtx ctx);
}

9
src/main/java/me/cortex/voxy/common/storage/StorageBackend.java → src/main/java/me/cortex/voxy/common/config/storage/StorageBackend.java
View File

@@ -1,6 +1,7 @@
package me.cortex.voxy.common.storage;
package me.cortex.voxy.common.config.storage;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.config.IMappingStorage;
import me.cortex.voxy.common.util.MemoryBuffer;
import java.nio.ByteBuffer;
@@ -8,7 +9,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongConsumer;
public abstract class StorageBackend {
public abstract class StorageBackend implements IMappingStorage {
public abstract void iterateStoredSectionPositions(LongConsumer consumer);
//Implementation may use the scratch buffer as the return value, it MUST NOT free the scratch buffer
@@ -18,10 +19,6 @@ public abstract class StorageBackend {
public abstract void deleteSectionData(long key);
public abstract void putIdMapping(int id, ByteBuffer data);
public abstract Int2ObjectOpenHashMap<byte[]> getIdMappingsData();
public abstract void flush();
public abstract void close();

4
src/main/java/me/cortex/voxy/common/storage/config/StorageConfig.java → src/main/java/me/cortex/voxy/common/config/storage/StorageConfig.java
View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.storage.config;
package me.cortex.voxy.common.config.storage;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.Serialization;
import me.cortex.voxy.common.storage.StorageBackend;
import java.util.ArrayList;
import java.util.List;

8
src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java → src/main/java/me/cortex/voxy/common/config/storage/inmemory/MemoryStorageBackend.java
View File

@@ -1,13 +1,13 @@
package me.cortex.voxy.common.storage.inmemory;
package me.cortex.voxy.common.config.storage.inmemory;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectCollection;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import net.minecraft.util.math.random.RandomSeed;
import org.apache.commons.lang3.stream.Streams;

4
src/main/java/me/cortex/voxy/common/storage/lmdb/Cursor.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/Cursor.java
View File

@@ -1,8 +1,8 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
import org.lwjgl.util.lmdb.MDBVal;
import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
import static me.cortex.voxy.common.config.storage.lmdb.LMDBInterface.E;
import static org.lwjgl.util.lmdb.LMDB.*;
public class Cursor implements AutoCloseable {

2
src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBInterface.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBInterface.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
import org.lwjgl.PointerBuffer;
import org.lwjgl.system.MemoryStack;

8
src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/LMDBStorageBackend.java
View File

@@ -1,10 +1,10 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import org.lwjgl.system.MemoryUtil;

2
src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionCallback.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionCallback.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
import org.lwjgl.system.MemoryStack;

2
src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionWrappedCallback.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionWrappedCallback.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
public interface TransactionWrappedCallback<T> {
T exec(TransactionWrapper wrapper);

4
src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionWrapper.java → src/main/java/me/cortex/voxy/common/config/storage/lmdb/TransactionWrapper.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.storage.lmdb;
package me.cortex.voxy.common.config.storage.lmdb;
import org.lwjgl.PointerBuffer;
import org.lwjgl.system.MemoryStack;
@@ -6,7 +6,7 @@ import org.lwjgl.util.lmdb.MDBVal;
import java.nio.ByteBuffer;
import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
import static me.cortex.voxy.common.config.storage.lmdb.LMDBInterface.E;
import static org.lwjgl.system.MemoryStack.stackPush;
import static org.lwjgl.util.lmdb.LMDB.*;

6
src/main/java/me/cortex/voxy/common/storage/other/BasicPathInsertionConfig.java → src/main/java/me/cortex/voxy/common/config/storage/other/BasicPathInsertionConfig.java
View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
//Very simple config that adds a path to the config builder
public class BasicPathInsertionConfig extends DelegateStorageConfig {

16
src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java → src/main/java/me/cortex/voxy/common/config/storage/other/CompressionStorageAdaptor.java
View File

@@ -1,16 +1,10 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.compressors.StorageCompressor;
import me.cortex.voxy.common.config.compressors.CompressorConfig;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.List;
//Compresses the section data
public class CompressionStorageAdaptor extends DelegatingStorageAdaptor {

8
src/main/java/me/cortex/voxy/common/storage/other/ConditionalStorageBackendConfig.java → src/main/java/me/cortex/voxy/common/config/storage/other/ConditionalStorageBackendConfig.java
View File

@@ -1,8 +1,8 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import org.apache.commons.lang3.NotImplementedException;
//A conditional storage backend depending on build time config, this enables conditional backends depending on the

4
src/main/java/me/cortex/voxy/common/storage/other/DelegateStorageConfig.java → src/main/java/me/cortex/voxy/common/config/storage/other/DelegateStorageConfig.java
View File

@@ -1,6 +1,6 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageConfig;
import java.util.List;

9
src/main/java/me/cortex/voxy/common/storage/other/DelegatingStorageAdaptor.java → src/main/java/me/cortex/voxy/common/config/storage/other/DelegatingStorageAdaptor.java
View File

@@ -1,13 +1,8 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.StorageCompressor;
import me.cortex.voxy.common.storage.config.CompressorConfig;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.List;

8
src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java → src/main/java/me/cortex/voxy/common/config/storage/other/FragmentedStorageBackendAdaptor.java
View File

@@ -1,11 +1,11 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import net.minecraft.util.math.random.RandomSeed;

9
src/main/java/me/cortex/voxy/common/storage/other/ReadonlyCachingLayer.java → src/main/java/me/cortex/voxy/common/config/storage/other/ReadonlyCachingLayer.java
View File

@@ -1,11 +1,10 @@
package me.cortex.voxy.common.storage.other;
package me.cortex.voxy.common.config.storage.other;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import org.lwjgl.system.MemoryUtil;
import java.nio.ByteBuffer;
import java.util.List;

8
src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java → src/main/java/me/cortex/voxy/common/config/storage/redis/RedisStorageBackend.java
View File

@@ -1,9 +1,9 @@
package me.cortex.voxy.common.storage.redis;
package me.cortex.voxy.common.config.storage.redis;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import redis.clients.jedis.JedisPool;

9
src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java → src/main/java/me/cortex/voxy/common/config/storage/rocksdb/RocksDBStorageBackend.java
View File

@@ -1,12 +1,11 @@
package me.cortex.voxy.common.storage.rocksdb;
package me.cortex.voxy.common.config.storage.rocksdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.config.storage.StorageBackend;
import me.cortex.voxy.common.config.ConfigBuildCtx;
import me.cortex.voxy.common.config.storage.StorageConfig;
import me.cortex.voxy.common.util.MemoryBuffer;
import me.cortex.voxy.common.util.UnsafeUtil;
import me.cortex.voxy.common.world.SaveLoadSystem;
import org.lwjgl.system.MemoryStack;
import org.lwjgl.system.MemoryUtil;
import org.rocksdb.*;

13
src/main/java/me/cortex/voxy/common/storage/config/GsonTypeAdaptor.java
View File

@@ -1,13 +0,0 @@
package me.cortex.voxy.common.storage.config;
import com.google.gson.Gson;
import com.google.gson.TypeAdapter;
import com.google.gson.TypeAdapterFactory;
import com.google.gson.reflect.TypeToken;
public class GsonTypeAdaptor<T> implements TypeAdapterFactory {
@Override
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
return null;
}
}

5
src/main/java/me/cortex/voxy/common/util/ThreadLocalMemoryBuffer.java
View File

@@ -1,12 +1,7 @@
package me.cortex.voxy.common.util;
import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
import java.lang.ref.Cleaner;
import static org.lwjgl.util.zstd.Zstd.ZSTD_createCCtx;
import static org.lwjgl.util.zstd.Zstd.ZSTD_freeCCtx;
public class ThreadLocalMemoryBuffer {
private static final Cleaner CLEANER = Cleaner.create();
private static MemoryBuffer createMemoryBuffer(long size) {

36
src/main/java/me/cortex/voxy/common/world/WorldEngine.java
View File

@@ -1,12 +1,12 @@
package me.cortex.voxy.common.world;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.config.section.SectionStorage;
import me.cortex.voxy.common.util.ThreadLocalMemoryBuffer;
import me.cortex.voxy.common.voxelization.VoxelizedSection;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.common.world.service.SectionSavingService;
import me.cortex.voxy.common.world.service.VoxelIngestService;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.thread.ServiceThreadPool;
import java.util.Arrays;
@@ -24,7 +24,7 @@ public class WorldEngine {
public interface ISectionChangeCallback {void accept(WorldSection section, int updateFlags);}
public final StorageBackend storage;
public final SectionStorage storage;
private final Mapper mapper;
private final ActiveSectionTracker sectionTracker;
public final VoxelIngestService ingestService;
@@ -39,43 +39,21 @@ public class WorldEngine {
public Mapper getMapper() {return this.mapper;}
public WorldEngine(StorageBackend storageBackend, ServiceThreadPool serviceThreadPool, int cacheCount) {
this(storageBackend, serviceThreadPool, MAX_LOD_LAYERS, cacheCount);
public WorldEngine(SectionStorage storage, ServiceThreadPool serviceThreadPool, int cacheCount) {
this(storage, serviceThreadPool, MAX_LOD_LAYERS, cacheCount);
}
private WorldEngine(StorageBackend storageBackend, ServiceThreadPool serviceThreadPool, int maxMipLayers, int cacheCount) {
private WorldEngine(SectionStorage storage, ServiceThreadPool serviceThreadPool, int maxMipLayers, int cacheCount) {
this.maxMipLevels = maxMipLayers;
this.storage = storageBackend;
this.storage = storage;
this.mapper = new Mapper(this.storage);
//4 cache size bits means that the section tracker has 16 separate maps that it uses
this.sectionTracker = new ActiveSectionTracker(4, this::unsafeLoadSection, cacheCount);
this.sectionTracker = new ActiveSectionTracker(4, storage::loadSection, cacheCount);
this.savingService = new SectionSavingService(this, serviceThreadPool);
this.ingestService = new VoxelIngestService(this, serviceThreadPool);
}
private static final ThreadLocalMemoryBuffer MEMORY_CACHE = new ThreadLocalMemoryBuffer(SaveLoadSystem.BIGGEST_SERIALIZED_SECTION_SIZE + 1024);
private int unsafeLoadSection(WorldSection into) {
var data = this.storage.getSectionData(into.key, MEMORY_CACHE.get().createUntrackedUnfreeableReference());
if (data != null) {
if (!SaveLoadSystem.deserialize(into, data)) {
this.storage.deleteSectionData(into.key);
//TODO: regenerate the section from children
Arrays.fill(into.data, Mapper.AIR);
Logger.error("Section " + into.lvl + ", " + into.x + ", " + into.y + ", " + into.z + " was unable to load, removing");
return -1;
} else {
return 0;
}
} else {
//TODO: if we need to fetch an lod from a server, send the request here and block until the request is finished
// the response should be put into the local db so that future data can just use that
// the server can also send arbitrary updates to the client for arbitrary lods
return 1;
}
}
public WorldSection acquireIfExists(int lvl, int x, int y, int z) {
return this.sectionTracker.acquire(lvl, x, y, z, true);
}

18
src/main/java/me/cortex/voxy/common/world/WorldSection.java
View File

@@ -80,6 +80,10 @@ public final class WorldSection {
ATOMIC_STATE_HANDLE.set(this, 1);
}
public long[] _unsafeGetRawDataArray() {
return this.data;
}
@Override
public int hashCode() {
return ((x*1235641+y)*8127451+z)*918267913+lvl;
@@ -120,7 +124,15 @@ public final class WorldSection {
}
}
if ((state>>1)==0) {
this.tracker.tryUnload(this);
if (this.tracker != null) {
this.tracker.tryUnload(this);
} else {
//This should _ONLY_ ever happen when its an untracked section
// If it is, try release it
if (this.trySetFreed()) {
this._releaseArray();
}
}
}
return state>>1;
}
@@ -239,4 +251,8 @@ public final class WorldSection {
} while (!NON_EMPTY_CHILD_HANDLE.compareAndSet(this, prev, next));
return prev != next;
}
public static WorldSection _createRawUntrackedUnsafeSection(int lvl, int x, int y, int z) {
return new WorldSection(lvl, x, y, z, null);
}
}

8
src/main/java/me/cortex/voxy/common/world/other/Mapper.java
View File

@@ -1,11 +1,11 @@
package me.cortex.voxy.common.world.other;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.config.IMappingStorage;
import me.cortex.voxy.common.config.section.SectionStorage;
import net.minecraft.block.Block;
import net.minecraft.block.BlockState;
import net.minecraft.block.Blocks;
import net.minecraft.block.RedstoneWireBlock;
import net.minecraft.nbt.NbtCompound;
import net.minecraft.nbt.NbtIo;
import net.minecraft.nbt.NbtOps;
@@ -30,7 +30,7 @@ public class Mapper {
private static final int BLOCK_STATE_TYPE = 1;
private static final int BIOME_TYPE = 2;
private final StorageBackend storage;
private final IMappingStorage storage;
public static final long UNKNOWN_MAPPING = -1;
public static final long AIR = 0;
@@ -41,7 +41,7 @@ public class Mapper {
private Consumer<StateEntry> newStateCallback;
private Consumer<BiomeEntry> newBiomeCallback;
public Mapper(StorageBackend storage) {
public Mapper(IMappingStorage storage) {
this.storage = storage;
//Insert air since its a special entry (index 0)
var airEntry = new StateEntry(0, Blocks.AIR.getDefaultState());

10
src/main/java/me/cortex/voxy/common/world/service/SectionSavingService.java
View File

@@ -1,5 +1,6 @@
package me.cortex.voxy.common.world.service;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.world.SaveLoadSystem;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.common.world.WorldSection;
@@ -28,12 +29,11 @@ public class SectionSavingService {
section.assertNotFree();
try {
section.inSaveQueue.set(false);
var saveData = SaveLoadSystem.serialize(section);
this.world.storage.setSectionData(section.key, saveData);
saveData.free();
this.world.storage.saveSection(section);
} catch (Exception e) {
e.printStackTrace();
MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal("Voxy saver had an exception while executing please check logs and report error"), true));
String err = "Voxy saver had an exception while executing please check logs and report error";
Logger.error(err, e);
MinecraftClient.getInstance().executeSync(()->MinecraftClient.getInstance().player.sendMessage(Text.literal(err), true));
}
section.release();
}