diff --git a/gradle.properties b/gradle.properties index 4e4d20e1..1b7922a5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ yarn_mappings=1.20.4+build.1 loader_version=0.15.0 # Mod Properties -mod_version = 0.0.5-alpha +mod_version = 0.0.6-alpha maven_group = me.cortex archives_base_name = voxy diff --git a/src/main/java/me/cortex/voxy/client/Voxy.java b/src/main/java/me/cortex/voxy/client/Voxy.java index f164b3bc..01cc453f 100644 --- a/src/main/java/me/cortex/voxy/client/Voxy.java +++ b/src/main/java/me/cortex/voxy/client/Voxy.java @@ -2,17 +2,16 @@ package me.cortex.voxy.client; import me.cortex.voxy.client.config.VoxyConfig; import me.cortex.voxy.client.core.VoxelCore; -import me.cortex.voxy.client.mixin.minecraft.MixinWorldRenderer; import me.cortex.voxy.client.terrain.WorldImportCommand; -import me.cortex.voxy.common.storage.CompressionStorageAdaptor; -import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor; +import me.cortex.voxy.common.storage.config.Serialization; +import me.cortex.voxy.common.storage.other.CompressionStorageAdaptor; import me.cortex.voxy.common.storage.StorageBackend; -import me.cortex.voxy.common.storage.ZSTDCompressor; +import me.cortex.voxy.common.storage.compressors.ZSTDCompressor; +import me.cortex.voxy.common.storage.other.FragmentedStorageBackendAdaptor; import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend; import me.cortex.voxy.common.world.WorldEngine; import net.fabricmc.api.ClientModInitializer; import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback; -import net.minecraft.client.render.WorldRenderer; import net.minecraft.client.world.ClientWorld; import java.io.File; @@ -20,13 +19,19 @@ import java.io.File; public class Voxy implements ClientModInitializer { @Override public void onInitializeClient() { + //var cfg = new CompressionStorageAdaptor.Config(); + //cfg.compressor = new ZSTDCompressor.Config(); + //cfg.backend = new FragmentedStorageBackendAdaptor.Config(); + //((FragmentedStorageBackendAdaptor.Config)cfg.backend).backends.add(new RocksDBStorageBackend.Config()); + //System.out.println(Serialization.GSON.toJson(cfg)); + ClientCommandRegistrationCallback.EVENT.register((dispatcher, registryAccess) -> { dispatcher.register(WorldImportCommand.register()); }); } public static VoxelCore createVoxelCore(ClientWorld world) { - StorageBackend storage = new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath)); + StorageBackend storage = new RocksDBStorageBackend(VoxyConfig.CONFIG.storagePath); //StorageBackend storage = new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath)); storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), storage); var engine = new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); diff --git a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java b/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java index e2d9c13d..7c46a846 100644 --- a/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java +++ b/src/main/java/me/cortex/voxy/client/config/VoxyConfig.java @@ -29,14 +29,6 @@ public class VoxyConfig { public int savingCompressionLevel = 7; public String storagePath = "voxy_db"; - transient StorageConfig storageConfig; - public static abstract class StorageConfig { } - public static class FragmentedStorageConfig extends StorageConfig { } - public static class LmdbStorageConfig extends StorageConfig { } - - - - public static VoxyConfig loadOrCreate() { var path = getConfigPath(); diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index e4bd5b64..6efbae99 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -6,12 +6,8 @@ import me.cortex.voxy.client.core.rendering.*; import me.cortex.voxy.client.core.rendering.building.RenderGenerationService; import me.cortex.voxy.client.core.rendering.post.PostProcessing; import me.cortex.voxy.client.core.util.DebugUtil; -import me.cortex.voxy.common.storage.CompressionStorageAdaptor; -import me.cortex.voxy.common.storage.ZSTDCompressor; -import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend; import me.cortex.voxy.common.world.WorldEngine; import me.cortex.voxy.client.importers.WorldImporter; -import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor; import net.minecraft.client.MinecraftClient; import net.minecraft.client.render.Camera; import net.minecraft.client.render.Frustum; diff --git a/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java b/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java index 5ad61937..24d1eff1 100644 --- a/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java +++ b/src/main/java/me/cortex/voxy/client/saver/SaveSelectionSystem.java @@ -1,11 +1,10 @@ package me.cortex.voxy.client.saver; import me.cortex.voxy.client.config.VoxyConfig; -import me.cortex.voxy.common.storage.CompressionStorageAdaptor; -import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor; -import me.cortex.voxy.common.storage.ZSTDCompressor; +import me.cortex.voxy.common.storage.other.CompressionStorageAdaptor; +import me.cortex.voxy.common.storage.other.FragmentedStorageBackendAdaptor; +import me.cortex.voxy.common.storage.compressors.ZSTDCompressor; import me.cortex.voxy.common.world.WorldEngine; -import net.minecraft.client.MinecraftClient; import java.io.File; import java.nio.file.Path; @@ -30,9 +29,6 @@ public class SaveSelectionSystem { // FragmentedStorageBackendAdaptorConfig(File) // RocksDBStorageBackendConfig(File) // RedisStorageBackendConfig(String, int, String) - - - var storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath))); - return new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5); + return null; } } diff --git a/src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java b/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java similarity index 62% rename from src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java rename to src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java index 7027834d..54a7f6c7 100644 --- a/src/main/java/me/cortex/voxy/common/storage/ZSTDCompressor.java +++ b/src/main/java/me/cortex/voxy/common/storage/compressors/ZSTDCompressor.java @@ -1,5 +1,8 @@ -package me.cortex.voxy.common.storage; +package me.cortex.voxy.common.storage.compressors; +import me.cortex.voxy.common.storage.StorageCompressor; +import me.cortex.voxy.common.storage.config.CompressorConfig; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; import org.lwjgl.system.MemoryUtil; import java.nio.ByteBuffer; @@ -34,4 +37,17 @@ public class ZSTDCompressor implements StorageCompressor { public void close() { } + + public static class Config extends CompressorConfig { + public int compressionLevel; + + @Override + public StorageCompressor build(ConfigBuildCtx ctx) { + return new ZSTDCompressor(this.compressionLevel); + } + + public static String getConfigTypeName() { + return "ZSTD"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/config/CompressorConfig.java b/src/main/java/me/cortex/voxy/common/storage/config/CompressorConfig.java new file mode 100644 index 00000000..574ef0d4 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/config/CompressorConfig.java @@ -0,0 +1,11 @@ +package me.cortex.voxy.common.storage.config; + +import me.cortex.voxy.common.storage.StorageCompressor; + +public abstract class CompressorConfig { + static { + Serialization.CONFIG_TYPES.add(CompressorConfig.class); + } + + public abstract StorageCompressor build(ConfigBuildCtx ctx); +} diff --git a/src/main/java/me/cortex/voxy/common/storage/config/ConfigBuildCtx.java b/src/main/java/me/cortex/voxy/common/storage/config/ConfigBuildCtx.java new file mode 100644 index 00000000..5c1d75c4 --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/config/ConfigBuildCtx.java @@ -0,0 +1,52 @@ +package me.cortex.voxy.common.storage.config; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; + +public class ConfigBuildCtx { + //List of tokens + public static final String BASE_LEVEL_PATH = "{base_level_path}"; + + //Pushes a path to the BuildCtx path stack so that when resolving with resolvePath it uses the entire path stack + public ConfigBuildCtx pushPath(String path) { + return this; + } + + public ConfigBuildCtx popPath() { + return this; + } + + /** + * Resolves a path with the current build context path + * @param other path to resolve against + * @return resolved path + */ + public String resolvePath(String other) { + return null; + } + + /** + * Substitutes special tokens in the string the configured values + * @param string the string to substitute + * @return substituted string + */ + public String substituteString(String string) { + //This is e.g. so you can have dbs spread across multiple disks if you want + return null; + } + + /** + * Ensures that the path provided exists, if not, create it + * @param path the path to make + * @return the input + */ + public String ensurePathExists(String path) { + try { + Files.createDirectories(new File(path).toPath()); + } catch (Exception e) { + throw new RuntimeException(e); + } + return path; + } +} diff --git a/src/main/java/me/cortex/voxy/common/storage/config/GsonTypeAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/config/GsonTypeAdaptor.java new file mode 100644 index 00000000..e994c11d --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/config/GsonTypeAdaptor.java @@ -0,0 +1,13 @@ +package me.cortex.voxy.common.storage.config; + +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.reflect.TypeToken; + +public class GsonTypeAdaptor implements TypeAdapterFactory { + @Override + public TypeAdapter create(Gson gson, TypeToken type) { + return null; + } +} diff --git a/src/main/java/me/cortex/voxy/common/storage/config/Serialization.java b/src/main/java/me/cortex/voxy/common/storage/config/Serialization.java new file mode 100644 index 00000000..7d8a552e --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/config/Serialization.java @@ -0,0 +1,101 @@ +package me.cortex.voxy.common.storage.config; + +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class Serialization { + public static final Set> CONFIG_TYPES = new HashSet<>(); + public static final Gson GSON; + + private static final class GsonConfigSerialization { + private final String typeField = "config_type"; + private final Class clz; + + private final Map> name2type = new HashMap<>(); + private final Map, String> type2name = new HashMap<>(); + + private GsonConfigSerialization(Class clz) { + this.clz = clz; + } + + public GsonConfigSerialization register(String typeName, Class cls) { + if (this.name2type.put(typeName, cls) != null) { + throw new IllegalStateException("Type name already registered: " + typeName); + } + if (this.type2name.put(cls, typeName) != null) { + throw new IllegalStateException("Class already registered with type name: " + typeName + ", " + cls); + } + return this; + } + } + + static { + Map, GsonConfigSerialization> serializers = new HashMap<>(); + outer: + for (var clzName : collectAllClasses("me.cortex.voxy.common.storage")) { + if (clzName.equals(Serialization.class.getName())) { + continue;//Dont want to load ourselves + } + + try { + var clz = Class.forName(clzName); + var original = clz; + while ((clz = clz.getSuperclass()) != null) { + if (CONFIG_TYPES.contains(clz)) { + Method nameMethod = null; + try { + nameMethod = original.getMethod("getConfigTypeName"); + nameMethod.setAccessible(true); + } catch (NoSuchMethodException e) {} + if (nameMethod == null) { + System.err.println("WARNING: Config class " + clzName + " doesnt contain a getConfigTypeName and thus wont be serializable"); + continue outer; + } + String name = (String) nameMethod.invoke(null); + serializers.computeIfAbsent(clz, GsonConfigSerialization::new) + .register(name, (Class) original); + System.out.println("Registered " + original.getSimpleName() + " as " + name + " for config type " + clz.getSimpleName()); + break; + } + } + } catch (Exception e) { + System.err.println("Error while setting up config serialization"); + e.printStackTrace(); + } + } + + var builder = new GsonBuilder(); + //for (var entry : serializers.entrySet()) { + // builder.registerTypeHierarchyAdapter(entry.getKey(), entry.getValue()); + //} + + GSON = builder.create(); + } + + private static List collectAllClasses(String pack) { + InputStream stream = Serialization.class.getClassLoader() + .getResourceAsStream(pack.replaceAll("[.]", "/")); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + return reader.lines().flatMap(inner -> { + if (inner.endsWith(".class")) { + return Stream.of(pack + "." + inner.replace(".class", "")); + } else if (!inner.contains(".")) { + return collectAllClasses(pack + "." + inner).stream(); + } else { + return Stream.of(); + } + }).collect(Collectors.toList()); + } +} diff --git a/src/main/java/me/cortex/voxy/common/storage/config/StorageConfig.java b/src/main/java/me/cortex/voxy/common/storage/config/StorageConfig.java new file mode 100644 index 00000000..1160638c --- /dev/null +++ b/src/main/java/me/cortex/voxy/common/storage/config/StorageConfig.java @@ -0,0 +1,11 @@ +package me.cortex.voxy.common.storage.config; + +import me.cortex.voxy.common.storage.StorageBackend; + +public abstract class StorageConfig { + static { + Serialization.CONFIG_TYPES.add(StorageConfig.class); + } + + public abstract StorageBackend build(ConfigBuildCtx ctx); +} diff --git a/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java index 098da75a..814105f3 100644 --- a/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/storage/inmemory/MemoryStorageBackend.java @@ -6,12 +6,13 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectCollection; import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import net.minecraft.util.math.random.RandomSeed; import org.apache.commons.lang3.stream.Streams; import org.lwjgl.system.MemoryUtil; import java.nio.ByteBuffer; -import java.util.stream.Stream; public class MemoryStorageBackend extends StorageBackend { private final Long2ObjectMap[] maps; @@ -105,4 +106,15 @@ public class MemoryStorageBackend extends StorageBackend { Streams.of(this.maps).map(Long2ObjectMap::values).flatMap(ObjectCollection::stream).forEach(MemoryUtil::memFree); this.idMappings.values().forEach(MemoryUtil::memFree); } + + public static class Config extends StorageConfig { + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + return new MemoryStorageBackend(); + } + + public static String getConfigTypeName() { + return "Memory"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java index 82722e2e..d8774d7b 100644 --- a/src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java @@ -2,10 +2,11 @@ package me.cortex.voxy.common.storage.lmdb; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import org.lwjgl.system.MemoryUtil; import org.lwjgl.util.lmdb.MDBVal; -import java.io.File; import java.nio.ByteBuffer; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -24,10 +25,10 @@ public class LMDBStorageBackend extends StorageBackend { private final LMDBInterface dbi; private final LMDBInterface.Database sectionDatabase; private final LMDBInterface.Database idMappingDatabase; - public LMDBStorageBackend(File file) { + public LMDBStorageBackend(String file) { this.dbi = new LMDBInterface.Builder() .setMaxDbs(2) - .open(file.getAbsolutePath(), MDB_NOSUBDIR)//MDB_NOLOCK (IF I DO THIS, must sync the db manually)// TODO: THIS + .open(file, MDB_NOSUBDIR)//MDB_NOLOCK (IF I DO THIS, must sync the db manually)// TODO: THIS .fetch(); this.dbi.setMapSize(GROW_SIZE); this.sectionDatabase = this.dbi.createDb("world_sections"); @@ -156,4 +157,17 @@ public class LMDBStorageBackend extends StorageBackend { this.idMappingDatabase.close(); this.dbi.close(); } + + public static class Config extends StorageConfig { + public String path; + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + return new LMDBStorageBackend(ctx.ensurePathExists(ctx.substituteString(ctx.resolvePath(this.path)))); + } + + public static String getConfigTypeName() { + return "LMDB"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java similarity index 66% rename from src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java rename to src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java index 7b95954f..8de682dc 100644 --- a/src/main/java/me/cortex/voxy/common/storage/CompressionStorageAdaptor.java +++ b/src/main/java/me/cortex/voxy/common/storage/other/CompressionStorageAdaptor.java @@ -1,17 +1,14 @@ -package me.cortex.voxy.common.storage; +package me.cortex.voxy.common.storage.other; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.objects.Object2IntMap; -import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; -import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend; -import net.minecraft.util.math.random.RandomSeed; +import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.StorageCompressor; +import me.cortex.voxy.common.storage.config.CompressorConfig; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import org.lwjgl.system.MemoryUtil; -import java.io.File; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.util.Arrays; //Compresses the section data public class CompressionStorageAdaptor extends StorageBackend { @@ -65,4 +62,18 @@ public class CompressionStorageAdaptor extends StorageBackend { this.compressor.close(); this.child.close(); } + + public static class Config extends StorageConfig { + public CompressorConfig compressor; + public StorageConfig backend; + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + return new CompressionStorageAdaptor(this.compressor.build(ctx), this.backend.build(ctx)); + } + + public static String getConfigTypeName() { + return "CompressionAdaptor"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/FragmentedStorageBackendAdaptor.java b/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java similarity index 75% rename from src/main/java/me/cortex/voxy/common/storage/FragmentedStorageBackendAdaptor.java rename to src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java index 072acd6d..a5b56d5f 100644 --- a/src/main/java/me/cortex/voxy/common/storage/FragmentedStorageBackendAdaptor.java +++ b/src/main/java/me/cortex/voxy/common/storage/other/FragmentedStorageBackendAdaptor.java @@ -1,42 +1,32 @@ -package me.cortex.voxy.common.storage; +package me.cortex.voxy.common.storage.other; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; -import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend; +import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import net.minecraft.util.math.random.RandomSeed; -import java.io.File; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; //Segments the section data into multiple dbs public class FragmentedStorageBackendAdaptor extends StorageBackend { - private final StorageBackend[] backends = new StorageBackend[32]; + private final StorageBackend[] backends; - public FragmentedStorageBackendAdaptor(File directory) { - try { - Files.createDirectories(directory.toPath()); - } catch (IOException e) { - throw new RuntimeException(e); - } - for (int i = 0; i < this.backends.length; i++) { - this.backends[i] = new LMDBStorageBackend(directory.toPath().resolve("storage-db-"+i+".db").toFile());// + public FragmentedStorageBackendAdaptor(StorageBackend... backends) { + this.backends = backends; + int len = backends.length; + if ((len&(len-1)) != (len-1)) { + throw new IllegalArgumentException("Backend count not a power of 2"); } } - - //public static long getWorldSectionId(int lvl, int x, int y, int z) { - // return ((long)lvl<<60)|((long)(y&0xFF)<<52)|((long)(z&((1<<24)-1))<<28)|((long)(x&((1<<24)-1))<<4);//NOTE: 4 bits spare for whatever - // } - //private int getSegmentId(long key) { - // return (int) (((key>>4)&1)|((key>>27)&0b10)|((key>>50)&0b100)); - //} - private int getSegmentId(long key) { - return (int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&0x1F); + return (int) (RandomSeed.mixStafford13(RandomSeed.mixStafford13(key)^key)&(this.backends.length-1)); } //TODO: reencode the key to be shifted one less OR @@ -140,4 +130,22 @@ public class FragmentedStorageBackendAdaptor extends StorageBackend { db.close(); } } + + public static class Config extends StorageConfig { + public List backends = new ArrayList<>(); + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + StorageBackend[] builtBackends = new StorageBackend[this.backends.size()]; + for (int i = 0; i < this.backends.size(); i++) { + //TODO: put each backend in a different folder? + builtBackends[i] = this.backends.get(i).build(ctx); + } + return new FragmentedStorageBackendAdaptor(builtBackends); + } + + public static String getConfigTypeName() { + return "FragmentationAdaptor"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java index e8de2ede..f3f7279e 100644 --- a/src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java @@ -2,6 +2,8 @@ package me.cortex.voxy.common.storage.redis; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import org.lwjgl.system.MemoryUtil; import redis.clients.jedis.JedisPool; @@ -9,17 +11,31 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; public class RedisStorageBackend extends StorageBackend { - private final JedisPool pool = new JedisPool("localhost", 6379); - private final byte[] WORLD = "world_sections".getBytes(StandardCharsets.UTF_8); - private final byte[] MAPPINGS = "id_mappings".getBytes(StandardCharsets.UTF_8); + private final JedisPool pool; + private final String user; + private final String password; + private final byte[] WORLD; + private final byte[] MAPPINGS; - public RedisStorageBackend() { + public RedisStorageBackend(String host, int port, String prefix) { + this(host, port, prefix, null, null); + } + public RedisStorageBackend(String host, int port, String prefix, String user, String password) { + this.pool = new JedisPool(host, port); + this.user = user; + this.password = password; + this.WORLD = (prefix+"world_sections").getBytes(StandardCharsets.UTF_8); + this.MAPPINGS = (prefix+"id_mappings").getBytes(StandardCharsets.UTF_8); } @Override public ByteBuffer getSectionData(long key) { try (var jedis = this.pool.getResource()) { + if (this.user != null) { + jedis.auth(this.user, this.password); + } + var result = jedis.hget(WORLD, longToBytes(key)); if (result == null) { return null; @@ -35,6 +51,10 @@ public class RedisStorageBackend extends StorageBackend { @Override public void setSectionData(long key, ByteBuffer data) { try (var jedis = this.pool.getResource()) { + if (this.user != null) { + jedis.auth(this.user, this.password); + } + var buffer = new byte[data.remaining()]; data.get(buffer); data.rewind(); @@ -45,6 +65,10 @@ public class RedisStorageBackend extends StorageBackend { @Override public void deleteSectionData(long key) { try (var jedis = this.pool.getResource()) { + if (this.user != null) { + jedis.auth(this.user, this.password); + } + jedis.hdel(WORLD, longToBytes(key)); } } @@ -52,6 +76,10 @@ public class RedisStorageBackend extends StorageBackend { @Override public void putIdMapping(int id, ByteBuffer data) { try (var jedis = this.pool.getResource()) { + if (this.user != null) { + jedis.auth(this.user, this.password); + } + var buffer = new byte[data.remaining()]; data.get(buffer); data.rewind(); @@ -62,6 +90,10 @@ public class RedisStorageBackend extends StorageBackend { @Override public Int2ObjectOpenHashMap getIdMappingsData() { try (var jedis = this.pool.getResource()) { + if (this.user != null) { + jedis.auth(this.user, this.password); + } + var mappings = jedis.hgetAll(MAPPINGS); var out = new Int2ObjectOpenHashMap(); if (mappings == null) { @@ -108,4 +140,19 @@ public class RedisStorageBackend extends StorageBackend { } return result; } + + public static class Config extends StorageConfig { + public String host; + public int port; + public String prefix; + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + return new RedisStorageBackend(this.host, this.port, ctx.substituteString(this.prefix)); + } + + public static String getConfigTypeName() { + return "Redis"; + } + } } diff --git a/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java b/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java index ac82cff5..d190f7f1 100644 --- a/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java +++ b/src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java @@ -2,14 +2,14 @@ package me.cortex.voxy.common.storage.rocksdb; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import me.cortex.voxy.common.storage.StorageBackend; +import me.cortex.voxy.common.storage.config.ConfigBuildCtx; +import me.cortex.voxy.common.storage.config.StorageConfig; import org.lwjgl.system.MemoryUtil; import org.rocksdb.*; -import redis.clients.jedis.JedisPool; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -23,9 +23,9 @@ public class RocksDBStorageBackend extends StorageBackend { //NOTE: closes in order private final List closeList = new ArrayList<>(); - public RocksDBStorageBackend(File path) { + public RocksDBStorageBackend(String path) { try { - var lockPath = path.toPath().resolve("LOCK"); + var lockPath = new File(path).toPath().resolve("LOCK"); if (Files.exists(lockPath)) { System.err.println("WARNING, deleting rocksdb LOCK file"); Files.delete(lockPath); @@ -50,7 +50,7 @@ public class RocksDBStorageBackend extends StorageBackend { try { this.db = RocksDB.open(options, - path.getPath(), cfDescriptors, + path, cfDescriptors, handles); this.closeList.addAll(handles); @@ -164,4 +164,17 @@ public class RocksDBStorageBackend extends StorageBackend { } return result; } + + public static class Config extends StorageConfig { + public String path; + + @Override + public StorageBackend build(ConfigBuildCtx ctx) { + return new RocksDBStorageBackend(ctx.ensurePathExists(ctx.substituteString(ctx.resolvePath(this.path)))); + } + + public static String getConfigTypeName() { + return "RocksDB"; + } + } }