spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serialization system works

Browse Source
This commit is contained in:
mcrcortex
2024-02-12 12:12:06 +10:00
parent 0c866c8af6
commit 4d236169ba
6 changed files with 106 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/me/cortex/voxy/client/saver/WorldSelectionSystem.java
View File

@@ -4,6 +4,8 @@ import me.cortex.voxy.client.config.VoxyConfig;
import me.cortex.voxy.common.storage.StorageBackend;
import me.cortex.voxy.common.storage.compressors.ZSTDCompressor;
import me.cortex.voxy.common.storage.config.ConfigBuildCtx;
import me.cortex.voxy.common.storage.config.Serialization;
import me.cortex.voxy.common.storage.config.StorageConfig;
import me.cortex.voxy.common.storage.other.CompressionStorageAdaptor;
import me.cortex.voxy.common.storage.other.TranslocatingStorageAdaptor;
import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend;
@@ -17,7 +19,9 @@ import java.util.List;
// this is a bit tricky as each world has its own config, e.g. storage configuration
public class WorldSelectionSystem {
public static class Selection {
public WorldEngine createEngine() {
private VoxyConfig config;
public StorageBackend createStorageBackend() {
var baseDB = new RocksDBStorageBackend.Config();
baseDB.path = VoxyConfig.CONFIG.storagePath;
@@ -33,13 +37,15 @@ public class WorldSelectionSystem {
translocator.transforms.add(new TranslocatingStorageAdaptor.BoxTransform(0,5,0, 200, 64, 200, 0, -5, 0));
var ctx = new ConfigBuildCtx();
var storage = translocator.build(ctx);
return new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
return translocator.build(ctx);
//StorageBackend storage = new RocksDBStorageBackend(VoxyConfig.CONFIG.storagePath);
////StorageBackend storage = new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath));
//storage = new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), storage);
//return new WorldEngine(storage, VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
//return new CompressionStorageAdaptor(new ZSTDCompressor(VoxyConfig.CONFIG.savingCompressionLevel), storage);
}
public WorldEngine createEngine() {
return new WorldEngine(this.createStorageBackend(), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, 5);
}
//Saves the config for the world selection or something, need to figure out how to make it work with dimensional configs maybe?

22
src/main/java/me/cortex/voxy/common/storage/config/ConfigBuildCtx.java
View File

@@ -3,14 +3,32 @@ package me.cortex.voxy.common.storage.config;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
public class ConfigBuildCtx {
//List of tokens
public static final String BASE_LEVEL_PATH = "{base_level_path}";
private final Map<String, String> properties = new HashMap<>();
private final Stack<String> pathStack = new Stack<>();
/**
* Sets a builder property
* @param property property name
* @param value property value
* @return the builder context
*/
public ConfigBuildCtx setProperty(String property, String value) {
if (!(property.startsWith("{") && property.endsWith("}"))) {
throw new IllegalArgumentException("Property name doesnt start with { and end with }");
}
this.properties.put(property, value);
return this;
}
/**
* Pushes a path to the build context so that when resolvePath is called it is with respect to the added path
* @param path the path to add to the stack
@@ -76,7 +94,9 @@ public class ConfigBuildCtx {
* @return substituted string
*/
public String substituteString(String string) {
//TODO: this
for (var entry : this.properties.entrySet()) {
string = string.replace(entry.getKey(), entry.getValue());
}
return string;
}

56
src/main/java/me/cortex/voxy/common/storage/config/Serialization.java
View File

@@ -1,8 +1,10 @@
package me.cortex.voxy.common.storage.config;
import com.google.gson.*;
import com.google.gson.internal.bind.JsonTreeWriter;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import com.google.gson.stream.JsonWriter;
import java.io.BufferedReader;
@@ -19,8 +21,8 @@ public class Serialization {
public static final Set<Class<?>> CONFIG_TYPES = new HashSet<>();
public static final Gson GSON;
private static final class GsonConfigSerialization <T> {
private final String typeField = "config_type";
private static final class GsonConfigSerialization <T> implements TypeAdapterFactory {
private final String typeField = "TYPE";
private final Class<T> clz;
private final Map<String, Class<? extends T>> name2type = new HashMap<>();
@@ -39,6 +41,50 @@ public class Serialization {
}
return this;
}
private T deserialize(Gson gson, JsonElement json) {
var retype = this.name2type.get(json.getAsJsonObject().remove(this.typeField).getAsString());
return gson.getDelegateAdapter(this, TypeToken.get(retype)).fromJsonTree(json);
}
private JsonElement serialize(Gson gson, T value) {
String name = this.type2name.get(value.getClass());
if (name == null) {
name = "UNKNOWN_TYPE_{" + value.getClass().getName() + "}";
}
var vjson = gson
.getDelegateAdapter(this, TypeToken.get((Class<T>) value.getClass()))
.toJsonTree(value);
//All of this is so that the config_type is at the top :blob_face:
var json = new JsonObject();
json.addProperty(this.typeField, name);
vjson.getAsJsonObject().asMap().forEach(json::add);
return json;
}
@Override
public <X> TypeAdapter<X> create(Gson gson, TypeToken<X> type) {
if (this.clz.isAssignableFrom(type.getRawType())) {
var jsonObjectAdapter = gson.getAdapter(JsonElement.class);
return (TypeAdapter<X>) new TypeAdapter<T>() {
@Override
public void write(JsonWriter out, T value) throws IOException {
jsonObjectAdapter.write(out, GsonConfigSerialization.this.serialize(gson, value));
}
@Override
public T read(JsonReader in) throws IOException {
var obj = jsonObjectAdapter.read(in);
return GsonConfigSerialization.this.deserialize(gson, obj);
}
};
}
return null;
}
}
static {
@@ -77,9 +123,9 @@ public class Serialization {
}
var builder = new GsonBuilder();
//for (var entry : serializers.entrySet()) {
// builder.registerTypeHierarchyAdapter(entry.getKey(), entry.getValue());
//}
for (var entry : serializers.entrySet()) {
builder.registerTypeAdapterFactory(entry.getValue());
}
GSON = builder.create();
}

18
src/main/java/me/cortex/voxy/common/storage/other/TranslocatingStorageAdaptor.java
View File

@@ -55,12 +55,26 @@ public class TranslocatingStorageAdaptor extends DelegatingStorageAdaptor {
@Override
public void setSectionData(long key, ByteBuffer data) {
super.setSectionData(this.transformPosition(key), data);
//Dont save data if its a transformed position
for (var transform : this.transforms) {
long tpos = transform.transformIfInBox(key);
if (tpos != -1) {
return;
}
}
super.setSectionData(key, data);
}
@Override
public void deleteSectionData(long key) {
super.deleteSectionData(this.transformPosition(key));
//Dont delete save data if its a transformed position
for (var transform : this.transforms) {
long tpos = transform.transformIfInBox(key);
if (tpos != -1) {
return;
}
}
super.deleteSectionData(key);
}
public static class Config extends StorageConfig {

12
src/main/java/me/cortex/voxy/common/world/SaveLoadSystem.java
View File

@@ -53,7 +53,7 @@ public class SaveLoadSystem {
return raw;
}
public static boolean deserialize(WorldSection section, ByteBuffer data) {
public static boolean deserialize(WorldSection section, ByteBuffer data, boolean ignoreMismatchPosition) {
long hash = 0;
long key = data.getLong();
int lutLen = data.getInt();
@@ -66,11 +66,11 @@ public class SaveLoadSystem {
hash ^= lut[i];
}
//if (section.key != key) {
// //throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
// System.err.println("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
// return false;
//}
if ((!ignoreMismatchPosition) && section.key != key) {
//throw new IllegalStateException("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
System.err.println("Decompressed section not the same as requested. got: " + key + " expected: " + section.key);
return false;
}
for (int i = 0; i < section.data.length; i++) {
short lutId = data.getShort();

2
src/main/java/me/cortex/voxy/common/world/WorldEngine.java
View File

@@ -44,7 +44,7 @@ public class WorldEngine {
var data = this.storage.getSectionData(into.key);
if (data != null) {
try {
if (!SaveLoadSystem.deserialize(into, data)) {
if (!SaveLoadSystem.deserialize(into, data, true)) {
this.storage.deleteSectionData(into.key);
//TODO: regenerate the section from children
Arrays.fill(into.data, Mapper.AIR);