spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added rocks db

Browse Source
This commit is contained in:
mcrcortex
2024-02-04 15:34:41 +10:00
parent 86f3b04b37
commit ba93e3f872
17 changed files with 320 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
build.gradle
View File

@@ -104,4 +104,7 @@ dependencies {
include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-windows")
include(runtimeOnly "org.lwjgl:lwjgl-lmdb:$lwjglVersion:natives-linux")
include(runtimeOnly "org.lwjgl:lwjgl-zstd:$lwjglVersion:natives-linux")
implementation 'org.rocksdb:rocksdbjni:8.10.0'
implementation 'redis.clients:jedis:5.1.0'
}

4
src/main/java/me/cortex/voxy/client/Test.java
View File

@@ -1,8 +1,8 @@
package me.cortex.voxy.client;
import me.cortex.voxy.common.world.storage.lmdb.LMDBInterface;
import me.cortex.voxy.common.storage.lmdb.LMDBInterface;
import me.cortex.voxy.client.importers.WorldImporter;
import me.cortex.voxy.common.world.storage.lmdb.LMDBStorageBackend;
import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend;
import org.lwjgl.system.MemoryUtil;
import java.io.File;

5
src/main/java/me/cortex/voxy/client/core/VoxelCore.java
View File

@@ -6,9 +6,10 @@ import me.cortex.voxy.client.core.rendering.*;
import me.cortex.voxy.client.core.rendering.building.RenderGenerationService;
import me.cortex.voxy.client.core.rendering.post.PostProcessing;
import me.cortex.voxy.client.core.util.DebugUtil;
import me.cortex.voxy.common.storage.rocksdb.RocksDBStorageBackend;
import me.cortex.voxy.common.world.WorldEngine;
import me.cortex.voxy.client.importers.WorldImporter;
import me.cortex.voxy.common.world.storage.FragmentedStorageBackendAdaptor;
import me.cortex.voxy.common.storage.FragmentedStorageBackendAdaptor;
import net.minecraft.client.MinecraftClient;
import net.minecraft.client.render.Camera;
import net.minecraft.client.render.Frustum;
@@ -53,7 +54,7 @@ public class VoxelCore {
SharedIndexBuffer.INSTANCE.id();
this.renderer = new Gl46FarWorldRenderer(VoxyConfig.CONFIG.geometryBufferSize, VoxyConfig.CONFIG.maxSections);
System.out.println("Renderer initialized");
this.world = new WorldEngine(new FragmentedStorageBackendAdaptor(new File(VoxyConfig.CONFIG.storagePath)), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, VoxyConfig.CONFIG.savingCompressionLevel, 5);//"storagefile.db"//"ethoslab.db"
this.world = new WorldEngine(new RocksDBStorageBackend(new File(VoxyConfig.CONFIG.storagePath)), VoxyConfig.CONFIG.ingestThreads, VoxyConfig.CONFIG.savingThreads, VoxyConfig.CONFIG.savingCompressionLevel, 5);//"storagefile.db"//"ethoslab.db"
System.out.println("World engine");
this.renderTracker = new RenderTracker(this.world, this.renderer);

4
src/main/java/me/cortex/voxy/common/world/storage/FragmentedStorageBackendAdaptor.java → src/main/java/me/cortex/voxy/common/storage/FragmentedStorageBackendAdaptor.java
View File

@@ -1,9 +1,9 @@
package me.cortex.voxy.common.world.storage;
package me.cortex.voxy.common.storage;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import me.cortex.voxy.common.world.storage.lmdb.LMDBStorageBackend;
import me.cortex.voxy.common.storage.lmdb.LMDBStorageBackend;
import net.minecraft.util.math.random.RandomSeed;
import java.io.File;

2
src/main/java/me/cortex/voxy/common/world/storage/StorageBackend.java → src/main/java/me/cortex/voxy/common/storage/StorageBackend.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.world.storage;
package me.cortex.voxy.common.storage;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;

4
src/main/java/me/cortex/voxy/common/world/storage/lmdb/Cursor.java → src/main/java/me/cortex/voxy/common/storage/lmdb/Cursor.java
View File

@@ -1,8 +1,8 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
import org.lwjgl.util.lmdb.MDBVal;
import static me.cortex.voxy.common.world.storage.lmdb.LMDBInterface.E;
import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
import static org.lwjgl.util.lmdb.LMDB.*;
public class Cursor implements AutoCloseable {

2
src/main/java/me/cortex/voxy/common/world/storage/lmdb/LMDBInterface.java → src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBInterface.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
import org.lwjgl.PointerBuffer;
import org.lwjgl.system.MemoryStack;

4
src/main/java/me/cortex/voxy/common/world/storage/lmdb/LMDBStorageBackend.java → src/main/java/me/cortex/voxy/common/storage/lmdb/LMDBStorageBackend.java
View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.world.storage.StorageBackend;
import me.cortex.voxy.common.storage.StorageBackend;
import org.lwjgl.system.MemoryUtil;
import org.lwjgl.util.lmdb.MDBVal;

2
src/main/java/me/cortex/voxy/common/world/storage/lmdb/TransactionCallback.java → src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionCallback.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
import org.lwjgl.system.MemoryStack;

2
src/main/java/me/cortex/voxy/common/world/storage/lmdb/TransactionWrappedCallback.java → src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionWrappedCallback.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
public interface TransactionWrappedCallback<T> {
T exec(TransactionWrapper wrapper);

4
src/main/java/me/cortex/voxy/common/world/storage/lmdb/TransactionWrapper.java → src/main/java/me/cortex/voxy/common/storage/lmdb/TransactionWrapper.java
View File

@@ -1,4 +1,4 @@
package me.cortex.voxy.common.world.storage.lmdb;
package me.cortex.voxy.common.storage.lmdb;
import org.lwjgl.PointerBuffer;
import org.lwjgl.system.MemoryStack;
@@ -6,7 +6,7 @@ import org.lwjgl.util.lmdb.MDBVal;
import java.nio.ByteBuffer;
import static me.cortex.voxy.common.world.storage.lmdb.LMDBInterface.E;
import static me.cortex.voxy.common.storage.lmdb.LMDBInterface.E;
import static org.lwjgl.system.MemoryStack.stackPush;
import static org.lwjgl.util.lmdb.LMDB.*;

111
src/main/java/me/cortex/voxy/common/storage/redis/RedisStorageBackend.java Normal file
View File

@@ -0,0 +1,111 @@
package me.cortex.voxy.common.storage.redis;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import org.lwjgl.system.MemoryUtil;
import redis.clients.jedis.JedisPool;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
public class RedisStorageBackend extends StorageBackend {
private final JedisPool pool = new JedisPool("localhost", 6379);
private final byte[] WORLD = "world_sections".getBytes(StandardCharsets.UTF_8);
private final byte[] MAPPINGS = "id_mappings".getBytes(StandardCharsets.UTF_8);
public RedisStorageBackend() {
}
@Override
public ByteBuffer getSectionData(long key) {
try (var jedis = this.pool.getResource()) {
var result = jedis.hget(WORLD, longToBytes(key));
if (result == null) {
return null;
}
//Need to copy to native memory
var buffer = MemoryUtil.memAlloc(result.length);
buffer.put(result);
buffer.rewind();
return buffer;
}
}
@Override
public void setSectionData(long key, ByteBuffer data) {
try (var jedis = this.pool.getResource()) {
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
jedis.hset(WORLD, longToBytes(key), buffer);
}
}
@Override
public void deleteSectionData(long key) {
try (var jedis = this.pool.getResource()) {
jedis.hdel(WORLD, longToBytes(key));
}
}
@Override
public void putIdMapping(int id, ByteBuffer data) {
try (var jedis = this.pool.getResource()) {
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
jedis.hset(MAPPINGS, intToBytes(id), buffer);
}
}
@Override
public Int2ObjectOpenHashMap<byte[]> getIdMappingsData() {
try (var jedis = this.pool.getResource()) {
var mappings = jedis.hgetAll(MAPPINGS);
var out = new Int2ObjectOpenHashMap<byte[]>();
if (mappings == null) {
return out;
}
for (var entry : mappings.entrySet()) {
out.put(bytesToInt(entry.getKey()), entry.getValue());
}
return out;
}
}
@Override
public void flush() {
}
@Override
public void close() {
this.pool.close();
}
private static byte[] intToBytes(int i) {
return new byte[] {(byte)(i>>24), (byte)(i>>16), (byte)(i>>8), (byte) i};
}
private static int bytesToInt(byte[] i) {
return (Byte.toUnsignedInt(i[0])<<24)|(Byte.toUnsignedInt(i[1])<<16)|(Byte.toUnsignedInt(i[2])<<8)|(Byte.toUnsignedInt(i[3]));
}
private static byte[] longToBytes(long l) {
byte[] result = new byte[Long.BYTES];
for (int i = Long.BYTES - 1; i >= 0; i--) {
result[i] = (byte)(l & 0xFF);
l >>= Byte.SIZE;
}
return result;
}
private static long bytesToLong(final byte[] b) {
long result = 0;
for (int i = 0; i < Long.BYTES; i++) {
result <<= Byte.SIZE;
result |= (b[i] & 0xFF);
}
return result;
}
}

155
src/main/java/me/cortex/voxy/common/storage/rocksdb/RocksDBStorageBackend.java Normal file
View File

@@ -0,0 +1,155 @@
package me.cortex.voxy.common.storage.rocksdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import me.cortex.voxy.common.storage.StorageBackend;
import org.lwjgl.system.MemoryUtil;
import org.rocksdb.*;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class RocksDBStorageBackend extends StorageBackend {
private final RocksDB db;
private final ColumnFamilyHandle worldSections;
private final ColumnFamilyHandle idMappings;
//NOTE: closes in order
private final List<AbstractImmutableNativeReference> closeList = new ArrayList<>();
public RocksDBStorageBackend(File path) {
final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().optimizeUniversalStyleCompaction();
final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts),
new ColumnFamilyDescriptor("world_sections".getBytes(), cfOpts),
new ColumnFamilyDescriptor("id_mappings".getBytes(), cfOpts)
);
final DBOptions options = new DBOptions()
.setCreateIfMissing(true)
.setCreateMissingColumnFamilies(true);
List<ColumnFamilyHandle> handles = new ArrayList<>();
try {
this.db = RocksDB.open(options,
path.getPath(), cfDescriptors,
handles);
this.closeList.addAll(handles);
this.closeList.add(this.db);
this.closeList.add(options);
this.closeList.add(cfOpts);
this.worldSections = handles.get(1);
this.idMappings = handles.get(2);
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public ByteBuffer getSectionData(long key) {
try {
var result = this.db.get(this.worldSections, longToBytes(key));
if (result == null) {
return null;
}
//Need to copy to native memory
var buffer = MemoryUtil.memAlloc(result.length);
buffer.put(result);
buffer.rewind();
return buffer;
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public void setSectionData(long key, ByteBuffer data) {
try {
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
this.db.put(this.worldSections, longToBytes(key), buffer);
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public void deleteSectionData(long key) {
try {
this.db.delete(this.worldSections, longToBytes(key));
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public void putIdMapping(int id, ByteBuffer data) {
try {
var buffer = new byte[data.remaining()];
data.get(buffer);
data.rewind();
this.db.put(this.idMappings, intToBytes(id), buffer);
} catch (
RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public Int2ObjectOpenHashMap<byte[]> getIdMappingsData() {
var iterator = this.db.newIterator(this.idMappings);
var out = new Int2ObjectOpenHashMap<byte[]>();
for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
out.put(bytesToInt(iterator.key()), iterator.value());
}
return out;
}
@Override
public void flush() {
try {
this.db.flushWal(true);
} catch (RocksDBException e) {
throw new RuntimeException(e);
}
}
@Override
public void close() {
this.closeList.forEach(AbstractImmutableNativeReference::close);
}
private static byte[] intToBytes(int i) {
return new byte[] {(byte)(i>>24), (byte)(i>>16), (byte)(i>>8), (byte) i};
}
private static int bytesToInt(byte[] i) {
return (Byte.toUnsignedInt(i[0])<<24)|(Byte.toUnsignedInt(i[1])<<16)|(Byte.toUnsignedInt(i[2])<<8)|(Byte.toUnsignedInt(i[3]));
}
private static byte[] longToBytes(long l) {
byte[] result = new byte[Long.BYTES];
for (int i = Long.BYTES - 1; i >= 0; i--) {
result[i] = (byte)(l & 0xFF);
l >>= Byte.SIZE;
}
return result;
}
private static long bytesToLong(final byte[] b) {
long result = 0;
for (int i = 0; i < Long.BYTES; i++) {
result <<= Byte.SIZE;
result |= (b[i] & 0xFF);
}
return result;
}
}

2
src/main/java/me/cortex/voxy/common/world/WorldEngine.java
View File

@@ -4,7 +4,7 @@ import me.cortex.voxy.common.voxelization.VoxelizedSection;
import me.cortex.voxy.common.world.other.Mapper;
import me.cortex.voxy.common.world.service.SectionSavingService;
import me.cortex.voxy.common.world.service.VoxelIngestService;
import me.cortex.voxy.common.world.storage.StorageBackend;
import me.cortex.voxy.common.storage.StorageBackend;
import org.lwjgl.system.MemoryUtil;
import java.util.Arrays;

2
src/main/java/me/cortex/voxy/common/world/other/Mapper.java
View File

@@ -1,7 +1,7 @@
package me.cortex.voxy.common.world.other;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import me.cortex.voxy.common.world.storage.StorageBackend;
import me.cortex.voxy.common.storage.StorageBackend;
import net.minecraft.block.Block;
import net.minecraft.block.BlockState;
import net.minecraft.block.Blocks;

2
src/main/resources/assets/voxy/shaders/lod/gl46/cmdgen.comp
View File

@@ -1,4 +1,4 @@
#version 460
#version 450
#extension GL_ARB_gpu_shader_int64 : enable
layout(local_size_x = 128, local_size_y = 1, local_size_x = 1) in;

56
src/main/resources/assets/voxy/shaders/post/ssao.comp
View File

@@ -1,4 +1,4 @@
#version 460
#version 450
layout(local_size_x = 32, local_size_y = 32, local_size_x = 1) in;
@@ -12,12 +12,6 @@ vec3 rev3d(vec3 clip) {
return view.xyz/view.w;
}
vec3 proj3dscreen(vec3 pos) {
vec4 view = MVP * vec4(pos, 1);
view.xyz /= view.w;
return view.xyz * 0.5 + 0.5;
}
vec3 reDeProject(vec3 pos) {
vec4 view = MVP * vec4(pos, 1);
view.xy /= view.w;
@@ -27,25 +21,6 @@ vec3 reDeProject(vec3 pos) {
return view.xyz/view.w;
}
vec3 reDeProjectContained(vec3 pos) {
vec4 view = MVP * vec4(pos, 1);
view.xy /= view.w;
float depth = texture(depthTex, clamp(view.xy*0.5+0.5, 0, 1)).x*2-1;
view.z = min(view.z, depth);
view.w = 1;
view = invMVP * view;
return view.xyz/view.w;
}
vec3 computeDifference(vec3 pos, vec3 offset) {
return reDeProject(pos + offset) - pos - offset;
}
float computeAngleDifference(vec3 pos, vec3 offset) {
vec3 repro = reDeProject(pos + offset) - pos;
return dot(repro, offset)/(length(repro) * length(offset));
}
float computeAOAngle(vec3 pos, float testHeight, vec3 normal) {
vec3 repro = reDeProject(pos + normal*testHeight) - pos;
float len = length(repro);
@@ -86,3 +61,32 @@ void main() {
//if (any(lessThan(ivec3(4), abs(repro-offset)))) {// || all(lessThan(abs(repro-offset), ivec3(0.01)))
//return;
//}
/*
vec3 proj3dscreen(vec3 pos) {
vec4 view = MVP * vec4(pos, 1);
view.xyz /= view.w;
return view.xyz * 0.5 + 0.5;
}
vec3 reDeProjectContained(vec3 pos) {
vec4 view = MVP * vec4(pos, 1);
view.xy /= view.w;
float depth = texture(depthTex, clamp(view.xy*0.5+0.5, 0, 1)).x*2-1;
view.z = min(view.z, depth);
view.w = 1;
view = invMVP * view;
return view.xyz/view.w;
}
vec3 computeDifference(vec3 pos, vec3 offset) {
return reDeProject(pos + offset) - pos - offset;
}
float computeAngleDifference(vec3 pos, vec3 offset) {
vec3 repro = reDeProject(pos + offset) - pos;
return dot(repro, offset)/(length(repro) * length(offset));
}
*/