spdis/voxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve ingest and import speed

Browse Source
This commit is contained in:
mcrcortex
2025-02-05 17:08:44 +10:00
parent d77addb416
commit 6d683503cd
7 changed files with 124 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@@ -77,6 +77,8 @@ dependencies {
modRuntimeOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric"
modCompileOnly "maven.modrinth:sodium:mc1.21.4-0.6.6-fabric"
modImplementation("maven.modrinth:lithium:mc1.21.4-0.14.7-fabric")
//modRuntimeOnly "maven.modrinth:nvidium:0.2.6-beta"
modCompileOnly "maven.modrinth:nvidium:0.2.8-beta"

17
src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java
View File

@@ -63,12 +63,17 @@ public class WorldImportWrapper {
var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false);
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar);
long start = System.currentTimeMillis();
factory.create(this.importer, (a, b)->
MinecraftClient.getInstance().executeSync(()-> {
Taskbar.INSTANCE.setProgress(a, Math.max(1,b));
bossBar.setPercent(((float) a)/((float) Math.max(1,b)));
bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks"));
}),
long[] ticker = new long[1];
factory.create(this.importer, (a, b)-> {
if (System.currentTimeMillis() - ticker[0] > 50) {
ticker[0] = System.currentTimeMillis();
MinecraftClient.getInstance().executeSync(() -> {
Taskbar.INSTANCE.setProgress(a, Math.max(1, b));
bossBar.setPercent(((float) a) / ((float) Math.max(1, b)));
bossBar.setName(Text.of("Voxy import: " + a + "/" + b + " chunks"));
});
}
},
chunkCount -> {
MinecraftClient.getInstance().executeSync(()-> {
MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID);

4
src/main/java/me/cortex/voxy/common/voxelization/ILightingSupplier.java
View File

@@ -1,7 +1,5 @@
package me.cortex.voxy.common.voxelization;
import net.minecraft.block.BlockState;
public interface ILightingSupplier {
byte supply(int x, int y, int z, BlockState state);
byte supply(int x, int y, int z);
}

129
src/main/java/me/cortex/voxy/common/voxelization/WorldConversionFactory.java
View File

@@ -1,33 +1,124 @@
package me.cortex.voxy.common.voxelization;
import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap;
import me.cortex.voxy.common.Logger;
import me.cortex.voxy.common.util.Pair;
import me.cortex.voxy.common.world.other.Mipper;
import me.cortex.voxy.common.world.other.Mapper;
import net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette;
import net.minecraft.block.BlockState;
import net.minecraft.registry.entry.RegistryEntry;
import net.minecraft.world.biome.Biome;
import net.minecraft.world.chunk.PalettedContainer;
import net.minecraft.world.chunk.ReadableContainer;
import net.minecraft.world.chunk.*;
import java.util.WeakHashMap;
public class WorldConversionFactory {
private static final class Cache {
private final int[] biomeCache = new int[4*4*4];
private final WeakHashMap<Mapper, Reference2IntOpenHashMap<BlockState>> localMapping = new WeakHashMap<>();
private int[] paletteCache = new int[1024];
private Reference2IntOpenHashMap<BlockState> getLocalMapping(Mapper mapper) {
return this.localMapping.computeIfAbsent(mapper, (a_)->new Reference2IntOpenHashMap<>());
}
private int[] getPaletteCache(int size) {
if (this.paletteCache.length < size) {
this.paletteCache = new int[size];
}
return this.paletteCache;
}
}
//TODO: create a mapping for world/mapper -> local mapping
private static final ThreadLocal<Pair<int[], WeakHashMap<Mapper, Reference2IntOpenHashMap<BlockState>>>> THREAD_LOCAL = ThreadLocal.withInitial(()->new Pair<>(new int[4*4*4], new WeakHashMap<>()));
private static final ThreadLocal<Cache> THREAD_LOCAL = ThreadLocal.withInitial(Cache::new);
private static void setupLocalPalette(Palette<BlockState> vp,Reference2IntOpenHashMap<BlockState> blockCache, Mapper mapper, int[] pc) {
{
if (vp instanceof ArrayPalette<BlockState>) {
for (int i = 0; i < vp.getSize(); i++) {
var state = vp.get(i);
int blockId = -1;
if (state != null) {
blockId = blockCache.getOrDefault(state, -1);
if (blockId == -1) {
blockId = mapper.getIdForBlockState(state);
blockCache.put(state, blockId);
}
}
pc[i] = blockId;
}
} else if (vp instanceof LithiumHashPalette<BlockState>) {
for (int i = 0; i < vp.getSize(); i++) {
BlockState state = null;
int blockId = -1;
try { state = vp.get(i); } catch (Exception e) {}
if (state != null) {
blockId = blockCache.getOrDefault(state, -1);
if (blockId == -1) {
blockId = mapper.getIdForBlockState(state);
blockCache.put(state, blockId);
}
}
pc[i] = blockId;
}
} else {
if (vp instanceof BiMapPalette<BlockState> pal) {
//var map = pal.map;
//TODO: heavily optimize this by reading the map directly
for (int i = 0; i < vp.getSize(); i++) {
BlockState state = null;
int blockId = -1;
try { state = vp.get(i); } catch (Exception e) {}
if (state != null) {
blockId = blockCache.getOrDefault(state, -1);
if (blockId == -1) {
blockId = mapper.getIdForBlockState(state);
blockCache.put(state, blockId);
}
}
pc[i] = blockId;
}
} else if (vp instanceof SingularPalette<BlockState>) {
int blockId = -1;
var state = vp.get(0);
if (state != null) {
blockId = blockCache.getOrDefault(state, -1);
if (blockId == -1) {
blockId = mapper.getIdForBlockState(state);
blockCache.put(state, blockId);
}
}
pc[0] = blockId;
} else {
Logger.error("Unknown palette type: " + vp);
}
}
}
}
public static VoxelizedSection convert(VoxelizedSection section,
Mapper stateMapper,
PalettedContainer<BlockState> blockContainer,
ReadableContainer<RegistryEntry<Biome>> biomeContainer,
ILightingSupplier lightSupplier) {
var threadLocal = THREAD_LOCAL.get();
var blockCache = threadLocal.right().computeIfAbsent(stateMapper, (mapper)->new Reference2IntOpenHashMap<>());
var biomes = threadLocal.left();
//Cheat by creating a local pallet then read the data directly
var cache = THREAD_LOCAL.get();
var blockCache = cache.getLocalMapping(stateMapper);
var biomes = cache.biomeCache;
var data = section.section;
int blockId = -1;
BlockState block = null;
var vp = blockContainer.data.palette;
var pc = cache.getPaletteCache(vp.getSize());
setupLocalPalette(vp, blockCache, stateMapper, pc);
{
int i = 0;
for (int y = 0; y < 4; y++) {
@@ -41,29 +132,15 @@ public class WorldConversionFactory {
var bDat = blockContainer.data;
var bStor = bDat.storage;
var bPall = bDat.palette;
int i = 0;
for (int y = 0; y < 16; y++) {
for (int z = 0; z < 16; z++) {
for (int x = 0; x < 16; x++) {
var state = bPall.get(bStor.get(i++));
int bId = pc[bStor.get(i++)];
byte light = lightSupplier.supply(x,y,z,state);
if (!(state.isAir() && (light==0))) {
if (block != state) {
if (state.isAir()) {
block = state;
blockId = 0;
} else {
blockId = blockCache.getOrDefault(state, -1);
if (blockId == -1) {
blockId = stateMapper.getIdForBlockState(state);
blockCache.put(state, blockId);
}
block = state;
}
}
data[G(x, y, z)] = Mapper.composeMappingId(light, blockId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]);
byte light = lightSupplier.supply(x,y,z);
if (!(bId==0 && (light==0))) {
data[G(x, y, z)] = Mapper.composeMappingId(light, bId, biomes[((y&0b1100)<<2)|(z&0b1100)|(x>>2)]);
} else {
data[G(x, y, z)] = Mapper.AIR;
}

2
src/main/java/me/cortex/voxy/common/world/WorldSection.java
View File

@@ -33,7 +33,7 @@ public final class WorldSection {
//TODO: should make it dynamically adjust the size allowance based on memory pressure/WorldSection allocation rate (e.g. is it doing a world import)
private static final int ARRAY_REUSE_CACHE_SIZE = 200;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes
private static final int ARRAY_REUSE_CACHE_SIZE = 300;//500;//32*32*32*8*ARRAY_REUSE_CACHE_SIZE == number of bytes
//TODO: maybe just swap this to a ConcurrentLinkedDeque
private static final Deque<long[]> ARRAY_REUSE_CACHE = new ArrayDeque<>(1024);

8
src/main/java/me/cortex/voxy/common/world/service/VoxelIngestService.java
View File

@@ -41,16 +41,16 @@ public class VoxelIngestService {
this.world.getMapper(),
section.getBlockStateContainer(),
section.getBiomeContainer(),
(x, y, z, state) -> {
(x, y, z) -> {
if (lighting == null || ((lighting.first() != null && lighting.first().isUninitialized())&&(lighting.second()!=null&&lighting.second().isUninitialized()))) {
return (byte) 0;
} else {
//Lighting is hell
int block = lighting.first()!=null?Math.min(15,lighting.first().get(x, y, z)):0;
int sky = lighting.second()!=null?Math.min(15,lighting.second().get(x, y, z)):0;
if (block<state.getLuminance()) {
block = state.getLuminance();
}
//if (block<state.getLuminance()) {
// block = state.getLuminance();
//}
return (byte) (sky|(block<<4));
}
}

3
src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java
View File

@@ -430,12 +430,13 @@ public class WorldImporter {
}
var blockStates = blockStatesRes.getPartialOrThrow();
var biomes = this.biomeCodec.parse(NbtOps.INSTANCE, section.getCompound("biomes")).result().orElse(this.defaultBiomeProvider);
VoxelizedSection csec = WorldConversionFactory.convert(
SECTION_CACHE.get().setPosition(x, y, z),
this.world.getMapper(),
blockStates,
biomes,
(bx, by, bz, state) -> {
(bx, by, bz) -> {
int block = 0;
int sky = 0;
if (blockLight != null) {