From 49aa9c2dd99c3f855682b2993a2c2e2c21d2000b Mon Sep 17 00:00:00 2001 From: mcrcortex <18544518+MCRcortex@users.noreply.github.com> Date: Wed, 5 Feb 2025 14:05:44 +1000 Subject: [PATCH] Add support for importing directly from zip files --- .../me/cortex/voxy/client/core/VoxelCore.java | 59 +-------- .../voxy/client/core/WorldImportWrapper.java | 89 +++++++++++++ .../client/terrain/WorldImportCommand.java | 62 +++++++-- .../voxy/common/world/WorldSection.java | 4 + .../commonImpl/importers/WorldImporter.java | 125 +++++++++++++----- 5 files changed, 242 insertions(+), 97 deletions(-) create mode 100644 src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java diff --git a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java index 891e84fa..b7ce9e04 100644 --- a/src/main/java/me/cortex/voxy/client/core/VoxelCore.java +++ b/src/main/java/me/cortex/voxy/client/core/VoxelCore.java @@ -4,15 +4,12 @@ import com.mojang.blaze3d.systems.RenderSystem; import me.cortex.voxy.client.config.VoxyConfig; import me.cortex.voxy.client.core.gl.Capabilities; import me.cortex.voxy.client.core.gl.GlBuffer; -import me.cortex.voxy.client.core.model.ColourDepthTextureData; import me.cortex.voxy.client.core.model.ModelBakerySubsystem; -import me.cortex.voxy.client.core.model.ModelTextureBakery; import me.cortex.voxy.client.core.rendering.*; import me.cortex.voxy.client.core.rendering.building.RenderDataFactory4; import me.cortex.voxy.client.core.rendering.building.RenderGenerationService; import me.cortex.voxy.client.core.rendering.post.PostProcessing; import me.cortex.voxy.client.core.rendering.util.DownloadStream; -import me.cortex.voxy.client.core.rendering.util.RawDownloadStream; import me.cortex.voxy.client.core.util.IrisUtil; import me.cortex.voxy.client.saver.ContextSelectionSystem; import me.cortex.voxy.client.taskbar.Taskbar; @@ -24,7 +21,7 @@ import me.cortex.voxy.common.thread.ServiceThreadPool; import me.cortex.voxy.common.world.WorldSection; import me.cortex.voxy.common.world.other.Mapper; import me.cortex.voxy.commonImpl.VoxyCommon; -import net.minecraft.block.Blocks; +import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource; import net.minecraft.client.MinecraftClient; import net.minecraft.client.gui.hud.ClientBossBar; import net.minecraft.client.render.Camera; @@ -37,11 +34,12 @@ import net.minecraft.world.World; import net.minecraft.world.chunk.WorldChunk; import org.joml.Matrix4f; import org.lwjgl.opengl.GL11; -import org.lwjgl.system.MemoryUtil; import java.io.File; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Predicate; import static org.lwjgl.opengl.GL30C.*; @@ -73,8 +71,7 @@ public class VoxelCore { private final PostProcessing postProcessing; private final ServiceThreadPool serviceThreadPool; - private WorldImporter importer; - private UUID importerBossBarUUID; + public final WorldImportWrapper importer; public VoxelCore(ContextSelectionSystem.Selection worldSelection) { var cfg = worldSelection.getConfig(); @@ -83,6 +80,8 @@ public class VoxelCore { this.world = worldSelection.createEngine(this.serviceThreadPool); Logger.info("Initializing voxy core"); + this.importer = new WorldImportWrapper(this.serviceThreadPool, this.world); + //Trigger the shared index buffer loading SharedIndexBuffer.INSTANCE.id(); Capabilities.init();//Ensure clinit is called @@ -240,10 +239,7 @@ public class VoxelCore { //} //this.world.getMapper().forceResaveStates(); - if (this.importer != null) { - Logger.info("Shutting down importer"); - try {this.importer.shutdown();this.importer = null;} catch (Exception e) {Logger.error("Error shutting down importer", e);} - } + this.importer.shutdown(); Logger.info("Shutting down rendering"); try {this.renderer.shutdown();} catch (Exception e) {Logger.error("Error shutting down renderer", e);} Logger.info("Shutting down post processor"); @@ -253,47 +249,6 @@ public class VoxelCore { Logger.info("Shutting down service thread pool"); this.serviceThreadPool.shutdown(); Logger.info("Voxel core shut down"); - //Remove bossbar - if (this.importerBossBarUUID != null) { - MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID); - Taskbar.INSTANCE.setIsNone(); - } - } - - public boolean createWorldImporter(World mcWorld, File worldPath) { - if (this.importer == null) { - this.importer = new WorldImporter(this.world, mcWorld, this.serviceThreadPool); - } - if (this.importer.isBusy()) { - return false; - } - - Taskbar.INSTANCE.setProgress(0,10000); - Taskbar.INSTANCE.setIsProgression(); - - this.importerBossBarUUID = MathHelper.randomUuid(); - var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false); - MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar); - long start = System.currentTimeMillis(); - this.importer.importWorldAsyncStart(worldPath, (a,b)-> - MinecraftClient.getInstance().executeSync(()-> { - Taskbar.INSTANCE.setProgress(a, b); - bossBar.setPercent(((float) a)/((float) b)); - bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks")); - }), - chunkCount -> { - MinecraftClient.getInstance().executeSync(()-> { - MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID); - this.importerBossBarUUID = null; - long delta = System.currentTimeMillis() - start; - - String msg = "Voxy world import finished in " + (delta/1000) + " seconds, averaging " + (chunkCount/(delta/1000)) + " chunks per second"; - MinecraftClient.getInstance().inGameHud.getChatHud().addMessage(Text.literal(msg)); - Logger.info(msg); - Taskbar.INSTANCE.setIsNone(); - }); - }); - return true; } public WorldEngine getWorldEngine() { diff --git a/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java b/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java new file mode 100644 index 00000000..d4ff61b9 --- /dev/null +++ b/src/main/java/me/cortex/voxy/client/core/WorldImportWrapper.java @@ -0,0 +1,89 @@ +package me.cortex.voxy.client.core; + +import me.cortex.voxy.client.taskbar.Taskbar; +import me.cortex.voxy.common.Logger; +import me.cortex.voxy.common.thread.ServiceThreadPool; +import me.cortex.voxy.common.world.WorldEngine; +import me.cortex.voxy.commonImpl.importers.WorldImporter; +import net.minecraft.client.MinecraftClient; +import net.minecraft.client.gui.hud.ClientBossBar; +import net.minecraft.entity.boss.BossBar; +import net.minecraft.text.Text; +import net.minecraft.util.math.MathHelper; +import net.minecraft.world.World; + +import java.util.UUID; +import java.util.function.Consumer; + +public class WorldImportWrapper { + + private WorldImporter importer; + private final ServiceThreadPool pool; + private final WorldEngine world; + private UUID importerBossBarUUID; + + public WorldImportWrapper(ServiceThreadPool pool, WorldEngine world) { + this.pool = pool; + this.world = world; + } + + public void shutdown() { + Logger.info("Shutting down importer"); + try {this.importer.shutdown();this.importer = null;} catch (Exception e) {Logger.error("Error shutting down importer", e);} + + //Remove bossbar + if (this.importerBossBarUUID != null) { + MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID); + Taskbar.INSTANCE.setIsNone(); + } + } + + public void stopImporter() { + if (this.isImporterRunning()) { + this.importer.shutdown(); + } + } + + public interface IImporterFactory { + void create(WorldImporter importer, WorldImporter.UpdateCallback updateCallback, Consumer onCompletion); + } + public boolean createWorldImporter(World mcWorld, IImporterFactory factory) { + if (this.importer == null) { + this.importer = new WorldImporter(this.world, mcWorld, this.pool); + } + if (this.importer.isBusy()) { + return false; + } + + Taskbar.INSTANCE.setProgress(0,10000); + Taskbar.INSTANCE.setIsProgression(); + + this.importerBossBarUUID = MathHelper.randomUuid(); + var bossBar = new ClientBossBar(this.importerBossBarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false); + MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar); + long start = System.currentTimeMillis(); + factory.create(this.importer, (a, b)-> + MinecraftClient.getInstance().executeSync(()-> { + Taskbar.INSTANCE.setProgress(a, b); + bossBar.setPercent(((float) a)/((float) b)); + bossBar.setName(Text.of("Voxy import: "+ a+"/"+b + " chunks")); + }), + chunkCount -> { + MinecraftClient.getInstance().executeSync(()-> { + MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.importerBossBarUUID); + this.importerBossBarUUID = null; + long delta = System.currentTimeMillis() - start; + + String msg = "Voxy world import finished in " + (delta/1000) + " seconds, averaging " + (chunkCount/(delta/1000)) + " chunks per second"; + MinecraftClient.getInstance().inGameHud.getChatHud().addMessage(Text.literal(msg)); + Logger.info(msg); + Taskbar.INSTANCE.setIsNone(); + }); + }); + return true; + } + + public boolean isImporterRunning() { + return this.importer != null; + } +} diff --git a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java index 6faca27b..693367e0 100644 --- a/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java +++ b/src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java @@ -6,6 +6,7 @@ import com.mojang.brigadier.context.CommandContext; import com.mojang.brigadier.suggestion.Suggestions; import com.mojang.brigadier.suggestion.SuggestionsBuilder; import me.cortex.voxy.client.core.IGetVoxelCore; +import me.cortex.voxy.client.core.VoxelCore; import net.fabricmc.fabric.api.client.command.v2.ClientCommandManager; import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource; import net.minecraft.client.MinecraftClient; @@ -14,43 +15,62 @@ import net.minecraft.command.CommandSource; import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.CompletableFuture; public class WorldImportCommand { public static LiteralArgumentBuilder register() { - return ClientCommandManager.literal("voxy").then( - ClientCommandManager.literal("import") + return ClientCommandManager.literal("voxy").requires((ctx)-> ((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore() != null) + .then(ClientCommandManager.literal("import") .then(ClientCommandManager.literal("world") .then(ClientCommandManager.argument("world_name", StringArgumentType.string()) .suggests(WorldImportCommand::importWorldSuggester) .executes(WorldImportCommand::importWorld))) .then(ClientCommandManager.literal("bobby") .then(ClientCommandManager.argument("world_name", StringArgumentType.string()) + .suggests(WorldImportCommand::importBobbySuggester) .executes(WorldImportCommand::importBobby))) .then(ClientCommandManager.literal("raw") .then(ClientCommandManager.argument("path", StringArgumentType.string()) - .executes(WorldImportCommand::importRaw)))); + .executes(WorldImportCommand::importRaw))) + .then(ClientCommandManager.literal("zip") + .then(ClientCommandManager.argument("zipPath", StringArgumentType.string()) + .executes(WorldImportCommand::importZip) + .then(ClientCommandManager.argument("innerPath", StringArgumentType.string()) + .executes(WorldImportCommand::importZip)))) + .then(ClientCommandManager.literal("cancel") + .requires((ctx)->((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore().importer.isImporterRunning()) + .executes((ctx)->{((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore().importer.stopImporter(); return 0;})) + ); } + private static boolean fileBasedImporter(File directory) { + var instance = MinecraftClient.getInstance(); + var core = ((IGetVoxelCore)instance.worldRenderer).getVoxelCore(); + return core.importer.createWorldImporter(instance.player.clientWorld, + (importer, up, done)->importer.importRegionDirectoryAsyncStart(directory, up, done)); + } private static int importRaw(CommandContext ctx) { - var instance = MinecraftClient.getInstance(); - var file = new File(ctx.getArgument("path", String.class)); - ((IGetVoxelCore)instance.worldRenderer).getVoxelCore().createWorldImporter(MinecraftClient.getInstance().player.clientWorld, file); - return 0; + return fileBasedImporter(new File(ctx.getArgument("path", String.class)))?0:1; } private static int importBobby(CommandContext ctx) { - var instance = MinecraftClient.getInstance(); var file = new File(".bobby").toPath().resolve(ctx.getArgument("world_name", String.class)).toFile(); - ((IGetVoxelCore)instance.worldRenderer).getVoxelCore().createWorldImporter(MinecraftClient.getInstance().player.clientWorld, file); - return 0; + return fileBasedImporter(file)?0:1; } private static CompletableFuture importWorldSuggester(CommandContext ctx, SuggestionsBuilder sb) { + return fileDirectorySuggester(MinecraftClient.getInstance().runDirectory.toPath().resolve("saves"), sb); + } + private static CompletableFuture importBobbySuggester(CommandContext ctx, SuggestionsBuilder sb) { + return fileDirectorySuggester(new File(".bobby").toPath(), sb); + } + + private static CompletableFuture fileDirectorySuggester(Path dir, SuggestionsBuilder sb) { try { - var worlds = Files.list(MinecraftClient.getInstance().runDirectory.toPath().resolve("saves")).toList(); + var worlds = Files.list(dir).toList(); for (var world : worlds) { if (!world.toFile().isDirectory()) { continue; @@ -71,10 +91,24 @@ public class WorldImportCommand { } private static int importWorld(CommandContext ctx) { - var instance = MinecraftClient.getInstance(); var file = new File("saves").toPath().resolve(ctx.getArgument("world_name", String.class)).resolve("region").toFile(); - ((IGetVoxelCore)instance.worldRenderer).getVoxelCore().createWorldImporter(MinecraftClient.getInstance().player.clientWorld, file); - return 0; + return fileBasedImporter(file)?0:1; } + private static int importZip(CommandContext ctx) { + var zip = new File(ctx.getArgument("zipPath", String.class)); + var innerDir = "region/"; + try { + innerDir = ctx.getArgument("innerPath", String.class); + } catch (Exception e) {} + + var instance = MinecraftClient.getInstance(); + var core = ((IGetVoxelCore)instance.worldRenderer).getVoxelCore(); + if (core == null) { + return 1; + } + String finalInnerDir = innerDir; + return core.importer.createWorldImporter(instance.player.clientWorld, + (importer, up, done)->importer.importZippedRegionDirectoryAsyncStart(zip, finalInnerDir, up, done))?0:1; + } } \ No newline at end of file diff --git a/src/main/java/me/cortex/voxy/common/world/WorldSection.java b/src/main/java/me/cortex/voxy/common/world/WorldSection.java index 50ae5737..a2537fcb 100644 --- a/src/main/java/me/cortex/voxy/common/world/WorldSection.java +++ b/src/main/java/me/cortex/voxy/common/world/WorldSection.java @@ -252,6 +252,10 @@ public final class WorldSection { return prev != next; } + public void _unsafeSetNonEmptyChildren(byte nonEmptyChildren) { + NON_EMPTY_CHILD_HANDLE.set(this, nonEmptyChildren); + } + public static WorldSection _createRawUntrackedUnsafeSection(int lvl, int x, int y, int z) { return new WorldSection(lvl, x, y, z, null); } diff --git a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java index 45c143ea..b2b67399 100644 --- a/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java +++ b/src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java @@ -1,7 +1,6 @@ package me.cortex.voxy.commonImpl.importers; import com.mojang.serialization.Codec; -import me.cortex.voxy.common.util.ByteBufferBackedInputStream; import me.cortex.voxy.common.Logger; import me.cortex.voxy.common.util.MemoryBuffer; import me.cortex.voxy.common.util.UnsafeUtil; @@ -26,13 +25,15 @@ import net.minecraft.world.chunk.ChunkStatus; import net.minecraft.world.chunk.PalettedContainer; import net.minecraft.world.chunk.ReadableContainer; import net.minecraft.world.storage.ChunkCompressionFormat; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.lwjgl.system.MemoryUtil; import java.io.*; -import java.nio.ByteOrder; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; -import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicInteger; @@ -127,37 +128,83 @@ public class WorldImporter { this.threadPool.shutdown(); } + private interface IImporterMethod { + void importRegion(T file) throws Exception; + } + private volatile Thread worker; private UpdateCallback updateCallback; - public void importWorldAsyncStart(File directory, UpdateCallback updateCallback, Consumer onCompletion) { + public void importRegionDirectoryAsyncStart(File directory, UpdateCallback updateCallback, Consumer onCompletion) { + var files = directory.listFiles((dir, name) -> { + var sections = name.split("\\."); + if (sections.length != 4 || (!sections[0].equals("r")) || (!sections[3].equals("mca"))) { + Logger.error("Unknown file: " + name); + return false; + } + return true; + }); + if (files == null) { + onCompletion.accept(0); + return; + } + Arrays.sort(files, File::compareTo); + this.importRegionsAsyncStart(files, this::importRegionFile, updateCallback, onCompletion); + } + + public void importZippedRegionDirectoryAsyncStart(File zip, String innerDirectory, UpdateCallback updateCallback, Consumer onCompletion) { + try { + innerDirectory = innerDirectory.replace("\\\\", "\\").replace("\\", "/"); + var file = ZipFile.builder().setFile(zip).get(); + ArrayList regions = new ArrayList<>(); + for (var e = file.getEntries(); e.hasMoreElements();) { + var entry = e.nextElement(); + if (entry.isDirectory()||!entry.getName().startsWith(innerDirectory)) { + continue; + } + var parts = entry.getName().split("/"); + var name = parts[parts.length-1]; + var sections = name.split("\\."); + if (sections.length != 4 || (!sections[0].equals("r")) || (!sections[3].equals("mca"))) { + Logger.error("Unknown file: " + name); + continue; + } + regions.add(entry); + } + this.importRegionsAsyncStart(regions.toArray(ZipArchiveEntry[]::new), (entry)->{ + var buf = new MemoryBuffer(entry.getSize()); + try (var channel = Channels.newChannel(file.getInputStream(entry))) { + if (channel.read(buf.asByteBuffer()) != buf.size) { + buf.free(); + throw new IllegalStateException("Could not read full zip entry"); + } + } + + var parts = entry.getName().split("/"); + var name = parts[parts.length-1]; + var sections = name.split("\\."); + this.importRegion(buf, Integer.parseInt(sections[1]), Integer.parseInt(sections[2])); + buf.free(); + + }, updateCallback, onCompletion); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private void importRegionsAsyncStart(T[] regionFiles, IImporterMethod importer, UpdateCallback updateCallback, Consumer onCompletion) { this.totalChunks.set(0); this.estimatedTotalChunks.set(0); this.chunksProcessed.set(0); this.updateCallback = updateCallback; this.worker = new Thread(() -> { this.isRunning = true; - var files = directory.listFiles(); - if (files == null) { - onCompletion.accept(0); - } - Arrays.sort(files, File::compareTo); - this.estimatedTotalChunks.addAndGet(files.length*1024); - for (var file : files) { - if (!file.isFile()) { - continue; - } - var name = file.getName(); - var sections = name.split("\\."); - if (sections.length != 4 || (!sections[0].equals("r")) || (!sections[3].equals("mca"))) { - Logger.error("Unknown file: " + name); - continue; - } - int rx = Integer.parseInt(sections[1]); - int rz = Integer.parseInt(sections[2]); + this.estimatedTotalChunks.addAndGet(regionFiles.length*1024); + for (var file : regionFiles) { this.estimatedTotalChunks.addAndGet(-1024); try { - this.importRegionFile(file.toPath(), rx, rz); - } catch (IOException e) { + importer.importRegion(file); + } catch (Exception e) { throw new RuntimeException(e); } while ((this.totalChunks.get()-this.chunksProcessed.get() > 10_000) && this.isRunning) { @@ -168,7 +215,9 @@ public class WorldImporter { } } if (!this.isRunning) { + this.threadPool.blockTillEmpty(); onCompletion.accept(this.totalChunks.get()); + this.worker = null; return; } } @@ -186,31 +235,41 @@ public class WorldImporter { }); this.worker.setName("World importer"); this.worker.start(); - } public boolean isBusy() { return this.worker != null; } - private void importRegionFile(Path file, int x, int z) throws IOException { - try (var fileStream = FileChannel.open(file, StandardOpenOption.READ)) { + private void importRegionFile(File file) throws IOException { + var name = file.getName(); + var sections = name.split("\\."); + if (sections.length != 4 || (!sections[0].equals("r")) || (!sections[3].equals("mca"))) { + Logger.error("Unknown file: " + name); + throw new IllegalStateException(); + } + int rx = Integer.parseInt(sections[1]); + int rz = Integer.parseInt(sections[2]); + + try (var fileStream = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { var fileData = new MemoryBuffer(fileStream.size()); if (fileStream.read(fileData.asByteBuffer(), 0) < 8192) { fileData.free(); Logger.warn("Header of region file invalid"); return; } - this.importRegionFile(fileData, x, z); + this.importRegion(fileData, rx, rz); fileData.free(); } } - private void importRegionFile(MemoryBuffer regionFile, int x, int z) throws IOException { - //if (true) return; - + private void importRegion(MemoryBuffer regionFile, int x, int z) { //Find and load all saved chunks + if (regionFile.size < 8192) {//File not big enough + Logger.warn("Header of region file invalid"); + return; + } for (int idx = 0; idx < 1024; idx++) { int sectorMeta = Integer.reverseBytes(MemoryUtil.memGetInt(regionFile.address+idx*4));//Assumes little endian if (sectorMeta == 0) { @@ -221,6 +280,10 @@ public class WorldImporter { int sectorCount = sectorMeta&((1<<8)-1); //TODO: create memory copy for each section + if (regionFile.size < (sectorCount+sectorStart)*4096L) { + Logger.warn("Cannot access chunk sector as it goes out of bounds. start: " + sectorStart + " count: " + sectorCount + " fileSize: " + regionFile.size); + continue; + } var data = new MemoryBuffer(sectorCount*4096).cpyFrom(regionFile.address+sectorStart*4096L); boolean addedToQueue = false;