Rewired import system

src/main/java/me/cortex/voxy/client/ClientImportManager.java

@@ -0,0 +1,61 @@
+package me.cortex.voxy.client;
+
+import me.cortex.voxy.client.taskbar.Taskbar;
+import me.cortex.voxy.common.Logger;
+import me.cortex.voxy.common.thread.ServiceThreadPool;
+import me.cortex.voxy.commonImpl.ImportManager;
+import me.cortex.voxy.commonImpl.importers.IDataImporter;
+import net.minecraft.client.MinecraftClient;
+import net.minecraft.client.gui.hud.ClientBossBar;
+import net.minecraft.entity.boss.BossBar;
+import net.minecraft.text.Text;
+import net.minecraft.util.math.MathHelper;
+
+import java.util.UUID;
+import java.util.function.BooleanSupplier;
+
+public class ClientImportManager extends ImportManager {
+    protected class ClientImportTask extends ImportTask {
+        private final UUID bossbarUUID;
+        private final ClientBossBar bossBar;
+        protected ClientImportTask(IDataImporter importer) {
+            super(importer);
+
+            this.bossbarUUID = MathHelper.randomUuid();
+            this.bossBar = new ClientBossBar(this.bossbarUUID, Text.of("Voxy world importer"), 0.0f, BossBar.Color.GREEN, BossBar.Style.PROGRESS, false, false, false);
+            MinecraftClient.getInstance().execute(()->{
+                MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.put(bossBar.getUuid(), bossBar);
+            });
+        }
+
+        @Override
+        protected boolean onUpdate(int completed, int outOf) {
+            if (!super.onUpdate(completed, outOf)) {
+                return false;
+            }
+            MinecraftClient.getInstance().execute(()->{
+                this.bossBar.setPercent((float) (((double)completed) / ((double) Math.max(1, outOf))));
+                this.bossBar.setName(Text.of("Voxy import: " + completed + "/" + outOf + " chunks"));
+            });
+            return true;
+        }
+
+        @Override
+        protected void onCompleted(int total) {
+            super.onCompleted(total);
+            MinecraftClient.getInstance().execute(()->{
+                MinecraftClient.getInstance().inGameHud.getBossBarHud().bossBars.remove(this.bossbarUUID);
+                long delta = Math.max(System.currentTimeMillis() - this.startTime, 1);
+
+                String msg = "Voxy world import finished in " + (delta/1000) + " seconds, averaging " + (int)(total/(delta/1000f)) + " chunks per second";
+                MinecraftClient.getInstance().inGameHud.getChatHud().addMessage(Text.literal(msg));
+                Logger.info(msg);
+            });
+        }
+    }
+
+    @Override
+    protected synchronized ImportTask createImportTask(IDataImporter importer) {
+        return new ClientImportTask(importer);
+    }
+}

src/main/java/me/cortex/voxy/client/VoxyClientInstance.java

@@ -3,27 +3,27 @@ package me.cortex.voxy.client;
 import me.cortex.voxy.client.config.VoxyConfig;
 import me.cortex.voxy.client.core.WorldImportWrapper;
 import me.cortex.voxy.client.saver.ContextSelectionSystem;
+import me.cortex.voxy.common.Logger;
 import me.cortex.voxy.common.world.WorldEngine;
 import me.cortex.voxy.commonImpl.IVoxyWorldGetter;
 import me.cortex.voxy.commonImpl.IVoxyWorldSetter;
+import me.cortex.voxy.commonImpl.ImportManager;
 import me.cortex.voxy.commonImpl.VoxyInstance;
+import me.cortex.voxy.commonImpl.importers.DHImporter;
 import net.minecraft.client.world.ClientWorld;
 
+import java.io.File;
+
 public class VoxyClientInstance extends VoxyInstance {
     private static final ContextSelectionSystem SELECTOR = new ContextSelectionSystem();
-    public WorldImportWrapper importWrapper;
 
     public VoxyClientInstance() {
         super(VoxyConfig.CONFIG.serviceThreads);
     }
 
     @Override
-    public void stopWorld(WorldEngine world) {
+    protected ImportManager createImportManager() {
-        if (this.importWrapper != null) {
+        return new ClientImportManager();
-            this.importWrapper.stopImporter();
-            this.importWrapper = null;
-        }
-        super.stopWorld(world);
     }
 
     public WorldEngine getOrMakeRenderWorld(ClientWorld world) {
@@ -31,7 +31,6 @@ public class VoxyClientInstance extends VoxyInstance {
         if (vworld == null) {
             vworld = this.createWorld(SELECTOR.getBestSelectionOrCreate(world).createSectionStorageBackend());
             ((IVoxyWorldSetter)world).setWorldEngine(vworld);
-            this.importWrapper = new WorldImportWrapper(this.threadPool, vworld);
         } else {
             if (!this.activeWorlds.contains(vworld)) {
                 throw new IllegalStateException("World referenced does not exist in instance");

src/main/java/me/cortex/voxy/client/core/model/IdNotYetComputedException.java

@@ -3,7 +3,6 @@ package me.cortex.voxy.client.core.model;
 public class IdNotYetComputedException extends RuntimeException {
     public final int id;
     public IdNotYetComputedException(int id) {
-        //super("Id not yet computed: " + id);
         super(null, null, false, false);
         this.id = id;
     }

src/main/java/me/cortex/voxy/client/terrain/WorldImportCommand.java

@@ -8,6 +8,8 @@ import com.mojang.brigadier.suggestion.SuggestionsBuilder;
 import me.cortex.voxy.client.VoxyClientInstance;
 import me.cortex.voxy.commonImpl.VoxyCommon;
 import me.cortex.voxy.commonImpl.VoxyInstance;
+import me.cortex.voxy.commonImpl.importers.DHImporter;
+import me.cortex.voxy.commonImpl.importers.WorldImporter;
 import net.fabricmc.fabric.api.client.command.v2.ClientCommandManager;
 import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource;
 import net.minecraft.client.MinecraftClient;
@@ -41,19 +43,47 @@ public class WorldImportCommand {
                                         .executes(WorldImportCommand::importZip)
                                         .then(ClientCommandManager.argument("innerPath", StringArgumentType.string())
                                                 .executes(WorldImportCommand::importZip))))
+                        .then(ClientCommandManager.literal("distant_horizons")
+                                .then(ClientCommandManager.argument("sqlDbPath", StringArgumentType.string())
+                                        .executes(WorldImportCommand::importDistantHorizons)))
                         .then(ClientCommandManager.literal("cancel")
-                                //.requires((ctx)->((IGetVoxelCore)MinecraftClient.getInstance().worldRenderer).getVoxelCore().importer.isImporterRunning())
                                 .executes(WorldImportCommand::cancelImport))
         );
     }
 
+    private static int importDistantHorizons(CommandContext<FabricClientCommandSource> ctx) {
+        var instance = (VoxyClientInstance)VoxyCommon.getInstance();
+        if (instance == null) {
+            return 1;
+        }
+        var dbFile = new File(ctx.getArgument("sqlDbPath", String.class));
+        if (!dbFile.exists()) {
+            return 1;
+        }
+        if (dbFile.isDirectory()) {
+            dbFile = dbFile.toPath().resolve("DistantHorizons.sqlite").toFile();
+            if (!dbFile.exists()) {
+                return 1;
+            }
+        }
+
+        File dbFile_ = dbFile;
+        var engine = instance.getOrMakeRenderWorld(MinecraftClient.getInstance().player.clientWorld);
+        return instance.getImportManager().makeAndRunIfNone(engine, ()->
+                new DHImporter(dbFile_, engine, MinecraftClient.getInstance().player.clientWorld, instance.getThreadPool(), instance.getSavingService()))?0:1;
+    }
+
     private static boolean fileBasedImporter(File directory) {
         var instance = (VoxyClientInstance)VoxyCommon.getInstance();
         if (instance == null) {
             return false;
         }
-        return instance.importWrapper.createWorldImporter(MinecraftClient.getInstance().player.clientWorld,
+        var engine = instance.getOrMakeRenderWorld(MinecraftClient.getInstance().player.clientWorld);
-                (importer)->importer.importRegionDirectoryAsyncStart(directory));
+        return instance.getImportManager().makeAndRunIfNone(engine, ()->{
+            var importer = new WorldImporter(engine, MinecraftClient.getInstance().player.clientWorld, instance.getThreadPool(), instance.getSavingService());
+            importer.importRegionDirectoryAsync(directory);
+            return importer;
+        });
     }
 
     private static int importRaw(CommandContext<FabricClientCommandSource> ctx) {
@@ -139,8 +169,13 @@ public class WorldImportCommand {
             return 1;
         }
         String finalInnerDir = innerDir;
-        return instance.importWrapper.createWorldImporter(MinecraftClient.getInstance().player.clientWorld,
+
-                (importer)->importer.importZippedRegionDirectoryAsyncStart(zip, finalInnerDir))?0:1;
+        var engine = instance.getOrMakeRenderWorld(MinecraftClient.getInstance().player.clientWorld);
+        return instance.getImportManager().makeAndRunIfNone(engine, ()->{
+            var importer = new WorldImporter(engine, MinecraftClient.getInstance().player.clientWorld, instance.getThreadPool(), instance.getSavingService());
+            importer.importZippedRegionDirectoryAsync(zip, finalInnerDir);
+            return importer;
+        })?0:1;
     }
 
     private static int cancelImport(CommandContext<FabricClientCommandSource> fabricClientCommandSourceCommandContext) {
@@ -148,7 +183,7 @@ public class WorldImportCommand {
         if (instance == null) {
             return 1;
         }
-        instance.importWrapper.stopImporter();
+        var world = instance.getOrMakeRenderWorld(MinecraftClient.getInstance().player.clientWorld);
-        return 0;
+        return instance.getImportManager().cancelImport(world)?0:1;
     }
 }

src/main/java/me/cortex/voxy/commonImpl/ImportManager.java

@@ -0,0 +1,121 @@
+package me.cortex.voxy.commonImpl;
+
+import me.cortex.voxy.common.thread.ServiceThreadPool;
+import me.cortex.voxy.common.world.WorldEngine;
+import me.cortex.voxy.commonImpl.importers.IDataImporter;
+import me.cortex.voxy.commonImpl.importers.WorldImporter;
+import net.minecraft.world.World;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.function.BooleanSupplier;
+import java.util.function.IntConsumer;
+import java.util.function.Supplier;
+
+public class ImportManager {
+    private final Map<WorldEngine, ImportTask> activeImporters = new HashMap<>();
+
+    protected class ImportTask {
+        protected final IDataImporter importer;
+        protected long startTime;
+        protected long timer;
+        protected long updateEvery = 50;
+
+        protected ImportTask(IDataImporter importer) {
+            this.importer = importer;
+            this.timer = System.currentTimeMillis();
+        }
+
+        private void start() {
+            if (this.importer.isRunning()) {
+                throw new IllegalStateException();
+            }
+            this.startTime = System.currentTimeMillis();
+            this.importer.runImport(this::onUpdate, this::onCompleted);
+        }
+
+        protected boolean onUpdate(int completed, int outOf) {
+            if (System.currentTimeMillis() - this.timer < this.updateEvery)
+                return false;
+            this.timer = System.currentTimeMillis();
+
+            //TODO: THING
+
+            return true;
+        }
+
+        protected void onCompleted(int total) {
+            ImportManager.this.jobFinished(this);
+        }
+
+        protected void shutdown() {
+            this.importer.shutdown();
+        }
+
+        protected boolean isCompleted() {
+            return !this.importer.isRunning();
+        }
+    }
+
+    protected synchronized ImportTask createImportTask(IDataImporter importer) {
+        return new ImportTask(importer);
+    }
+
+    public boolean tryRunImport(IDataImporter importer) {
+        ImportTask task;
+        synchronized (this) {
+            {
+                var importerTask = this.activeImporters.get(importer.getEngine());
+                if (importerTask != null) {
+                    if (!importerTask.isCompleted()) {
+                        return false;
+                    } else {
+                        throw new IllegalStateException();
+                    }
+                }
+            }
+            task = this.createImportTask(importer);
+            this.activeImporters.put(importer.getEngine(), task);
+        }
+        task.start();
+        return true;
+    }
+
+    public boolean makeAndRunIfNone(WorldEngine engine, Supplier<IDataImporter> factory) {
+        synchronized (this) {
+            if (this.activeImporters.containsKey(engine)) {
+                return false;
+            }
+        }
+        return this.tryRunImport(factory.get());
+    }
+
+    public boolean cancelImport(WorldEngine engine) {
+        ImportTask task;
+        synchronized (this) {
+            task = this.activeImporters.get(engine);
+            if (task == null) {
+                return false;
+            }
+        }
+        task.shutdown();
+        synchronized (this) {
+            this.activeImporters.remove(engine);
+        }
+        return true;
+    }
+
+    private synchronized void jobFinished(ImportTask task) {
+        if (!task.isCompleted()) {
+            throw new IllegalStateException();
+        }
+
+        var remTask = this.activeImporters.remove(task.importer.getEngine());
+        if (remTask != null) {
+            if (remTask != task) {
+                throw new IllegalStateException();
+            }
+        }
+    }
+}

src/main/java/me/cortex/voxy/commonImpl/VoxyInstance.java

@@ -1,5 +1,6 @@
 package me.cortex.voxy.commonImpl;
 
+import me.cortex.voxy.client.core.WorldImportWrapper;
 import me.cortex.voxy.common.Logger;
 import me.cortex.voxy.common.config.section.SectionStorage;
 import me.cortex.voxy.common.thread.ServiceThreadPool;
@@ -20,11 +21,18 @@ public class VoxyInstance {
     protected final VoxelIngestService ingestService;
     protected final Set<WorldEngine> activeWorlds = new HashSet<>();
 
+    protected final ImportManager importManager;
+
     public VoxyInstance(int threadCount) {
         Logger.info("Initializing voxy instance");
         this.threadPool = new ServiceThreadPool(threadCount);
         this.savingService = new SectionSavingService(this.threadPool);
         this.ingestService = new VoxelIngestService(this.threadPool);
+        this.importManager = this.createImportManager();
+    }
+
+    protected ImportManager createImportManager() {
+        return new ImportManager();
     }
 
     public void addDebug(List<String> debug) {
@@ -36,6 +44,12 @@ public class VoxyInstance {
     public void shutdown() {
         Logger.info("Shutdown voxy instance");
 
+        if (!this.activeWorlds.isEmpty()) {
+            for (var world : this.activeWorlds) {
+                this.importManager.cancelImport(world);
+            }
+        }
+
         try {this.ingestService.shutdown();} catch (Exception e) {Logger.error(e);}
         try {this.savingService.shutdown();} catch (Exception e) {Logger.error(e);}
 
@@ -65,6 +79,10 @@ public class VoxyInstance {
         return this.savingService;
     }
 
+    public ImportManager getImportManager() {
+        return this.importManager;
+    }
+
     public void flush() {
         try {
             while (this.ingestService.getTaskCount() != 0) {
@@ -106,6 +124,8 @@ public class VoxyInstance {
             throw new IllegalStateException("World cannot be in world set and not alive");
         }
 
+        this.importManager.cancelImport(world);
+
         this.flush();
 
         world.free();

src/main/java/me/cortex/voxy/commonImpl/importers/DHImporter.java

@@ -337,6 +337,8 @@ public class DHImporter implements IDataImporter {
             return;
         }
         this.isRunning = false;
+        while (!this.tasks.isEmpty())
+            this.tasks.poll();
         try {
             if (this.runner != Thread.currentThread()) {
                 this.runner.join();
@@ -359,6 +361,11 @@ public class DHImporter implements IDataImporter {
         return this.isRunning;
     }
 
+    @Override
+    public WorldEngine getEngine() {
+        return this.engine;
+    }
+
     private static VarHandle create(Class<?> viewArrayClass) {
         return MethodHandles.byteArrayViewVarHandle(viewArrayClass, ByteOrder.BIG_ENDIAN);
     }

src/main/java/me/cortex/voxy/commonImpl/importers/IDataImporter.java

@@ -0,0 +1,15 @@
+package me.cortex.voxy.commonImpl.importers;
+
+import me.cortex.voxy.common.world.WorldEngine;
+
+public interface IDataImporter {
+    interface ICompletionCallback{void onCompletion(int chunks);}
+    interface IUpdateCallback{void onUpdate(int finished, int outOf);}
+
+    void runImport(IUpdateCallback updateCallback, ICompletionCallback completionCallback);
+
+    WorldEngine getEngine();
+
+    void shutdown();
+    boolean isRunning();
+}

src/main/java/me/cortex/voxy/commonImpl/importers/WorldImporter.java

@@ -128,6 +128,11 @@ public class WorldImporter implements IDataImporter {
         this.worker.start();
     }
 
+    @Override
+    public WorldEngine getEngine() {
+        return this.world;
+    }
+
     public void shutdown() {
         this.isRunning = false;
         if (this.worker != null) {
@@ -149,7 +154,7 @@ public class WorldImporter implements IDataImporter {
     private volatile Thread worker;
     private IUpdateCallback updateCallback;
     private ICompletionCallback completionCallback;
-    public void importRegionDirectoryAsyncStart(File directory) {
+    public void importRegionDirectoryAsync(File directory) {
         var files = directory.listFiles((dir, name) -> {
             var sections = name.split("\\.");
             if (sections.length != 4 || (!sections[0].equals("r")) || (!sections[3].equals("mca"))) {
@@ -162,10 +167,10 @@ public class WorldImporter implements IDataImporter {
             return;
         }
         Arrays.sort(files, File::compareTo);
-        this.importRegionsAsyncStart(files, this::importRegionFile);
+        this.importRegionsAsync(files, this::importRegionFile);
     }
 
-    public void importZippedRegionDirectoryAsyncStart(File zip, String innerDirectory) {
+    public void importZippedRegionDirectoryAsync(File zip, String innerDirectory) {
         try {
             innerDirectory = innerDirectory.replace("\\\\", "\\").replace("\\", "/");
             var file = ZipFile.builder().setFile(zip).get();
@@ -184,7 +189,7 @@ public class WorldImporter implements IDataImporter {
                 }
                 regions.add(entry);
             }
-            this.importRegionsAsyncStart(regions.toArray(ZipArchiveEntry[]::new), (entry)->{
+            this.importRegionsAsync(regions.toArray(ZipArchiveEntry[]::new), (entry)->{
                 var buf = new MemoryBuffer(entry.getSize());
                 try (var channel = Channels.newChannel(file.getInputStream(entry))) {
                     if (channel.read(buf.asByteBuffer()) != buf.size) {
@@ -206,7 +211,7 @@ public class WorldImporter implements IDataImporter {
 
     }
 
-    private <T> void importRegionsAsyncStart(T[] regionFiles, IImporterMethod<T> importer) {
+    private <T> void importRegionsAsync(T[] regionFiles, IImporterMethod<T> importer) {
         this.totalChunks.set(0);
         this.estimatedTotalChunks.set(0);
         this.chunksProcessed.set(0);