From 81b5b4812a00b8cc64b7294ae39684d839adfeb4 Mon Sep 17 00:00:00 2001 From: IzzelAliz Date: Tue, 20 Apr 2021 19:49:51 +0800 Subject: [PATCH] Implement plugin class cache Plugins are expected to load faster on second run. --- .../mixin/bukkit/PluginClassLoaderMixin.java | 29 +- .../mod/util/remapper/ArclightClassCache.java | 262 ++++++++++++++++++ .../util/remapper/ClassLoaderRemapper.java | 49 +++- .../generated/ArclightReflectionHandler.java | 3 +- .../generated/RemappingURLClassLoader.java | 3 +- build.gradle | 2 +- .../arclight/i18n/conf/OptimizationSpec.java | 7 + .../src/main/resources/META-INF/arclight.conf | 1 + 8 files changed, 338 insertions(+), 18 deletions(-) create mode 100644 arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ArclightClassCache.java diff --git a/arclight-common/src/main/java/io/izzel/arclight/common/mixin/bukkit/PluginClassLoaderMixin.java b/arclight-common/src/main/java/io/izzel/arclight/common/mixin/bukkit/PluginClassLoaderMixin.java index c4a296d2..8214caf5 100644 --- a/arclight-common/src/main/java/io/izzel/arclight/common/mixin/bukkit/PluginClassLoaderMixin.java +++ b/arclight-common/src/main/java/io/izzel/arclight/common/mixin/bukkit/PluginClassLoaderMixin.java @@ -23,6 +23,7 @@ import java.net.URLConnection; import java.security.CodeSigner; import java.security.CodeSource; import java.util.Map; +import java.util.concurrent.Callable; import java.util.jar.Manifest; @Mixin(targets = "org.bukkit.plugin.java.PluginClassLoader", remap = false) @@ -71,29 +72,31 @@ public class PluginClassLoaderMixin extends URLClassLoader implements RemappingC URL url = this.findResource(path); if (url != null) { - byte[] classBytes; URLConnection connection; CodeSigner[] signers; + Callable byteSource; try { connection = url.openConnection(); - try (InputStream is = connection.getInputStream()) { - classBytes = ByteStreams.toByteArray(is); - if (connection instanceof JarURLConnection) { - signers = ((JarURLConnection) connection).getJarEntry().getCodeSigners(); - } else { - signers = new CodeSigner[0]; - } - } catch (IOException ex) { - throw new ClassNotFoundException(name, ex); + connection.connect(); + if (connection instanceof JarURLConnection) { + signers = ((JarURLConnection) connection).getJarEntry().getCodeSigners(); + } else { + signers = new CodeSigner[0]; } + byteSource = () -> { + try (InputStream is = connection.getInputStream()) { + byte[] classBytes = ByteStreams.toByteArray(is); + classBytes = SwitchTableFixer.INSTANCE.processClass(classBytes); + classBytes = Bukkit.getUnsafe().processClass(description, path, classBytes); + return classBytes; + } + }; } catch (IOException e) { throw new ClassNotFoundException(name, e); } - classBytes = SwitchTableFixer.INSTANCE.processClass(classBytes); - classBytes = Bukkit.getUnsafe().processClass(description, path, classBytes); - classBytes = this.getRemapper().remapClass(classBytes); + byte[] classBytes = this.getRemapper().remapClass(name, byteSource, connection); int dot = name.lastIndexOf('.'); if (dot != -1) { diff --git a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ArclightClassCache.java b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ArclightClassCache.java new file mode 100644 index 00000000..61261ca4 --- /dev/null +++ b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ArclightClassCache.java @@ -0,0 +1,262 @@ +package io.izzel.arclight.common.mod.util.remapper; + +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import cpw.mods.modlauncher.api.LamdbaExceptionUtils; +import io.izzel.arclight.common.mod.ArclightMod; +import io.izzel.arclight.i18n.ArclightConfig; +import io.izzel.tools.product.Product; +import io.izzel.tools.product.Product2; +import io.izzel.tools.product.Product4; +import net.minecraftforge.fml.ModList; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.JarURLConnection; +import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.jar.JarFile; + +public abstract class ArclightClassCache implements AutoCloseable { + + public abstract CacheSegment makeSegment(URLConnection connection) throws IOException; + + public abstract void save() throws IOException; + + public interface CacheSegment { + + Optional findByName(String name) throws IOException; + + void addToCache(String name, byte[] value); + + void save() throws IOException; + } + + private static final Marker MARKER = MarkerManager.getMarker("CLCACHE"); + private static final ArclightClassCache INSTANCE = new Impl(); + + public static ArclightClassCache instance() { + return INSTANCE; + } + + private static class Impl extends ArclightClassCache { + + private final boolean enabled = ArclightConfig.spec().getOptimization().isCachePluginClass(); + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + private final Path basePath = Paths.get(".arclight/class_cache"); + private ScheduledExecutorService executor; + + public Impl() { + if (!enabled) return; + executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r); + thread.setName("arclight class cache saving thread"); + thread.setDaemon(true); + return thread; + }); + executor.scheduleWithFixedDelay(() -> { + try { + this.save(); + } catch (IOException e) { + ArclightMod.LOGGER.error(MARKER, "Failed to save class cache", e); + } + }, 1, 10, TimeUnit.MINUTES); + try { + if (Files.isRegularFile(basePath)) { + Files.delete(basePath); + } + if (!Files.isDirectory(basePath)) { + Files.createDirectories(basePath); + } + String current = ModList.get().getModContainerById("arclight") + .orElseThrow(IllegalStateException::new).getModInfo().getVersion().toString(); + String store; + Path version = basePath.resolve(".version"); + if (Files.exists(version)) { + store = new String(Files.readAllBytes(version), StandardCharsets.UTF_8); + } else { + store = null; + } + boolean obsolete = !Objects.equals(current, store); + Path index = basePath.resolve("index"); + if (obsolete) { + FileUtils.deleteDirectory(index.toFile()); + } + if (!Files.exists(index)) { + Files.createDirectories(index); + } + Path blob = basePath.resolve("blob"); + if (obsolete) { + FileUtils.deleteDirectory(blob.toFile()); + } + if (!Files.exists(blob)) { + Files.createDirectories(blob); + } + if (obsolete) { + Files.write(version, current.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + ArclightMod.LOGGER.info(MARKER, "Obsolete plugin class cache is cleared"); + } + } catch (IOException e) { + ArclightMod.LOGGER.error(MARKER, "Failed to initialize class cache", e); + } + Thread thread = new Thread(() -> { + try { + this.close(); + } catch (Exception e) { + ArclightMod.LOGGER.error(MARKER, "Failed to close class cache", e); + } + }, "arclight class cache cleanup"); + thread.setDaemon(true); + Runtime.getRuntime().addShutdownHook(thread); + } + + @Override + public CacheSegment makeSegment(URLConnection connection) throws IOException { + if (enabled && connection instanceof JarURLConnection) { + JarFile file = ((JarURLConnection) connection).getJarFile(); + return this.map.computeIfAbsent(file.getName(), LamdbaExceptionUtils.rethrowFunction(JarSegment::new)); + } else { + return new EmptySegment(); + } + } + + @Override + public void save() throws IOException { + if (enabled) { + for (CacheSegment segment : map.values()) { + segment.save(); + } + } + } + + @Override + public void close() throws Exception { + if (enabled) { + save(); + executor.shutdownNow(); + } + } + + private class JarSegment implements CacheSegment { + + private final Map> rangeMap = new ConcurrentHashMap<>(); + private final ConcurrentLinkedQueue> savingQueue = new ConcurrentLinkedQueue<>(); + private final AtomicLong sizeAllocator; + private final Path indexPath, blobPath; + + private JarSegment(String fileName) throws IOException { + Path jarFile = new File(fileName).toPath(); + Hasher hasher = Hashing.sha256().newHasher(); + hasher.putBytes(Files.readAllBytes(jarFile)); + String hash = hasher.hash().toString(); + this.indexPath = basePath.resolve("index").resolve(hash); + this.blobPath = basePath.resolve("blob").resolve(hash); + if (!Files.exists(indexPath)) { + Files.createFile(indexPath); + } + if (!Files.exists(blobPath)) { + Files.createFile(blobPath); + } + sizeAllocator = new AtomicLong(Files.size(blobPath)); + read(); + } + + @Override + public Optional findByName(String name) throws IOException { + Product2 product2 = rangeMap.get(name); + if (product2 != null) { + long off = product2._1; + int len = product2._2; + try (SeekableByteChannel channel = Files.newByteChannel(blobPath)) { + channel.position(off); + ByteBuffer buffer = ByteBuffer.allocate(len); + channel.read(buffer); + return Optional.of(buffer.array()); + } + } else { + return Optional.empty(); + } + } + + @Override + public void addToCache(String name, byte[] value) { + int len = value.length; + long off = sizeAllocator.getAndAdd(len); + savingQueue.add(Product.of(name, value, off, len)); + } + + @Override + public synchronized void save() throws IOException { + if (savingQueue.isEmpty()) return; + List> list = new ArrayList<>(); + while (!savingQueue.isEmpty()) { + list.add(savingQueue.poll()); + } + try (OutputStream outIndex = Files.newOutputStream(indexPath, StandardOpenOption.APPEND); + DataOutputStream dataOutIndex = new DataOutputStream(outIndex); + SeekableByteChannel channel = Files.newByteChannel(blobPath, StandardOpenOption.WRITE)) { + for (Product4 product4 : list) { + channel.position(product4._3); + channel.write(ByteBuffer.wrap(product4._2)); + dataOutIndex.writeUTF(product4._1); + dataOutIndex.writeLong(product4._3); + dataOutIndex.writeInt(product4._4); + rangeMap.put(product4._1, Product.of(product4._3, product4._4)); + } + } + } + + private synchronized void read() throws IOException { + try (InputStream inputStream = Files.newInputStream(indexPath); + DataInputStream dataIn = new DataInputStream(inputStream)) { + while (dataIn.available() > 0) { + String name = dataIn.readUTF(); + long off = dataIn.readLong(); + int len = dataIn.readInt(); + rangeMap.put(name, Product.of(off, len)); + } + } + } + } + + private static class EmptySegment implements CacheSegment { + + @Override + public Optional findByName(String name) { + return Optional.empty(); + } + + @Override + public void addToCache(String name, byte[] value) { + } + + @Override + public void save() { + } + } + } +} diff --git a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ClassLoaderRemapper.java b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ClassLoaderRemapper.java index 016115c4..c69373f8 100644 --- a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ClassLoaderRemapper.java +++ b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/ClassLoaderRemapper.java @@ -6,6 +6,7 @@ import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; import io.izzel.arclight.api.Unsafe; import io.izzel.arclight.common.mod.util.remapper.generated.ArclightReflectionHandler; +import io.izzel.arclight.i18n.ArclightConfig; import net.md_5.specialsource.JarMapping; import net.md_5.specialsource.JarRemapper; import net.md_5.specialsource.RemappingClassAdapter; @@ -27,12 +28,15 @@ import java.io.File; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.net.URLConnection; import java.nio.file.Files; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; @@ -40,12 +44,14 @@ public class ClassLoaderRemapper extends LenientJarRemapper { private static final Logger LOGGER = LogManager.getLogger("Arclight"); private static final String PREFIX = "net/minecraft/"; + private static final String REPLACED_NAME = Type.getInternalName(ArclightReflectionHandler.class); private final JarMapping toBukkitMapping; private final JarRemapper toBukkitRemapper; private final ClassLoader classLoader; private final String generatedHandler; private final Class generatedHandlerClass; + private final GeneratedHandlerAdapter generatedHandlerAdapter; public ClassLoaderRemapper(JarMapping jarMapping, JarMapping toBukkitMapping, ClassLoader classLoader) { super(jarMapping); @@ -57,6 +63,7 @@ public class ClassLoaderRemapper extends LenientJarRemapper { this.toBukkitRemapper = new LenientJarRemapper(this.toBukkitMapping); this.generatedHandlerClass = generateReflectionHandler(); this.generatedHandler = Type.getInternalName(generatedHandlerClass); + this.generatedHandlerAdapter = new GeneratedHandlerAdapter(REPLACED_NAME, generatedHandler); GlobalClassRepo.INSTANCE.addRepo(new ClassLoaderRepo(this.classLoader)); } @@ -272,8 +279,28 @@ public class ClassLoaderRemapper extends LenientJarRemapper { return Maps.immutableEntry(owner, mapped); } - public byte[] remapClass(byte[] arr) { - return remapClassFile(arr, GlobalClassRepo.INSTANCE); + public byte[] remapClass(String className, Callable byteSource, URLConnection connection) throws ClassNotFoundException { + try { + ArclightClassCache.CacheSegment segment = ArclightClassCache.instance().makeSegment(connection); + Optional optional = segment.findByName(className); + if (optional.isPresent()) { + byte[] bytes = optional.get(); + ClassWriter cw = new ClassWriter(0); + new ClassReader(bytes).accept(new ClassRemapper(cw, generatedHandlerAdapter), 0); + return cw.toByteArray(); + } else { + byte[] bytes = remapClassFile(byteSource.call(), GlobalClassRepo.INSTANCE); + if (ArclightConfig.spec().getOptimization().isCachePluginClass()) { + ClassWriter cw = new ClassWriter(0); + new ClassReader(bytes).accept(new ClassRemapper(cw, new GeneratedHandlerAdapter(generatedHandler, REPLACED_NAME)), 0); + byte[] store = cw.toByteArray(); + segment.addToCache(className, store); + } + return bytes; + } + } catch (Exception e) { + throw new ClassNotFoundException(className, e); + } } @Override @@ -368,7 +395,25 @@ public class ClassLoaderRemapper extends LenientJarRemapper { } return ArclightRemapper.getNmsMapper().map(node.superName); } + } + private static class GeneratedHandlerAdapter extends Remapper { + + private final String from, to; + + private GeneratedHandlerAdapter(String from, String to) { + this.from = from; + this.to = to; + } + + @Override + public String map(String internalName) { + if (from.equals(internalName)) { + return to; + } else { + return internalName; + } + } } static class WrappedMethod { diff --git a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/ArclightReflectionHandler.java b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/ArclightReflectionHandler.java index 9c7eb9cf..ce5ea1b5 100644 --- a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/ArclightReflectionHandler.java +++ b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/ArclightReflectionHandler.java @@ -4,6 +4,7 @@ import io.izzel.arclight.api.ArclightVersion; import io.izzel.arclight.api.Unsafe; import io.izzel.arclight.common.mod.util.remapper.ArclightRedirectAdapter; import io.izzel.arclight.common.mod.util.remapper.ClassLoaderRemapper; +import io.izzel.arclight.common.mod.util.remapper.GlobalClassRepo; import io.izzel.arclight.common.mod.util.remapper.RemappingClassLoader; import io.izzel.arclight.common.util.Enumerations; import org.objectweb.asm.ClassReader; @@ -471,7 +472,7 @@ public class ArclightReflectionHandler extends ClassLoader { } } if (rcl != null) { - return rcl.getRemapper().remapClass(bytes); + return rcl.getRemapper().remapClassFile(bytes, GlobalClassRepo.INSTANCE); } else { ArclightRedirectAdapter.scanMethod(bytes); return bytes; diff --git a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/RemappingURLClassLoader.java b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/RemappingURLClassLoader.java index 75320039..77c1ca0f 100644 --- a/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/RemappingURLClassLoader.java +++ b/arclight-common/src/main/java/io/izzel/arclight/common/mod/util/remapper/generated/RemappingURLClassLoader.java @@ -3,6 +3,7 @@ package io.izzel.arclight.common.mod.util.remapper.generated; import com.google.common.io.ByteStreams; import io.izzel.arclight.common.mod.util.remapper.ArclightRemapper; import io.izzel.arclight.common.mod.util.remapper.ClassLoaderRemapper; +import io.izzel.arclight.common.mod.util.remapper.GlobalClassRepo; import io.izzel.arclight.common.mod.util.remapper.RemappingClassLoader; import java.io.IOException; @@ -39,7 +40,7 @@ public class RemappingURLClassLoader extends URLClassLoader implements Remapping if (resource != null) { try { URLConnection connection = resource.openConnection(); - byte[] bytes = getRemapper().remapClass(ByteStreams.toByteArray(connection.getInputStream())); + byte[] bytes = getRemapper().remapClassFile(ByteStreams.toByteArray(connection.getInputStream()), GlobalClassRepo.INSTANCE); int i = name.lastIndexOf('.'); if (i != -1) { String pkgName = name.substring(0, i); diff --git a/build.gradle b/build.gradle index 836eea44..45c97307 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ allprojects { group 'io.izzel.arclight' - version '1.0.17' + version '1.0.18-SNAPSHOT' ext { agpVersion = '1.15' diff --git a/i18n-config/src/main/java/io/izzel/arclight/i18n/conf/OptimizationSpec.java b/i18n-config/src/main/java/io/izzel/arclight/i18n/conf/OptimizationSpec.java index a0ed7ad9..60595ca7 100644 --- a/i18n-config/src/main/java/io/izzel/arclight/i18n/conf/OptimizationSpec.java +++ b/i18n-config/src/main/java/io/izzel/arclight/i18n/conf/OptimizationSpec.java @@ -9,7 +9,14 @@ public class OptimizationSpec { @Setting("remove-stream") private boolean removeStream; + @Setting("cache-plugin-class") + private boolean cachePluginClass; + public boolean isRemoveStream() { return removeStream; } + + public boolean isCachePluginClass() { + return cachePluginClass; + } } diff --git a/i18n-config/src/main/resources/META-INF/arclight.conf b/i18n-config/src/main/resources/META-INF/arclight.conf index 26425bcd..9f55a453 100644 --- a/i18n-config/src/main/resources/META-INF/arclight.conf +++ b/i18n-config/src/main/resources/META-INF/arclight.conf @@ -5,6 +5,7 @@ locale { } optimization { remove-stream = true + cache-plugin-class = true } compatibility { material-property-overrides {