From c2794831dab7f167d23a785971ed4a0aa0900670 Mon Sep 17 00:00:00 2001 From: Robbert van der Helm Date: Sat, 16 Apr 2022 18:21:02 +0200 Subject: [PATCH] Add (for now, hardcoded) chainloader libraries --- CHANGELOG.md | 22 +++++ docs/architecture.md | 37 ++++--- meson.build | 43 ++++++++ src/chainloader/meson.build | 16 +++ src/chainloader/vst2-chainloader.cpp | 105 ++++++++++++++++++++ src/chainloader/vst3-chainloader.cpp | 143 +++++++++++++++++++++++++++ 6 files changed, 355 insertions(+), 11 deletions(-) create mode 100644 src/chainloader/meson.build create mode 100644 src/chainloader/vst2-chainloader.cpp create mode 100644 src/chainloader/vst3-chainloader.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 217baa99..056fe25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,28 @@ Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Yabridge 4.0 introduces a completely new way to load plugins that allows + yabridge to be updated without breaking any plugins, saves disk space on + filesystems that don't support reflinks, and makes the `yabridgectl sync` + process faster. It does this by chainloading the actual plugin libraries from + these new tiny, dependencyless shim libraries. The way yabridge has always + worked is that whenever you run `yabridgectl sync`, yabridgectl will create + copies of `libyabridge-vst2.so` or `libyabridge-vst3.so` for every Windows + plugin it finds. When your DAW then loads those plugin libraries, yabridge + will find the corresponding Windows plugin and starts doing its magic. + Yabridge 4.0 changes this process by no longer copying the full + `libyabridge-*.so` libraries, and instead using these shim libraries that will + find and then chainload the actual yabridge plugin libraries. The result is + that instead of having to copy large files, yabridgectl now only needs to copy + these small shim libraries while the actual plugin libraries stay in + yabridge's installation directory. That not only saves disk space, but it also + means that it's no longer possible for yabridge to be out of sync after an + update. If you use a distro packaged version of yabridge, then that means + yabridge can now be updated safely without requiring any action from your + side. + ### Changed - Almost the entirety of yabridge's backend has been rewritten to get rid of all diff --git a/docs/architecture.md b/docs/architecture.md index 29f11181..3786bdd0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,20 +9,35 @@ ## General architecture -The project consists of multiple components: several native Linux plugins -(`libyabridge-vst2.so` for VST2 plugins, and `libyabridge-vst3.so` for VST3 -plugins) and a few different plugin host applications that can run under Wine +The project consists of multiple components: a number of native Linux plugin +libraries (`libyabridge-vst2.so` for VST2 plugins, and `libyabridge-vst3.so` for +VST3 plugins), matching chainloader libraries (`libyabridge-chainloader-vst2.so` +for VST2 plugins, and `libyabridge-chainloader-vst3.so` for VST3 plugins) that +act as stubs to load the former libraries, and a few different plugin host +applications that can run under Wine (`yabridge-host.exe`/`yabridge-host.exe.so`, and `yabridge-host-32.exe`/`yabridge-host-32.exe.so` if the bitbridge is enabled). -The main idea is that when the host loads a plugin, the plugin will try to -locate the corresponding Windows plugin, and it will then start a Wine process -to host that Windows plugin. Depending on the architecture of the Windows plugin -and the configuration in the `yabridge.toml` config files (see the readme for -more information), yabridge will pick between the four plugin host applications -named above. When a plugin has been configured to use plugin groups, instead of -spawning a new host process the plugin will try to connect to an existing group -host process first and ask it to host the Windows plugin within that process. +The main idea is that when the host loads a (chainloader) plugin, the plugin +will try to locate the corresponding Windows plugin, and it will then start a +Wine process to host that Windows plugin. Depending on the architecture of the +Windows plugin and the configuration in the `yabridge.toml` config files (see +the readme for more information), yabridge will pick between the four plugin +host applications named above. When a plugin has been configured to use plugin +groups, instead of spawning a new host process the plugin will try to connect to +an existing group host process first and ask it to host the Windows plugin +within that process. + +The chainloader libraries are compact dependencyless shims that load the +corresponding plugin library and forward calls to the plugin API's entry poitn +functions. This allows the plugin library to be updated without needing to +replace existing copies of the chainloader library. That makes using a +distro-packaged version of yabridge more convenient soname rebuilds won't +require a `yabridgectl sync` for yabridge to keep working. It also means that +multiple plugins can all share the same yabridge plugin bridge library instance, +since the same library will be `dlopen()`'d into a single process multiple +times. This can help increase the L1i cache hit rate when using multiple +yabridge plugins. ### Communication diff --git a/meson.build b/meson.build index 92d07f14..740962ee 100644 --- a/meson.build +++ b/meson.build @@ -49,6 +49,12 @@ compiler_options = [ '-msse2', ] +chainloader_compiler_options = [ + # We use our process library for sending notifications from the chainloaders, + # but we don't need the Asio pipe support there + '-DPROCESS_NO_ASIO', +] + # HACK: Some stuff from `windows.h` that we don't need results in conflicting # definitions, so we'll try to exclude those bits wine_compiler_options = [ @@ -234,6 +240,7 @@ subdir('src/common/config') # directory and it's much more convenient having all of the important files # directory under `build/`. # https://github.com/mesonbuild/meson/pull/4037 +subdir('src/chainloader') subdir('src/plugin') subdir('src/wine-host') @@ -260,6 +267,24 @@ shared_library( # Wine plugin host binaries override_options : ['b_lto=true'], ) +shared_library( + 'yabridge-chainloader-vst2', + vst2_chainloader_sources, + native : true, + dependencies : [ + configuration_dep, + + dl_dep, + ghc_filesystem_dep, + rt_dep, + ], + cpp_args : compiler_options + chainloader_compiler_options, + # LTO currently doesn't work with winelibs, so instead we'll explicitly enable + # it for all other targets (which is extra important for the chainloaders as + # they'd otherwise pull in a bunch of unused symbols) without affecting the + # Wine plugin host binaries + override_options : ['b_lto=true'], +) if with_vst3 # This is the VST3 equivalent of `libyabridge-vst2.so`. The Wine host @@ -289,6 +314,24 @@ if with_vst3 # without affecting the Wine plugin host binaries override_options : ['b_lto=true'], ) + shared_library( + 'yabridge-chainloader-vst3', + vst3_chainloader_sources, + native : true, + dependencies : [ + configuration_dep, + + dl_dep, + ghc_filesystem_dep, + rt_dep, + ], + cpp_args : compiler_options + chainloader_compiler_options, + # LTO currently doesn't work with winelibs, so instead we'll explicitly + # enable it for all other targets (which is extra important for the + # chainloaders as they'd otherwise pull in a bunch of unused symbols) + # without affecting the Wine plugin host binaries + override_options : ['b_lto=true'], + ) endif if is_64bit_system diff --git a/src/chainloader/meson.build b/src/chainloader/meson.build new file mode 100644 index 00000000..195e1367 --- /dev/null +++ b/src/chainloader/meson.build @@ -0,0 +1,16 @@ +# Like for the other libraries, the actual `shared_library()` call is in the +# main `meson.build` file so everything gets bundled to a single directory. + +vst2_chainloader_sources = files( + 'vst2-chainloader.cpp', + '../common/linking.cpp', + '../common/notifications.cpp', + '../common/process.cpp', +) + +vst3_chainloader_sources = files( + 'vst3-chainloader.cpp', + '../common/linking.cpp', + '../common/notifications.cpp', + '../common/process.cpp', +) diff --git a/src/chainloader/vst2-chainloader.cpp b/src/chainloader/vst2-chainloader.cpp new file mode 100644 index 00000000..a6da808a --- /dev/null +++ b/src/chainloader/vst2-chainloader.cpp @@ -0,0 +1,105 @@ +// yabridge: a Wine plugin bridge +// Copyright (C) 2020-2022 Robbert van der Helm +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include + +#include + +#include "../common/linking.h" +#include "../common/utils.h" + +// These chainloader libraries are tiny, mostly dependencyless libraries that +// `dlopen()` the actual `libyabridge-{vst2,vst3}.so` files and forward the +// entry point function calls from this library to those. Or technically, these +// libraries use dedicated entry point functions because multiple chainloader +// libraries may all dynamically link to the exact same plugin library, so we +// can't store any bridge information in a global there. This approach avoids +// wasting disk space on copies on file systems that don't support reflinking, +// but more importantly it also avoids the need to rerun `yabridgectl sync` +// whenever yabridge is updated. This is even more important when considering +// distro packaging, because updates to Boost might require the package to be +// rebuilt, which in turn would also require a resync. + +// TODO: Order to check in: See Discord + +namespace fs = ghc::filesystem; + +using audioMasterCallback = void*; +using AEffect = void; + +// These functions are loaded from `libyabridge-vst3.so` the first time +// `VSTPluginMain` gets called +AEffect* (*yabridge_plugin_init)(audioMasterCallback host_callback, + const char* plugin_path) = nullptr; + +/** + * The first time one of the exported functions from this library gets called, + * we'll need to load the corresponding `libyabridge-*.so` file and fetch the + * the entry point functions from that file. + */ +bool initialize_library() { + static void* library_handle = nullptr; + static std::mutex library_handle_mutex; + + std::lock_guard lock(library_handle_mutex); + + // There should be no situation where this library gets loaded and then two + // threads immediately start calling functions, but we'll handle that + // situation just in case it does happen + if (library_handle) { + return true; + } + + // FIXME: Hardcoded path + library_handle = dlopen( + "/home/robbert/Documenten/projecten/yabridge/build/libyabridge-vst2.so", + RTLD_LAZY | RTLD_LOCAL); + assert(library_handle); + +#define LOAD_FUNCTION(name) \ + do { \ + (name) = \ + reinterpret_cast(dlsym(library_handle, #name)); \ + assert(name); \ + } while (false) + + LOAD_FUNCTION(yabridge_plugin_init); + +#undef LOAD_FUNCTION + + return true; +} + +extern "C" YABRIDGE_EXPORT AEffect* VSTPluginMain( + audioMasterCallback host_callback) { + assert(host_callback); + + initialize_library(); + + const fs::path this_plugin_path = get_this_file_location(); + return yabridge_plugin_init(host_callback, this_plugin_path.c_str()); +} + +// XXX: GCC doens't seem to have a clean way to let you define an arbitrary +// function called 'main'. Even JUCE does it this way, so it should be +// safe. +extern "C" YABRIDGE_EXPORT AEffect* deprecated_main( + audioMasterCallback audioMaster) asm("main"); +YABRIDGE_EXPORT AEffect* deprecated_main(audioMasterCallback audioMaster) { + return VSTPluginMain(audioMaster); +} diff --git a/src/chainloader/vst3-chainloader.cpp b/src/chainloader/vst3-chainloader.cpp new file mode 100644 index 00000000..8b476050 --- /dev/null +++ b/src/chainloader/vst3-chainloader.cpp @@ -0,0 +1,143 @@ +// yabridge: a Wine plugin bridge +// Copyright (C) 2020-2022 Robbert van der Helm +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include + +#include + +#include "../common/linking.h" +#include "../common/utils.h" + +// These chainloader libraries are tiny, mostly dependencyless libraries that +// `dlopen()` the actual `libyabridge-{vst2,vst3}.so` files and forward the +// entry point function calls from this library to those. Or technically, these +// libraries use dedicated entry point functions because multiple chainloader +// libraries may all dynamically link to the exact same plugin library, so we +// can't store any bridge information in a global there. This approach avoids +// wasting disk space on copies on file systems that don't support reflinking, +// but more importantly it also avoids the need to rerun `yabridgectl sync` +// whenever yabridge is updated. This is even more important when considering +// distro packaging, because updates to Boost might require the package to be +// rebuilt, which in turn would also require a resync. + +// TODO: Order to check in: See Discord + +namespace fs = ghc::filesystem; + +using Vst3PluginBridge = void; +using PluginFactory = void; + +// These functions are loaded from `libyabridge-vst3.so` the first time +// `ModuleEntry()` gets called +Vst3PluginBridge* (*yabridge_module_init)(const char* plugin_path) = nullptr; +void (*yabridge_module_free)(Vst3PluginBridge* instance) = nullptr; +PluginFactory* (*yabridge_module_get_plugin_factory)( + Vst3PluginBridge* instance) = nullptr; + +/** + * The bridge instance for this chainloader. This is initialized when + * `ModuleEntry()` first gets called. + */ +std::unique_ptr bridge( + nullptr, + nullptr); +/** + * The number of active instances. Incremented when `ModuleEntry()` is called, + * decremented when `ModuleExit()` is called. We'll initialize the bridge when + * this is first incremented from 0, and we'll free the bridge again when a + * `ModuleExit()` call causes this to return back to 0. + */ +std::atomic_size_t active_instances = 0; +std::mutex bridge_mutex; + +/** + * The first time one of the exported functions from this library gets called, + * we'll need to load the corresponding `libyabridge-*.so` file and fetch the + * the entry point functions from that file. + */ +bool initialize_library() { + static void* library_handle = nullptr; + static std::mutex library_handle_mutex; + + std::lock_guard lock(library_handle_mutex); + + // There should be no situation where this library gets loaded and then two + // threads immediately start calling functions, but we'll handle that + // situation just in case it does happen + if (library_handle) { + return true; + } + + // FIXME: Hardcoded path + library_handle = dlopen( + "/home/robbert/Documenten/projecten/yabridge/build/libyabridge-vst3.so", + RTLD_LAZY | RTLD_LOCAL); + assert(library_handle); + +#define LOAD_FUNCTION(name) \ + do { \ + (name) = \ + reinterpret_cast(dlsym(library_handle, #name)); \ + assert(name); \ + } while (false) + + LOAD_FUNCTION(yabridge_module_init); + LOAD_FUNCTION(yabridge_module_free); + LOAD_FUNCTION(yabridge_module_get_plugin_factory); + +#undef LOAD_FUNCTION + + return true; +} + +extern "C" YABRIDGE_EXPORT bool ModuleEntry(void*) { + // This function can be called multiple times, so we should make sure to + // only initialize the bridge on the first call + if (active_instances.fetch_add(1, std::memory_order_seq_cst) == 0) { + assert(initialize_library()); + + // You can't change the deleter function with `.reset()` so we'll need + // this abomination instead + const fs::path this_plugin_path = get_this_file_location(); + bridge = + decltype(bridge)(yabridge_module_init(this_plugin_path.c_str()), + yabridge_module_free); + if (!bridge) { + return false; + } + } + + return true; +} + +extern "C" YABRIDGE_EXPORT bool ModuleExit() { + // We'll free the bridge when this exits brings the reference count back to + // zero + if (active_instances.fetch_sub(1, std::memory_order_seq_cst) == 1) { + bridge.reset(); + } + + return true; +} + +extern "C" YABRIDGE_EXPORT PluginFactory* GetPluginFactory() { + // The host should have called `InitModule()` first + assert(bridge); + + return yabridge_module_get_plugin_factory(bridge.get()); +}