Make realtime priority setting more granular

On the plugin side.

CHANGELOG.md

@@ -57,13 +57,13 @@ TODO: Add an updated screenshot with some fancy VST3-only plugins to the readme
 - `libyabridge.so` is now called `libyabridge-vst2.so`. If you're using
   yabridgectl then nothing changes here. **To avoid any confusion in the future,
   please remove the old `libyabridge.so` file before upgrading.**
-- Slightly increased responsiveness when resizing plugin GUIs by preventing
-  unnecessary blitting. This also reduces flickering with plugins that don't do
-  double buffering.
 - Window closing is now deferred. This means that when closing the editor
   window, the host no longer has to wait for Wine to fully close the window.
   Most hosts already do something similar themselves, so this may not make any
   difference in responsiveness.
+- Slightly increased responsiveness when resizing plugin GUIs by preventing
+  unnecessary blitting. This also reduces flickering with plugins that don't do
+  double buffering.
 - VST2 editor idle events are now handled slightly differently. This should
   result in even more responsive GUIs for VST2 plugins.
 - Win32 and X11 events in the Wine plugin host are now handled with lower
@@ -71,6 +71,15 @@ TODO: Add an updated screenshot with some fancy VST3-only plugins to the readme
   drawing should not affect DSP load at all, but this should help with less than
   optimal setups where some people were getting DSP latency spikes while having
   plugin editor open.
+- Yabridge handles realtime priority now slightly differently:
+
+  - Realtime priority on the plugin side is now a more granular. Instead of
+    setting everything to use `SCHED_FIFO`, only the spawned threads will be set
+    to realtime priority. This prevents changing the scheduling policy of your
+    host's GUI thread if your host instantiates plugins from its GUI thread like
+    REAPER does.
+  - TODO: Next up is periodically synchronizing audio thread priorities.
+
 - Opening and closing plugin editors is now also no longer done with realtime
   priority. This should get rid of any latency spikes during those operations,
   as this could otherwise steal resources away from the threads that are

src/plugin/bridges/common.h

@@ -16,6 +16,7 @@
 
 #pragma once
 
+#include <future>
 #include <iomanip>
 
 // Generated inside of the build directory
@@ -88,8 +89,12 @@ class PluginBridge {
                                             info.windows_plugin_path.string(),
                                         .endpoint_base_dir =
                                             sockets.base_dir.string()}))),
-          has_realtime_priority(set_realtime_priority(true)),
-          wine_io_handler([&]() { io_context.run(); }) {}
+          has_realtime_priority(has_realtime_priority_promise.get_future()),
+          wine_io_handler([&]() {
+              has_realtime_priority_promise.set_value(
+                  set_realtime_priority(true));
+              io_context.run();
+          }) {}
 
     virtual ~PluginBridge(){};
 
@@ -108,8 +113,9 @@ class PluginBridge {
                  << "'" << std::endl;
         init_msg << "plugin type:  '" << plugin_type_to_string(info.plugin_type)
                  << "'" << std::endl;
-        init_msg << "realtime:     '" << (has_realtime_priority ? "yes" : "no")
-                 << "'" << std::endl;
+        init_msg << "realtime:     '"
+                 << (has_realtime_priority.get() ? "yes" : "no") << "'"
+                 << std::endl;
         init_msg << "sockets:      '" << sockets.base_dir.string() << "'"
                  << std::endl;
         init_msg << "wine prefix:  '";
@@ -300,12 +306,19 @@ class PluginBridge {
      */
     std::unique_ptr<HostProcess> plugin_host;
 
+   private:
     /**
-     * Whether this process runs with realtime priority. We'll set this _after_
-     * spawning the Wine process because from my testing running wineserver with
-     * realtime priority can actually increase latency.
+     * The promise belonging to `has_realtime_priority` below.
      */
-    bool has_realtime_priority;
+    std::promise<bool> has_realtime_priority_promise;
+
+   public:
+    /**
+     * Whether this process runs with realtime priority. This is set on the
+     * thread that's relaying STDOUT and STDERR output from Wine, hence the need
+     * for a future.
+     */
+    std::future<bool> has_realtime_priority;
 
     /**
      * Runs the Boost.Asio `io_context` thread for logging the Wine process

src/plugin/bridges/vst2.cpp

@@ -72,6 +72,8 @@ Vst2PluginBridge::Vst2PluginBridge(audioMasterCallback host_callback)
     // instead of asynchronous IO since communication has to be handled in
     // lockstep anyway
     host_callback_handler = std::jthread([&]() {
+        set_realtime_priority(true);
+
         sockets.vst_host_callback.receive_events(
             std::pair<Vst2Logger&, bool>(logger, false),
             [&](Event& event, bool /*on_main_thread*/) {

src/plugin/bridges/vst3-impls/plug-view-proxy.h

@@ -244,6 +244,8 @@ class Vst3PlugViewProxyImpl : public Vst3PlugViewProxy {
         // to  from this thread
         std::promise<TResponse> response_promise{};
         std::jthread sending_thread([&]() {
+            set_realtime_priority(true);
+
             const TResponse response = bridge.send_message(object);
 
             // Stop accepting additional work to be run from the calling thread

src/plugin/bridges/vst3.cpp

@@ -47,6 +47,8 @@ Vst3PluginBridge::Vst3PluginBridge()
     // first thing, the Wine VST host will ask us for a copy of the
     // configuration.
     host_callback_handler = std::jthread([&]() {
+        set_realtime_priority(true);
+
         sockets.vst_host_callback.receive_messages(
             std::pair<Vst3Logger&, bool>(logger, false),
             overload{

src/plugin/host-process.cpp

@@ -21,6 +21,8 @@
 #include <boost/process/io.hpp>
 #include <boost/process/start_dir.hpp>
 
+#include "src/common/utils.h"
+
 namespace bp = boost::process;
 namespace fs = boost::filesystem;
 
@@ -191,6 +193,8 @@ GroupHost::GroupHost(boost::asio::io_context& io_context,
         const pid_t group_host_pid = group_host.id();
         group_host_connect_handler =
             std::jthread([this, connect, group_host_pid]() {
+                set_realtime_priority(true);
+
                 using namespace std::literals::chrono_literals;
 
                 // We'll first try to connect to the group host we just spawned