Always handle IPlugView::onSize() from UI thread

This requires a super hacky workaround because the UI thread can be
currently blocked by the plugin calling `IPlugFrame::resizeView()` from
the Win32 message loop.

src/wine-host/bridges/vst3-impls/plug-frame-proxy.cpp

@@ -46,8 +46,81 @@ Vst3PlugFrameProxyImpl::resizeView(Steinberg::IPlugView* /*view*/,
         // XXX: Since VST3 currently only support a single view type we'll
         //      assume `view` is the `IPlugView*` returned by the last call to
         //      `IEditController::createView()`
-        return bridge.send_message(YaPlugFrame::ResizeView{
-            .owner_instance_id = owner_instance_id(), .new_size = *newSize});
+
+        // HACK: This ia bit of a weird one and requires special handling. A
+        //       plugin will call this function from the Win32 message loop so
+        //       the call blocks the loop. The host will then check with the
+        //       plugin if it can actually resize itself to `*newSize`, and it
+        //       will then call `IPlugView::onSize()` with the new size. The
+        //       issue is that the `IPlugView::onSize()` call also has to be
+        //       called from within the UI thread, but that thread is currently
+        //       being blocked by the call to this function.
+        //       As a workaround, we'll send the message for the call to
+        //       `IPlugFrame::resizeView()` on another thread. We then wait for
+        //       either that request to finish immediately (meaning the host
+        //       hasn't resized the window), or for `IPlugView::onSize()` to be
+        //       called by the host. If the host does call `IPlugView::onSize()`
+        //       while the other thread is handling `IPlugFrame::resizeView()`,
+        //       then we'll awaken this thread using a condition variable so we
+        //       can do the actual call to `IPlugView::onSize()` from here, from
+        //       within the Win32 loop.
+        // TODO: Can we someone use Boost.Asio strands to make this cleaner?
+        {
+            std::lock_guard lock(on_size_interrupt_mutex);
+            on_size_interrupt_waiting = true;
+            on_size_interrupt.reset();
+            on_size_interrupt_result.reset();
+        }
+
+        std::promise<tresult> resize_result_promise{};
+        std::future<tresult> resize_result_future =
+            resize_result_promise.get_future();
+        Win32Thread resize_thread([&]() {
+            const tresult result = bridge.send_message(YaPlugFrame::ResizeView{
+                .owner_instance_id = owner_instance_id(),
+                .new_size = *newSize});
+
+            resize_result_promise.set_value(result);
+
+            {
+                std::lock_guard lock(on_size_interrupt_mutex);
+                if (BOOST_LIKELY(!on_size_interrupt_waiting)) {
+                    return;
+                }
+
+                // If the call to `IPlugFrame::resizeView()` finish without the
+                // host calling `IPlugView::onSize()` (e.g. when the call
+                // failed), then we'll have to manually unblock the thread below
+                // by providing a noop function.
+                on_size_interrupt = []() { return Steinberg::kResultOk; };
+            }
+            on_size_interrupt_cv.notify_one();
+
+            // We don't need this result value, but we should still wait for
+            // it
+            std::unique_lock lock(on_size_interrupt_mutex);
+            on_size_interrupt_cv.wait(
+                lock, [&]() { return on_size_interrupt_result.has_value(); });
+        });
+
+        // Wait for `IPlugView::onSize()` to be called by the host and handle
+        // the call here on this thread, or wait for the call to
+        // `IPlugFrame::resizeView()` in the above thread to finish. In the last
+        // case we'll execute a noop function.
+        std::unique_lock lock(on_size_interrupt_mutex);
+        on_size_interrupt_cv.wait(
+            lock, [&]() { return on_size_interrupt.has_value(); });
+
+        // When we get a function through one of the two above means, we'll
+        // store the result and then awake the calling thread again so it can
+        // access the result
+        on_size_interrupt_result = (*on_size_interrupt)();
+        on_size_interrupt_waiting = false;
+
+        lock.unlock();
+        on_size_interrupt_cv.notify_one();
+
+        return resize_result_future.get();
     } else {
         std::cerr
             << "WARNING: Null pointer passed to 'IPlugFrame::resizeView()'"

src/wine-host/bridges/vst3-impls/plug-frame-proxy.h

@@ -30,10 +30,58 @@ class Vst3PlugFrameProxyImpl : public Vst3PlugFrameProxy {
     tresult PLUGIN_API queryInterface(const Steinberg::TUID _iid,
                                       void** obj) override;
 
+    /**
+     * This is needed to be able to handle a call to `IPlugView::onSize()` from
+     * the UI thread while the plugin is currently calling
+     * `IPlugFrame::resizeView()` from that same thread.. This is probably the
+     * hackiest (and most error prone, probably) part of this VST3
+     * implementation. The details on how this works and why it is necessary are
+     * explained in the comment in `YaPlugFrameProxyImpl::resizeView()`.
+     *
+     * If there is currently a call to `resizeView()` being processed, then this
+     * will run `on_size` from the same thread that's currently processing a
+     * call to `resizeView()` and return the result from the function call.
+     * Otherwise this will return a nullopt and `on_size` should be passed to
+     * `main_context.run_in_context<tresult>()`.
+     */
+    template <typename F>
+    std::optional<tresult> maybe_run_on_size_from_ui_thread(F on_size) {
+        {
+            std::lock_guard lock(on_size_interrupt_mutex);
+            if (!on_size_interrupt_waiting) {
+                return std::nullopt;
+            }
+
+            on_size_interrupt = std::move(on_size);
+        }
+        on_size_interrupt_cv.notify_one();
+
+        // Since `on_size` is run from another thread, we now have to wait to be
+        // woken up again when the result is ready
+        std::unique_lock lock(on_size_interrupt_mutex);
+        on_size_interrupt_cv.wait(
+            lock, [&]() { return on_size_interrupt_result.has_value(); });
+
+        return *on_size_interrupt_result;
+    }
+
     // From `IPlugFrame`
     tresult PLUGIN_API resizeView(Steinberg::IPlugView* view,
                                   Steinberg::ViewRect* newSize) override;
 
    private:
     Vst3Bridge& bridge;
+
+    /**
+     * A function that will be used to run `IPlugView::onSize()` on the thread
+     * that originally called `IPlugFrame::resizeView()` to work around a Win32
+     * limitation, along with its result, and whether or not we're currently
+     * waiting for this function to be provided by some other thread. See the
+     * comment in `onSize()` for more information.
+     */
+    bool on_size_interrupt_waiting = false;
+    std::optional<fu2::unique_function<tresult()>> on_size_interrupt;
+    std::optional<tresult> on_size_interrupt_result;
+    std::condition_variable on_size_interrupt_cv;
+    std::mutex on_size_interrupt_mutex;
 };

src/wine-host/bridges/vst3.cpp

@@ -409,12 +409,31 @@ void Vst3Bridge::run() {
                     .result = result, .updated_size = request.size};
             },
             [&](YaPlugView::OnSize& request) -> YaPlugView::OnSize::Response {
-                return main_context
-                    .run_in_context<tresult>([&]() {
-                        return object_instances[request.owner_instance_id]
+                auto call_on_size = [&]() {
+                    const auto result =
+                        object_instances[request.owner_instance_id]
                             .plug_view->onSize(&request.new_size);
-                    })
-                    .get();
+                    return result;
+                };
+
+                // HACK: As explained in the docstring of
+                //       `YaPlugFrameProxyImpl::resizeView()`, calls to
+                //       `IPlugView::onSize()` have to be handled from the UI
+                //       thread, even if that thread is currently making a call
+                //       to `IPlugFrame::resizeView()`. We use some special
+                //       handling to execute `call_on_size()` on that thread if
+                //       it's currently waiting for a call to
+                //       `IPlugFrame::resizeView()`, and we'll just run in
+                //       within the main context as usual otherwise.
+                if (auto result =
+                        object_instances[request.owner_instance_id]
+                            .plug_frame_proxy_impl
+                            ->maybe_run_on_size_from_ui_thread(call_on_size)) {
+                    return *result;
+                } else {
+                    return main_context.run_in_context<tresult>(call_on_size)
+                        .get();
+                }
             },
             [&](const YaPlugView::OnFocus& request)
                 -> YaPlugView::OnFocus::Response {
@@ -431,12 +450,18 @@ void Vst3Bridge::run() {
                 // pass that to the `setFrame()` function. The lifetime of this
                 // object is tied to that of the actual `IPlugFrame` object
                 // we're passing this proxy to.
+                // We'll also store an unmanaged pointer to the actual
+                // implementation so we can do some sneakiness for
+                // `IPlugView::onSize()`.
+                object_instances[request.owner_instance_id]
+                    .plug_frame_proxy_impl = new Vst3PlugFrameProxyImpl(
+                    *this, std::move(request.plug_frame_args));
+                object_instances[request.owner_instance_id].plug_frame_proxy =
+                    Steinberg::owned(object_instances[request.owner_instance_id]
+                                         .plug_frame_proxy_impl);
+
                 // TODO: Does this have to be run from the UI thread? Figure out
                 //       if it does
-                object_instances[request.owner_instance_id].plug_frame_proxy =
-                    Steinberg::owned(new Vst3PlugFrameProxyImpl(
-                        *this, std::move(request.plug_frame_args)));
-
                 return object_instances[request.owner_instance_id]
                     .plug_view->setFrame(
                         object_instances[request.owner_instance_id]

src/wine-host/bridges/vst3.h

@@ -25,6 +25,9 @@
 #include "../editor.h"
 #include "common.h"
 
+// Forward declarations
+class Vst3PlugFrameProxyImpl;
+
 /**
  * A holder for plugin object instance created from the factory. This stores all
  * relevant interface smart pointers to that object so we can handle control
@@ -72,6 +75,13 @@ struct InstanceInterfaces {
      */
     Steinberg::IPtr<Vst3PlugFrameProxy> plug_frame_proxy;
 
+    /**
+     * An unmanaged raw pointer for the actual implementation behind
+     * `plug_frame_proxy`. This is needed for some special handling for
+     * `IPlugView::onSize()`.
+     */
+    Vst3PlugFrameProxyImpl* plug_frame_proxy_impl;
+
     /**
      * The base object we cast from.
      */