Proxy host context menu items for VST3 plugins

This wasn't implemented yet because no plugin tried using the interface
in this way before this, but Surge XT incorporates the host's context
menu items into their own (much more elaborate) context menu. To
accommodate this, we now copy over all of the host's prepopulated
context menu items to the Wine plugin host, and calling the targets
associated with any of those items will cause the target on the
associated context menu item on the host to be called.

This is slightly more complicated than what would otherwise be necessary
because Bitwig does not assign tags to their context menu items and
instead always uses 0.

src/common/serialization/vst3-impls/context-menu-target.h

@@ -22,6 +22,9 @@
  * This implementation used to live in `src/plugin/bridges/vst3-impls`, but
  * since plugins can also call context menu items added by the host this is
  * needed on both sides.
+ *
+ * NOTE: Bitwig does not actually set the tags here, so host menu items need to
+ *       be identified through their item ID, not through the tag.
  */
 template <typename Bridge>
 class YaContextMenuTargetImpl : public YaContextMenuTarget {
@@ -45,11 +48,16 @@ class YaContextMenuTargetImpl : public YaContextMenuTarget {
 
     // From `IContextMenuTarget`
     tresult PLUGIN_API executeMenuItem(int32 tag) override {
-        return bridge_.send_message(YaContextMenuTarget::ExecuteMenuItem{
-            .owner_instance_id = owner_instance_id(),
-            .context_menu_id = context_menu_id(),
-            .target_tag = target_tag(),
-            .tag = tag});
+        // NOTE: This requires mutual recursion, because REAPER will call
+        //       `getState()` whle the context menu is open, and `getState()`
+        //       also has to be handled from the GUI thread
+        return bridge_.send_mutually_recursive_message(
+            YaContextMenuTarget::ExecuteMenuItem{
+                .owner_instance_id = owner_instance_id(),
+                .context_menu_id = context_menu_id(),
+                .item_id = item_id(),
+                .target_tag = target_tag(),
+                .tag = tag});
     }
 
    private:

src/common/serialization/vst3.h

@@ -260,10 +260,10 @@ using CallbackRequest =
                  // `IConnectionPoint::notify` calls through
                  // there
                  YaConnectionPoint::Notify,
-                 YaContextMenu::GetItemCount,
                  YaContextMenu::AddItem,
                  YaContextMenu::RemoveItem,
                  YaContextMenu::Popup,
+                 YaContextMenuTarget::ExecuteMenuItem,
                  YaHostApplication::GetName,
                  YaPlugFrame::ResizeView,
                  YaPlugInterfaceSupport::IsPlugInterfaceSupported,

src/common/serialization/vst3/context-menu-proxy.cpp

@@ -21,7 +21,7 @@ Vst3ContextMenuProxy::ConstructArgs::ConstructArgs() noexcept {}
 Vst3ContextMenuProxy::ConstructArgs::ConstructArgs(
     Steinberg::IPtr<Steinberg::FUnknown> object,
     size_t owner_instance_id,
-    size_t context_menu_id) noexcept
+    size_t context_menu_id)
     : owner_instance_id(owner_instance_id),
       context_menu_id(context_menu_id),
       context_menu_args(object) {}

src/common/serialization/vst3/context-menu-proxy.h

@@ -49,7 +49,7 @@ class Vst3ContextMenuProxy : public YaContextMenu {
          */
         ConstructArgs(Steinberg::IPtr<FUnknown> object,
                       size_t owner_instance_id,
-                      size_t context_menu_id) noexcept;
+                      size_t context_menu_id);
 
         /**
          * The unique instance identifier of the proxy object instance this

src/common/serialization/vst3/context-menu-target.cpp

@@ -16,16 +16,6 @@
 
 #include "context-menu-target.h"
 
-YaContextMenuTarget::ConstructArgs::ConstructArgs() noexcept {}
-
-YaContextMenuTarget::ConstructArgs::ConstructArgs(
-    native_size_t owner_instance_id,
-    native_size_t context_menu_id,
-    int32 tag) noexcept
-    : owner_instance_id(owner_instance_id),
-      context_menu_id(context_menu_id),
-      tag(tag) {}
-
 YaContextMenuTarget::YaContextMenuTarget(ConstructArgs&& args) noexcept
     : arguments_(std::move(args)){FUNKNOWN_CTOR}
 

src/common/serialization/vst3/context-menu-target.h

@@ -37,30 +37,26 @@ class YaContextMenuTarget : public Steinberg::Vst::IContextMenuTarget {
      * `YaContextMenuTargetImpl`.
      */
     struct ConstructArgs {
-        ConstructArgs() noexcept;
-
-        /**
-         * Read from an existing object. We will try to mimic this object, so
-         * we'll support any interfaces this object also supports.
-         *
-         * @param owner_instance_id The object instance that this target's
-         * context menu belongs to.
-         * @param context_menu_id The unique ID of the context menu requested by
-         *   `owwner_instance_id`.
-         * @param tag The tag of the menu item this target belongs to.
-         */
-        ConstructArgs(native_size_t owner_instance_id,
-                      native_size_t context_menu_id,
-                      int32 tag) noexcept;
-
         native_size_t owner_instance_id;
         native_size_t context_menu_id;
+        /**
+         * The ID of the menu item this target belongs to, only used when
+         * calling host targets from the plugin.
+         *
+         * NOTE: Needed to work around a Bitwig bug, see the comment in
+         *       `ExecuteMenuItem`
+         */
+        int32 item_id;
+        /**
+         * The tag of the menu item this target belongs to.
+         */
         int32 tag;
 
         template <typename S>
         void serialize(S& s) {
             s.value8b(owner_instance_id);
             s.value8b(context_menu_id);
+            s.value4b(item_id);
             s.value4b(tag);
         }
     };
@@ -87,6 +83,12 @@ class YaContextMenuTarget : public Steinberg::Vst::IContextMenuTarget {
      */
     inline size_t context_menu_id() const { return arguments_.context_menu_id; }
 
+    /**
+     * Get the ID of the menu item this target was obtained from. This value is
+     * only actually used when calling host context menu items from a plugin.
+     */
+    inline int32 item_id() const { return arguments_.item_id; }
+
     /**
      * Get the tag of the menu item this target was passed to.
      */
@@ -103,8 +105,22 @@ class YaContextMenuTarget : public Steinberg::Vst::IContextMenuTarget {
         native_size_t owner_instance_id;
         native_size_t context_menu_id;
         /**
-         * The tag this target was passed for. This should be the same as `tag`,
-         * but it doesn't have to be.
+         * The menu item ID this target belongs to.
+         *
+         * This is used when calling host context menu items from the plugin's
+         * side.
+         *
+         * NOTE: This is needed because Bitwig identifies its own menu items by
+         *       opaque ID, and not through the tag. They use 0 for all tags.
+         */
+        int32 item_id;
+        /**
+         * The tag of the target this method was called on. Presumably this
+         * would always be the same as the `tag` argument passed to this
+         * function, but it doesn't have to be.
+         *
+         * This is used when calling plugin context menu items from the host's
+         * side.
          */
         int32 target_tag;
 
@@ -114,6 +130,7 @@ class YaContextMenuTarget : public Steinberg::Vst::IContextMenuTarget {
         void serialize(S& s) {
             s.value8b(owner_instance_id);
             s.value8b(context_menu_id);
+            s.value4b(item_id);
             s.value4b(target_tag);
             s.value4b(tag);
         }

src/common/serialization/vst3/context-menu/context-menu.cpp

@@ -20,7 +20,24 @@ YaContextMenu::ConstructArgs::ConstructArgs() noexcept {}
 
 YaContextMenu::ConstructArgs::ConstructArgs(
     Steinberg::IPtr<Steinberg::FUnknown> object) noexcept
-    : supported(Steinberg::FUnknownPtr<Steinberg::Vst::IContextMenu>(object)) {}
+    : supported(Steinberg::FUnknownPtr<Steinberg::Vst::IContextMenu>(object)) {
+    Steinberg::FUnknownPtr<Steinberg::Vst::IContextMenu> context_menu(object);
+    if (context_menu) {
+        // Can't trust plugins to check for null pointers, so we'll just always
+        // pass something
+        Steinberg::Vst::IContextMenuTarget* dummyTarget = nullptr;
+
+        // Prepopulate the context menu with these targets
+        // NOTE: Bitwig does not actually set the tags here, so host menu items
+        //       need to be identified through their item ID, not through the
+        //       tag
+        items.resize(context_menu->getItemCount());
+        for (size_t i = 0; i < items.size(); i++) {
+            context_menu->getItem(static_cast<int32>(i), items[i],
+                                  &dummyTarget);
+        }
+    }
+}
 
 YaContextMenu::YaContextMenu(ConstructArgs&& args) noexcept
     : arguments_(std::move(args)) {}

src/common/serialization/vst3/context-menu/context-menu.h

@@ -29,6 +29,9 @@
 /**
  * Wraps around `IContextMenu` for serialization purposes. This is instantiated
  * as part of `Vst3ContextMenuProxy`.
+ *
+ * Plugins can also call context menu items created by the host, in which case
+ * we'll proxy that call through to the host.
  */
 class YaContextMenu : public Steinberg::Vst::IContextMenu {
    public:
@@ -49,9 +52,18 @@ class YaContextMenu : public Steinberg::Vst::IContextMenu {
          */
         bool supported;
 
+        /**
+         * The context menu items prepopulated by the host so the plugin can
+         * call them. These items will receive `YaContextMenuTarget` proxy
+         * targets in `Vst3ContextMenuProxyImpl`, so when the plugin calls them
+         * it will dispatch a call to the host instead.
+         */
+        std::vector<Steinberg::Vst::IContextMenuItem> items;
+
         template <typename S>
         void serialize(S& s) {
             s.value1b(supported);
+            s.container(items, 1 << 16);
         }
     };
 
@@ -63,28 +75,10 @@ class YaContextMenu : public Steinberg::Vst::IContextMenu {
 
     inline bool supported() const noexcept { return arguments_.supported; }
 
-    /**
-     * Message to pass through a call to `IContextMenu::getItemCount()` to the
-     * corresponding context menu instance returned by the host.
-     */
-    struct GetItemCount {
-        using Response = PrimitiveWrapper<int32>;
-
-        native_size_t owner_instance_id;
-        native_size_t context_menu_id;
-
-        template <typename S>
-        void serialize(S& s) {
-            s.value8b(owner_instance_id);
-            s.value8b(context_menu_id);
-        }
-    };
-
+    // Since we pass along a list of initial items, we don't need to proxy this
+    // unless the host somehow adds more items after the plugin adds an item
     virtual int32 PLUGIN_API getItemCount() override = 0;
-
-    // XXX: Can a plugin call this to get items created by the host? Why would
-    //      they do that? We should find a host/plugin combination that supports
-    //      `IComponentHandler3` first.
+    // Plugins can also call context menu items created by the host
     virtual tresult PLUGIN_API
     getItem(int32 index,
             Steinberg::Vst::IContextMenuItem& item /*out*/,