Add an optional double editor embedding mode #27

This sounds like it would the simplest way to work around the issue of
E27 calculating its own coordinates based on the parent window's
coordinates. I have not noticed any weird issues with having this
enabled all the time, but less moving parts is always better so it's
still behind an option.

src/wine-host/bridges/vst2.cpp

@@ -373,11 +373,11 @@ intptr_t Vst2Bridge::dispatch_wrapper(AEffect* plugin,
             // should get a unique window class
             const std::string window_class =
                 "yabridge plugin " + socket_endpoint.path();
-            Editor& editor_instance =
+            Editor& editor_instance = editor.emplace<Editor>(
-                editor.emplace<Editor>(window_class, plugin, x11_handle);
+                config, window_class, x11_handle, plugin);
 
             return plugin->dispatcher(plugin, opcode, index, value,
-                                      editor_instance.win32_handle.get(),
+                                      editor_instance.get_win32_handle(),
                                       option);
         } break;
         case effEditClose: {

src/wine-host/editor.cpp

@@ -55,9 +55,10 @@ WindowClass::~WindowClass() {
     UnregisterClass(reinterpret_cast<LPCSTR>(atom), GetModuleHandle(nullptr));
 }
 
-Editor::Editor(const std::string& window_class_name,
+Editor::Editor(const Configuration& config,
-               AEffect* effect,
+               const std::string& window_class_name,
-               const size_t parent_window_handle)
+               const size_t parent_window_handle,
+               AEffect* effect)
     : x11_connection(xcb_connect(nullptr, nullptr), xcb_disconnect),
       client_area(get_maximum_screen_dimensions(*x11_connection)),
       window_class(window_class_name),
@@ -79,9 +80,27 @@ Editor::Editor(const std::string& window_class_name,
                                   GetModuleHandle(nullptr),
                                   this),
                    DestroyWindow),
+      win32_child_handle(
+          config.editor_double_embed
+              ? std::make_optional<std::unique_ptr<std::remove_pointer_t<HWND>,
+                                                   decltype(&DestroyWindow)>>(
+                    CreateWindowEx(0,
+                                   reinterpret_cast<LPCSTR>(window_class.atom),
+                                   "yabridge plugin child",
+                                   WS_CHILD,
+                                   CW_USEDEFAULT,
+                                   CW_USEDEFAULT,
+                                   client_area.width,
+                                   client_area.height,
+                                   win32_handle.get(),
+                                   nullptr,
+                                   GetModuleHandle(nullptr),
+                                   nullptr),
+                    DestroyWindow)
+              : std::nullopt),
       idle_timer(win32_handle.get(), idle_timer_id, 100),
       parent_window(parent_window_handle),
-      child_window(get_x11_handle(win32_handle.get())),
+      wine_window(get_x11_handle(win32_handle.get())),
       topmost_window(find_topmost_window(*x11_connection, parent_window)),
       // Needed to send update messages on a timer
       plugin(effect) {
@@ -107,12 +126,14 @@ Editor::Editor(const std::string& window_class_name,
     // using the XEmbed protocol, we'll register a few events and manage the
     // child window ourselves. This is a hack to work around the issue's
     // described in `Editor`'s docstring'.
-    xcb_reparent_window(x11_connection.get(), child_window, parent_window, 0,
+    xcb_reparent_window(x11_connection.get(), wine_window, parent_window, 0, 0);
-                        0);
+    xcb_map_window(x11_connection.get(), wine_window);
-    xcb_map_window(x11_connection.get(), child_window);
     xcb_flush(x11_connection.get());
 
     ShowWindow(win32_handle.get(), SW_SHOWNORMAL);
+    if (win32_child_handle) {
+        ShowWindow(win32_child_handle->get(), SW_SHOWNORMAL);
+    }
 }
 
 Editor::~Editor() {
@@ -124,7 +145,7 @@ Editor::~Editor() {
         xcb_setup_roots_iterator(xcb_get_setup(x11_connection.get()))
             .data->root;
 
-    xcb_reparent_window(x11_connection.get(), child_window, root, 0, 0);
+    xcb_reparent_window(x11_connection.get(), wine_window, root, 0, 0);
     xcb_flush(x11_connection.get());
 
     // FIXME: I have no idea why, but for some reason the window still hangs
@@ -132,9 +153,18 @@ Editor::~Editor() {
     //        `std::unique_ptr`` to the window handle` (which calls
     //        `DestroyWindow()`), even though the behavior should be identical
     //        without this line.
+    win32_child_handle.reset();
     win32_handle.reset();
 }
 
+HWND Editor::get_win32_handle() {
+    if (win32_child_handle) {
+        return win32_child_handle->get();
+    } else {
+        return win32_handle.get();
+    }
+}
+
 void Editor::send_idle_event() {
     plugin->dispatcher(plugin, effEditIdle, 0, 0, nullptr, 0);
 }
@@ -191,7 +221,7 @@ void Editor::handle_x11_events() const {
                 // the main Bitwig Studio window instead of allowing the child
                 // window to handle those events.
                 xcb_set_input_focus(x11_connection.get(),
-                                    XCB_INPUT_FOCUS_PARENT, child_window,
+                                    XCB_INPUT_FOCUS_PARENT, wine_window,
                                     XCB_CURRENT_TIME);
                 xcb_flush(x11_connection.get());
                 break;
@@ -228,8 +258,8 @@ void Editor::fix_local_coordinates() const {
 
     xcb_configure_notify_event_t translated_event{};
     translated_event.response_type = XCB_CONFIGURE_NOTIFY;
-    translated_event.event = child_window;
+    translated_event.event = wine_window;
-    translated_event.window = child_window;
+    translated_event.window = wine_window;
     // This should be set to the same sizes the window was created on. Since
     // we're not using `SetWindowPos` to resize the Window, Wine can get a bit
     // confused when we suddenly report a different client area size. Without
@@ -240,7 +270,7 @@ void Editor::fix_local_coordinates() const {
     translated_event.y = translated_coordinates->dst_y;
 
     xcb_send_event(
-        x11_connection.get(), false, child_window,
+        x11_connection.get(), false, wine_window,
         XCB_EVENT_MASK_STRUCTURE_NOTIFY | XCB_EVENT_MASK_SUBSTRUCTURE_NOTIFY,
         reinterpret_cast<char*>(&translated_event));
     xcb_flush(x11_connection.get());

src/wine-host/editor.h

@@ -32,6 +32,7 @@
 #include <optional>
 #include <string>
 
+#include "../common/configuration.h"
 #include "utils.h"
 
 /**
@@ -80,21 +81,31 @@ class Editor {
      * Open a window, embed it into the DAW's parent window and create a handle
      * to the new Win32 window that can be used by the hosted VST plugin.
      *
+     * @param config This instance's configuration, used to enable alternative
+     *   editor behaviours.
      * @param window_class_name The name for the window class for editor
      *   windows.
-     * @param effect The plugin this window is being created for. Used to send
-     *   `effEditIdle` messages on a timer.
      * @param parent_window_handle The X11 window handle passed by the VST host
      *   for the editor to embed itself into.
+     * @param effect The plugin this window is being created for. Used to send
+     *   `effEditIdle` messages on a timer.
      *
      * @see win32_handle
      */
-    Editor(const std::string& window_class_name,
+    Editor(const Configuration& config,
-           AEffect* effect,
+           const std::string& window_class_name,
-           const size_t parent_window_handle);
+           const size_t parent_window_handle,
+           AEffect* effect);
 
     ~Editor();
 
+    /**
+     * Get the Win32 window handle so it can be passed to an `effEditOpen()`
+     * call. This will return the child window's handle if double editor
+     * embedding is enabled.
+     */
+    HWND get_win32_handle();
+
     /**
      * Send a single `effEditIdle` event to the plugin to allow it to update its
      * GUI state. This is called periodically from a timer while the GUI is
@@ -144,7 +155,6 @@ class Editor {
      */
     const WindowClass window_class;
 
-   public:
     /**
      * The handle for the window created through Wine that the plugin uses to
      * embed itself in.
@@ -152,6 +162,18 @@ class Editor {
     std::unique_ptr<std::remove_pointer_t<HWND>, decltype(&DestroyWindow)>
         win32_handle;
 
+    /**
+     * A child window embedded inside of `win32_handle`. This is only used if
+     * the `editor_double_embed` option is enabled. It can be used as a
+     * workaround for plugins that rely on their parent window's screen
+     * coordinates instead of their own (see the 'Editor hosting modes' section
+     * of the readme for more details). The plugin should then embed itself
+     * within this child window.
+     */
+    std::optional<
+        std::unique_ptr<std::remove_pointer_t<HWND>, decltype(&DestroyWindow)>>
+        win32_child_handle;
+
    private:
     /**
      * The Win32 API will block the `DispatchMessage` call when opening e.g. a
@@ -169,7 +191,7 @@ class Editor {
     /**
      * The X11 window handle of the window belonging to  `win32_handle`.
      */
-    const xcb_window_t child_window;
+    const xcb_window_t wine_window;
     /**
      * The X11 window that's at the top of the window tree starting from
      * `parent_window`, i.e. a direct child of the root window. In most cases