Move all window handling to the Wine host side

src/common/events.h

@@ -202,14 +202,23 @@ void passthrough_event(boost::asio::local::stream_protocol::socket& socket,
             [&](const std::string& s) -> void* {
                 return const_cast<char*>(s.c_str());
             },
+            [&](size_t& window_handle) -> void* {
+                // This is the X11 window handle that the editor should reparent
+                // itself to. We have a special wrapper around the dispatch
+                // function that intercepts `effEditOpen` events and creates a
+                // Win32 window and then finally embeds the X11 window Wine
+                // created into this wnidow handle.
+                // TODO: Check if the host passes the window handle like this,
+                //       or if the window handle is behind the pointer
+                return reinterpret_cast<void*>(window_handle);
+            },
             [&](const AEffect&) -> void* { return nullptr; },
             [&](DynamicVstEvents& events) -> void* {
                 return &events.as_c_events();
             },
             [&](WantsChunkBuffer&) -> void* { return string_buffer.data(); },
             [&](const WantsVstTimeInfo&) -> void* { return nullptr; },
-            [&](WantsString&) -> void* { return string_buffer.data(); },
-            [&](WantsWindowHandle&) -> void* { return string_buffer.data(); }},
+            [&](WantsString&) -> void* { return string_buffer.data(); }},
         event.payload);
 
     const intptr_t return_value = callback(plugin, event.opcode, event.index,
@@ -264,17 +273,6 @@ void passthrough_event(boost::asio::local::stream_protocol::socket& socket,
                  },
                  [&](WantsString&) -> EventResposnePayload {
                      return std::string(static_cast<char*>(data));
-                 },
-                 [&](WantsWindowHandle&) -> EventResposnePayload {
-                     // This is a bit of a hack, but I couldn't think of a nicer
-                     // way to do this since it's only needed for the
-                     // `effEditOpen` event. We override the callback function
-                     // to create a Win32 window, pass that to the plugin, and
-                     // then write the corresponding X11 window handle to the
-                     // data pointer.
-                     // TODO: I really want to build a more obvious mechanism
-                     //       for this
-                     return *reinterpret_cast<intptr_t*>(data);
                  }},
         event.payload);
 

src/common/logging.cpp

@@ -167,6 +167,7 @@ void Logger::log_event(bool is_dispatch,
                         message << "<" << s.size() << " bytes>";
                     }
                 },
+                [&](const intptr_t&) { message << "<nullptr>"; },
                 [&](const AEffect&) { message << "<nullptr>"; },
                 [&](const DynamicVstEvents& events) {
                     message << "<" << events.events.size() << " midi_events>";
@@ -175,8 +176,7 @@ void Logger::log_event(bool is_dispatch,
                     message << "<writable_buffer>";
                 },
                 [&](const WantsVstTimeInfo&) { message << "<nullptr>"; },
-                [&](const WantsString&) { message << "<writable_string>"; },
-                [&](const WantsWindowHandle&) { message << "<nullptr>"; }},
+                [&](const WantsString&) { message << "<writable_string>"; }},
             payload);
 
         message << ")";
@@ -210,10 +210,7 @@ void Logger::log_event_response(bool is_dispatch,
                          }
                      },
                      [&](const AEffect&) { message << ", <AEffect_object>"; },
-                     [&](const VstTimeInfo&) { message << ", <time_info>"; },
-                     [&](const intptr_t& window_handle) {
-                         message << ", window #" << window_handle;
-                     }},
+                     [&](const VstTimeInfo&) { message << ", <time_info>"; }},
             payload);
 
         log(message.str());

src/common/serialization.h

@@ -164,12 +164,6 @@ struct WantsVstTimeInfo {};
  */
 struct WantsString {};
 
-/**
- * The event handler (the Wine VST host) should create a Winn32 window, pass
- * that to the plugin, and return an X11 window handle.
- */
-struct WantsWindowHandle {};
-
 /**
  * VST events are passed a void pointer that can contain a variety of different
  * data types depending on the event's opcode. This is typically either:
@@ -183,6 +177,7 @@ struct WantsWindowHandle {};
  *         size. We can replace `std::string` with `char*` once it does for
  *         clarity's sake.
  *
+ * - An X11 window handle.
  * - Specific data structures from `aeffextx.h`. For instance an event with the
  *   opcode `effProcessEvents` the hosts passes a `VstEvents` struct containing
  *   midi events, and `audioMasterIOChanged` lets the host know that the
@@ -199,12 +194,12 @@ struct WantsWindowHandle {};
  */
 using EventPayload = std::variant<std::nullptr_t,
                                   std::string,
+                                  size_t,
                                   AEffect,
                                   DynamicVstEvents,
                                   WantsChunkBuffer,
                                   WantsVstTimeInfo,
-                                  WantsString,
-                                  WantsWindowHandle>;
+                                  WantsString>;
 
 template <typename S>
 void serialize(S& s, EventPayload& payload) {
@@ -216,6 +211,7 @@ void serialize(S& s, EventPayload& payload) {
                   // also use this to send back large chunk data
                   s.text1b(string, binary_buffer_size);
               },
+              [](S& s, size_t& window_handle) { s.value8b(window_handle); },
               [](S& s, AEffect& effect) { s.object(effect); },
               [](S& s, DynamicVstEvents& events) {
                   s.container(
@@ -223,7 +219,7 @@ void serialize(S& s, EventPayload& payload) {
                       [](S& s, VstEvent& event) { s.container1b(event.dump); });
               },
               [](S&, WantsChunkBuffer&) {}, [](S&, WantsVstTimeInfo&) {},
-              [](S&, WantsString&) {}, [](S&, WantsWindowHandle&) {}});
+              [](S&, WantsString&) {}});
 }
 
 /**
@@ -279,7 +275,7 @@ struct Event {
  * - An X11 window pointer for the editor window.
  */
 using EventResposnePayload =
-    std::variant<std::monostate, std::string, AEffect, VstTimeInfo, intptr_t>;
+    std::variant<std::monostate, std::string, AEffect, VstTimeInfo>;
 
 template <typename S>
 void serialize(S& s, EventResposnePayload& payload) {
@@ -293,8 +289,7 @@ void serialize(S& s, EventResposnePayload& payload) {
                   s.text1b(string, binary_buffer_size);
               },
               [](S& s, AEffect& effect) { s.object(effect); },
-              [](S& s, VstTimeInfo& time_info) { s.object(time_info); },
-              [](S& s, intptr_t& v) { s.value8b(v); }});
+              [](S& s, VstTimeInfo& time_info) { s.object(time_info); }});
 }
 
 /**

src/plugin/host-bridge.cpp

@@ -195,7 +195,13 @@ class DispatchDataConverter : DefaultDataConverter {
                 return std::nullopt;
                 break;
             case effEditOpen:
-                return WantsWindowHandle();
+                // The host will have passed us an X11 window handle in the void
+                // pointer. In the Wine VST host we'll create a Win32 window,
+                // ask the plugin to embed itself in that and then embed that
+                // window into this X11 window handle.
+                // TODO: Check if the host passes the window handle like this,
+                //       or if the window handle is behind the pointer
+                return reinterpret_cast<size_t>(data);
                 break;
             case effGetChunk:
                 return WantsChunkBuffer();
@@ -216,10 +222,6 @@ class DispatchDataConverter : DefaultDataConverter {
 
     void write(const int opcode, void* data, const EventResult& response) {
         switch (opcode) {
-            case effEditOpen:
-                // TODO: Write the returned window handle somewhere
-                DefaultDataConverter::write(opcode, data, response);
-                break;
             case effGetChunk:
                 // Write the chunk data to some publically accessible place in
                 // `HostBridge` and write a pointer to that struct to the data
@@ -312,26 +314,6 @@ intptr_t HostBridge::dispatch(AEffect* /*plugin*/,
                 return return_value;
             }
             break;
-        case effEditOpen:
-            // TODO: Should work as follows:
-            //       1. Create a window in the Wine host using the Windows APIs
-            //       2. Return the X11 window handle
-            //       4. Use XEmbed to embmed the Wine window inside the parent
-            //          window stored in the data void pointer
-            //       5. Return a handle to the X11 window
-            {
-                // The plugin will return 0 if it does not actually support
-                // opening a window
-                const auto return_value =
-                    send_event(host_vst_dispatch, dispatch_semaphore, converter,
-                               std::pair<Logger&, bool>(logger, true), opcode,
-                               index, value, data, option);
-                if (return_value == 0) {
-                    return 0;
-                }
-
-                return return_value;
-            }
     }
 
     // TODO: Maybe reuse buffers here when dealing with chunk data

src/wine-host/plugin-bridge.cpp

@@ -131,18 +131,21 @@ PluginBridge::PluginBridge(std::string plugin_dll_path,
                     // We have to intercept GUI open calls since we can't use
                     // the X11 window handle passed by the host
                     if (opcode == effEditOpen) {
-                        const auto handle = editor.open();
+                        const auto win32_handle = editor.open();
 
                         // The plugin will return 0 if it can not open its
                         // editor window (or if it does not support it, but in
                         // that case the DAW should be hiding the option)
                         const intptr_t return_value = plugin->dispatcher(
-                            plugin, opcode, index, value, handle, option);
+                            plugin, opcode, index, value, win32_handle, option);
                         if (return_value == 0) {
                             return 0;
                         }
 
-                        // TODO: Return X11 handle
+                        // TODO: Embed the Win32 window into the X11 window
+                        // handle
+                        const auto x11_handle = reinterpret_cast<size_t>(data);
+
                         return return_value;
                     }