Add a limit to all Win32 message loops #28

This works around Waves plugins causing an infinite message loop. Since
we run the loop 30 times per second anyways splitting the loop up into
chunks of 20 shouldn't be an issue.

CHANGELOG.md

@@ -12,6 +12,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 - Don't print calls to `effIdle()` when `YABRIDGE_DEBUG_LEVEL` is set to 1.
 
+### Fixed
+
+- Fix Waves plugins from freezing the plugin process by preventing them from
+  causing an infinite message loop.
+
 ## [1.4.1] - 2020-07-27
 
 ### yabridgectl

src/wine-host/bridges/group.cpp

@@ -262,12 +262,14 @@ void GroupBridge::async_handle_events() {
             MSG msg;
 
             // Keep the loop responsive by not handling too many events at once
-            // TODO: For some reason the Melda plugins run into a seemingly
-            //       infinite timer loop for a little while after opening a
-            //       second editor. Without this limit everything will get
-            //       blocked indefinitely. How could this be fixed?
-            for (int i = 0;
-                 i < 20 && PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE); i++) {
+            //
+            // For some reason the Melda plugins run into a seemingly infinite
+            // timer loop for a little while after opening a second editor.
+            // Without this limit everything will get blocked indefinitely. How
+            // could this be fixed?
+            for (int i = 0; i < max_win32_messages &&
+                            PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE);
+                 i++) {
                 TranslateMessage(&msg);
                 DispatchMessage(&msg);
             }

src/wine-host/bridges/vst2.cpp

@@ -397,22 +397,23 @@ intptr_t Vst2Bridge::dispatch_wrapper(AEffect* plugin,
 }
 
 void Vst2Bridge::handle_win32_events() {
-    std::visit(
-        overload{[](Editor& editor) { editor.handle_win32_events(); },
-                 [](std::monostate&) {
-                     MSG msg;
+    std::visit(overload{[](Editor& editor) { editor.handle_win32_events(); },
+                        [](std::monostate&) {
+                            MSG msg;
 
-                     while (PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE)) {
-                         TranslateMessage(&msg);
-                         DispatchMessage(&msg);
-                     }
-                 },
-                 [](EditorOpening&) {
-                     // Don't handle any events in this
-                     // particular case as explained in
-                     // `Vst2Bridge::editor`
-                 }},
-        editor);
+                            for (int i = 0;
+                                 i < max_win32_messages &&
+                                 PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE);
+                                 i++) {
+                                TranslateMessage(&msg);
+                                DispatchMessage(&msg);
+                            }
+                        },
+                        [](EditorOpening&) {
+                            // Don't handle any events in this particular case
+                            // as explained in `Vst2Bridge::editor`
+                        }},
+               editor);
 }
 
 void Vst2Bridge::handle_x11_events() {

src/wine-host/editor.cpp

@@ -173,8 +173,12 @@ void Editor::handle_win32_events() const {
     MSG msg;
 
     // The null value for the second argument is needed to handle interaction
-    // with child GUI components
-    while (PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE)) {
+    // with child GUI components. So far limiting this to `max_win32_messages`
+    // messages has only been needed for Waves plugins as they otherwise cause
+    // an infinite message loop.
+    for (int i = 0;
+         i < max_win32_messages && PeekMessage(&msg, nullptr, 0, 0, PM_REMOVE);
+         i++) {
         // This timer would periodically send `effEditIdle` events so the editor
         // remains responsive even during blocking GUI operations such as open
         // dropdowns or message boxes. This is only needed when the GUI is

src/wine-host/editor.h

@@ -35,6 +35,17 @@
 #include "../common/configuration.h"
 #include "utils.h"
 
+/**
+ * The maximum number of Win32 messages to handle per message loop. This is
+ * needed because otherwise some plugins can run into an infinite loop. I've
+ * observed this with:
+ *
+ * - Waves plugins
+ * - Melda plugins when having multiple editor windows open within a single
+ *   plugin group
+ */
+constexpr int max_win32_messages = 20;
+
 /**
  * Used to store the maximum width and height of a screen.
  */