Allow resizing shared memory buffers in setActive

REAPER apparently changes bus arrangements between
`IAudioProcessor::setupProcessing()` and `IAudioProcessor::setActive()`.

CHANGELOG.md

@@ -69,6 +69,8 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixed
 
+- Fixed manually changing channel counts with supported VST3 plugins in
+  **REAPER** not working.
 - Fixed an obscure issue with VST3 plugins crashing in **Ardour** on
   Arch/Manjaro because of misreported parameter queue lengths.
 - Some of yabridge's socket file names contained extremely aesthetically

src/common/logging/vst3.cpp

@@ -1897,6 +1897,19 @@ void Vst3Logger::log_response(
     });
 }
 
+void Vst3Logger::log_response(bool is_host_vst,
+                              const YaComponent::SetActiveResponse& response) {
+    log_response_base(is_host_vst, [&](auto& message) {
+        message << response.result.string();
+        if (response.result == Steinberg::kResultOk &&
+            response.updated_audio_buffers_config) {
+            message << ", <new shared memory configuration for \""
+                    << response.updated_audio_buffers_config->name << "\", "
+                    << response.updated_audio_buffers_config->size << " bytes>";
+        }
+    });
+}
+
 void Vst3Logger::log_response(
     bool is_host_vst,
     const YaPrefetchableSupport::GetPrefetchableSupportResponse& response) {

src/common/logging/vst3.h

@@ -319,6 +319,7 @@ class Vst3Logger {
                       bool from_cache = false);
     void log_response(bool is_host_vst,
                       const YaComponent::GetRoutingInfoResponse&);
+    void log_response(bool is_host_vst, const YaComponent::SetActiveResponse&);
     void log_response(
         bool is_host_vst,
         const YaPrefetchableSupport::GetPrefetchableSupportResponse&);

src/common/serialization/vst3/plugin/audio-processor.h

@@ -194,6 +194,11 @@ class YaAudioProcessor : public Steinberg::Vst::IAudioProcessor {
     /**
      * Message to pass through a call to
      * `IAudioProcessor::setupProcessing(setup)` to the Wine plugin host.
+     *
+     * Here Wine plugin host will set up the shared memory buffers.
+     *
+     * NOTE: This process is repeated as part of `SetActive`. Apparently REAPER
+     *       can change bus arrangements after the processing has been set up.
      */
     struct SetupProcessing {
         using Response = SetupProcessingResponse;

src/common/serialization/vst3/plugin/component.h

@@ -18,6 +18,7 @@
 
 #include <pluginterfaces/vst/ivstcomponent.h>
 
+#include "../../../audio-shm.h"
 #include "../../../bitsery/ext/in-place-optional.h"
 #include "../../common.h"
 #include "../base.h"
@@ -251,12 +252,32 @@ class YaComponent : public Steinberg::Vst::IComponent {
                                            int32 index,
                                            TBool state) override = 0;
 
+    /**
+     * The response code and written state for a call to
+     * `IAudioProcessor::setupProcessing(setup)`.
+     */
+    struct SetActiveResponse {
+        UniversalTResult result;
+        std::optional<AudioShmBuffer::Config> updated_audio_buffers_config;
+
+        template <typename S>
+        void serialize(S& s) {
+            s.object(result);
+            s.ext(updated_audio_buffers_config,
+                  bitsery::ext::InPlaceOptional{});
+        }
+    };
+
     /**
      * Message to pass through a call to `IComponent::setActive(state)` to the
      * Wine plugin host.
+     *
+     * NOTE: REAPER may change a plugin's bus arrangements after the processing
+     *       has been set up, so we need to check for this on every
+     *       `setActive()` call
      */
     struct SetActive {
-        using Response = UniversalTResult;
+        using Response = SetActiveResponse;
 
         native_size_t instance_id;
 

src/plugin/bridges/vst3-impls/plugin-proxy.cpp

@@ -416,14 +416,31 @@ Vst3PluginProxyImpl::activateBus(Steinberg::Vst::MediaType type,
 
 tresult PLUGIN_API Vst3PluginProxyImpl::setActive(TBool state) {
     // HACK: Even though we initially implemented this cache specifically for
-    //       REAPER, REAPER doesn't use `IComponent::setProcessing` properly and
-    //       calls it before doing setting up input and output busses. So now
-    //       our workaround to get acceptable performance in REAPER needs a
-    //       workaround of its ownn.  Great!
+    //       REAPER, REAPER doesn't use `IComponent::setProcessing()` properly
+    //       and calls it before doing setting up input and output busses. So
+    //       now our workaround to get acceptable performance in REAPER needs a
+    //       workaround of its own. Great!
     clear_bus_cache();
 
-    return bridge_.send_audio_processor_message(
+    const SetActiveResponse response = bridge_.send_audio_processor_message(
         YaComponent::SetActive{.instance_id = instance_id(), .state = state});
+
+    // NOTE: REAPER may (and will) change a plugin's channel layout after
+    //       calling `setupProcessing()`. Because of that, we need to test
+    //       whether this has happened any time the plugin gets reactivated.
+    //       It's technically legal, so we need to support it.
+    if (response.updated_audio_buffers_config) {
+        // The host should absolutely not call this function before
+        // `setupProcessing()` so this should already contain a value, but you
+        // never know...
+        if (!process_buffers_) {
+            process_buffers_.emplace(*response.updated_audio_buffers_config);
+        } else {
+            process_buffers_->resize(*response.updated_audio_buffers_config);
+        }
+    }
+
+    return response.result;
 }
 
 tresult PLUGIN_API Vst3PluginProxyImpl::setState(Steinberg::IBStream* state) {

src/wine-host/bridges/vst3.cpp

@@ -1351,20 +1351,18 @@ Vst3Bridge::get_instance(size_t instance_id) noexcept {
         object_instances_.at(instance_id), std::move(lock));
 }
 
-AudioShmBuffer::Config Vst3Bridge::setup_shared_audio_buffers(
+std::optional<AudioShmBuffer::Config> Vst3Bridge::setup_shared_audio_buffers(
     size_t instance_id) {
     const auto& [instance, _] = get_instance(instance_id);
-    // TODO: When also calling this in setActive, return a nullopt instead of
-    //       asserting
-    assert(instance.process_buffers);
 
     const Steinberg::IPtr<Steinberg::Vst::IComponent> component =
         instance.interfaces.component;
     const Steinberg::IPtr<Steinberg::Vst::IAudioProcessor> audio_processor =
         instance.interfaces.audio_processor;
-    // TODO: When also calling this in setActive, return a nullopt instead of
-    //       asserting
-    assert(component && audio_processor);
+
+    if (!instance.process_setup || !component || !audio_processor) {
+        return std::nullopt;
+    }
 
     // We'll query the plugin for its audio bus layouts, and then create
     // calculate the offsets in a large memory buffer for the different audio
@@ -1575,7 +1573,7 @@ size_t Vst3Bridge::register_object_instance(
                         // buffers.
                         instance.process_setup = request.setup;
                         const AudioShmBuffer::Config audio_buffers_config =
-                            setup_shared_audio_buffers(request.instance_id);
+                            *setup_shared_audio_buffers(request.instance_id);
 
                         return YaAudioProcessor::SetupProcessingResponse{
                             .result = result,
@@ -1739,12 +1737,28 @@ size_t Vst3Bridge::register_object_instance(
                         //       deadlocks caused by mutually recursive function
                         //       calls.
                         return do_mutual_recursion_on_off_thread(
-                            [&]() -> tresult {
+                            [&]() -> YaComponent::SetActive::Response {
                                 const auto& [instance, _] =
                                     get_instance(request.instance_id);
 
-                                return instance.interfaces.component->setActive(
+                                const tresult result =
+                                    instance.interfaces.component->setActive(
                                         request.state);
+
+                                // NOTE: REAPER may change the bus layout after
+                                //       calling
+                                //       `IAudioProcessor::setupProcessing`. In
+                                //       that case we'll need to resize the
+                                //       shared memory buffers here.
+                                const std::optional<AudioShmBuffer::Config>
+                                    updated_audio_buffers_config =
+                                        setup_shared_audio_buffers(
+                                            request.instance_id);
+
+                                return YaComponent::SetActiveResponse{
+                                    .result = result,
+                                    .updated_audio_buffers_config = std::move(
+                                        updated_audio_buffers_config)};
                             });
                     },
                     [&](const YaPrefetchableSupport::GetPrefetchableSupport&

src/wine-host/bridges/vst3.h

@@ -456,10 +456,12 @@ class Vst3Bridge : public HostBridge {
      * instance and return the configuration so the native plugin can connect to
      * it as well.
      *
-     * This uses the `Vst3PluginInstance::process_setup` field, so that needs to
-     * be set first or this function raise SIGABRT.
+     * This returns a nullopt when `Vst3PluginInstance::process_setup` is not
+     * set, or when the object instance does not support the `IAudioProcessor`
+     * interface.
      */
-    AudioShmBuffer::Config setup_shared_audio_buffers(size_t instance_id);
+    std::optional<AudioShmBuffer::Config> setup_shared_audio_buffers(
+        size_t instance_id);
 
     /**
      * Assign a unique identifier to an object and add it to