diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ac4363..388ab699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ Versioning](https://semver.org/spec/v2.0.0.html). this can significantly reduce the overhead of bridging VST3 plugins under those hosts. +### Changed + +- Redesigned the VST3 audio socket handling to be able to reuse the process data + objects on both sides. This greatly reduces the overhead of our VST3 bridging + by getting rid of all memory allocations during audio processing. + ## [3.2.0] - 2021-05-03 ### Added diff --git a/src/common/communication/vst3.h b/src/common/communication/vst3.h index c818c7fc..0a5c015e 100644 --- a/src/common/communication/vst3.h +++ b/src/common/communication/vst3.h @@ -237,7 +237,12 @@ class Vst3MessageHandler : public AdHocSocketHandler { auto [logger, is_host_vst] = *logging; return logger.log_request(is_host_vst, object); }, - request); + // In the case of `AudioProcessorRequest`, we need to + // actually fetch the variant field since our object + // also contains a persistent object to store process + // data into so we can prevent allocations during audio + // processing + get_request_variant(request)); } // We do the visiting here using a templated lambda. This way we @@ -258,7 +263,8 @@ class Vst3MessageHandler : public AdHocSocketHandler { write_object(socket, response); } }, - request); + // See above + get_request_variant(request)); }; this->receive_multi(logging @@ -442,6 +448,20 @@ class Vst3Sockets : public Sockets { audio_processor_buffers.at(object.instance_id)); } + /** + * Overload for use with `MessageReference`, since we cannot + * directly get the instance ID there. + */ + template + typename T::Response send_audio_processor_message( + const MessageReference& object_ref, + std::optional> logging) { + return audio_processor_sockets.at(object_ref.get().instance_id) + .send_message( + object_ref, logging, + audio_processor_buffers.at(object_ref.get().instance_id)); + } + /** * For sending messages from the host to the plugin. After we have a better * idea of what our communication model looks like we'll probably want to diff --git a/src/common/logging/vst3.cpp b/src/common/logging/vst3.cpp index 5fefe136..9b1458e5 100644 --- a/src/common/logging/vst3.cpp +++ b/src/common/logging/vst3.cpp @@ -992,13 +992,15 @@ bool Vst3Logger::log_request(bool is_host_vst, }); } -bool Vst3Logger::log_request(bool is_host_vst, - const YaAudioProcessor::Process& request) { +bool Vst3Logger::log_request( + bool is_host_vst, + const MessageReference& request_wrapper) { return log_request_base( is_host_vst, Logger::Verbosity::all_events, [&](auto& message) { // This is incredibly verbose, but if you're really a plugin that // handles processing in a weird way you're going to need all of // this + const YaAudioProcessor::Process& request = request_wrapper.get(); std::ostringstream num_input_channels; num_input_channels << "["; diff --git a/src/common/logging/vst3.h b/src/common/logging/vst3.h index ad167825..8e184342 100644 --- a/src/common/logging/vst3.h +++ b/src/common/logging/vst3.h @@ -191,7 +191,8 @@ class Vst3Logger { bool log_request(bool is_host_vst, const YaAudioProcessor::SetupProcessing&); bool log_request(bool is_host_vst, const YaAudioProcessor::SetProcessing&); - bool log_request(bool is_host_vst, const YaAudioProcessor::Process&); + bool log_request(bool is_host_vst, + const MessageReference&); bool log_request(bool is_host_vst, const YaAudioProcessor::GetTailSamples&); bool log_request(bool is_host_vst, const YaComponent::GetControllerClassId&); diff --git a/src/common/serialization/vst3.h b/src/common/serialization/vst3.h index 3f922a1a..0e2e5148 100644 --- a/src/common/serialization/vst3.h +++ b/src/common/serialization/vst3.h @@ -20,6 +20,7 @@ #include +#include "../bitsery/ext/message-reference.h" #include "../configuration.h" #include "../utils.h" #include "common.h" @@ -153,31 +154,78 @@ void serialize(S& s, ControlRequest& payload) { * A subset of all functions a host can call on a plugin. These functions are * called from a hot loop every processing cycle, so we want a dedicated socket * for these for every plugin instance. + * + * We use a separate struct for this so we can keep the + * `YaAudioProcessor::Process` object, which also contains the entire audio + * processing data struct, alive as a thread local static object on the Wine + * side, and as a regular field in `Vst3PluginProxyImpl` on the plugin side. In + * our variant we then store a `MessageReference` that points to this object, + * and we'll do some magic to be able to serialize and deserialize this object + * without needing to create copies. See `MessageReference` and + * `bitsery::ext::MessageReference` for more information. */ -using AudioProcessorRequest = - std::variant; +struct AudioProcessorRequest { + AudioProcessorRequest() {} -template -void serialize(S& s, AudioProcessorRequest& payload) { - // All of the objects in `AudioProcessorRequest` should have their own - // serialization function. - s.ext(payload, bitsery::ext::StdVariant{}); -} + /** + * Initialize the variant with an object. In `Vst3Sockets::send_message()` + * the object gets implicitly converted to the this variant. + */ + template + AudioProcessorRequest(T request) : payload(std::move(request)) {} + + using Payload = + std::variant, + YaAudioProcessor::GetTailSamples, + YaComponent::GetControllerClassId, + YaComponent::SetIoMode, + YaComponent::GetBusCount, + YaComponent::GetBusInfo, + YaComponent::GetRoutingInfo, + YaComponent::ActivateBus, + YaComponent::SetActive, + YaPrefetchableSupport::GetPrefetchableSupport>; + + Payload payload; + + template + void serialize(S& s) { + s.ext( + payload, + bitsery::ext::StdVariant{ + [&](S& s, + MessageReference& request_ref) { + // When serializing this reference we'll read the data + // directly from the referred to object. During + // deserializing we'll deserialize into the persistent and + // thread local `process_request` object (see + // `Vst3Sockets::add_audio_processor_and_listen`) and then + // reassign the reference to point to that boject. + s.ext(request_ref, + bitsery::ext::MessageReference(process_request)); + }, + [](S& s, auto& request) { s.object(request); }}); + } + + /** + * Used for deserializing the `MessageReference` + * variant. When we encounter this variant, we'll actually deserialize the + * object into this object, and we'll then reassign the reference to point + * to this object. That way we can keep it around as a thread local object + * to prevent unnecessary allocations. + */ + std::optional process_request; +}; /** * When we do a callback from the Wine VST host to the plugin, this encodes the @@ -222,3 +270,26 @@ void serialize(S& s, CallbackRequest& payload) { // serialization function. s.ext(payload, bitsery::ext::StdVariant{}); } + +/** + * Get the actual variant for a request. We need a function for this to be able + * to handle composite types, like `AudioProcessorRequest` that use + * `MesasgeReference` to be able to store persistent objects in the message + * variant. + */ +template +std::variant& get_request_variant(std::variant& request) { + return request; +} + +/** + * Fetch the `std::variant<>` from an audio processor request object. This will + * let us use our regular, simple function call dispatch code, but we can still + * store the process data in a separate field (to reduce allocations). + * + * @overload + */ +inline AudioProcessorRequest::Payload& get_request_variant( + AudioProcessorRequest& request) { + return request.payload; +} diff --git a/src/common/serialization/vst3/process-data.cpp b/src/common/serialization/vst3/process-data.cpp index 213a4609..72369367 100644 --- a/src/common/serialization/vst3/process-data.cpp +++ b/src/common/serialization/vst3/process-data.cpp @@ -281,6 +281,9 @@ YaProcessDataResponse YaProcessData::move_outputs_to_response() { // `ProcessData` object generated in `get()` here sicne these of // course are not references or pointers like all other fields, so // they're not implicitly copied like all of our other fields + // FIXME: Instead of moving, the `YaProcessDataResponse` should be an + // (optional) field. Moving defeats the point of us trying to reuse + // these objects. for (int i = 0; i < reconstructed_process_data.numOutputs; i++) { outputs[i].silence_flags = reconstructed_process_data.outputs[i].silenceFlags; diff --git a/src/plugin/bridges/vst3-impls/plugin-proxy.cpp b/src/plugin/bridges/vst3-impls/plugin-proxy.cpp index 78a5ffc5..a0177105 100644 --- a/src/plugin/bridges/vst3-impls/plugin-proxy.cpp +++ b/src/plugin/bridges/vst3-impls/plugin-proxy.cpp @@ -216,13 +216,12 @@ Vst3PluginProxyImpl::process(Steinberg::Vst::ProcessData& data) { } // We reuse this existing object to avoid allocations - process_data.repopulate(data); + process_request.instance_id = instance_id(); + process_request.data.repopulate(data); + process_request.new_realtime_priority = new_realtime_priority; - ProcessResponse response = - bridge.send_audio_processor_message(YaAudioProcessor::Process{ - .instance_id = instance_id(), - .data = process_data, - .new_realtime_priority = new_realtime_priority}); + ProcessResponse response = bridge.send_audio_processor_message( + MessageReference(process_request)); response.output_data.write_back_outputs(data); diff --git a/src/plugin/bridges/vst3-impls/plugin-proxy.h b/src/plugin/bridges/vst3-impls/plugin-proxy.h index 41a69e6f..315da918 100644 --- a/src/plugin/bridges/vst3-impls/plugin-proxy.h +++ b/src/plugin/bridges/vst3-impls/plugin-proxy.h @@ -478,10 +478,16 @@ class Vst3PluginProxyImpl : public Vst3PluginProxy { std::atomic_size_t current_context_menu_id; /** - * We'll reuse this process data object and simply fill the objects - * contained with new data to avoid allocations during audio processing. + * NOTE: We'll reuse the request objects for the audio processor so we can + * keep the process data object (which contains vectors and other heap + * allocated data structure) alive. We'll then just fill this object + * with new data every processing cycle to prevent allocations. Then, + * we pass a `MessageReference` to our + * sockets. This together with `bitisery::ext::MessageReference` will + * let us serialize from and to existing objects without having to + * copy or reallocate them. */ - YaProcessData process_data; + YaAudioProcessor::Process process_request; // Caches diff --git a/src/wine-host/bridges/vst3.cpp b/src/wine-host/bridges/vst3.cpp index abb8f3c7..6f4147d0 100644 --- a/src/wine-host/bridges/vst3.cpp +++ b/src/wine-host/bridges/vst3.cpp @@ -1235,8 +1235,18 @@ size_t Vst3Bridge::register_object_instance( return object_instances[request.instance_id] .audio_processor->setProcessing(request.state); }, - [&](YaAudioProcessor::Process& request) + [&](MessageReference& + request_ref) -> YaAudioProcessor::Process::Response { + // NOTE: To prevent allocations we keep this actual + // `YaAudioProcessor::Process` object around as + // part of a static thread local + // `AudioProcessorRequest` object, and we only + // store a reference to it in our variant (this is + // done during the deserialization in + // `bitsery::ext::MessageReference`) + YaAudioProcessor::Process& request = request_ref.get(); + // Most plugins will already enable FTZ, but there are a // handful of plugins that don't that suffer from // extreme DSP load increases when they start producing @@ -1251,6 +1261,9 @@ size_t Vst3Bridge::register_object_instance( true, *request.new_realtime_priority); } + // TODO: This `get()` now moves data. We should avoid + // that, since that would require reallocating the + // process data next iteration. const tresult result = object_instances[request.instance_id] .audio_processor->process(request.data.get());