Add serialization wrapper for process data

This isn't used yet, but in theory we can just hook this up now and
audio processing will work.

src/common/serialization/clap.h

@@ -32,6 +32,7 @@
 #include "clap/ext/tail.h"
 #include "clap/host.h"
 #include "clap/plugin-factory.h"
+#include "clap/process.h"
 #include "common.h"
 
 // The CLAP communication strategy is identical to what we do for VST3.

src/common/serialization/clap/process.cpp

@@ -0,0 +1,242 @@
+// yabridge: a Wine plugin bridge
+// Copyright (C) 2020-2022 Robbert van der Helm
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more destates.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+#include "process.h"
+
+namespace clap {
+namespace process {
+
+Process::Process() noexcept {}
+
+void Process::repopulate(const clap_process_t& process,
+                         AudioShmBuffer& shared_audio_buffers) {
+    assert(process.in_events && process.out_events);
+    if (process.audio_inputs_count > 0) {
+        assert(process.audio_inputs);
+    }
+    if (process.audio_outputs_count > 0) {
+        assert(process.audio_outputs);
+    }
+
+    // In this function and in every function we call, we should be careful to
+    // not use `push_back`/`emplace_back` anywhere. Resizing vectors and
+    // modifying them in place performs much better because that avoids
+    // destroying and creating objects most of the time.
+    steady_time_ = process.steady_time;
+    frames_count_ = process.frames_count;
+
+    if (process.transport) {
+        transport_.emplace(*process.transport);
+    } else {
+        transport_.reset();
+    }
+
+    // The actual audio is stored in an accompanying `AudioShmBuffer` object, so
+    // these inputs and outputs objects are only used to serialize metadata
+    // about the input and output audio bus buffers
+    audio_inputs_.resize(process.audio_inputs_count);
+    audio_inputs_type_.resize(process.audio_inputs_count);
+    for (size_t port = 0; port < process.audio_inputs_count; port++) {
+        // NOTE: With VST3 plugins sometimes hosts provided more ports than the
+        //       plugin asked for (or sometimes fewer, fun). So we'll account
+        //       for both cases just to be safe.
+        audio_inputs_[port].channel_count =
+            std::min(static_cast<uint32_t>(
+                         shared_audio_buffers.num_input_channels(port)),
+                     process.audio_inputs[port].channel_count);
+        audio_inputs_[port].latency = process.audio_inputs[port].latency;
+        audio_inputs_[port].constant_mask =
+            process.audio_inputs[port].constant_mask;
+
+        // We'll encode the port type using a separate vector because we can't
+        // store it in place without creating dangling pointers
+        if (process.audio_inputs[port].data32) {
+            audio_inputs_type_[port] =
+                clap::audio_buffer::AudioBufferType::Float32;
+
+            // We copy the actual input audio for every bus to the shared memory
+            // object
+            for (uint32_t channel = 0;
+                 channel < process.audio_inputs[port].channel_count;
+                 channel++) {
+                std::copy_n(process.audio_inputs[port].data32[channel],
+                            frames_count_,
+                            shared_audio_buffers.input_channel_ptr<float>(
+                                port, channel));
+            }
+        } else if (process.audio_inputs[port].data64) {
+            audio_inputs_type_[port] =
+                clap::audio_buffer::AudioBufferType::Double64;
+
+            for (uint32_t channel = 0;
+                 channel < process.audio_inputs[port].channel_count;
+                 channel++) {
+                std::copy_n(process.audio_inputs[port].data64[channel],
+                            frames_count_,
+                            shared_audio_buffers.input_channel_ptr<double>(
+                                port, channel));
+            }
+        } else {
+            // Only reasonable-ish (it's still not reasonable) time where
+            // neither of the pointers is set
+            assert(process.audio_inputs[port].channel_count == 0);
+        }
+    }
+
+    audio_outputs_.resize(process.audio_outputs_count);
+    audio_outputs_type_.resize(process.audio_outputs_count);
+    for (size_t port = 0; port < process.audio_outputs_count; port++) {
+        // The same notes apply to the outputs
+        audio_outputs_[port].channel_count =
+            std::min(static_cast<uint32_t>(
+                         shared_audio_buffers.num_output_channels(port)),
+                     process.audio_outputs[port].channel_count);
+        audio_outputs_[port].latency = process.audio_outputs[port].latency;
+        // Shouldn't be any reason to bridge this, but who knows what will
+        // happen when we don't
+        audio_outputs_[port].constant_mask =
+            process.audio_outputs[port].constant_mask;
+
+        if (process.audio_outputs[port].data32) {
+            audio_outputs_type_[port] =
+                clap::audio_buffer::AudioBufferType::Float32;
+        } else if (process.audio_outputs[port].data64) {
+            audio_outputs_type_[port] =
+                clap::audio_buffer::AudioBufferType::Double64;
+        } else {
+            // Only reasonable-ish (it's still not reasonable) time where
+            // neither of the pointers is set
+            assert(process.audio_outputs[port].channel_count == 0);
+        }
+    }
+
+    in_events_.repopulate(*process.in_events);
+}
+
+const clap_process_t& Process::reconstruct(
+    std::vector<std::vector<void*>>& input_pointers,
+    std::vector<std::vector<void*>>& output_pointers) {
+    reconstructed_process_data_.steady_time = steady_time_;
+    reconstructed_process_data_.frames_count = frames_count_;
+    reconstructed_process_data_.transport = transport_ ? &*transport_ : nullptr;
+
+    // The actual audio data is contained within a shared memory object, and the
+    // input and output pointers point to regions in that object. These pointers
+    // are calculated during `clap_plugin::activate()`.
+    assert(audio_inputs_.size() <= input_pointers.size() &&
+           audio_outputs_.size() <= output_pointers.size() &&
+           audio_inputs_type_.size() == audio_inputs_.size() &&
+           audio_outputs_type_.size() == audio_outputs_.size());
+    for (size_t port = 0; port < audio_inputs_.size(); port++) {
+        // The sample depth depends on whether the plugin claimed to support
+        // 64-bit or not and whether the host ended up passing us 32-bit or
+        // 64-bit audio
+        switch (audio_inputs_type_[port]) {
+            case clap::audio_buffer::AudioBufferType::Float32:
+            default:
+                audio_inputs_[port].data32 =
+                    reinterpret_cast<float**>(input_pointers[port].data());
+                break;
+            case clap::audio_buffer::AudioBufferType::Double64:
+                audio_inputs_[port].data64 =
+                    reinterpret_cast<double**>(input_pointers[port].data());
+                break;
+        }
+    }
+    for (size_t port = 0; port < audio_outputs_.size(); port++) {
+        switch (audio_outputs_type_[port]) {
+            case clap::audio_buffer::AudioBufferType::Float32:
+            default:
+                audio_outputs_[port].data32 =
+                    reinterpret_cast<float**>(output_pointers[port].data());
+                break;
+            case clap::audio_buffer::AudioBufferType::Double64:
+                audio_outputs_[port].data64 =
+                    reinterpret_cast<double**>(output_pointers[port].data());
+                break;
+        }
+    }
+
+    reconstructed_process_data_.audio_inputs = audio_inputs_.data();
+    reconstructed_process_data_.audio_outputs = audio_outputs_.data();
+    reconstructed_process_data_.audio_inputs_count =
+        static_cast<uint32_t>(audio_inputs_.size());
+    reconstructed_process_data_.audio_outputs_count =
+        static_cast<uint32_t>(audio_outputs_.size());
+
+    out_events_.clear();
+    reconstructed_process_data_.in_events = in_events_.input_events();
+    reconstructed_process_data_.out_events = out_events_.output_events();
+
+    return reconstructed_process_data_;
+}
+
+Process::Response& Process::create_response() noexcept {
+    // This response object acts as an optimization. It stores pointers to the
+    // original fields in our objects, so we can both only serialize those
+    // fields when sending the response from the Wine side. This lets us avoid
+    // allocations by not having to copy or move the data. On the plugin side we
+    // need to be careful to deserialize into an existing
+    // `clap::plugin::ProcessResponse` object with a response object that
+    // belongs to an actual process data object, because with these changes it's
+    // no longer possible to deserialize those results into a new ad-hoc created
+    // object.
+    response_object_.audio_outputs = &audio_outputs_;
+    response_object_.out_events = &out_events_;
+
+    return response_object_;
+}
+
+void Process::write_back_outputs(const clap_process_t& process,
+                                 const AudioShmBuffer& shared_audio_buffers) {
+    assert(process.audio_outputs && process.out_events);
+
+    assert(audio_outputs_.size() == process.audio_outputs_count);
+    for (size_t port = 0; port < audio_outputs_.size(); port++) {
+        process.audio_outputs[port].constant_mask =
+            audio_outputs_[port].constant_mask;
+        // Don't think the plugin is supposed to change this, but uh may as well
+        process.audio_outputs[port].latency = audio_outputs_[port].latency;
+
+        // `audio_outputs_[port].channel_count` is the minimum of the plugin's
+        // and the host's channel count
+        for (size_t channel = 0; channel < audio_outputs_[port].channel_count;
+             channel++) {
+            // We copy the output audio for every bus from the shared memory
+            // object back to the buffer provided by the host
+            switch (audio_outputs_type_[port]) {
+                case clap::audio_buffer::AudioBufferType::Float32:
+                default:
+                    std::copy_n(shared_audio_buffers.output_channel_ptr<float>(
+                                    port, channel),
+                                process.frames_count,
+                                process.audio_outputs[port].data32[channel]);
+                    break;
+                case clap::audio_buffer::AudioBufferType::Double64:
+                    std::copy_n(shared_audio_buffers.output_channel_ptr<double>(
+                                    port, channel),
+                                process.frames_count,
+                                process.audio_outputs[port].data64[channel]);
+                    break;
+            }
+        }
+    }
+
+    out_events_.write_back_outputs(*process.out_events);
+}
+
+}  // namespace process
+}  // namespace clap

src/common/serialization/clap/process.h

@@ -0,0 +1,247 @@
+// yabridge: a Wine plugin bridge
+// Copyright (C) 2020-2022 Robbert van der Helm
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more destates.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+#pragma once
+
+#include <optional>
+
+#include <clap/process.h>
+#include <llvm/small-vector.h>
+
+#include "../../audio-shm.h"
+#include "../../bitsery/ext/in-place-optional.h"
+#include "audio-buffer.h"
+#include "events.h"
+
+// Serialization messages for `clap/process.h`
+
+namespace clap {
+namespace process {
+
+/**
+ * A serializable wrapper around `clap_process_t`. This works exactly the same
+ * as the process data wrapper for VST3. At the start of a process cycle all
+ * audio data is copied to the already set up shared memory buffers and the
+ * event and transport data is copied to this object. Then on the Wine side
+ * we'll reconstruct the original `clap_process_t` and pass that to the plugin.
+ * The output events are then sent back to the native plugin, where it can write
+ * those along with the audio outputs (which are also in that shared memory
+ * buffer) back to the host. The response is serialized in the
+ * `clap::process::Process::Response` object since only a small amount of
+ * information needs to be sent back.
+ *
+ * As mentioned earlier, the audio data is not stored in this object but it is
+ * instead stored in a shared memory object shared by a `clap_plugin_proxy` and
+ * `ClapPluginInstance` pair. Since the amount of audio data sent per processing
+ * call is fixed, this halves the number of required memory copies.
+ *
+ * Be sure to double check how `clap::process::Process::Response` is used. We do
+ * some pointer tricks there to avoid copies and moves when serializing the
+ * results of our audio processing.
+ */
+class Process {
+   public:
+    /**
+     * Initialize the process data. We only provide a default constructor here,
+     * because we need to fill the existing object with new data every
+     * processing cycle to avoid reallocating a new object every time.
+     */
+    Process() noexcept;
+
+    /**
+     * Copy data from a host provided `clap_process_T` object to this struct
+     * during a process call. This struct can then be serialized, and
+     * `reconstruct()` can then be used again to recreate the original
+     * `clap_process_t` object. This avoids allocating unless it's absolutely
+     * necessary (e.g. when we receive more events than we've received in
+     * previous calls).
+     *
+     * The input audio buffer will be copied to `shared_audio_buffers`. There's
+     * no direct link between this `Process` object and those buffers, but they
+     * should be treated as a pair. This is a bit ugly, but optimizations sadly
+     * never made code prettier.
+     */
+    void repopulate(const clap_process_t& process,
+                    AudioShmBuffer& shared_audio_buffers);
+
+    /**
+     * Reconstruct the original `clap_process_t` object passed to `repopulate()`
+     * and return it. This is used in the Wine plugin host when handling a
+     * `clap_plugin::process()` call.
+     *
+     * Because the actual audio is stored in an `AudioShmBuffer` outside of this
+     * object, we need to make sure that the `AudioBusBuffers` objects we're
+     * using point to the correct buffer even after a resize. To make it more
+     * difficult for us to mess this up, we'll store those bus-channel pointers
+     * in `ClapBridge::ClapPluginInstance` and we'll point the pointers in our
+     * `inputs` and `outputs` fields directly to those pointers. They will have
+     * been set up during `clap_plugin::activate()`.
+     *
+     * CLAP allows mixed float and double precision audio if the plugin opts
+     * into it. The audio buffers thus always contain enough space for double
+     * precision if a port supports it. The actual sample format used is stored
+     * in our `clap::audio_buffer::AudioBuffer` serialization wrapper.
+     */
+    const clap_process_t& reconstruct(
+        std::vector<std::vector<void*>>& input_pointers,
+        std::vector<std::vector<void*>>& output_pointers);
+
+    /**
+     * A serializable wrapper around the output fields of `clap_process_t`, so
+     * we only have to copy the information back that's actually important.
+     * These fields are pointers to the corresponding fields in `Process`. On
+     * the plugin side this information can then be written back to the host.
+     * The actual output audio is stored in the shared memory object.
+     *
+     * HACK: All of this is an optimization to avoid unnecessarily copying or
+     *       moving and reallocating. Directly serializing and deserializing
+     *       from and to pointers does make all of this very error prone, hence
+     *       all of the assertions.
+     *
+     * @see YaProcessData
+     */
+    struct Response {
+        // We store raw pointers instead of references so we can default
+        // initialize this object during deserialization
+        llvm::SmallVectorImpl<clap_audio_buffer_t>* audio_outputs = nullptr;
+        clap::events::EventList* out_events = nullptr;
+
+        template <typename S>
+        void serialize(S& s) {
+            assert(audio_outputs && out_events);
+            // Since these fields are references to the corresponding fields on
+            // the surrounding object, we're actually serializing those fields.
+            // This means that on the plugin side we can _only_ deserialize into
+            // an existing object, since our serializing code doesn't touch the
+            // actual pointers.
+            s.container(*audio_outputs, 1 << 14);
+            s.object(*out_events);
+        }
+    };
+
+    /**
+     * Create a `clap::process::Process::Response` object that refers to the
+     * output fields in this object. The object doesn't store any actual data,
+     * and may not outlive this object. We use this so we only have to copy the
+     * relevant fields back to the host. On the Wine side this function should
+     * only be called after the plugin's `clap_plugin::process()` function has
+     * been called with the reconstructed process data obtained from
+     * `Process::reconstruct()`.
+     *
+     * On the plugin side this should be used to create a response object that
+     * **must** be received into, since we're deserializing directly into some
+     * pointers.
+     */
+    Response& create_response() noexcept;
+
+    /**
+     * Write all of this output data back to the host's `clap_process_t` object.
+     * During this process we'll also write the output audio from the
+     * corresponding shared memory audio buffers back.
+     */
+    void write_back_outputs(const clap_process_t& process,
+                            const AudioShmBuffer& shared_audio_buffers);
+
+    template <typename S>
+    void serialize(S& s) {
+        s.value8b(steady_time_);
+        s.value4b(frames_count_);
+
+        s.ext(transport_, bitsery::ext::InPlaceOptional{});
+
+        // Both `audio_inputs_` and `audio_outputs_` only store metadata. The
+        // actual audio is sent using an accompanying `AudioShmBuffer` object.
+        s.container(audio_inputs_, 1 << 14);
+        s.container1b(audio_inputs_type_, 1 << 14);
+        s.container(audio_outputs_, 1 << 14);
+        s.container1b(audio_outputs_type_, 1 << 14);
+
+        // We don't need to serialize the output events because this will always
+        // be empty on the Wine side. The response is sent back through the
+        // separate `Response` object
+        s.object(in_events_);
+    }
+
+    // These fields are input and context data read from the original
+    // `clap_process_t` object
+
+    int64_t steady_time_ = 0;
+    uint32_t frames_count_ = 0;
+
+    // This is an optional field
+    std::optional<clap_event_transport_t> transport_;
+
+    /**
+     * The audio input buffers for every port. We'll only serialize the metadata
+     * During `reconstruct()` the channel pointers pointers in these objects
+     * will be set to point to our shared memory surface that holds the actual
+     * audio data.
+     */
+    llvm::SmallVector<clap_audio_buffer_t, 8> audio_inputs_;
+    /**
+     * The types corresponding to each buffer in `audio_inputs_`. This needs to
+     * be serialized separately since this information is encoded by setting one
+     * of the two pointers instead of through a flag.
+     */
+    llvm::SmallVector<clap::audio_buffer::AudioBufferType, 8>
+        audio_inputs_type_;
+
+    /**
+     * The audio output buffers for every port. We'll only serialize the
+     * metadata During `reconstruct()` the channel pointers pointers in these
+     * objects will be set to point to our shared memory surface that holds the
+     * actual audio data.
+     */
+    llvm::SmallVector<clap_audio_buffer_t, 8> audio_outputs_;
+    /**
+     * The types corresponding to each buffer in `audio_outputs_`. This needs to
+     * be serialized separately since this information is encoded by setting one
+     * of the two pointers instead of through a flag.
+     */
+    llvm::SmallVector<clap::audio_buffer::AudioBufferType, 8>
+        audio_outputs_type_;
+
+    clap::events::EventList in_events_;
+    clap::events::EventList out_events_;
+
+   private:
+    // These last few members are used on the Wine plugin host side to
+    // reconstruct the original `clap_process_t` object. Here we also initialize
+    // these output fields so the Windows CLAP plugin can write to them though a
+    // regular `ProcessData` object. Finally we can wrap these output fields
+    // back into a `clap::process::Process::Response` using `create_response()`.
+    // so they can be serialized and written back to the host's `clap_process_t`
+    // object.
+
+    /**
+     * This is a `Response` object that contains pointers to other fields in
+     * this struct so we can serialize to and from them.
+     *
+     * NOTE: We use this on the plugin side as an optimization to be able to
+     *       directly receive data into this object, avoiding the need for any
+     *       allocations.
+     */
+    Response response_object_;
+
+    /**
+     * The process data we reconstruct from the other fields during
+     * `reconstruct()`.
+     */
+    clap_process_t reconstructed_process_data_{};
+};
+
+}  // namespace process
+}  // namespace clap

src/plugin/meson.build

@@ -83,6 +83,7 @@ if with_clap
     '../common/serialization/clap/events.cpp',
     '../common/serialization/clap/host.cpp',
     '../common/serialization/clap/plugin.cpp',
+    '../common/serialization/clap/process.cpp',
     '../common/serialization/clap/stream.cpp',
     '../common/utils.cpp',
     '../include/llvm/small-vector.cpp',

src/wine-host/meson.build

@@ -86,6 +86,7 @@ if with_clap
     '../common/serialization/clap/events.cpp',
     '../common/serialization/clap/host.cpp',
     '../common/serialization/clap/plugin.cpp',
+    '../common/serialization/clap/process.cpp',
     '../common/serialization/clap/stream.cpp',
     'bridges/clap-impls/host-proxy.cpp',
     'bridges/clap.cpp',