Choose a host application by parsing PE headers

Now yabridge can load both 64 and 32 bit plugins without having to
specify which are which.

src/plugin/host-bridge.cpp

@@ -49,7 +49,8 @@ constexpr char alphanumeric_characters[] =
 
 std::string create_logger_prefix(const fs::path& socket_path);
 fs::path find_vst_plugin();
-fs::path find_wine_vst_host();
+PluginArchitecture find_plugin_architecture(fs::path);
+fs::path find_wine_vst_host(PluginArchitecture plugin_arch);
 std::optional<fs::path> find_wineprefix();
 fs::path generate_endpoint_name();
 bp::environment set_wineprefix();
@@ -70,8 +71,9 @@ HostBridge& get_bridge_instance(const AEffect& plugin) {
 }
 
 HostBridge::HostBridge(audioMasterCallback host_callback)
-    : vst_host_path(find_wine_vst_host()),
-      vst_plugin_path(find_vst_plugin()),
+    : vst_plugin_path(find_vst_plugin()),
+      vst_plugin_arch(find_plugin_architecture(vst_plugin_path)),
+      vst_host_path(find_wine_vst_host(vst_plugin_arch)),
       // All the fields should be zero initialized because
       // `Vst2PluginInstance::vstAudioMasterCallback` from Bitwig's plugin
       // bridge will crash otherwise
@@ -491,8 +493,8 @@ std::string create_logger_prefix(const fs::path& socket_path) {
 }
 
 /**
- * Finds the Wine VST hsot (named `yabridge-host.exe`). For this we will search
- * in two places:
+ * Finds the Wine VST hsot (either `yabridge-host.exe` or `yabridge-host.exe`
+ * depending on the plugin). For this we will search in two places:
  *
  *   1. Alongside libyabridge.so if the file got symlinked. This is useful
  *      when developing, as you can simply symlink the the libyabridge.so
@@ -500,25 +502,31 @@ std::string create_logger_prefix(const fs::path& socket_path) {
  *      /usr.
  *   2. In the regular search path.
  *
+ * @param plugin_arch The architecture of the plugin, either 64-bit or 32-bit.
+ *   Used to determine which host application to use, if available.
+ *
  * @return The a path to the VST host, if found.
  * @throw std::runtime_error If the Wine VST host could not be found.
  */
-fs::path find_wine_vst_host() {
+fs::path find_wine_vst_host(PluginArchitecture plugin_arch) {
+    auto host_name = yabridge_wine_host_name;
+    if (plugin_arch == PluginArchitecture::vst_32) {
+        host_name = yabridge_wine_host_name_32bit;
+    }
+
     fs::path host_path =
         fs::canonical(boost::dll::this_line_location()).remove_filename() /
-        yabridge_wine_host_name;
+        host_name;
     if (fs::exists(host_path)) {
         return host_path;
     }
 
-    // TODO: First, check whether the plugin is 32-bit or 64-bit, and then
-    //       search for the correct binary accordingly
     // Bosot will return an empty path if the file could not be found in the
     // search path
-    const fs::path vst_host_path = bp::search_path(yabridge_wine_host_name);
+    const fs::path vst_host_path = bp::search_path(host_name);
     if (vst_host_path == "") {
-        throw std::runtime_error("Could not locate '" +
-                                 std::string(yabridge_wine_host_name) + "'");
+        throw std::runtime_error("Could not locate '" + std::string(host_name) +
+                                 "'");
     }
 
     return vst_host_path;
@@ -568,10 +576,66 @@ fs::path find_vst_plugin() {
             "VST plugin .dll file.");
     }
 
-    // Also resolve symlinks here, mostly for development purposes
+    // Also resolve symlinks here
     return fs::canonical(plugin_path);
 }
 
+/**
+ * Determine the architecture of a VST plugin (or rather, a .dll file) based on
+ * it's header values.
+ *
+ * See https://docs.microsoft.com/en-us/windows/win32/debug/pe-format for more
+ * information on the PE32 format.
+ *
+ * @param plugin_path The path to the .dll file we're going to check.
+ *
+ * @return The detected architecture.
+ * @throw std::runtime_error If the file is not a .dll file.
+ */
+PluginArchitecture find_plugin_architecture(fs::path plugin_path) {
+    std::ifstream file(plugin_path, std::ifstream::binary | std::ifstream::in);
+
+    // The linker will place the offset where the PE signature is placed at the
+    // end of the MS-DOS stub, at this offset
+    uint32_t pe_signature_offset;
+    file.seekg(0x3c);
+    file.read(reinterpret_cast<char*>(&pe_signature_offset),
+              sizeof(pe_signature_offset));
+
+    // The PE32 signature will be followed by a magic number.
+    // file >> pe_signature_offset;
+    uint32_t pe_signature;
+    uint16_t machine_type;
+    file.seekg(pe_signature_offset);
+    file.read(reinterpret_cast<char*>(&pe_signature), sizeof(pe_signature));
+    file.read(reinterpret_cast<char*>(&machine_type), sizeof(machine_type));
+
+    constexpr char expected_pe_signature[4] = {'P', 'E', '\0', '\0'};
+    if (pe_signature !=
+        *reinterpret_cast<const uint32_t*>(expected_pe_signature)) {
+        throw std::runtime_error("'" + plugin_path.string() +
+                                 "' is not a valid .dll file");
+    }
+
+    // These constants are specified in
+    // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
+    switch (machine_type) {
+        case 0x014c:  // IMAGE_FILE_MACHINE_I386
+            return PluginArchitecture::vst_32;
+            break;
+        case 0x8664:  // IMAGE_FILE_MACHINE_AMD64
+        case 0x0000:  // IMAGE_FILE_MACHINE_UNKNOWN
+            return PluginArchitecture::vst_64;
+            break;
+        default:
+            throw std::runtime_error(
+                "'" + plugin_path.string() +
+                "' does not have a supported architecture: " +
+                std::to_string(machine_type));
+            break;
+    }
+}
+
 /**
  * Generate a unique name for the Unix domain socket endpoint based on the VST
  * plugin's name. This will also generate the parent directory if it does not

src/plugin/host-bridge.h

@@ -45,6 +45,12 @@ class patched_async_pipe : public boost::process::async_pipe {
     typedef typename handle_type::executor_type executor_type;
 };
 
+/**
+ * A tag to differentiate between 32 and 64-bit plugins, used to determine which
+ * host application to use.
+ */
+enum class PluginArchitecture { vst_32, vst_64 };
+
 /**
  * This handles the communication between the Linux native VST plugin and the
  * Wine VST host. The functions below should be used as callback functions in an
@@ -90,14 +96,20 @@ class HostBridge {
     float get_parameter(AEffect* plugin, int index);
     void set_parameter(AEffect* plugin, int index, float value);
 
-    /**
-     * The path to `yabridge-host.exe`.
-     */
-    const boost::filesystem::path vst_host_path;
     /**
      * The path to the .dll being loaded in the Wine VST host.
      */
     const boost::filesystem::path vst_plugin_path;
+    /**
+     * Whether the plugin is 64-bit or 32-bit.
+     */
+    const PluginArchitecture vst_plugin_arch;
+    /**
+     * The path to the host application (i.e. a path to either
+     * `yabridge-host.exe` or `yabridge-host-32.exe`). The host application will
+     * be chosen depending on the architecture of the VST plugin .dll file.
+     */
+    const boost::filesystem::path vst_host_path;
 
     /**
      * This AEffect struct will be populated using the data passed by the Wine