From b2a15620f32e2e03d01cafe84623192769c74463 Mon Sep 17 00:00:00 2001 From: Robbert van der Helm Date: Thu, 14 Apr 2022 23:31:14 +0200 Subject: [PATCH] Swap Boost.Container's small_vector out for LLVM's This implementation misses a shrink to fit function, but reassigning the vector with a fresh one should be equivalent. --- src/common/bitsery/traits/small-vector.h | 42 +- src/common/communication/common.h | 32 +- src/common/communication/vst2.h | 16 +- src/common/logging/vst3.h | 2 - src/common/serialization/vst2.h | 13 +- src/common/serialization/vst3/event-list.h | 9 +- .../serialization/vst3/param-value-queue.h | 12 +- .../serialization/vst3/parameter-changes.h | 4 +- src/common/serialization/vst3/process-data.h | 8 +- src/include/llvm/small-vector.cpp | 163 ++ src/include/llvm/small-vector.h | 1365 +++++++++++++++++ src/plugin/bridges/vst2.h | 2 +- src/plugin/meson.build | 2 + src/wine-host/bridges/vst2.h | 2 +- src/wine-host/editor.cpp | 9 +- src/wine-host/meson.build | 1 + src/wine-host/xdnd-proxy.cpp | 8 +- src/wine-host/xdnd-proxy.h | 8 +- 18 files changed, 1601 insertions(+), 97 deletions(-) create mode 100644 src/include/llvm/small-vector.cpp create mode 100644 src/include/llvm/small-vector.h diff --git a/src/common/bitsery/traits/small-vector.h b/src/common/bitsery/traits/small-vector.h index 7c151cee..54a346e2 100644 --- a/src/common/bitsery/traits/small-vector.h +++ b/src/common/bitsery/traits/small-vector.h @@ -17,44 +17,30 @@ #pragma once #include -#include -#include +#include namespace bitsery { namespace traits { -template -struct ContainerTraits> - : public StdContainer, - true, - true> { - // Unlike `std::vector`, I'm pretty sure - // `boost::container::small_vector` is contiguous. So hopefully - // this assertion does its thing. - static_assert(boost::container::dtl::is_contiguous_container< - boost::container::small_vector>::value); +template +struct ContainerTraits> + : public StdContainer, true, true> { + // The small vector implementation needs to be contiguous for this to work }; -template -struct BufferAdapterTraits> - : public StdContainerForBufferAdapter< - boost::container::small_vector> {}; +template +struct BufferAdapterTraits> + : public StdContainerForBufferAdapter> {}; // And the same extensions again for the type erased version -template -struct ContainerTraits> - : public StdContainer, - true, - true> { - static_assert(boost::container::dtl::is_contiguous_container< - boost::container::small_vector_base>::value); -}; +template +struct ContainerTraits> + : public StdContainer, true, true> {}; -template -struct BufferAdapterTraits> - : public StdContainerForBufferAdapter< - boost::container::small_vector_base> {}; +template +struct BufferAdapterTraits> + : public StdContainerForBufferAdapter> {}; } // namespace traits } // namespace bitsery diff --git a/src/common/communication/common.h b/src/common/communication/common.h index bd528ce5..c5963acf 100644 --- a/src/common/communication/common.h +++ b/src/common/communication/common.h @@ -26,11 +26,11 @@ #ifdef __WINE__ #include "../wine-host/asio-fix.h" #endif +#include #include #include #include #include -#include #include #include "../bitsery/traits/small-vector.h" @@ -71,37 +71,35 @@ using InputAdapter = * this way. */ template -using SerializationBuffer = boost::container::small_vector; +using SerializationBuffer = llvm::SmallVector; /** * The class `SerializationBuffer` is derived from, so we can erase the * buffer's initial capacity from all functions that work with them. */ -using SerializationBufferBase = boost::container::small_vector_base; +using SerializationBufferBase = llvm::SmallVectorImpl; namespace asio { -template -inline ASIO_MUTABLE_BUFFER buffer( - boost::container::small_vector_base& data) +// These are copied verbatim `asio::buffer(std::vector&, +// std::size_t)`, since `llvm::SmallVector` is mostly compatible with the STL +// vector. +template +inline ASIO_MUTABLE_BUFFER buffer(llvm::SmallVectorImpl& data) ASIO_NOEXCEPT { return ASIO_MUTABLE_BUFFER( data.size() ? &data[0] : 0, data.size() * sizeof(PodType) #if defined(ASIO_ENABLE_BUFFER_DEBUGGING) , - detail::buffer_debug_check::iterator>(data.begin()) + detail::buffer_debug_check< + typename llvm::SmallVectorImpl::iterator>(data.begin()) #endif // ASIO_ENABLE_BUFFER_DEBUGGING ); } -// These are copied verbatim `asio::buffer(std::vector&, std::size_t)`, since `boost::container::small_vector` is -// compatible with the STL vector. -template -inline ASIO_MUTABLE_BUFFER buffer( - boost::container::small_vector_base& data, - std::size_t max_size_in_bytes) ASIO_NOEXCEPT { +template +inline ASIO_MUTABLE_BUFFER buffer(llvm::SmallVectorImpl& data, + std::size_t max_size_in_bytes) ASIO_NOEXCEPT { return ASIO_MUTABLE_BUFFER( data.size() ? &data[0] : 0, data.size() * sizeof(PodType) < max_size_in_bytes @@ -109,8 +107,8 @@ inline ASIO_MUTABLE_BUFFER buffer( : max_size_in_bytes #if defined(ASIO_ENABLE_BUFFER_DEBUGGING) , - detail::buffer_debug_check::iterator>(data.begin()) + detail::buffer_debug_check< + typename llvm::SmallVectorImpl::iterator>(data.begin()) #endif // ASIO_ENABLE_BUFFER_DEBUGGING ); } diff --git a/src/common/communication/vst2.h b/src/common/communication/vst2.h index c9fea0cd..12562915 100644 --- a/src/common/communication/vst2.h +++ b/src/common/communication/vst2.h @@ -205,8 +205,8 @@ class Vst2EventHandler : public AdHocSocketHandler { // from the socket, so we can override this for specific function calls // that potentially need to have their responses handled on the same // calling thread (i.e. mutual recursion). - const Vst2EventResult response = this->send( - [&](asio::local::stream_protocol::socket& socket) { + const Vst2EventResult response = + this->send([&](asio::local::stream_protocol::socket& socket) { return data_converter.send_event(socket, event, serialization_buffer()); }); @@ -294,7 +294,7 @@ class Vst2EventHandler : public AdHocSocketHandler { * thread at all cost we'll just predefine a large buffer for every thread. */ SerializationBufferBase& serialization_buffer() { - // This object also contains a `boost::container::small_vector` that has + // This object also contains a `llvm::SmallVector` that has // capacity for a large-ish number of events so we don't have to // allocate under normal circumstances. constexpr size_t initial_events_size = sizeof(DynamicVstEvents); @@ -304,9 +304,13 @@ class Vst2EventHandler : public AdHocSocketHandler { // when sending and receiving preset data. In such cases we do want to // reallocate the buffer on the next event to free up memory again. This // won't happen during audio processing. - if (buffer.size() > initial_events_size) { - buffer.resize(initial_events_size); - buffer.shrink_to_fit(); + if (buffer.size() > initial_events_size * 2) { + // NOTE: There's no `.shrink_to_fit()` implementation here, so we'll + // just YOLO it and reinitialize the vector since we don't + // need the old data + buffer = SerializationBuffer{}; + // buffer.resize(initial_events_size); + // buffer.shrink_to_fit(); } return buffer; diff --git a/src/common/logging/vst3.h b/src/common/logging/vst3.h index b4af31b6..15d9f738 100644 --- a/src/common/logging/vst3.h +++ b/src/common/logging/vst3.h @@ -18,8 +18,6 @@ #include -#include - #include "../serialization/vst3.h" #include "common.h" diff --git a/src/common/serialization/vst2.h b/src/common/serialization/vst2.h index 229889cf..ddab41e1 100644 --- a/src/common/serialization/vst2.h +++ b/src/common/serialization/vst2.h @@ -20,8 +20,8 @@ #include #include +#include #include -#include #include "../audio-shm.h" #include "../bitsery/ext/in-place-optional.h" @@ -123,18 +123,17 @@ class alignas(16) DynamicVstEvents { * the `sysex_data_` field before dumping everything to * `vst_events_buffer_`. */ - boost::container::small_vector events_; + llvm::SmallVector events_; /** * If the host or a plugin sends SysEx data, then we will store that data * here. I've only seen this happen with the combination of an Arturia * MiniLab keyboard, REAPER, and D16 Group plugins. We'll store this as an * associative list of `(index, data)` pairs, where `index` corresponds to - * an event in `events`. There's no 'small_unordered_map' in - * Boost.Container, so this will have to do. + * an event in `events`. There's no 'SmallUnorderedMap' equivalent to the + * `SmallVector`. */ - boost::container::small_vector, 8> - sysex_data_; + llvm::SmallVector, 8> sysex_data_; template void serialize(S& s) { @@ -161,7 +160,7 @@ class alignas(16) DynamicVstEvents { * MIDI events the host can send at once we have to build this object on the * heap by hand. */ - boost::container::small_vector< + llvm::SmallVector< uint8_t, sizeof(VstEvents) + ((64 - 1) * diff --git a/src/common/serialization/vst3/event-list.h b/src/common/serialization/vst3/event-list.h index 1b344e39..57ba9c44 100644 --- a/src/common/serialization/vst3/event-list.h +++ b/src/common/serialization/vst3/event-list.h @@ -16,18 +16,15 @@ #pragma once +#include #include -#include #include "../../bitsery/ext/in-place-variant.h" +#include "../../bitsery/traits/small-vector.h" #include "base.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -// FIXME: When used in a Boost.Containers small vector, GCC somehow complains -// that the fields in `YaEvent` may be uninitialized (during the -// deserialization). This warning only shows up during a unity build. -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" /** * A wrapper around `DataEvent` for serialization purposes, as this event @@ -270,7 +267,7 @@ class YaEventList : public Steinberg::Vst::IEventList { } private: - boost::container::small_vector events_; + llvm::SmallVector events_; }; #pragma GCC diagnostic pop diff --git a/src/common/serialization/vst3/param-value-queue.h b/src/common/serialization/vst3/param-value-queue.h index db1e8b5f..05e4be2f 100644 --- a/src/common/serialization/vst3/param-value-queue.h +++ b/src/common/serialization/vst3/param-value-queue.h @@ -16,20 +16,14 @@ #pragma once +#include #include -#include #include "../../bitsery/traits/small-vector.h" #include "base.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -// FIXME: When used in a Boost.Containers small vector, GCC somehow complains -// that the fields in this class may be uninitialized (during the -// deserialization). This warning only shows up during a unity build. -#if defined(__GNUC__) && !defined(__llvm__) -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif /** * Wraps around `IParamValueQueue` for serializing a queue containing changes to @@ -102,9 +96,7 @@ class alignas(16) YaParamValueQueue : public Steinberg::Vst::IParamValueQueue { * * This contains pairs of `(sample_offset, value)`. */ - boost::container::small_vector, - 16> - queue_; + llvm::SmallVector, 16> queue_; }; #pragma GCC diagnostic pop diff --git a/src/common/serialization/vst3/parameter-changes.h b/src/common/serialization/vst3/parameter-changes.h index 96bcfb55..97f16f69 100644 --- a/src/common/serialization/vst3/parameter-changes.h +++ b/src/common/serialization/vst3/parameter-changes.h @@ -16,8 +16,8 @@ #pragma once +#include #include -#include #include "../../bitsery/traits/small-vector.h" #include "base.h" @@ -85,7 +85,7 @@ class YaParameterChanges : public Steinberg::Vst::IParameterChanges { /** * The parameter value changes queues. */ - boost::container::small_vector queues_; + llvm::SmallVector queues_; }; #pragma GCC diagnostic pop diff --git a/src/common/serialization/vst3/process-data.h b/src/common/serialization/vst3/process-data.h index b19abc66..1980e9f0 100644 --- a/src/common/serialization/vst3/process-data.h +++ b/src/common/serialization/vst3/process-data.h @@ -112,8 +112,8 @@ class YaProcessData { struct Response { // We store raw pointers instead of references so we can default // initialize this object during deserialization - boost::container::small_vector_base* - outputs = nullptr; + llvm::SmallVectorImpl* outputs = + nullptr; std::optional* output_parameter_changes = nullptr; std::optional* output_events = nullptr; @@ -206,7 +206,7 @@ class YaProcessData { * be set to point to our shared memory surface that holds the actual audio * data. */ - boost::container::small_vector inputs_; + llvm::SmallVector inputs_; /** * This contains metadata about the output buffers for every bus. During @@ -214,7 +214,7 @@ class YaProcessData { * be set to point to our shared memory surface that holds the actual audio * data. */ - boost::container::small_vector outputs_; + llvm::SmallVector outputs_; /** * Incoming parameter changes. diff --git a/src/include/llvm/small-vector.cpp b/src/include/llvm/small-vector.cpp new file mode 100644 index 00000000..02eb7a77 --- /dev/null +++ b/src/include/llvm/small-vector.cpp @@ -0,0 +1,163 @@ +//===- llvm/ADT/SmallVector.cpp - 'Normally small' vectors ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the SmallVector class. +// +//===----------------------------------------------------------------------===// + +// Taken from +// https://github.com/llvm/llvm-project/blob/f14334ffa1191ad734d2a994609cf8220c5b7abf/llvm/lib/Support/SmallVector.cpp + +#include "small-vector.h" + +#define LLVM_ENABLE_EXCEPTIONS + +#include +#ifdef LLVM_ENABLE_EXCEPTIONS +#include +#endif +using namespace llvm; + +// Check that no bytes are wasted and everything is well-aligned. +namespace { +// These structures may cause binary compat warnings on AIX. Suppress the +// warning since we are only using these types for the static assertions below. +#if defined(_AIX) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Waix-compat" +#endif +struct Struct16B { + alignas(16) void* X; +}; +struct Struct32B { + alignas(32) void* X; +}; +#if defined(_AIX) +#pragma GCC diagnostic pop +#endif +} // namespace +static_assert(sizeof(SmallVector) == + sizeof(unsigned) * 2 + sizeof(void*), + "wasted space in SmallVector size 0"); +static_assert(alignof(SmallVector) >= alignof(Struct16B), + "wrong alignment for 16-byte aligned T"); +static_assert(alignof(SmallVector) >= alignof(Struct32B), + "wrong alignment for 32-byte aligned T"); +static_assert(sizeof(SmallVector) >= alignof(Struct16B), + "missing padding for 16-byte aligned T"); +static_assert(sizeof(SmallVector) >= alignof(Struct32B), + "missing padding for 32-byte aligned T"); +static_assert(sizeof(SmallVector) == + sizeof(unsigned) * 2 + sizeof(void*) * 2, + "wasted space in SmallVector size 1"); + +static_assert(sizeof(SmallVector) == sizeof(void*) * 2 + sizeof(void*), + "1 byte elements have word-sized type for size and capacity"); + +/// Report that MinSize doesn't fit into this vector's size type. Throws +/// std::length_error or calls report_fatal_error. +[[noreturn]] static void report_size_overflow(size_t MinSize, size_t MaxSize); +static void report_size_overflow(size_t MinSize, size_t MaxSize) { + std::string Reason = "SmallVector unable to grow. Requested capacity (" + + std::to_string(MinSize) + + ") is larger than maximum value for size type (" + + std::to_string(MaxSize) + ")"; +#ifdef LLVM_ENABLE_EXCEPTIONS + throw std::length_error(Reason); +#else + report_fatal_error(Twine(Reason)); +#endif +} + +/// Report that this vector is already at maximum capacity. Throws +/// std::length_error or calls report_fatal_error. +[[noreturn]] static void report_at_maximum_capacity(size_t MaxSize); +static void report_at_maximum_capacity(size_t MaxSize) { + std::string Reason = + "SmallVector capacity unable to grow. Already at maximum size " + + std::to_string(MaxSize); +#ifdef LLVM_ENABLE_EXCEPTIONS + throw std::length_error(Reason); +#else + report_fatal_error(Twine(Reason)); +#endif +} + +// Note: Moving this function into the header may cause performance regression. +template +static size_t getNewCapacity(size_t MinSize, + size_t /*TSize*/, + size_t OldCapacity) { + constexpr size_t MaxSize = std::numeric_limits::max(); + + // Ensure we can fit the new capacity. + // This is only going to be applicable when the capacity is 32 bit. + if (MinSize > MaxSize) + report_size_overflow(MinSize, MaxSize); + + // Ensure we can meet the guarantee of space for at least one more element. + // The above check alone will not catch the case where grow is called with a + // default MinSize of 0, but the current capacity cannot be increased. + // This is only going to be applicable when the capacity is 32 bit. + if (OldCapacity == MaxSize) + report_at_maximum_capacity(MaxSize); + + // In theory 2*capacity can overflow if the capacity is 64 bit, but the + // original capacity would never be large enough for this to be a problem. + size_t NewCapacity = 2 * OldCapacity + 1; // Always grow. + return std::min(std::max(NewCapacity, MinSize), MaxSize); +} + +// Note: Moving this function into the header may cause performance regression. +template +void* SmallVectorBase::mallocForGrow(size_t MinSize, + size_t TSize, + size_t& NewCapacity) { + NewCapacity = getNewCapacity(MinSize, TSize, this->capacity()); + // return llvm::safe_malloc(NewCapacity * TSize); + return malloc(NewCapacity * TSize); +} + +// Note: Moving this function into the header may cause performance regression. +template +void SmallVectorBase::grow_pod(void* FirstEl, + size_t MinSize, + size_t TSize) { + size_t NewCapacity = + getNewCapacity(MinSize, TSize, this->capacity()); + void* NewElts; + if (BeginX == FirstEl) { + NewElts = malloc(NewCapacity * TSize); + + // Copy the elements over. No need to run dtors on PODs. + memcpy(NewElts, this->BeginX, size() * TSize); + } else { + // If this wasn't grown from the inline copy, grow the allocated space. + NewElts = realloc(this->BeginX, NewCapacity * TSize); + } + + this->BeginX = NewElts; + this->Capacity = NewCapacity; +} + +template class llvm::SmallVectorBase; + +// Disable the uint64_t instantiation for 32-bit builds. +// Both uint32_t and uint64_t instantiations are needed for 64-bit builds. +// This instantiation will never be used in 32-bit builds, and will cause +// warnings when sizeof(Size_T) > sizeof(size_t). +#if SIZE_MAX > UINT32_MAX +template class llvm::SmallVectorBase; + +// Assertions to ensure this #if stays in sync with SmallVectorSizeType. +static_assert(sizeof(SmallVectorSizeType) == sizeof(uint64_t), + "Expected SmallVectorBase variant to be in use."); +#else +static_assert(sizeof(SmallVectorSizeType) == sizeof(uint32_t), + "Expected SmallVectorBase variant to be in use."); +#endif diff --git a/src/include/llvm/small-vector.h b/src/include/llvm/small-vector.h new file mode 100644 index 00000000..3c419b56 --- /dev/null +++ b/src/include/llvm/small-vector.h @@ -0,0 +1,1365 @@ +//===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines the SmallVector class. +/// +//===----------------------------------------------------------------------===// + +// Taken from +// https://github.com/llvm/llvm-project/blob/c4f059e5094edd216761fd8345ceb6345836d91d/llvm/include/llvm/ADT/SmallVector.h + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// The original version depends on some additional LLVM macros, we'll define +// those inline + +// llvm/Support/Compiler.h +#define LLVM_NODISCARD [[nodiscard]] + +// llvm/Support/Compiler.h +#if __has_cpp_attribute(gsl::Owner) +#define LLVM_GSL_OWNER [[gsl::Owner]] +#else +#define LLVM_GSL_OWNER +#endif + +// llvm/Support/Compiler.h +#define LLVM_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true) +#define LLVM_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false) + +namespace llvm { + +template +class iterator_range; + +/// This is all the stuff common to all SmallVectors. +/// +/// The template parameter specifies the type which should be used to hold the +/// Size and Capacity of the SmallVector, so it can be adjusted. +/// Using 32 bit size is desirable to shrink the size of the SmallVector. +/// Using 64 bit size is desirable for cases like SmallVector, where a +/// 32 bit size would limit the vector to ~4GB. SmallVectors are used for +/// buffering bitcode output - which can exceed 4GB. +template +class SmallVectorBase { + protected: + void* BeginX; + Size_T Size = 0, Capacity; + + /// The maximum value of the Size_T used. + static constexpr size_t SizeTypeMax() { + return std::numeric_limits::max(); + } + + SmallVectorBase() = delete; + SmallVectorBase(void* FirstEl, size_t TotalCapacity) + : BeginX(FirstEl), Capacity(TotalCapacity) {} + + /// This is a helper for \a grow() that's out of line to reduce code + /// duplication. This function will report a fatal error if it can't grow + /// at least to \p MinSize. + void* mallocForGrow(size_t MinSize, size_t TSize, size_t& NewCapacity); + + /// This is an implementation of the grow() method which only works + /// on POD-like data types and is out of line to reduce code duplication. + /// This function will report a fatal error if it cannot increase capacity. + void grow_pod(void* FirstEl, size_t MinSize, size_t TSize); + + public: + size_t size() const { return Size; } + size_t capacity() const { return Capacity; } + + LLVM_NODISCARD bool empty() const { return !Size; } + + protected: + /// Set the array size to \p N, which the current array must have enough + /// capacity for. + /// + /// This does not construct or destroy any elements in the vector. + void set_size(size_t N) { + assert(N <= capacity()); + Size = N; + } +}; + +template +using SmallVectorSizeType = typename std:: + conditional= 8, uint64_t, uint32_t>::type; + +/// Figure out the offset of the first element. +template +struct SmallVectorAlignmentAndSize { + alignas(SmallVectorBase>) char Base[sizeof( + SmallVectorBase>)]; + alignas(T) char FirstEl[sizeof(T)]; +}; + +/// This is the part of SmallVectorTemplateBase which does not depend on whether +/// the type T is a POD. The extra dummy template argument is used by ArrayRef +/// to avoid unnecessarily requiring T to be complete. +template +class SmallVectorTemplateCommon + : public SmallVectorBase> { + using Base = SmallVectorBase>; + + /// Find the address of the first element. For this pointer math to be + /// valid with small-size of 0 for T with lots of alignment, it's important + /// that SmallVectorStorage is properly-aligned even for small-size of 0. + void* getFirstEl() const { + return const_cast(reinterpret_cast( + reinterpret_cast(this) + + offsetof(SmallVectorAlignmentAndSize, FirstEl))); + } + // Space after 'FirstEl' is clobbered, do not add any instance vars after + // it. + + protected: + SmallVectorTemplateCommon(size_t Size) : Base(getFirstEl(), Size) {} + + void grow_pod(size_t MinSize, size_t TSize) { + Base::grow_pod(getFirstEl(), MinSize, TSize); + } + + /// Return true if this is a smallvector which has not had dynamic + /// memory allocated for it. + bool isSmall() const { return this->BeginX == getFirstEl(); } + + /// Put this vector in a state of being small. + void resetToSmall() { + this->BeginX = getFirstEl(); + this->Size = this->Capacity = + 0; // FIXME: Setting Capacity to 0 is suspect. + } + + /// Return true if V is an internal reference to the given range. + bool isReferenceToRange(const void* V, + const void* First, + const void* Last) const { + // Use std::less to avoid UB. + std::less<> LessThan; + return !LessThan(V, First) && LessThan(V, Last); + } + + /// Return true if V is an internal reference to this vector. + bool isReferenceToStorage(const void* V) const { + return isReferenceToRange(V, this->begin(), this->end()); + } + + /// Return true if First and Last form a valid (possibly empty) range in + /// this vector's storage. + bool isRangeInStorage(const void* First, const void* Last) const { + // Use std::less to avoid UB. + std::less<> LessThan; + return !LessThan(First, this->begin()) && !LessThan(Last, First) && + !LessThan(this->end(), Last); + } + + /// Return true unless Elt will be invalidated by resizing the vector to + /// NewSize. + bool isSafeToReferenceAfterResize(const void* Elt, size_t NewSize) { + // Past the end. + if (LLVM_LIKELY(!isReferenceToStorage(Elt))) + return true; + + // Return false if Elt will be destroyed by shrinking. + if (NewSize <= this->size()) + return Elt < this->begin() + NewSize; + + // Return false if we need to grow. + return NewSize <= this->capacity(); + } + + /// Check whether Elt will be invalidated by resizing the vector to NewSize. + void assertSafeToReferenceAfterResize(const void* Elt, size_t NewSize) { + assert( + isSafeToReferenceAfterResize(Elt, NewSize) && + "Attempting to reference an element of the vector in an operation " + "that invalidates it"); + } + + /// Check whether Elt will be invalidated by increasing the size of the + /// vector by N. + void assertSafeToAdd(const void* Elt, size_t N = 1) { + this->assertSafeToReferenceAfterResize(Elt, this->size() + N); + } + + /// Check whether any part of the range will be invalidated by clearing. + void assertSafeToReferenceAfterClear(const T* From, const T* To) { + if (From == To) + return; + this->assertSafeToReferenceAfterResize(From, 0); + this->assertSafeToReferenceAfterResize(To - 1, 0); + } + template < + class ItTy, + std::enable_if_t, T*>::value, + bool> = false> + void assertSafeToReferenceAfterClear(ItTy, ItTy) {} + + /// Check whether any part of the range will be invalidated by growing. + void assertSafeToAddRange(const T* From, const T* To) { + if (From == To) + return; + this->assertSafeToAdd(From, To - From); + this->assertSafeToAdd(To - 1, To - From); + } + template < + class ItTy, + std::enable_if_t, T*>::value, + bool> = false> + void assertSafeToAddRange(ItTy, ItTy) {} + + /// Reserve enough space to add one element, and return the updated element + /// pointer in case it was a reference to the storage. + template + static const T* reserveForParamAndGetAddressImpl(U* This, + const T& Elt, + size_t N) { + size_t NewSize = This->size() + N; + if (LLVM_LIKELY(NewSize <= This->capacity())) + return &Elt; + + bool ReferencesStorage = false; + int64_t Index = -1; + if (!U::TakesParamByValue) { + if (LLVM_UNLIKELY(This->isReferenceToStorage(&Elt))) { + ReferencesStorage = true; + Index = &Elt - This->begin(); + } + } + This->grow(NewSize); + return ReferencesStorage ? This->begin() + Index : &Elt; + } + + public: + using size_type = size_t; + using difference_type = ptrdiff_t; + using value_type = T; + using iterator = T*; + using const_iterator = const T*; + + using const_reverse_iterator = std::reverse_iterator; + using reverse_iterator = std::reverse_iterator; + + using reference = T&; + using const_reference = const T&; + using pointer = T*; + using const_pointer = const T*; + + using Base::capacity; + using Base::empty; + using Base::size; + + // forward iterator creation methods. + iterator begin() { return (iterator)this->BeginX; } + const_iterator begin() const { return (const_iterator)this->BeginX; } + iterator end() { return begin() + size(); } + const_iterator end() const { return begin() + size(); } + + // reverse iterator creation methods. + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + size_type size_in_bytes() const { return size() * sizeof(T); } + size_type max_size() const { + return std::min(this->SizeTypeMax(), size_type(-1) / sizeof(T)); + } + + size_t capacity_in_bytes() const { return capacity() * sizeof(T); } + + /// Return a pointer to the vector's buffer, even if empty(). + pointer data() { return pointer(begin()); } + /// Return a pointer to the vector's buffer, even if empty(). + const_pointer data() const { return const_pointer(begin()); } + + reference operator[](size_type idx) { + assert(idx < size()); + return begin()[idx]; + } + const_reference operator[](size_type idx) const { + assert(idx < size()); + return begin()[idx]; + } + + reference front() { + assert(!empty()); + return begin()[0]; + } + const_reference front() const { + assert(!empty()); + return begin()[0]; + } + + reference back() { + assert(!empty()); + return end()[-1]; + } + const_reference back() const { + assert(!empty()); + return end()[-1]; + } +}; + +/// SmallVectorTemplateBase - This is where we put +/// method implementations that are designed to work with non-trivial T's. +/// +/// We approximate is_trivially_copyable with trivial move/copy construction and +/// trivial destruction. While the standard doesn't specify that you're allowed +/// copy these types with memcpy, there is no way for the type to observe this. +/// This catches the important case of std::pair, which is not +/// trivially assignable. +template ::value) && + (std::is_trivially_move_constructible::value) && + std::is_trivially_destructible::value> +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { + friend class SmallVectorTemplateCommon; + + protected: + static constexpr bool TakesParamByValue = false; + using ValueParamT = const T&; + + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + static void destroy_range(T* S, T* E) { + while (S != E) { + --E; + E->~T(); + } + } + + /// Move the range [I, E) into the uninitialized memory starting with + /// "Dest", constructing elements as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(std::make_move_iterator(I), + std::make_move_iterator(E), Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory starting with + /// "Dest", constructing elements as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(I, E, Dest); + } + + /// Grow the allocated memory (without initializing new elements), doubling + /// the size of the allocated memory. Guarantees space for at least one more + /// element, or MinSize more elements if specified. + void grow(size_t MinSize = 0); + + /// Create a new allocation big enough for \p MinSize and pass back its size + /// in \p NewCapacity. This is the first section of \a grow(). + T* mallocForGrow(size_t MinSize, size_t& NewCapacity) { + return static_cast( + SmallVectorBase>::mallocForGrow( + MinSize, sizeof(T), NewCapacity)); + } + + /// Move existing elements over to the new allocation \p NewElts, the middle + /// section of \a grow(). + void moveElementsForGrow(T* NewElts); + + /// Transfer ownership of the allocation, finishing up \a grow(). + void takeAllocationForGrow(T* NewElts, size_t NewCapacity); + + /// Reserve enough space to add one element, and return the updated element + /// pointer in case it was a reference to the storage. + const T* reserveForParamAndGetAddress(const T& Elt, size_t N = 1) { + return this->reserveForParamAndGetAddressImpl(this, Elt, N); + } + + /// Reserve enough space to add one element, and return the updated element + /// pointer in case it was a reference to the storage. + T* reserveForParamAndGetAddress(T& Elt, size_t N = 1) { + return const_cast( + this->reserveForParamAndGetAddressImpl(this, Elt, N)); + } + + static T&& forward_value_param(T&& V) { return std::move(V); } + static const T& forward_value_param(const T& V) { return V; } + + void growAndAssign(size_t NumElts, const T& Elt) { + // Grow manually in case Elt is an internal reference. + size_t NewCapacity; + T* NewElts = mallocForGrow(NumElts, NewCapacity); + std::uninitialized_fill_n(NewElts, NumElts, Elt); + this->destroy_range(this->begin(), this->end()); + takeAllocationForGrow(NewElts, NewCapacity); + this->set_size(NumElts); + } + + template + T& growAndEmplaceBack(ArgTypes&&... Args) { + // Grow manually in case one of Args is an internal reference. + size_t NewCapacity; + T* NewElts = mallocForGrow(0, NewCapacity); + ::new ((void*)(NewElts + this->size())) + T(std::forward(Args)...); + moveElementsForGrow(NewElts); + takeAllocationForGrow(NewElts, NewCapacity); + this->set_size(this->size() + 1); + return this->back(); + } + + public: + void push_back(const T& Elt) { + const T* EltPtr = reserveForParamAndGetAddress(Elt); + ::new ((void*)this->end()) T(*EltPtr); + this->set_size(this->size() + 1); + } + + void push_back(T&& Elt) { + T* EltPtr = reserveForParamAndGetAddress(Elt); + ::new ((void*)this->end()) T(::std::move(*EltPtr)); + this->set_size(this->size() + 1); + } + + void pop_back() { + this->set_size(this->size() - 1); + this->end()->~T(); + } +}; + +// Define this out-of-line to dissuade the C++ compiler from inlining it. +template +void SmallVectorTemplateBase::grow(size_t MinSize) { + size_t NewCapacity; + T* NewElts = mallocForGrow(MinSize, NewCapacity); + moveElementsForGrow(NewElts); + takeAllocationForGrow(NewElts, NewCapacity); +} + +// Define this out-of-line to dissuade the C++ compiler from inlining it. +template +void SmallVectorTemplateBase::moveElementsForGrow( + T* NewElts) { + // Move the elements over. + this->uninitialized_move(this->begin(), this->end(), NewElts); + + // Destroy the original elements. + destroy_range(this->begin(), this->end()); +} + +// Define this out-of-line to dissuade the C++ compiler from inlining it. +template +void SmallVectorTemplateBase::takeAllocationForGrow( + T* NewElts, + size_t NewCapacity) { + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + free(this->begin()); + + this->BeginX = NewElts; + this->Capacity = NewCapacity; +} + +/// SmallVectorTemplateBase - This is where we put +/// method implementations that are designed to work with trivially copyable +/// T's. This allows using memcpy in place of copy/move construction and +/// skipping destruction. +template +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { + friend class SmallVectorTemplateCommon; + + protected: + /// True if it's cheap enough to take parameters by value. Doing so avoids + /// overhead related to mitigations for reference invalidation. + static constexpr bool TakesParamByValue = sizeof(T) <= 2 * sizeof(void*); + + /// Either const T& or T, depending on whether it's cheap enough to take + /// parameters by value. + using ValueParamT = + typename std::conditional::type; + + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + // No need to do a destroy loop for POD's. + static void destroy_range(T*, T*) {} + + /// Move the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + // Just do a copy. + uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + // Arbitrary iterator types; just use the basic implementation. + std::uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy( + T1* I, + T1* E, + T2* Dest, + std::enable_if_t::type, + T2>::value>* = nullptr) { + // Use memcpy for PODs iterated by pointers (which includes SmallVector + // iterators): std::uninitialized_copy optimizes to memmove, but we can + // use memcpy here. Note that I and E are iterators and thus might be + // invalid for memcpy if they are equal. + if (I != E) + memcpy(reinterpret_cast(Dest), I, (E - I) * sizeof(T)); + } + + /// Double the size of the allocated memory, guaranteeing space for at + /// least one more element or MinSize if specified. + void grow(size_t MinSize = 0) { this->grow_pod(MinSize, sizeof(T)); } + + /// Reserve enough space to add one element, and return the updated element + /// pointer in case it was a reference to the storage. + const T* reserveForParamAndGetAddress(const T& Elt, size_t N = 1) { + return this->reserveForParamAndGetAddressImpl(this, Elt, N); + } + + /// Reserve enough space to add one element, and return the updated element + /// pointer in case it was a reference to the storage. + T* reserveForParamAndGetAddress(T& Elt, size_t N = 1) { + return const_cast( + this->reserveForParamAndGetAddressImpl(this, Elt, N)); + } + + /// Copy \p V or return a reference, depending on \a ValueParamT. + static ValueParamT forward_value_param(ValueParamT V) { return V; } + + void growAndAssign(size_t NumElts, T Elt) { + // Elt has been copied in case it's an internal reference, side-stepping + // reference invalidation problems without losing the realloc + // optimization. + this->set_size(0); + this->grow(NumElts); + std::uninitialized_fill_n(this->begin(), NumElts, Elt); + this->set_size(NumElts); + } + + template + T& growAndEmplaceBack(ArgTypes&&... Args) { + // Use push_back with a copy in case Args has an internal reference, + // side-stepping reference invalidation problems without losing the + // realloc optimization. + push_back(T(std::forward(Args)...)); + return this->back(); + } + + public: + void push_back(ValueParamT Elt) { + const T* EltPtr = reserveForParamAndGetAddress(Elt); + memcpy(reinterpret_cast(this->end()), EltPtr, sizeof(T)); + this->set_size(this->size() + 1); + } + + void pop_back() { this->set_size(this->size() - 1); } +}; + +/// This class consists of common code factored out of the SmallVector class to +/// reduce code duplication based on the SmallVector 'N' template parameter. +template +class SmallVectorImpl : public SmallVectorTemplateBase { + using SuperClass = SmallVectorTemplateBase; + + public: + using iterator = typename SuperClass::iterator; + using const_iterator = typename SuperClass::const_iterator; + using reference = typename SuperClass::reference; + using size_type = typename SuperClass::size_type; + + protected: + using SmallVectorTemplateBase::TakesParamByValue; + using ValueParamT = typename SuperClass::ValueParamT; + + // Default ctor - Initialize to empty. + explicit SmallVectorImpl(unsigned N) : SmallVectorTemplateBase(N) {} + + void assignRemote(SmallVectorImpl&& RHS) { + this->destroy_range(this->begin(), this->end()); + if (!this->isSmall()) + free(this->begin()); + this->BeginX = RHS.BeginX; + this->Size = RHS.Size; + this->Capacity = RHS.Capacity; + RHS.resetToSmall(); + } + + public: + SmallVectorImpl(const SmallVectorImpl&) = delete; + + ~SmallVectorImpl() { + // Subclass has already destructed this vector's elements. + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + free(this->begin()); + } + + void clear() { + this->destroy_range(this->begin(), this->end()); + this->Size = 0; + } + + private: + // Make set_size() private to avoid misuse in subclasses. + using SuperClass::set_size; + + template + void resizeImpl(size_type N) { + if (N == this->size()) + return; + + if (N < this->size()) { + this->truncate(N); + return; + } + + this->reserve(N); + for (auto I = this->end(), E = this->begin() + N; I != E; ++I) + if (ForOverwrite) + new (&*I) T; + else + new (&*I) T(); + this->set_size(N); + } + + public: + void resize(size_type N) { resizeImpl(N); } + + /// Like resize, but \ref T is POD, the new values won't be initialized. + void resize_for_overwrite(size_type N) { resizeImpl(N); } + + /// Like resize, but requires that \p N is less than \a size(). + void truncate(size_type N) { + assert(this->size() >= N && "Cannot increase size with truncate"); + this->destroy_range(this->begin() + N, this->end()); + this->set_size(N); + } + + void resize(size_type N, ValueParamT NV) { + if (N == this->size()) + return; + + if (N < this->size()) { + this->truncate(N); + return; + } + + // N > this->size(). Defer to append. + this->append(N - this->size(), NV); + } + + void reserve(size_type N) { + if (this->capacity() < N) + this->grow(N); + } + + void pop_back_n(size_type NumItems) { + assert(this->size() >= NumItems); + truncate(this->size() - NumItems); + } + + LLVM_NODISCARD T pop_back_val() { + T Result = ::std::move(this->back()); + this->pop_back(); + return Result; + } + + void swap(SmallVectorImpl& RHS); + + /// Add the specified range to the end of the SmallVector. + template ::iterator_category, + std::input_iterator_tag>::value>> + void append(in_iter in_start, in_iter in_end) { + this->assertSafeToAddRange(in_start, in_end); + size_type NumInputs = std::distance(in_start, in_end); + this->reserve(this->size() + NumInputs); + this->uninitialized_copy(in_start, in_end, this->end()); + this->set_size(this->size() + NumInputs); + } + + /// Append \p NumInputs copies of \p Elt to the end. + void append(size_type NumInputs, ValueParamT Elt) { + const T* EltPtr = this->reserveForParamAndGetAddress(Elt, NumInputs); + std::uninitialized_fill_n(this->end(), NumInputs, *EltPtr); + this->set_size(this->size() + NumInputs); + } + + void append(std::initializer_list IL) { append(IL.begin(), IL.end()); } + + void append(const SmallVectorImpl& RHS) { append(RHS.begin(), RHS.end()); } + + void assign(size_type NumElts, ValueParamT Elt) { + // Note that Elt could be an internal reference. + if (NumElts > this->capacity()) { + this->growAndAssign(NumElts, Elt); + return; + } + + // Assign over existing elements. + std::fill_n(this->begin(), std::min(NumElts, this->size()), Elt); + if (NumElts > this->size()) + std::uninitialized_fill_n(this->end(), NumElts - this->size(), Elt); + else if (NumElts < this->size()) + this->destroy_range(this->begin() + NumElts, this->end()); + this->set_size(NumElts); + } + + // FIXME: Consider assigning over existing elements, rather than clearing & + // re-initializing them - for all assign(...) variants. + + template ::iterator_category, + std::input_iterator_tag>::value>> + void assign(in_iter in_start, in_iter in_end) { + this->assertSafeToReferenceAfterClear(in_start, in_end); + clear(); + append(in_start, in_end); + } + + void assign(std::initializer_list IL) { + clear(); + append(IL); + } + + void assign(const SmallVectorImpl& RHS) { assign(RHS.begin(), RHS.end()); } + + iterator erase(const_iterator CI) { + // Just cast away constness because this is a non-const member function. + iterator I = const_cast(CI); + + assert(this->isReferenceToStorage(CI) && + "Iterator to erase is out of bounds."); + + iterator N = I; + // Shift all elts down one. + std::move(I + 1, this->end(), I); + // Drop the last elt. + this->pop_back(); + return (N); + } + + iterator erase(const_iterator CS, const_iterator CE) { + // Just cast away constness because this is a non-const member function. + iterator S = const_cast(CS); + iterator E = const_cast(CE); + + assert(this->isRangeInStorage(S, E) && + "Range to erase is out of bounds."); + + iterator N = S; + // Shift all elts down. + iterator I = std::move(E, this->end(), S); + // Drop the last elts. + this->destroy_range(I, this->end()); + this->set_size(I - this->begin()); + return (N); + } + + private: + template + iterator insert_one_impl(iterator I, ArgType&& Elt) { + // Callers ensure that ArgType is derived from T. + static_assert( + std::is_same>, + T>::value, + "ArgType must be derived from T!"); + + if (I == this->end()) { // Important special case for empty vector. + this->push_back(::std::forward(Elt)); + return this->end() - 1; + } + + assert(this->isReferenceToStorage(I) && + "Insertion iterator is out of bounds."); + + // Grow if necessary. + size_t Index = I - this->begin(); + std::remove_reference_t* EltPtr = + this->reserveForParamAndGetAddress(Elt); + I = this->begin() + Index; + + ::new ((void*)this->end()) T(::std::move(this->back())); + // Push everything else over. + std::move_backward(I, this->end() - 1, this->end()); + this->set_size(this->size() + 1); + + // If we just moved the element we're inserting, be sure to update + // the reference (never happens if TakesParamByValue). + static_assert(!TakesParamByValue || std::is_same::value, + "ArgType must be 'T' when taking by value!"); + if (!TakesParamByValue && + this->isReferenceToRange(EltPtr, I, this->end())) + ++EltPtr; + + *I = ::std::forward(*EltPtr); + return I; + } + + public: + iterator insert(iterator I, T&& Elt) { + return insert_one_impl(I, this->forward_value_param(std::move(Elt))); + } + + iterator insert(iterator I, const T& Elt) { + return insert_one_impl(I, this->forward_value_param(Elt)); + } + + iterator insert(iterator I, size_type NumToInsert, ValueParamT Elt) { + // Convert iterator to elt# to avoid invalidating iterator when we + // reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(NumToInsert, Elt); + return this->begin() + InsertElt; + } + + assert(this->isReferenceToStorage(I) && + "Insertion iterator is out of bounds."); + + // Ensure there is enough space, and get the (maybe updated) address of + // Elt. + const T* EltPtr = this->reserveForParamAndGetAddress(Elt, NumToInsert); + + // Uninvalidate the iterator. + I = this->begin() + InsertElt; + + // If there are more elements between the insertion point and the end of + // the range than there are being inserted, we can use a simple approach + // to insertion. Since we already reserved space, we know that this + // won't reallocate the vector. + if (size_t(this->end() - I) >= NumToInsert) { + T* OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd - NumToInsert, OldEnd); + + // If we just moved the element we're inserting, be sure to update + // the reference (never happens if TakesParamByValue). + if (!TakesParamByValue && I <= EltPtr && EltPtr < this->end()) + EltPtr += NumToInsert; + + std::fill_n(I, NumToInsert, *EltPtr); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and + // we're not inserting at the end. + + // Move over the elements that we're about to overwrite. + T* OldEnd = this->end(); + this->set_size(this->size() + NumToInsert); + size_t NumOverwritten = OldEnd - I; + this->uninitialized_move(I, OldEnd, this->end() - NumOverwritten); + + // If we just moved the element we're inserting, be sure to update + // the reference (never happens if TakesParamByValue). + if (!TakesParamByValue && I <= EltPtr && EltPtr < this->end()) + EltPtr += NumToInsert; + + // Replace the overwritten part. + std::fill_n(I, NumOverwritten, *EltPtr); + + // Insert the non-overwritten middle part. + std::uninitialized_fill_n(OldEnd, NumToInsert - NumOverwritten, + *EltPtr); + return I; + } + + template ::iterator_category, + std::input_iterator_tag>::value>> + iterator insert(iterator I, ItTy From, ItTy To) { + // Convert iterator to elt# to avoid invalidating iterator when we + // reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(From, To); + return this->begin() + InsertElt; + } + + assert(this->isReferenceToStorage(I) && + "Insertion iterator is out of bounds."); + + // Check that the reserve that follows doesn't invalidate the iterators. + this->assertSafeToAddRange(From, To); + + size_t NumToInsert = std::distance(From, To); + + // Ensure there is enough space. + reserve(this->size() + NumToInsert); + + // Uninvalidate the iterator. + I = this->begin() + InsertElt; + + // If there are more elements between the insertion point and the end of + // the range than there are being inserted, we can use a simple approach + // to insertion. Since we already reserved space, we know that this + // won't reallocate the vector. + if (size_t(this->end() - I) >= NumToInsert) { + T* OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd - NumToInsert, OldEnd); + + std::copy(From, To, I); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and + // we're not inserting at the end. + + // Move over the elements that we're about to overwrite. + T* OldEnd = this->end(); + this->set_size(this->size() + NumToInsert); + size_t NumOverwritten = OldEnd - I; + this->uninitialized_move(I, OldEnd, this->end() - NumOverwritten); + + // Replace the overwritten part. + for (T* J = I; NumOverwritten > 0; --NumOverwritten) { + *J = *From; + ++J; + ++From; + } + + // Insert the non-overwritten middle part. + this->uninitialized_copy(From, To, OldEnd); + return I; + } + + void insert(iterator I, std::initializer_list IL) { + insert(I, IL.begin(), IL.end()); + } + + template + reference emplace_back(ArgTypes&&... Args) { + if (LLVM_UNLIKELY(this->size() >= this->capacity())) + return this->growAndEmplaceBack(std::forward(Args)...); + + ::new ((void*)this->end()) T(std::forward(Args)...); + this->set_size(this->size() + 1); + return this->back(); + } + + SmallVectorImpl& operator=(const SmallVectorImpl& RHS); + + SmallVectorImpl& operator=(SmallVectorImpl&& RHS); + + bool operator==(const SmallVectorImpl& RHS) const { + if (this->size() != RHS.size()) + return false; + return std::equal(this->begin(), this->end(), RHS.begin()); + } + bool operator!=(const SmallVectorImpl& RHS) const { + return !(*this == RHS); + } + + bool operator<(const SmallVectorImpl& RHS) const { + return std::lexicographical_compare(this->begin(), this->end(), + RHS.begin(), RHS.end()); + } +}; + +template +void SmallVectorImpl::swap(SmallVectorImpl& RHS) { + if (this == &RHS) + return; + + // We can only avoid copying elements if neither vector is small. + if (!this->isSmall() && !RHS.isSmall()) { + std::swap(this->BeginX, RHS.BeginX); + std::swap(this->Size, RHS.Size); + std::swap(this->Capacity, RHS.Capacity); + return; + } + this->reserve(RHS.size()); + RHS.reserve(this->size()); + + // Swap the shared elements. + size_t NumShared = this->size(); + if (NumShared > RHS.size()) + NumShared = RHS.size(); + for (size_type i = 0; i != NumShared; ++i) + std::swap((*this)[i], RHS[i]); + + // Copy over the extra elts. + if (this->size() > RHS.size()) { + size_t EltDiff = this->size() - RHS.size(); + this->uninitialized_copy(this->begin() + NumShared, this->end(), + RHS.end()); + RHS.set_size(RHS.size() + EltDiff); + this->destroy_range(this->begin() + NumShared, this->end()); + this->set_size(NumShared); + } else if (RHS.size() > this->size()) { + size_t EltDiff = RHS.size() - this->size(); + this->uninitialized_copy(RHS.begin() + NumShared, RHS.end(), + this->end()); + this->set_size(this->size() + EltDiff); + this->destroy_range(RHS.begin() + NumShared, RHS.end()); + RHS.set_size(NumShared); + } +} + +template +SmallVectorImpl& SmallVectorImpl::operator=( + const SmallVectorImpl& RHS) { + // Avoid self-assignment. + if (this == &RHS) + return *this; + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd; + if (RHSSize) + NewEnd = + std::copy(RHS.begin(), RHS.begin() + RHSSize, this->begin()); + else + NewEnd = this->begin(); + + // Destroy excess elements. + this->destroy_range(NewEnd, this->end()); + + // Trim. + this->set_size(RHSSize); + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: don't do this if they're efficiently moveable. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->clear(); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::copy(RHS.begin(), RHS.begin() + CurSize, this->begin()); + } + + // Copy construct the new elements in place. + this->uninitialized_copy(RHS.begin() + CurSize, RHS.end(), + this->begin() + CurSize); + + // Set end. + this->set_size(RHSSize); + return *this; +} + +template +SmallVectorImpl& SmallVectorImpl::operator=(SmallVectorImpl&& RHS) { + // Avoid self-assignment. + if (this == &RHS) + return *this; + + // If the RHS isn't small, clear this vector and then steal its buffer. + if (!RHS.isSmall()) { + this->assignRemote(std::move(RHS)); + return *this; + } + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd = this->begin(); + if (RHSSize) + NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd); + + // Destroy excess elements and trim the bounds. + this->destroy_range(NewEnd, this->end()); + this->set_size(RHSSize); + + // Clear the RHS. + RHS.clear(); + + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: this may not actually make any sense if we can efficiently move + // elements. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->clear(); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::move(RHS.begin(), RHS.begin() + CurSize, this->begin()); + } + + // Move-construct the new elements in place. + this->uninitialized_move(RHS.begin() + CurSize, RHS.end(), + this->begin() + CurSize); + + // Set end. + this->set_size(RHSSize); + + RHS.clear(); + return *this; +} + +/// Storage for the SmallVector elements. This is specialized for the N=0 case +/// to avoid allocating unnecessary storage. +template +struct SmallVectorStorage { + alignas(T) char InlineElts[N * sizeof(T)]; +}; + +/// We need the storage to be properly aligned even for small-size of 0 so that +/// the pointer math in \a SmallVectorTemplateCommon::getFirstEl() is +/// well-defined. +template +struct alignas(T) SmallVectorStorage {}; + +/// Forward declaration of SmallVector so that +/// calculateSmallVectorDefaultInlinedElements can reference +/// `sizeof(SmallVector)`. +template +class LLVM_GSL_OWNER SmallVector; + +/// Helper class for calculating the default number of inline elements for +/// `SmallVector`. +/// +/// This should be migrated to a constexpr function when our minimum +/// compiler support is enough for multi-statement constexpr functions. +template +struct CalculateSmallVectorDefaultInlinedElements { + // Parameter controlling the default number of inlined elements + // for `SmallVector`. + // + // The default number of inlined elements ensures that + // 1. There is at least one inlined element. + // 2. `sizeof(SmallVector) <= kPreferredSmallVectorSizeof` unless + // it contradicts 1. + static constexpr size_t kPreferredSmallVectorSizeof = 64; + + // static_assert that sizeof(T) is not "too big". + // + // Because our policy guarantees at least one inlined element, it is + // possible for an arbitrarily large inlined element to allocate an + // arbitrarily large amount of inline storage. We generally consider it an + // antipattern for a SmallVector to allocate an excessive amount of inline + // storage, so we want to call attention to these cases and make sure that + // users are making an intentional decision if they request a lot of inline + // storage. + // + // We want this assertion to trigger in pathological cases, but otherwise + // not be too easy to hit. To accomplish that, the cutoff is actually + // somewhat larger than kPreferredSmallVectorSizeof (otherwise, + // `SmallVector>` would be one easy way to trip it, and that + // pattern seems useful in practice). + // + // One wrinkle is that this assertion is in theory non-portable, since + // sizeof(T) is in general platform-dependent. However, we don't expect this + // to be much of an issue, because most LLVM development happens on 64-bit + // hosts, and therefore sizeof(T) is expected to *decrease* when compiled + // for 32-bit hosts, dodging the issue. The reverse situation, where + // development happens on a 32-bit host and then fails due to sizeof(T) + // *increasing* on a 64-bit host, is expected to be very rare. + static_assert( + sizeof(T) <= 256, + "You are trying to use a default number of inlined elements for " + "`SmallVector` but `sizeof(T)` is really big! Please use an " + "explicit number of inlined elements with `SmallVector` to make " + "sure you really want that much inline storage."); + + // Discount the size of the header itself when calculating the maximum + // inline bytes. + static constexpr size_t PreferredInlineBytes = + kPreferredSmallVectorSizeof - sizeof(SmallVector); + static constexpr size_t NumElementsThatFit = + PreferredInlineBytes / sizeof(T); + static constexpr size_t value = + NumElementsThatFit == 0 ? 1 : NumElementsThatFit; +}; + +/// This is a 'vector' (really, a variable-sized array), optimized +/// for the case when the array is small. It contains some number of elements +/// in-place, which allows it to avoid heap allocation when the actual number of +/// elements is below that threshold. This allows normal "small" cases to be +/// fast without losing generality for large inputs. +/// +/// \note +/// In the absence of a well-motivated choice for the number of inlined +/// elements \p N, it is recommended to use \c SmallVector (that is, +/// omitting the \p N). This will choose a default number of inlined elements +/// reasonable for allocation on the stack (for example, trying to keep \c +/// sizeof(SmallVector) around 64 bytes). +/// +/// \warning This does not attempt to be exception safe. +/// +/// \see https://llvm.org/docs/ProgrammersManual.html#llvm-adt-smallvector-h +template ::value> +class LLVM_GSL_OWNER SmallVector : public SmallVectorImpl, + SmallVectorStorage { + public: + SmallVector() : SmallVectorImpl(N) {} + + ~SmallVector() { + // Destroy the constructed elements in the vector. + this->destroy_range(this->begin(), this->end()); + } + + explicit SmallVector(size_t Size, const T& Value = T()) + : SmallVectorImpl(N) { + this->assign(Size, Value); + } + + template ::iterator_category, + std::input_iterator_tag>::value>> + SmallVector(ItTy S, ItTy E) : SmallVectorImpl(N) { + this->append(S, E); + } + + template + explicit SmallVector(const iterator_range& R) + : SmallVectorImpl(N) { + this->append(R.begin(), R.end()); + } + + SmallVector(std::initializer_list IL) : SmallVectorImpl(N) { + this->assign(IL); + } + + SmallVector(const SmallVector& RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(RHS); + } + + SmallVector& operator=(const SmallVector& RHS) { + SmallVectorImpl::operator=(RHS); + return *this; + } + + SmallVector(SmallVector&& RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + SmallVector(SmallVectorImpl&& RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + SmallVector& operator=(SmallVector&& RHS) { + if (N) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + // SmallVectorImpl::operator= does not leverage N==0. Optimize the + // case. + if (this == &RHS) + return *this; + if (RHS.empty()) { + this->destroy_range(this->begin(), this->end()); + this->Size = 0; + } else { + this->assignRemote(std::move(RHS)); + } + return *this; + } + + SmallVector& operator=(SmallVectorImpl&& RHS) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + + SmallVector& operator=(std::initializer_list IL) { + this->assign(IL); + return *this; + } +}; + +template +inline size_t capacity_in_bytes(const SmallVector& X) { + return X.capacity_in_bytes(); +} + +template +using ValueTypeFromRangeType = + typename std::remove_const()))>::type>::type; + +/// Given a range of type R, iterate the entire range and return a +/// SmallVector with elements of the vector. This is useful, for example, +/// when you want to iterate a range and then sort the results. +template +SmallVector, Size> to_vector(R&& Range) { + return {std::begin(Range), std::end(Range)}; +} +template +SmallVector, + CalculateSmallVectorDefaultInlinedElements< + ValueTypeFromRangeType>::value> +to_vector(R&& Range) { + return {std::begin(Range), std::end(Range)}; +} + +} // end namespace llvm + +namespace std { + +/// Implement std::swap in terms of SmallVector swap. +template +inline void swap(llvm::SmallVectorImpl& LHS, llvm::SmallVectorImpl& RHS) { + LHS.swap(RHS); +} + +/// Implement std::swap in terms of SmallVector swap. +template +inline void swap(llvm::SmallVector& LHS, llvm::SmallVector& RHS) { + LHS.swap(RHS); +} + +} // end namespace std diff --git a/src/plugin/bridges/vst2.h b/src/plugin/bridges/vst2.h index ce010bca..ea7e65dd 100644 --- a/src/plugin/bridges/vst2.h +++ b/src/plugin/bridges/vst2.h @@ -196,7 +196,7 @@ class Vst2PluginBridge : PluginBridge> { * we receive so we can send them to host on the audio thread at the end of * `process_replacing()`. */ - boost::container::small_vector incoming_midi_events_; + llvm::SmallVector incoming_midi_events_; /** * Mutex for locking the above event queue, since recieving and processing * now happens in two different threads. diff --git a/src/plugin/meson.build b/src/plugin/meson.build index aeaee5dc..b582ee28 100644 --- a/src/plugin/meson.build +++ b/src/plugin/meson.build @@ -12,6 +12,7 @@ vst2_plugin_sources = files( '../common/plugins.cpp', '../common/process.cpp', '../common/utils.cpp', + '../include/llvm/small-vector.cpp', 'bridges/vst2.cpp', 'host-process.cpp', 'utils.cpp', @@ -82,6 +83,7 @@ vst3_plugin_sources = files( '../common/plugins.cpp', '../common/process.cpp', '../common/utils.cpp', + '../include/llvm/small-vector.cpp', 'bridges/vst3.cpp', 'bridges/vst3-impls/plugin-factory-proxy.cpp', 'bridges/vst3-impls/plug-view-proxy.cpp', diff --git a/src/wine-host/bridges/vst2.h b/src/wine-host/bridges/vst2.h index 98e36a77..cf6c35e5 100644 --- a/src/wine-host/bridges/vst2.h +++ b/src/wine-host/bridges/vst2.h @@ -245,7 +245,7 @@ class Vst2Bridge : public HostBridge { * practice every host will bundle all events in a single * `effProcessEvents()` call. */ - boost::container::small_vector + llvm::SmallVector next_audio_buffer_midi_events_; /** * Whether `next_audio_buffer_midi_events` should be cleared before diff --git a/src/wine-host/editor.cpp b/src/wine-host/editor.cpp index 525cb510..eead40d9 100644 --- a/src/wine-host/editor.cpp +++ b/src/wine-host/editor.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include using namespace std::literals::chrono_literals; using namespace std::literals::string_literals; @@ -132,7 +132,7 @@ static const HCURSOR arrow_cursor = LoadCursor(nullptr, IDC_ARROW); * @return A non-empty list containing `starting_at` and all of its ancestor * windows `starting_at`. */ -boost::container::small_vector find_ancestor_windows( +llvm::SmallVector find_ancestor_windows( xcb_connection_t& x11_connection, xcb_window_t starting_at); @@ -1238,7 +1238,7 @@ LRESULT CALLBACK window_proc(HWND handle, return DefWindowProc(handle, message, wParam, lParam); } -boost::container::small_vector find_ancestor_windows( +llvm::SmallVector find_ancestor_windows( xcb_connection_t& x11_connection, xcb_window_t starting_at) { xcb_window_t current_window = starting_at; @@ -1250,8 +1250,7 @@ boost::container::small_vector find_ancestor_windows( THROW_X11_ERROR(error); const xcb_window_t root = query_reply->root; - boost::container::small_vector ancestor_windows{ - current_window}; + llvm::SmallVector ancestor_windows{current_window}; while (query_reply->parent != root) { current_window = query_reply->parent; ancestor_windows.push_back(current_window); diff --git a/src/wine-host/meson.build b/src/wine-host/meson.build index ed357ae5..d2907d85 100644 --- a/src/wine-host/meson.build +++ b/src/wine-host/meson.build @@ -66,6 +66,7 @@ host_common_sources = files( '../common/plugins.cpp', '../common/process.cpp', '../common/utils.cpp', + '../include/llvm/small-vector.cpp', 'bridges/common.cpp', 'bridges/vst2.cpp', 'editor.cpp', diff --git a/src/wine-host/xdnd-proxy.cpp b/src/wine-host/xdnd-proxy.cpp index 61d5d8f2..e0f25265 100644 --- a/src/wine-host/xdnd-proxy.cpp +++ b/src/wine-host/xdnd-proxy.cpp @@ -192,9 +192,9 @@ WineXdndProxy::Handle WineXdndProxy::get_handle() { return Handle(instance); } -void WineXdndProxy::begin_xdnd(const boost::container::small_vector_base< - ghc::filesystem::path>& file_paths, - HWND tracker_window) { +void WineXdndProxy::begin_xdnd( + const llvm::SmallVectorImpl& file_paths, + HWND tracker_window) { if (file_paths.empty()) { throw std::runtime_error("Cannot drag-and-drop without any files"); } @@ -815,7 +815,7 @@ void CALLBACK dnd_winevent_callback(HWINEVENTHOOK /*hWinEventHook*/, } // This will contain the normal, Unix-style paths to the files - boost::container::small_vector dragged_files; + llvm::SmallVector dragged_files; for (unsigned int format_idx = 0; format_idx < num_formats; format_idx++) { STGMEDIUM storage{}; if (HRESULT result = tracker_info->dataObject->GetData( diff --git a/src/wine-host/xdnd-proxy.h b/src/wine-host/xdnd-proxy.h index 08d56b28..dcb208e2 100644 --- a/src/wine-host/xdnd-proxy.h +++ b/src/wine-host/xdnd-proxy.h @@ -26,8 +26,8 @@ #include #pragma pop_macro("_WIN32") +#include #include -#include #include #include "utils.h" @@ -149,9 +149,9 @@ class WineXdndProxy { * Initiate the XDDN protocol by taking ownership of the `XdndSelection` * selection and setting up the event listeners. */ - void begin_xdnd(const boost::container::small_vector_base< - ghc::filesystem::path>& file_paths, - HWND tracker_window); + void begin_xdnd( + const llvm::SmallVectorImpl& file_paths, + HWND tracker_window); /** * Release ownership of the selection stop listening for X11 events.