diff --git a/NAM/linear.cpp b/NAM/linear.cpp index 186ef8ae..3695e3c6 100644 --- a/NAM/linear.cpp +++ b/NAM/linear.cpp @@ -1,8 +1,11 @@ #include "linear.h" #include +#include +#include #include #include +#include #include #include "registry.h" @@ -11,51 +14,74 @@ namespace { -constexpr int _LINEAR_AUTO_DIRECT_MAX_TAPS = 256; -constexpr int _LINEAR_FFT_SMALL_BLOCK_SIZE = 256; -constexpr int _LINEAR_FFT_MEDIUM_BLOCK_SIZE = 512; -constexpr int _LINEAR_FFT_LARGE_BLOCK_SIZE = 1024; +struct LinearFFTDispatchEntry +{ + int max_taps; + nam::LinearImplementation implementation; + nam::LinearFFTPlan plan; +}; + +// Tuned using tools/bench_linear. The table keeps model-size policy separate +// from the DSP implementation. +constexpr std::array _LINEAR_FFT_DISPATCH{{ + {1024, nam::LinearImplementation::Direct, {128, 256}}, + {2048, nam::LinearImplementation::FFT, {128, 512}}, + {4096, nam::LinearImplementation::FFT, {128, 1024}}, + {8192, nam::LinearImplementation::FFT, {128, 2048}}, + {48000, nam::LinearImplementation::FFT, {64, 4096}}, + {240000, nam::LinearImplementation::FFT, {64, 8192}}, + {std::numeric_limits::max(), nam::LinearImplementation::FFT, {64, 8192}}, +}}; int _ceil_div(const int numerator, const int denominator) { return (numerator + denominator - 1) / denominator; } -int _choose_linear_fft_block_size(const int receptive_field) -{ - if (receptive_field <= 2048) - return _LINEAR_FFT_SMALL_BLOCK_SIZE; - if (receptive_field <= 8192) - return _LINEAR_FFT_MEDIUM_BLOCK_SIZE; - return _LINEAR_FFT_LARGE_BLOCK_SIZE; -} - } // namespace struct nam::LinearFFTState { using Complex = std::complex; - struct ChannelState + struct TierChannelState { std::vector input_time; std::vector> input_spectra; - std::vector output_ring; + std::vector accumulator; + std::vector ifft_time; int input_pos = 0; int spectrum_write_index = 0; + int job_spectrum_write_index = 0; + size_t job_work_index = 0; + int job_ticks_remaining = 0; + long long job_block_start = 0; + bool job_active = false; + }; + + struct Tier + { + Eigen::FFT fft; + int offset = 0; + int block_size = 0; + int fft_size = 0; + int spectrum_size = 0; + int num_partitions = 0; + bool runs_inline = false; + std::vector> kernel_spectra; + std::vector channels; + }; + + struct OutputChannelState + { + std::vector output_ring; }; - Eigen::FFT fft; - int block_size = 0; - int fft_size = 0; int direct_taps = 0; - int num_partitions = 0; int output_ring_size = 0; long long sample_index = 0; - std::vector> kernel_spectra; - std::vector channels; - std::vector accumulator; - std::vector ifft_time; + std::vector tiers; + std::vector output_channels; }; nam::Linear::Linear(const int in_channels, const int out_channels, const int receptive_field, const bool _bias, @@ -103,8 +129,7 @@ void nam::Linear::_configure_implementation() else if (this->_requested_implementation == LinearImplementation::FFT) this->_active_implementation = LinearImplementation::FFT; else - this->_active_implementation = - this->_receptive_field <= _LINEAR_AUTO_DIRECT_MAX_TAPS ? LinearImplementation::Direct : LinearImplementation::FFT; + this->_active_implementation = linear::select_implementation(this->_receptive_field); if (this->_active_implementation == LinearImplementation::FFT) this->_configure_fft_state(); @@ -116,52 +141,84 @@ void nam::Linear::_configure_fft_state() { this->_fft_state = std::make_unique(); auto& state = *this->_fft_state; - - state.block_size = _choose_linear_fft_block_size(this->_receptive_field); - state.fft_size = 2 * state.block_size; - state.direct_taps = std::min(this->_receptive_field, state.block_size); - state.num_partitions = this->_receptive_field > state.direct_taps - ? _ceil_div(this->_receptive_field - state.direct_taps, state.block_size) - : 0; - state.output_ring_size = 4 * state.block_size; + const auto plan = linear::select_fft_plan(this->_receptive_field); + state.direct_taps = std::min(this->_receptive_field, plan.direct_taps); state.sample_index = 0; this->_fft_direct_weight.resize(state.direct_taps); for (int i = 0; i < state.direct_taps; i++) this->_fft_direct_weight(i) = this->_impulse_response[state.direct_taps - 1 - i]; - state.kernel_spectra.assign(state.num_partitions, std::vector(state.fft_size)); - std::vector kernel_time(state.fft_size, 0.0f); - for (int partition = 0; partition < state.num_partitions; partition++) + // The inline tier covers [head, 4 * head). Every subsequent power-of-two + // tier starts at twice its block size, which gives it one full block of + // scheduling slack. Once the tuned maximum block size is reached, the last + // tier simply contains as many uniform partitions as are needed. + int offset = state.direct_taps; + int block_size = state.direct_taps; + while (offset < this->_receptive_field) { - std::fill(kernel_time.begin(), kernel_time.end(), 0.0f); - const int start = state.direct_taps + partition * state.block_size; - const int partition_size = std::min(state.block_size, this->_receptive_field - start); - for (int i = 0; i < partition_size; i++) - kernel_time[i] = this->_impulse_response[start + i]; - state.fft.fwd(state.kernel_spectra[partition].data(), kernel_time.data(), state.fft_size); + const bool first_tier = state.tiers.empty(); + const int partitions = first_tier ? std::min(3, _ceil_div(this->_receptive_field - offset, block_size)) + : block_size == plan.max_partition_size + ? _ceil_div(this->_receptive_field - offset, block_size) + : std::min(2, _ceil_div(this->_receptive_field - offset, block_size)); + + auto& tier = state.tiers.emplace_back(); + tier.fft.SetFlag(Eigen::FFT::HalfSpectrum); + tier.offset = offset; + tier.block_size = block_size; + tier.fft_size = 2 * block_size; + tier.spectrum_size = block_size + 1; + tier.num_partitions = partitions; + tier.runs_inline = first_tier; + tier.kernel_spectra.assign(partitions, std::vector(tier.spectrum_size)); + + std::vector kernel_time(tier.fft_size, 0.0f); + for (int partition = 0; partition < partitions; partition++) + { + std::fill(kernel_time.begin(), kernel_time.end(), 0.0f); + const int start = offset + partition * block_size; + const int partition_size = std::min(block_size, this->_receptive_field - start); + std::copy_n(this->_impulse_response.begin() + start, partition_size, kernel_time.begin()); + tier.fft.fwd(tier.kernel_spectra[partition].data(), kernel_time.data(), tier.fft_size); + } + + offset += partitions * block_size; + if (block_size < plan.max_partition_size) + block_size = std::min(2 * block_size, plan.max_partition_size); } const int channels_to_process = std::min(NumInputChannels(), NumOutputChannels()); - state.channels.resize(channels_to_process); - for (auto& channel : state.channels) + int largest_block_size = state.direct_taps; + for (auto& tier : state.tiers) { - channel.input_time.assign(state.fft_size, 0.0f); - channel.input_spectra.assign( - state.num_partitions, std::vector(state.fft_size, LinearFFTState::Complex{})); - channel.output_ring.assign(state.output_ring_size, 0.0f); - channel.input_pos = 0; - channel.spectrum_write_index = 0; + largest_block_size = std::max(largest_block_size, tier.block_size); + tier.channels.resize(channels_to_process); + for (auto& channel : tier.channels) + { + channel.input_time.assign(tier.fft_size, 0.0f); + channel.input_spectra.assign( + tier.num_partitions, std::vector(tier.spectrum_size, LinearFFTState::Complex{})); + channel.accumulator.assign(tier.spectrum_size, LinearFFTState::Complex{}); + channel.ifft_time.assign(tier.fft_size, 0.0f); + // Half-block phase offsets keep power-of-two tiers from all transforming + // in the same callback. The leading inline tier must remain unshifted. + channel.input_pos = tier.runs_inline ? 0 : tier.block_size / 2; + } } - state.accumulator.assign(state.fft_size, LinearFFTState::Complex{}); - state.ifft_time.assign(state.fft_size, 0.0f); - if (state.num_partitions > 0) + state.output_ring_size = 4 * largest_block_size; + state.output_channels.resize(channels_to_process); + for (auto& channel : state.output_channels) + channel.output_ring.assign(state.output_ring_size, 0.0f); + + // Create all FFT plans outside the audio callback. + for (auto& tier : state.tiers) { - std::vector warm_spectrum(state.fft_size); - std::vector warm_time(state.fft_size, 0.0f); - state.fft.fwd(warm_spectrum.data(), warm_time.data(), state.fft_size); - state.fft.inv(warm_time.data(), warm_spectrum.data(), state.fft_size); + std::vector warm_spectrum(tier.spectrum_size); + std::vector warm_time(tier.fft_size, 0.0f); + tier.fft.fwd(warm_spectrum.data(), warm_time.data(), tier.fft_size); + tier.fft.inv(warm_time.data(), warm_spectrum.data(), tier.fft_size); } } @@ -213,20 +270,27 @@ void nam::Linear::_process_fft(NAM_SAMPLE** input, NAM_SAMPLE** output, const in const long direct_offset = this->_input_buffer_offset - direct_taps + i + 1; for (int ch = 0; ch < channels_to_process; ch++) { + this->_advance_fft_jobs(ch); + const int ring_index = (int)(state.sample_index % state.output_ring_size); - const float tail = state.channels[ch].output_ring[ring_index]; - state.channels[ch].output_ring[ring_index] = 0.0f; + const float tail = state.output_channels[ch].output_ring[ring_index]; + state.output_channels[ch].output_ring[ring_index] = 0.0f; auto input_vec = Eigen::Map(&this->_input_buffers[ch][direct_offset], direct_taps); output[ch][i] = this->_bias + this->_fft_direct_weight.dot(input_vec) + tail; - if (state.num_partitions > 0) + for (size_t tier_index = 0; tier_index < state.tiers.size(); ++tier_index) { - auto& channel = state.channels[ch]; + auto& tier = state.tiers[tier_index]; + auto& channel = tier.channels[ch]; channel.input_time[channel.input_pos] = (float)input[ch][i]; channel.input_pos++; - if (channel.input_pos == state.block_size) - this->_run_fft_block(ch); + if (channel.input_pos == tier.block_size) + { + const long long block_start = state.sample_index - tier.block_size + 1; + this->_start_fft_block((int)tier_index, ch, block_start); + channel.input_pos = 0; + } } } @@ -239,42 +303,80 @@ void nam::Linear::_process_fft(NAM_SAMPLE** input, NAM_SAMPLE** output, const in nam::Buffer::_advance_input_buffer_(num_frames); } -void nam::Linear::_run_fft_block(const int channel_index) +void nam::Linear::_advance_fft_jobs(const int channel_index) { - auto& state = *this->_fft_state; - auto& channel = state.channels[channel_index]; - - auto& current_spectrum = channel.input_spectra[channel.spectrum_write_index]; - state.fft.fwd(current_spectrum.data(), channel.input_time.data(), state.fft_size); + for (size_t tier_index = 0; tier_index < this->_fft_state->tiers.size(); ++tier_index) + this->_advance_fft_job((int)tier_index, channel_index); +} - std::fill(state.accumulator.begin(), state.accumulator.end(), LinearFFTState::Complex{}); - for (int partition = 0; partition < state.num_partitions; partition++) +void nam::Linear::_advance_fft_job(const int tier_index, const int channel_index) +{ + auto& tier = this->_fft_state->tiers[tier_index]; + auto& channel = tier.channels[channel_index]; + if (!channel.job_active) + return; + + const size_t total_work = (size_t)tier.num_partitions * tier.spectrum_size; + const size_t remaining_work = total_work - channel.job_work_index; + const size_t work_this_tick = + (remaining_work + (size_t)channel.job_ticks_remaining - 1) / (size_t)channel.job_ticks_remaining; + const size_t work_end = std::min(total_work, channel.job_work_index + work_this_tick); + while (channel.job_work_index < work_end) { - int input_spectrum_index = channel.spectrum_write_index - partition; + const int partition = (int)(channel.job_work_index / (size_t)tier.spectrum_size); + const int bin = (int)(channel.job_work_index % (size_t)tier.spectrum_size); + int input_spectrum_index = channel.job_spectrum_write_index - partition; if (input_spectrum_index < 0) - input_spectrum_index += state.num_partitions; - const auto& input_spectrum = channel.input_spectra[input_spectrum_index]; - const auto& kernel_spectrum = state.kernel_spectra[partition]; - for (int bin = 0; bin < state.fft_size; bin++) - state.accumulator[bin] += input_spectrum[bin] * kernel_spectrum[bin]; + input_spectrum_index += tier.num_partitions; + channel.accumulator[bin] += channel.input_spectra[input_spectrum_index][bin] * tier.kernel_spectra[partition][bin]; + channel.job_work_index++; } + channel.job_ticks_remaining--; + if (channel.job_work_index == total_work) + this->_finish_fft_block(tier_index, channel_index); +} - state.fft.inv(state.ifft_time.data(), state.accumulator.data(), state.fft_size); +void nam::Linear::_start_fft_block(const int tier_index, const int channel_index, const long long block_start) +{ + auto& tier = this->_fft_state->tiers[tier_index]; + auto& channel = tier.channels[channel_index]; + assert(!channel.job_active); + + channel.job_spectrum_write_index = channel.spectrum_write_index; + auto& current_spectrum = channel.input_spectra[channel.job_spectrum_write_index]; + tier.fft.fwd(current_spectrum.data(), channel.input_time.data(), tier.fft_size); + std::fill(channel.accumulator.begin(), channel.accumulator.end(), LinearFFTState::Complex{}); + channel.job_work_index = 0; + channel.job_ticks_remaining = tier.runs_inline ? 1 : tier.block_size; + channel.job_block_start = block_start; + channel.job_active = true; - const long long block_start = state.sample_index - state.block_size + 1; - const long long output_start = block_start + state.direct_taps; - auto& output_ring = channel.output_ring; - for (int i = 0; i < state.fft_size - 1; i++) + channel.spectrum_write_index++; + if (channel.spectrum_write_index == tier.num_partitions) + channel.spectrum_write_index = 0; + + std::fill(channel.input_time.begin(), channel.input_time.begin() + tier.block_size, 0.0f); + + if (tier.runs_inline) + this->_advance_fft_job(tier_index, channel_index); +} + +void nam::Linear::_finish_fft_block(const int tier_index, const int channel_index) +{ + auto& state = *this->_fft_state; + auto& tier = state.tiers[tier_index]; + auto& channel = tier.channels[channel_index]; + + tier.fft.inv(channel.ifft_time.data(), channel.accumulator.data(), tier.fft_size); + + const long long output_start = channel.job_block_start + tier.offset; + auto& output_ring = state.output_channels[channel_index].output_ring; + for (int i = 0; i < tier.fft_size - 1; i++) { const int ring_index = (int)((output_start + i) % state.output_ring_size); - output_ring[ring_index] += state.ifft_time[i]; + output_ring[ring_index] += channel.ifft_time[i]; } - - std::fill(channel.input_time.begin(), channel.input_time.begin() + state.block_size, 0.0f); - channel.input_pos = 0; - channel.spectrum_write_index++; - if (channel.spectrum_write_index == state.num_partitions) - channel.spectrum_write_index = 0; + channel.job_active = false; } nam::LinearImplementation nam::linear::parse_implementation(const std::string& implementation) @@ -303,6 +405,22 @@ std::string nam::linear::implementation_to_string(const LinearImplementation imp throw std::runtime_error("Unsupported Linear implementation enum"); } +nam::LinearFFTPlan nam::linear::select_fft_plan(const int receptive_field) +{ + for (const auto& entry : _LINEAR_FFT_DISPATCH) + if (receptive_field <= entry.max_taps) + return entry.plan; + throw std::runtime_error("No Linear FFT dispatch entry for receptive field"); +} + +nam::LinearImplementation nam::linear::select_implementation(const int receptive_field) +{ + for (const auto& entry : _LINEAR_FFT_DISPATCH) + if (receptive_field <= entry.max_taps) + return entry.implementation; + throw std::runtime_error("No Linear implementation dispatch entry for receptive field"); +} + nam::linear::LinearConfig nam::linear::parse_config_json(const nlohmann::json& config) { LinearConfig c; diff --git a/NAM/linear.h b/NAM/linear.h index d3559bcf..03ef4156 100644 --- a/NAM/linear.h +++ b/NAM/linear.h @@ -7,6 +7,12 @@ namespace nam struct LinearFFTState; +struct LinearFFTPlan +{ + int direct_taps; + int max_partition_size; +}; + /// \brief Selects the convolution engine used by Linear models. enum class LinearImplementation { @@ -62,7 +68,10 @@ class Linear : public Buffer void _configure_fft_state(); void _process_direct(NAM_SAMPLE** input, NAM_SAMPLE** output, const int num_frames); void _process_fft(NAM_SAMPLE** input, NAM_SAMPLE** output, const int num_frames); - void _run_fft_block(const int channel); + void _advance_fft_job(const int tier, const int channel); + void _advance_fft_jobs(const int channel); + void _start_fft_block(const int tier, const int channel, const long long block_start); + void _finish_fft_block(const int tier, const int channel); }; namespace linear @@ -86,6 +95,12 @@ LinearImplementation parse_implementation(const std::string& implementation); /// \brief String name for a Linear implementation. std::string implementation_to_string(const LinearImplementation implementation); +/// \brief Select the tuned convolution plan for an impulse-response length. +LinearFFTPlan select_fft_plan(int receptive_field); + +/// \brief Select the default implementation for an impulse-response length. +LinearImplementation select_implementation(int receptive_field); + /// \brief Parse Linear configuration from JSON /// \param config JSON configuration object /// \return LinearConfig diff --git a/tools/BENCHMARK_LINEAR.md b/tools/BENCHMARK_LINEAR.md new file mode 100644 index 00000000..e1ae6ff5 --- /dev/null +++ b/tools/BENCHMARK_LINEAR.md @@ -0,0 +1,52 @@ +# Linear convolution benchmark + +`bench_linear` measures callback-time distribution for synthetic linear models. It intentionally reports callback +times rather than only throughput: a convolution implementation can have a low average cost and still cause audio +dropouts when periodic work exceeds the callback deadline. + +Build and run a Release benchmark with: + +```sh +cmake -S . -B build-release -DCMAKE_BUILD_TYPE=Release +cmake --build build-release --target bench_linear -j +build-release/tools/bench_linear +``` + +The optional arguments are `taps`, `callback size`, `seconds`, `fft|direct`, and `cold|verify`. The `cold` mode +touches a 64 MiB buffer before each timed callback to expose cache-sensitive plans. `verify` renders an impulse +through the entire requested filter—including a one-minute filter—and compares every output sample with its tap. + +## Dispatch tuning + +The dispatch table was tuned on an Apple M1 at 48 kHz using the same `-O3` optimization level as the Release plugin. +The main constraint was the 666.67 microsecond deadline of a 32-sample callback. Candidate plans varied direct-head +sizes from 64 through 256 samples and maximum FFT partitions from 1,024 through 32,768 samples. + +The selected FFT plans produced these representative 10-second results (the default dispatcher uses direct +convolution for the 1,024-tap case): + +| Taps | Direct head | Maximum partition | Mean (us) | p99 (us) | Max (us) | Overruns | +|---:|---:|---:|---:|---:|---:|---:| +| 1,024 | 128 | 256 | 2.37 | 10.62 | 55.67 | 0 | +| 2,048 | 128 | 512 | 2.90 | 14.54 | 63.38 | 0 | +| 4,096 | 128 | 1,024 | 3.65 | 23.79 | 58.33 | 0 | +| 8,192 | 128 | 2,048 | 4.63 | 49.12 | 81.79 | 0 | +| 48,000 | 64 | 4,096 | 6.45 | 48.83 | 123.42 | 0 | +| 240,000 | 64 | 8,192 | 8.21 | 91.12 | 218.33 | 0 | +| 1,200,000 | 64 | 8,192 | 14.41 | 99.79 | 297.58 | 0 | +| 2,880,000 | 64 | 8,192 | 24.22 | 112.00 | 324.62 | 0 | + +The old 1,024-sample uniform implementation measured about 2.7--3.1 ms at p99 for the 1,208,121-tap atmospheric +model and overran 32- and 64-sample callback deadlines. The non-uniform plan reduces that periodic burst by more than +an order of magnitude. + +The table favors bounded callback time over the absolute minimum mean: + +- A 64-sample direct head had lower p99 times for long filters; a 128-sample head reduced overhead for 2K--8K filters. +- A 16,384-sample maximum partition slightly reduced mean time for multi-second filters but roughly doubled p99 and + produced deadline outliers. A 4,096 maximum kept transforms smaller but nearly doubled one-minute steady-state cost. +- Direct convolution through 1,024 taps had a flatter callback profile than FFT convolution at essentially the same + mean cost, so `Auto` selects it for that range. + +These numbers are hardware-specific. When changing the FFT backend, data layout, or dispatch table, rerun warm and +cold-cache tests on every supported architecture and optimize for the worst callback distribution, not only mean CPU. diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 492fb676..51c7e7ea 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -18,6 +18,18 @@ include_directories(tools ${AUDIO_DSP_TOOLS_DIR}/dsp) add_executable(loadmodel loadmodel.cpp ${NAM_SOURCES}) add_executable(benchmodel benchmodel.cpp ${NAM_SOURCES}) +add_executable(bench_linear bench_linear.cpp ${NAM_SOURCES}) +target_compile_features(bench_linear PUBLIC cxx_std_20) +set_target_properties(bench_linear PROPERTIES + CXX_VISIBILITY_PRESET hidden + INTERPROCEDURAL_OPTIMIZATION TRUE + PREFIX "" +) +if (MSVC) + target_compile_options(bench_linear PRIVATE "$<$:/O2>") +else() + target_compile_options(bench_linear PRIVATE "$<$:-O3>") +endif() add_executable(render render.cpp ${NAM_SOURCES} ${AUDIO_DSP_TOOLS_WAV_SOURCES}) target_compile_features(render PUBLIC cxx_std_20) # AudioDSPTools wav.cpp has sign-compare issues; don't fail build diff --git a/tools/bench_linear.cpp b/tools/bench_linear.cpp new file mode 100644 index 00000000..b100742a --- /dev/null +++ b/tools/bench_linear.cpp @@ -0,0 +1,128 @@ +#include "NAM/linear.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +using Clock = std::chrono::steady_clock; + +std::vector make_weights(const int taps) +{ + std::vector weights(taps); + for (int i = 0; i < taps; ++i) + weights[i] = 0.001f * std::exp(-6.0f * (float)i / (float)taps) * std::sin(0.031f * (float)(i + 1)); + return weights; +} + +double percentile(const std::vector& sorted, const double fraction) +{ + const size_t index = std::min(sorted.size() - 1, (size_t)std::floor(fraction * (double)(sorted.size() - 1))); + return sorted[index]; +} + +bool verify_impulse_response(const int taps, const int callback_size, const int sample_rate) +{ + auto weights = make_weights(taps); + nam::Linear model(1, 1, taps, false, weights, sample_rate, nam::LinearImplementation::FFT); + model.Reset(sample_rate, callback_size); + + std::vector input(callback_size, 0.0); + std::vector output(callback_size, 0.0); + NAM_SAMPLE* inputs[] = {input.data()}; + NAM_SAMPLE* outputs[] = {output.data()}; + double max_abs_error = 0.0; + int sample = 0; + const int samples_to_process = taps + 2 * callback_size; + while (sample < samples_to_process) + { + std::fill(input.begin(), input.end(), 0.0); + if (sample == 0) + input[0] = 1.0; + const int frames = std::min(callback_size, samples_to_process - sample); + model.process(inputs, outputs, frames); + for (int i = 0; i < frames; ++i) + { + const double expected = sample + i < taps ? weights[sample + i] : 0.0; + max_abs_error = std::max(max_abs_error, std::abs((double)output[i] - expected)); + } + sample += frames; + } + std::cout << "verified_taps=" << taps << ",max_abs_error=" << std::scientific << max_abs_error << '\n'; + return max_abs_error < 5.0e-5; +} + +void run(const int taps, const int callback_size, const int sample_rate, const double seconds, + const nam::LinearImplementation implementation, const bool cold_cache) +{ + constexpr int max_callback_size = 512; + auto weights = make_weights(taps); + nam::Linear model(1, 1, taps, false, weights, sample_rate, implementation); + model.Reset(sample_rate, max_callback_size); + + std::vector input(max_callback_size); + std::vector output(max_callback_size); + for (int i = 0; i < max_callback_size; ++i) + input[i] = (NAM_SAMPLE)(0.1 * std::sin(0.017 * i) + 0.03 * std::cos(0.043 * i)); + NAM_SAMPLE* inputs[] = {input.data()}; + NAM_SAMPLE* outputs[] = {output.data()}; + + for (int i = 0; i < 4096 / callback_size; ++i) + model.process(inputs, outputs, callback_size); + + const int callbacks = std::max(1, (int)std::ceil(seconds * sample_rate / callback_size)); + std::vector durations; + durations.reserve(callbacks); + std::vector cache_polluter(cold_cache ? 64 * 1024 * 1024 : 0, 1); + volatile NAM_SAMPLE sink = 0.0; + volatile unsigned int cache_sink = 0; + for (int i = 0; i < callbacks; ++i) + { + for (size_t byte = 0; byte < cache_polluter.size(); byte += 64) + cache_sink = cache_sink + cache_polluter[byte]; + const auto start = Clock::now(); + model.process(inputs, outputs, callback_size); + const auto end = Clock::now(); + sink = sink + output[i % callback_size]; + durations.push_back(std::chrono::duration(end - start).count()); + } + + std::sort(durations.begin(), durations.end()); + const double deadline = 1.0e6 * callback_size / sample_rate; + const auto overruns = + std::count_if(durations.begin(), durations.end(), [deadline](double us) { return us > deadline; }); + const double mean = std::accumulate(durations.begin(), durations.end(), 0.0) / durations.size(); + std::cout << taps << ',' << callback_size << ',' << std::fixed << std::setprecision(2) << mean << ',' + << percentile(durations, 0.50) << ',' << percentile(durations, 0.95) << ',' << percentile(durations, 0.99) + << ',' << durations.back() << ',' << deadline << ',' << overruns << ',' << sink + cache_sink * 0.0f << '\n'; +} +} // namespace + +int main(int argc, char** argv) +{ + const int sample_rate = 48000; + const double seconds = argc > 3 ? std::atof(argv[3]) : 2.0; + const nam::LinearImplementation implementation = + argc > 4 && std::string(argv[4]) == "direct" ? nam::LinearImplementation::Direct : nam::LinearImplementation::FFT; + const bool cold_cache = argc > 5 && std::string(argv[5]) == "cold"; + const bool verify = argc > 5 && std::string(argv[5]) == "verify"; + const std::vector taps = argc > 1 ? std::vector{std::atoi(argv[1])} + : std::vector{1024, 2048, 4096, 8192, 48000, 240000, 1200000, 2880000}; + const std::vector callbacks = argc > 2 ? std::vector{std::atoi(argv[2])} : std::vector{32, 64, 128}; + + if (verify) + return verify_impulse_response(taps.front(), callbacks.front(), sample_rate) ? 0 : 1; + + std::cout << "taps,callback,mean_us,p50_us,p95_us,p99_us,max_us,deadline_us,overruns,sink\n"; + for (const int tap_count : taps) + for (const int callback_size : callbacks) + run(tap_count, callback_size, sample_rate, seconds, implementation, cold_cache); + return 0; +} diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index db82922b..0c8cd932 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -100,6 +100,8 @@ int main() test_linear::test_direct_known_values(); test_linear::test_fft_matches_direct_irregular_chunks(); test_linear::test_auto_selection(); + test_linear::test_fft_dispatch_table(); + test_linear::test_fft_impulse_response_across_dispatch_sizes(); test_linear::test_parse_implementation(); test_linear::test_direct_process_realtime_safe(); test_linear::test_fft_process_realtime_safe(); diff --git a/tools/test/test_linear.cpp b/tools/test/test_linear.cpp index ada26e70..d087ab0f 100644 --- a/tools/test/test_linear.cpp +++ b/tools/test/test_linear.cpp @@ -140,17 +140,45 @@ void test_auto_selection() assert(short_model.GetRequestedImplementation() == nam::LinearImplementation::Auto); assert(short_model.GetActiveImplementation() == nam::LinearImplementation::Direct); - const auto cutoff_weights = make_weights(256, false); - nam::Linear cutoff_model(1, 1, 256, false, cutoff_weights, 48000.0); + const auto cutoff_weights = make_weights(1024, false); + nam::Linear cutoff_model(1, 1, 1024, false, cutoff_weights, 48000.0); assert(cutoff_model.GetRequestedImplementation() == nam::LinearImplementation::Auto); assert(cutoff_model.GetActiveImplementation() == nam::LinearImplementation::Direct); - const auto fft_weights = make_weights(512, false); - nam::Linear fft_model(1, 1, 512, false, fft_weights, 48000.0); + const auto fft_weights = make_weights(2048, false); + nam::Linear fft_model(1, 1, 2048, false, fft_weights, 48000.0); assert(fft_model.GetRequestedImplementation() == nam::LinearImplementation::Auto); assert(fft_model.GetActiveImplementation() == nam::LinearImplementation::FFT); } +void test_fft_dispatch_table() +{ + assert(nam::linear::select_implementation(1024) == nam::LinearImplementation::Direct); + assert(nam::linear::select_implementation(1025) == nam::LinearImplementation::FFT); + assert(nam::linear::select_fft_plan(1024).direct_taps == 128); + assert(nam::linear::select_fft_plan(8192).max_partition_size == 2048); + assert(nam::linear::select_fft_plan(48000).max_partition_size == 4096); + assert(nam::linear::select_fft_plan(240000).max_partition_size == 8192); + assert(nam::linear::select_fft_plan(2880000).max_partition_size == 8192); +} + +void test_fft_impulse_response_across_dispatch_sizes() +{ + const std::vector receptive_fields{1024, 2048, 4096, 8192, 48000}; + for (const int receptive_field : receptive_fields) + { + const auto weights = make_weights(receptive_field, false); + nam::Linear model(1, 1, receptive_field, false, weights, 48000.0, nam::LinearImplementation::FFT); + std::vector input(receptive_field + 257, 0.0); + input[0] = 1.0; + const auto output = process_model(model, input, {1, 17, 32, 63, 128, 511}); + for (int i = 0; i < receptive_field; ++i) + assert_near(output[i], weights[i], 5.0e-5); + for (size_t i = receptive_field; i < output.size(); ++i) + assert_near(output[i], 0.0, 5.0e-5); + } +} + void test_parse_implementation() { assert(nam::linear::parse_implementation("auto") == nam::LinearImplementation::Auto);