Michal Vrba
b784e91b22
Adds temporal voting queues and aggregation logic for note and chord detection, improving stability and reducing spurious results. Enhances chord detection with new harmonicity, peak ratio, and separation checks. Introduces HPS-based fundamental frequency estimation. Updates SensitivityProfile with new tuning parameters. All changes aim to make detection more reliable and musically meaningful.
1077 lines
34 KiB
C#
1077 lines
34 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.Numerics;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FractalVisualizer.Models;
|
|
using MathNet.Numerics.IntegralTransforms;
|
|
|
|
namespace FractalVisualizer.Audio;
|
|
|
|
public sealed class AudioAnalyzer : IDisposable
|
|
{
|
|
private readonly AudioCapture _capture;
|
|
private readonly int _fftSize;
|
|
private readonly int _spectrumBins;
|
|
private readonly float[] _hannWindow;
|
|
private readonly float[] _timeSamples;
|
|
private readonly Complex[] _fftBuffer;
|
|
private readonly float[] _spectrumAccumulator;
|
|
private readonly int[] _spectrumAccumulatorCounts;
|
|
private readonly float[] _chroma = new float[12];
|
|
private readonly float[] _noteVoteScores = new float[12];
|
|
private readonly float[] _chordVoteScores = new float[24];
|
|
private readonly float[][] _spectrumRefsByMode;
|
|
private readonly float[] _levelRefsByMode = new float[2];
|
|
private readonly float[] _bassRefsByMode = new float[2];
|
|
private readonly float[] _midRefsByMode = new float[2];
|
|
private readonly float[] _highRefsByMode = new float[2];
|
|
private readonly Stopwatch _clock = new();
|
|
|
|
private readonly object _lifecycleSync = new();
|
|
|
|
private CancellationTokenSource? _cts;
|
|
private Task? _analysisTask;
|
|
private int _sampleRate;
|
|
|
|
private float _smoothedLevel;
|
|
private float _smoothedBass;
|
|
private float _smoothedMid;
|
|
private float _smoothedHigh;
|
|
private float _smoothedBeat;
|
|
private float _energyBaseline;
|
|
private float _smoothedBpm;
|
|
private float _lastBeatTimeSeconds = -10f;
|
|
private bool _beatGateOpen = true;
|
|
private readonly Queue<float> _beatIntervals = new();
|
|
private readonly Queue<NoteVote> _noteVotes = new();
|
|
private readonly Queue<ChordVote> _chordVotes = new();
|
|
private const float TemporalVoteWindowSeconds = 0.30f;
|
|
private const int MaxBeatIntervals = 8;
|
|
private int _lastChordRoot = -1;
|
|
private int _lastChordType = -1;
|
|
private float _lastChordChangeTimeSeconds;
|
|
private int _stableNoteIndex = -1;
|
|
private float _stableNoteStrength;
|
|
private float _lastNoteChangeTimeSeconds;
|
|
private bool _noteIsActive;
|
|
private int _activeNoteIndex = -1;
|
|
|
|
private Snapshot _latestSnapshot;
|
|
private bool _disposed;
|
|
|
|
public AudioAnalyzer(AudioCapture capture, int fftSize = 2048, int spectrumBins = 64)
|
|
{
|
|
if (fftSize <= 0 || (fftSize & (fftSize - 1)) != 0)
|
|
{
|
|
throw new ArgumentOutOfRangeException(nameof(fftSize), "FFT size must be a power of 2.");
|
|
}
|
|
|
|
if (spectrumBins <= 0)
|
|
{
|
|
throw new ArgumentOutOfRangeException(nameof(spectrumBins));
|
|
}
|
|
|
|
_capture = capture ?? throw new ArgumentNullException(nameof(capture));
|
|
_fftSize = fftSize;
|
|
_spectrumBins = spectrumBins;
|
|
_sampleRate = capture.SampleRate;
|
|
|
|
_hannWindow = BuildHannWindow(fftSize);
|
|
_timeSamples = new float[fftSize];
|
|
_fftBuffer = new Complex[fftSize];
|
|
_spectrumAccumulator = new float[spectrumBins];
|
|
_spectrumAccumulatorCounts = new int[spectrumBins];
|
|
_spectrumRefsByMode =
|
|
[
|
|
new float[spectrumBins],
|
|
new float[spectrumBins]
|
|
];
|
|
_latestSnapshot = new Snapshot(AudioFeatures.CreateEmpty(spectrumBins));
|
|
|
|
_capture.SampleRateChanged += OnSampleRateChanged;
|
|
}
|
|
|
|
public int FftSize => _fftSize;
|
|
|
|
public int SpectrumBins => _spectrumBins;
|
|
|
|
public void Start()
|
|
{
|
|
ThrowIfDisposed();
|
|
|
|
lock (_lifecycleSync)
|
|
{
|
|
if (_analysisTask is not null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_clock.Restart();
|
|
_cts = new CancellationTokenSource();
|
|
_analysisTask = Task.Run(() => AnalysisLoop(_cts.Token), _cts.Token);
|
|
}
|
|
}
|
|
|
|
public void Stop()
|
|
{
|
|
Task? taskToWait;
|
|
CancellationTokenSource? ctsToDispose;
|
|
|
|
lock (_lifecycleSync)
|
|
{
|
|
if (_analysisTask is null || _cts is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
taskToWait = _analysisTask;
|
|
ctsToDispose = _cts;
|
|
_analysisTask = null;
|
|
_cts = null;
|
|
}
|
|
|
|
ctsToDispose.Cancel();
|
|
try
|
|
{
|
|
taskToWait.Wait();
|
|
}
|
|
catch (AggregateException ex) when (ex.InnerException is OperationCanceledException)
|
|
{
|
|
}
|
|
finally
|
|
{
|
|
ctsToDispose.Dispose();
|
|
}
|
|
}
|
|
|
|
public AudioFeatures GetLatestFeatures()
|
|
{
|
|
return Volatile.Read(ref _latestSnapshot).Features;
|
|
}
|
|
|
|
private void AnalysisLoop(CancellationToken cancellationToken)
|
|
{
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
if (!_capture.RingBuffer.TryRead(_timeSamples))
|
|
{
|
|
Thread.Sleep(2);
|
|
continue;
|
|
}
|
|
|
|
ProcessWindow();
|
|
}
|
|
}
|
|
|
|
private void ProcessWindow()
|
|
{
|
|
AudioInputMode mode = _capture.CurrentMode;
|
|
SensitivityProfile profile = mode == AudioInputMode.Microphone ? SensitivityProfile.Microphone : SensitivityProfile.SystemOutput;
|
|
int modeIndex = mode == AudioInputMode.Microphone ? 0 : 1;
|
|
int sampleRate = Volatile.Read(ref _sampleRate);
|
|
int half = _fftSize / 2;
|
|
|
|
for (int i = 0; i < _fftSize; i++)
|
|
{
|
|
_fftBuffer[i] = new Complex(_timeSamples[i] * _hannWindow[i], 0.0);
|
|
}
|
|
|
|
Fourier.Forward(_fftBuffer, FourierOptions.Matlab);
|
|
|
|
Array.Clear(_spectrumAccumulator);
|
|
Array.Clear(_spectrumAccumulatorCounts);
|
|
|
|
float rmsAccumulator = 0f;
|
|
for (int i = 0; i < _fftSize; i++)
|
|
{
|
|
float sample = _timeSamples[i];
|
|
rmsAccumulator += sample * sample;
|
|
}
|
|
|
|
float rms = MathF.Sqrt(rmsAccumulator / _fftSize);
|
|
|
|
float bassEnergy = 0f;
|
|
float midEnergy = 0f;
|
|
float highEnergy = 0f;
|
|
float dominantMagnitude = 0f;
|
|
float dominantFrequency = 0f;
|
|
Array.Clear(_chroma);
|
|
|
|
for (int bin = 1; bin < half; bin++)
|
|
{
|
|
float frequency = (sampleRate * bin) / (float)_fftSize;
|
|
float magnitude = (float)_fftBuffer[bin].Magnitude;
|
|
|
|
int spectrumIndex = (int)((bin / (float)half) * _spectrumBins);
|
|
if (spectrumIndex >= _spectrumBins)
|
|
{
|
|
spectrumIndex = _spectrumBins - 1;
|
|
}
|
|
|
|
_spectrumAccumulator[spectrumIndex] += magnitude;
|
|
_spectrumAccumulatorCounts[spectrumIndex]++;
|
|
|
|
if (frequency >= 20f && frequency < 200f)
|
|
{
|
|
bassEnergy += magnitude;
|
|
}
|
|
else if (frequency >= 200f && frequency < 2000f)
|
|
{
|
|
midEnergy += magnitude;
|
|
}
|
|
else if (frequency >= 2000f && frequency <= 16000f)
|
|
{
|
|
highEnergy += magnitude;
|
|
}
|
|
|
|
if (frequency >= 120f && frequency <= 6000f && magnitude > dominantMagnitude)
|
|
{
|
|
dominantMagnitude = magnitude;
|
|
dominantFrequency = frequency;
|
|
}
|
|
|
|
if (frequency >= 60f && frequency <= 5000f)
|
|
{
|
|
int pitchClass = GetNoteIndexFromFrequency(frequency);
|
|
if (pitchClass >= 0)
|
|
{
|
|
_chroma[pitchClass] += magnitude;
|
|
}
|
|
}
|
|
}
|
|
|
|
float[] spectrum = new float[_spectrumBins];
|
|
for (int i = 0; i < _spectrumBins; i++)
|
|
{
|
|
if (_spectrumAccumulatorCounts[i] == 0)
|
|
{
|
|
spectrum[i] = 0f;
|
|
continue;
|
|
}
|
|
|
|
float raw = _spectrumAccumulator[i] / _spectrumAccumulatorCounts[i];
|
|
spectrum[i] = NormalizeWithReference(raw, ref _spectrumRefsByMode[modeIndex][i], profile.SpectrumGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
|
|
}
|
|
|
|
float instantEnergy = rms * rms;
|
|
_energyBaseline = Lerp(_energyBaseline, instantEnergy, 0.02f);
|
|
float beat = 0f;
|
|
if (_energyBaseline > 1e-7f && instantEnergy > _energyBaseline * 1.5f)
|
|
{
|
|
beat = Math.Clamp((instantEnergy / _energyBaseline) - 1f, 0f, 1f);
|
|
}
|
|
|
|
float levelNorm = NormalizeWithReference(rms, ref _levelRefsByMode[modeIndex], profile.LevelGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
|
|
float bassNorm = NormalizeWithReference(bassEnergy, ref _bassRefsByMode[modeIndex], profile.BassGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
|
|
float midNorm = NormalizeWithReference(midEnergy, ref _midRefsByMode[modeIndex], profile.MidGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
|
|
float highNorm = NormalizeWithReference(highEnergy, ref _highRefsByMode[modeIndex], profile.HighGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
|
|
|
|
_smoothedLevel = Smooth(_smoothedLevel, levelNorm, 0.45f, 0.08f);
|
|
_smoothedBass = Smooth(_smoothedBass, bassNorm, 0.40f, 0.10f);
|
|
_smoothedMid = Smooth(_smoothedMid, midNorm, 0.30f, 0.08f);
|
|
_smoothedHigh = Smooth(_smoothedHigh, highNorm, 0.30f, 0.08f);
|
|
_smoothedBeat = MathF.Max(beat, _smoothedBeat * 0.85f);
|
|
|
|
float now = (float)_clock.Elapsed.TotalSeconds;
|
|
bool beatHit = beat > 0.62f && _beatGateOpen;
|
|
if (beatHit)
|
|
{
|
|
float delta = now - _lastBeatTimeSeconds;
|
|
if (delta > 0.22f && delta < 2.0f)
|
|
{
|
|
_beatIntervals.Enqueue(delta);
|
|
while (_beatIntervals.Count > MaxBeatIntervals)
|
|
{
|
|
_beatIntervals.Dequeue();
|
|
}
|
|
}
|
|
|
|
_lastBeatTimeSeconds = now;
|
|
_beatGateOpen = false;
|
|
}
|
|
else if (beat < 0.20f)
|
|
{
|
|
_beatGateOpen = true;
|
|
}
|
|
|
|
float bpmInstant = CalculateBpm();
|
|
_smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f);
|
|
|
|
EstimateFundamentalFrequencyHps(sampleRate, out float hpsFrequency, out float hpsMagnitude, out float harmonicity);
|
|
float noteFrequency = hpsFrequency > 0f ? hpsFrequency : dominantFrequency;
|
|
float noteMagnitude = hpsFrequency > 0f ? hpsMagnitude : dominantMagnitude;
|
|
int noteIndex = GetNoteIndexFromFrequency(noteFrequency);
|
|
float whistleBoost = GetWhistleBoost(noteFrequency);
|
|
float dominantStrength = Clamp01(noteMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f) * (0.58f + (0.42f * harmonicity)));
|
|
|
|
DetectChord(_chroma, _smoothedLevel, profile, harmonicity, out int chordRoot, out int chordType, out float chordConfidence);
|
|
ApplyTemporalVoting(
|
|
now,
|
|
profile,
|
|
noteIndex,
|
|
dominantStrength,
|
|
chordRoot,
|
|
chordType,
|
|
chordConfidence,
|
|
out int votedNoteIndex,
|
|
out float votedNoteStrength,
|
|
out int votedChordRoot,
|
|
out int votedChordType,
|
|
out float votedChordConfidence);
|
|
ApplyDetectionStability(
|
|
now,
|
|
profile,
|
|
votedNoteIndex,
|
|
votedNoteStrength,
|
|
votedChordRoot,
|
|
votedChordType,
|
|
votedChordConfidence,
|
|
out int stableNoteIndex,
|
|
out float stableNoteStrength,
|
|
out int stableChordRoot,
|
|
out int stableChordType,
|
|
out float stableChordConfidence);
|
|
bool noteOnset = DetectNoteOnset(stableNoteIndex, stableNoteStrength, profile);
|
|
|
|
AudioFeatures features = new()
|
|
{
|
|
Level = _smoothedLevel,
|
|
Bass = _smoothedBass,
|
|
Mid = _smoothedMid,
|
|
High = _smoothedHigh,
|
|
Beat = _smoothedBeat,
|
|
Bpm = _smoothedBpm,
|
|
DominantFrequencyHz = noteFrequency,
|
|
DetectedNoteIndex = stableNoteIndex,
|
|
DetectedNoteStrength = stableNoteStrength,
|
|
NoteOnset = noteOnset,
|
|
DetectedChordRootIndex = stableChordRoot,
|
|
DetectedChordType = stableChordType,
|
|
DetectedChordConfidence = stableChordConfidence,
|
|
Spectrum = spectrum,
|
|
TimeSeconds = now
|
|
};
|
|
|
|
Volatile.Write(ref _latestSnapshot, new Snapshot(features));
|
|
}
|
|
|
|
private void OnSampleRateChanged(int sampleRate)
|
|
{
|
|
Volatile.Write(ref _sampleRate, sampleRate);
|
|
}
|
|
|
|
private static float[] BuildHannWindow(int length)
|
|
{
|
|
float[] window = new float[length];
|
|
if (length == 1)
|
|
{
|
|
window[0] = 1f;
|
|
return window;
|
|
}
|
|
|
|
for (int i = 0; i < length; i++)
|
|
{
|
|
window[i] = 0.5f * (1f - MathF.Cos((2f * MathF.PI * i) / (length - 1)));
|
|
}
|
|
|
|
return window;
|
|
}
|
|
|
|
private static float Smooth(float current, float target, float attack, float release)
|
|
{
|
|
float factor = target > current ? attack : release;
|
|
return current + ((target - current) * factor);
|
|
}
|
|
|
|
private static float Lerp(float a, float b, float t)
|
|
{
|
|
return a + ((b - a) * t);
|
|
}
|
|
|
|
private float CalculateBpm()
|
|
{
|
|
if (_beatIntervals.Count == 0)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
float sum = 0f;
|
|
foreach (float interval in _beatIntervals)
|
|
{
|
|
sum += interval;
|
|
}
|
|
|
|
float avg = sum / _beatIntervals.Count;
|
|
if (avg <= 1e-5f)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
float bpm = 60f / avg;
|
|
return Math.Clamp(bpm, 40f, 220f);
|
|
}
|
|
|
|
private static int GetNoteIndexFromFrequency(float frequencyHz)
|
|
{
|
|
if (frequencyHz <= 0f)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
float midi = 69f + (12f * MathF.Log2(frequencyHz / 440f));
|
|
int note = ((int)MathF.Round(midi)) % 12;
|
|
if (note < 0)
|
|
{
|
|
note += 12;
|
|
}
|
|
|
|
return note;
|
|
}
|
|
|
|
private void DetectChord(float[] chroma, float level, SensitivityProfile profile, float harmonicity, out int root, out int type, out float confidence)
|
|
{
|
|
if (level < profile.ChordLevelThreshold || harmonicity < profile.ChordHarmonicityThreshold)
|
|
{
|
|
root = -1;
|
|
type = -1;
|
|
confidence = 0f;
|
|
return;
|
|
}
|
|
|
|
float total = 0f;
|
|
for (int i = 0; i < chroma.Length; i++)
|
|
{
|
|
total += chroma[i];
|
|
}
|
|
|
|
if (total <= 1e-6f)
|
|
{
|
|
root = -1;
|
|
type = -1;
|
|
confidence = 0f;
|
|
return;
|
|
}
|
|
|
|
float logSum = 0f;
|
|
float peak = 0f;
|
|
for (int i = 0; i < chroma.Length; i++)
|
|
{
|
|
float c = chroma[i];
|
|
if (c > peak)
|
|
{
|
|
peak = c;
|
|
}
|
|
|
|
logSum += MathF.Log(c + 1e-9f);
|
|
}
|
|
|
|
float arithmeticMean = total / chroma.Length;
|
|
float geometricMean = MathF.Exp(logSum / chroma.Length);
|
|
float flatness = geometricMean / (arithmeticMean + 1e-9f);
|
|
float chromaHarmonicity = Clamp01(1f - (flatness * 1.25f));
|
|
if (chromaHarmonicity < profile.ChordHarmonicityThreshold)
|
|
{
|
|
root = -1;
|
|
type = -1;
|
|
confidence = 0f;
|
|
return;
|
|
}
|
|
|
|
float peakRatio = peak / (total + 1e-6f);
|
|
if (peakRatio < profile.ChordPeakRatioThreshold)
|
|
{
|
|
root = -1;
|
|
type = -1;
|
|
confidence = 0f;
|
|
return;
|
|
}
|
|
|
|
float bestScore = 0f;
|
|
float secondBestScore = 0f;
|
|
int bestRoot = -1;
|
|
int bestType = -1;
|
|
for (int r = 0; r < 12; r++)
|
|
{
|
|
float major = ScoreChord(chroma, r, 0);
|
|
if (major > bestScore)
|
|
{
|
|
secondBestScore = bestScore;
|
|
bestScore = major;
|
|
bestRoot = r;
|
|
bestType = 0;
|
|
}
|
|
else if (major > secondBestScore)
|
|
{
|
|
secondBestScore = major;
|
|
}
|
|
|
|
float minor = ScoreChord(chroma, r, 1);
|
|
if (minor > bestScore)
|
|
{
|
|
secondBestScore = bestScore;
|
|
bestScore = minor;
|
|
bestRoot = r;
|
|
bestType = 1;
|
|
}
|
|
else if (minor > secondBestScore)
|
|
{
|
|
secondBestScore = minor;
|
|
}
|
|
}
|
|
|
|
float conf = Clamp01(bestScore / (total + 1e-6f));
|
|
float separation = (bestScore - secondBestScore) / (total + 1e-6f);
|
|
if (conf < 0.17f || separation < profile.ChordSeparationThreshold)
|
|
{
|
|
root = -1;
|
|
type = -1;
|
|
confidence = 0f;
|
|
return;
|
|
}
|
|
|
|
if (_lastChordRoot >= 0 && _lastChordType >= 0)
|
|
{
|
|
float held = ScoreChord(chroma, _lastChordRoot, _lastChordType);
|
|
if ((bestRoot != _lastChordRoot || bestType != _lastChordType) && held * 1.08f > bestScore)
|
|
{
|
|
bestRoot = _lastChordRoot;
|
|
bestType = _lastChordType;
|
|
bestScore = held;
|
|
conf = Clamp01(bestScore / (total + 1e-6f));
|
|
}
|
|
}
|
|
|
|
root = bestRoot;
|
|
type = bestType;
|
|
confidence = Clamp01(conf * (0.55f + (0.45f * harmonicity)));
|
|
}
|
|
|
|
private void EstimateFundamentalFrequencyHps(int sampleRate, out float frequencyHz, out float magnitude, out float harmonicity)
|
|
{
|
|
frequencyHz = 0f;
|
|
magnitude = 0f;
|
|
harmonicity = 0f;
|
|
|
|
int half = _fftSize / 2;
|
|
int minBin = Math.Max(2, (int)(70f * _fftSize / sampleRate));
|
|
int maxBinByFrequency = Math.Min(half - 1, (int)(4200f * _fftSize / sampleRate));
|
|
int maxBin = Math.Min(maxBinByFrequency, (half / 4) - 1);
|
|
if (maxBin <= minBin)
|
|
{
|
|
return;
|
|
}
|
|
|
|
float bestLogScore = float.NegativeInfinity;
|
|
int bestBin = -1;
|
|
const float epsilon = 1e-12f;
|
|
for (int bin = minBin; bin <= maxBin; bin++)
|
|
{
|
|
float m1 = (float)_fftBuffer[bin].Magnitude;
|
|
float m2 = (float)_fftBuffer[bin * 2].Magnitude;
|
|
float m3 = (float)_fftBuffer[bin * 3].Magnitude;
|
|
float m4 = (float)_fftBuffer[bin * 4].Magnitude;
|
|
float logScore =
|
|
MathF.Log(m1 + epsilon) +
|
|
(0.90f * MathF.Log(m2 + epsilon)) +
|
|
(0.72f * MathF.Log(m3 + epsilon)) +
|
|
(0.55f * MathF.Log(m4 + epsilon));
|
|
if (logScore > bestLogScore)
|
|
{
|
|
bestLogScore = logScore;
|
|
bestBin = bin;
|
|
}
|
|
}
|
|
|
|
if (bestBin < 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
float fundamental = (float)_fftBuffer[bestBin].Magnitude;
|
|
float second = (float)_fftBuffer[bestBin * 2].Magnitude;
|
|
float third = (float)_fftBuffer[bestBin * 3].Magnitude;
|
|
float fourth = (float)_fftBuffer[bestBin * 4].Magnitude;
|
|
|
|
frequencyHz = (sampleRate * bestBin) / (float)_fftSize;
|
|
magnitude = fundamental;
|
|
harmonicity = Clamp01(((second + third + fourth) / 3f) / (fundamental + 1e-6f));
|
|
}
|
|
|
|
private static float ScoreChord(float[] chroma, int root, int type)
|
|
{
|
|
int third = type == 0 ? (root + 4) % 12 : (root + 3) % 12;
|
|
int fifth = (root + 7) % 12;
|
|
int second = (root + 2) % 12;
|
|
int fourth = (root + 5) % 12;
|
|
float score = chroma[root] * 1.0f + chroma[third] * 0.9f + chroma[fifth] * 0.8f;
|
|
score -= (chroma[second] + chroma[fourth]) * 0.15f;
|
|
return Math.Max(0f, score);
|
|
}
|
|
|
|
private static float Clamp01(float value)
|
|
{
|
|
if (value < 0f)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
if (value > 1f)
|
|
{
|
|
return 1f;
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
private static float GetWhistleBoost(float frequencyHz)
|
|
{
|
|
if (frequencyHz <= 0f)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
// Human whistles usually sit around ~1kHz-4kHz, with strong note clarity in this band.
|
|
if (frequencyHz < 800f)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
if (frequencyHz > 5000f)
|
|
{
|
|
return 0f;
|
|
}
|
|
|
|
float centered = (frequencyHz - 800f) / (5000f - 800f);
|
|
return Clamp01(centered);
|
|
}
|
|
|
|
private static float NormalizeWithReference(float value, ref float reference, float gain, float noiseFloor, float referenceRise, float referenceDecay)
|
|
{
|
|
float v = MathF.Max(0f, value - noiseFloor);
|
|
if (v > reference)
|
|
{
|
|
reference = Lerp(reference, v, referenceRise);
|
|
}
|
|
else
|
|
{
|
|
reference = Lerp(reference, v, referenceDecay);
|
|
}
|
|
|
|
if (reference < 1e-6f)
|
|
{
|
|
reference = 1e-6f;
|
|
}
|
|
|
|
return Clamp01((v / reference) * gain);
|
|
}
|
|
|
|
private void ApplyTemporalVoting(
|
|
float now,
|
|
SensitivityProfile profile,
|
|
int noteIndex,
|
|
float noteStrength,
|
|
int chordRoot,
|
|
int chordType,
|
|
float chordConfidence,
|
|
out int votedNoteIndex,
|
|
out float votedNoteStrength,
|
|
out int votedChordRoot,
|
|
out int votedChordType,
|
|
out float votedChordConfidence)
|
|
{
|
|
float noteWeight = noteIndex >= 0 ? MathF.Max(0f, noteStrength) : profile.SilenceVoteWeight;
|
|
_noteVotes.Enqueue(new NoteVote(now, noteIndex, noteWeight));
|
|
PruneOldVotes(_noteVotes, now);
|
|
ResolveNoteVote(profile, out votedNoteIndex, out votedNoteStrength);
|
|
|
|
bool hasChordCandidate = chordRoot >= 0 && chordConfidence >= (profile.ChordConfidenceThreshold * 0.55f);
|
|
float chordWeight = hasChordCandidate ? chordConfidence : profile.SilenceVoteWeight;
|
|
_chordVotes.Enqueue(new ChordVote(now, hasChordCandidate ? chordRoot : -1, hasChordCandidate ? chordType : -1, chordWeight));
|
|
PruneOldVotes(_chordVotes, now);
|
|
ResolveChordVote(profile, out votedChordRoot, out votedChordType, out votedChordConfidence);
|
|
}
|
|
|
|
private void ResolveNoteVote(SensitivityProfile profile, out int noteIndex, out float noteStrength)
|
|
{
|
|
Array.Clear(_noteVoteScores);
|
|
float totalWeight = 0f;
|
|
float silenceWeight = 0f;
|
|
|
|
foreach (NoteVote vote in _noteVotes)
|
|
{
|
|
float weight = MathF.Max(0f, vote.Weight);
|
|
totalWeight += weight;
|
|
if (vote.NoteIndex >= 0)
|
|
{
|
|
_noteVoteScores[vote.NoteIndex] += weight;
|
|
}
|
|
else
|
|
{
|
|
silenceWeight += weight;
|
|
}
|
|
}
|
|
|
|
if (totalWeight <= 1e-6f)
|
|
{
|
|
noteIndex = -1;
|
|
noteStrength = 0f;
|
|
return;
|
|
}
|
|
|
|
float best = 0f;
|
|
float second = 0f;
|
|
int bestIndex = -1;
|
|
for (int i = 0; i < _noteVoteScores.Length; i++)
|
|
{
|
|
float score = _noteVoteScores[i];
|
|
if (score > best)
|
|
{
|
|
second = best;
|
|
best = score;
|
|
bestIndex = i;
|
|
}
|
|
else if (score > second)
|
|
{
|
|
second = score;
|
|
}
|
|
}
|
|
|
|
float consensus = best / (totalWeight + 1e-6f);
|
|
float separation = (best - second) / (totalWeight + 1e-6f);
|
|
if (bestIndex < 0 ||
|
|
best < profile.NoteVoteMinScore ||
|
|
consensus < profile.NoteVoteConsensus ||
|
|
separation < profile.NoteVoteSeparation ||
|
|
silenceWeight > best * 1.08f)
|
|
{
|
|
noteIndex = -1;
|
|
noteStrength = 0f;
|
|
return;
|
|
}
|
|
|
|
noteIndex = bestIndex;
|
|
noteStrength = Clamp01((consensus * 0.72f) + (separation * 0.28f));
|
|
}
|
|
|
|
private void ResolveChordVote(SensitivityProfile profile, out int chordRoot, out int chordType, out float chordConfidence)
|
|
{
|
|
Array.Clear(_chordVoteScores);
|
|
float totalWeight = 0f;
|
|
float silenceWeight = 0f;
|
|
|
|
foreach (ChordVote vote in _chordVotes)
|
|
{
|
|
float weight = MathF.Max(0f, vote.Weight);
|
|
totalWeight += weight;
|
|
if (vote.Root >= 0 && vote.Type >= 0)
|
|
{
|
|
int idx = (vote.Root * 2) + vote.Type;
|
|
_chordVoteScores[idx] += weight;
|
|
}
|
|
else
|
|
{
|
|
silenceWeight += weight;
|
|
}
|
|
}
|
|
|
|
if (totalWeight <= 1e-6f)
|
|
{
|
|
chordRoot = -1;
|
|
chordType = -1;
|
|
chordConfidence = 0f;
|
|
return;
|
|
}
|
|
|
|
float best = 0f;
|
|
float second = 0f;
|
|
int bestIndex = -1;
|
|
for (int i = 0; i < _chordVoteScores.Length; i++)
|
|
{
|
|
float score = _chordVoteScores[i];
|
|
if (score > best)
|
|
{
|
|
second = best;
|
|
best = score;
|
|
bestIndex = i;
|
|
}
|
|
else if (score > second)
|
|
{
|
|
second = score;
|
|
}
|
|
}
|
|
|
|
float consensus = best / (totalWeight + 1e-6f);
|
|
float separation = (best - second) / (totalWeight + 1e-6f);
|
|
if (bestIndex < 0 ||
|
|
best < profile.ChordVoteMinScore ||
|
|
consensus < profile.ChordVoteConsensus ||
|
|
separation < profile.ChordVoteSeparation ||
|
|
silenceWeight > best * 1.10f)
|
|
{
|
|
chordRoot = -1;
|
|
chordType = -1;
|
|
chordConfidence = 0f;
|
|
return;
|
|
}
|
|
|
|
chordRoot = bestIndex / 2;
|
|
chordType = bestIndex % 2;
|
|
chordConfidence = Clamp01((consensus * 0.75f) + (separation * 0.25f));
|
|
}
|
|
|
|
private void PruneOldVotes(Queue<NoteVote> votes, float now)
|
|
{
|
|
float cutoff = now - TemporalVoteWindowSeconds;
|
|
while (votes.Count > 0 && votes.Peek().Time < cutoff)
|
|
{
|
|
votes.Dequeue();
|
|
}
|
|
}
|
|
|
|
private void PruneOldVotes(Queue<ChordVote> votes, float now)
|
|
{
|
|
float cutoff = now - TemporalVoteWindowSeconds;
|
|
while (votes.Count > 0 && votes.Peek().Time < cutoff)
|
|
{
|
|
votes.Dequeue();
|
|
}
|
|
}
|
|
|
|
private void ApplyDetectionStability(
|
|
float now,
|
|
SensitivityProfile profile,
|
|
int noteIndex,
|
|
float noteStrength,
|
|
int chordRoot,
|
|
int chordType,
|
|
float chordConfidence,
|
|
out int stableNoteIndex,
|
|
out float stableNoteStrength,
|
|
out int stableChordRoot,
|
|
out int stableChordType,
|
|
out float stableChordConfidence)
|
|
{
|
|
stableNoteIndex = _stableNoteIndex;
|
|
stableNoteStrength = _stableNoteStrength * 0.96f;
|
|
|
|
if (noteIndex >= 0 && noteStrength >= profile.NoteConfidenceThreshold)
|
|
{
|
|
bool changed = _stableNoteIndex != noteIndex;
|
|
bool canSwitch = (now - _lastNoteChangeTimeSeconds) >= profile.SwitchHoldSeconds;
|
|
if (!changed || canSwitch || noteStrength > _stableNoteStrength * 1.25f)
|
|
{
|
|
stableNoteIndex = noteIndex;
|
|
stableNoteStrength = noteStrength;
|
|
if (changed)
|
|
{
|
|
_lastNoteChangeTimeSeconds = now;
|
|
}
|
|
}
|
|
}
|
|
|
|
_stableNoteIndex = stableNoteIndex;
|
|
_stableNoteStrength = stableNoteStrength;
|
|
|
|
stableChordRoot = _lastChordRoot;
|
|
stableChordType = _lastChordType;
|
|
stableChordConfidence = 0f;
|
|
if (stableChordRoot >= 0)
|
|
{
|
|
stableChordConfidence = Clamp01(chordConfidence * 0.85f);
|
|
}
|
|
|
|
if (chordRoot >= 0 && chordConfidence >= profile.ChordConfidenceThreshold)
|
|
{
|
|
bool changed = _lastChordRoot != chordRoot || _lastChordType != chordType;
|
|
bool canSwitch = (now - _lastChordChangeTimeSeconds) >= profile.SwitchHoldSeconds;
|
|
if (!changed || canSwitch || chordConfidence > stableChordConfidence * 1.2f)
|
|
{
|
|
_lastChordRoot = chordRoot;
|
|
_lastChordType = chordType;
|
|
if (changed)
|
|
{
|
|
_lastChordChangeTimeSeconds = now;
|
|
}
|
|
}
|
|
|
|
stableChordRoot = _lastChordRoot;
|
|
stableChordType = _lastChordType;
|
|
stableChordConfidence = chordConfidence;
|
|
}
|
|
else if (_lastChordRoot >= 0 && (_smoothedLevel <= profile.ChordLevelThreshold || stableNoteStrength <= profile.NoteReleaseThreshold))
|
|
{
|
|
// Clear stale chord label when audio falls back to near-silence.
|
|
if ((now - _lastChordChangeTimeSeconds) >= profile.SwitchHoldSeconds)
|
|
{
|
|
_lastChordRoot = -1;
|
|
_lastChordType = -1;
|
|
stableChordRoot = -1;
|
|
stableChordType = -1;
|
|
stableChordConfidence = 0f;
|
|
}
|
|
}
|
|
}
|
|
|
|
private bool DetectNoteOnset(int stableNoteIndex, float stableNoteStrength, SensitivityProfile profile)
|
|
{
|
|
if (stableNoteIndex >= 0 && stableNoteStrength >= profile.NoteOnsetThreshold)
|
|
{
|
|
if (!_noteIsActive || _activeNoteIndex != stableNoteIndex)
|
|
{
|
|
_noteIsActive = true;
|
|
_activeNoteIndex = stableNoteIndex;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
if (_noteIsActive && stableNoteStrength <= profile.NoteReleaseThreshold)
|
|
{
|
|
_noteIsActive = false;
|
|
_activeNoteIndex = -1;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private readonly struct SensitivityProfile(
|
|
float levelGain,
|
|
float bassGain,
|
|
float midGain,
|
|
float highGain,
|
|
float spectrumGain,
|
|
float noteStrengthGain,
|
|
float noiseFloor,
|
|
float referenceRise,
|
|
float referenceDecay,
|
|
float switchHoldSeconds,
|
|
float noteConfidenceThreshold,
|
|
float chordConfidenceThreshold,
|
|
float noteOnsetThreshold,
|
|
float noteReleaseThreshold,
|
|
float chordLevelThreshold,
|
|
float chordHarmonicityThreshold,
|
|
float chordPeakRatioThreshold,
|
|
float chordSeparationThreshold,
|
|
float silenceVoteWeight,
|
|
float noteVoteMinScore,
|
|
float noteVoteConsensus,
|
|
float noteVoteSeparation,
|
|
float chordVoteMinScore,
|
|
float chordVoteConsensus,
|
|
float chordVoteSeparation)
|
|
{
|
|
public float LevelGain { get; } = levelGain;
|
|
public float BassGain { get; } = bassGain;
|
|
public float MidGain { get; } = midGain;
|
|
public float HighGain { get; } = highGain;
|
|
public float SpectrumGain { get; } = spectrumGain;
|
|
public float NoteStrengthGain { get; } = noteStrengthGain;
|
|
public float NoiseFloor { get; } = noiseFloor;
|
|
public float ReferenceRise { get; } = referenceRise;
|
|
public float ReferenceDecay { get; } = referenceDecay;
|
|
public float SwitchHoldSeconds { get; } = switchHoldSeconds;
|
|
public float NoteConfidenceThreshold { get; } = noteConfidenceThreshold;
|
|
public float ChordConfidenceThreshold { get; } = chordConfidenceThreshold;
|
|
public float NoteOnsetThreshold { get; } = noteOnsetThreshold;
|
|
public float NoteReleaseThreshold { get; } = noteReleaseThreshold;
|
|
public float ChordLevelThreshold { get; } = chordLevelThreshold;
|
|
public float ChordHarmonicityThreshold { get; } = chordHarmonicityThreshold;
|
|
public float ChordPeakRatioThreshold { get; } = chordPeakRatioThreshold;
|
|
public float ChordSeparationThreshold { get; } = chordSeparationThreshold;
|
|
public float SilenceVoteWeight { get; } = silenceVoteWeight;
|
|
public float NoteVoteMinScore { get; } = noteVoteMinScore;
|
|
public float NoteVoteConsensus { get; } = noteVoteConsensus;
|
|
public float NoteVoteSeparation { get; } = noteVoteSeparation;
|
|
public float ChordVoteMinScore { get; } = chordVoteMinScore;
|
|
public float ChordVoteConsensus { get; } = chordVoteConsensus;
|
|
public float ChordVoteSeparation { get; } = chordVoteSeparation;
|
|
|
|
public static SensitivityProfile Microphone { get; } = new(
|
|
levelGain: 1.20f,
|
|
bassGain: 1.25f,
|
|
midGain: 1.20f,
|
|
highGain: 1.15f,
|
|
spectrumGain: 1.40f,
|
|
noteStrengthGain: 0.036f,
|
|
noiseFloor: 0.0009f,
|
|
referenceRise: 0.16f,
|
|
referenceDecay: 0.010f,
|
|
switchHoldSeconds: 0.18f,
|
|
noteConfidenceThreshold: 0.11f,
|
|
chordConfidenceThreshold: 0.18f,
|
|
noteOnsetThreshold: 0.20f,
|
|
noteReleaseThreshold: 0.08f,
|
|
chordLevelThreshold: 0.10f,
|
|
chordHarmonicityThreshold: 0.24f,
|
|
chordPeakRatioThreshold: 0.12f,
|
|
chordSeparationThreshold: 0.040f,
|
|
silenceVoteWeight: 0.055f,
|
|
noteVoteMinScore: 0.24f,
|
|
noteVoteConsensus: 0.30f,
|
|
noteVoteSeparation: 0.08f,
|
|
chordVoteMinScore: 0.26f,
|
|
chordVoteConsensus: 0.32f,
|
|
chordVoteSeparation: 0.070f);
|
|
|
|
public static SensitivityProfile SystemOutput { get; } = new(
|
|
levelGain: 1.00f,
|
|
bassGain: 1.00f,
|
|
midGain: 1.00f,
|
|
highGain: 1.00f,
|
|
spectrumGain: 1.18f,
|
|
noteStrengthGain: 0.022f,
|
|
noiseFloor: 0.0004f,
|
|
referenceRise: 0.20f,
|
|
referenceDecay: 0.012f,
|
|
switchHoldSeconds: 0.18f,
|
|
noteConfidenceThreshold: 0.12f,
|
|
chordConfidenceThreshold: 0.14f,
|
|
noteOnsetThreshold: 0.18f,
|
|
noteReleaseThreshold: 0.07f,
|
|
chordLevelThreshold: 0.08f,
|
|
chordHarmonicityThreshold: 0.18f,
|
|
chordPeakRatioThreshold: 0.10f,
|
|
chordSeparationThreshold: 0.030f,
|
|
silenceVoteWeight: 0.050f,
|
|
noteVoteMinScore: 0.20f,
|
|
noteVoteConsensus: 0.26f,
|
|
noteVoteSeparation: 0.06f,
|
|
chordVoteMinScore: 0.22f,
|
|
chordVoteConsensus: 0.28f,
|
|
chordVoteSeparation: 0.055f);
|
|
}
|
|
|
|
private readonly record struct NoteVote(float Time, int NoteIndex, float Weight);
|
|
|
|
private readonly record struct ChordVote(float Time, int Root, int Type, float Weight);
|
|
|
|
private void ThrowIfDisposed()
|
|
{
|
|
if (_disposed)
|
|
{
|
|
throw new ObjectDisposedException(nameof(AudioAnalyzer));
|
|
}
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
if (_disposed)
|
|
{
|
|
return;
|
|
}
|
|
|
|
Stop();
|
|
_capture.SampleRateChanged -= OnSampleRateChanged;
|
|
_disposed = true;
|
|
}
|
|
|
|
private sealed class Snapshot(AudioFeatures features)
|
|
{
|
|
public AudioFeatures Features { get; } = features;
|
|
}
|
|
}
|