Temporal voting for robust note/chord detection

Adds temporal voting queues and aggregation logic for note and chord detection, improving stability and reducing spurious results. Enhances chord detection with new harmonicity, peak ratio, and separation checks. Introduces HPS-based fundamental frequency estimation. Updates SensitivityProfile with new tuning parameters. All changes aim to make detection more reliable and musically meaningful.
2026-02-13 21:31:06 +01:00
parent 11a4311bc4
commit b784e91b22
1 changed files with 349 additions and 14 deletions
@@ -20,6 +20,8 @@ public sealed class AudioAnalyzer : IDisposable
    private readonly float[] _spectrumAccumulator;
    private readonly int[] _spectrumAccumulatorCounts;
    private readonly float[] _chroma = new float[12];
+    private readonly float[] _noteVoteScores = new float[12];
+    private readonly float[] _chordVoteScores = new float[24];
    private readonly float[][] _spectrumRefsByMode;
    private readonly float[] _levelRefsByMode = new float[2];
    private readonly float[] _bassRefsByMode = new float[2];
@@ -43,6 +45,9 @@ public sealed class AudioAnalyzer : IDisposable
    private float _lastBeatTimeSeconds = -10f;
    private bool _beatGateOpen = true;
    private readonly Queue<float> _beatIntervals = new();
+    private readonly Queue<NoteVote> _noteVotes = new();
+    private readonly Queue<ChordVote> _chordVotes = new();
+    private const float TemporalVoteWindowSeconds = 0.30f;
    private const int MaxBeatIntervals = 8;
    private int _lastChordRoot = -1;
    private int _lastChordType = -1;
@@ -294,11 +299,15 @@ public sealed class AudioAnalyzer : IDisposable
        float bpmInstant = CalculateBpm();
        _smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f);

-        int noteIndex = GetNoteIndexFromFrequency(dominantFrequency);
-        float whistleBoost = GetWhistleBoost(dominantFrequency);
-        float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f));
-        DetectChord(_chroma, _smoothedLevel, profile, out int chordRoot, out int chordType, out float chordConfidence);
-        ApplyDetectionStability(
+        EstimateFundamentalFrequencyHps(sampleRate, out float hpsFrequency, out float hpsMagnitude, out float harmonicity);
+        float noteFrequency = hpsFrequency > 0f ? hpsFrequency : dominantFrequency;
+        float noteMagnitude = hpsFrequency > 0f ? hpsMagnitude : dominantMagnitude;
+        int noteIndex = GetNoteIndexFromFrequency(noteFrequency);
+        float whistleBoost = GetWhistleBoost(noteFrequency);
+        float dominantStrength = Clamp01(noteMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f) * (0.58f + (0.42f * harmonicity)));
+
+        DetectChord(_chroma, _smoothedLevel, profile, harmonicity, out int chordRoot, out int chordType, out float chordConfidence);
+        ApplyTemporalVoting(
            now,
            profile,
            noteIndex,
@@ -306,6 +315,19 @@ public sealed class AudioAnalyzer : IDisposable
            chordRoot,
            chordType,
            chordConfidence,
+            out int votedNoteIndex,
+            out float votedNoteStrength,
+            out int votedChordRoot,
+            out int votedChordType,
+            out float votedChordConfidence);
+        ApplyDetectionStability(
+            now,
+            profile,
+            votedNoteIndex,
+            votedNoteStrength,
+            votedChordRoot,
+            votedChordType,
+            votedChordConfidence,
            out int stableNoteIndex,
            out float stableNoteStrength,
            out int stableChordRoot,
@@ -321,7 +343,7 @@ public sealed class AudioAnalyzer : IDisposable
            High = _smoothedHigh,
            Beat = _smoothedBeat,
            Bpm = _smoothedBpm,
-            DominantFrequencyHz = dominantFrequency,
+            DominantFrequencyHz = noteFrequency,
            DetectedNoteIndex = stableNoteIndex,
            DetectedNoteStrength = stableNoteStrength,
            NoteOnset = noteOnset,
@@ -408,9 +430,9 @@ public sealed class AudioAnalyzer : IDisposable
        return note;
    }

-    private void DetectChord(float[] chroma, float level, SensitivityProfile profile, out int root, out int type, out float confidence)
+    private void DetectChord(float[] chroma, float level, SensitivityProfile profile, float harmonicity, out int root, out int type, out float confidence)
    {
-        if (level < profile.ChordLevelThreshold)
+        if (level < profile.ChordLevelThreshold || harmonicity < profile.ChordHarmonicityThreshold)
        {
            root = -1;
            type = -1;
@@ -432,7 +454,42 @@ public sealed class AudioAnalyzer : IDisposable
            return;
        }

+        float logSum = 0f;
+        float peak = 0f;
+        for (int i = 0; i < chroma.Length; i++)
+        {
+            float c = chroma[i];
+            if (c > peak)
+            {
+                peak = c;
+            }
+
+            logSum += MathF.Log(c + 1e-9f);
+        }
+
+        float arithmeticMean = total / chroma.Length;
+        float geometricMean = MathF.Exp(logSum / chroma.Length);
+        float flatness = geometricMean / (arithmeticMean + 1e-9f);
+        float chromaHarmonicity = Clamp01(1f - (flatness * 1.25f));
+        if (chromaHarmonicity < profile.ChordHarmonicityThreshold)
+        {
+            root = -1;
+            type = -1;
+            confidence = 0f;
+            return;
+        }
+
+        float peakRatio = peak / (total + 1e-6f);
+        if (peakRatio < profile.ChordPeakRatioThreshold)
+        {
+            root = -1;
+            type = -1;
+            confidence = 0f;
+            return;
+        }
+
        float bestScore = 0f;
+        float secondBestScore = 0f;
        int bestRoot = -1;
        int bestType = -1;
        for (int r = 0; r < 12; r++)
@@ -440,26 +497,37 @@ public sealed class AudioAnalyzer : IDisposable
            float major = ScoreChord(chroma, r, 0);
            if (major > bestScore)
            {
+                secondBestScore = bestScore;
                bestScore = major;
                bestRoot = r;
                bestType = 0;
            }
+            else if (major > secondBestScore)
+            {
+                secondBestScore = major;
+            }

            float minor = ScoreChord(chroma, r, 1);
            if (minor > bestScore)
            {
+                secondBestScore = bestScore;
                bestScore = minor;
                bestRoot = r;
                bestType = 1;
            }
+            else if (minor > secondBestScore)
+            {
+                secondBestScore = minor;
+            }
        }

        float conf = Clamp01(bestScore / (total + 1e-6f));
-        if (conf < 0.17f)
+        float separation = (bestScore - secondBestScore) / (total + 1e-6f);
+        if (conf < 0.17f || separation < profile.ChordSeparationThreshold)
        {
            root = -1;
            type = -1;
-            confidence = conf;
+            confidence = 0f;
            return;
        }

@@ -477,7 +545,58 @@ public sealed class AudioAnalyzer : IDisposable

        root = bestRoot;
        type = bestType;
-        confidence = conf;
+        confidence = Clamp01(conf * (0.55f + (0.45f * harmonicity)));
+    }
+
+    private void EstimateFundamentalFrequencyHps(int sampleRate, out float frequencyHz, out float magnitude, out float harmonicity)
+    {
+        frequencyHz = 0f;
+        magnitude = 0f;
+        harmonicity = 0f;
+
+        int half = _fftSize / 2;
+        int minBin = Math.Max(2, (int)(70f * _fftSize / sampleRate));
+        int maxBinByFrequency = Math.Min(half - 1, (int)(4200f * _fftSize / sampleRate));
+        int maxBin = Math.Min(maxBinByFrequency, (half / 4) - 1);
+        if (maxBin <= minBin)
+        {
+            return;
+        }
+
+        float bestLogScore = float.NegativeInfinity;
+        int bestBin = -1;
+        const float epsilon = 1e-12f;
+        for (int bin = minBin; bin <= maxBin; bin++)
+        {
+            float m1 = (float)_fftBuffer[bin].Magnitude;
+            float m2 = (float)_fftBuffer[bin * 2].Magnitude;
+            float m3 = (float)_fftBuffer[bin * 3].Magnitude;
+            float m4 = (float)_fftBuffer[bin * 4].Magnitude;
+            float logScore =
+                MathF.Log(m1 + epsilon) +
+                (0.90f * MathF.Log(m2 + epsilon)) +
+                (0.72f * MathF.Log(m3 + epsilon)) +
+                (0.55f * MathF.Log(m4 + epsilon));
+            if (logScore > bestLogScore)
+            {
+                bestLogScore = logScore;
+                bestBin = bin;
+            }
+        }
+
+        if (bestBin < 0)
+        {
+            return;
+        }
+
+        float fundamental = (float)_fftBuffer[bestBin].Magnitude;
+        float second = (float)_fftBuffer[bestBin * 2].Magnitude;
+        float third = (float)_fftBuffer[bestBin * 3].Magnitude;
+        float fourth = (float)_fftBuffer[bestBin * 4].Magnitude;
+
+        frequencyHz = (sampleRate * bestBin) / (float)_fftSize;
+        magnitude = fundamental;
+        harmonicity = Clamp01(((second + third + fourth) / 3f) / (fundamental + 1e-6f));
    }

    private static float ScoreChord(float[] chroma, int root, int type)
@@ -548,6 +667,178 @@ public sealed class AudioAnalyzer : IDisposable
        return Clamp01((v / reference) * gain);
    }

+    private void ApplyTemporalVoting(
+        float now,
+        SensitivityProfile profile,
+        int noteIndex,
+        float noteStrength,
+        int chordRoot,
+        int chordType,
+        float chordConfidence,
+        out int votedNoteIndex,
+        out float votedNoteStrength,
+        out int votedChordRoot,
+        out int votedChordType,
+        out float votedChordConfidence)
+    {
+        float noteWeight = noteIndex >= 0 ? MathF.Max(0f, noteStrength) : profile.SilenceVoteWeight;
+        _noteVotes.Enqueue(new NoteVote(now, noteIndex, noteWeight));
+        PruneOldVotes(_noteVotes, now);
+        ResolveNoteVote(profile, out votedNoteIndex, out votedNoteStrength);
+
+        bool hasChordCandidate = chordRoot >= 0 && chordConfidence >= (profile.ChordConfidenceThreshold * 0.55f);
+        float chordWeight = hasChordCandidate ? chordConfidence : profile.SilenceVoteWeight;
+        _chordVotes.Enqueue(new ChordVote(now, hasChordCandidate ? chordRoot : -1, hasChordCandidate ? chordType : -1, chordWeight));
+        PruneOldVotes(_chordVotes, now);
+        ResolveChordVote(profile, out votedChordRoot, out votedChordType, out votedChordConfidence);
+    }
+
+    private void ResolveNoteVote(SensitivityProfile profile, out int noteIndex, out float noteStrength)
+    {
+        Array.Clear(_noteVoteScores);
+        float totalWeight = 0f;
+        float silenceWeight = 0f;
+
+        foreach (NoteVote vote in _noteVotes)
+        {
+            float weight = MathF.Max(0f, vote.Weight);
+            totalWeight += weight;
+            if (vote.NoteIndex >= 0)
+            {
+                _noteVoteScores[vote.NoteIndex] += weight;
+            }
+            else
+            {
+                silenceWeight += weight;
+            }
+        }
+
+        if (totalWeight <= 1e-6f)
+        {
+            noteIndex = -1;
+            noteStrength = 0f;
+            return;
+        }
+
+        float best = 0f;
+        float second = 0f;
+        int bestIndex = -1;
+        for (int i = 0; i < _noteVoteScores.Length; i++)
+        {
+            float score = _noteVoteScores[i];
+            if (score > best)
+            {
+                second = best;
+                best = score;
+                bestIndex = i;
+            }
+            else if (score > second)
+            {
+                second = score;
+            }
+        }
+
+        float consensus = best / (totalWeight + 1e-6f);
+        float separation = (best - second) / (totalWeight + 1e-6f);
+        if (bestIndex < 0 ||
+            best < profile.NoteVoteMinScore ||
+            consensus < profile.NoteVoteConsensus ||
+            separation < profile.NoteVoteSeparation ||
+            silenceWeight > best * 1.08f)
+        {
+            noteIndex = -1;
+            noteStrength = 0f;
+            return;
+        }
+
+        noteIndex = bestIndex;
+        noteStrength = Clamp01((consensus * 0.72f) + (separation * 0.28f));
+    }
+
+    private void ResolveChordVote(SensitivityProfile profile, out int chordRoot, out int chordType, out float chordConfidence)
+    {
+        Array.Clear(_chordVoteScores);
+        float totalWeight = 0f;
+        float silenceWeight = 0f;
+
+        foreach (ChordVote vote in _chordVotes)
+        {
+            float weight = MathF.Max(0f, vote.Weight);
+            totalWeight += weight;
+            if (vote.Root >= 0 && vote.Type >= 0)
+            {
+                int idx = (vote.Root * 2) + vote.Type;
+                _chordVoteScores[idx] += weight;
+            }
+            else
+            {
+                silenceWeight += weight;
+            }
+        }
+
+        if (totalWeight <= 1e-6f)
+        {
+            chordRoot = -1;
+            chordType = -1;
+            chordConfidence = 0f;
+            return;
+        }
+
+        float best = 0f;
+        float second = 0f;
+        int bestIndex = -1;
+        for (int i = 0; i < _chordVoteScores.Length; i++)
+        {
+            float score = _chordVoteScores[i];
+            if (score > best)
+            {
+                second = best;
+                best = score;
+                bestIndex = i;
+            }
+            else if (score > second)
+            {
+                second = score;
+            }
+        }
+
+        float consensus = best / (totalWeight + 1e-6f);
+        float separation = (best - second) / (totalWeight + 1e-6f);
+        if (bestIndex < 0 ||
+            best < profile.ChordVoteMinScore ||
+            consensus < profile.ChordVoteConsensus ||
+            separation < profile.ChordVoteSeparation ||
+            silenceWeight > best * 1.10f)
+        {
+            chordRoot = -1;
+            chordType = -1;
+            chordConfidence = 0f;
+            return;
+        }
+
+        chordRoot = bestIndex / 2;
+        chordType = bestIndex % 2;
+        chordConfidence = Clamp01((consensus * 0.75f) + (separation * 0.25f));
+    }
+
+    private void PruneOldVotes(Queue<NoteVote> votes, float now)
+    {
+        float cutoff = now - TemporalVoteWindowSeconds;
+        while (votes.Count > 0 && votes.Peek().Time < cutoff)
+        {
+            votes.Dequeue();
+        }
+    }
+
+    private void PruneOldVotes(Queue<ChordVote> votes, float now)
+    {
+        float cutoff = now - TemporalVoteWindowSeconds;
+        while (votes.Count > 0 && votes.Peek().Time < cutoff)
+        {
+            votes.Dequeue();
+        }
+    }
+
    private void ApplyDetectionStability(
        float now,
        SensitivityProfile profile,
@@ -661,7 +952,17 @@ public sealed class AudioAnalyzer : IDisposable
        float chordConfidenceThreshold,
        float noteOnsetThreshold,
        float noteReleaseThreshold,
-        float chordLevelThreshold)
+        float chordLevelThreshold,
+        float chordHarmonicityThreshold,
+        float chordPeakRatioThreshold,
+        float chordSeparationThreshold,
+        float silenceVoteWeight,
+        float noteVoteMinScore,
+        float noteVoteConsensus,
+        float noteVoteSeparation,
+        float chordVoteMinScore,
+        float chordVoteConsensus,
+        float chordVoteSeparation)
    {
        public float LevelGain { get; } = levelGain;
        public float BassGain { get; } = bassGain;
@@ -678,6 +979,16 @@ public sealed class AudioAnalyzer : IDisposable
        public float NoteOnsetThreshold { get; } = noteOnsetThreshold;
        public float NoteReleaseThreshold { get; } = noteReleaseThreshold;
        public float ChordLevelThreshold { get; } = chordLevelThreshold;
+        public float ChordHarmonicityThreshold { get; } = chordHarmonicityThreshold;
+        public float ChordPeakRatioThreshold { get; } = chordPeakRatioThreshold;
+        public float ChordSeparationThreshold { get; } = chordSeparationThreshold;
+        public float SilenceVoteWeight { get; } = silenceVoteWeight;
+        public float NoteVoteMinScore { get; } = noteVoteMinScore;
+        public float NoteVoteConsensus { get; } = noteVoteConsensus;
+        public float NoteVoteSeparation { get; } = noteVoteSeparation;
+        public float ChordVoteMinScore { get; } = chordVoteMinScore;
+        public float ChordVoteConsensus { get; } = chordVoteConsensus;
+        public float ChordVoteSeparation { get; } = chordVoteSeparation;

        public static SensitivityProfile Microphone { get; } = new(
            levelGain: 1.20f,
@@ -694,7 +1005,17 @@ public sealed class AudioAnalyzer : IDisposable
            chordConfidenceThreshold: 0.18f,
            noteOnsetThreshold: 0.20f,
            noteReleaseThreshold: 0.08f,
-            chordLevelThreshold: 0.10f);
+            chordLevelThreshold: 0.10f,
+            chordHarmonicityThreshold: 0.24f,
+            chordPeakRatioThreshold: 0.12f,
+            chordSeparationThreshold: 0.040f,
+            silenceVoteWeight: 0.055f,
+            noteVoteMinScore: 0.24f,
+            noteVoteConsensus: 0.30f,
+            noteVoteSeparation: 0.08f,
+            chordVoteMinScore: 0.26f,
+            chordVoteConsensus: 0.32f,
+            chordVoteSeparation: 0.070f);

        public static SensitivityProfile SystemOutput { get; } = new(
            levelGain: 1.00f,
@@ -711,9 +1032,23 @@ public sealed class AudioAnalyzer : IDisposable
            chordConfidenceThreshold: 0.14f,
            noteOnsetThreshold: 0.18f,
            noteReleaseThreshold: 0.07f,
-            chordLevelThreshold: 0.08f);
+            chordLevelThreshold: 0.08f,
+            chordHarmonicityThreshold: 0.18f,
+            chordPeakRatioThreshold: 0.10f,
+            chordSeparationThreshold: 0.030f,
+            silenceVoteWeight: 0.050f,
+            noteVoteMinScore: 0.20f,
+            noteVoteConsensus: 0.26f,
+            noteVoteSeparation: 0.06f,
+            chordVoteMinScore: 0.22f,
+            chordVoteConsensus: 0.28f,
+            chordVoteSeparation: 0.055f);
    }

+    private readonly record struct NoteVote(float Time, int NoteIndex, float Weight);
+
+    private readonly record struct ChordVote(float Time, int Root, int Type, float Weight);
+
    private void ThrowIfDisposed()
    {
        if (_disposed)