Temporal voting for robust note/chord detection

Adds temporal voting queues and aggregation logic for note and chord detection, improving stability and reducing spurious results. Enhances chord detection with new harmonicity, peak ratio, and separation checks. Introduces HPS-based fundamental frequency estimation. Updates SensitivityProfile with new tuning parameters. All changes aim to make detection more reliable and musically meaningful.

FractalVisualizer/Audio/AudioAnalyzer.cs

@@ -20,6 +20,8 @@ public sealed class AudioAnalyzer : IDisposable
     private readonly float[] _spectrumAccumulator;
     private readonly int[] _spectrumAccumulatorCounts;
     private readonly float[] _chroma = new float[12];
+    private readonly float[] _noteVoteScores = new float[12];
+    private readonly float[] _chordVoteScores = new float[24];
     private readonly float[][] _spectrumRefsByMode;
     private readonly float[] _levelRefsByMode = new float[2];
     private readonly float[] _bassRefsByMode = new float[2];
@@ -43,6 +45,9 @@ public sealed class AudioAnalyzer : IDisposable
     private float _lastBeatTimeSeconds = -10f;
     private bool _beatGateOpen = true;
     private readonly Queue<float> _beatIntervals = new();
+    private readonly Queue<NoteVote> _noteVotes = new();
+    private readonly Queue<ChordVote> _chordVotes = new();
+    private const float TemporalVoteWindowSeconds = 0.30f;
     private const int MaxBeatIntervals = 8;
     private int _lastChordRoot = -1;
     private int _lastChordType = -1;
@@ -294,11 +299,15 @@ public sealed class AudioAnalyzer : IDisposable
         float bpmInstant = CalculateBpm();
         _smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f);
 
-        int noteIndex = GetNoteIndexFromFrequency(dominantFrequency);
+        EstimateFundamentalFrequencyHps(sampleRate, out float hpsFrequency, out float hpsMagnitude, out float harmonicity);
-        float whistleBoost = GetWhistleBoost(dominantFrequency);
+        float noteFrequency = hpsFrequency > 0f ? hpsFrequency : dominantFrequency;
-        float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f));
+        float noteMagnitude = hpsFrequency > 0f ? hpsMagnitude : dominantMagnitude;
-        DetectChord(_chroma, _smoothedLevel, profile, out int chordRoot, out int chordType, out float chordConfidence);
+        int noteIndex = GetNoteIndexFromFrequency(noteFrequency);
-        ApplyDetectionStability(
+        float whistleBoost = GetWhistleBoost(noteFrequency);
+        float dominantStrength = Clamp01(noteMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f) * (0.58f + (0.42f * harmonicity)));
+
+        DetectChord(_chroma, _smoothedLevel, profile, harmonicity, out int chordRoot, out int chordType, out float chordConfidence);
+        ApplyTemporalVoting(
             now,
             profile,
             noteIndex,
@@ -306,6 +315,19 @@ public sealed class AudioAnalyzer : IDisposable
             chordRoot,
             chordType,
             chordConfidence,
+            out int votedNoteIndex,
+            out float votedNoteStrength,
+            out int votedChordRoot,
+            out int votedChordType,
+            out float votedChordConfidence);
+        ApplyDetectionStability(
+            now,
+            profile,
+            votedNoteIndex,
+            votedNoteStrength,
+            votedChordRoot,
+            votedChordType,
+            votedChordConfidence,
             out int stableNoteIndex,
             out float stableNoteStrength,
             out int stableChordRoot,
@@ -321,7 +343,7 @@ public sealed class AudioAnalyzer : IDisposable
             High = _smoothedHigh,
             Beat = _smoothedBeat,
             Bpm = _smoothedBpm,
-            DominantFrequencyHz = dominantFrequency,
+            DominantFrequencyHz = noteFrequency,
             DetectedNoteIndex = stableNoteIndex,
             DetectedNoteStrength = stableNoteStrength,
             NoteOnset = noteOnset,
@@ -408,9 +430,9 @@ public sealed class AudioAnalyzer : IDisposable
         return note;
     }
 
-    private void DetectChord(float[] chroma, float level, SensitivityProfile profile, out int root, out int type, out float confidence)
+    private void DetectChord(float[] chroma, float level, SensitivityProfile profile, float harmonicity, out int root, out int type, out float confidence)
     {
-        if (level < profile.ChordLevelThreshold)
+        if (level < profile.ChordLevelThreshold || harmonicity < profile.ChordHarmonicityThreshold)
         {
             root = -1;
             type = -1;
@@ -432,7 +454,42 @@ public sealed class AudioAnalyzer : IDisposable
             return;
         }
 
+        float logSum = 0f;
+        float peak = 0f;
+        for (int i = 0; i < chroma.Length; i++)
+        {
+            float c = chroma[i];
+            if (c > peak)
+            {
+                peak = c;
+            }
+
+            logSum += MathF.Log(c + 1e-9f);
+        }
+
+        float arithmeticMean = total / chroma.Length;
+        float geometricMean = MathF.Exp(logSum / chroma.Length);
+        float flatness = geometricMean / (arithmeticMean + 1e-9f);
+        float chromaHarmonicity = Clamp01(1f - (flatness * 1.25f));
+        if (chromaHarmonicity < profile.ChordHarmonicityThreshold)
+        {
+            root = -1;
+            type = -1;
+            confidence = 0f;
+            return;
+        }
+
+        float peakRatio = peak / (total + 1e-6f);
+        if (peakRatio < profile.ChordPeakRatioThreshold)
+        {
+            root = -1;
+            type = -1;
+            confidence = 0f;
+            return;
+        }
+
         float bestScore = 0f;
+        float secondBestScore = 0f;
         int bestRoot = -1;
         int bestType = -1;
         for (int r = 0; r < 12; r++)
@@ -440,26 +497,37 @@ public sealed class AudioAnalyzer : IDisposable
             float major = ScoreChord(chroma, r, 0);
             if (major > bestScore)
             {
+                secondBestScore = bestScore;
                 bestScore = major;
                 bestRoot = r;
                 bestType = 0;
             }
+            else if (major > secondBestScore)
+            {
+                secondBestScore = major;
+            }
 
             float minor = ScoreChord(chroma, r, 1);
             if (minor > bestScore)
             {
+                secondBestScore = bestScore;
                 bestScore = minor;
                 bestRoot = r;
                 bestType = 1;
             }
+            else if (minor > secondBestScore)
+            {
+                secondBestScore = minor;
+            }
         }
 
         float conf = Clamp01(bestScore / (total + 1e-6f));
-        if (conf < 0.17f)
+        float separation = (bestScore - secondBestScore) / (total + 1e-6f);
+        if (conf < 0.17f || separation < profile.ChordSeparationThreshold)
         {
             root = -1;
             type = -1;
-            confidence = conf;
+            confidence = 0f;
             return;
         }
 
@@ -477,7 +545,58 @@ public sealed class AudioAnalyzer : IDisposable
 
         root = bestRoot;
         type = bestType;
-        confidence = conf;
+        confidence = Clamp01(conf * (0.55f + (0.45f * harmonicity)));
+    }
+
+    private void EstimateFundamentalFrequencyHps(int sampleRate, out float frequencyHz, out float magnitude, out float harmonicity)
+    {
+        frequencyHz = 0f;
+        magnitude = 0f;
+        harmonicity = 0f;
+
+        int half = _fftSize / 2;
+        int minBin = Math.Max(2, (int)(70f * _fftSize / sampleRate));
+        int maxBinByFrequency = Math.Min(half - 1, (int)(4200f * _fftSize / sampleRate));
+        int maxBin = Math.Min(maxBinByFrequency, (half / 4) - 1);
+        if (maxBin <= minBin)
+        {
+            return;
+        }
+
+        float bestLogScore = float.NegativeInfinity;
+        int bestBin = -1;
+        const float epsilon = 1e-12f;
+        for (int bin = minBin; bin <= maxBin; bin++)
+        {
+            float m1 = (float)_fftBuffer[bin].Magnitude;
+            float m2 = (float)_fftBuffer[bin * 2].Magnitude;
+            float m3 = (float)_fftBuffer[bin * 3].Magnitude;
+            float m4 = (float)_fftBuffer[bin * 4].Magnitude;
+            float logScore =
+                MathF.Log(m1 + epsilon) +
+                (0.90f * MathF.Log(m2 + epsilon)) +
+                (0.72f * MathF.Log(m3 + epsilon)) +
+                (0.55f * MathF.Log(m4 + epsilon));
+            if (logScore > bestLogScore)
+            {
+                bestLogScore = logScore;
+                bestBin = bin;
+            }
+        }
+
+        if (bestBin < 0)
+        {
+            return;
+        }
+
+        float fundamental = (float)_fftBuffer[bestBin].Magnitude;
+        float second = (float)_fftBuffer[bestBin * 2].Magnitude;
+        float third = (float)_fftBuffer[bestBin * 3].Magnitude;
+        float fourth = (float)_fftBuffer[bestBin * 4].Magnitude;
+
+        frequencyHz = (sampleRate * bestBin) / (float)_fftSize;
+        magnitude = fundamental;
+        harmonicity = Clamp01(((second + third + fourth) / 3f) / (fundamental + 1e-6f));
     }
 
     private static float ScoreChord(float[] chroma, int root, int type)
@@ -548,6 +667,178 @@ public sealed class AudioAnalyzer : IDisposable
         return Clamp01((v / reference) * gain);
     }
 
+    private void ApplyTemporalVoting(
+        float now,
+        SensitivityProfile profile,
+        int noteIndex,
+        float noteStrength,
+        int chordRoot,
+        int chordType,
+        float chordConfidence,
+        out int votedNoteIndex,
+        out float votedNoteStrength,
+        out int votedChordRoot,
+        out int votedChordType,
+        out float votedChordConfidence)
+    {
+        float noteWeight = noteIndex >= 0 ? MathF.Max(0f, noteStrength) : profile.SilenceVoteWeight;
+        _noteVotes.Enqueue(new NoteVote(now, noteIndex, noteWeight));
+        PruneOldVotes(_noteVotes, now);
+        ResolveNoteVote(profile, out votedNoteIndex, out votedNoteStrength);
+
+        bool hasChordCandidate = chordRoot >= 0 && chordConfidence >= (profile.ChordConfidenceThreshold * 0.55f);
+        float chordWeight = hasChordCandidate ? chordConfidence : profile.SilenceVoteWeight;
+        _chordVotes.Enqueue(new ChordVote(now, hasChordCandidate ? chordRoot : -1, hasChordCandidate ? chordType : -1, chordWeight));
+        PruneOldVotes(_chordVotes, now);
+        ResolveChordVote(profile, out votedChordRoot, out votedChordType, out votedChordConfidence);
+    }
+
+    private void ResolveNoteVote(SensitivityProfile profile, out int noteIndex, out float noteStrength)
+    {
+        Array.Clear(_noteVoteScores);
+        float totalWeight = 0f;
+        float silenceWeight = 0f;
+
+        foreach (NoteVote vote in _noteVotes)
+        {
+            float weight = MathF.Max(0f, vote.Weight);
+            totalWeight += weight;
+            if (vote.NoteIndex >= 0)
+            {
+                _noteVoteScores[vote.NoteIndex] += weight;
+            }
+            else
+            {
+                silenceWeight += weight;
+            }
+        }
+
+        if (totalWeight <= 1e-6f)
+        {
+            noteIndex = -1;
+            noteStrength = 0f;
+            return;
+        }
+
+        float best = 0f;
+        float second = 0f;
+        int bestIndex = -1;
+        for (int i = 0; i < _noteVoteScores.Length; i++)
+        {
+            float score = _noteVoteScores[i];
+            if (score > best)
+            {
+                second = best;
+                best = score;
+                bestIndex = i;
+            }
+            else if (score > second)
+            {
+                second = score;
+            }
+        }
+
+        float consensus = best / (totalWeight + 1e-6f);
+        float separation = (best - second) / (totalWeight + 1e-6f);
+        if (bestIndex < 0 ||
+            best < profile.NoteVoteMinScore ||
+            consensus < profile.NoteVoteConsensus ||
+            separation < profile.NoteVoteSeparation ||
+            silenceWeight > best * 1.08f)
+        {
+            noteIndex = -1;
+            noteStrength = 0f;
+            return;
+        }
+
+        noteIndex = bestIndex;
+        noteStrength = Clamp01((consensus * 0.72f) + (separation * 0.28f));
+    }
+
+    private void ResolveChordVote(SensitivityProfile profile, out int chordRoot, out int chordType, out float chordConfidence)
+    {
+        Array.Clear(_chordVoteScores);
+        float totalWeight = 0f;
+        float silenceWeight = 0f;
+
+        foreach (ChordVote vote in _chordVotes)
+        {
+            float weight = MathF.Max(0f, vote.Weight);
+            totalWeight += weight;
+            if (vote.Root >= 0 && vote.Type >= 0)
+            {
+                int idx = (vote.Root * 2) + vote.Type;
+                _chordVoteScores[idx] += weight;
+            }
+            else
+            {
+                silenceWeight += weight;
+            }
+        }
+
+        if (totalWeight <= 1e-6f)
+        {
+            chordRoot = -1;
+            chordType = -1;
+            chordConfidence = 0f;
+            return;
+        }
+
+        float best = 0f;
+        float second = 0f;
+        int bestIndex = -1;
+        for (int i = 0; i < _chordVoteScores.Length; i++)
+        {
+            float score = _chordVoteScores[i];
+            if (score > best)
+            {
+                second = best;
+                best = score;
+                bestIndex = i;
+            }
+            else if (score > second)
+            {
+                second = score;
+            }
+        }
+
+        float consensus = best / (totalWeight + 1e-6f);
+        float separation = (best - second) / (totalWeight + 1e-6f);
+        if (bestIndex < 0 ||
+            best < profile.ChordVoteMinScore ||
+            consensus < profile.ChordVoteConsensus ||
+            separation < profile.ChordVoteSeparation ||
+            silenceWeight > best * 1.10f)
+        {
+            chordRoot = -1;
+            chordType = -1;
+            chordConfidence = 0f;
+            return;
+        }
+
+        chordRoot = bestIndex / 2;
+        chordType = bestIndex % 2;
+        chordConfidence = Clamp01((consensus * 0.75f) + (separation * 0.25f));
+    }
+
+    private void PruneOldVotes(Queue<NoteVote> votes, float now)
+    {
+        float cutoff = now - TemporalVoteWindowSeconds;
+        while (votes.Count > 0 && votes.Peek().Time < cutoff)
+        {
+            votes.Dequeue();
+        }
+    }
+
+    private void PruneOldVotes(Queue<ChordVote> votes, float now)
+    {
+        float cutoff = now - TemporalVoteWindowSeconds;
+        while (votes.Count > 0 && votes.Peek().Time < cutoff)
+        {
+            votes.Dequeue();
+        }
+    }
+
     private void ApplyDetectionStability(
         float now,
         SensitivityProfile profile,
@@ -661,7 +952,17 @@ public sealed class AudioAnalyzer : IDisposable
         float chordConfidenceThreshold,
         float noteOnsetThreshold,
         float noteReleaseThreshold,
-        float chordLevelThreshold)
+        float chordLevelThreshold,
+        float chordHarmonicityThreshold,
+        float chordPeakRatioThreshold,
+        float chordSeparationThreshold,
+        float silenceVoteWeight,
+        float noteVoteMinScore,
+        float noteVoteConsensus,
+        float noteVoteSeparation,
+        float chordVoteMinScore,
+        float chordVoteConsensus,
+        float chordVoteSeparation)
     {
         public float LevelGain { get; } = levelGain;
         public float BassGain { get; } = bassGain;
@@ -678,6 +979,16 @@ public sealed class AudioAnalyzer : IDisposable
         public float NoteOnsetThreshold { get; } = noteOnsetThreshold;
         public float NoteReleaseThreshold { get; } = noteReleaseThreshold;
         public float ChordLevelThreshold { get; } = chordLevelThreshold;
+        public float ChordHarmonicityThreshold { get; } = chordHarmonicityThreshold;
+        public float ChordPeakRatioThreshold { get; } = chordPeakRatioThreshold;
+        public float ChordSeparationThreshold { get; } = chordSeparationThreshold;
+        public float SilenceVoteWeight { get; } = silenceVoteWeight;
+        public float NoteVoteMinScore { get; } = noteVoteMinScore;
+        public float NoteVoteConsensus { get; } = noteVoteConsensus;
+        public float NoteVoteSeparation { get; } = noteVoteSeparation;
+        public float ChordVoteMinScore { get; } = chordVoteMinScore;
+        public float ChordVoteConsensus { get; } = chordVoteConsensus;
+        public float ChordVoteSeparation { get; } = chordVoteSeparation;
 
         public static SensitivityProfile Microphone { get; } = new(
             levelGain: 1.20f,
@@ -694,7 +1005,17 @@ public sealed class AudioAnalyzer : IDisposable
             chordConfidenceThreshold: 0.18f,
             noteOnsetThreshold: 0.20f,
             noteReleaseThreshold: 0.08f,
-            chordLevelThreshold: 0.10f);
+            chordLevelThreshold: 0.10f,
+            chordHarmonicityThreshold: 0.24f,
+            chordPeakRatioThreshold: 0.12f,
+            chordSeparationThreshold: 0.040f,
+            silenceVoteWeight: 0.055f,
+            noteVoteMinScore: 0.24f,
+            noteVoteConsensus: 0.30f,
+            noteVoteSeparation: 0.08f,
+            chordVoteMinScore: 0.26f,
+            chordVoteConsensus: 0.32f,
+            chordVoteSeparation: 0.070f);
 
         public static SensitivityProfile SystemOutput { get; } = new(
             levelGain: 1.00f,
@@ -711,9 +1032,23 @@ public sealed class AudioAnalyzer : IDisposable
             chordConfidenceThreshold: 0.14f,
             noteOnsetThreshold: 0.18f,
             noteReleaseThreshold: 0.07f,
-            chordLevelThreshold: 0.08f);
+            chordLevelThreshold: 0.08f,
+            chordHarmonicityThreshold: 0.18f,
+            chordPeakRatioThreshold: 0.10f,
+            chordSeparationThreshold: 0.030f,
+            silenceVoteWeight: 0.050f,
+            noteVoteMinScore: 0.20f,
+            noteVoteConsensus: 0.26f,
+            noteVoteSeparation: 0.06f,
+            chordVoteMinScore: 0.22f,
+            chordVoteConsensus: 0.28f,
+            chordVoteSeparation: 0.055f);
     }
 
+    private readonly record struct NoteVote(float Time, int NoteIndex, float Weight);
+
+    private readonly record struct ChordVote(float Time, int Root, int Type, float Weight);
+
     private void ThrowIfDisposed()
     {
         if (_disposed)