diff --git a/FractalVisualizer/Audio/AudioAnalyzer.cs b/FractalVisualizer/Audio/AudioAnalyzer.cs index 812aa47..733efcc 100644 --- a/FractalVisualizer/Audio/AudioAnalyzer.cs +++ b/FractalVisualizer/Audio/AudioAnalyzer.cs @@ -20,6 +20,8 @@ public sealed class AudioAnalyzer : IDisposable private readonly float[] _spectrumAccumulator; private readonly int[] _spectrumAccumulatorCounts; private readonly float[] _chroma = new float[12]; + private readonly float[] _noteVoteScores = new float[12]; + private readonly float[] _chordVoteScores = new float[24]; private readonly float[][] _spectrumRefsByMode; private readonly float[] _levelRefsByMode = new float[2]; private readonly float[] _bassRefsByMode = new float[2]; @@ -43,6 +45,9 @@ public sealed class AudioAnalyzer : IDisposable private float _lastBeatTimeSeconds = -10f; private bool _beatGateOpen = true; private readonly Queue _beatIntervals = new(); + private readonly Queue _noteVotes = new(); + private readonly Queue _chordVotes = new(); + private const float TemporalVoteWindowSeconds = 0.30f; private const int MaxBeatIntervals = 8; private int _lastChordRoot = -1; private int _lastChordType = -1; @@ -294,11 +299,15 @@ public sealed class AudioAnalyzer : IDisposable float bpmInstant = CalculateBpm(); _smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f); - int noteIndex = GetNoteIndexFromFrequency(dominantFrequency); - float whistleBoost = GetWhistleBoost(dominantFrequency); - float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f)); - DetectChord(_chroma, _smoothedLevel, profile, out int chordRoot, out int chordType, out float chordConfidence); - ApplyDetectionStability( + EstimateFundamentalFrequencyHps(sampleRate, out float hpsFrequency, out float hpsMagnitude, out float harmonicity); + float noteFrequency = hpsFrequency > 0f ? hpsFrequency : dominantFrequency; + float noteMagnitude = hpsFrequency > 0f ? hpsMagnitude : dominantMagnitude; + int noteIndex = GetNoteIndexFromFrequency(noteFrequency); + float whistleBoost = GetWhistleBoost(noteFrequency); + float dominantStrength = Clamp01(noteMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f) * (0.58f + (0.42f * harmonicity))); + + DetectChord(_chroma, _smoothedLevel, profile, harmonicity, out int chordRoot, out int chordType, out float chordConfidence); + ApplyTemporalVoting( now, profile, noteIndex, @@ -306,6 +315,19 @@ public sealed class AudioAnalyzer : IDisposable chordRoot, chordType, chordConfidence, + out int votedNoteIndex, + out float votedNoteStrength, + out int votedChordRoot, + out int votedChordType, + out float votedChordConfidence); + ApplyDetectionStability( + now, + profile, + votedNoteIndex, + votedNoteStrength, + votedChordRoot, + votedChordType, + votedChordConfidence, out int stableNoteIndex, out float stableNoteStrength, out int stableChordRoot, @@ -321,7 +343,7 @@ public sealed class AudioAnalyzer : IDisposable High = _smoothedHigh, Beat = _smoothedBeat, Bpm = _smoothedBpm, - DominantFrequencyHz = dominantFrequency, + DominantFrequencyHz = noteFrequency, DetectedNoteIndex = stableNoteIndex, DetectedNoteStrength = stableNoteStrength, NoteOnset = noteOnset, @@ -408,9 +430,9 @@ public sealed class AudioAnalyzer : IDisposable return note; } - private void DetectChord(float[] chroma, float level, SensitivityProfile profile, out int root, out int type, out float confidence) + private void DetectChord(float[] chroma, float level, SensitivityProfile profile, float harmonicity, out int root, out int type, out float confidence) { - if (level < profile.ChordLevelThreshold) + if (level < profile.ChordLevelThreshold || harmonicity < profile.ChordHarmonicityThreshold) { root = -1; type = -1; @@ -432,7 +454,42 @@ public sealed class AudioAnalyzer : IDisposable return; } + float logSum = 0f; + float peak = 0f; + for (int i = 0; i < chroma.Length; i++) + { + float c = chroma[i]; + if (c > peak) + { + peak = c; + } + + logSum += MathF.Log(c + 1e-9f); + } + + float arithmeticMean = total / chroma.Length; + float geometricMean = MathF.Exp(logSum / chroma.Length); + float flatness = geometricMean / (arithmeticMean + 1e-9f); + float chromaHarmonicity = Clamp01(1f - (flatness * 1.25f)); + if (chromaHarmonicity < profile.ChordHarmonicityThreshold) + { + root = -1; + type = -1; + confidence = 0f; + return; + } + + float peakRatio = peak / (total + 1e-6f); + if (peakRatio < profile.ChordPeakRatioThreshold) + { + root = -1; + type = -1; + confidence = 0f; + return; + } + float bestScore = 0f; + float secondBestScore = 0f; int bestRoot = -1; int bestType = -1; for (int r = 0; r < 12; r++) @@ -440,26 +497,37 @@ public sealed class AudioAnalyzer : IDisposable float major = ScoreChord(chroma, r, 0); if (major > bestScore) { + secondBestScore = bestScore; bestScore = major; bestRoot = r; bestType = 0; } + else if (major > secondBestScore) + { + secondBestScore = major; + } float minor = ScoreChord(chroma, r, 1); if (minor > bestScore) { + secondBestScore = bestScore; bestScore = minor; bestRoot = r; bestType = 1; } + else if (minor > secondBestScore) + { + secondBestScore = minor; + } } float conf = Clamp01(bestScore / (total + 1e-6f)); - if (conf < 0.17f) + float separation = (bestScore - secondBestScore) / (total + 1e-6f); + if (conf < 0.17f || separation < profile.ChordSeparationThreshold) { root = -1; type = -1; - confidence = conf; + confidence = 0f; return; } @@ -477,7 +545,58 @@ public sealed class AudioAnalyzer : IDisposable root = bestRoot; type = bestType; - confidence = conf; + confidence = Clamp01(conf * (0.55f + (0.45f * harmonicity))); + } + + private void EstimateFundamentalFrequencyHps(int sampleRate, out float frequencyHz, out float magnitude, out float harmonicity) + { + frequencyHz = 0f; + magnitude = 0f; + harmonicity = 0f; + + int half = _fftSize / 2; + int minBin = Math.Max(2, (int)(70f * _fftSize / sampleRate)); + int maxBinByFrequency = Math.Min(half - 1, (int)(4200f * _fftSize / sampleRate)); + int maxBin = Math.Min(maxBinByFrequency, (half / 4) - 1); + if (maxBin <= minBin) + { + return; + } + + float bestLogScore = float.NegativeInfinity; + int bestBin = -1; + const float epsilon = 1e-12f; + for (int bin = minBin; bin <= maxBin; bin++) + { + float m1 = (float)_fftBuffer[bin].Magnitude; + float m2 = (float)_fftBuffer[bin * 2].Magnitude; + float m3 = (float)_fftBuffer[bin * 3].Magnitude; + float m4 = (float)_fftBuffer[bin * 4].Magnitude; + float logScore = + MathF.Log(m1 + epsilon) + + (0.90f * MathF.Log(m2 + epsilon)) + + (0.72f * MathF.Log(m3 + epsilon)) + + (0.55f * MathF.Log(m4 + epsilon)); + if (logScore > bestLogScore) + { + bestLogScore = logScore; + bestBin = bin; + } + } + + if (bestBin < 0) + { + return; + } + + float fundamental = (float)_fftBuffer[bestBin].Magnitude; + float second = (float)_fftBuffer[bestBin * 2].Magnitude; + float third = (float)_fftBuffer[bestBin * 3].Magnitude; + float fourth = (float)_fftBuffer[bestBin * 4].Magnitude; + + frequencyHz = (sampleRate * bestBin) / (float)_fftSize; + magnitude = fundamental; + harmonicity = Clamp01(((second + third + fourth) / 3f) / (fundamental + 1e-6f)); } private static float ScoreChord(float[] chroma, int root, int type) @@ -548,6 +667,178 @@ public sealed class AudioAnalyzer : IDisposable return Clamp01((v / reference) * gain); } + private void ApplyTemporalVoting( + float now, + SensitivityProfile profile, + int noteIndex, + float noteStrength, + int chordRoot, + int chordType, + float chordConfidence, + out int votedNoteIndex, + out float votedNoteStrength, + out int votedChordRoot, + out int votedChordType, + out float votedChordConfidence) + { + float noteWeight = noteIndex >= 0 ? MathF.Max(0f, noteStrength) : profile.SilenceVoteWeight; + _noteVotes.Enqueue(new NoteVote(now, noteIndex, noteWeight)); + PruneOldVotes(_noteVotes, now); + ResolveNoteVote(profile, out votedNoteIndex, out votedNoteStrength); + + bool hasChordCandidate = chordRoot >= 0 && chordConfidence >= (profile.ChordConfidenceThreshold * 0.55f); + float chordWeight = hasChordCandidate ? chordConfidence : profile.SilenceVoteWeight; + _chordVotes.Enqueue(new ChordVote(now, hasChordCandidate ? chordRoot : -1, hasChordCandidate ? chordType : -1, chordWeight)); + PruneOldVotes(_chordVotes, now); + ResolveChordVote(profile, out votedChordRoot, out votedChordType, out votedChordConfidence); + } + + private void ResolveNoteVote(SensitivityProfile profile, out int noteIndex, out float noteStrength) + { + Array.Clear(_noteVoteScores); + float totalWeight = 0f; + float silenceWeight = 0f; + + foreach (NoteVote vote in _noteVotes) + { + float weight = MathF.Max(0f, vote.Weight); + totalWeight += weight; + if (vote.NoteIndex >= 0) + { + _noteVoteScores[vote.NoteIndex] += weight; + } + else + { + silenceWeight += weight; + } + } + + if (totalWeight <= 1e-6f) + { + noteIndex = -1; + noteStrength = 0f; + return; + } + + float best = 0f; + float second = 0f; + int bestIndex = -1; + for (int i = 0; i < _noteVoteScores.Length; i++) + { + float score = _noteVoteScores[i]; + if (score > best) + { + second = best; + best = score; + bestIndex = i; + } + else if (score > second) + { + second = score; + } + } + + float consensus = best / (totalWeight + 1e-6f); + float separation = (best - second) / (totalWeight + 1e-6f); + if (bestIndex < 0 || + best < profile.NoteVoteMinScore || + consensus < profile.NoteVoteConsensus || + separation < profile.NoteVoteSeparation || + silenceWeight > best * 1.08f) + { + noteIndex = -1; + noteStrength = 0f; + return; + } + + noteIndex = bestIndex; + noteStrength = Clamp01((consensus * 0.72f) + (separation * 0.28f)); + } + + private void ResolveChordVote(SensitivityProfile profile, out int chordRoot, out int chordType, out float chordConfidence) + { + Array.Clear(_chordVoteScores); + float totalWeight = 0f; + float silenceWeight = 0f; + + foreach (ChordVote vote in _chordVotes) + { + float weight = MathF.Max(0f, vote.Weight); + totalWeight += weight; + if (vote.Root >= 0 && vote.Type >= 0) + { + int idx = (vote.Root * 2) + vote.Type; + _chordVoteScores[idx] += weight; + } + else + { + silenceWeight += weight; + } + } + + if (totalWeight <= 1e-6f) + { + chordRoot = -1; + chordType = -1; + chordConfidence = 0f; + return; + } + + float best = 0f; + float second = 0f; + int bestIndex = -1; + for (int i = 0; i < _chordVoteScores.Length; i++) + { + float score = _chordVoteScores[i]; + if (score > best) + { + second = best; + best = score; + bestIndex = i; + } + else if (score > second) + { + second = score; + } + } + + float consensus = best / (totalWeight + 1e-6f); + float separation = (best - second) / (totalWeight + 1e-6f); + if (bestIndex < 0 || + best < profile.ChordVoteMinScore || + consensus < profile.ChordVoteConsensus || + separation < profile.ChordVoteSeparation || + silenceWeight > best * 1.10f) + { + chordRoot = -1; + chordType = -1; + chordConfidence = 0f; + return; + } + + chordRoot = bestIndex / 2; + chordType = bestIndex % 2; + chordConfidence = Clamp01((consensus * 0.75f) + (separation * 0.25f)); + } + + private void PruneOldVotes(Queue votes, float now) + { + float cutoff = now - TemporalVoteWindowSeconds; + while (votes.Count > 0 && votes.Peek().Time < cutoff) + { + votes.Dequeue(); + } + } + + private void PruneOldVotes(Queue votes, float now) + { + float cutoff = now - TemporalVoteWindowSeconds; + while (votes.Count > 0 && votes.Peek().Time < cutoff) + { + votes.Dequeue(); + } + } + private void ApplyDetectionStability( float now, SensitivityProfile profile, @@ -661,7 +952,17 @@ public sealed class AudioAnalyzer : IDisposable float chordConfidenceThreshold, float noteOnsetThreshold, float noteReleaseThreshold, - float chordLevelThreshold) + float chordLevelThreshold, + float chordHarmonicityThreshold, + float chordPeakRatioThreshold, + float chordSeparationThreshold, + float silenceVoteWeight, + float noteVoteMinScore, + float noteVoteConsensus, + float noteVoteSeparation, + float chordVoteMinScore, + float chordVoteConsensus, + float chordVoteSeparation) { public float LevelGain { get; } = levelGain; public float BassGain { get; } = bassGain; @@ -678,6 +979,16 @@ public sealed class AudioAnalyzer : IDisposable public float NoteOnsetThreshold { get; } = noteOnsetThreshold; public float NoteReleaseThreshold { get; } = noteReleaseThreshold; public float ChordLevelThreshold { get; } = chordLevelThreshold; + public float ChordHarmonicityThreshold { get; } = chordHarmonicityThreshold; + public float ChordPeakRatioThreshold { get; } = chordPeakRatioThreshold; + public float ChordSeparationThreshold { get; } = chordSeparationThreshold; + public float SilenceVoteWeight { get; } = silenceVoteWeight; + public float NoteVoteMinScore { get; } = noteVoteMinScore; + public float NoteVoteConsensus { get; } = noteVoteConsensus; + public float NoteVoteSeparation { get; } = noteVoteSeparation; + public float ChordVoteMinScore { get; } = chordVoteMinScore; + public float ChordVoteConsensus { get; } = chordVoteConsensus; + public float ChordVoteSeparation { get; } = chordVoteSeparation; public static SensitivityProfile Microphone { get; } = new( levelGain: 1.20f, @@ -694,7 +1005,17 @@ public sealed class AudioAnalyzer : IDisposable chordConfidenceThreshold: 0.18f, noteOnsetThreshold: 0.20f, noteReleaseThreshold: 0.08f, - chordLevelThreshold: 0.10f); + chordLevelThreshold: 0.10f, + chordHarmonicityThreshold: 0.24f, + chordPeakRatioThreshold: 0.12f, + chordSeparationThreshold: 0.040f, + silenceVoteWeight: 0.055f, + noteVoteMinScore: 0.24f, + noteVoteConsensus: 0.30f, + noteVoteSeparation: 0.08f, + chordVoteMinScore: 0.26f, + chordVoteConsensus: 0.32f, + chordVoteSeparation: 0.070f); public static SensitivityProfile SystemOutput { get; } = new( levelGain: 1.00f, @@ -711,9 +1032,23 @@ public sealed class AudioAnalyzer : IDisposable chordConfidenceThreshold: 0.14f, noteOnsetThreshold: 0.18f, noteReleaseThreshold: 0.07f, - chordLevelThreshold: 0.08f); + chordLevelThreshold: 0.08f, + chordHarmonicityThreshold: 0.18f, + chordPeakRatioThreshold: 0.10f, + chordSeparationThreshold: 0.030f, + silenceVoteWeight: 0.050f, + noteVoteMinScore: 0.20f, + noteVoteConsensus: 0.26f, + noteVoteSeparation: 0.06f, + chordVoteMinScore: 0.22f, + chordVoteConsensus: 0.28f, + chordVoteSeparation: 0.055f); } + private readonly record struct NoteVote(float Time, int NoteIndex, float Weight); + + private readonly record struct ChordVote(float Time, int Root, int Type, float Weight); + private void ThrowIfDisposed() { if (_disposed)