From 488e3c64a3be04bd83d74be10f487c598b94baca Mon Sep 17 00:00:00 2001 From: Michal Vrba Date: Fri, 13 Feb 2026 21:01:09 +0100 Subject: [PATCH] Adaptive normalization & stability for audio features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improves audio analysis by introducing adaptive normalization and per-mode sensitivity profiles for spectrum, level, and band features. Adds temporal stability to note and chord detection, reducing jitter and false positives. Enhances note detection and visual feedback for whistling frequencies (800 Hz–5 kHz). These changes make both analysis and visualization more robust and responsive to varying input conditions. --- FractalVisualizer/Audio/AudioAnalyzer.cs | 222 +++++++++++++++++- .../Rendering/FractalRenderer.cs | 14 +- 2 files changed, 223 insertions(+), 13 deletions(-) diff --git a/FractalVisualizer/Audio/AudioAnalyzer.cs b/FractalVisualizer/Audio/AudioAnalyzer.cs index e42c736..1834370 100644 --- a/FractalVisualizer/Audio/AudioAnalyzer.cs +++ b/FractalVisualizer/Audio/AudioAnalyzer.cs @@ -20,6 +20,11 @@ public sealed class AudioAnalyzer : IDisposable private readonly float[] _spectrumAccumulator; private readonly int[] _spectrumAccumulatorCounts; private readonly float[] _chroma = new float[12]; + private readonly float[][] _spectrumRefsByMode; + private readonly float[] _levelRefsByMode = new float[2]; + private readonly float[] _bassRefsByMode = new float[2]; + private readonly float[] _midRefsByMode = new float[2]; + private readonly float[] _highRefsByMode = new float[2]; private readonly Stopwatch _clock = new(); private readonly object _lifecycleSync = new(); @@ -41,6 +46,10 @@ public sealed class AudioAnalyzer : IDisposable private const int MaxBeatIntervals = 8; private int _lastChordRoot = -1; private int _lastChordType = -1; + private float _lastChordChangeTimeSeconds; + private int _stableNoteIndex = -1; + private float _stableNoteStrength; + private float _lastNoteChangeTimeSeconds; private Snapshot _latestSnapshot; private bool _disposed; @@ -67,6 +76,11 @@ public sealed class AudioAnalyzer : IDisposable _fftBuffer = new Complex[fftSize]; _spectrumAccumulator = new float[spectrumBins]; _spectrumAccumulatorCounts = new int[spectrumBins]; + _spectrumRefsByMode = + [ + new float[spectrumBins], + new float[spectrumBins] + ]; _latestSnapshot = new Snapshot(AudioFeatures.CreateEmpty(spectrumBins)); _capture.SampleRateChanged += OnSampleRateChanged; @@ -146,6 +160,9 @@ public sealed class AudioAnalyzer : IDisposable private void ProcessWindow() { + AudioInputMode mode = _capture.CurrentMode; + SensitivityProfile profile = mode == AudioInputMode.Microphone ? SensitivityProfile.Microphone : SensitivityProfile.SystemOutput; + int modeIndex = mode == AudioInputMode.Microphone ? 0 : 1; int sampleRate = Volatile.Read(ref _sampleRate); int half = _fftSize / 2; @@ -202,7 +219,7 @@ public sealed class AudioAnalyzer : IDisposable highEnergy += magnitude; } - if (frequency >= 80f && frequency <= 2000f && magnitude > dominantMagnitude) + if (frequency >= 120f && frequency <= 6000f && magnitude > dominantMagnitude) { dominantMagnitude = magnitude; dominantFrequency = frequency; @@ -227,7 +244,8 @@ public sealed class AudioAnalyzer : IDisposable continue; } - spectrum[i] = _spectrumAccumulator[i] / _spectrumAccumulatorCounts[i]; + float raw = _spectrumAccumulator[i] / _spectrumAccumulatorCounts[i]; + spectrum[i] = NormalizeWithReference(raw, ref _spectrumRefsByMode[modeIndex][i], profile.SpectrumGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay); } float instantEnergy = rms * rms; @@ -238,10 +256,15 @@ public sealed class AudioAnalyzer : IDisposable beat = Math.Clamp((instantEnergy / _energyBaseline) - 1f, 0f, 1f); } - _smoothedLevel = Smooth(_smoothedLevel, rms, 0.45f, 0.08f); - _smoothedBass = Smooth(_smoothedBass, bassEnergy, 0.40f, 0.10f); - _smoothedMid = Smooth(_smoothedMid, midEnergy, 0.30f, 0.08f); - _smoothedHigh = Smooth(_smoothedHigh, highEnergy, 0.30f, 0.08f); + float levelNorm = NormalizeWithReference(rms, ref _levelRefsByMode[modeIndex], profile.LevelGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay); + float bassNorm = NormalizeWithReference(bassEnergy, ref _bassRefsByMode[modeIndex], profile.BassGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay); + float midNorm = NormalizeWithReference(midEnergy, ref _midRefsByMode[modeIndex], profile.MidGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay); + float highNorm = NormalizeWithReference(highEnergy, ref _highRefsByMode[modeIndex], profile.HighGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay); + + _smoothedLevel = Smooth(_smoothedLevel, levelNorm, 0.45f, 0.08f); + _smoothedBass = Smooth(_smoothedBass, bassNorm, 0.40f, 0.10f); + _smoothedMid = Smooth(_smoothedMid, midNorm, 0.30f, 0.08f); + _smoothedHigh = Smooth(_smoothedHigh, highNorm, 0.30f, 0.08f); _smoothedBeat = MathF.Max(beat, _smoothedBeat * 0.85f); float now = (float)_clock.Elapsed.TotalSeconds; @@ -270,8 +293,22 @@ public sealed class AudioAnalyzer : IDisposable _smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f); int noteIndex = GetNoteIndexFromFrequency(dominantFrequency); - float dominantStrength = Clamp01(dominantMagnitude * 0.02f); + float whistleBoost = GetWhistleBoost(dominantFrequency); + float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f)); DetectChord(_chroma, out int chordRoot, out int chordType, out float chordConfidence); + ApplyDetectionStability( + now, + profile, + noteIndex, + dominantStrength, + chordRoot, + chordType, + chordConfidence, + out int stableNoteIndex, + out float stableNoteStrength, + out int stableChordRoot, + out int stableChordType, + out float stableChordConfidence); AudioFeatures features = new() { @@ -282,11 +319,11 @@ public sealed class AudioAnalyzer : IDisposable Beat = _smoothedBeat, Bpm = _smoothedBpm, DominantFrequencyHz = dominantFrequency, - DetectedNoteIndex = noteIndex, - DetectedNoteStrength = dominantStrength, - DetectedChordRootIndex = chordRoot, - DetectedChordType = chordType, - DetectedChordConfidence = chordConfidence, + DetectedNoteIndex = stableNoteIndex, + DetectedNoteStrength = stableNoteStrength, + DetectedChordRootIndex = stableChordRoot, + DetectedChordType = stableChordType, + DetectedChordConfidence = stableChordConfidence, Spectrum = spectrum, TimeSeconds = now }; @@ -459,6 +496,167 @@ public sealed class AudioAnalyzer : IDisposable return value; } + private static float GetWhistleBoost(float frequencyHz) + { + if (frequencyHz <= 0f) + { + return 0f; + } + + // Human whistles usually sit around ~1kHz-4kHz, with strong note clarity in this band. + if (frequencyHz < 800f) + { + return 0f; + } + + if (frequencyHz > 5000f) + { + return 0f; + } + + float centered = (frequencyHz - 800f) / (5000f - 800f); + return Clamp01(centered); + } + + private static float NormalizeWithReference(float value, ref float reference, float gain, float noiseFloor, float referenceRise, float referenceDecay) + { + float v = MathF.Max(0f, value - noiseFloor); + if (v > reference) + { + reference = Lerp(reference, v, referenceRise); + } + else + { + reference = Lerp(reference, v, referenceDecay); + } + + if (reference < 1e-6f) + { + reference = 1e-6f; + } + + return Clamp01((v / reference) * gain); + } + + private void ApplyDetectionStability( + float now, + SensitivityProfile profile, + int noteIndex, + float noteStrength, + int chordRoot, + int chordType, + float chordConfidence, + out int stableNoteIndex, + out float stableNoteStrength, + out int stableChordRoot, + out int stableChordType, + out float stableChordConfidence) + { + stableNoteIndex = _stableNoteIndex; + stableNoteStrength = _stableNoteStrength * 0.96f; + + if (noteIndex >= 0 && noteStrength >= profile.NoteConfidenceThreshold) + { + bool changed = _stableNoteIndex != noteIndex; + bool canSwitch = (now - _lastNoteChangeTimeSeconds) >= profile.SwitchHoldSeconds; + if (!changed || canSwitch || noteStrength > _stableNoteStrength * 1.25f) + { + stableNoteIndex = noteIndex; + stableNoteStrength = noteStrength; + if (changed) + { + _lastNoteChangeTimeSeconds = now; + } + } + } + + _stableNoteIndex = stableNoteIndex; + _stableNoteStrength = stableNoteStrength; + + stableChordRoot = _lastChordRoot; + stableChordType = _lastChordType; + stableChordConfidence = 0f; + if (stableChordRoot >= 0) + { + stableChordConfidence = Clamp01(chordConfidence * 0.85f); + } + + if (chordRoot >= 0 && chordConfidence >= profile.ChordConfidenceThreshold) + { + bool changed = _lastChordRoot != chordRoot || _lastChordType != chordType; + bool canSwitch = (now - _lastChordChangeTimeSeconds) >= profile.SwitchHoldSeconds; + if (!changed || canSwitch || chordConfidence > stableChordConfidence * 1.2f) + { + _lastChordRoot = chordRoot; + _lastChordType = chordType; + if (changed) + { + _lastChordChangeTimeSeconds = now; + } + } + + stableChordRoot = _lastChordRoot; + stableChordType = _lastChordType; + stableChordConfidence = chordConfidence; + } + } + + private readonly struct SensitivityProfile( + float levelGain, + float bassGain, + float midGain, + float highGain, + float spectrumGain, + float noteStrengthGain, + float noiseFloor, + float referenceRise, + float referenceDecay, + float switchHoldSeconds, + float noteConfidenceThreshold, + float chordConfidenceThreshold) + { + public float LevelGain { get; } = levelGain; + public float BassGain { get; } = bassGain; + public float MidGain { get; } = midGain; + public float HighGain { get; } = highGain; + public float SpectrumGain { get; } = spectrumGain; + public float NoteStrengthGain { get; } = noteStrengthGain; + public float NoiseFloor { get; } = noiseFloor; + public float ReferenceRise { get; } = referenceRise; + public float ReferenceDecay { get; } = referenceDecay; + public float SwitchHoldSeconds { get; } = switchHoldSeconds; + public float NoteConfidenceThreshold { get; } = noteConfidenceThreshold; + public float ChordConfidenceThreshold { get; } = chordConfidenceThreshold; + + public static SensitivityProfile Microphone { get; } = new( + levelGain: 1.20f, + bassGain: 1.25f, + midGain: 1.20f, + highGain: 1.15f, + spectrumGain: 1.40f, + noteStrengthGain: 0.036f, + noiseFloor: 0.0009f, + referenceRise: 0.16f, + referenceDecay: 0.010f, + switchHoldSeconds: 0.18f, + noteConfidenceThreshold: 0.11f, + chordConfidenceThreshold: 0.18f); + + public static SensitivityProfile SystemOutput { get; } = new( + levelGain: 1.00f, + bassGain: 1.00f, + midGain: 1.00f, + highGain: 1.00f, + spectrumGain: 1.18f, + noteStrengthGain: 0.022f, + noiseFloor: 0.0004f, + referenceRise: 0.20f, + referenceDecay: 0.012f, + switchHoldSeconds: 0.18f, + noteConfidenceThreshold: 0.12f, + chordConfidenceThreshold: 0.14f); + } + private void ThrowIfDisposed() { if (_disposed) diff --git a/FractalVisualizer/Rendering/FractalRenderer.cs b/FractalVisualizer/Rendering/FractalRenderer.cs index ef72bba..2323542 100644 --- a/FractalVisualizer/Rendering/FractalRenderer.cs +++ b/FractalVisualizer/Rendering/FractalRenderer.cs @@ -1204,7 +1204,8 @@ public sealed class FractalRenderer : IDisposable } } - float spawnRate = features.DetectedNoteStrength * (0.3f + Clamp01(features.Beat) * 1.3f); + float whistleBoost = GetWhistleSpawnBoost(features.DominantFrequencyHz); + float spawnRate = features.DetectedNoteStrength * (0.9f + whistleBoost * 1.7f + Clamp01(features.Beat) * 0.35f); _detectedNoteAccumulator += spawnRate * _runtimePerformanceScale; while (_detectedNoteAccumulator >= 1f) { @@ -1231,6 +1232,17 @@ public sealed class FractalRenderer : IDisposable } } + private static float GetWhistleSpawnBoost(float frequencyHz) + { + if (frequencyHz < 800f || frequencyHz > 5000f) + { + return 0f; + } + + float norm = (frequencyHz - 800f) / (5000f - 800f); + return Clamp01(norm); + } + private void RenderChordCenter(AudioFeatures features, float timeSeconds) { for (int y = 0; y < _internalHeight; y++)