Michal/SoundFlix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adaptive normalization & stability for audio features

Browse Source 
Improves audio analysis by introducing adaptive normalization and per-mode sensitivity profiles for spectrum, level, and band features. Adds temporal stability to note and chord detection, reducing jitter and false positives. Enhances note detection and visual feedback for whistling frequencies (800 Hz–5 kHz). These changes make both analysis and visualization more robust and responsive to varying input conditions.
This commit is contained in:
Michal Vrba
2026-02-13 21:01:09 +01:00
parent 2d535addd0
commit 488e3c64a3
2 changed files with 223 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FractalVisualizer/Audio/AudioAnalyzer.cs
+210 -12
View File
@@ -20,6 +20,11 @@ public sealed class AudioAnalyzer : IDisposable
private readonly float[] _spectrumAccumulator;
private readonly int[] _spectrumAccumulatorCounts;
private readonly float[] _chroma = new float[12];
private readonly float[][] _spectrumRefsByMode;
private readonly float[] _levelRefsByMode = new float[2];
private readonly float[] _bassRefsByMode = new float[2];
private readonly float[] _midRefsByMode = new float[2];
private readonly float[] _highRefsByMode = new float[2];
private readonly Stopwatch _clock = new();
private readonly object _lifecycleSync = new();
@@ -41,6 +46,10 @@ public sealed class AudioAnalyzer : IDisposable
private const int MaxBeatIntervals = 8;
private int _lastChordRoot = -1;
private int _lastChordType = -1;
private float _lastChordChangeTimeSeconds;
private int _stableNoteIndex = -1;
private float _stableNoteStrength;
private float _lastNoteChangeTimeSeconds;
private Snapshot _latestSnapshot;
private bool _disposed;
@@ -67,6 +76,11 @@ public sealed class AudioAnalyzer : IDisposable
_fftBuffer = new Complex[fftSize];
_spectrumAccumulator = new float[spectrumBins];
_spectrumAccumulatorCounts = new int[spectrumBins];
_spectrumRefsByMode =
[
new float[spectrumBins],
new float[spectrumBins]
];
_latestSnapshot = new Snapshot(AudioFeatures.CreateEmpty(spectrumBins));
_capture.SampleRateChanged += OnSampleRateChanged;
@@ -146,6 +160,9 @@ public sealed class AudioAnalyzer : IDisposable
private void ProcessWindow()
{
AudioInputMode mode = _capture.CurrentMode;
SensitivityProfile profile = mode == AudioInputMode.Microphone ? SensitivityProfile.Microphone : SensitivityProfile.SystemOutput;
int modeIndex = mode == AudioInputMode.Microphone ? 0 : 1;
int sampleRate = Volatile.Read(ref _sampleRate);
int half = _fftSize / 2;
@@ -202,7 +219,7 @@ public sealed class AudioAnalyzer : IDisposable
highEnergy += magnitude;
}
if (frequency >= 80f && frequency <= 2000f && magnitude > dominantMagnitude)
if (frequency >= 120f && frequency <= 6000f && magnitude > dominantMagnitude)
{
dominantMagnitude = magnitude;
dominantFrequency = frequency;
@@ -227,7 +244,8 @@ public sealed class AudioAnalyzer : IDisposable
continue;
}
spectrum[i] = _spectrumAccumulator[i] / _spectrumAccumulatorCounts[i];
float raw = _spectrumAccumulator[i] / _spectrumAccumulatorCounts[i];
spectrum[i] = NormalizeWithReference(raw, ref _spectrumRefsByMode[modeIndex][i], profile.SpectrumGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
}
float instantEnergy = rms * rms;
@@ -238,10 +256,15 @@ public sealed class AudioAnalyzer : IDisposable
beat = Math.Clamp((instantEnergy / _energyBaseline) - 1f, 0f, 1f);
}
_smoothedLevel = Smooth(_smoothedLevel, rms, 0.45f, 0.08f);
_smoothedBass = Smooth(_smoothedBass, bassEnergy, 0.40f, 0.10f);
_smoothedMid = Smooth(_smoothedMid, midEnergy, 0.30f, 0.08f);
_smoothedHigh = Smooth(_smoothedHigh, highEnergy, 0.30f, 0.08f);
float levelNorm = NormalizeWithReference(rms, ref _levelRefsByMode[modeIndex], profile.LevelGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
float bassNorm = NormalizeWithReference(bassEnergy, ref _bassRefsByMode[modeIndex], profile.BassGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
float midNorm = NormalizeWithReference(midEnergy, ref _midRefsByMode[modeIndex], profile.MidGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
float highNorm = NormalizeWithReference(highEnergy, ref _highRefsByMode[modeIndex], profile.HighGain, profile.NoiseFloor, profile.ReferenceRise, profile.ReferenceDecay);
_smoothedLevel = Smooth(_smoothedLevel, levelNorm, 0.45f, 0.08f);
_smoothedBass = Smooth(_smoothedBass, bassNorm, 0.40f, 0.10f);
_smoothedMid = Smooth(_smoothedMid, midNorm, 0.30f, 0.08f);
_smoothedHigh = Smooth(_smoothedHigh, highNorm, 0.30f, 0.08f);
_smoothedBeat = MathF.Max(beat, _smoothedBeat * 0.85f);
float now = (float)_clock.Elapsed.TotalSeconds;
@@ -270,8 +293,22 @@ public sealed class AudioAnalyzer : IDisposable
_smoothedBpm = Smooth(_smoothedBpm, bpmInstant, 0.18f, 0.04f);
int noteIndex = GetNoteIndexFromFrequency(dominantFrequency);
float dominantStrength = Clamp01(dominantMagnitude * 0.02f);
float whistleBoost = GetWhistleBoost(dominantFrequency);
float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f));
DetectChord(_chroma, out int chordRoot, out int chordType, out float chordConfidence);
ApplyDetectionStability(
now,
profile,
noteIndex,
dominantStrength,
chordRoot,
chordType,
chordConfidence,
out int stableNoteIndex,
out float stableNoteStrength,
out int stableChordRoot,
out int stableChordType,
out float stableChordConfidence);
AudioFeatures features = new()
{
@@ -282,11 +319,11 @@ public sealed class AudioAnalyzer : IDisposable
Beat = _smoothedBeat,
Bpm = _smoothedBpm,
DominantFrequencyHz = dominantFrequency,
DetectedNoteIndex = noteIndex,
DetectedNoteStrength = dominantStrength,
DetectedChordRootIndex = chordRoot,
DetectedChordType = chordType,
DetectedChordConfidence = chordConfidence,
DetectedNoteIndex = stableNoteIndex,
DetectedNoteStrength = stableNoteStrength,
DetectedChordRootIndex = stableChordRoot,
DetectedChordType = stableChordType,
DetectedChordConfidence = stableChordConfidence,
Spectrum = spectrum,
TimeSeconds = now
};
@@ -459,6 +496,167 @@ public sealed class AudioAnalyzer : IDisposable
return value;
}
private static float GetWhistleBoost(float frequencyHz)
{
if (frequencyHz <= 0f)
{
return 0f;
}
// Human whistles usually sit around ~1kHz-4kHz, with strong note clarity in this band.
if (frequencyHz < 800f)
{
return 0f;
}
if (frequencyHz > 5000f)
{
return 0f;
}
float centered = (frequencyHz - 800f) / (5000f - 800f);
return Clamp01(centered);
}
private static float NormalizeWithReference(float value, ref float reference, float gain, float noiseFloor, float referenceRise, float referenceDecay)
{
float v = MathF.Max(0f, value - noiseFloor);
if (v > reference)
{
reference = Lerp(reference, v, referenceRise);
}
else
{
reference = Lerp(reference, v, referenceDecay);
}
if (reference < 1e-6f)
{
reference = 1e-6f;
}
return Clamp01((v / reference) * gain);
}
private void ApplyDetectionStability(
float now,
SensitivityProfile profile,
int noteIndex,
float noteStrength,
int chordRoot,
int chordType,
float chordConfidence,
out int stableNoteIndex,
out float stableNoteStrength,
out int stableChordRoot,
out int stableChordType,
out float stableChordConfidence)
{
stableNoteIndex = _stableNoteIndex;
stableNoteStrength = _stableNoteStrength * 0.96f;
if (noteIndex >= 0 && noteStrength >= profile.NoteConfidenceThreshold)
{
bool changed = _stableNoteIndex != noteIndex;
bool canSwitch = (now - _lastNoteChangeTimeSeconds) >= profile.SwitchHoldSeconds;
if (!changed || canSwitch || noteStrength > _stableNoteStrength * 1.25f)
{
stableNoteIndex = noteIndex;
stableNoteStrength = noteStrength;
if (changed)
{
_lastNoteChangeTimeSeconds = now;
}
}
}
_stableNoteIndex = stableNoteIndex;
_stableNoteStrength = stableNoteStrength;
stableChordRoot = _lastChordRoot;
stableChordType = _lastChordType;
stableChordConfidence = 0f;
if (stableChordRoot >= 0)
{
stableChordConfidence = Clamp01(chordConfidence * 0.85f);
}
if (chordRoot >= 0 && chordConfidence >= profile.ChordConfidenceThreshold)
{
bool changed = _lastChordRoot != chordRoot || _lastChordType != chordType;
bool canSwitch = (now - _lastChordChangeTimeSeconds) >= profile.SwitchHoldSeconds;
if (!changed || canSwitch || chordConfidence > stableChordConfidence * 1.2f)
{
_lastChordRoot = chordRoot;
_lastChordType = chordType;
if (changed)
{
_lastChordChangeTimeSeconds = now;
}
}
stableChordRoot = _lastChordRoot;
stableChordType = _lastChordType;
stableChordConfidence = chordConfidence;
}
}
private readonly struct SensitivityProfile(
float levelGain,
float bassGain,
float midGain,
float highGain,
float spectrumGain,
float noteStrengthGain,
float noiseFloor,
float referenceRise,
float referenceDecay,
float switchHoldSeconds,
float noteConfidenceThreshold,
float chordConfidenceThreshold)
{
public float LevelGain { get; } = levelGain;
public float BassGain { get; } = bassGain;
public float MidGain { get; } = midGain;
public float HighGain { get; } = highGain;
public float SpectrumGain { get; } = spectrumGain;
public float NoteStrengthGain { get; } = noteStrengthGain;
public float NoiseFloor { get; } = noiseFloor;
public float ReferenceRise { get; } = referenceRise;
public float ReferenceDecay { get; } = referenceDecay;
public float SwitchHoldSeconds { get; } = switchHoldSeconds;
public float NoteConfidenceThreshold { get; } = noteConfidenceThreshold;
public float ChordConfidenceThreshold { get; } = chordConfidenceThreshold;
public static SensitivityProfile Microphone { get; } = new(
levelGain: 1.20f,
bassGain: 1.25f,
midGain: 1.20f,
highGain: 1.15f,
spectrumGain: 1.40f,
noteStrengthGain: 0.036f,
noiseFloor: 0.0009f,
referenceRise: 0.16f,
referenceDecay: 0.010f,
switchHoldSeconds: 0.18f,
noteConfidenceThreshold: 0.11f,
chordConfidenceThreshold: 0.18f);
public static SensitivityProfile SystemOutput { get; } = new(
levelGain: 1.00f,
bassGain: 1.00f,
midGain: 1.00f,
highGain: 1.00f,
spectrumGain: 1.18f,
noteStrengthGain: 0.022f,
noiseFloor: 0.0004f,
referenceRise: 0.20f,
referenceDecay: 0.012f,
switchHoldSeconds: 0.18f,
noteConfidenceThreshold: 0.12f,
chordConfidenceThreshold: 0.14f);
}
private void ThrowIfDisposed()
{
if (_disposed)