Michal Vrba
069c2a84c5
Enhanced AudioAnalyzer with bass chroma, integrated chroma, and chord frame accumulation for more robust and stable chord detection. Added spectral peak note estimation and note continuity logic. Refined chord scoring and introduced chord lock mechanism to prevent rapid switching. Added methods for tuning reset, set, and snapshot retrieval. Updated SensitivityProfile with chord lock parameters. In Game1, added keyboard shortcuts and JSON persistence for detection tuning profiles. Improved HUD with confidence bars and new indicators. These changes increase detection accuracy, stability, and user customization.
465 lines
16 KiB
C#
465 lines
16 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Text.Json;
|
|
using FractalVisualizer.Audio;
|
|
using FractalVisualizer.Models;
|
|
using FractalVisualizer.Rendering;
|
|
using FractalVisualizer.UI;
|
|
using Microsoft.Xna.Framework;
|
|
using Microsoft.Xna.Framework.Graphics;
|
|
using Microsoft.Xna.Framework.Input;
|
|
|
|
namespace FractalVisualizer;
|
|
|
|
public class Game1 : Game
|
|
{
|
|
private readonly GraphicsDeviceManager _graphics;
|
|
|
|
private SpriteBatch? _spriteBatch;
|
|
private Texture2D? _pixel;
|
|
private HudBitmapFont? _hudFont;
|
|
|
|
private readonly AudioCapture _audioCapture;
|
|
private readonly AudioAnalyzer _audioAnalyzer;
|
|
private FractalRenderer? _fractalRenderer;
|
|
|
|
private KeyboardState _previousKeyboardState;
|
|
private AudioInputMode _currentInputMode;
|
|
private AudioFeatures _latestFeatures;
|
|
private bool _isPaused;
|
|
private bool _showHud = true;
|
|
private bool _showLegend = true;
|
|
private int _fractalPresetIndex;
|
|
private int _fps;
|
|
private int _frameCounter;
|
|
private TimeSpan _fpsElapsed;
|
|
private const int TargetFps = 120;
|
|
private RenderQuality _renderQuality = RenderQuality.Medium;
|
|
private readonly (int Width, int Height)[] _windowedResolutions =
|
|
[
|
|
(1280, 720),
|
|
(1600, 900),
|
|
(1920, 1080)
|
|
];
|
|
private int _resolutionIndex = 0;
|
|
private const string TuningProfilePath = "detection-tuning.json";
|
|
|
|
public Game1()
|
|
{
|
|
_graphics = new GraphicsDeviceManager(this);
|
|
_graphics.SynchronizeWithVerticalRetrace = false;
|
|
_graphics.PreferredBackBufferWidth = 1280;
|
|
_graphics.PreferredBackBufferHeight = 720;
|
|
Content.RootDirectory = "Content";
|
|
IsMouseVisible = true;
|
|
Window.AllowUserResizing = true;
|
|
IsFixedTimeStep = true;
|
|
TargetElapsedTime = TimeSpan.FromSeconds(1.0 / 120.0);
|
|
|
|
_audioCapture = new AudioCapture();
|
|
_audioAnalyzer = new AudioAnalyzer(_audioCapture, fftSize: 2048, spectrumBins: 64);
|
|
_currentInputMode = AudioInputMode.Microphone;
|
|
_latestFeatures = AudioFeatures.CreateEmpty(64);
|
|
}
|
|
|
|
protected override void Initialize()
|
|
{
|
|
_audioCapture.Start();
|
|
_audioAnalyzer.Start();
|
|
Window.Title = "Fractal Visualizer";
|
|
base.Initialize();
|
|
}
|
|
|
|
protected override void LoadContent()
|
|
{
|
|
_spriteBatch = new SpriteBatch(GraphicsDevice);
|
|
_fractalRenderer = new FractalRenderer(GraphicsDevice);
|
|
_fractalRenderer.SetQuality(_renderQuality);
|
|
_pixel = new Texture2D(GraphicsDevice, 1, 1, false, SurfaceFormat.Color);
|
|
_pixel.SetData([Color.White]);
|
|
_hudFont = new HudBitmapFont();
|
|
}
|
|
|
|
protected override void Update(GameTime gameTime)
|
|
{
|
|
KeyboardState keyboard = Keyboard.GetState();
|
|
if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
|
|
{
|
|
Exit();
|
|
return;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Escape))
|
|
{
|
|
if (_graphics.IsFullScreen)
|
|
{
|
|
_graphics.ToggleFullScreen();
|
|
}
|
|
else
|
|
{
|
|
Exit();
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F11))
|
|
{
|
|
ToggleFullScreen();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F))
|
|
{
|
|
ToggleFullScreen();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.R))
|
|
{
|
|
CycleWindowResolution();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D1))
|
|
{
|
|
SetRenderQuality(RenderQuality.Low);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D2))
|
|
{
|
|
SetRenderQuality(RenderQuality.Medium);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D3))
|
|
{
|
|
SetRenderQuality(RenderQuality.High);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Space))
|
|
{
|
|
_isPaused = !_isPaused;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Tab))
|
|
{
|
|
_showHud = !_showHud;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.H))
|
|
{
|
|
_showLegend = !_showLegend;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Back))
|
|
{
|
|
_audioAnalyzer.ResetTuning(_currentInputMode);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F5))
|
|
{
|
|
SaveTuningProfiles();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F6))
|
|
{
|
|
LoadTuningProfiles();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Left))
|
|
{
|
|
_fractalPresetIndex = Math.Max(0, _fractalPresetIndex - 1);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Right))
|
|
{
|
|
_fractalPresetIndex = Math.Min(23, _fractalPresetIndex + 1);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.M))
|
|
{
|
|
_currentInputMode = _currentInputMode == AudioInputMode.Microphone
|
|
? AudioInputMode.SystemOutput
|
|
: AudioInputMode.Microphone;
|
|
_audioCapture.SwitchMode(_currentInputMode);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemOpenBrackets))
|
|
{
|
|
_audioAnalyzer.AdjustChordHarmonicity(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemCloseBrackets))
|
|
{
|
|
_audioAnalyzer.AdjustChordHarmonicity(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemSemicolon))
|
|
{
|
|
_audioAnalyzer.AdjustChordSeparation(_currentInputMode, -0.005f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemQuotes))
|
|
{
|
|
_audioAnalyzer.AdjustChordSeparation(_currentInputMode, +0.005f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemComma))
|
|
{
|
|
_audioAnalyzer.AdjustNoteVoteConsensus(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemPeriod))
|
|
{
|
|
_audioAnalyzer.AdjustNoteVoteConsensus(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemMinus))
|
|
{
|
|
_audioAnalyzer.AdjustChordVoteConsensus(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemPlus))
|
|
{
|
|
_audioAnalyzer.AdjustChordVoteConsensus(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
_latestFeatures = _audioAnalyzer.GetLatestFeatures();
|
|
_fpsElapsed += gameTime.ElapsedGameTime;
|
|
_frameCounter++;
|
|
if (_fpsElapsed >= TimeSpan.FromSeconds(1))
|
|
{
|
|
_fps = _frameCounter;
|
|
_frameCounter = 0;
|
|
_fpsElapsed -= TimeSpan.FromSeconds(1);
|
|
}
|
|
|
|
Window.Title = $"Fractal Visualizer | {GetPresetName(_fractalPresetIndex)} | {_renderQuality} | Mode: {_currentInputMode} | FPS:{_fps} | BPM:{_latestFeatures.Bpm:F1}";
|
|
_previousKeyboardState = keyboard;
|
|
base.Update(gameTime);
|
|
}
|
|
|
|
protected override void Draw(GameTime gameTime)
|
|
{
|
|
GraphicsDevice.Clear(Color.Black);
|
|
|
|
if (_fractalRenderer is not null)
|
|
{
|
|
_fractalRenderer.UpdatePerformanceHint(_fps, TargetFps);
|
|
float renderTime = _isPaused ? _latestFeatures.TimeSeconds : (float)gameTime.TotalGameTime.TotalSeconds;
|
|
_fractalRenderer.Draw(_latestFeatures, renderTime, _fractalPresetIndex);
|
|
}
|
|
|
|
if (_showHud && _spriteBatch is not null && _hudFont is not null && _pixel is not null)
|
|
{
|
|
var tuning = _audioAnalyzer.GetTuning(_currentInputMode);
|
|
_spriteBatch.Begin(samplerState: SamplerState.PointClamp, blendState: BlendState.AlphaBlend);
|
|
string modeText = _currentInputMode == AudioInputMode.Microphone ? "MIC" : "SYS";
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "MODE " + modeText + " M SWITCH", 12, 10, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "PRESET " + GetPresetName(_fractalPresetIndex) + " LEFT RIGHT", 12, 30, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "LEVEL " + _latestFeatures.Level.ToString("0.000"), 12, 50, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BASS " + _latestFeatures.Bass.ToString("0.000"), 12, 70, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "MID " + _latestFeatures.Mid.ToString("0.000"), 12, 90, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "HIGH " + _latestFeatures.High.ToString("0.000"), 12, 110, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BEAT " + _latestFeatures.Beat.ToString("0.000"), 12, 130, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BPM " + _latestFeatures.Bpm.ToString("0.0"), 12, 150, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "NOTE " + GetDetectedNoteName(_latestFeatures.DetectedNoteIndex), 12, 170, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "CHORD " + GetDetectedChordName(_latestFeatures.DetectedChordRootIndex, _latestFeatures.DetectedChordType, _latestFeatures.DetectedChordConfidence), 12, 190, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "NOCHORD " + (_latestFeatures.DetectedChordRootIndex < 0 ? "YES" : "NO"), 12, 210, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "PAUSED " + (_isPaused ? "YES" : "NO"), 12, 230, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "FPS " + _fps, 12, 250, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "QUALITY " + _renderQuality + " 1 2 3", 12, 270, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "F OR F11 FULLSCREEN R RESIZE H HELP", 12, 290, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "[ ] HARM " + tuning.ChordHarmonicityThreshold.ToString("0.000"), 12, 310, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "; ' SEP " + tuning.ChordSeparationThreshold.ToString("0.000"), 12, 330, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, ", . NCONS " + tuning.NoteVoteConsensus.ToString("0.000"), 12, 350, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "- + CCONS " + tuning.ChordVoteConsensus.ToString("0.000"), 12, 370, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BACK RESET MODE F5 SAVE F6 LOAD", 12, 390, 2, Color.White);
|
|
if (_showLegend)
|
|
{
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "LEGEND SPACE PAUSE TAB HUD M INPUT <- -> PRESET", 12, 410, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "TUNING [ ] HARM ; ' SEP , . NOTE CONS - + CHORD CONS", 12, 430, 2, Color.White);
|
|
}
|
|
|
|
DrawConfidenceBar(820, 14, 420, 12, _latestFeatures.DetectedNoteStrength, new Color(48, 72, 118), new Color(120, 220, 255), "NOTE CONF", 798, 12);
|
|
DrawConfidenceBar(820, 40, 420, 12, _latestFeatures.DetectedChordConfidence, new Color(64, 52, 48), new Color(255, 182, 92), "CHORD CONF", 790, 38);
|
|
_spriteBatch.End();
|
|
}
|
|
|
|
base.Draw(gameTime);
|
|
}
|
|
|
|
protected override void Dispose(bool disposing)
|
|
{
|
|
if (disposing)
|
|
{
|
|
_audioAnalyzer.Dispose();
|
|
_audioCapture.Dispose();
|
|
_fractalRenderer?.Dispose();
|
|
_pixel?.Dispose();
|
|
_spriteBatch?.Dispose();
|
|
}
|
|
|
|
base.Dispose(disposing);
|
|
}
|
|
|
|
private bool IsPressed(KeyboardState keyboardState, Keys key)
|
|
{
|
|
return keyboardState.IsKeyDown(key) && _previousKeyboardState.IsKeyUp(key);
|
|
}
|
|
|
|
private static string GetPresetName(int presetIndex)
|
|
{
|
|
return presetIndex switch
|
|
{
|
|
0 => "JULIA",
|
|
1 => "MANDELBULB",
|
|
2 => "KALEIDO",
|
|
3 => "FLAME",
|
|
4 => "PLASMA",
|
|
5 => "NEBULA",
|
|
6 => "FLAMEGRILL",
|
|
7 => "NOTES",
|
|
8 => "FIREWORKS",
|
|
9 => "EQCITY",
|
|
10 => "METABALL",
|
|
11 => "GALAXY",
|
|
12 => "AURORA",
|
|
13 => "VORTEX",
|
|
14 => "LASERGRID",
|
|
15 => "RAINRIPPLE",
|
|
16 => "INKSMOKE",
|
|
17 => "MOSAIC",
|
|
18 => "RIBBON",
|
|
19 => "GRAINBARS",
|
|
20 => "NOTEDROP",
|
|
21 => "CHORDS",
|
|
22 => "NOTESBOOK",
|
|
23 => "NOTESCROLL",
|
|
_ => "JULIA"
|
|
};
|
|
}
|
|
|
|
private static string GetDetectedNoteName(int noteIndex)
|
|
{
|
|
return noteIndex switch
|
|
{
|
|
0 => "C",
|
|
1 => "C#",
|
|
2 => "D",
|
|
3 => "D#",
|
|
4 => "E",
|
|
5 => "F",
|
|
6 => "F#",
|
|
7 => "G",
|
|
8 => "G#",
|
|
9 => "A",
|
|
10 => "A#",
|
|
11 => "B",
|
|
_ => "-"
|
|
};
|
|
}
|
|
|
|
private static string GetDetectedChordName(int rootIndex, int chordType, float confidence)
|
|
{
|
|
if (confidence < 0.16f)
|
|
{
|
|
return "-";
|
|
}
|
|
|
|
string root = GetDetectedNoteName(rootIndex);
|
|
if (root == "-")
|
|
{
|
|
return "-";
|
|
}
|
|
|
|
return chordType switch
|
|
{
|
|
0 => root + " MAJ",
|
|
1 => root + " MIN",
|
|
2 => root + " SUS2",
|
|
3 => root + " SUS4",
|
|
4 => root + " DIM",
|
|
5 => root + " AUG",
|
|
6 => root + " 7",
|
|
7 => root + " MAJ7",
|
|
8 => root + " MIN7",
|
|
_ => root + " MAJ"
|
|
};
|
|
}
|
|
|
|
private void ToggleFullScreen()
|
|
{
|
|
_graphics.ToggleFullScreen();
|
|
}
|
|
|
|
private void SetRenderQuality(RenderQuality quality)
|
|
{
|
|
_renderQuality = quality;
|
|
_fractalRenderer?.SetQuality(quality);
|
|
}
|
|
|
|
private void CycleWindowResolution()
|
|
{
|
|
if (_graphics.IsFullScreen)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_resolutionIndex = (_resolutionIndex + 1) % _windowedResolutions.Length;
|
|
(int width, int height) = _windowedResolutions[_resolutionIndex];
|
|
_graphics.PreferredBackBufferWidth = width;
|
|
_graphics.PreferredBackBufferHeight = height;
|
|
_graphics.ApplyChanges();
|
|
}
|
|
|
|
private void DrawConfidenceBar(int x, int y, int width, int height, float value, Color bg, Color fg, string label, int labelX, int labelY)
|
|
{
|
|
if (_spriteBatch is null || _pixel is null || _hudFont is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
float clamped = Math.Clamp(value, 0f, 1f);
|
|
int fillWidth = (int)MathF.Round(width * clamped);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x, y, width, height), bg);
|
|
if (fillWidth > 0)
|
|
{
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x, y, fillWidth, height), fg);
|
|
}
|
|
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y - 1, width + 2, 1), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y + height, width + 2, 1), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y - 1, 1, height + 2), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x + width, y - 1, 1, height + 2), Color.White * 0.45f);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, label + " " + clamped.ToString("0.000"), labelX, labelY, 2, Color.White);
|
|
}
|
|
|
|
private void SaveTuningProfiles()
|
|
{
|
|
TuningProfiles profile = new(
|
|
_audioAnalyzer.GetTuning(AudioInputMode.Microphone),
|
|
_audioAnalyzer.GetTuning(AudioInputMode.SystemOutput));
|
|
string json = JsonSerializer.Serialize(profile, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText(TuningProfilePath, json);
|
|
}
|
|
|
|
private void LoadTuningProfiles()
|
|
{
|
|
if (!File.Exists(TuningProfilePath))
|
|
{
|
|
return;
|
|
}
|
|
|
|
string json = File.ReadAllText(TuningProfilePath);
|
|
TuningProfiles? profile = JsonSerializer.Deserialize<TuningProfiles>(json);
|
|
if (profile is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_audioAnalyzer.SetTuning(AudioInputMode.Microphone, profile.Microphone);
|
|
_audioAnalyzer.SetTuning(AudioInputMode.SystemOutput, profile.SystemOutput);
|
|
}
|
|
|
|
private readonly record struct TuningProfiles(
|
|
AudioAnalyzer.DetectionTuningSnapshot Microphone,
|
|
AudioAnalyzer.DetectionTuningSnapshot SystemOutput);
|
|
}
|