466 lines
16 KiB
C#
466 lines
16 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Text.Json;
|
|
using FractalVisualizer.Audio;
|
|
using FractalVisualizer.Models;
|
|
using FractalVisualizer.Rendering;
|
|
using FractalVisualizer.UI;
|
|
using Microsoft.Xna.Framework;
|
|
using Microsoft.Xna.Framework.Graphics;
|
|
using Microsoft.Xna.Framework.Input;
|
|
|
|
namespace FractalVisualizer;
|
|
|
|
public class Game1 : Game
|
|
{
|
|
private readonly GraphicsDeviceManager _graphics;
|
|
|
|
private SpriteBatch? _spriteBatch;
|
|
private Texture2D? _pixel;
|
|
private HudBitmapFont? _hudFont;
|
|
|
|
private readonly AudioCapture _audioCapture;
|
|
private readonly AudioAnalyzer _audioAnalyzer;
|
|
private FractalRenderer? _fractalRenderer;
|
|
|
|
private KeyboardState _previousKeyboardState;
|
|
private AudioInputMode _currentInputMode;
|
|
private AudioFeatures _latestFeatures;
|
|
private bool _isPaused;
|
|
private bool _showHud = true;
|
|
private bool _showLegend = true;
|
|
private int _fractalPresetIndex;
|
|
private int _fps;
|
|
private int _frameCounter;
|
|
private TimeSpan _fpsElapsed;
|
|
private const int TargetFps = 120;
|
|
private RenderQuality _renderQuality = RenderQuality.Medium;
|
|
private readonly (int Width, int Height)[] _windowedResolutions =
|
|
[
|
|
(1280, 720),
|
|
(1600, 900),
|
|
(1920, 1080)
|
|
];
|
|
private int _resolutionIndex = 0;
|
|
private const string TuningProfilePath = "detection-tuning.json";
|
|
|
|
public Game1()
|
|
{
|
|
_graphics = new GraphicsDeviceManager(this);
|
|
_graphics.SynchronizeWithVerticalRetrace = false;
|
|
_graphics.PreferredBackBufferWidth = 1280;
|
|
_graphics.PreferredBackBufferHeight = 720;
|
|
Content.RootDirectory = "Content";
|
|
IsMouseVisible = true;
|
|
Window.AllowUserResizing = true;
|
|
IsFixedTimeStep = true;
|
|
TargetElapsedTime = TimeSpan.FromSeconds(1.0 / 120.0);
|
|
|
|
_audioCapture = new AudioCapture();
|
|
_audioAnalyzer = new AudioAnalyzer(_audioCapture, fftSize: 2048, spectrumBins: 64);
|
|
_currentInputMode = AudioInputMode.Microphone;
|
|
_latestFeatures = AudioFeatures.CreateEmpty(64);
|
|
}
|
|
|
|
protected override void Initialize()
|
|
{
|
|
_audioCapture.Start();
|
|
_audioAnalyzer.Start();
|
|
Window.Title = "Fractal Visualizer";
|
|
base.Initialize();
|
|
}
|
|
|
|
protected override void LoadContent()
|
|
{
|
|
_spriteBatch = new SpriteBatch(GraphicsDevice);
|
|
_fractalRenderer = new FractalRenderer(GraphicsDevice);
|
|
_fractalRenderer.SetQuality(_renderQuality);
|
|
_pixel = new Texture2D(GraphicsDevice, 1, 1, false, SurfaceFormat.Color);
|
|
_pixel.SetData([Color.White]);
|
|
_hudFont = new HudBitmapFont();
|
|
}
|
|
|
|
protected override void Update(GameTime gameTime)
|
|
{
|
|
KeyboardState keyboard = Keyboard.GetState();
|
|
if (GamePad.GetState(PlayerIndex.One).Buttons.Back == ButtonState.Pressed)
|
|
{
|
|
Exit();
|
|
return;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Escape))
|
|
{
|
|
if (_graphics.IsFullScreen)
|
|
{
|
|
_graphics.ToggleFullScreen();
|
|
}
|
|
else
|
|
{
|
|
Exit();
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F11))
|
|
{
|
|
ToggleFullScreen();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F))
|
|
{
|
|
ToggleFullScreen();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.R))
|
|
{
|
|
CycleWindowResolution();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D1))
|
|
{
|
|
SetRenderQuality(RenderQuality.Low);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D2))
|
|
{
|
|
SetRenderQuality(RenderQuality.Medium);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.D3))
|
|
{
|
|
SetRenderQuality(RenderQuality.High);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Space))
|
|
{
|
|
_isPaused = !_isPaused;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Tab))
|
|
{
|
|
_showHud = !_showHud;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.H))
|
|
{
|
|
_showLegend = !_showLegend;
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Back))
|
|
{
|
|
_audioAnalyzer.ResetTuning(_currentInputMode);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F5))
|
|
{
|
|
SaveTuningProfiles();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.F6))
|
|
{
|
|
LoadTuningProfiles();
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Left))
|
|
{
|
|
_fractalPresetIndex = Math.Max(0, _fractalPresetIndex - 1);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.Right))
|
|
{
|
|
_fractalPresetIndex = Math.Min(23, _fractalPresetIndex + 1);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.M))
|
|
{
|
|
_currentInputMode = _currentInputMode == AudioInputMode.Microphone
|
|
? AudioInputMode.SystemOutput
|
|
: AudioInputMode.Microphone;
|
|
_audioCapture.SwitchMode(_currentInputMode);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemOpenBrackets))
|
|
{
|
|
_audioAnalyzer.AdjustChordHarmonicity(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemCloseBrackets))
|
|
{
|
|
_audioAnalyzer.AdjustChordHarmonicity(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemSemicolon))
|
|
{
|
|
_audioAnalyzer.AdjustChordSeparation(_currentInputMode, -0.005f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemQuotes))
|
|
{
|
|
_audioAnalyzer.AdjustChordSeparation(_currentInputMode, +0.005f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemComma))
|
|
{
|
|
_audioAnalyzer.AdjustNoteVoteConsensus(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemPeriod))
|
|
{
|
|
_audioAnalyzer.AdjustNoteVoteConsensus(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemMinus))
|
|
{
|
|
_audioAnalyzer.AdjustChordVoteConsensus(_currentInputMode, -0.01f);
|
|
}
|
|
|
|
if (IsPressed(keyboard, Keys.OemPlus))
|
|
{
|
|
_audioAnalyzer.AdjustChordVoteConsensus(_currentInputMode, +0.01f);
|
|
}
|
|
|
|
_latestFeatures = _audioAnalyzer.GetLatestFeatures();
|
|
_fpsElapsed += gameTime.ElapsedGameTime;
|
|
_frameCounter++;
|
|
if (_fpsElapsed >= TimeSpan.FromSeconds(1))
|
|
{
|
|
_fps = _frameCounter;
|
|
_frameCounter = 0;
|
|
_fpsElapsed -= TimeSpan.FromSeconds(1);
|
|
}
|
|
|
|
Window.Title = $"Fractal Visualizer | {GetPresetName(_fractalPresetIndex)} | {_renderQuality} | Mode: {_currentInputMode} | FPS:{_fps} | BPM:{_latestFeatures.Bpm:F1}";
|
|
_previousKeyboardState = keyboard;
|
|
base.Update(gameTime);
|
|
}
|
|
|
|
protected override void Draw(GameTime gameTime)
|
|
{
|
|
GraphicsDevice.Clear(Color.Black);
|
|
|
|
if (_fractalRenderer is not null)
|
|
{
|
|
_fractalRenderer.UpdatePerformanceHint(_fps, TargetFps);
|
|
float renderTime = _isPaused ? _latestFeatures.TimeSeconds : (float)gameTime.TotalGameTime.TotalSeconds;
|
|
_fractalRenderer.Draw(_latestFeatures, renderTime, _fractalPresetIndex);
|
|
}
|
|
|
|
if (_showHud && _spriteBatch is not null && _hudFont is not null && _pixel is not null)
|
|
{
|
|
var tuning = _audioAnalyzer.GetTuning(_currentInputMode);
|
|
_spriteBatch.Begin(samplerState: SamplerState.PointClamp, blendState: BlendState.AlphaBlend);
|
|
string modeText = _currentInputMode == AudioInputMode.Microphone ? "MIC" : "SYS";
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "MODE " + modeText + " M SWITCH", 12, 10, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "PRESET " + GetPresetName(_fractalPresetIndex) + " LEFT RIGHT", 12, 30, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "LEVEL " + _latestFeatures.Level.ToString("0.000"), 12, 50, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BASS " + _latestFeatures.Bass.ToString("0.000"), 12, 70, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "MID " + _latestFeatures.Mid.ToString("0.000"), 12, 90, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "HIGH " + _latestFeatures.High.ToString("0.000"), 12, 110, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BEAT " + _latestFeatures.Beat.ToString("0.000"), 12, 130, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BPM " + _latestFeatures.Bpm.ToString("0.0"), 12, 150, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "NOTE " + GetDetectedNoteName(_latestFeatures.DetectedNoteIndex), 12, 170, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "CHORD " + GetDetectedChordName(_latestFeatures.DetectedChordRootIndex, _latestFeatures.DetectedChordType, _latestFeatures.DetectedChordConfidence), 12, 190, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "NOCHORD " + (_latestFeatures.DetectedChordRootIndex < 0 ? "YES" : "NO"), 12, 210, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "PAUSED " + (_isPaused ? "YES" : "NO"), 12, 230, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "FPS " + _fps, 12, 250, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "QUALITY " + _renderQuality + " 1 2 3", 12, 270, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "F OR F11 FULLSCREEN R RESIZE H HELP", 12, 290, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "[ ] HARM " + tuning.ChordHarmonicityThreshold.ToString("0.000"), 12, 310, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "; ' SEP " + tuning.ChordSeparationThreshold.ToString("0.000"), 12, 330, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, ", . NCONS " + tuning.NoteVoteConsensus.ToString("0.000"), 12, 350, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "- + CCONS " + tuning.ChordVoteConsensus.ToString("0.000"), 12, 370, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "BACK RESET MODE F5 SAVE F6 LOAD", 12, 390, 2, Color.White);
|
|
if (_showLegend)
|
|
{
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "LEGEND SPACE PAUSE TAB HUD M INPUT <- -> PRESET", 12, 410, 2, Color.White);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, "TUNING [ ] HARM ; ' SEP , . NOTE CONS - + CHORD CONS", 12, 430, 2, Color.White);
|
|
}
|
|
|
|
DrawConfidenceBar(820, 14, 420, 12, _latestFeatures.DetectedNoteStrength, new Color(48, 72, 118), new Color(120, 220, 255), "NOTE CONF", 798, 12);
|
|
DrawConfidenceBar(820, 40, 420, 12, _latestFeatures.DetectedChordConfidence, new Color(64, 52, 48), new Color(255, 182, 92), "CHORD CONF", 790, 38);
|
|
_spriteBatch.End();
|
|
}
|
|
|
|
base.Draw(gameTime);
|
|
}
|
|
|
|
protected override void Dispose(bool disposing)
|
|
{
|
|
if (disposing)
|
|
{
|
|
_audioAnalyzer.Dispose();
|
|
_audioCapture.Dispose();
|
|
_fractalRenderer?.Dispose();
|
|
_pixel?.Dispose();
|
|
_spriteBatch?.Dispose();
|
|
}
|
|
|
|
base.Dispose(disposing);
|
|
}
|
|
|
|
private bool IsPressed(KeyboardState keyboardState, Keys key)
|
|
{
|
|
return keyboardState.IsKeyDown(key) && _previousKeyboardState.IsKeyUp(key);
|
|
}
|
|
|
|
private static string GetPresetName(int presetIndex)
|
|
{
|
|
return presetIndex switch
|
|
{
|
|
0 => "JULIA",
|
|
1 => "MANDELBULB",
|
|
2 => "KALEIDO",
|
|
3 => "FLAME",
|
|
4 => "PLASMA",
|
|
5 => "NEBULA",
|
|
6 => "FLAMEGRILL",
|
|
7 => "NOTES",
|
|
8 => "FIREWORKS",
|
|
9 => "EQCITY",
|
|
10 => "METABALL",
|
|
11 => "GALAXY",
|
|
12 => "AURORA",
|
|
13 => "VORTEX",
|
|
14 => "LASERGRID",
|
|
15 => "RAINRIPPLE",
|
|
16 => "INKSMOKE",
|
|
17 => "MOSAIC",
|
|
18 => "RIBBON",
|
|
19 => "GRAINBARS",
|
|
20 => "NOTEDROP",
|
|
21 => "CHORDS",
|
|
22 => "NOTESBOOK",
|
|
23 => "NOTESCROLL",
|
|
_ => "JULIA"
|
|
};
|
|
}
|
|
|
|
private static string GetDetectedNoteName(int noteIndex)
|
|
{
|
|
return noteIndex switch
|
|
{
|
|
0 => "C",
|
|
1 => "C#",
|
|
2 => "D",
|
|
3 => "D#",
|
|
4 => "E",
|
|
5 => "F",
|
|
6 => "F#",
|
|
7 => "G",
|
|
8 => "G#",
|
|
9 => "A",
|
|
10 => "A#",
|
|
11 => "B",
|
|
_ => "-"
|
|
};
|
|
}
|
|
|
|
private static string GetDetectedChordName(int rootIndex, int chordType, float confidence)
|
|
{
|
|
if (confidence < 0.16f)
|
|
{
|
|
return "-";
|
|
}
|
|
|
|
string root = GetDetectedNoteName(rootIndex);
|
|
if (root == "-")
|
|
{
|
|
return "-";
|
|
}
|
|
|
|
return chordType switch
|
|
{
|
|
0 => root + " MAJ",
|
|
1 => root + " MIN",
|
|
2 => root + " SUS2",
|
|
3 => root + " SUS4",
|
|
4 => root + " DIM",
|
|
5 => root + " AUG",
|
|
6 => root + " 7",
|
|
7 => root + " MAJ7",
|
|
8 => root + " MIN7",
|
|
_ => root + " MAJ"
|
|
};
|
|
}
|
|
|
|
private void ToggleFullScreen()
|
|
{
|
|
_graphics.ToggleFullScreen();
|
|
}
|
|
|
|
private void SetRenderQuality(RenderQuality quality)
|
|
{
|
|
_renderQuality = quality;
|
|
_fractalRenderer?.SetQuality(quality);
|
|
}
|
|
|
|
private void CycleWindowResolution()
|
|
{
|
|
if (_graphics.IsFullScreen)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_resolutionIndex = (_resolutionIndex + 1) % _windowedResolutions.Length;
|
|
(int width, int height) = _windowedResolutions[_resolutionIndex];
|
|
_graphics.PreferredBackBufferWidth = width;
|
|
_graphics.PreferredBackBufferHeight = height;
|
|
_graphics.ApplyChanges();
|
|
}
|
|
|
|
private void DrawConfidenceBar(int x, int y, int width, int height, float value, Color bg, Color fg, string label, int labelX, int labelY)
|
|
{
|
|
if (_spriteBatch is null || _pixel is null || _hudFont is null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
float clamped = Math.Clamp(value, 0f, 1f);
|
|
int fillWidth = (int)MathF.Round(width * clamped);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x, y, width, height), bg);
|
|
if (fillWidth > 0)
|
|
{
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x, y, fillWidth, height), fg);
|
|
}
|
|
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y - 1, width + 2, 1), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y + height, width + 2, 1), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x - 1, y - 1, 1, height + 2), Color.White * 0.45f);
|
|
_spriteBatch.Draw(_pixel, new Rectangle(x + width, y - 1, 1, height + 2), Color.White * 0.45f);
|
|
_hudFont.DrawText(_spriteBatch, _pixel, label + " " + clamped.ToString("0.000"), labelX, labelY, 2, Color.White);
|
|
}
|
|
|
|
private void SaveTuningProfiles()
|
|
{
|
|
TuningProfiles profile = new(
|
|
_audioAnalyzer.GetTuning(AudioInputMode.Microphone),
|
|
_audioAnalyzer.GetTuning(AudioInputMode.SystemOutput));
|
|
string json = JsonSerializer.Serialize(profile, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText(TuningProfilePath, json);
|
|
}
|
|
|
|
private void LoadTuningProfiles()
|
|
{
|
|
if (!File.Exists(TuningProfilePath))
|
|
{
|
|
return;
|
|
}
|
|
|
|
string json = File.ReadAllText(TuningProfilePath);
|
|
TuningProfiles? profile = JsonSerializer.Deserialize<TuningProfiles>(json);
|
|
if (!profile.HasValue)
|
|
{
|
|
return;
|
|
}
|
|
|
|
TuningProfiles value = profile.Value;
|
|
_audioAnalyzer.SetTuning(AudioInputMode.Microphone, value.Microphone);
|
|
_audioAnalyzer.SetTuning(AudioInputMode.SystemOutput, value.SystemOutput);
|
|
}
|
|
|
|
private readonly record struct TuningProfiles(
|
|
AudioAnalyzer.DetectionTuningSnapshot Microphone,
|
|
AudioAnalyzer.DetectionTuningSnapshot SystemOutput);
|
|
}
|