diff --git a/.gitignore b/.gitignore index 9491a2f..009e6ee 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,7 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd +/.dotnet-home/.dotnet/TelemetryStorageService +/.dotnet-home/.dotnet +/.nuget diff --git a/FractalVisualizer/Audio/AudioAnalyzer.cs b/FractalVisualizer/Audio/AudioAnalyzer.cs index 1834370..812aa47 100644 --- a/FractalVisualizer/Audio/AudioAnalyzer.cs +++ b/FractalVisualizer/Audio/AudioAnalyzer.cs @@ -50,6 +50,8 @@ public sealed class AudioAnalyzer : IDisposable private int _stableNoteIndex = -1; private float _stableNoteStrength; private float _lastNoteChangeTimeSeconds; + private bool _noteIsActive; + private int _activeNoteIndex = -1; private Snapshot _latestSnapshot; private bool _disposed; @@ -295,7 +297,7 @@ public sealed class AudioAnalyzer : IDisposable int noteIndex = GetNoteIndexFromFrequency(dominantFrequency); float whistleBoost = GetWhistleBoost(dominantFrequency); float dominantStrength = Clamp01(dominantMagnitude * profile.NoteStrengthGain * (1f + whistleBoost * 0.65f)); - DetectChord(_chroma, out int chordRoot, out int chordType, out float chordConfidence); + DetectChord(_chroma, _smoothedLevel, profile, out int chordRoot, out int chordType, out float chordConfidence); ApplyDetectionStability( now, profile, @@ -309,6 +311,7 @@ public sealed class AudioAnalyzer : IDisposable out int stableChordRoot, out int stableChordType, out float stableChordConfidence); + bool noteOnset = DetectNoteOnset(stableNoteIndex, stableNoteStrength, profile); AudioFeatures features = new() { @@ -321,6 +324,7 @@ public sealed class AudioAnalyzer : IDisposable DominantFrequencyHz = dominantFrequency, DetectedNoteIndex = stableNoteIndex, DetectedNoteStrength = stableNoteStrength, + NoteOnset = noteOnset, DetectedChordRootIndex = stableChordRoot, DetectedChordType = stableChordType, DetectedChordConfidence = stableChordConfidence, @@ -404,8 +408,16 @@ public sealed class AudioAnalyzer : IDisposable return note; } - private void DetectChord(float[] chroma, out int root, out int type, out float confidence) + private void DetectChord(float[] chroma, float level, SensitivityProfile profile, out int root, out int type, out float confidence) { + if (level < profile.ChordLevelThreshold) + { + root = -1; + type = -1; + confidence = 0f; + return; + } + float total = 0f; for (int i = 0; i < chroma.Length; i++) { @@ -463,8 +475,6 @@ public sealed class AudioAnalyzer : IDisposable } } - _lastChordRoot = bestRoot; - _lastChordType = bestType; root = bestRoot; type = bestType; confidence = conf; @@ -599,6 +609,41 @@ public sealed class AudioAnalyzer : IDisposable stableChordType = _lastChordType; stableChordConfidence = chordConfidence; } + else if (_lastChordRoot >= 0 && (_smoothedLevel <= profile.ChordLevelThreshold || stableNoteStrength <= profile.NoteReleaseThreshold)) + { + // Clear stale chord label when audio falls back to near-silence. + if ((now - _lastChordChangeTimeSeconds) >= profile.SwitchHoldSeconds) + { + _lastChordRoot = -1; + _lastChordType = -1; + stableChordRoot = -1; + stableChordType = -1; + stableChordConfidence = 0f; + } + } + } + + private bool DetectNoteOnset(int stableNoteIndex, float stableNoteStrength, SensitivityProfile profile) + { + if (stableNoteIndex >= 0 && stableNoteStrength >= profile.NoteOnsetThreshold) + { + if (!_noteIsActive || _activeNoteIndex != stableNoteIndex) + { + _noteIsActive = true; + _activeNoteIndex = stableNoteIndex; + return true; + } + + return false; + } + + if (_noteIsActive && stableNoteStrength <= profile.NoteReleaseThreshold) + { + _noteIsActive = false; + _activeNoteIndex = -1; + } + + return false; } private readonly struct SensitivityProfile( @@ -613,7 +658,10 @@ public sealed class AudioAnalyzer : IDisposable float referenceDecay, float switchHoldSeconds, float noteConfidenceThreshold, - float chordConfidenceThreshold) + float chordConfidenceThreshold, + float noteOnsetThreshold, + float noteReleaseThreshold, + float chordLevelThreshold) { public float LevelGain { get; } = levelGain; public float BassGain { get; } = bassGain; @@ -627,6 +675,9 @@ public sealed class AudioAnalyzer : IDisposable public float SwitchHoldSeconds { get; } = switchHoldSeconds; public float NoteConfidenceThreshold { get; } = noteConfidenceThreshold; public float ChordConfidenceThreshold { get; } = chordConfidenceThreshold; + public float NoteOnsetThreshold { get; } = noteOnsetThreshold; + public float NoteReleaseThreshold { get; } = noteReleaseThreshold; + public float ChordLevelThreshold { get; } = chordLevelThreshold; public static SensitivityProfile Microphone { get; } = new( levelGain: 1.20f, @@ -640,7 +691,10 @@ public sealed class AudioAnalyzer : IDisposable referenceDecay: 0.010f, switchHoldSeconds: 0.18f, noteConfidenceThreshold: 0.11f, - chordConfidenceThreshold: 0.18f); + chordConfidenceThreshold: 0.18f, + noteOnsetThreshold: 0.20f, + noteReleaseThreshold: 0.08f, + chordLevelThreshold: 0.10f); public static SensitivityProfile SystemOutput { get; } = new( levelGain: 1.00f, @@ -654,7 +708,10 @@ public sealed class AudioAnalyzer : IDisposable referenceDecay: 0.012f, switchHoldSeconds: 0.18f, noteConfidenceThreshold: 0.12f, - chordConfidenceThreshold: 0.14f); + chordConfidenceThreshold: 0.14f, + noteOnsetThreshold: 0.18f, + noteReleaseThreshold: 0.07f, + chordLevelThreshold: 0.08f); } private void ThrowIfDisposed() diff --git a/FractalVisualizer/Game1.cs b/FractalVisualizer/Game1.cs index 426e775..37cb2f0 100644 --- a/FractalVisualizer/Game1.cs +++ b/FractalVisualizer/Game1.cs @@ -145,7 +145,7 @@ public class Game1 : Game if (IsPressed(keyboard, Keys.Right)) { - _fractalPresetIndex = Math.Min(21, _fractalPresetIndex + 1); + _fractalPresetIndex = Math.Min(23, _fractalPresetIndex + 1); } if (IsPressed(keyboard, Keys.M)) @@ -195,7 +195,7 @@ public class Game1 : Game _hudFont.DrawText(_spriteBatch, _pixel, "BEAT " + _latestFeatures.Beat.ToString("0.000"), 12, 130, 2, Color.White); _hudFont.DrawText(_spriteBatch, _pixel, "BPM " + _latestFeatures.Bpm.ToString("0.0"), 12, 150, 2, Color.White); _hudFont.DrawText(_spriteBatch, _pixel, "NOTE " + GetDetectedNoteName(_latestFeatures.DetectedNoteIndex), 12, 170, 2, Color.White); - _hudFont.DrawText(_spriteBatch, _pixel, "CHORD " + GetDetectedChordName(_latestFeatures.DetectedChordRootIndex, _latestFeatures.DetectedChordType), 12, 190, 2, Color.White); + _hudFont.DrawText(_spriteBatch, _pixel, "CHORD " + GetDetectedChordName(_latestFeatures.DetectedChordRootIndex, _latestFeatures.DetectedChordType, _latestFeatures.DetectedChordConfidence), 12, 190, 2, Color.White); _hudFont.DrawText(_spriteBatch, _pixel, "PAUSED " + (_isPaused ? "YES" : "NO"), 12, 210, 2, Color.White); _hudFont.DrawText(_spriteBatch, _pixel, "FPS " + _fps, 12, 230, 2, Color.White); _hudFont.DrawText(_spriteBatch, _pixel, "QUALITY " + _renderQuality + " 1 2 3", 12, 250, 2, Color.White); @@ -251,6 +251,8 @@ public class Game1 : Game 19 => "GRAINBARS", 20 => "NOTEDROP", 21 => "CHORDS", + 22 => "NOTESBOOK", + 23 => "NOTESCROLL", _ => "JULIA" }; } @@ -275,8 +277,13 @@ public class Game1 : Game }; } - private static string GetDetectedChordName(int rootIndex, int chordType) + private static string GetDetectedChordName(int rootIndex, int chordType, float confidence) { + if (confidence < 0.16f) + { + return "-"; + } + string root = GetDetectedNoteName(rootIndex); if (root == "-") { diff --git a/FractalVisualizer/Models/AudioFeatures.cs b/FractalVisualizer/Models/AudioFeatures.cs index fd5be7d..83d1b68 100644 --- a/FractalVisualizer/Models/AudioFeatures.cs +++ b/FractalVisualizer/Models/AudioFeatures.cs @@ -11,6 +11,7 @@ public struct AudioFeatures public float DominantFrequencyHz; public int DetectedNoteIndex; public float DetectedNoteStrength; + public bool NoteOnset; public int DetectedChordRootIndex; public int DetectedChordType; public float DetectedChordConfidence; diff --git a/FractalVisualizer/Rendering/FractalRenderer.cs b/FractalVisualizer/Rendering/FractalRenderer.cs index 2323542..c066315 100644 --- a/FractalVisualizer/Rendering/FractalRenderer.cs +++ b/FractalVisualizer/Rendering/FractalRenderer.cs @@ -19,6 +19,8 @@ public sealed class FractalRenderer : IDisposable private Color[] _previousFrame; private readonly List _notes = []; private readonly List _detectedNotes = []; + private readonly List _staffNotes = []; + private readonly List _scrollStaffNotes = []; private readonly List _fireworkRockets = []; private readonly List _fireworkSparks = []; private readonly Random _rng = new(); @@ -45,8 +47,9 @@ public sealed class FractalRenderer : IDisposable }; private float _noteSpawnAccumulator; - private float _detectedNoteAccumulator; private float _fireworkSpawnAccumulator; + private float _sheetOffsetX; + private float _lastRenderTimeSeconds = -1f; private int _lastChordCode = -1; private float _chordFlash; private float _runtimePerformanceScale = 1f; @@ -140,11 +143,14 @@ public sealed class FractalRenderer : IDisposable _previousFrame = new Color[_internalWidth * _internalHeight]; _notes.Clear(); _detectedNotes.Clear(); + _staffNotes.Clear(); + _scrollStaffNotes.Clear(); _fireworkRockets.Clear(); _fireworkSparks.Clear(); _noteSpawnAccumulator = 0f; - _detectedNoteAccumulator = 0f; _fireworkSpawnAccumulator = 0f; + _sheetOffsetX = 0f; + _lastRenderTimeSeconds = -1f; } private int ScaleCount(int baseCount, int minCount = 1) @@ -234,6 +240,12 @@ public sealed class FractalRenderer : IDisposable case 21: RenderChordCenter(features, timeSeconds); break; + case 22: + RenderMusicStaffNotes(features, timeSeconds); + break; + case 23: + RenderScrollingMusicStaff(features, timeSeconds); + break; default: RenderJulia(features, timeSeconds); break; @@ -1204,13 +1216,9 @@ public sealed class FractalRenderer : IDisposable } } - float whistleBoost = GetWhistleSpawnBoost(features.DominantFrequencyHz); - float spawnRate = features.DetectedNoteStrength * (0.9f + whistleBoost * 1.7f + Clamp01(features.Beat) * 0.35f); - _detectedNoteAccumulator += spawnRate * _runtimePerformanceScale; - while (_detectedNoteAccumulator >= 1f) + if (features.NoteOnset) { SpawnDetectedNote(features); - _detectedNoteAccumulator -= 1f; } for (int i = _detectedNotes.Count - 1; i >= 0; i--) @@ -1232,17 +1240,6 @@ public sealed class FractalRenderer : IDisposable } } - private static float GetWhistleSpawnBoost(float frequencyHz) - { - if (frequencyHz < 800f || frequencyHz > 5000f) - { - return 0f; - } - - float norm = (frequencyHz - 800f) / (5000f - 800f); - return Clamp01(norm); - } - private void RenderChordCenter(AudioFeatures features, float timeSeconds) { for (int y = 0; y < _internalHeight; y++) @@ -1284,6 +1281,248 @@ public sealed class FractalRenderer : IDisposable DrawTextPixels(chordText, x0, y0, scale, textColor, 0.95f); } + private void RenderMusicStaffNotes(AudioFeatures features, float timeSeconds) + { + for (int y = 0; y < _internalHeight; y++) + { + float t = y / (float)(_internalHeight - 1); + Color bg = new(0.98f - t * 0.05f, 0.97f - t * 0.05f, 0.93f - t * 0.04f, 1f); + int row = y * _internalWidth; + for (int x = 0; x < _internalWidth; x++) + { + _pixelBuffer[row + x] = bg; + } + } + + int left = (int)(_internalWidth * 0.08f); + int right = (int)(_internalWidth * 0.92f); + int top = (int)(_internalHeight * 0.32f); + int spacing = Math.Max(5, _internalHeight / 24); + + for (int i = 0; i < 5; i++) + { + int y = top + i * spacing; + DrawRect(left, y, right - left, 1, new Vector3(0.15f, 0.15f, 0.18f), 0.85f); + } + + if (features.NoteOnset) + { + SpawnStaffNote(features, left, right, top, spacing); + } + + for (int i = _staffNotes.Count - 1; i >= 0; i--) + { + StaffNoteParticle n = _staffNotes[i]; + n.Life -= 0.0065f; + n.X += n.Vx; + if (n.Life <= 0f || n.X > right + 20) + { + _staffNotes.RemoveAt(i); + continue; + } + + _staffNotes[i] = n; + DrawStaffNote(n, top, spacing); + } + } + + private void RenderScrollingMusicStaff(AudioFeatures features, float timeSeconds) + { + float dt; + if (_lastRenderTimeSeconds < 0f) + { + dt = 1f / 120f; + } + else + { + dt = Math.Clamp(timeSeconds - _lastRenderTimeSeconds, 1f / 240f, 1f / 20f); + } + + _lastRenderTimeSeconds = timeSeconds; + + for (int y = 0; y < _internalHeight; y++) + { + float t = y / (float)(_internalHeight - 1); + Color bg = new(0.985f - t * 0.04f, 0.98f - t * 0.04f, 0.95f - t * 0.03f, 1f); + int row = y * _internalWidth; + for (int x = 0; x < _internalWidth; x++) + { + _pixelBuffer[row + x] = bg; + } + } + + int left = (int)(_internalWidth * 0.06f); + int right = (int)(_internalWidth * 0.94f); + int top = (int)(_internalHeight * 0.34f); + int spacing = Math.Max(5, _internalHeight / 24); + + float bpm = features.Bpm > 0f ? features.Bpm : 90f; + float pxPerSecond = 14f + bpm * 0.24f; + float speed = pxPerSecond * dt; + _sheetOffsetX += speed; + int measureSpacing = Math.Max(30, _internalWidth / 6); + if (_sheetOffsetX >= measureSpacing) + { + _sheetOffsetX -= measureSpacing; + } + + for (int i = 0; i < 5; i++) + { + int y = top + i * spacing; + DrawRect(left, y, right - left, 1, new Vector3(0.14f, 0.14f, 0.16f), 0.85f); + } + + for (int x = right - (int)_sheetOffsetX; x > left; x -= measureSpacing) + { + DrawRect(x, top - spacing / 2, 1, spacing * 5, new Vector3(0.2f, 0.2f, 0.24f), 0.55f); + } + + if (features.NoteOnset) + { + SpawnScrollingStaffNote(features, right, top, spacing); + } + + for (int i = _scrollStaffNotes.Count - 1; i >= 0; i--) + { + ScrollingStaffNoteParticle n = _scrollStaffNotes[i]; + n.X -= speed * (0.95f + n.SpeedJitter); + n.Life -= 0.0045f; + if (n.Life <= 0f || n.X < left - 30) + { + _scrollStaffNotes.RemoveAt(i); + continue; + } + + _scrollStaffNotes[i] = n; + DrawScrollingStaffNote(n, top, spacing); + } + } + + private void SpawnScrollingStaffNote(AudioFeatures features, int right, int top, int spacing) + { + if (features.DetectedNoteIndex < 0 || _scrollStaffNotes.Count >= ScaleCount(180, 30)) + { + return; + } + + int staffStep = NoteToStaffStep(features.DetectedNoteIndex); + int staffCenterY = top + spacing * 4; + int y = staffCenterY - (staffStep * (spacing / 2)); + int x = right - 8 + _rng.Next(0, 8); + + _scrollStaffNotes.Add(new ScrollingStaffNoteParticle + { + X = x, + Y = y, + Life = 1f, + UpStem = _rng.NextDouble() > 0.5, + Filled = features.DetectedNoteStrength > 0.20f, + SpeedJitter = (float)_rng.NextDouble() * 0.35f + }); + } + + private void DrawScrollingStaffNote(ScrollingStaffNoteParticle n, int top, int spacing) + { + int x = (int)n.X; + int y = n.Y; + float a = Clamp01(n.Life); + Vector3 ink = new(0.07f, 0.07f, 0.1f); + + int rx = Math.Max(2, spacing / 3); + int ry = Math.Max(2, spacing / 4 + 1); + DrawEllipse(x, y, rx, ry, ink, a, n.Filled); + + int stemH = spacing * 3; + if (n.UpStem) + { + DrawRect(x + rx, y - stemH, 1, stemH, ink, a); + } + else + { + DrawRect(x - rx, y, 1, stemH, ink, a); + } + + int staffTop = top; + int staffBottom = top + spacing * 4; + if (y < staffTop - spacing / 2 || y > staffBottom + spacing / 2) + { + DrawRect(x - spacing / 2, y, spacing, 1, ink, a * 0.9f); + } + } + + private void SpawnStaffNote(AudioFeatures features, int left, int right, int top, int spacing) + { + if (features.DetectedNoteIndex < 0 || _staffNotes.Count >= ScaleCount(120, 20)) + { + return; + } + + int noteIndex = features.DetectedNoteIndex; + int staffStep = NoteToStaffStep(noteIndex); + int staffCenterY = top + spacing * 4; + int y = staffCenterY - (staffStep * (spacing / 2)); + int x = left + _rng.Next(0, Math.Max(10, right - left - 20)); + + _staffNotes.Add(new StaffNoteParticle + { + X = x, + Y = y, + Vx = 0.20f + (float)_rng.NextDouble() * 0.35f, + Life = 1f, + UpStem = _rng.NextDouble() > 0.5, + Filled = features.DetectedNoteStrength > 0.22f + }); + } + + private static int NoteToStaffStep(int noteIndex) + { + return noteIndex switch + { + 0 => -1, + 1 => -1, + 2 => 0, + 3 => 0, + 4 => 1, + 5 => 2, + 6 => 2, + 7 => 3, + 8 => 3, + 9 => 4, + 10 => 4, + 11 => 5, + _ => 0 + }; + } + + private void DrawStaffNote(StaffNoteParticle n, int top, int spacing) + { + int x = (int)n.X; + int y = n.Y; + float a = Clamp01(n.Life); + Vector3 ink = new(0.08f, 0.08f, 0.1f); + + int rx = Math.Max(2, spacing / 3); + int ry = Math.Max(2, spacing / 4 + 1); + DrawEllipse(x, y, rx, ry, ink, a, n.Filled); + + int stemH = spacing * 3; + if (n.UpStem) + { + DrawRect(x + rx, y - stemH, 1, stemH, ink, a); + } + else + { + DrawRect(x - rx, y, 1, stemH, ink, a); + } + + int staffTop = top; + int staffBottom = top + spacing * 4; + if (y < staffTop - spacing / 2 || y > staffBottom + spacing / 2) + { + DrawRect(x - spacing / 2, y, spacing, 1, ink, a * 0.9f); + } + } + private void SpawnDetectedNote(AudioFeatures features) { if (features.DetectedNoteIndex < 0 || features.DetectedNoteIndex > 11) @@ -1577,6 +1816,49 @@ public sealed class FractalRenderer : IDisposable } } + private void DrawEllipse(int cx, int cy, int rx, int ry, Vector3 color, float alpha, bool filled) + { + int minX = cx - rx - 1; + int maxX = cx + rx + 1; + int minY = cy - ry - 1; + int maxY = cy + ry + 1; + float irx = 1f / (rx * rx + 1e-6f); + float iry = 1f / (ry * ry + 1e-6f); + for (int y = minY; y <= maxY; y++) + { + if (y < 0 || y >= _internalHeight) + { + continue; + } + + for (int x = minX; x <= maxX; x++) + { + if (x < 0 || x >= _internalWidth) + { + continue; + } + + float dx = x - cx; + float dy = y - cy; + float d = dx * dx * irx + dy * dy * iry; + if (filled) + { + if (d <= 1f) + { + BlendPixel(x, y, color, alpha); + } + } + else + { + if (d >= 0.7f && d <= 1.2f) + { + BlendPixel(x, y, color, alpha); + } + } + } + } + } + private void BlendPixel(int x, int y, Vector3 color, float alpha) { int idx = y * _internalWidth + x; @@ -1705,6 +1987,26 @@ public sealed class FractalRenderer : IDisposable public Vector3 Color; } + private struct StaffNoteParticle + { + public float X; + public int Y; + public float Vx; + public float Life; + public bool UpStem; + public bool Filled; + } + + private struct ScrollingStaffNoteParticle + { + public float X; + public int Y; + public float Life; + public bool UpStem; + public bool Filled; + public float SpeedJitter; + } + private struct FireworkRocket { public float X;