diff --git a/pytest/modules/Speech_Recognition/test_reference_lyrics_aligner.py b/pytest/modules/Speech_Recognition/test_reference_lyrics_aligner.py index 2794460c..d9f91582 100644 --- a/pytest/modules/Speech_Recognition/test_reference_lyrics_aligner.py +++ b/pytest/modules/Speech_Recognition/test_reference_lyrics_aligner.py @@ -8,6 +8,7 @@ parse_lrc_synced_lyrics, _compute_note_for_word, _split_word_at_pitch_changes, + _trim_word_to_voiced, create_midi_segments_from_reference_lyrics, align_lyrics_to_audio, ) @@ -196,6 +197,51 @@ def test_boundaries_preserved(self): assert result[-1].end == pytest.approx(4.5) +# --------------------------------------------------------------------------- +# Voiced-region trimming +# --------------------------------------------------------------------------- + + +class TestTrimWordToVoiced: + """Tests for _trim_word_to_voiced.""" + + def _word_with_trailing_silence(self): + """A word whose second half is silence, so trimming kicks in.""" + fps = 62.5 + n_frames = int(4.0 * fps) + times = [i / fps for i in range(n_frames)] + freqs, confs = [], [] + for t in times: + if 1.0 <= t <= 2.0: + freqs.append(440.0) + confs.append(0.95) + else: + freqs.append(0.0) + confs.append(0.0) + pd = PitchedData(times=times, frequencies=freqs, confidence=confs) + # CTC stretched the word until 3.5s although singing ends at 2.0s + word = {"word": "night", "start": 1.0, "end": 3.5, + "backing": True, "line_end": True} + return word, pd + + def test_trims_trailing_silence(self): + word, pd = self._word_with_trailing_silence() + trimmed = _trim_word_to_voiced(word, pd) + assert trimmed["end"] < word["end"] + + def test_preserves_flags_when_trimming(self): + """Regression: trimming used to rebuild the dict and drop the + line_end/backing flags. CTC stretches precisely the last word of + each LRC line, so this silently killed nearly every LRCLIB + linebreak in the output file.""" + word, pd = self._word_with_trailing_silence() + trimmed = _trim_word_to_voiced(word, pd) + assert trimmed["end"] < word["end"] # trimming actually happened + assert trimmed["line_end"] is True + assert trimmed["backing"] is True + assert trimmed["word"] == "night" + + # --------------------------------------------------------------------------- # Integration (with mocked WhisperX) # --------------------------------------------------------------------------- diff --git a/pytest/modules/UltraStar/test_ultrastar_writer.py b/pytest/modules/UltraStar/test_ultrastar_writer.py index 659a7ce9..97face21 100644 --- a/pytest/modules/UltraStar/test_ultrastar_writer.py +++ b/pytest/modules/UltraStar/test_ultrastar_writer.py @@ -10,6 +10,7 @@ create_ultrastar_txt, silence_threshold, calculate_silent_beat_length, format_separated_string, add_score_to_ultrastar_txt, add_game_score_to_ultrastar_txt, + _compute_linebreak_indices, _enforce_max_line_length, ) from src.modules.Midi.MidiSegment import MidiSegment from src.modules.Ultrastar.ultrastar_txt import UltrastarTxtValue, UltrastarTxtTag @@ -569,5 +570,134 @@ def test_add_score_to_ultrastar_txt_still_works(self): os.remove(path) +def _seg(word, start, end, line_break_after=False): + return MidiSegment("A4", start, end, word, + line_break_after=line_break_after) + + +def _make_line(words, start=0.0, note_s=0.4, gap_s=0.05): + """Build consecutive word segments with a uniform small gap.""" + segments = [] + t = start + for w in words: + segments.append(_seg(w + " ", t, t + note_s)) + t += note_s + gap_s + return segments + + +class TestComputeLinebreakIndices(unittest.TestCase): + def test_lrclib_flags_win_over_silence(self): + segments = _make_line(["one", "two", "three", "four"]) + segments[1].line_break_after = True + breaks = _compute_linebreak_indices(segments, silence_split_duration=0.01) + self.assertEqual(breaks, {1}) + + def test_lrclib_flag_on_last_segment_is_ignored(self): + segments = _make_line(["one", "two"]) + segments[-1].line_break_after = True + breaks = _compute_linebreak_indices(segments, silence_split_duration=None) + self.assertEqual(breaks, set()) + + def test_silence_fallback_breaks_at_long_gaps(self): + segments = _make_line(["one", "two", "three"]) + # Insert a much longer gap after "two" + long_gap = 2.0 + offset = long_gap - 0.05 + for s in segments[2:]: + s.start += offset + s.end += offset + breaks = _compute_linebreak_indices(segments, silence_split_duration=1.0) + self.assertEqual(breaks, {1}) + + def test_no_threshold_no_fallback_breaks(self): + segments = _make_line(["one", "two", "three"]) + breaks = _compute_linebreak_indices(segments, silence_split_duration=None) + self.assertEqual(breaks, set()) + + +class TestEnforceMaxLineLength(unittest.TestCase): + def test_short_line_untouched(self): + segments = _make_line(["short", "line"]) + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=45) + self.assertEqual(breaks, set()) + + def test_overlong_line_splits_at_largest_gap(self): + words = ["alpha", "bravo", "charlie", "delta", "echo", + "foxtrot", "golf", "hotel"] + segments = _make_line(words) + # Clear pause after "delta" (index 3) + offset = 1.0 + for s in segments[4:]: + s.start += offset + s.end += offset + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=30) + self.assertIn(3, breaks) + + def test_overlong_line_without_gaps_splits_near_middle(self): + words = ["aaaa", "bbbb", "cccc", "dddd", "eeee", "ffff"] + segments = [] + t = 0.0 + for w in words: + segments.append(_seg(w + " ", t, t + 0.4)) + t += 0.4 # zero gap between notes + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=20) + self.assertTrue(breaks) + # All resulting lines must respect the limit (rendered length = + # raw concatenation of the note texts, trailing space stripped) + idx = sorted(breaks) + [len(segments) - 1] + start = 0 + for end in idx: + chars = len("".join(s.word for s in segments[start:end + 1]).rstrip()) + self.assertLessEqual(chars, 20) + start = end + 1 + + def test_syllable_notes_are_not_overcounted(self): + """Syllable notes without trailing spaces attach directly in the + rendered line - they must not be counted as separate words. + 'Sunshine holiday' rendered is 16 chars, far under the limit.""" + segments = [ + _seg("Sun", 0.0, 0.3), + _seg("shine ", 0.4, 0.7), + _seg("ho", 0.8, 1.1), + _seg("li", 1.2, 1.5), + _seg("day", 1.6, 1.9), + ] + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=17) + self.assertEqual(breaks, set()) + + def test_never_breaks_inside_melisma(self): + """Continuation notes (no trailing space) are not split points.""" + segments = [ + _seg("looooooooooooong ", 0.0, 0.4), + _seg("~", 0.5, 0.9), + _seg("~", 1.0, 1.4), + _seg("~", 1.5, 1.9), + _seg("meloooooooooody ", 2.0, 2.4), + _seg("here", 2.5, 2.9), + ] + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=15) + for b in breaks: + self.assertTrue(segments[b].word.endswith(" ")) + + def test_nested_split_handles_very_long_lines(self): + words = ["word%02d" % i for i in range(20)] + segments = _make_line(words) + breaks = set() + _enforce_max_line_length(segments, breaks, max_chars=25) + self.assertGreaterEqual(len(breaks), 3) + + def test_existing_breaks_respected(self): + """Lines already short enough through existing breaks stay as-is.""" + segments = _make_line(["one", "two", "three", "four"]) + breaks = {1} + _enforce_max_line_length(segments, breaks, max_chars=45) + self.assertEqual(breaks, {1}) + + if __name__ == "__main__": unittest.main() diff --git a/src/modules/Speech_Recognition/reference_lyrics_aligner.py b/src/modules/Speech_Recognition/reference_lyrics_aligner.py index af4091b4..d181aba6 100644 --- a/src/modules/Speech_Recognition/reference_lyrics_aligner.py +++ b/src/modules/Speech_Recognition/reference_lyrics_aligner.py @@ -428,7 +428,12 @@ def _trim_word_to_voiced( if new_end - new_start < 0.05: return word - return {"word": word["word"], "start": new_start, "end": new_end} + # Preserve ALL other keys (backing, line_end, ...): CTC stretches + # precisely the LAST word of each LRC line into the following pause, + # so almost every line-end word lands here - rebuilding the dict from + # scratch used to silently drop the line_end flag, which killed nearly + # all LRCLIB linebreaks in the output file. + return {**word, "start": new_start, "end": new_end} def _split_word_at_silence_gaps( diff --git a/src/modules/Ultrastar/ultrastar_writer.py b/src/modules/Ultrastar/ultrastar_writer.py index 01a1b708..a85db186 100644 --- a/src/modules/Ultrastar/ultrastar_writer.py +++ b/src/modules/Ultrastar/ultrastar_writer.py @@ -95,14 +95,14 @@ def create_ultrastar_txt( file.write(f"#{UltrastarTxtTag.TAGS.value}:{ultrastar_class.tags}\n") file.write(f"#{UltrastarTxtTag.CREATOR.value}:{ultrastar_class.creator}\n") - # Check if any LRCLIB linebreaks are available - has_lrclib_linebreaks = any( - getattr(ms, "line_break_after", False) for ms in midi_segments + # Decide after which notes a linebreak is written (LRCLIB flags or + # silence-based fallback, plus a maximum-line-length safety net). + break_indices = _compute_linebreak_indices( + midi_segments, silence_split_duration ) # Write the singing part previous_end_beat = 0 - separated_word_silence = [] # This is a workaround for separated words that get his ends to far away for i, midi_segment in enumerate(midi_segments): start_time = (midi_segment.start - gap) * multiplication @@ -120,12 +120,6 @@ def create_ultrastar_txt( start_beat = previous_end_beat previous_end_beat = start_beat + duration - # Calculate the silence between the words - if i < len(midi_segments) - 1: - silence = (midi_segments[i + 1].start - midi_segment.end) - else: - silence = 0 - # Use note_type from LRCLIB metadata (: normal, F freestyle) note_type = getattr(midi_segment, "note_type", UltrastarTxtNoteTypeTag.NORMAL.value) @@ -143,38 +137,142 @@ def create_ultrastar_txt( file.write(line) - # Linebreak logic: prefer LRCLIB linebreaks when available, - # fall back to silence-based detection otherwise - if has_lrclib_linebreaks: - # LRCLIB mode: use line_break_after flag from reference lyrics - if getattr(midi_segment, "line_break_after", False) and i != len(midi_segments) - 1: - show_next = ( - second_to_beat(midi_segment.end - gap, real_bpm) - * multiplication - ) - linebreak = f"{UltrastarTxtTag.LINEBREAK.value} " \ - f"{str(math.floor(show_next))}\n" - file.write(linebreak) - else: - # Fallback: silence-based linebreak detection - if not midi_segment.word.endswith(" "): - separated_word_silence.append(silence) - continue - - if silence_split_duration is not None and i != len(midi_segments) - 1 and ( - silence > silence_split_duration - or any(s > silence_split_duration for s in separated_word_silence)): - show_next = ( - second_to_beat(midi_segment.end - gap, real_bpm) - * multiplication - ) - linebreak = f"{UltrastarTxtTag.LINEBREAK.value} " \ - f"{str(math.floor(show_next))}\n" - file.write(linebreak) - separated_word_silence = [] + if i in break_indices: + show_next = ( + second_to_beat(midi_segment.end - gap, real_bpm) + * multiplication + ) + linebreak = f"{UltrastarTxtTag.LINEBREAK.value} " \ + f"{str(math.floor(show_next))}\n" + file.write(linebreak) file.write(f"{UltrastarTxtTag.FILE_END.value}") +# Maximum visible characters per line before the safety net forces an extra +# break. Manually-authored charts stay around 25-40 characters; beyond ~45 +# the notes get compressed to unreadable slivers (each note's width shrinks +# with the line's total duration) and the lyric line overflows the screen. +_MAX_LINE_CHARS = 45 +# Never create a line shorter than this via forced splits. +_MIN_SPLIT_CHARS = 10 + + +def _compute_linebreak_indices( + midi_segments: list[MidiSegment], + silence_split_duration: float | None, +) -> set[int]: + """Return the indices after which a linebreak is written. + + Primary source: LRCLIB ``line_break_after`` flags when present, + otherwise silence-based detection (only the longest gaps become + breaks). Both are followed by a maximum-line-length safety net that + splits overlong lines at the largest pause on a word boundary - one + huge line compresses every note to an unreadable sliver and pushes + the lyrics off-screen. + """ + n = len(midi_segments) + breaks: set[int] = set() + + has_lrclib_linebreaks = any( + getattr(ms, "line_break_after", False) for ms in midi_segments + ) + if has_lrclib_linebreaks: + breaks = { + i for i, ms in enumerate(midi_segments) + if getattr(ms, "line_break_after", False) and i != n - 1 + } + elif silence_split_duration is not None: + # Workaround for separated words whose ends drift too far away + separated_word_silence: list[float] = [] + for i, ms in enumerate(midi_segments): + silence = (midi_segments[i + 1].start - ms.end) if i < n - 1 else 0 + if not ms.word.endswith(" "): + separated_word_silence.append(silence) + continue + if i != n - 1 and ( + silence > silence_split_duration + or any(s > silence_split_duration + for s in separated_word_silence)): + breaks.add(i) + separated_word_silence = [] + + _enforce_max_line_length(midi_segments, breaks) + return breaks + + +def _enforce_max_line_length( + midi_segments: list[MidiSegment], + breaks: set[int], + max_chars: int = _MAX_LINE_CHARS, +) -> None: + """Split lines longer than ``max_chars`` at the best word boundary. + + Modifies ``breaks`` in place. The split point must end a word (its + text carries the trailing space that marks word boundaries) and + leave at least ``_MIN_SPLIT_CHARS`` visible characters on each side. + Among the eligible positions, the one with the longest pause to the + next note wins (that is where a screen change feels natural); when + no pause stands out, the split closest to the middle wins. + """ + n = len(midi_segments) + if n == 0: + return + + def visible_chars(a: int, b: int) -> int: + # The rendered line is the raw concatenation of the note texts: + # word notes carry their own trailing space, syllable/continuation + # notes attach directly without one - so measure exactly that. + text = "".join(midi_segments[k].word for k in range(a, b + 1)) + return len(text.rstrip()) + + def gap_after(k: int) -> float: + if k >= n - 1: + return 0.0 + return max(0.0, midi_segments[k + 1].start - midi_segments[k].end) + + # Build the current lines and process them with a worklist; each split + # re-queues both halves so nested overlength lines are handled too. + pending: list[tuple[int, int]] = [] + start = 0 + for i in sorted(breaks): + pending.append((start, i)) + start = i + 1 + pending.append((start, n - 1)) + + while pending: + a, b = pending.pop() + if a >= b or visible_chars(a, b) <= max_chars: + continue + + candidates = [] + left_chars = 0 + for k in range(a, b): + left_chars += len(midi_segments[k].word) + if not midi_segments[k].word.endswith(" "): + continue + right_chars = visible_chars(k + 1, b) + if left_chars - 1 < _MIN_SPLIT_CHARS or right_chars < _MIN_SPLIT_CHARS: + continue + candidates.append((k, left_chars - 1)) + if not candidates: + continue + + max_gap = max(gap_after(k) for k, _ in candidates) + mid = visible_chars(a, b) / 2 + if max_gap > 0.0: + # Prefer long pauses; among near-equal pauses (within 10 %) + # take the one closest to the middle of the line. + eligible = [(k, c) for k, c in candidates + if gap_after(k) >= 0.9 * max_gap] + else: + eligible = candidates + best, _ = min(eligible, key=lambda kc: abs(kc[1] - mid)) + + breaks.add(best) + pending.append((a, best)) + pending.append((best + 1, b)) + + def silence_threshold( silence_parts: list[float], percentile: float = 85 ) -> float | None: