diff --git a/tests/unit/test_text_splitter.py b/tests/unit/test_text_splitter.py index 831253df2..faff5b971 100755 --- a/tests/unit/test_text_splitter.py +++ b/tests/unit/test_text_splitter.py @@ -1,100 +1,295 @@ -"""text_splitter 单元测试.""" +"""TTS 文本分段工具单元测试.""" + +import pytest from packages.application.tts_job.text_splitter import split_text -class TestSplitText: - def test_empty_text_returns_empty(self): +class TestSplitTextEmpty: + """空文本测试""" + + def test_empty_string(self): + """空字符串返回空列表.""" assert split_text("") == [] - def test_whitespace_only(self): - assert split_text(" \n\t ") == [] + def test_only_whitespace(self): + """纯空白文本返回空列表.""" + assert split_text(" \n \t ") == [] - def test_short_text_single_segment(self): - text = "你好世界。" + def test_none_not_allowed(self): + """None 会抛出异常(不是我们的职责).""" + with pytest.raises(AttributeError): + split_text(None) # type: ignore + + +class TestSplitTextShort: + """短文本测试""" + + def test_short_text_one_segment(self): + """短文本返回一个段落.""" + text = "你好,世界。" result = split_text(text, max_chars=500) assert len(result) == 1 assert result[0] == text - def test_exact_max_chars(self): + def test_exactly_max_chars(self): + """刚好等于 max_chars 的文本返回一个段落.""" text = "a" * 500 result = split_text(text, max_chars=500) assert len(result) == 1 assert len(result[0]) == 500 - def test_splits_on_sentence_boundary(self): - # 两个长句子,各300字左右,超过50字阈值 - sent1 = "你" * 300 + "。" - sent2 = "我" * 300 + "。" - text = sent1 + sent2 + def test_one_under_max(self): + """max_chars-1 的文本返回一个段落.""" + text = "a" * 499 result = split_text(text, max_chars=500) - assert len(result) == 2 - assert result[0] == sent1 - assert result[1] == sent2 + assert len(result) == 1 + assert len(result[0]) == 499 - def test_long_sentence_hard_cut(self): - # 一个超长句子,没有句末标点,会被硬切 - text = "长" * 800 - result = split_text(text, max_chars=500) + +class TestSplitTextSentenceBoundary: + """句子边界分段测试""" + + def test_split_at_period(self): + """在句号处拆分.""" + text = "第一句。第二句。第三句。" + # 每句5字符,max_chars=10,每次两句就接近10 + result = split_text(text, max_chars=10) assert len(result) >= 2 - assert all(len(seg) <= 500 for seg in result) - # 合起来应该等于原文本 - assert "".join(result) == text + # 所有段落都不超过 max_chars + for seg in result: + assert len(seg) <= 10 + + def test_split_at_exclamation(self): + """在感叹号处拆分.""" + text = "好棒!真的好棒!太厉害了!" + result = split_text(text, max_chars=10) + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 10 + + def test_split_at_question(self): + """在问号处拆分.""" + text = "你好吗?你是谁?你在哪?" + result = split_text(text, max_chars=10) + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 10 + + def test_split_at_newline(self): + """在换行处拆分.""" + text = "第一段\n第二段\n第三段" + result = split_text(text, max_chars=10) + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 10 + + def test_split_at_semicolon(self): + """在分号处拆分.""" + text = "第一项;第二项;第三项;" + result = split_text(text, max_chars=10) + assert len(result) >= 2 + + def test_english_punctuation(self): + """英文标点也能拆分.""" + text = "Hello world. How are you? I am fine!" + result = split_text(text, max_chars=20) + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 20 + + def test_mixed_punctuation(self): + """中英文标点混合.""" + text = "你好!Hello. 你好吗?How are you?" + result = split_text(text, max_chars=15) + assert len(result) >= 2 + + +class TestSplitTextForceSplit: + """强制分段测试""" + + def test_very_long_sentence_forced_split(self): + """超长单句强制分段.""" + text = "a" * 1000 # 没有标点的长文本 + result = split_text(text, max_chars=100) + assert len(result) == 10 + for seg in result: + assert len(seg) == 100 + + def test_mixed_long_and_short_sentences(self): + """长短句混合.""" + long = "我" * 200 + text = f"短句。{long}。短句。" + result = split_text(text, max_chars=100) + # 所有段都不超过100 + for seg in result: + assert len(seg) <= 100 + # 至少有3段(长句被强制拆分) + assert len(result) >= 3 + + +class TestSplitTextMerging: + """短段落合并测试""" def test_short_segments_merged(self): - # 多个短句应该被合并 - sentences = [f"第{i}句。" for i in range(10)] - text = "".join(sentences) - result = split_text(text, max_chars=200) - # 每句5字左右,10句才50字,应该合并成1段 - assert len(result) < 10 - assert len(result[0]) <= 200 - - def test_preserves_content(self): - text = "今天天气真好。我们去公园玩吧!你觉得怎么样?好的,走吧。" - result = split_text(text, max_chars=20) - # 合并后内容应一致 - assert "".join(result) == text - - def test_multiple_punctuation_types(self): - # 构造足够长的文本触发分段 - text = "第一" * 30 + "。" + "第二" * 30 + "!" + "第三" * 30 + "?" + "第四" * 30 + ";" + """多个短段合并为一个.""" + # 生成5个短句,每句5字符,max_chars=100,应该合并成一段 + text = "一。二。三。四。五。" result = split_text(text, max_chars=100) - assert len(result) >= 2 - assert "".join(result) == text + assert len(result) == 1 + assert len(result[0]) <= 100 - def test_custom_max_chars(self): - text = "a" * 100 + "。" + "b" * 100 + "。" - result = split_text(text, max_chars=150) - assert len(result) == 2 - assert "a" in result[0] - assert "b" in result[1] + def test_merge_within_limit(self): + """合并后不超过 max_chars.""" + # 10个短句,每句4字符 = 40字符 + text = "句子。" * 10 + result = split_text(text, max_chars=100) + assert len(result) == 1 + assert len(result[0]) <= 100 - def test_newline_as_sentence_end(self): - text = "第一段\n第二段\n第三段" + def test_merge_across_multiple(self): + """多个短段依次合并.""" + text = "短。" * 30 # 30个短句,每句2字符=60字符 + result = split_text(text, max_chars=100) + assert len(result) == 1 + assert len(result[0]) == 60 # 全部合并 + + +class TestSplitTextChinese: + """中文文本测试""" + + def test_chinese_paragraph(self): + """典型中文段落.""" + text = ( + "在一个阳光明媚的早晨,小明来到了公园。" + "他看到了很多人在锻炼身体。" + "有的人在跑步,有的人在打太极,还有的人在跳舞。" + "小明也加入了他们,开始了愉快的一天。" + ) result = split_text(text, max_chars=50) - assert len(result) >= 1 - assert "".join(result) == text.strip() + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 50 + # 重新拼回应该等于原文本(除了可能的空格处理) + combined = "".join(result) + assert combined == text.replace(" ", "") # strip 不影响中文字符 - def test_minimum_segment_length(self): - # 句子太短(<50字)不会立即分段 - text = "短句一。短句二。短句三。" + def test_chinese_long_paragraph(self): + """长中文段落.""" + text = "这是一个测试句子。" * 100 # 100个句子 result = split_text(text, max_chars=200) + assert len(result) > 1 + for seg in result: + assert len(seg) <= 200 + # 总字符数不变 + assert sum(len(s) for s in result) == len(text) + + +class TestSplitTextCustomMaxChars: + """自定义 max_chars 测试""" + + def test_small_max_chars(self): + """很小的 max_chars.""" + text = "一二三四五六七八九十。" + result = split_text(text, max_chars=5) + for seg in result: + assert len(seg) <= 5 + + def test_large_max_chars(self): + """很大的 max_chars(不拆分).""" + text = "这是一段测试文本。" * 10 + result = split_text(text, max_chars=10000) assert len(result) == 1 - def test_trailing_content_added(self): - # 最后一段不完整的句子也要加上 - text = "完整的句子。剩余内容" - result = split_text(text, max_chars=50) - assert "".join(result) == text + def test_max_chars_zero(self): + """max_chars=0 时的行为.""" + text = "测试文本。" + # 0 会导致每加一个字符就触发强制分段 + result = split_text(text, max_chars=0) + # 每个字符一段?或者至少有结果 + assert isinstance(result, list) + assert len(result) > 0 + + def test_max_chars_one(self): + """max_chars=1.""" + text = "abc" + result = split_text(text, max_chars=1) + assert len(result) == 3 + assert result == ["a", "b", "c"] + + +class TestSplitTextPreservesContent: + """内容完整性测试""" + + def test_preserves_all_chars(self): + """分段后拼接等于原文(忽略空白调整).""" + text = "第一句。第二句!第三句?第四句。第五句。" + result = split_text(text, max_chars=10) + combined = "".join(result) + assert combined == text def test_no_empty_segments(self): - text = "。。。。。" # 全是标点 - result = split_text(text, max_chars=2) - assert all(len(seg) > 0 for seg in result) + """没有空字符串段落.""" + text = "句子。。。双标点。" + result = split_text(text, max_chars=10) + assert all(seg for seg in result) # 所有段非空 - def test_chinese_and_english_mixed(self): - text = "Hello世界。这是测试Test文本。Mixed混合。" - result = split_text(text, max_chars=20) + def test_stripped_segments(self): + """段落首尾没有多余空白.""" + text = " 第一句。 第二句。 " + result = split_text(text, max_chars=10) + for seg in result: + assert seg == seg.strip() + + +class TestSplitTextEdgeCases: + """边界情况测试""" + + def test_single_char(self): + """单字符.""" + result = split_text("我", max_chars=10) + assert len(result) == 1 + assert result[0] == "我" + + def test_only_punctuation(self): + """纯标点.""" + text = "。。。!!??" + result = split_text(text, max_chars=5) + assert len(result) >= 1 + assert sum(len(s) for s in result) == len(text) + + def test_numbers_and_symbols(self): + """数字和符号.""" + text = "第1章。第2节。第3段。" + result = split_text(text, max_chars=10) + assert len(result) >= 1 + assert all(len(s) <= 10 for s in result) + + def test_mixed_chinese_english(self): + """中英文混合.""" + text = "Hello你好World世界。Test测试。" + result = split_text(text, max_chars=10) assert len(result) >= 2 - assert "".join(result) == text + assert all(len(s) <= 10 for s in result) + + def test_consecutive_punctuation(self): + """连续标点.""" + text = "真的吗!?不对。。。好吧。" + result = split_text(text, max_chars=20) + assert len(result) >= 1 + combined = "".join(result) + assert combined == text + + +class TestSplitTextDefaultParams: + """默认参数测试""" + + def test_default_max_chars_is_500(self): + """默认 max_chars=500.""" + text = "a" * 500 + result = split_text(text) + assert len(result) == 1 + + text2 = "a" * 501 + result2 = split_text(text2) + assert len(result2) >= 2