"""文本分段工具单元测试.""" from __future__ import annotations import pytest from packages.application.tts_job.text_splitter import split_text class TestSplitText: """split_text 函数测试""" def test_empty_string_returns_empty_list(self): """空字符串返回空列表""" assert split_text("") == [] def test_whitespace_only_returns_empty_list(self): """纯空白字符返回空列表""" assert split_text(" \n \t ") == [] def test_short_text_returns_single_segment(self): """短文本直接返回单段""" text = "这是一段短文本。" result = split_text(text, max_chars=500) assert result == [text] def test_text_length_equals_max_chars(self): """文本长度恰好等于 max_chars 时返回单段""" text = "a" * 100 result = split_text(text, max_chars=100) assert len(result) == 1 assert len(result[0]) == 100 def test_splits_on_sentence_boundary(self): """在句子边界处分段""" # 构造长文本,确保超过 max_chars sentences = ["今天天气真好。我们一起去公园散步吧。", "公园里有很多花。还有很多小朋友在玩耍。"] * 10 text = "".join(sentences) result = split_text(text, max_chars=200) assert len(result) >= 2 # 每段都不超过 max_chars for seg in result: assert len(seg) <= 200 def test_all_segments_within_max_chars(self): """所有分段都不超过 max_chars""" text = "这是第一句话。这是第二句话。这是第三句话。这是第四句话。这是第五句话。" * 10 result = split_text(text, max_chars=100) for seg in result: assert len(seg) <= 100 def test_long_single_sentence_hard_cut(self): """超长单句会被硬切""" text = "a" * 1000 # 没有标点 result = split_text(text, max_chars=200) assert len(result) > 1 for seg in result: assert len(seg) <= 200 def test_newline_is_sentence_end(self): """换行符作为句子结束符""" text = "第一行内容\n第二行内容\n第三行内容" * 10 result = split_text(text, max_chars=50) assert len(result) > 1 for seg in result: assert len(seg) <= 50 def test_chinese_punctuation(self): """中文标点(。!?;)作为句子结束符""" text = "你好!今天吃什么?我吃米饭;你呢?我也吃米饭。" * 10 result = split_text(text, max_chars=80) for seg in result: assert len(seg) <= 80 def test_english_punctuation(self): """英文标点(.!?;)作为句子结束符""" text = "Hello! How are you? I'm fine; thank you. Good bye." * 10 result = split_text(text, max_chars=80) for seg in result: assert len(seg) <= 80 def test_merged_short_segments(self): """过短的段落会被合并""" # 构造很多短句 text = "你好。再见。谢谢。抱歉。好的。不行。可以。去吧。" * 5 # 每句3-4字 result = split_text(text, max_chars=100) # 合并后段数应该比单纯按句切的少 assert len(result) < len(text) // 3 # 粗略估计 for seg in result: assert len(seg) <= 100 def test_preserves_content(self): """分段后内容总和与原文基本一致(忽略strip的空白)""" text = "这是测试文本。包含多个句子。用来验证分段正确性。" * 5 result = split_text(text, max_chars=50) # 合并所有分段,去掉空白后应该与原文去掉空白后基本一致 combined = "".join(result).replace(" ", "") original = text.strip().replace(" ", "") assert combined == original def test_custom_max_chars(self): """支持自定义 max_chars""" text = "测试" * 100 # 200字 result_50 = split_text(text, max_chars=50) result_100 = split_text(text, max_chars=100) # max_chars 越小,段数应该越多 assert len(result_50) >= len(result_100) def test_single_char_text(self): """单字符文本""" assert split_text("好", max_chars=10) == ["好"] def test_text_with_only_punctuation(self): """纯标点文本""" text = "。。。。。。。。。。" # 10个句号 result = split_text(text, max_chars=5) assert len(result) >= 1 for seg in result: assert len(seg) <= 5 def test_mixed_content(self): """中英文混合内容""" text = "今天的天气是 sunny and warm。我们去了 park 玩。真的很开心!" * 5 result = split_text(text, max_chars=80) for seg in result: assert len(seg) <= 80 # ── 短文本与空文本补充 ────────────────────────────────────────────────────── class TestSplitTextEmptyAndShort: """空文本与短文本补充场景.""" def test_whitespace_only_returns_empty(self): """纯空白文本返回空列表.""" assert split_text(" \n\t ") == [] def test_single_char(self): """单字符文本.""" assert split_text("好", max_chars=10) == ["好"] def test_exactly_max_chars_no_split(self): """刚好等于 max_chars 不分割.""" text = "a" * 100 result = split_text(text, max_chars=100) assert len(result) == 1 assert result[0] == text def test_one_over_max_chars_splits(self): """超过 max_chars 1 个字符就会分割.""" text = "a" * 101 result = split_text(text, max_chars=100) assert len(result) >= 2 def test_none_raises(self): """None 输入抛 AttributeError(strip 失败).""" with pytest.raises(AttributeError): split_text(None) # ── 句子边界分段补充 ────────────────────────────────────────────────────── class TestSplitTextSentenceBoundaries: """句子边界分段补充场景.""" def test_split_on_fullwidth_period(self): """全角句号分段.""" text = "第一句很长的内容。" * 20 result = split_text(text, max_chars=60) assert len(result) > 1 for seg in result: assert len(seg) <= 60 def test_split_on_fullwidth_question(self): """全角问号分段.""" text = "你知道这是为什么吗?" + "是的。" * 20 result = split_text(text, max_chars=60) assert len(result) > 1 def test_split_on_fullwidth_exclamation(self): """全角感叹号分段.""" text = "真是太棒了!" + "内容。" * 20 result = split_text(text, max_chars=60) assert len(result) > 1 def test_split_on_newline(self): """换行符分段.""" lines = ["这是第一行很长的一段文字内容" * 3 for _ in range(5)] text = "\n".join(lines) result = split_text(text, max_chars=80) assert len(result) > 1 def test_split_on_semicolon(self): """全角分号分段.""" text = "第一项内容;" + "其他内容。" * 20 result = split_text(text, max_chars=60) assert len(result) > 1 def test_english_period_splits(self): """英文句号分段.""" text = "Hello world. " * 30 result = split_text(text, max_chars=80) assert len(result) > 1 def test_short_sentences_stay_merged(self): """短句(都 < 50字的句子不会单独成段,会累积到一起.""" text = "你好。我好。大家好。" result = split_text(text, max_chars=200) assert len(result) == 1 # ── 长句强制切段补充 ────────────────────────────────────────────────────── class TestSplitTextLongSentenceForce: """超长单句强制切段补充.""" def test_no_punctuation_forced_split(self): """完全没有标点的超长文本硬切.""" text = "字" * 300 result = split_text(text, max_chars=100) assert len(result) == 3 for seg in result: assert len(seg) == 100 def test_force_split_preserves_content(self): """硬切不丢字符.""" text = "a" * 250 result = split_text(text, max_chars=100) assert sum(len(s) for s in result) == 250 def test_mixed_long_and_short(self): """长句短句混合.""" long_part = "非常长的句子没有标点符号" * 15 text = long_part + "。结尾。" result = split_text(text, max_chars=100) assert len(result) > 1 for seg in result: assert len(seg) <= 100 # ── 短段合并补充 ───────────────────────────────────────────────────────── class TestSplitTextShortSegmentMerge: """短段合并补充场景.""" def test_multiple_short_sentences_merged(self): """多个短句合并成一段.""" sentences = ["你好。", "我好。", "大家好。", "天气好。", "心情好。"] text = "".join(sentences) result = split_text(text, max_chars=200) assert len(result) == 1 def test_short_tail_merged(self): """尾部短段被合并到前一段.""" # 前面一段接近 max_chars,尾部很短 long_part = "一二三四五六七八九十" * 9 + "。" # ~90字 tail = "完。" # 2字 text = long_part + tail result = split_text(text, max_chars=100) # 尾部短的应该被合并 assert len(result) <= 2 # ── 边界情况补充 ───────────────────────────────────────────────────────── class TestSplitTextEdgeCases: """边界情况补充.""" def test_only_punctuation(self): """纯标点符号.""" text = "。。。。。" result = split_text(text, max_chars=10) assert len(result) == 1 def test_mixed_chinese_english(self): """中英文混合.""" text = "你好Hello。World!" * 20 result = split_text(text, max_chars=100) assert len(result) > 1 for seg in result: assert len(seg) <= 100 def test_strip_whitespace(self): """首尾空白被去除.""" text = " 你好世界。 " result = split_text(text, max_chars=100) assert result == ["你好世界。"] def test_total_length_preserved(self): """分段后总长度等于原文 strip 后长度.""" text = "这是一段用于测试的文本内容。" * 20 result = split_text(text, max_chars=100) assert "".join(result) == text.strip() def test_custom_small_max_chars(self): """很小的 max_chars.""" text = "一二三四五六七八九十。" * 5 result = split_text(text, max_chars=20) assert len(result) > 1 for seg in result: assert len(seg) <= 20 # ── 更多边界场景补充 ───────────────────────────────────────────────────────── class TestSplitTextMoreEdgeCases: """更多边界场景补充""" def test_max_chars_one(self): """max_chars=1 每个字符一段""" text = "一二三四五" result = split_text(text, max_chars=1) assert len(result) == 5 for seg in result: assert len(seg) == 1 def test_consecutive_newlines(self): """连续多个换行符""" text = "第一段\n\n\n第二段\n\n第三段" result = split_text(text, max_chars=100) # 合并后应该是一段(内容不长且合并逻辑会被合并) assert len(result) >= 1 assert "第一段" in result[0] for seg in result: assert len(seg) <= 100 def test_only_newlines_only(self): """只有换行符(纯空白被strip掉返回空""" assert split_text("\n\n\n\n") == [] def test_leading_trailing_whitespace(self): """首尾空白被去除""" text = " 你好世界。 " result = split_text(text, max_chars=100) assert result == ["你好世界。"] def test_very_long_single_sentence_many_segments(self): """超长单句被切成很多段""" text = "字" * 1000 result = split_text(text, max_chars=100) assert len(result) == 10 for seg in result: assert len(seg) == 100 def test_mixed_punctuation_types(self): """全角半角标点混合""" text = "你好!再见。谢谢?抱歉;好的" result = split_text(text, max_chars=200) assert len(result) == 1 def test_last_segment_short_merged_to_previous(self): """尾部极短段被合并到前一段""" # 构造第一段接近max_chars,结尾有个短句尾巴 long_part = "一二三四五六七八九十" * 9 + "。" # ~90字 tail = "完" # 1字 text = long_part + tail result = split_text(text, max_chars=100) # 尾巴应该被合并 combined = "".join(result) assert combined == text.strip() assert len(result) <= 2 def test_all_short_sentences_merged_into_one(self): """大量短句全部合并成一段""" sentences = ["你好。", "我好。", "他好。", "大家好。", "才是真的好。"] text = "".join(sentences) result = split_text(text, max_chars=200) assert len(result) == 1 def test_punctuation_only_long(self): """很长的纯标点文本""" text = "。" * 200 result = split_text(text, max_chars=50) assert len(result) >= 4 for seg in result: assert len(seg) <= 50 def test_tab_not_sentence_end(self): """制表符不是句子结束符""" text = "这是一段\t包含制表符的文本内容" + "字" * 100 result = split_text(text, max_chars=50) # 制表符不在句子结束符集合中,不会触发分段 # 制表符会保留在分段内容中 has_tab = any("\t" in seg for seg in result) assert has_tab