From 76a0fdd00ca9997708b9912fb1ee9bb81012686a Mon Sep 17 00:00:00 2001 From: xiaoxia Date: Wed, 22 Jul 2026 16:28:25 +0800 Subject: [PATCH] =?UTF-8?q?test(text=5Fsplitter):=20P3-1=E7=AC=AC=E5=8D=81?= =?UTF-8?q?=E4=BA=94=E6=B3=A2=20=E9=95=BF=E6=96=87=E6=9C=AC=E5=88=86?= =?UTF-8?q?=E6=AE=B5=E5=8D=95=E6=B5=8B=2032=E4=B8=AA=20(#719)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/test_text_splitter.py | 315 +++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100755 tests/unit/test_text_splitter.py diff --git a/tests/unit/test_text_splitter.py b/tests/unit/test_text_splitter.py new file mode 100755 index 000000000..81fc33a06 --- /dev/null +++ b/tests/unit/test_text_splitter.py @@ -0,0 +1,315 @@ +""" +text_splitter 长文本分段工具单元测试 + +覆盖： +- 空文本 / 短文本 +- 句子边界分段（。！？；\n . ! ? ;） +- 超长句子硬切 +- 过短段落合并 +- max_chars 参数 +- 中英文混合 +""" + +import pytest + +from packages.application.tts_job.text_splitter import split_text + +# ============================================================ +# 基础场景 +# ============================================================ + + +class TestBasicCases: + """基础场景""" + + def test_empty_text_returns_empty_list(self): + assert split_text("") == [] + + def test_whitespace_only_returns_empty(self): + assert split_text(" \n\n ") == [] + + def test_short_text_single_segment(self): + text = "这是一段短文本。" + result = split_text(text, max_chars=500) + assert result == [text] + + def test_exactly_max_chars_single_segment(self): + text = "a" * 500 + result = split_text(text, max_chars=500) + assert len(result) == 1 + assert len(result[0]) == 500 + + def test_text_stripped(self): + text = " 你好世界。 " + result = split_text(text, max_chars=500) + assert result == ["你好世界。"] + + +# ============================================================ +# 句子边界分段 +# ============================================================ + + +class TestSentenceBoundarySplitting: + """句子边界分段""" + + def test_split_by_chinese_period(self): + text = "第一句。第二句。第三句。" + # 三句都很短，应该合并成一段 + result = split_text(text, max_chars=500) + assert len(result) == 1 + + def test_split_by_chinese_period_long_text(self): + """多段长句子，按句号分段""" + sentence1 = "我是第一句" + "啊" * 100 + "。" + sentence2 = "我是第二句" + "哦" * 100 + "。" + sentence3 = "我是第三句" + "嗯" * 100 + "。" + text = sentence1 + sentence2 + sentence3 + + result = split_text(text, max_chars=150) + # 每句106字符，超过150的阈值？不，106<150 + # 但累计到一定程度会切 + assert len(result) >= 2 + # 每段都不超过 max_chars + for seg in result: + assert len(seg) <= 150 + + def test_split_by_question_mark(self): + text = "你是谁？你从哪里来？你要到哪里去？" + result = split_text(text, max_chars=500) + # 三句都很短，合并成一段 + assert len(result) == 1 + + def test_split_by_exclamation_mark(self): + text = "太棒了！太厉害了！太牛了！" + result = split_text(text, max_chars=500) + assert len(result) == 1 + + def test_split_by_newline(self): + text = "第一段\n第二段\n第三段" + result = split_text(text, max_chars=500) + assert len(result) == 1 + + def test_split_by_semicolon(self): + text = "第一部分；第二部分；第三部分。" + result = split_text(text, max_chars=500) + assert len(result) == 1 + + def test_mixed_punctuation(self): + """混合标点符号的句子边界""" + parts = [] + for i in range(20): + parts.append(f"第{i}句的内容" + "字" * 30 + "。") + text = "".join(parts) + + result = split_text(text, max_chars=200) + # 每句约35字符，200字符大约能放5-6句 + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 200 + + def test_english_period_splitting(self): + text = "Hello. How are you. I am fine." + result = split_text(text, max_chars=500) + assert len(result) == 1 + + def test_english_question(self): + text = "What? Why? How?" + result = split_text(text, max_chars=500) + assert len(result) == 1 + + +# ============================================================ +# 超长硬切 +# ============================================================ + + +class TestLongSentenceHardCut: + """超长句子硬切""" + + def test_single_very_long_sentence_hard_cut(self): + """单个超长句子，没有标点，硬切""" + text = "字" * 1000 + result = split_text(text, max_chars=500) + assert len(result) == 2 + assert len(result[0]) == 500 + assert len(result[1]) == 500 + + def test_three_times_max_chars(self): + text = "字" * 1500 + result = split_text(text, max_chars=500) + assert len(result) == 3 + for seg in result: + assert len(seg) == 500 + + def test_not_exact_multiple(self): + text = "字" * 1250 + result = split_text(text, max_chars=500) + assert len(result) == 3 + assert len(result[0]) == 500 + assert len(result[1]) == 500 + assert len(result[2]) == 250 + + def test_all_segments_within_limit(self): + """所有段都不超过 max_chars""" + import random + + random.seed(42) + # 生成随机长度的文本 + text = "".join(random.choices("字字字字。！？；\n", k=5000)) + for max_chars in [100, 200, 500]: + result = split_text(text, max_chars=max_chars) + for i, seg in enumerate(result): + assert len(seg) <= max_chars, f"Segment {i} length {len(seg)} > {max_chars}" + + +# ============================================================ +# 过短段落合并 +# ============================================================ + + +class TestShortSegmentMerging: + """过短段落合并""" + + def test_short_final_segment_merged(self): + """最后一段过短，应该合并到前一段""" + # 构造：前一段接近上限，后一段很短 + long_part = "字" * 480 + "。" + short_part = "好的。" + text = long_part + short_part + + result = split_text(text, max_chars=500) + # 两段加起来 481+3=484 < 500，可能合并 + # 但要看具体实现... + # 至少验证所有段不超长 + for seg in result: + assert len(seg) <= 500 + + def test_multiple_short_segments(self): + """多个短段落应该合并""" + sentences = ["你好。", "我好。", "大家好。", "今天天气不错。", "适合出去玩。"] + text = "".join(sentences) + result = split_text(text, max_chars=500) + # 5个短句子，应该合并成一段 + assert len(result) == 1 + + +# ============================================================ +# max_chars 参数 +# ============================================================ + + +class TestMaxCharsParameter: + """max_chars 参数""" + + def test_small_max_chars(self): + text = "一二三四五六七八九十一二三四五六七八九十。" + result = split_text(text, max_chars=10) + # 应该被切成多段 + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 10 + + def test_custom_max_chars_200(self): + text = "测试文本" * 100 # 400字符 + result = split_text(text, max_chars=200) + assert len(result) == 2 + assert len(result[0]) == 200 + assert len(result[1]) == 200 + + def test_very_small_max_chars(self): + text = "abcdefghij" + result = split_text(text, max_chars=3) + assert len(result) >= 3 + for seg in result: + assert len(seg) <= 3 + + +# ============================================================ +# 中英文混合 +# ============================================================ + + +class TestMixedContent: + """中英文混合内容""" + + def test_chinese_english_mixed(self): + text = "今天天气很好，Today is sunny. 我们去公园玩吧！Let's go to the park." + result = split_text(text, max_chars=500) + assert len(result) == 1 + assert result[0] == text.strip() + + def test_mixed_long_text(self): + parts = [] + for i in range(50): + parts.append(f"第{i}段中文内容" + "字" * 20 + ". English part " + "word " * 10 + "。") + text = "".join(parts) + + result = split_text(text, max_chars=300) + assert len(result) >= 2 + for seg in result: + assert len(seg) <= 300 + + +# ============================================================ +# 输出完整性 +# ============================================================ + + +class TestOutputIntegrity: + """输出完整性验证""" + + def test_combined_length_equals_original(self): + """所有段拼接起来（去掉空段）应该等于原文长度""" + text = "这是第一段。这是第二段。这是第三段。这是第四段。这是第五段。" * 20 + result = split_text(text, max_chars=100) + combined = "".join(result) + # 由于 strip 可能去掉一些空格，原文也 strip 比较 + assert len(combined) == len(text.strip()) + + def test_order_preserved(self): + """分段后再拼接，文本顺序不变""" + text = "第一。第二。第三。第四。第五。" * 10 + result = split_text(text, max_chars=50) + combined = "".join(result) + assert combined == text.strip() + + def test_no_empty_strings_in_result(self): + """结果中没有空字符串""" + text = "句子一。句子二。句子三。" + result = split_text(text, max_chars=10) + for seg in result: + assert seg != "" + assert len(seg) > 0 + + +# ============================================================ +# 边界情况 +# ============================================================ + + +class TestEdgeCases: + """边界情况""" + + def test_single_character(self): + assert split_text("一", max_chars=500) == ["一"] + + def test_only_punctuation(self): + text = "。。。。。" + result = split_text(text, max_chars=500) + # 都是标点，也算文本 + assert len(result) == 1 + + def test_only_newlines(self): + text = "\n\n\n" + result = split_text(text, max_chars=500) + assert result == [] + + def test_long_text_many_sentences(self): + """大量句子的长文本""" + sentences = [f"第{i}句的完整内容。" for i in range(100)] + text = "".join(sentences) + result = split_text(text, max_chars=200) + assert len(result) >= 5 + for seg in result: + assert len(seg) <= 200