test(text_splitter): 补充长文本分段工具32个单元测试

覆盖空文本/短文本/句子边界分段/超长硬切/
过短段落合并/max_chars参数/中英文混合/输出完整性

tests/unit/test_text_splitter.py

@@ -0,0 +1,315 @@
+"""
+text_splitter 长文本分段工具单元测试
+
+覆盖：
+- 空文本 / 短文本
+- 句子边界分段（。！？；\n . ! ? ;）
+- 超长句子硬切
+- 过短段落合并
+- max_chars 参数
+- 中英文混合
+"""
+
+import pytest
+
+from packages.application.tts_job.text_splitter import split_text
+
+
+# ============================================================
+# 基础场景
+# ============================================================
+
+
+class TestBasicCases:
+    """基础场景"""
+
+    def test_empty_text_returns_empty_list(self):
+        assert split_text("") == []
+
+    def test_whitespace_only_returns_empty(self):
+        assert split_text("   \n\n  ") == []
+
+    def test_short_text_single_segment(self):
+        text = "这是一段短文本。"
+        result = split_text(text, max_chars=500)
+        assert result == [text]
+
+    def test_exactly_max_chars_single_segment(self):
+        text = "a" * 500
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+        assert len(result[0]) == 500
+
+    def test_text_stripped(self):
+        text = "   你好世界。   "
+        result = split_text(text, max_chars=500)
+        assert result == ["你好世界。"]
+
+
+# ============================================================
+# 句子边界分段
+# ============================================================
+
+
+class TestSentenceBoundarySplitting:
+    """句子边界分段"""
+
+    def test_split_by_chinese_period(self):
+        text = "第一句。第二句。第三句。"
+        # 三句都很短，应该合并成一段
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+    def test_split_by_chinese_period_long_text(self):
+        """多段长句子，按句号分段"""
+        sentence1 = "我是第一句" + "啊" * 100 + "。"
+        sentence2 = "我是第二句" + "哦" * 100 + "。"
+        sentence3 = "我是第三句" + "嗯" * 100 + "。"
+        text = sentence1 + sentence2 + sentence3
+
+        result = split_text(text, max_chars=150)
+        # 每句106字符，超过150的阈值？不，106<150
+        # 但累计到一定程度会切
+        assert len(result) >= 2
+        # 每段都不超过 max_chars
+        for seg in result:
+            assert len(seg) <= 150
+
+    def test_split_by_question_mark(self):
+        text = "你是谁？你从哪里来？你要到哪里去？"
+        result = split_text(text, max_chars=500)
+        # 三句都很短，合并成一段
+        assert len(result) == 1
+
+    def test_split_by_exclamation_mark(self):
+        text = "太棒了！太厉害了！太牛了！"
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+    def test_split_by_newline(self):
+        text = "第一段\n第二段\n第三段"
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+    def test_split_by_semicolon(self):
+        text = "第一部分；第二部分；第三部分。"
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+    def test_mixed_punctuation(self):
+        """混合标点符号的句子边界"""
+        parts = []
+        for i in range(20):
+            parts.append(f"第{i}句的内容" + "字" * 30 + "。")
+        text = "".join(parts)
+
+        result = split_text(text, max_chars=200)
+        # 每句约35字符，200字符大约能放5-6句
+        assert len(result) >= 2
+        for seg in result:
+            assert len(seg) <= 200
+
+    def test_english_period_splitting(self):
+        text = "Hello. How are you. I am fine."
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+    def test_english_question(self):
+        text = "What? Why? How?"
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+
+
+# ============================================================
+# 超长硬切
+# ============================================================
+
+
+class TestLongSentenceHardCut:
+    """超长句子硬切"""
+
+    def test_single_very_long_sentence_hard_cut(self):
+        """单个超长句子，没有标点，硬切"""
+        text = "字" * 1000
+        result = split_text(text, max_chars=500)
+        assert len(result) == 2
+        assert len(result[0]) == 500
+        assert len(result[1]) == 500
+
+    def test_three_times_max_chars(self):
+        text = "字" * 1500
+        result = split_text(text, max_chars=500)
+        assert len(result) == 3
+        for seg in result:
+            assert len(seg) == 500
+
+    def test_not_exact_multiple(self):
+        text = "字" * 1250
+        result = split_text(text, max_chars=500)
+        assert len(result) == 3
+        assert len(result[0]) == 500
+        assert len(result[1]) == 500
+        assert len(result[2]) == 250
+
+    def test_all_segments_within_limit(self):
+        """所有段都不超过 max_chars"""
+        import random
+        random.seed(42)
+        # 生成随机长度的文本
+        text = "".join(random.choices("字字字字。！？；\n", k=5000))
+        for max_chars in [100, 200, 500]:
+            result = split_text(text, max_chars=max_chars)
+            for i, seg in enumerate(result):
+                assert len(seg) <= max_chars, f"Segment {i} length {len(seg)} > {max_chars}"
+
+
+# ============================================================
+# 过短段落合并
+# ============================================================
+
+
+class TestShortSegmentMerging:
+    """过短段落合并"""
+
+    def test_short_final_segment_merged(self):
+        """最后一段过短，应该合并到前一段"""
+        # 构造：前一段接近上限，后一段很短
+        long_part = "字" * 480 + "。"
+        short_part = "好的。"
+        text = long_part + short_part
+
+        result = split_text(text, max_chars=500)
+        # 两段加起来 481+3=484 < 500，可能合并
+        # 但要看具体实现...
+        # 至少验证所有段不超长
+        for seg in result:
+            assert len(seg) <= 500
+
+    def test_multiple_short_segments(self):
+        """多个短段落应该合并"""
+        sentences = ["你好。", "我好。", "大家好。", "今天天气不错。", "适合出去玩。"]
+        text = "".join(sentences)
+        result = split_text(text, max_chars=500)
+        # 5个短句子，应该合并成一段
+        assert len(result) == 1
+
+
+# ============================================================
+# max_chars 参数
+# ============================================================
+
+
+class TestMaxCharsParameter:
+    """max_chars 参数"""
+
+    def test_small_max_chars(self):
+        text = "一二三四五六七八九十一二三四五六七八九十。"
+        result = split_text(text, max_chars=10)
+        # 应该被切成多段
+        assert len(result) >= 2
+        for seg in result:
+            assert len(seg) <= 10
+
+    def test_custom_max_chars_200(self):
+        text = "测试文本" * 100  # 400字符
+        result = split_text(text, max_chars=200)
+        assert len(result) == 2
+        assert len(result[0]) == 200
+        assert len(result[1]) == 200
+
+    def test_very_small_max_chars(self):
+        text = "abcdefghij"
+        result = split_text(text, max_chars=3)
+        assert len(result) >= 3
+        for seg in result:
+            assert len(seg) <= 3
+
+
+# ============================================================
+# 中英文混合
+# ============================================================
+
+
+class TestMixedContent:
+    """中英文混合内容"""
+
+    def test_chinese_english_mixed(self):
+        text = "今天天气很好，Today is sunny. 我们去公园玩吧！Let's go to the park."
+        result = split_text(text, max_chars=500)
+        assert len(result) == 1
+        assert result[0] == text.strip()
+
+    def test_mixed_long_text(self):
+        parts = []
+        for i in range(50):
+            parts.append(f"第{i}段中文内容" + "字" * 20 + ". English part " + "word " * 10 + "。")
+        text = "".join(parts)
+
+        result = split_text(text, max_chars=300)
+        assert len(result) >= 2
+        for seg in result:
+            assert len(seg) <= 300
+
+
+# ============================================================
+# 输出完整性
+# ============================================================
+
+
+class TestOutputIntegrity:
+    """输出完整性验证"""
+
+    def test_combined_length_equals_original(self):
+        """所有段拼接起来（去掉空段）应该等于原文长度"""
+        text = "这是第一段。这是第二段。这是第三段。这是第四段。这是第五段。" * 20
+        result = split_text(text, max_chars=100)
+        combined = "".join(result)
+        # 由于 strip 可能去掉一些空格，原文也 strip 比较
+        assert len(combined) == len(text.strip())
+
+    def test_order_preserved(self):
+        """分段后再拼接，文本顺序不变"""
+        text = "第一。第二。第三。第四。第五。" * 10
+        result = split_text(text, max_chars=50)
+        combined = "".join(result)
+        assert combined == text.strip()
+
+    def test_no_empty_strings_in_result(self):
+        """结果中没有空字符串"""
+        text = "句子一。句子二。句子三。"
+        result = split_text(text, max_chars=10)
+        for seg in result:
+            assert seg != ""
+            assert len(seg) > 0
+
+
+# ============================================================
+# 边界情况
+# ============================================================
+
+
+class TestEdgeCases:
+    """边界情况"""
+
+    def test_single_character(self):
+        assert split_text("一", max_chars=500) == ["一"]
+
+    def test_only_punctuation(self):
+        text = "。。。。。"
+        result = split_text(text, max_chars=500)
+        # 都是标点，也算文本
+        assert len(result) == 1
+
+    def test_only_newlines(self):
+        text = "\n\n\n"
+        result = split_text(text, max_chars=500)
+        assert result == []
+
+    def test_long_text_many_sentences(self):
+        """大量句子的长文本"""
+        sentences = [f"第{i}句的完整内容。" for i in range(100)]
+        text = "".join(sentences)
+        result = split_text(text, max_chars=200)
+        assert len(result) >= 5
+        for seg in result:
+            assert len(seg) <= 200