xiaoxia-saas/tests/unit/test_text_splitter.py

"""
text_splitter 长文本分段工具单元测试

覆盖：
- 空文本 / 短文本
- 句子边界分段（。！？；\n . ! ? ;）
- 超长句子硬切
- 过短段落合并
- max_chars 参数
- 中英文混合
"""

import pytest

from packages.application.tts_job.text_splitter import split_text

# ============================================================
# 基础场景
# ============================================================


class TestBasicCases:
    """基础场景"""

    def test_empty_text_returns_empty_list(self):
        assert split_text("") == []

    def test_whitespace_only_returns_empty(self):
        assert split_text("   \n\n  ") == []

    def test_short_text_single_segment(self):
        text = "这是一段短文本。"
        result = split_text(text, max_chars=500)
        assert result == [text]

    def test_exactly_max_chars_single_segment(self):
        text = "a" * 500
        result = split_text(text, max_chars=500)
        assert len(result) == 1
        assert len(result[0]) == 500

    def test_text_stripped(self):
        text = "   你好世界。   "
        result = split_text(text, max_chars=500)
        assert result == ["你好世界。"]


# ============================================================
# 句子边界分段
# ============================================================


class TestSentenceBoundarySplitting:
    """句子边界分段"""

    def test_split_by_chinese_period(self):
        text = "第一句。第二句。第三句。"
        # 三句都很短，应该合并成一段
        result = split_text(text, max_chars=500)
        assert len(result) == 1

    def test_split_by_chinese_period_long_text(self):
        """多段长句子，按句号分段"""
        sentence1 = "我是第一句" + "啊" * 100 + "。"
        sentence2 = "我是第二句" + "哦" * 100 + "。"
        sentence3 = "我是第三句" + "嗯" * 100 + "。"
        text = sentence1 + sentence2 + sentence3

        result = split_text(text, max_chars=150)
        # 每句106字符，超过150的阈值？不，106<150
        # 但累计到一定程度会切
        assert len(result) >= 2
        # 每段都不超过 max_chars
        for seg in result:
            assert len(seg) <= 150

    def test_split_by_question_mark(self):
        text = "你是谁？你从哪里来？你要到哪里去？"
        result = split_text(text, max_chars=500)
        # 三句都很短，合并成一段
        assert len(result) == 1

    def test_split_by_exclamation_mark(self):
        text = "太棒了！太厉害了！太牛了！"
        result = split_text(text, max_chars=500)
        assert len(result) == 1

    def test_split_by_newline(self):
        text = "第一段\n第二段\n第三段"
        result = split_text(text, max_chars=500)
        assert len(result) == 1

    def test_split_by_semicolon(self):
        text = "第一部分；第二部分；第三部分。"
        result = split_text(text, max_chars=500)
        assert len(result) == 1

    def test_mixed_punctuation(self):
        """混合标点符号的句子边界"""
        parts = []
        for i in range(20):
            parts.append(f"第{i}句的内容" + "字" * 30 + "。")
        text = "".join(parts)

        result = split_text(text, max_chars=200)
        # 每句约35字符，200字符大约能放5-6句
        assert len(result) >= 2
        for seg in result:
            assert len(seg) <= 200

    def test_english_period_splitting(self):
        text = "Hello. How are you. I am fine."
        result = split_text(text, max_chars=500)
        assert len(result) == 1

    def test_english_question(self):
        text = "What? Why? How?"
        result = split_text(text, max_chars=500)
        assert len(result) == 1


# ============================================================
# 超长硬切
# ============================================================


class TestLongSentenceHardCut:
    """超长句子硬切"""

    def test_single_very_long_sentence_hard_cut(self):
        """单个超长句子，没有标点，硬切"""
        text = "字" * 1000
        result = split_text(text, max_chars=500)
        assert len(result) == 2
        assert len(result[0]) == 500
        assert len(result[1]) == 500

    def test_three_times_max_chars(self):
        text = "字" * 1500
        result = split_text(text, max_chars=500)
        assert len(result) == 3
        for seg in result:
            assert len(seg) == 500

    def test_not_exact_multiple(self):
        text = "字" * 1250
        result = split_text(text, max_chars=500)
        assert len(result) == 3
        assert len(result[0]) == 500
        assert len(result[1]) == 500
        assert len(result[2]) == 250

    def test_all_segments_within_limit(self):
        """所有段都不超过 max_chars"""
        import random

        random.seed(42)
        # 生成随机长度的文本
        text = "".join(random.choices("字字字字。！？；\n", k=5000))
        for max_chars in [100, 200, 500]:
            result = split_text(text, max_chars=max_chars)
            for i, seg in enumerate(result):
                assert len(seg) <= max_chars, f"Segment {i} length {len(seg)} > {max_chars}"


# ============================================================
# 过短段落合并
# ============================================================


class TestShortSegmentMerging:
    """过短段落合并"""

    def test_short_final_segment_merged(self):
        """最后一段过短，应该合并到前一段"""
        # 构造：前一段接近上限，后一段很短
        long_part = "字" * 480 + "。"
        short_part = "好的。"
        text = long_part + short_part

        result = split_text(text, max_chars=500)
        # 两段加起来 481+3=484 < 500，可能合并
        # 但要看具体实现...
        # 至少验证所有段不超长
        for seg in result:
            assert len(seg) <= 500

    def test_multiple_short_segments(self):
        """多个短段落应该合并"""
        sentences = ["你好。", "我好。", "大家好。", "今天天气不错。", "适合出去玩。"]
        text = "".join(sentences)
        result = split_text(text, max_chars=500)
        # 5个短句子，应该合并成一段
        assert len(result) == 1


# ============================================================
# max_chars 参数
# ============================================================


class TestMaxCharsParameter:
    """max_chars 参数"""

    def test_small_max_chars(self):
        text = "一二三四五六七八九十一二三四五六七八九十。"
        result = split_text(text, max_chars=10)
        # 应该被切成多段
        assert len(result) >= 2
        for seg in result:
            assert len(seg) <= 10

    def test_custom_max_chars_200(self):
        text = "测试文本" * 100  # 400字符
        result = split_text(text, max_chars=200)
        assert len(result) == 2
        assert len(result[0]) == 200
        assert len(result[1]) == 200

    def test_very_small_max_chars(self):
        text = "abcdefghij"
        result = split_text(text, max_chars=3)
        assert len(result) >= 3
        for seg in result:
            assert len(seg) <= 3


# ============================================================
# 中英文混合
# ============================================================


class TestMixedContent:
    """中英文混合内容"""

    def test_chinese_english_mixed(self):
        text = "今天天气很好，Today is sunny. 我们去公园玩吧！Let's go to the park."
        result = split_text(text, max_chars=500)
        assert len(result) == 1
        assert result[0] == text.strip()

    def test_mixed_long_text(self):
        parts = []
        for i in range(50):
            parts.append(f"第{i}段中文内容" + "字" * 20 + ". English part " + "word " * 10 + "。")
        text = "".join(parts)

        result = split_text(text, max_chars=300)
        assert len(result) >= 2
        for seg in result:
            assert len(seg) <= 300


# ============================================================
# 输出完整性
# ============================================================


class TestOutputIntegrity:
    """输出完整性验证"""

    def test_combined_length_equals_original(self):
        """所有段拼接起来（去掉空段）应该等于原文长度"""
        text = "这是第一段。这是第二段。这是第三段。这是第四段。这是第五段。" * 20
        result = split_text(text, max_chars=100)
        combined = "".join(result)
        # 由于 strip 可能去掉一些空格，原文也 strip 比较
        assert len(combined) == len(text.strip())

    def test_order_preserved(self):
        """分段后再拼接，文本顺序不变"""
        text = "第一。第二。第三。第四。第五。" * 10
        result = split_text(text, max_chars=50)
        combined = "".join(result)
        assert combined == text.strip()

    def test_no_empty_strings_in_result(self):
        """结果中没有空字符串"""
        text = "句子一。句子二。句子三。"
        result = split_text(text, max_chars=10)
        for seg in result:
            assert seg != ""
            assert len(seg) > 0


# ============================================================
# 边界情况
# ============================================================


class TestEdgeCases:
    """边界情况"""

    def test_single_character(self):
        assert split_text("一", max_chars=500) == ["一"]

    def test_only_punctuation(self):
        text = "。。。。。"
        result = split_text(text, max_chars=500)
        # 都是标点，也算文本
        assert len(result) == 1

    def test_only_newlines(self):
        text = "\n\n\n"
        result = split_text(text, max_chars=500)
        assert result == []

    def test_long_text_many_sentences(self):
        """大量句子的长文本"""
        sentences = [f"第{i}句的完整内容。" for i in range(100)]
        text = "".join(sentences)
        result = split_text(text, max_chars=200)
        assert len(result) >= 5
        for seg in result:
            assert len(seg) <= 200