xiaoxia-saas/tests/unit/test_text_splitter.py

"""文本分段工具单元测试."""

from __future__ import annotations

import pytest

from packages.application.tts_job.text_splitter import split_text


class TestSplitText:
    """split_text 函数测试"""

    def test_empty_string_returns_empty_list(self):
        """空字符串返回空列表"""
        assert split_text("") == []

    def test_whitespace_only_returns_empty_list(self):
        """纯空白字符返回空列表"""
        assert split_text("   \n  \t  ") == []

    def test_short_text_returns_single_segment(self):
        """短文本直接返回单段"""
        text = "这是一段短文本。"
        result = split_text(text, max_chars=500)
        assert result == [text]

    def test_text_length_equals_max_chars(self):
        """文本长度恰好等于 max_chars 时返回单段"""
        text = "a" * 100
        result = split_text(text, max_chars=100)
        assert len(result) == 1
        assert len(result[0]) == 100

    def test_splits_on_sentence_boundary(self):
        """在句子边界处分段"""
        # 构造长文本，确保超过 max_chars
        sentences = ["今天天气真好。我们一起去公园散步吧。", "公园里有很多花。还有很多小朋友在玩耍。"] * 10
        text = "".join(sentences)

        result = split_text(text, max_chars=200)

        assert len(result) >= 2
        # 每段都不超过 max_chars
        for seg in result:
            assert len(seg) <= 200

    def test_all_segments_within_max_chars(self):
        """所有分段都不超过 max_chars"""
        text = "这是第一句话。这是第二句话。这是第三句话。这是第四句话。这是第五句话。" * 10

        result = split_text(text, max_chars=100)

        for seg in result:
            assert len(seg) <= 100

    def test_long_single_sentence_hard_cut(self):
        """超长单句会被硬切"""
        text = "a" * 1000  # 没有标点

        result = split_text(text, max_chars=200)

        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 200

    def test_newline_is_sentence_end(self):
        """换行符作为句子结束符"""
        text = "第一行内容\n第二行内容\n第三行内容" * 10

        result = split_text(text, max_chars=50)

        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 50

    def test_chinese_punctuation(self):
        """中文标点（。！？；）作为句子结束符"""
        text = "你好！今天吃什么？我吃米饭；你呢？我也吃米饭。" * 10

        result = split_text(text, max_chars=80)

        for seg in result:
            assert len(seg) <= 80

    def test_english_punctuation(self):
        """英文标点（.!?;）作为句子结束符"""
        text = "Hello! How are you? I'm fine; thank you. Good bye." * 10

        result = split_text(text, max_chars=80)

        for seg in result:
            assert len(seg) <= 80

    def test_merged_short_segments(self):
        """过短的段落会被合并"""
        # 构造很多短句
        text = "你好。再见。谢谢。抱歉。好的。不行。可以。去吧。" * 5  # 每句3-4字

        result = split_text(text, max_chars=100)

        # 合并后段数应该比单纯按句切的少
        assert len(result) < len(text) // 3  # 粗略估计
        for seg in result:
            assert len(seg) <= 100

    def test_preserves_content(self):
        """分段后内容总和与原文基本一致（忽略strip的空白）"""
        text = "这是测试文本。包含多个句子。用来验证分段正确性。" * 5

        result = split_text(text, max_chars=50)

        # 合并所有分段，去掉空白后应该与原文去掉空白后基本一致
        combined = "".join(result).replace(" ", "")
        original = text.strip().replace(" ", "")
        assert combined == original

    def test_custom_max_chars(self):
        """支持自定义 max_chars"""
        text = "测试" * 100  # 200字

        result_50 = split_text(text, max_chars=50)
        result_100 = split_text(text, max_chars=100)

        # max_chars 越小，段数应该越多
        assert len(result_50) >= len(result_100)

    def test_single_char_text(self):
        """单字符文本"""
        assert split_text("好", max_chars=10) == ["好"]

    def test_text_with_only_punctuation(self):
        """纯标点文本"""
        text = "。。。。。。。。。。"  # 10个句号
        result = split_text(text, max_chars=5)

        assert len(result) >= 1
        for seg in result:
            assert len(seg) <= 5

    def test_mixed_content(self):
        """中英文混合内容"""
        text = "今天的天气是 sunny and warm。我们去了 park 玩。真的很开心！" * 5

        result = split_text(text, max_chars=80)

        for seg in result:
            assert len(seg) <= 80


# ── 短文本与空文本补充 ──────────────────────────────────────────────────────


class TestSplitTextEmptyAndShort:
    """空文本与短文本补充场景."""

    def test_whitespace_only_returns_empty(self):
        """纯空白文本返回空列表."""
        assert split_text("   \n\t  ") == []

    def test_single_char(self):
        """单字符文本."""
        assert split_text("好", max_chars=10) == ["好"]

    def test_exactly_max_chars_no_split(self):
        """刚好等于 max_chars 不分割."""
        text = "a" * 100
        result = split_text(text, max_chars=100)
        assert len(result) == 1
        assert result[0] == text

    def test_one_over_max_chars_splits(self):
        """超过 max_chars 1 个字符就会分割."""
        text = "a" * 101
        result = split_text(text, max_chars=100)
        assert len(result) >= 2

    def test_none_raises(self):
        """None 输入抛 AttributeError（strip 失败）."""
        with pytest.raises(AttributeError):
            split_text(None)


# ── 句子边界分段补充 ──────────────────────────────────────────────────────


class TestSplitTextSentenceBoundaries:
    """句子边界分段补充场景."""

    def test_split_on_fullwidth_period(self):
        """全角句号分段."""
        text = "第一句很长的内容。" * 20
        result = split_text(text, max_chars=60)
        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 60

    def test_split_on_fullwidth_question(self):
        """全角问号分段."""
        text = "你知道这是为什么吗？" + "是的。" * 20
        result = split_text(text, max_chars=60)
        assert len(result) > 1

    def test_split_on_fullwidth_exclamation(self):
        """全角感叹号分段."""
        text = "真是太棒了！" + "内容。" * 20
        result = split_text(text, max_chars=60)
        assert len(result) > 1

    def test_split_on_newline(self):
        """换行符分段."""
        lines = ["这是第一行很长的一段文字内容" * 3 for _ in range(5)]
        text = "\n".join(lines)
        result = split_text(text, max_chars=80)
        assert len(result) > 1

    def test_split_on_semicolon(self):
        """全角分号分段."""
        text = "第一项内容；" + "其他内容。" * 20
        result = split_text(text, max_chars=60)
        assert len(result) > 1

    def test_english_period_splits(self):
        """英文句号分段."""
        text = "Hello world. " * 30
        result = split_text(text, max_chars=80)
        assert len(result) > 1

    def test_short_sentences_stay_merged(self):
        """短句（都 < 50字的句子不会单独成段，会累积到一起."""
        text = "你好。我好。大家好。"
        result = split_text(text, max_chars=200)
        assert len(result) == 1


# ── 长句强制切段补充 ──────────────────────────────────────────────────────


class TestSplitTextLongSentenceForce:
    """超长单句强制切段补充."""

    def test_no_punctuation_forced_split(self):
        """完全没有标点的超长文本硬切."""
        text = "字" * 300
        result = split_text(text, max_chars=100)
        assert len(result) == 3
        for seg in result:
            assert len(seg) == 100

    def test_force_split_preserves_content(self):
        """硬切不丢字符."""
        text = "a" * 250
        result = split_text(text, max_chars=100)
        assert sum(len(s) for s in result) == 250

    def test_mixed_long_and_short(self):
        """长句短句混合."""
        long_part = "非常长的句子没有标点符号" * 15
        text = long_part + "。结尾。"
        result = split_text(text, max_chars=100)
        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 100


# ── 短段合并补充 ─────────────────────────────────────────────────────────


class TestSplitTextShortSegmentMerge:
    """短段合并补充场景."""

    def test_multiple_short_sentences_merged(self):
        """多个短句合并成一段."""
        sentences = ["你好。", "我好。", "大家好。", "天气好。", "心情好。"]
        text = "".join(sentences)
        result = split_text(text, max_chars=200)
        assert len(result) == 1

    def test_short_tail_merged(self):
        """尾部短段被合并到前一段."""
        # 前面一段接近 max_chars，尾部很短
        long_part = "一二三四五六七八九十" * 9 + "。"  # ~90字
        tail = "完。"  # 2字
        text = long_part + tail
        result = split_text(text, max_chars=100)
        # 尾部短的应该被合并
        assert len(result) <= 2


# ── 边界情况补充 ─────────────────────────────────────────────────────────


class TestSplitTextEdgeCases:
    """边界情况补充."""

    def test_only_punctuation(self):
        """纯标点符号."""
        text = "。。。。。"
        result = split_text(text, max_chars=10)
        assert len(result) == 1

    def test_mixed_chinese_english(self):
        """中英文混合."""
        text = "你好Hello。World!" * 20
        result = split_text(text, max_chars=100)
        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 100

    def test_strip_whitespace(self):
        """首尾空白被去除."""
        text = "   你好世界。   "
        result = split_text(text, max_chars=100)
        assert result == ["你好世界。"]

    def test_total_length_preserved(self):
        """分段后总长度等于原文 strip 后长度."""
        text = "这是一段用于测试的文本内容。" * 20
        result = split_text(text, max_chars=100)
        assert "".join(result) == text.strip()

    def test_custom_small_max_chars(self):
        """很小的 max_chars."""
        text = "一二三四五六七八九十。" * 5
        result = split_text(text, max_chars=20)
        assert len(result) > 1
        for seg in result:
            assert len(seg) <= 20


# ── 更多边界场景补充 ─────────────────────────────────────────────────────────


class TestSplitTextMoreEdgeCases:
    """更多边界场景补充"""

    def test_max_chars_one(self):
        """max_chars=1 每个字符一段"""
        text = "一二三四五"
        result = split_text(text, max_chars=1)
        assert len(result) == 5
        for seg in result:
            assert len(seg) == 1

    def test_consecutive_newlines(self):
        """连续多个换行符"""
        text = "第一段\n\n\n第二段\n\n第三段"
        result = split_text(text, max_chars=100)
        # 合并后应该是一段（内容不长且合并逻辑会被合并）
        assert len(result) >= 1
        assert "第一段" in result[0]
        for seg in result:
            assert len(seg) <= 100

    def test_only_newlines_only(self):
        """只有换行符（纯空白被strip掉返回空"""
        assert split_text("\n\n\n\n") == []

    def test_leading_trailing_whitespace(self):
        """首尾空白被去除"""
        text = "   你好世界。   "
        result = split_text(text, max_chars=100)
        assert result == ["你好世界。"]

    def test_very_long_single_sentence_many_segments(self):
        """超长单句被切成很多段"""
        text = "字" * 1000
        result = split_text(text, max_chars=100)
        assert len(result) == 10
        for seg in result:
            assert len(seg) == 100

    def test_mixed_punctuation_types(self):
        """全角半角标点混合"""
        text = "你好!再见。谢谢?抱歉；好的"
        result = split_text(text, max_chars=200)
        assert len(result) == 1

    def test_last_segment_short_merged_to_previous(self):
        """尾部极短段被合并到前一段"""
        # 构造第一段接近max_chars，结尾有个短句尾巴
        long_part = "一二三四五六七八九十" * 9 + "。"  # ~90字
        tail = "完"  # 1字
        text = long_part + tail
        result = split_text(text, max_chars=100)
        # 尾巴应该被合并
        combined = "".join(result)
        assert combined == text.strip()
        assert len(result) <= 2

    def test_all_short_sentences_merged_into_one(self):
        """大量短句全部合并成一段"""
        sentences = ["你好。", "我好。", "他好。", "大家好。", "才是真的好。"]
        text = "".join(sentences)
        result = split_text(text, max_chars=200)
        assert len(result) == 1

    def test_punctuation_only_long(self):
        """很长的纯标点文本"""
        text = "。" * 200
        result = split_text(text, max_chars=50)
        assert len(result) >= 4
        for seg in result:
            assert len(seg) <= 50

    def test_tab_not_sentence_end(self):
        """制表符不是句子结束符"""
        text = "这是一段\t包含制表符的文本内容" + "字" * 100
        result = split_text(text, max_chars=50)
        # 制表符不在句子结束符集合中，不会触发分段
        # 制表符会保留在分段内容中
        has_tab = any("\t" in seg for seg in result)
        assert has_tab