xiaoxia-saas/tests/unit/test_text_splitter.py

"""text_splitter 单元测试."""

from packages.application.tts_job.text_splitter import split_text


class TestSplitText:
    def test_empty_text_returns_empty(self):
        assert split_text("") == []

    def test_whitespace_only(self):
        assert split_text("   \n\t  ") == []

    def test_short_text_single_segment(self):
        text = "你好世界。"
        result = split_text(text, max_chars=500)
        assert len(result) == 1
        assert result[0] == text

    def test_exact_max_chars(self):
        text = "a" * 500
        result = split_text(text, max_chars=500)
        assert len(result) == 1
        assert len(result[0]) == 500

    def test_splits_on_sentence_boundary(self):
        # 两个长句子，各300字左右，超过50字阈值
        sent1 = "你" * 300 + "。"
        sent2 = "我" * 300 + "。"
        text = sent1 + sent2
        result = split_text(text, max_chars=500)
        assert len(result) == 2
        assert result[0] == sent1
        assert result[1] == sent2

    def test_long_sentence_hard_cut(self):
        # 一个超长句子，没有句末标点，会被硬切
        text = "长" * 800
        result = split_text(text, max_chars=500)
        assert len(result) >= 2
        assert all(len(seg) <= 500 for seg in result)
        # 合起来应该等于原文本
        assert "".join(result) == text

    def test_short_segments_merged(self):
        # 多个短句应该被合并
        sentences = [f"第{i}句。" for i in range(10)]
        text = "".join(sentences)
        result = split_text(text, max_chars=200)
        # 每句5字左右，10句才50字，应该合并成1段
        assert len(result) < 10
        assert len(result[0]) <= 200

    def test_preserves_content(self):
        text = "今天天气真好。我们去公园玩吧！你觉得怎么样？好的，走吧。"
        result = split_text(text, max_chars=20)
        # 合并后内容应一致
        assert "".join(result) == text

    def test_multiple_punctuation_types(self):
        # 构造足够长的文本触发分段
        text = "第一" * 30 + "。" + "第二" * 30 + "！" + "第三" * 30 + "？" + "第四" * 30 + "；"
        result = split_text(text, max_chars=100)
        assert len(result) >= 2
        assert "".join(result) == text

    def test_custom_max_chars(self):
        text = "a" * 100 + "。" + "b" * 100 + "。"
        result = split_text(text, max_chars=150)
        assert len(result) == 2
        assert "a" in result[0]
        assert "b" in result[1]

    def test_newline_as_sentence_end(self):
        text = "第一段\n第二段\n第三段"
        result = split_text(text, max_chars=50)
        assert len(result) >= 1
        assert "".join(result) == text.strip()

    def test_minimum_segment_length(self):
        # 句子太短（<50字）不会立即分段
        text = "短句一。短句二。短句三。"
        result = split_text(text, max_chars=200)
        assert len(result) == 1

    def test_trailing_content_added(self):
        # 最后一段不完整的句子也要加上
        text = "完整的句子。剩余内容"
        result = split_text(text, max_chars=50)
        assert "".join(result) == text

    def test_no_empty_segments(self):
        text = "。。。。。"  # 全是标点
        result = split_text(text, max_chars=2)
        assert all(len(seg) > 0 for seg in result)

    def test_chinese_and_english_mixed(self):
        text = "Hello世界。这是测试Test文本。Mixed混合。"
        result = split_text(text, max_chars=20)
        assert len(result) >= 2
        assert "".join(result) == text