""" text_splitter 长文本分段工具单元测试 覆盖： - 空文本 / 短文本 - 句子边界分段（。！？；\n . ! ? ;） - 超长句子硬切 - 过短段落合并 - max_chars 参数 - 中英文混合 """ import pytest from packages.application.tts_job.text_splitter import split_text # ============================================================ # 基础场景 # ============================================================ class TestBasicCases: """基础场景""" def test_empty_text_returns_empty_list(self): assert split_text("") == [] def test_whitespace_only_returns_empty(self): assert split_text(" \n\n ") == [] def test_short_text_single_segment(self): text = "这是一段短文本。" result = split_text(text, max_chars=500) assert result == [text] def test_exactly_max_chars_single_segment(self): text = "a" * 500 result = split_text(text, max_chars=500) assert len(result) == 1 assert len(result[0]) == 500 def test_text_stripped(self): text = " 你好世界。 " result = split_text(text, max_chars=500) assert result == ["你好世界。"] # ============================================================ # 句子边界分段 # ============================================================ class TestSentenceBoundarySplitting: """句子边界分段""" def test_split_by_chinese_period(self): text = "第一句。第二句。第三句。" # 三句都很短，应该合并成一段 result = split_text(text, max_chars=500) assert len(result) == 1 def test_split_by_chinese_period_long_text(self): """多段长句子，按句号分段""" sentence1 = "我是第一句" + "啊" * 100 + "。" sentence2 = "我是第二句" + "哦" * 100 + "。" sentence3 = "我是第三句" + "嗯" * 100 + "。" text = sentence1 + sentence2 + sentence3 result = split_text(text, max_chars=150) # 每句106字符，超过150的阈值？不，106<150 # 但累计到一定程度会切 assert len(result) >= 2 # 每段都不超过 max_chars for seg in result: assert len(seg) <= 150 def test_split_by_question_mark(self): text = "你是谁？你从哪里来？你要到哪里去？" result = split_text(text, max_chars=500) # 三句都很短，合并成一段 assert len(result) == 1 def test_split_by_exclamation_mark(self): text = "太棒了！太厉害了！太牛了！" result = split_text(text, max_chars=500) assert len(result) == 1 def test_split_by_newline(self): text = "第一段\n第二段\n第三段" result = split_text(text, max_chars=500) assert len(result) == 1 def test_split_by_semicolon(self): text = "第一部分；第二部分；第三部分。" result = split_text(text, max_chars=500) assert len(result) == 1 def test_mixed_punctuation(self): """混合标点符号的句子边界""" parts = [] for i in range(20): parts.append(f"第{i}句的内容" + "字" * 30 + "。") text = "".join(parts) result = split_text(text, max_chars=200) # 每句约35字符，200字符大约能放5-6句 assert len(result) >= 2 for seg in result: assert len(seg) <= 200 def test_english_period_splitting(self): text = "Hello. How are you. I am fine." result = split_text(text, max_chars=500) assert len(result) == 1 def test_english_question(self): text = "What? Why? How?" result = split_text(text, max_chars=500) assert len(result) == 1 # ============================================================ # 超长硬切 # ============================================================ class TestLongSentenceHardCut: """超长句子硬切""" def test_single_very_long_sentence_hard_cut(self): """单个超长句子，没有标点，硬切""" text = "字" * 1000 result = split_text(text, max_chars=500) assert len(result) == 2 assert len(result[0]) == 500 assert len(result[1]) == 500 def test_three_times_max_chars(self): text = "字" * 1500 result = split_text(text, max_chars=500) assert len(result) == 3 for seg in result: assert len(seg) == 500 def test_not_exact_multiple(self): text = "字" * 1250 result = split_text(text, max_chars=500) assert len(result) == 3 assert len(result[0]) == 500 assert len(result[1]) == 500 assert len(result[2]) == 250 def test_all_segments_within_limit(self): """所有段都不超过 max_chars""" import random random.seed(42) # 生成随机长度的文本 text = "".join(random.choices("字字字字。！？；\n", k=5000)) for max_chars in [100, 200, 500]: result = split_text(text, max_chars=max_chars) for i, seg in enumerate(result): assert len(seg) <= max_chars, f"Segment {i} length {len(seg)} > {max_chars}" # ============================================================ # 过短段落合并 # ============================================================ class TestShortSegmentMerging: """过短段落合并""" def test_short_final_segment_merged(self): """最后一段过短，应该合并到前一段""" # 构造：前一段接近上限，后一段很短 long_part = "字" * 480 + "。" short_part = "好的。" text = long_part + short_part result = split_text(text, max_chars=500) # 两段加起来 481+3=484 < 500，可能合并 # 但要看具体实现... # 至少验证所有段不超长 for seg in result: assert len(seg) <= 500 def test_multiple_short_segments(self): """多个短段落应该合并""" sentences = ["你好。", "我好。", "大家好。", "今天天气不错。", "适合出去玩。"] text = "".join(sentences) result = split_text(text, max_chars=500) # 5个短句子，应该合并成一段 assert len(result) == 1 # ============================================================ # max_chars 参数 # ============================================================ class TestMaxCharsParameter: """max_chars 参数""" def test_small_max_chars(self): text = "一二三四五六七八九十一二三四五六七八九十。" result = split_text(text, max_chars=10) # 应该被切成多段 assert len(result) >= 2 for seg in result: assert len(seg) <= 10 def test_custom_max_chars_200(self): text = "测试文本" * 100 # 400字符 result = split_text(text, max_chars=200) assert len(result) == 2 assert len(result[0]) == 200 assert len(result[1]) == 200 def test_very_small_max_chars(self): text = "abcdefghij" result = split_text(text, max_chars=3) assert len(result) >= 3 for seg in result: assert len(seg) <= 3 # ============================================================ # 中英文混合 # ============================================================ class TestMixedContent: """中英文混合内容""" def test_chinese_english_mixed(self): text = "今天天气很好，Today is sunny. 我们去公园玩吧！Let's go to the park." result = split_text(text, max_chars=500) assert len(result) == 1 assert result[0] == text.strip() def test_mixed_long_text(self): parts = [] for i in range(50): parts.append(f"第{i}段中文内容" + "字" * 20 + ". English part " + "word " * 10 + "。") text = "".join(parts) result = split_text(text, max_chars=300) assert len(result) >= 2 for seg in result: assert len(seg) <= 300 # ============================================================ # 输出完整性 # ============================================================ class TestOutputIntegrity: """输出完整性验证""" def test_combined_length_equals_original(self): """所有段拼接起来（去掉空段）应该等于原文长度""" text = "这是第一段。这是第二段。这是第三段。这是第四段。这是第五段。" * 20 result = split_text(text, max_chars=100) combined = "".join(result) # 由于 strip 可能去掉一些空格，原文也 strip 比较 assert len(combined) == len(text.strip()) def test_order_preserved(self): """分段后再拼接，文本顺序不变""" text = "第一。第二。第三。第四。第五。" * 10 result = split_text(text, max_chars=50) combined = "".join(result) assert combined == text.strip() def test_no_empty_strings_in_result(self): """结果中没有空字符串""" text = "句子一。句子二。句子三。" result = split_text(text, max_chars=10) for seg in result: assert seg != "" assert len(seg) > 0 # ============================================================ # 边界情况 # ============================================================ class TestEdgeCases: """边界情况""" def test_single_character(self): assert split_text("一", max_chars=500) == ["一"] def test_only_punctuation(self): text = "。。。。。" result = split_text(text, max_chars=500) # 都是标点，也算文本 assert len(result) == 1 def test_only_newlines(self): text = "\n\n\n" result = split_text(text, max_chars=500) assert result == [] def test_long_text_many_sentences(self): """大量句子的长文本""" sentences = [f"第{i}句的完整内容。" for i in range(100)] text = "".join(sentences) result = split_text(text, max_chars=200) assert len(result) >= 5 for seg in result: assert len(seg) <= 200