CI Bot
37b3bf8db2
- test_url_security: URL安全校验（SSRF防护/魔数校验/可信域名/直接IP拦截） ~85个 - test_pagination: 分页器（参数校验/偏移计算/元数据/内存分页） ~35个 - test_text_splitter: 文本分段器（句子边界/强制切段/短段合并） ~24个 全部纯逻辑，无外部依赖
220 lines
7.7 KiB
Python
Executable File
220 lines
7.7 KiB
Python
Executable File
"""文本分段器纯逻辑测试 — split_text.
|
||
|
||
覆盖空文本、短文本、句子边界分段、超长句强制切段、过短段合并等场景。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from application.tts_job.text_splitter import split_text
|
||
|
||
|
||
class TestSplitTextEmptyOrShort:
|
||
"""空文本与短文本."""
|
||
|
||
def test_empty_string_returns_empty_list(self):
|
||
"""空字符串返回空列表."""
|
||
assert split_text("") == []
|
||
|
||
def test_whitespace_only_returns_empty_list(self):
|
||
"""纯空白字符串返回空列表."""
|
||
assert split_text(" \n\t ") == []
|
||
|
||
def test_none_not_supported(self):
|
||
"""None不支持（strip会报错）."""
|
||
with pytest.raises(AttributeError):
|
||
split_text(None)
|
||
|
||
def test_short_text_single_segment(self):
|
||
"""短文本不分割，单段返回."""
|
||
text = "你好世界。"
|
||
result = split_text(text, max_chars=500)
|
||
assert result == [text]
|
||
|
||
def test_exactly_max_chars_single_segment(self):
|
||
"""刚好等于max_chars时不分段."""
|
||
text = "a" * 100
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) == 1
|
||
assert len(result[0]) == 100
|
||
|
||
def test_under_max_chars_single_segment(self):
|
||
"""少于max_chars时不分段."""
|
||
text = "a" * 50
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) == 1
|
||
assert len(result[0]) == 50
|
||
|
||
|
||
class TestSplitTextSentenceBoundary:
|
||
"""句子边界分段."""
|
||
|
||
def test_split_by_period(self):
|
||
"""按句号分段."""
|
||
text = "第一句内容。" + "第二句内容。" * 50
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) > 1
|
||
# 每段都不超过max_chars
|
||
for seg in result:
|
||
assert len(seg) <= 100
|
||
# 合并后等于原文（去空格后近似）
|
||
assert "".join(result) == text.replace(" ", "")
|
||
|
||
def test_split_by_question_mark(self):
|
||
"""按问号分段."""
|
||
text = "你好吗？" + "我很好。" * 50
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
for seg in result:
|
||
assert len(seg) <= 80
|
||
|
||
def test_split_by_exclamation_mark(self):
|
||
"""按感叹号分段."""
|
||
text = "太棒了！" + "真的好。" * 50
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
|
||
def test_split_by_newline(self):
|
||
"""按换行符分段."""
|
||
lines = ["这是第一行很长的内容" * 5 for _ in range(10)]
|
||
text = "\n".join(lines)
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) > 1
|
||
for seg in result:
|
||
assert len(seg) <= 100
|
||
|
||
def test_split_by_semicolon_fullwidth(self):
|
||
"""按全角分号分段."""
|
||
text = "第一项；" + "第二项内容。" * 40
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
|
||
def test_split_by_english_period(self):
|
||
"""按英文句号也分段（_SENTENCE_ENDS包含.）."""
|
||
text = "Hello. " + "World. " * 50
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
|
||
def test_split_by_english_question(self):
|
||
"""按英文问号分段."""
|
||
text = "Really? " + "Yes. " * 50
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
|
||
def test_split_by_english_exclamation(self):
|
||
"""按英文感叹号分段."""
|
||
text = "Wow! " + "Great. " * 50
|
||
result = split_text(text, max_chars=80)
|
||
assert len(result) > 1
|
||
|
||
def test_short_sentences_not_split(self):
|
||
"""短句（<50字）即使有句号也不立刻切，等累积到一定长度."""
|
||
# 每句5字，即使有句号也不会在50字前切
|
||
text = "你好。" * 5 # 15字符
|
||
result = split_text(text, max_chars=100)
|
||
# 因为每段至少50字才在句子边界切，所以15字的文本应该是1段
|
||
assert len(result) == 1
|
||
|
||
|
||
class TestSplitTextLongSentenceForceSplit:
|
||
"""超长单句强制切段."""
|
||
|
||
def test_single_very_long_sentence_forced_split(self):
|
||
"""单句超长时强制切段."""
|
||
text = "啊" * 200 # 没有标点，200字
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) >= 2
|
||
for seg in result:
|
||
assert len(seg) <= 100
|
||
|
||
def test_force_split_preserves_all_chars(self):
|
||
"""强制切段不丢字符."""
|
||
text = "a" * 250
|
||
result = split_text(text, max_chars=100)
|
||
# 所有段的总长度应等于原文（去掉空白可能有细微差异，但纯字母应该不变）
|
||
assert sum(len(s) for s in result) == 250
|
||
|
||
def test_mixed_long_and_short_sentences(self):
|
||
"""长句短句混合."""
|
||
long_part = "非常长的句子没有标点符号" * 20
|
||
text = long_part + "。结束句。"
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) > 1
|
||
for seg in result:
|
||
assert len(seg) <= 100
|
||
|
||
|
||
class TestSplitTextShortSegmentMerging:
|
||
"""过短段落合并."""
|
||
|
||
def test_short_final_segment_merged(self):
|
||
"""最后一段过短会被合并到前一段（如果不超限）."""
|
||
# 构造两段，第二段很短
|
||
text = "第一部分内容" * 10 + "。" + "短尾巴。"
|
||
result = split_text(text, max_chars=200)
|
||
# 短尾巴应该被合并，不会单独成为一段
|
||
assert len(result) <= 2 # 可能1段或2段，但不会有3段
|
||
|
||
def test_very_short_segments_combined(self):
|
||
"""多个极短段会被合并."""
|
||
# 构造多个短句，都<50字
|
||
sentences = ["你好。", "我好。", "大家好。", "天气不错。", "一起玩吧。", "好的。"]
|
||
text = "".join(sentences)
|
||
result = split_text(text, max_chars=200)
|
||
# 总长度很短，应该合并成1段
|
||
assert len(result) == 1
|
||
|
||
|
||
class TestSplitTextEdgeCases:
|
||
"""边界情况."""
|
||
|
||
def test_single_character(self):
|
||
"""单字符."""
|
||
result = split_text("好", max_chars=10)
|
||
assert result == ["好"]
|
||
|
||
def test_only_punctuation(self):
|
||
"""纯标点符号."""
|
||
text = "。。。。。"
|
||
result = split_text(text, max_chars=10)
|
||
assert len(result) == 1
|
||
|
||
def test_custom_max_chars_small(self):
|
||
"""很小的max_chars."""
|
||
text = "一二三四五六七八九十。" * 5
|
||
result = split_text(text, max_chars=20)
|
||
assert len(result) > 1
|
||
for seg in result:
|
||
assert len(seg) <= 20
|
||
|
||
def test_mixed_chinese_english(self):
|
||
"""中英文混合."""
|
||
text = "今天天气很好。Today is a nice day. 我们出去玩吧！Let's go out and play." * 20
|
||
result = split_text(text, max_chars=150)
|
||
assert len(result) > 1
|
||
for seg in result:
|
||
assert len(seg) <= 150
|
||
|
||
def test_no_punctuation_long_text(self):
|
||
"""完全没有标点的长文本，只能硬切."""
|
||
text = "字" * 500
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) == 5
|
||
for seg in result:
|
||
assert len(seg) == 100
|
||
|
||
def test_strip_leading_trailing_whitespace(self):
|
||
"""首尾空白被去除."""
|
||
text = " 你好世界。 "
|
||
result = split_text(text, max_chars=100)
|
||
assert result == ["你好世界。"]
|
||
|
||
def test_total_length_preserved(self):
|
||
"""分段后总字符数大致等于原文（去除首尾空白后）."""
|
||
text = "这是一段测试文本。" * 30
|
||
result = split_text(text, max_chars=100)
|
||
joined = "".join(result)
|
||
# 因为strip的原因可能略有差异，但应该接近
|
||
assert len(joined) == len(text.strip())
|