xiaoxia
de201436ea
CI/CD Pipeline / Check if frontend-only change (push) Has been cancelled
CI/CD Pipeline / Validate - Code Quality (push) Has been cancelled
CI/CD Pipeline / Validate - Type Check (mypy) (push) Has been cancelled
CI/CD Pipeline / Validate - Migration (alembic) (push) Has been cancelled
CI/CD Pipeline / Unit Tests (push) Has been cancelled
CI/CD Pipeline / Integration Tests (push) Has been cancelled
CI/CD Pipeline / Frontend Lint (push) Has been cancelled
CI/CD Pipeline / Frontend Unit Tests (push) Has been cancelled
CI/CD Pipeline / PR Build API Image (push) Has been cancelled
CI/CD Pipeline / PR Build Web Image (push) Has been cancelled
CI/CD Pipeline / PR Build Worker Image (push) Has been cancelled
CI/CD Pipeline / Build Staging API Image (push) Has been cancelled
CI/CD Pipeline / Build Staging Web Image (push) Has been cancelled
CI/CD Pipeline / Build Staging Worker Image (push) Has been cancelled
CI/CD Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Has been cancelled
CI/CD Pipeline / Staging E2E Tests (push) Has been cancelled
CI/CD Pipeline / Staging API Integration Tests (push) Has been cancelled
CI/CD Pipeline / Build Production API Image (push) Has been cancelled
CI/CD Pipeline / Build Production Web Image (push) Has been cancelled
CI/CD Pipeline / Build Production Worker Image (push) Has been cancelled
CI/CD Pipeline / Deploy Production (push) Has been cancelled
CI/CD Pipeline / Production Browser E2E (push) Has been cancelled
CI/CD Pipeline / ACR Image Cleanup (push) Has been cancelled
CI/CD Pipeline / Canary Release to Production (push) Has been cancelled
CI/CD Pipeline / CI Gate (push) Has been cancelled
101 lines
3.6 KiB
Python
Executable File
101 lines
3.6 KiB
Python
Executable File
"""text_splitter 单元测试."""
|
||
|
||
from packages.application.tts_job.text_splitter import split_text
|
||
|
||
|
||
class TestSplitText:
|
||
def test_empty_text_returns_empty(self):
|
||
assert split_text("") == []
|
||
|
||
def test_whitespace_only(self):
|
||
assert split_text(" \n\t ") == []
|
||
|
||
def test_short_text_single_segment(self):
|
||
text = "你好世界。"
|
||
result = split_text(text, max_chars=500)
|
||
assert len(result) == 1
|
||
assert result[0] == text
|
||
|
||
def test_exact_max_chars(self):
|
||
text = "a" * 500
|
||
result = split_text(text, max_chars=500)
|
||
assert len(result) == 1
|
||
assert len(result[0]) == 500
|
||
|
||
def test_splits_on_sentence_boundary(self):
|
||
# 两个长句子，各300字左右，超过50字阈值
|
||
sent1 = "你" * 300 + "。"
|
||
sent2 = "我" * 300 + "。"
|
||
text = sent1 + sent2
|
||
result = split_text(text, max_chars=500)
|
||
assert len(result) == 2
|
||
assert result[0] == sent1
|
||
assert result[1] == sent2
|
||
|
||
def test_long_sentence_hard_cut(self):
|
||
# 一个超长句子，没有句末标点，会被硬切
|
||
text = "长" * 800
|
||
result = split_text(text, max_chars=500)
|
||
assert len(result) >= 2
|
||
assert all(len(seg) <= 500 for seg in result)
|
||
# 合起来应该等于原文本
|
||
assert "".join(result) == text
|
||
|
||
def test_short_segments_merged(self):
|
||
# 多个短句应该被合并
|
||
sentences = [f"第{i}句。" for i in range(10)]
|
||
text = "".join(sentences)
|
||
result = split_text(text, max_chars=200)
|
||
# 每句5字左右，10句才50字，应该合并成1段
|
||
assert len(result) < 10
|
||
assert len(result[0]) <= 200
|
||
|
||
def test_preserves_content(self):
|
||
text = "今天天气真好。我们去公园玩吧！你觉得怎么样？好的，走吧。"
|
||
result = split_text(text, max_chars=20)
|
||
# 合并后内容应一致
|
||
assert "".join(result) == text
|
||
|
||
def test_multiple_punctuation_types(self):
|
||
# 构造足够长的文本触发分段
|
||
text = "第一" * 30 + "。" + "第二" * 30 + "！" + "第三" * 30 + "？" + "第四" * 30 + "；"
|
||
result = split_text(text, max_chars=100)
|
||
assert len(result) >= 2
|
||
assert "".join(result) == text
|
||
|
||
def test_custom_max_chars(self):
|
||
text = "a" * 100 + "。" + "b" * 100 + "。"
|
||
result = split_text(text, max_chars=150)
|
||
assert len(result) == 2
|
||
assert "a" in result[0]
|
||
assert "b" in result[1]
|
||
|
||
def test_newline_as_sentence_end(self):
|
||
text = "第一段\n第二段\n第三段"
|
||
result = split_text(text, max_chars=50)
|
||
assert len(result) >= 1
|
||
assert "".join(result) == text.strip()
|
||
|
||
def test_minimum_segment_length(self):
|
||
# 句子太短（<50字）不会立即分段
|
||
text = "短句一。短句二。短句三。"
|
||
result = split_text(text, max_chars=200)
|
||
assert len(result) == 1
|
||
|
||
def test_trailing_content_added(self):
|
||
# 最后一段不完整的句子也要加上
|
||
text = "完整的句子。剩余内容"
|
||
result = split_text(text, max_chars=50)
|
||
assert "".join(result) == text
|
||
|
||
def test_no_empty_segments(self):
|
||
text = "。。。。。" # 全是标点
|
||
result = split_text(text, max_chars=2)
|
||
assert all(len(seg) > 0 for seg in result)
|
||
|
||
def test_chinese_and_english_mixed(self):
|
||
text = "Hello世界。这是测试Test文本。Mixed混合。"
|
||
result = split_text(text, max_chars=20)
|
||
assert len(result) >= 2
|
||
assert "".join(result) == text
|