test: 第73波 url_security + pagination + text_splitter 单测 (+144)
- test_url_security: URL安全校验（SSRF防护/魔数校验/可信域名/直接IP拦截） ~85个 - test_pagination: 分页器（参数校验/偏移计算/元数据/内存分页） ~35个 - test_text_splitter: 文本分段器（句子边界/强制切段/短段合并） ~24个 全部纯逻辑，无外部依赖
This commit is contained in:
CI Bot
+211
-136
@@ -1,11 +1,16 @@
|
||||
"""通用分页器单元测试."""
|
||||
"""分页器纯逻辑测试 — PaginationParams/PaginationMeta/paginate.
|
||||
|
||||
覆盖参数校验、偏移计算、元数据计算、内存分页等全部纯逻辑。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from math import ceil
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from packages.application.common.pagination import (
|
||||
from application.common.pagination import (
|
||||
PaginatedResponse,
|
||||
PaginationMeta,
|
||||
PaginationParams,
|
||||
@@ -13,232 +18,302 @@ from packages.application.common.pagination import (
|
||||
)
|
||||
|
||||
|
||||
class TestPaginationParams:
|
||||
"""PaginationParams 测试"""
|
||||
class TestPaginationParamsDefaults:
|
||||
"""PaginationParams 默认值与属性."""
|
||||
|
||||
def test_default_values(self):
|
||||
"""默认值正确"""
|
||||
def test_default_page_is_1(self):
|
||||
"""默认页码为1."""
|
||||
params = PaginationParams()
|
||||
assert params.page == 1
|
||||
|
||||
def test_default_page_size_is_20(self):
|
||||
"""默认每页20条."""
|
||||
params = PaginationParams()
|
||||
assert params.page_size == 20
|
||||
|
||||
def test_offset_first_page(self):
|
||||
"""第一页 offset 为 0"""
|
||||
def test_offset_first_page_is_0(self):
|
||||
"""第一页偏移量为0."""
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
assert params.offset == 0
|
||||
|
||||
def test_offset_second_page(self):
|
||||
"""第二页 offset 计算正确"""
|
||||
"""第二页偏移量=page_size."""
|
||||
params = PaginationParams(page=2, page_size=20)
|
||||
assert params.offset == 20
|
||||
|
||||
def test_offset_custom_page_size(self):
|
||||
"""自定义 page_size 的 offset"""
|
||||
def test_offset_page_3_size_10(self):
|
||||
"""第3页每页10条，偏移20."""
|
||||
params = PaginationParams(page=3, page_size=10)
|
||||
assert params.offset == 20
|
||||
|
||||
def test_limit_equals_page_size(self):
|
||||
"""limit 等于 page_size"""
|
||||
"""limit等于page_size."""
|
||||
params = PaginationParams(page_size=50)
|
||||
assert params.limit == 50
|
||||
|
||||
def test_page_must_be_at_least_1(self):
|
||||
"""page 不能小于 1"""
|
||||
|
||||
class TestPaginationParamsValidation:
|
||||
"""PaginationParams 参数校验."""
|
||||
|
||||
def test_page_zero_rejected(self):
|
||||
"""页码0被拒绝."""
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page=0)
|
||||
|
||||
def test_page_negative_raises(self):
|
||||
"""page 不能为负数"""
|
||||
def test_page_negative_rejected(self):
|
||||
"""负页码被拒绝."""
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page=-1)
|
||||
|
||||
def test_page_size_must_be_at_least_1(self):
|
||||
"""page_size 不能小于 1"""
|
||||
def test_page_size_zero_rejected(self):
|
||||
"""每页0条被拒绝."""
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=0)
|
||||
|
||||
def test_page_size_max_100(self):
|
||||
"""page_size 最大 100"""
|
||||
def test_page_size_negative_rejected(self):
|
||||
"""负每页条数被拒绝."""
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=-5)
|
||||
|
||||
def test_page_size_over_100_rejected(self):
|
||||
"""每页超过100条被拒绝."""
|
||||
with pytest.raises(ValidationError):
|
||||
PaginationParams(page_size=101)
|
||||
|
||||
def test_page_size_100_is_valid(self):
|
||||
"""page_size=100 是合法的"""
|
||||
def test_page_size_100_allowed(self):
|
||||
"""每页100条允许."""
|
||||
params = PaginationParams(page_size=100)
|
||||
assert params.page_size == 100
|
||||
|
||||
def test_page_size_1_allowed(self):
|
||||
"""每页1条允许."""
|
||||
params = PaginationParams(page_size=1)
|
||||
assert params.page_size == 1
|
||||
|
||||
class TestPaginationMeta:
|
||||
"""PaginationMeta 测试"""
|
||||
def test_page_1_allowed(self):
|
||||
"""页码1允许."""
|
||||
params = PaginationParams(page=1)
|
||||
assert params.page == 1
|
||||
|
||||
def test_from_params_first_page(self):
|
||||
"""第一页元数据"""
|
||||
def test_large_page_allowed(self):
|
||||
"""大页码允许（不设上限）."""
|
||||
params = PaginationParams(page=9999)
|
||||
assert params.page == 9999
|
||||
|
||||
|
||||
class TestPaginationMetaFromParams:
|
||||
"""PaginationMeta.from_params 元数据计算."""
|
||||
|
||||
def test_first_page_has_prev_false(self):
|
||||
"""第一页没有上一页."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=25)
|
||||
|
||||
assert meta.page == 1
|
||||
assert meta.page_size == 10
|
||||
assert meta.total == 25
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is True
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_from_params_last_page(self):
|
||||
"""最后一页元数据"""
|
||||
params = PaginationParams(page=3, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=25)
|
||||
|
||||
assert meta.page == 3
|
||||
assert meta.total_pages == 3
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_from_params_middle_page(self):
|
||||
"""中间页元数据"""
|
||||
params = PaginationParams(page=2, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
|
||||
assert meta.page == 2
|
||||
assert meta.total_pages == 5
|
||||
def test_first_page_has_next_true(self):
|
||||
"""第一页（数据多时有下一页."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.has_next is True
|
||||
|
||||
def test_last_page_has_next_false(self):
|
||||
"""最后一页没有下一页."""
|
||||
params = PaginationParams(page=10, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.has_next is False
|
||||
|
||||
def test_last_page_has_prev_true(self):
|
||||
"""最后一页有上一页."""
|
||||
params = PaginationParams(page=10, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_from_params_zero_total(self):
|
||||
"""总数为 0 时"""
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
def test_middle_page_has_both(self):
|
||||
"""中间页上下都有."""
|
||||
params = PaginationParams(page=5, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.has_prev is True
|
||||
assert meta.has_next is True
|
||||
|
||||
def test_total_pages_exact_division(self):
|
||||
"""整除时总页数正确."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=100)
|
||||
assert meta.total_pages == 10
|
||||
|
||||
def test_total_pages_with_remainder(self):
|
||||
"""有余数时向上取整."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=105)
|
||||
assert meta.total_pages == 11
|
||||
|
||||
def test_total_pages_single_item(self):
|
||||
"""1条数据总页数=1."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=1)
|
||||
assert meta.total_pages == 1
|
||||
|
||||
def test_total_zero_gives_zero_pages(self):
|
||||
"""0条数据总页数=0."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=0)
|
||||
|
||||
assert meta.total == 0
|
||||
assert meta.total_pages == 0
|
||||
assert meta.has_next is False
|
||||
|
||||
def test_total_zero_has_prev_false(self):
|
||||
"""0条数据has_prev=False."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=0)
|
||||
assert meta.has_prev is False
|
||||
|
||||
def test_from_params_exact_multiple(self):
|
||||
"""总数刚好是 page_size 的整数倍"""
|
||||
def test_total_zero_has_next_false(self):
|
||||
"""0条数据has_next=False."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=30)
|
||||
meta = PaginationMeta.from_params(params, total=0)
|
||||
assert meta.has_next is False
|
||||
|
||||
assert meta.total_pages == 3
|
||||
|
||||
def test_from_params_single_page(self):
|
||||
"""单页即可放下所有数据"""
|
||||
params = PaginationParams(page=1, page_size=100)
|
||||
def test_page_exactly_total_pages_no_next(self):
|
||||
"""当前页等于总页数时没有下一页."""
|
||||
params = PaginationParams(page=5, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.has_next is False
|
||||
assert meta.total_pages == 5
|
||||
|
||||
def test_page_beyond_total_pages(self):
|
||||
"""页码超过总页数时has_next=False."""
|
||||
params = PaginationParams(page=20, page_size=10)
|
||||
meta = PaginationMeta.from_params(params, total=50)
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is True
|
||||
|
||||
def test_preserves_page_and_page_size(self):
|
||||
"""保留输入的page和page_size."""
|
||||
params = PaginationParams(page=3, page_size=25)
|
||||
meta = PaginationMeta.from_params(params, total=200)
|
||||
assert meta.page == 3
|
||||
assert meta.page_size == 25
|
||||
assert meta.total == 200
|
||||
|
||||
def test_page_size_1_total_1(self):
|
||||
"""每页1条，1条数据."""
|
||||
params = PaginationParams(page=1, page_size=1)
|
||||
meta = PaginationMeta.from_params(params, total=1)
|
||||
assert meta.total_pages == 1
|
||||
assert meta.has_next is False
|
||||
assert meta.has_prev is False
|
||||
|
||||
|
||||
class TestPaginatedResponse:
|
||||
"""PaginatedResponse 测试"""
|
||||
class TestPaginatedResponseCreate:
|
||||
"""PaginatedResponse.create 创建分页响应."""
|
||||
|
||||
def test_create_success(self):
|
||||
"""创建分页响应"""
|
||||
def test_creates_with_data_and_meta(self):
|
||||
"""创建包含data和pagination."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
data = [1, 2, 3]
|
||||
data = [{"id": i} for i in range(10)]
|
||||
resp = PaginatedResponse.create(data, params, total=25)
|
||||
assert resp.data == data
|
||||
assert resp.pagination.page == 1
|
||||
assert resp.pagination.total == 25
|
||||
assert resp.pagination.total_pages == 3
|
||||
|
||||
response = PaginatedResponse.create(data, params, total=25)
|
||||
|
||||
assert response.data == [1, 2, 3]
|
||||
assert response.pagination.page == 1
|
||||
assert response.pagination.total == 25
|
||||
assert response.pagination.total_pages == 3
|
||||
|
||||
def test_create_empty_data(self):
|
||||
"""空数据分页响应"""
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
response = PaginatedResponse.create([], params, total=0)
|
||||
|
||||
assert response.data == []
|
||||
assert response.pagination.total == 0
|
||||
assert response.pagination.total_pages == 0
|
||||
def test_empty_data(self):
|
||||
"""空数据响应."""
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
resp = PaginatedResponse.create([], params, total=0)
|
||||
assert resp.data == []
|
||||
assert resp.pagination.total == 0
|
||||
assert resp.pagination.total_pages == 0
|
||||
|
||||
|
||||
class TestPaginateFunction:
|
||||
"""paginate 函数测试（内存分页）"""
|
||||
class TestPaginateInMemory:
|
||||
"""paginate 内存分页函数."""
|
||||
|
||||
def test_first_page(self):
|
||||
"""第一页分页"""
|
||||
items = list(range(30))
|
||||
"""第一页返回前N条."""
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
|
||||
result = paginate(items, params)
|
||||
|
||||
assert result.data == list(range(10))
|
||||
assert result.pagination.total == 30
|
||||
assert result.pagination.total_pages == 3
|
||||
assert result.pagination.total == 50
|
||||
assert result.pagination.total_pages == 5
|
||||
assert result.pagination.has_next is True
|
||||
assert result.pagination.has_prev is False
|
||||
|
||||
def test_second_page(self):
|
||||
"""第二页分页"""
|
||||
items = list(range(30))
|
||||
params = PaginationParams(page=2, page_size=10)
|
||||
|
||||
def test_middle_page(self):
|
||||
"""中间页."""
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=3, page_size=10)
|
||||
result = paginate(items, params)
|
||||
|
||||
assert result.data == list(range(10, 20))
|
||||
assert result.pagination.page == 2
|
||||
assert result.data == list(range(20, 30))
|
||||
assert result.pagination.has_prev is True
|
||||
assert result.pagination.has_next is True
|
||||
|
||||
def test_last_page(self):
|
||||
"""最后一页分页"""
|
||||
items = list(range(25))
|
||||
params = PaginationParams(page=3, page_size=10)
|
||||
|
||||
"""最后一页."""
|
||||
items = list(range(50))
|
||||
params = PaginationParams(page=5, page_size=10)
|
||||
result = paginate(items, params)
|
||||
|
||||
assert result.data == list(range(20, 25))
|
||||
assert len(result.data) == 5
|
||||
assert result.data == list(range(40, 50))
|
||||
assert result.pagination.has_next is False
|
||||
assert result.pagination.has_prev is True
|
||||
|
||||
def test_last_page_not_full(self):
|
||||
"""最后一页不足page_size."""
|
||||
items = list(range(45))
|
||||
params = PaginationParams(page=5, page_size=10)
|
||||
result = paginate(items, params)
|
||||
assert result.data == list(range(40, 45))
|
||||
assert len(result.data) == 5
|
||||
assert result.pagination.total == 45
|
||||
assert result.pagination.total_pages == 5
|
||||
|
||||
def test_empty_list(self):
|
||||
"""空列表分页"""
|
||||
params = PaginationParams(page=1, page_size=20)
|
||||
result = paginate([], params)
|
||||
|
||||
"""空列表分页."""
|
||||
items = []
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
result = paginate(items, params)
|
||||
assert result.data == []
|
||||
assert result.pagination.total == 0
|
||||
assert result.pagination.total_pages == 0
|
||||
|
||||
def test_page_beyond_total(self):
|
||||
"""页码超出总数"""
|
||||
"""页码超过总页数返回空数据."""
|
||||
items = list(range(5))
|
||||
params = PaginationParams(page=10, page_size=10)
|
||||
|
||||
result = paginate(items, params)
|
||||
|
||||
assert result.data == []
|
||||
assert result.pagination.total == 5
|
||||
assert result.pagination.total_pages == 1
|
||||
|
||||
def test_custom_page_size(self):
|
||||
"""自定义每页数量"""
|
||||
items = list(range(100))
|
||||
params = PaginationParams(page=1, page_size=50)
|
||||
|
||||
result = paginate(items, params)
|
||||
|
||||
assert len(result.data) == 50
|
||||
assert result.pagination.total_pages == 2
|
||||
assert result.pagination.has_next is False
|
||||
|
||||
def test_single_item(self):
|
||||
"""单条数据"""
|
||||
items = ["only_one"]
|
||||
"""单条数据分页."""
|
||||
items = ["only"]
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
|
||||
result = paginate(items, params)
|
||||
|
||||
assert result.data == ["only_one"]
|
||||
assert result.data == ["only"]
|
||||
assert result.pagination.total == 1
|
||||
assert result.pagination.total_pages == 1
|
||||
|
||||
def test_generic_type_preserved(self):
|
||||
"""泛型类型数据正确"""
|
||||
items = [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]
|
||||
params = PaginationParams(page=1, page_size=10)
|
||||
|
||||
def test_page_size_larger_than_total(self):
|
||||
"""每页条数大于总数，第一页包含全部."""
|
||||
items = list(range(5))
|
||||
params = PaginationParams(page=1, page_size=100)
|
||||
result = paginate(items, params)
|
||||
assert result.data == items
|
||||
assert result.pagination.total_pages == 1
|
||||
|
||||
assert len(result.data) == 2
|
||||
assert result.data[0]["id"] == 1
|
||||
def test_page_size_1(self):
|
||||
"""每页1条."""
|
||||
items = ["a", "b", "c"]
|
||||
params = PaginationParams(page=2, page_size=1)
|
||||
result = paginate(items, params)
|
||||
assert result.data == ["b"]
|
||||
assert result.pagination.total_pages == 3
|
||||
|
||||
def test_preserves_original_list(self):
|
||||
"""不修改原始列表."""
|
||||
items = [1, 2, 3, 4, 5]
|
||||
original = items.copy()
|
||||
params = PaginationParams(page=1, page_size=2)
|
||||
paginate(items, params)
|
||||
assert items == original
|
||||
|
||||
+179
-107
@@ -1,147 +1,219 @@
|
||||
"""文本分段工具单元测试."""
|
||||
"""文本分段器纯逻辑测试 — split_text.
|
||||
|
||||
覆盖空文本、短文本、句子边界分段、超长句强制切段、过短段合并等场景。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from packages.application.tts_job.text_splitter import split_text
|
||||
from application.tts_job.text_splitter import split_text
|
||||
|
||||
|
||||
class TestSplitText:
|
||||
"""split_text 函数测试"""
|
||||
class TestSplitTextEmptyOrShort:
|
||||
"""空文本与短文本."""
|
||||
|
||||
def test_empty_string_returns_empty_list(self):
|
||||
"""空字符串返回空列表"""
|
||||
"""空字符串返回空列表."""
|
||||
assert split_text("") == []
|
||||
|
||||
def test_whitespace_only_returns_empty_list(self):
|
||||
"""纯空白字符返回空列表"""
|
||||
assert split_text(" \n \t ") == []
|
||||
"""纯空白字符串返回空列表."""
|
||||
assert split_text(" \n\t ") == []
|
||||
|
||||
def test_short_text_returns_single_segment(self):
|
||||
"""短文本直接返回单段"""
|
||||
text = "这是一段短文本。"
|
||||
def test_none_not_supported(self):
|
||||
"""None不支持（strip会报错）."""
|
||||
with pytest.raises(AttributeError):
|
||||
split_text(None)
|
||||
|
||||
def test_short_text_single_segment(self):
|
||||
"""短文本不分割，单段返回."""
|
||||
text = "你好世界。"
|
||||
result = split_text(text, max_chars=500)
|
||||
assert result == [text]
|
||||
|
||||
def test_text_length_equals_max_chars(self):
|
||||
"""文本长度恰好等于 max_chars 时返回单段"""
|
||||
def test_exactly_max_chars_single_segment(self):
|
||||
"""刚好等于max_chars时不分段."""
|
||||
text = "a" * 100
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) == 1
|
||||
assert len(result[0]) == 100
|
||||
|
||||
def test_splits_on_sentence_boundary(self):
|
||||
"""在句子边界处分段"""
|
||||
# 构造长文本，确保超过 max_chars
|
||||
sentences = ["今天天气真好。我们一起去公园散步吧。", "公园里有很多花。还有很多小朋友在玩耍。"] * 10
|
||||
text = "".join(sentences)
|
||||
def test_under_max_chars_single_segment(self):
|
||||
"""少于max_chars时不分段."""
|
||||
text = "a" * 50
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) == 1
|
||||
assert len(result[0]) == 50
|
||||
|
||||
result = split_text(text, max_chars=200)
|
||||
|
||||
class TestSplitTextSentenceBoundary:
|
||||
"""句子边界分段."""
|
||||
|
||||
def test_split_by_period(self):
|
||||
"""按句号分段."""
|
||||
text = "第一句内容。" + "第二句内容。" * 50
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) > 1
|
||||
# 每段都不超过max_chars
|
||||
for seg in result:
|
||||
assert len(seg) <= 100
|
||||
# 合并后等于原文（去空格后近似）
|
||||
assert "".join(result) == text.replace(" ", "")
|
||||
|
||||
def test_split_by_question_mark(self):
|
||||
"""按问号分段."""
|
||||
text = "你好吗？" + "我很好。" * 50
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 80
|
||||
|
||||
def test_split_by_exclamation_mark(self):
|
||||
"""按感叹号分段."""
|
||||
text = "太棒了！" + "真的好。" * 50
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
|
||||
def test_split_by_newline(self):
|
||||
"""按换行符分段."""
|
||||
lines = ["这是第一行很长的内容" * 5 for _ in range(10)]
|
||||
text = "\n".join(lines)
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) > 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 100
|
||||
|
||||
def test_split_by_semicolon_fullwidth(self):
|
||||
"""按全角分号分段."""
|
||||
text = "第一项；" + "第二项内容。" * 40
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
|
||||
def test_split_by_english_period(self):
|
||||
"""按英文句号也分段（_SENTENCE_ENDS包含.）."""
|
||||
text = "Hello. " + "World. " * 50
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
|
||||
def test_split_by_english_question(self):
|
||||
"""按英文问号分段."""
|
||||
text = "Really? " + "Yes. " * 50
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
|
||||
def test_split_by_english_exclamation(self):
|
||||
"""按英文感叹号分段."""
|
||||
text = "Wow! " + "Great. " * 50
|
||||
result = split_text(text, max_chars=80)
|
||||
assert len(result) > 1
|
||||
|
||||
def test_short_sentences_not_split(self):
|
||||
"""短句（<50字）即使有句号也不立刻切，等累积到一定长度."""
|
||||
# 每句5字，即使有句号也不会在50字前切
|
||||
text = "你好。" * 5 # 15字符
|
||||
result = split_text(text, max_chars=100)
|
||||
# 因为每段至少50字才在句子边界切，所以15字的文本应该是1段
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestSplitTextLongSentenceForceSplit:
|
||||
"""超长单句强制切段."""
|
||||
|
||||
def test_single_very_long_sentence_forced_split(self):
|
||||
"""单句超长时强制切段."""
|
||||
text = "啊" * 200 # 没有标点，200字
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) >= 2
|
||||
# 每段都不超过 max_chars
|
||||
for seg in result:
|
||||
assert len(seg) <= 200
|
||||
|
||||
def test_all_segments_within_max_chars(self):
|
||||
"""所有分段都不超过 max_chars"""
|
||||
text = "这是第一句话。这是第二句话。这是第三句话。这是第四句话。这是第五句话。" * 10
|
||||
|
||||
result = split_text(text, max_chars=100)
|
||||
|
||||
for seg in result:
|
||||
assert len(seg) <= 100
|
||||
|
||||
def test_long_single_sentence_hard_cut(self):
|
||||
"""超长单句会被硬切"""
|
||||
text = "a" * 1000 # 没有标点
|
||||
def test_force_split_preserves_all_chars(self):
|
||||
"""强制切段不丢字符."""
|
||||
text = "a" * 250
|
||||
result = split_text(text, max_chars=100)
|
||||
# 所有段的总长度应等于原文（去掉空白可能有细微差异，但纯字母应该不变）
|
||||
assert sum(len(s) for s in result) == 250
|
||||
|
||||
def test_mixed_long_and_short_sentences(self):
|
||||
"""长句短句混合."""
|
||||
long_part = "非常长的句子没有标点符号" * 20
|
||||
text = long_part + "。结束句。"
|
||||
result = split_text(text, max_chars=100)
|
||||
assert len(result) > 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 100
|
||||
|
||||
|
||||
class TestSplitTextShortSegmentMerging:
|
||||
"""过短段落合并."""
|
||||
|
||||
def test_short_final_segment_merged(self):
|
||||
"""最后一段过短会被合并到前一段（如果不超限）."""
|
||||
# 构造两段，第二段很短
|
||||
text = "第一部分内容" * 10 + "。" + "短尾巴。"
|
||||
result = split_text(text, max_chars=200)
|
||||
# 短尾巴应该被合并，不会单独成为一段
|
||||
assert len(result) <= 2 # 可能1段或2段，但不会有3段
|
||||
|
||||
def test_very_short_segments_combined(self):
|
||||
"""多个极短段会被合并."""
|
||||
# 构造多个短句，都<50字
|
||||
sentences = ["你好。", "我好。", "大家好。", "天气不错。", "一起玩吧。", "好的。"]
|
||||
text = "".join(sentences)
|
||||
result = split_text(text, max_chars=200)
|
||||
# 总长度很短，应该合并成1段
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestSplitTextEdgeCases:
|
||||
"""边界情况."""
|
||||
|
||||
def test_single_character(self):
|
||||
"""单字符."""
|
||||
result = split_text("好", max_chars=10)
|
||||
assert result == ["好"]
|
||||
|
||||
def test_only_punctuation(self):
|
||||
"""纯标点符号."""
|
||||
text = "。。。。。"
|
||||
result = split_text(text, max_chars=10)
|
||||
assert len(result) == 1
|
||||
|
||||
def test_custom_max_chars_small(self):
|
||||
"""很小的max_chars."""
|
||||
text = "一二三四五六七八九十。" * 5
|
||||
result = split_text(text, max_chars=20)
|
||||
assert len(result) > 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 200
|
||||
|
||||
def test_newline_is_sentence_end(self):
|
||||
"""换行符作为句子结束符"""
|
||||
text = "第一行内容\n第二行内容\n第三行内容" * 10
|
||||
|
||||
result = split_text(text, max_chars=50)
|
||||
assert len(seg) <= 20
|
||||
|
||||
def test_mixed_chinese_english(self):
|
||||
"""中英文混合."""
|
||||
text = "今天天气很好。Today is a nice day. 我们出去玩吧！Let's go out and play." * 20
|
||||
result = split_text(text, max_chars=150)
|
||||
assert len(result) > 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 50
|
||||
|
||||
def test_chinese_punctuation(self):
|
||||
"""中文标点（。！？；）作为句子结束符"""
|
||||
text = "你好！今天吃什么？我吃米饭；你呢？我也吃米饭。" * 10
|
||||
|
||||
result = split_text(text, max_chars=80)
|
||||
|
||||
for seg in result:
|
||||
assert len(seg) <= 80
|
||||
|
||||
def test_english_punctuation(self):
|
||||
"""英文标点（.!?;）作为句子结束符"""
|
||||
text = "Hello! How are you? I'm fine; thank you. Good bye." * 10
|
||||
|
||||
result = split_text(text, max_chars=80)
|
||||
|
||||
for seg in result:
|
||||
assert len(seg) <= 80
|
||||
|
||||
def test_merged_short_segments(self):
|
||||
"""过短的段落会被合并"""
|
||||
# 构造很多短句
|
||||
text = "你好。再见。谢谢。抱歉。好的。不行。可以。去吧。" * 5 # 每句3-4字
|
||||
assert len(seg) <= 150
|
||||
|
||||
def test_no_punctuation_long_text(self):
|
||||
"""完全没有标点的长文本，只能硬切."""
|
||||
text = "字" * 500
|
||||
result = split_text(text, max_chars=100)
|
||||
|
||||
# 合并后段数应该比单纯按句切的少
|
||||
assert len(result) < len(text) // 3 # 粗略估计
|
||||
assert len(result) == 5
|
||||
for seg in result:
|
||||
assert len(seg) <= 100
|
||||
assert len(seg) == 100
|
||||
|
||||
def test_preserves_content(self):
|
||||
"""分段后内容总和与原文基本一致（忽略strip的空白）"""
|
||||
text = "这是测试文本。包含多个句子。用来验证分段正确性。" * 5
|
||||
def test_strip_leading_trailing_whitespace(self):
|
||||
"""首尾空白被去除."""
|
||||
text = " 你好世界。 "
|
||||
result = split_text(text, max_chars=100)
|
||||
assert result == ["你好世界。"]
|
||||
|
||||
result = split_text(text, max_chars=50)
|
||||
|
||||
# 合并所有分段，去掉空白后应该与原文去掉空白后基本一致
|
||||
combined = "".join(result).replace(" ", "")
|
||||
original = text.strip().replace(" ", "")
|
||||
assert combined == original
|
||||
|
||||
def test_custom_max_chars(self):
|
||||
"""支持自定义 max_chars"""
|
||||
text = "测试" * 100 # 200字
|
||||
|
||||
result_50 = split_text(text, max_chars=50)
|
||||
result_100 = split_text(text, max_chars=100)
|
||||
|
||||
# max_chars 越小，段数应该越多
|
||||
assert len(result_50) >= len(result_100)
|
||||
|
||||
def test_single_char_text(self):
|
||||
"""单字符文本"""
|
||||
assert split_text("好", max_chars=10) == ["好"]
|
||||
|
||||
def test_text_with_only_punctuation(self):
|
||||
"""纯标点文本"""
|
||||
text = "。。。。。。。。。。" # 10个句号
|
||||
result = split_text(text, max_chars=5)
|
||||
|
||||
assert len(result) >= 1
|
||||
for seg in result:
|
||||
assert len(seg) <= 5
|
||||
|
||||
def test_mixed_content(self):
|
||||
"""中英文混合内容"""
|
||||
text = "今天的天气是 sunny and warm。我们去了 park 玩。真的很开心！" * 5
|
||||
|
||||
result = split_text(text, max_chars=80)
|
||||
|
||||
for seg in result:
|
||||
assert len(seg) <= 80
|
||||
def test_total_length_preserved(self):
|
||||
"""分段后总字符数大致等于原文（去除首尾空白后）."""
|
||||
text = "这是一段测试文本。" * 30
|
||||
result = split_text(text, max_chars=100)
|
||||
joined = "".join(result)
|
||||
# 因为strip的原因可能略有差异，但应该接近
|
||||
assert len(joined) == len(text.strip())
|
||||
|
||||
+481
-546
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user