Codex
46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
"""
|
||
文档分块策略
|
||
"""
|
||
from .base import BaseChunker, ChunkResult
|
||
from .recursive import RecursiveChunker
|
||
from .markdown import MarkdownChunker
|
||
from .fixed import FixedChunker
|
||
from .qa_chunker import QAChunker
|
||
from .sentence import SentenceChunker
|
||
from .auto import AutoChunker
|
||
|
||
__all__ = [
|
||
'BaseChunker',
|
||
'ChunkResult',
|
||
'RecursiveChunker',
|
||
'MarkdownChunker',
|
||
'FixedChunker',
|
||
'QAChunker',
|
||
'SentenceChunker',
|
||
'AutoChunker',
|
||
'get_chunker',
|
||
]
|
||
|
||
|
||
def get_chunker(strategy: str, chunk_size: int = 500, chunk_overlap: int = 50, separator: str = None, **kwargs) -> BaseChunker:
|
||
"""
|
||
根据策略名称获取分块器实例
|
||
|
||
Args:
|
||
strategy: 分块策略名称（recursive/markdown/fixed/qa/sentence/auto）
|
||
chunk_size: 分块大小
|
||
chunk_overlap: 分块重叠
|
||
separator: 自定义分隔符
|
||
**kwargs: 额外参数（如 QAChunker 的 llm_caller）
|
||
"""
|
||
chunkers = {
|
||
'recursive': RecursiveChunker,
|
||
'markdown': MarkdownChunker,
|
||
'fixed': FixedChunker,
|
||
'qa': QAChunker,
|
||
'sentence': SentenceChunker,
|
||
'auto': AutoChunker,
|
||
}
|
||
chunker_cls = chunkers.get(strategy, RecursiveChunker)
|
||
return chunker_cls(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separator=separator, **kwargs)
|