Build lightweight AI agent admin

2026-06-08 18:14:59 +08:00
commit e164840f43
2530 changed files with 435693 additions and 0 deletions
@@ -0,0 +1,45 @@
+"""
+文档分块策略
+"""
+from .base import BaseChunker, ChunkResult
+from .recursive import RecursiveChunker
+from .markdown import MarkdownChunker
+from .fixed import FixedChunker
+from .qa_chunker import QAChunker
+from .sentence import SentenceChunker
+from .auto import AutoChunker
+
+__all__ = [
+    'BaseChunker',
+    'ChunkResult',
+    'RecursiveChunker',
+    'MarkdownChunker',
+    'FixedChunker',
+    'QAChunker',
+    'SentenceChunker',
+    'AutoChunker',
+    'get_chunker',
+]
+
+
+def get_chunker(strategy: str, chunk_size: int = 500, chunk_overlap: int = 50, separator: str = None, **kwargs) -> BaseChunker:
+    """
+    根据策略名称获取分块器实例
+    
+    Args:
+        strategy: 分块策略名称（recursive/markdown/fixed/qa/sentence/auto）
+        chunk_size: 分块大小
+        chunk_overlap: 分块重叠
+        separator: 自定义分隔符
+        **kwargs: 额外参数（如 QAChunker 的 llm_caller）
+    """
+    chunkers = {
+        'recursive': RecursiveChunker,
+        'markdown': MarkdownChunker,
+        'fixed': FixedChunker,
+        'qa': QAChunker,
+        'sentence': SentenceChunker,
+        'auto': AutoChunker,
+    }
+    chunker_cls = chunkers.get(strategy, RecursiveChunker)
+    return chunker_cls(chunk_size=chunk_size, chunk_overlap=chunk_overlap, separator=separator, **kwargs)