Build lightweight AI agent admin

2026-06-08 18:14:59 +08:00
commit e164840f43
2530 changed files with 435693 additions and 0 deletions
@@ -0,0 +1,92 @@
+"""
+分块策略基类
+"""
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Dict, Any, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ChunkResult:
+    """分块结果"""
+    content: str
+    position: int = 0
+    char_count: int = 0
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if not self.char_count:
+            self.char_count = len(self.content)
+
+
+class BaseChunker(ABC):
+    """
+    分块策略基类
+    
+    所有分块策略必须继承此类并实现 chunk 方法
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 500,
+        chunk_overlap: int = 50,
+        separator: Optional[str] = None,
+    ):
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.separator = separator
+
+    @abstractmethod
+    def chunk(self, text: str, metadata: Dict[str, Any] = None) -> List[ChunkResult]:
+        """
+        将文本分块
+        
+        Args:
+            text: 原始文本
+            metadata: 文档元数据
+            
+        Returns:
+            分块结果列表
+        """
+        pass
+
+    def _clean_text(self, text: str) -> str:
+        """清理文本：去除多余空白"""
+        import re
+        # 合并连续空行为单个空行
+        text = re.sub(r'\n{3,}', '\n\n', text)
+        # 去除行尾空白
+        text = '\n'.join(line.rstrip() for line in text.split('\n'))
+        return text.strip()
+
+    def _merge_small_chunks(self, chunks: List[str], min_size: int = 50) -> List[str]:
+        """合并过小的分块"""
+        if not chunks:
+            return []
+
+        merged = []
+        buffer = ""
+
+        for chunk in chunks:
+            if not chunk.strip():
+                continue
+            if buffer and len(buffer) + len(chunk) <= self.chunk_size:
+                buffer = buffer + "\n" + chunk
+            elif buffer and len(buffer) < min_size:
+                buffer = buffer + "\n" + chunk
+            else:
+                if buffer:
+                    merged.append(buffer)
+                buffer = chunk
+
+        if buffer:
+            # 最后一个 buffer 如果太小，合并到前一个
+            if merged and len(buffer) < min_size:
+                merged[-1] = merged[-1] + "\n" + buffer
+            else:
+                merged.append(buffer)
+
+        return merged