Build lightweight AI agent admin

2026-06-08 18:14:59 +08:00
commit e164840f43
2530 changed files with 435693 additions and 0 deletions
@@ -0,0 +1,57 @@
+"""
+固定大小分块策略
+
+按固定字符数分割文本，最简单的分块方式
+"""
+import logging
+from typing import Dict, Any, List, Optional
+
+from .base import BaseChunker, ChunkResult
+
+logger = logging.getLogger(__name__)
+
+
+class FixedChunker(BaseChunker):
+    """
+    固定大小分块器
+    
+    按固定字符数分割文本，相邻分块之间有 overlap 重叠
+    """
+
+    def chunk(self, text: str, metadata: Dict[str, Any] = None) -> List[ChunkResult]:
+        """固定大小分块"""
+        if not text or not text.strip():
+            return []
+
+        text = self._clean_text(text)
+        metadata = metadata or {}
+
+        if len(text) <= self.chunk_size:
+            return [ChunkResult(
+                content=text,
+                position=0,
+                metadata={**metadata},
+            )]
+
+        chunks = []
+        start = 0
+        position = 0
+        step = self.chunk_size - self.chunk_overlap
+
+        while start < len(text):
+            end = min(start + self.chunk_size, len(text))
+            chunk_text = text[start:end].strip()
+
+            if chunk_text:
+                chunks.append(ChunkResult(
+                    content=chunk_text,
+                    position=position,
+                    metadata={**metadata},
+                ))
+                position += 1
+
+            start += step
+            if step <= 0:
+                break
+
+        return chunks