""" 固定大小分块策略 按固定字符数分割文本,最简单的分块方式 """ import logging from typing import Dict, Any, List, Optional from .base import BaseChunker, ChunkResult logger = logging.getLogger(__name__) class FixedChunker(BaseChunker): """ 固定大小分块器 按固定字符数分割文本,相邻分块之间有 overlap 重叠 """ def chunk(self, text: str, metadata: Dict[str, Any] = None) -> List[ChunkResult]: """固定大小分块""" if not text or not text.strip(): return [] text = self._clean_text(text) metadata = metadata or {} if len(text) <= self.chunk_size: return [ChunkResult( content=text, position=0, metadata={**metadata}, )] chunks = [] start = 0 position = 0 step = self.chunk_size - self.chunk_overlap while start < len(text): end = min(start + self.chunk_size, len(text)) chunk_text = text[start:end].strip() if chunk_text: chunks.append(ChunkResult( content=chunk_text, position=position, metadata={**metadata}, )) position += 1 start += step if step <= 0: break return chunks