Codex
58 lines
1.4 KiB
Python
58 lines
1.4 KiB
Python
"""
|
|
固定大小分块策略
|
|
|
|
按固定字符数分割文本，最简单的分块方式
|
|
"""
|
|
import logging
|
|
from typing import Dict, Any, List, Optional
|
|
|
|
from .base import BaseChunker, ChunkResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class FixedChunker(BaseChunker):
|
|
"""
|
|
固定大小分块器
|
|
|
|
按固定字符数分割文本，相邻分块之间有 overlap 重叠
|
|
"""
|
|
|
|
def chunk(self, text: str, metadata: Dict[str, Any] = None) -> List[ChunkResult]:
|
|
"""固定大小分块"""
|
|
if not text or not text.strip():
|
|
return []
|
|
|
|
text = self._clean_text(text)
|
|
metadata = metadata or {}
|
|
|
|
if len(text) <= self.chunk_size:
|
|
return [ChunkResult(
|
|
content=text,
|
|
position=0,
|
|
metadata={**metadata},
|
|
)]
|
|
|
|
chunks = []
|
|
start = 0
|
|
position = 0
|
|
step = self.chunk_size - self.chunk_overlap
|
|
|
|
while start < len(text):
|
|
end = min(start + self.chunk_size, len(text))
|
|
chunk_text = text[start:end].strip()
|
|
|
|
if chunk_text:
|
|
chunks.append(ChunkResult(
|
|
content=chunk_text,
|
|
position=position,
|
|
metadata={**metadata},
|
|
))
|
|
position += 1
|
|
|
|
start += step
|
|
if step <= 0:
|
|
break
|
|
|
|
return chunks
|