#!/usr/bin/env python # -*- coding: utf-8 -*- """ 表单数据操作服务（异步版本） 支持动态操作不同数据表，适配 PostgreSQL、MySQL """ import asyncio import json import logging import uuid from contextlib import asynccontextmanager from datetime import date, datetime from decimal import Decimal from io import BytesIO from typing import Any, Dict, List, Optional, Set, Tuple from openpyxl import Workbook, load_workbook from openpyxl.styles import Font, PatternFill, Alignment, Border, Side from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession from online_dev.form_manager.model import FormMeta, FormSubTable from online_dev.form_data_manager.db_adapter import ( PLATFORM_SCHEMA, FormDataDbAdapter, create_form_data_adapter, create_platform_adapter, create_platform_sql_builder, resolve_form_sql_context, ) from online_dev.form_data_manager.dynamic_sql_builder import DynamicSQLBuilder from online_dev.form_data_manager.exceptions import ( FormDataException, FormNotFoundException, RecordNotFoundException, UniqueConstraintError, ColumnNotFoundError, FormDataValidationError, ForeignKeyConstraintError, ) from online_dev.form_data_manager.db_error_handler import ( translate_db_error, format_error_message, is_aborted_transaction_error, ) from core.database_manager.service import AsyncDatabaseManagerService from app.timezone import APP_TIMEZONE logger = logging.getLogger(__name__) def get_max_import_rows() -> int: """根据服务器物理内存计算最大导入/导出行数""" import psutil total_gb = psutil.virtual_memory().total / (1024 ** 3) thresholds = [(2, 100_000), (4, 200_000), (8, 600_000), (16, 1_048_575)] if total_gb <= 2: return 100_000 for i in range(len(thresholds) - 1): low_gb, low_rows = thresholds[i] high_gb, high_rows = thresholds[i + 1] if total_gb <= high_gb: ratio = (total_gb - low_gb) / (high_gb - low_gb) return int(low_rows + ratio * (high_rows - low_rows)) return 1_048_575 def get_server_memory_gb() -> float: """获取服务器物理内存（GB）""" import psutil return round(psutil.virtual_memory().total / (1024 ** 3), 1) MAX_IMPORT_EXPORT_ROWS = get_max_import_rows() SERVER_MEMORY_GB = get_server_memory_gb() logger.info(f"服务器内存: {SERVER_MEMORY_GB}GB, 导入导出行数上限: {MAX_IMPORT_EXPORT_ROWS}") class FormDataService: """ 表单数据操作服务（异步版本） 根据表单配置动态操作数据表，支持主表和子表的 CRUD 操作 """ def __init__( self, form_meta: FormMeta, sub_tables: List[FormSubTable], db_adapter: FormDataDbAdapter, ): """ 初始化服务 Args: form_meta: 表单元数据 sub_tables: 子表配置列表 db_adapter: 数据库执行适配器（平台库或第三方连接） """ self.form_meta = form_meta self.sub_tables = sub_tables self.db_adapter = db_adapter self.db_type = db_adapter.db_type self.db_config = db_adapter.db_config from online_dev.form_data_manager.db_adapter import create_sql_builder_for_adapter self.sql_builder = create_sql_builder_for_adapter(db_adapter) self._platform_adapter: Optional[FormDataDbAdapter] = None self._platform_sql_builder = create_platform_sql_builder() self._adapter_cache: Dict[str, FormDataDbAdapter] = {} self._sql_builder_cache: Dict[str, DynamicSQLBuilder] = {} # 缓存标量属性，避免 session 过期后懒加载导致 MissingGreenlet self._form_code = form_meta.code self._form_config = form_meta.form_config or {} self._list_config = form_meta.list_config or {} async def _resolve_schema(self, db: AsyncSession, schema: str) -> Optional[str]: """ 解析 schema 中的变量 如果 schema 包含 {{application_code}} 变量，则从表单关联的应用中获取 application_code Args: db: 数据库会话 schema: 原始 schema 配置（可能包含变量） Returns: 解析后的 schema 值 """ if not schema: return None # 检查是否包含变量语法 if '{{' in schema and '}}' in schema: # 提取变量名 import re match = re.search(r'\{\{([^}]+)\}\}', schema) if match: var_name = match.group(1).strip() # 如果是 application_code 变量，从应用中获取 if 'application_code' in var_name and self.form_meta.application_id: from core.application.model import Application result = await db.execute( select(Application.code).where(Application.id == self.form_meta.application_id) ) app_code = result.scalar_one_or_none() if app_code: # 替换变量 return schema.replace(f'{{{{{var_name}}}}}', app_code) return schema @classmethod async def create_service(cls, db: AsyncSession, form_code: str) -> "FormDataService": """ 工厂方法：创建服务实例 Args: db: 数据库会话 form_code: 表单编码 """ # 加载表单元数据 stmt = select(FormMeta).where( FormMeta.code == form_code, FormMeta.is_deleted == False ) result = await db.execute(stmt) form_meta = result.scalar_one_or_none() if not form_meta: raise FormNotFoundException(form_code) # 加载子表配置 sub_stmt = select(FormSubTable).where( FormSubTable.form_id == form_meta.id, FormSubTable.is_deleted == False ).order_by(FormSubTable.sort) sub_result = await db.execute(sub_stmt) sub_tables = list(sub_result.scalars().all()) db_config = (form_meta.db_config or "default").strip() or "default" db_adapter = await create_form_data_adapter(db_config, db) service = cls(form_meta, sub_tables, db_adapter) service._platform_adapter = await create_platform_adapter(db) service._adapter_cache[db_config] = db_adapter service._sql_builder_cache[db_config] = service.sql_builder return service async def _get_platform_adapter(self, db: AsyncSession) -> FormDataDbAdapter: if self._platform_adapter is None: self._platform_adapter = await create_platform_adapter(db) return self._platform_adapter async def _execute_platform_query( self, db: AsyncSession, sql: str, params: Any = None ) -> List[Dict[str, Any]]: adapter = await self._get_platform_adapter(db) return await adapter.execute_query(sql, params) async def _execute_platform_command( self, db: AsyncSession, sql: str, params: Any = None ) -> int: adapter = await self._get_platform_adapter(db) return await adapter.execute_command(sql, params) def _resolve_exec_database(self, database: Optional[str] = None) -> Optional[str]: """第三方 PostgreSQL 执行/事务时切换连接库（连接未配置 default 时用表单/子表 database）。""" if not self.db_adapter.is_external or self.db_adapter.db_type != "postgresql": return None explicit = (database or "").strip() if explicit: return explicit form_db = (self.form_meta.main_table_database or "").strip() if form_db: return form_db conn_default = (getattr(self.db_adapter, "default_database", "") or "").strip() return conn_default or None def _table_exec_database(self, table_database: Optional[str] = None) -> Optional[str]: """主表/子表操作对应的 PG 连接库名（子表 table_database 优先，否则主表）。""" effective = (table_database or "").strip() or ( self.form_meta.main_table_database or "" ).strip() return self._resolve_exec_database(effective or None) @asynccontextmanager async def _business_transaction(self, database: Optional[str] = None): """业务库事务：第三方走 handler 事务域，default 与平台 Session 一致。""" if self.db_adapter.is_external: exec_db = self._resolve_exec_database(database) async with self.db_adapter.transaction(database=exec_db): yield else: yield async def _create_savepoint(self, db: AsyncSession, name: str) -> None: if self.db_adapter.is_external: await self.db_adapter.create_savepoint(name) else: await db.execute(text(f"SAVEPOINT {name}")) async def _release_savepoint(self, db: AsyncSession, name: str) -> None: if self.db_adapter.is_external: await self.db_adapter.release_savepoint(name) else: await db.execute(text(f"RELEASE SAVEPOINT {name}")) async def _rollback_to_savepoint(self, db: AsyncSession, name: str) -> None: if self.db_adapter.is_external: await self.db_adapter.rollback_to_savepoint(name) else: await db.execute(text(f"ROLLBACK TO SAVEPOINT {name}")) # ============ 字段白名单 ============ def _get_allowed_fields(self, table_type: str = "main", table_name: str = None) -> Set[str]: """ 从表单配置中提取允许的字段（白名单） """ fields = set() form_config = self._form_config items = form_config.get("items", []) def traverse(item_list: List[Dict], in_sub_table: str = None): """递归遍历表单项""" for item in item_list: item_type = item.get("type", "") field = item.get("field", "") # 子表单组件 if item_type == "sub-table": sub_table_name = field children = item.get("children", []) traverse(children, sub_table_name) continue # 布局/展示组件，递归处理或跳过 if item_type in ("grid", "tabs", "collapse", "steps", "divider", "alert", "timeline", "text", "html", "spacer", "title", "button"): if item.get("columns"): for col in item["columns"]: traverse(col.get("children", []), in_sub_table) if item.get("items"): for sub_item in item["items"]: traverse(sub_item.get("children", []), in_sub_table) continue # 普通字段（排除虚拟字段，虚拟字段不对应数据库列） if field: props = item.get("props", {}) if props.get("isVirtualField"): continue if table_type == "main" and in_sub_table is None: fields.add(field) elif table_type == "sub" and in_sub_table == table_name: fields.add(field) traverse(items) # 始终允许 id 字段 fields.add("id") return fields def _filter_fields(self, data: Dict[str, Any], allowed_fields: Set[str]) -> Dict[str, Any]: """过滤字段，只保留白名单中的字段""" return {k: v for k, v in data.items() if k in allowed_fields} def _fill_system_fields_for_create(self, data: Dict[str, Any]) -> Dict[str, Any]: """ 填充新增时的系统字段 sys_create_datetime, sys_update_datetime, sys_creator_id, sys_modifier_id, sys_dept_id, sort """ from utils.context import get_current_user_info_from_context now = datetime.now() user_info = get_current_user_info_from_context() user_id = user_info.get('user_id') if user_info else None dept_id = user_info.get('dept_id') if user_info else None # 设置创建时间和更新时间 data['sys_create_datetime'] = now data['sys_update_datetime'] = now # 设置创建人和修改人 if user_id: data['sys_creator_id'] = user_id data['sys_modifier_id'] = user_id # 设置部门 if dept_id: data['sys_dept_id'] = dept_id # 设置排序字段默认值（如果表中有 sort 字段且未提供值） if 'sort' not in data or data.get('sort') is None: data['sort'] = 0 return data def _fill_system_fields_for_update(self, data: Dict[str, Any]) -> Dict[str, Any]: """ 填充更新时的系统字段 sys_update_datetime, sys_modifier_id """ from utils.context import get_current_user_info_from_context now = datetime.now() user_info = get_current_user_info_from_context() user_id = user_info.get('user_id') if user_info else None # 设置更新时间 data['sys_update_datetime'] = now # 设置修改人 if user_id: data['sys_modifier_id'] = user_id # 移除不应该被更新的系统字段 data.pop('sys_create_datetime', None) data.pop('sys_creator_id', None) data.pop('sys_dept_id', None) return data def _apply_data_scope_filter( self, filters: Dict[str, Any], data_scope: Dict[str, Any] ) -> Dict[str, Any]: """ 应用数据权限过滤条件 Args: filters: 现有过滤条件 data_scope: 数据权限配置，格式： { 'filter_type': 'all' | 'self' | 'dept' | 'dept_and_children' | 'custom', 'scope': 0-4, 'user_id': str | None, 'dept_id': str | None, 'dept_ids': List[str] | None } Returns: 合并后的过滤条件 """ if not filters: filters = {} filter_type = data_scope.get('filter_type', 'all') if filter_type == 'all': # 全部数据，不添加过滤条件 return filters elif filter_type == 'self': # 仅本人数据：需要表中有 sys_creator_id 字段 user_id = data_scope.get('user_id') if user_id: filters['sys_creator_id'] = user_id elif filter_type == 'dept': # 本部门数据：需要表中有 dept_id 字段 dept_id = data_scope.get('dept_id') if dept_id: filters['dept_id'] = dept_id elif filter_type == 'dept_and_children': # 本部门及下级：需要表中有 dept_id 字段 dept_ids = data_scope.get('dept_ids', []) if dept_ids: filters['dept_id'] = {"type": "in", "value": dept_ids} elif filter_type == 'custom': # 自定义部门 dept_ids = data_scope.get('dept_ids', []) if dept_ids: filters['dept_id'] = {"type": "in", "value": dept_ids} return filters async def _apply_data_scope_filter_safe( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], filters: Dict[str, Any], data_scope: Dict[str, Any] ) -> Dict[str, Any]: """ 安全地应用数据权限过滤（检查字段是否存在） 如果表中不存在所需的字段，则跳过该过滤条件 """ if not filters: filters = {} filter_type = data_scope.get('filter_type', 'all') if filter_type == 'all': # 全部数据，不添加过滤条件 return filters # 获取表的所有列 columns = await self._get_table_columns(db, table, schema, database) if filter_type == 'self': # 仅本人数据：需要表中有 sys_creator_id 字段 # 注意：字段为空时所有人可见（使用 eq_or_null） if 'sys_creator_id' in columns: user_id = data_scope.get('user_id') if user_id: filters['sys_creator_id'] = {"type": "eq_or_null", "value": user_id} else: logger.warning(f"表 {table} 中不存在 sys_creator_id 字段，跳过数据权限过滤") elif filter_type == 'dept': # 本部门数据：需要表中有 dept_id 或 sys_dept_id 字段 # 注意：字段为空时所有人可见（使用 eq_or_null） dept_field = 'sys_dept_id' if 'sys_dept_id' in columns else ('dept_id' if 'dept_id' in columns else None) if dept_field: dept_id = data_scope.get('dept_id') if dept_id: filters[dept_field] = {"type": "eq_or_null", "value": dept_id} else: logger.warning(f"表 {table} 中不存在 dept_id 或 sys_dept_id 字段，跳过数据权限过滤") elif filter_type == 'dept_and_children': # 本部门及下级：需要表中有 dept_id 或 sys_dept_id 字段 # 注意：字段为空时所有人可见（使用 in_or_null） dept_field = 'sys_dept_id' if 'sys_dept_id' in columns else ('dept_id' if 'dept_id' in columns else None) if dept_field: dept_ids = data_scope.get('dept_ids', []) if dept_ids: filters[dept_field] = {"type": "in_or_null", "value": dept_ids} else: # 如果没有部门ID，只返回字段为空的记录 filters[dept_field] = {"type": "null"} else: logger.warning(f"表 {table} 中不存在 dept_id 或 sys_dept_id 字段，跳过数据权限过滤") elif filter_type == 'custom': # 自定义部门 # 注意：字段为空时所有人可见（使用 in_or_null） dept_field = 'sys_dept_id' if 'sys_dept_id' in columns else ('dept_id' if 'dept_id' in columns else None) if dept_field: dept_ids = data_scope.get('dept_ids', []) if dept_ids: filters[dept_field] = {"type": "in_or_null", "value": dept_ids} else: # 如果没有部门ID，只返回字段为空的记录 filters[dept_field] = {"type": "null"} else: logger.warning(f"表 {table} 中不存在 dept_id 或 sys_dept_id 字段，跳过数据权限过滤") return filters # ============ 数据库操作 ============ async def _try_enrich_foreign_key_error( self, db: AsyncSession, exc: ForeignKeyConstraintError, ) -> ForeignKeyConstraintError: """在传入会话上解析引用表名；事务已中止时 rollback 后重试。""" referenced_table = exc.context.get("referenced_table") if not referenced_table: return exc display_name: Optional[str] = None try: display_name = await self._resolve_table_display_name(db, referenced_table) except Exception as first_err: if not is_aborted_transaction_error(first_err): logger.warning( "Failed to enrich foreign key error for table %s: %s", referenced_table, first_err, ) return exc try: await db.rollback() display_name = await self._resolve_table_display_name( db, referenced_table ) except Exception as enrich_err: logger.warning( "Failed to enrich foreign key error for table %s: %s", referenced_table, enrich_err, ) return exc if not display_name or display_name == referenced_table: return exc message = str(exc) if "无法删除" in message: message = ( f"无法删除，该数据已被「{display_name}」引用，" "请先删除或解除关联数据" ) elif referenced_table in message: message = message.replace(referenced_table, display_name) else: message = f"操作失败，引用的「{display_name}」数据不存在或无效" return ForeignKeyConstraintError( message=message, detail=exc.context.get("detail", ""), referenced_table=referenced_table, constraint_name=exc.context.get("constraint_name", ""), ) async def _resolve_table_display_name( self, db: AsyncSession, table_name: str, ) -> str: stmt = select(FormMeta.name).where( FormMeta.main_table == table_name, FormMeta.is_deleted == False, ).limit(1) result = await db.execute(stmt) form_name = result.scalar_one_or_none() if form_name: return form_name sub_stmt = ( select(FormMeta.name) .join(FormSubTable, FormSubTable.form_id == FormMeta.id) .where( FormSubTable.table_name == table_name, FormMeta.is_deleted == False, ) .limit(1) ) sub_result = await db.execute(sub_stmt) return sub_result.scalar_one_or_none() or table_name async def _execute_query( self, db: AsyncSession, sql: str, params: Any = None, *, database: Optional[str] = None, enrich_fk: bool = True, ) -> List[Dict[str, Any]]: """执行查询，自动将数据库异常转换为业务异常""" if params is None: params = {} try: exec_db = self._resolve_exec_database(database) return await self.db_adapter.execute_query( sql, params, database=exec_db ) except FormDataException: raise except Exception as e: biz_exc = translate_db_error(e) if enrich_fk and isinstance(biz_exc, ForeignKeyConstraintError): biz_exc = await self._try_enrich_foreign_key_error(db, biz_exc) raise biz_exc from e async def _execute_command( self, db: AsyncSession, sql: str, params: Any = None, *, database: Optional[str] = None, enrich_fk: bool = True, ) -> int: """执行命令，返回影响行数，自动将数据库异常转换为业务异常""" if params is None: params = {} try: self.db_adapter.ensure_write_allowed() exec_db = self._resolve_exec_database(database) return await self.db_adapter.execute_command( sql, params, database=exec_db ) except FormDataException: raise except Exception as e: biz_exc = translate_db_error(e) if enrich_fk and isinstance(biz_exc, ForeignKeyConstraintError): biz_exc = await self._try_enrich_foreign_key_error(db, biz_exc) raise biz_exc from e async def _get_table_columns( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], *, sql_builder: Optional[DynamicSQLBuilder] = None, db_adapter: Optional[FormDataDbAdapter] = None, ) -> List[str]: """获取表的所有列名""" sb = sql_builder or self.sql_builder adapter = db_adapter or self.db_adapter try: if sb.db_type == "postgresql": schema_name = schema or "public" sql = """ SELECT column_name FROM information_schema.columns WHERE table_schema = :schema AND table_name = :table """ params = {"schema": schema_name, "table": table} elif sb.db_type == "mysql": db_name = database or "information_schema" sql = """ SELECT column_name FROM information_schema.columns WHERE table_schema = :database AND table_name = :table """ params = {"database": db_name, "table": table} elif sb.db_type == "sqlserver": schema_name = schema or "dbo" sql = """ SELECT column_name FROM information_schema.columns WHERE table_schema = :schema AND table_name = :table """ params = {"schema": schema_name, "table": table} elif sb.db_type == "oracle": owner = (schema or "").upper() sql = """ SELECT column_name FROM all_tab_columns WHERE owner = :schema AND table_name = :table """ params = {"schema": owner, "table": table.upper()} else: return [] exec_db = None if adapter.is_external and adapter.db_type == "postgresql": exec_db = (database or "").strip() or self._resolve_exec_database() result = await adapter.execute_query(sql, params, database=exec_db) return [row["column_name"] for row in result] except Exception as e: logger.error(f"获取表列信息失败: {str(e)}") return [] async def _check_column_exists( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], column_name: str, *, sql_builder: Optional[DynamicSQLBuilder] = None, db_adapter: Optional[FormDataDbAdapter] = None, ) -> bool: """检查表中是否存在指定列""" columns = await self._get_table_columns( db, table, schema, database, sql_builder=sql_builder, db_adapter=db_adapter, ) return column_name in columns async def _validate_sort_field( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], sort_field: str ) -> Optional[str]: """验证排序字段是否存在于表中，如果不存在则返回 None""" try: # 构建表名 table_name = self.sql_builder.build_table_name(table, schema, database) # 获取表的列信息 columns = await self._get_table_columns(db, table, schema, database) if not columns: # 无法获取列信息，直接返回字段（不验证） return sort_field # 检查排序字段是否存在 if sort_field in columns: return sort_field logger.warning(f"排序字段 {sort_field} 不存在于表 {table_name} 中，可用字段: {columns}") return None except Exception as e: logger.error(f"验证排序字段失败: {str(e)}") # 验证失败时返回 None，使用默认排序 return None # ============ 查询操作 ============ async def list( self, db: AsyncSession, page: int = 1, page_size: int = 20, filters: Dict[str, Any] = None, sort_list: List[Dict[str, str]] = None, data_scope: Dict[str, Any] = None, search: str = None, search_fields: List[str] = None ) -> Dict[str, Any]: """分页查询列表（仅主表，支持数据权限过滤） Args: sort_list: 排序列表，格式为 [{"field": "name", "order": "asc"}, ...] data_scope: 数据权限过滤配置 search: 搜索关键词 search_fields: 搜索字段列表（多字段模糊搜索，OR 关系） """ import time _t0 = time.perf_counter() table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # 应用数据权限过滤（先检查字段是否存在） if data_scope: filters = await self._apply_data_scope_filter_safe(db, table, schema, database, filters, data_scope) _t1 = time.perf_counter() logger.info(f"[list 耗时] 数据权限过滤: {(_t1 - _t0) * 1000:.1f}ms") # 处理关联字段的名称搜索（转换为 ID 搜索） if filters: filters = await self._convert_relation_name_to_id(db, filters) # 转换日期字符串为 date 对象（PostgreSQL asyncpg 需要） filters = self._convert_date_strings(filters) _t2 = time.perf_counter() logger.info(f"[list 耗时] 关联字段转换+日期转换: {(_t2 - _t1) * 1000:.1f}ms") # 处理多字段搜索（OR 关系） if search and search_fields: if not filters: filters = {} # 使用特殊的 __search__ 键来标记多字段搜索 filters['__search__'] = { 'keyword': search, 'fields': search_fields } # 构建 ORDER BY（支持多字段排序） order_by = None if sort_list: order_clauses = [] for sort_item in sort_list: sort_field = sort_item.get('field') sort_order = sort_item.get('order', 'desc') if not sort_field: continue # 验证排序字段是否存在于表中 valid_sort_field = await self._validate_sort_field(db, table, schema, database, sort_field) if valid_sort_field: direction = "DESC" if sort_order.lower() == "desc" else "ASC" order_clauses.append(f"{self.sql_builder.quote_identifier(valid_sort_field)} {direction}") else: logger.warning(f"排序字段 {sort_field} 不存在于表 {table} 中，跳过") if order_clauses: order_by = ", ".join(order_clauses) _t3 = time.perf_counter() logger.info(f"[list 耗时] 排序字段验证: {(_t3 - _t2) * 1000:.1f}ms") # 查询总数 count_sql, count_params = self.sql_builder.build_count( table, schema, database, filters ) main_exec_db = self._table_exec_database(database) count_result = await self._execute_query( db, count_sql, count_params, database=main_exec_db ) total = count_result[0]["total"] if count_result else 0 _t4 = time.perf_counter() logger.info(f"[list 耗时] COUNT 查询 (total={total}): {(_t4 - _t3) * 1000:.1f}ms | SQL: {count_sql}") # 查询数据 offset = (page - 1) * page_size data_sql, data_params = self.sql_builder.build_select( table=table, schema=schema, database=database, where=filters, order_by=order_by, limit=page_size, offset=offset ) rows = await self._execute_query( db, data_sql, data_params, database=main_exec_db ) _t5 = time.perf_counter() logger.info(f"[list 耗时] 数据查询 (rows={len(rows)}): {(_t5 - _t4) * 1000:.1f}ms | SQL: {data_sql}") # 处理特殊类型 items = [self._serialize_row(row) for row in rows] _t6 = time.perf_counter() logger.info(f"[list 耗时] 序列化: {(_t6 - _t5) * 1000:.1f}ms") # 填充关联字段的显示名称 relation_fields = self._get_relation_fields() logger.debug(f"识别到的关联字段: {relation_fields}") if relation_fields and items: items = await self._fill_relation_display_names(db, items, relation_fields) _t7 = time.perf_counter() logger.info(f"[list 耗时] 填充关联字段显示名称 (fields={len(relation_fields)}): {(_t7 - _t6) * 1000:.1f}ms") # 填充虚拟字段的值（基于值关联配置） items = await self._fill_virtual_fields(db, items, relation_fields, context="list") _t8 = time.perf_counter() logger.info(f"[list 耗时] 填充虚拟字段: {(_t8 - _t7) * 1000:.1f}ms") # 应用字段权限过滤 items = await self.apply_field_permissions(items, db) _t9 = time.perf_counter() logger.info(f"[list 耗时] 字段权限过滤: {(_t9 - _t8) * 1000:.1f}ms") logger.info(f"[list 耗时] ===== 总耗时: {(_t9 - _t0) * 1000:.1f}ms (table={table}, page={page}, page_size={page_size}, total={total}) =====") return { "items": items, "total": total, "page": page, "page_size": page_size } async def list_cursor( self, db: AsyncSession, page_size: int = 20, cursor: Optional[str] = None, cursor_direction: str = "next", filters: Dict[str, Any] = None, sort_list: List[Dict[str, str]] = None, data_scope: Dict[str, Any] = None, search: str = None, search_fields: List[str] = None ) -> Dict[str, Any]: """游标分页查询列表（跳过 COUNT，使用 keyset pagination）""" import base64 import time _t0 = time.perf_counter() table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None if data_scope: filters = await self._apply_data_scope_filter_safe(db, table, schema, database, filters, data_scope) if filters: filters = await self._convert_relation_name_to_id(db, filters) filters = self._convert_date_strings(filters) if search and search_fields: if not filters: filters = {} filters['__search__'] = { 'keyword': search, 'fields': search_fields } # 构建排序字段列表 [(field, "ASC"|"DESC"), ...] order_fields: List[Tuple[str, str]] = [] if sort_list: for sort_item in sort_list: sort_field = sort_item.get('field') sort_order = sort_item.get('order', 'desc') if not sort_field: continue valid_sort_field = await self._validate_sort_field(db, table, schema, database, sort_field) if valid_sort_field: direction = "DESC" if sort_order.lower() == "desc" else "ASC" order_fields.append((valid_sort_field, direction)) # 确保 id 作为 tie-breaker 保证排序唯一性 id_already = any(f == "id" for f, _ in order_fields) if not id_already: last_dir = order_fields[-1][1] if order_fields else "DESC" order_fields.append(("id", last_dir)) if not order_fields: order_fields = [("id", "DESC")] # 解码游标 cursor_values = None if cursor: try: padding = 4 - len(cursor) % 4 if padding != 4: cursor += "=" * padding cursor_json = base64.urlsafe_b64decode(cursor).decode("utf-8") cursor_data = json.loads(cursor_json) cursor_values = cursor_data.get("v", {}) # asyncpg 要求 datetime/date 列传入原生对象而非字符串 if cursor_values: for k, v in cursor_values.items(): if not isinstance(v, str): continue try: parsed = datetime.fromisoformat(v) cursor_values[k] = parsed if "T" in v or " " in v else parsed.date() except (ValueError, TypeError): pass except Exception as e: logger.warning(f"游标解码失败: {e}, 忽略游标从头查询") cursor_values = None _t1 = time.perf_counter() data_sql, data_params = self.sql_builder.build_cursor_select( table=table, schema=schema, database=database, where=filters, order_fields=order_fields, cursor_values=cursor_values, direction=cursor_direction, limit=page_size ) rows = await self._execute_query(db, data_sql, data_params) _t2 = time.perf_counter() logger.info(f"[list_cursor 耗时] 数据查询 (rows={len(rows)}): {(_t2 - _t1) * 1000:.1f}ms | SQL: {data_sql}") # 判断 has_more：查询了 page_size + 1 条 has_more = len(rows) > page_size if has_more: rows = rows[:page_size] # prev 方向查询时结果是反序的，需要翻转回来 if cursor_direction == "prev": rows = list(reversed(rows)) items = [self._serialize_row(row) for row in rows] relation_fields = self._get_relation_fields() if relation_fields and items: items = await self._fill_relation_display_names(db, items, relation_fields) items = await self._fill_virtual_fields(db, items, relation_fields, context="list") items = await self.apply_field_permissions(items, db) # 编码游标 def _encode_cursor(row_data: Dict[str, Any]) -> Optional[str]: if not row_data: return None v = {} for field, _ in order_fields: val = row_data.get(field) if isinstance(val, (datetime, date)): val = val.isoformat() elif isinstance(val, Decimal): val = float(val) v[field] = val cursor_obj = json.dumps({"v": v}, ensure_ascii=False, default=str) return base64.urlsafe_b64encode(cursor_obj.encode("utf-8")).decode("utf-8").rstrip("=") next_cursor = None prev_cursor = None if items: # 用原始行数据（序列化前）生成游标，确保值精确 last_raw = rows[-1] if rows else None first_raw = rows[0] if rows else None if cursor_direction == "prev": last_raw = rows[0] if rows else None first_raw = rows[-1] if rows else None if has_more: if cursor_direction == "next": next_cursor = _encode_cursor(dict(last_raw) if last_raw else None) else: prev_cursor = _encode_cursor(dict(first_raw) if first_raw else None) # 始终提供反方向游标（如果有游标传入，说明可以往回翻） if cursor: if cursor_direction == "next": prev_cursor = _encode_cursor(dict(rows[0]) if rows else None) if has_more: next_cursor = _encode_cursor(dict(rows[-1]) if rows else None) else: next_cursor = None else: next_cursor = _encode_cursor(dict(rows[-1]) if rows else None) if has_more: prev_cursor = _encode_cursor(dict(rows[0]) if rows else None) else: prev_cursor = None else: # 第一页：没有 prev_cursor prev_cursor = None if has_more: next_cursor = _encode_cursor(dict(rows[-1]) if rows else None) _t3 = time.perf_counter() logger.info(f"[list_cursor 耗时] ===== 总耗时: {(_t3 - _t0) * 1000:.1f}ms (table={table}, page_size={page_size}, rows={len(items)}) =====") return { "items": items, "has_more": has_more if cursor_direction == "next" else (cursor is not None), "next_cursor": next_cursor, "prev_cursor": prev_cursor, "page_size": page_size } async def get_tree_children( self, db: AsyncSession, parent_id: str = None, parent_field: str = "parent_id", data_scope: Dict[str, Any] = None ) -> List[Dict[str, Any]]: """获取树形数据的子节点（用于懒加载模式） Args: parent_id: 父节点ID，为空时获取根节点 parent_field: 父节点字段名，默认为 parent_id data_scope: 数据权限过滤配置 Returns: 子节点列表，每个节点包含 has_children 标记 """ table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # 验证 parent_field 是否存在 valid_parent_field = await self._validate_sort_field(db, table, schema, database, parent_field) if not valid_parent_field: raise ColumnNotFoundError(column=parent_field, table=table) # 构建过滤条件 filters = {} if parent_id: # 获取指定父节点的子节点 filters[parent_field] = {"type": "eq", "value": parent_id} else: # 获取根节点（parent_field 为空或 NULL） filters[parent_field] = {"type": "null"} # 应用数据权限过滤 if data_scope: filters = await self._apply_data_scope_filter_safe(db, table, schema, database, filters, data_scope) # 查询数据（按 sort 字段排序） order_by = f"{self.sql_builder.quote_identifier('sort')} ASC" data_sql, data_params = self.sql_builder.build_select( table=table, schema=schema, database=database, where=filters, order_by=order_by ) rows = await self._execute_query(db, data_sql, data_params) items = [self._serialize_row(row) for row in rows] # 填充关联字段的显示名称 relation_fields = self._get_relation_fields() if relation_fields and items: items = await self._fill_relation_display_names(db, items, relation_fields) # 为每个节点检查是否有子节点 if items: table_name = self.sql_builder.build_table_name(table, schema, database) quoted_parent_field = self.sql_builder.quote_identifier(parent_field) # 检查表是否有 is_deleted 字段 has_is_deleted = await self._check_column_exists(db, table, schema, database, "is_deleted") # 批量查询所有节点的子节点数量 ids = [item.get('id') for item in items if item.get('id')] if ids: # 构建 IN 查询 placeholders = ", ".join([f":id_{i}" for i in range(len(ids))]) where_clause = f"{quoted_parent_field} IN ({placeholders})" if has_is_deleted: where_clause += self.sql_builder.is_deleted_and_clause() count_sql = f""" SELECT {quoted_parent_field} as parent_id, COUNT(*) as child_count FROM {table_name} WHERE {where_clause} GROUP BY {quoted_parent_field} """ count_params = {f"id_{i}": id_val for i, id_val in enumerate(ids)} try: count_result = await self._execute_query(db, count_sql, count_params) # 构建父ID到子节点数量的映射 child_count_map = {row['parent_id']: row['child_count'] for row in count_result} # 为每个节点设置 has_children 标记 for item in items: item_id = item.get('id') item['has_children'] = child_count_map.get(item_id, 0) > 0 except Exception as e: logger.warning(f"查询子节点数量失败: {e}") # 如果查询失败，默认设置为 False for item in items: item['has_children'] = False return items async def get_field_values( self, db: AsyncSession, field_name: str, page: int = 1, page_size: int = 20, search: str = None ) -> Dict[str, Any]: """ 获取指定字段的唯一值列表（用于过滤选项） Args: db: 数据库会话 field_name: 字段名 page: 页码 page_size: 每页数量 search: 搜索关键词（模糊匹配） Returns: { "items": [{"value": "值", "label": "显示文本", "count": 数量}], "total": 总数, "hasMore": 是否有更多 } """ table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # 验证字段是否存在 valid_field = await self._validate_sort_field(db, table, schema, database, field_name) if not valid_field: raise ColumnNotFoundError(column=field_name, table=table) # 构建表名 table_name = self.sql_builder.build_table_name(table, schema, database) quoted_field = self.sql_builder.quote_identifier(field_name) # 构建 WHERE 条件（只检查 NOT NULL，不检查空字符串，因为非字符串类型会报错） where_clause = f"{quoted_field} IS NOT NULL" params = {} if search: if self.sql_builder.db_type == "postgresql": where_clause += f" AND CAST({quoted_field} AS TEXT) ILIKE :search" else: where_clause += ( f" AND LOWER(CAST({quoted_field} AS VARCHAR(4000))) " f"LIKE LOWER(:search)" ) params["search"] = f"%{search}%" # 检查表是否有 is_deleted 字段 has_is_deleted = await self._check_column_exists(db, table, schema, database, "is_deleted") if has_is_deleted: where_clause += self.sql_builder.is_deleted_and_clause() # 查询总数 count_sql = f""" SELECT COUNT(DISTINCT {quoted_field}) as total FROM {table_name} WHERE {where_clause} """ count_result = await self._execute_query(db, count_sql, params) total = count_result[0]["total"] if count_result else 0 # 查询唯一值（带分页） offset = (page - 1) * page_size data_sql = f""" SELECT {quoted_field} as value, COUNT(*) as count FROM {table_name} WHERE {where_clause} GROUP BY {quoted_field} ORDER BY count DESC, {quoted_field} ASC """ data_sql = self.sql_builder._append_limit_offset(data_sql, page_size, offset) rows = await self._execute_query(db, data_sql, params) # 处理结果，获取显示标签 items = [] field_options = self._get_field_options(field_name) for row in rows: value = row["value"] # 序列化特殊类型 if isinstance(value, (datetime, date)): value = value.isoformat() elif isinstance(value, Decimal): value = float(value) elif isinstance(value, uuid.UUID): value = str(value) # 获取显示标签 label = str(value) if field_options: for opt in field_options: if str(opt.get("value")) == str(value): label = opt.get("label", str(value)) break items.append({ "value": value, "label": label, "count": row["count"] }) return { "items": items, "total": total, "hasMore": (page * page_size) < total } def _get_unique_check_fields(self) -> List[str]: """ 从表单配置中提取启用了唯一性校验的主表字段名列表 支持两种配置方式： 1. 表单设计器中的 props.uniqueCheck（字段级别） 2. 数据库设计中的 tableConfigs[].fields[].uniqueCheck（表级别） """ unique_fields = set() form_config = self._form_config # 方式1：从表单设计器的 items 中获取 items = form_config.get("items", []) def traverse(item_list: List[Dict], in_sub_table: str = None): for item in item_list: item_type = item.get("type", "") field = item.get("field", "") if item_type == "sub-table": traverse(item.get("children", []), field) continue if item_type in ("grid", "tabs", "collapse", "steps", "divider", "alert", "timeline", "text", "html", "spacer", "title", "button"): if item.get("columns"): for col in item["columns"]: traverse(col.get("children", []), in_sub_table) if item.get("items"): for sub_item in item["items"]: traverse(sub_item.get("children", []), in_sub_table) continue if field and in_sub_table is None: props = item.get("props", {}) if props.get("uniqueCheck"): unique_fields.add(field) traverse(items) # 方式2：从数据库设计的 tableConfigs 中获取（主表字段） table_configs = form_config.get("tableConfigs", []) for table_config in table_configs: if table_config.get("type") == "main": for field in table_config.get("fields", []): if field.get("uniqueCheck"): unique_fields.add(field.get("name")) return list(unique_fields) async def _validate_unique_fields( self, db: AsyncSession, data: Dict[str, Any], exclude_id: str = None ) -> None: """ 批量校验数据中启用了唯一性校验的字段，不通过则抛出 FormDataException Args: db: 数据库会话 data: 待写入的主表数据 exclude_id: 编辑时排除的记录ID """ unique_fields = self._get_unique_check_fields() if not unique_fields: return duplicate_fields = [] for field_name in unique_fields: value = data.get(field_name) if value is None or value == '': continue is_unique = await self.check_unique(db, field_name, str(value), exclude_id) if not is_unique: duplicate_fields.append(field_name) if duplicate_fields: field_labels = self._get_field_labels(duplicate_fields) messages = [f"'{field_labels.get(f, f)}'" for f in duplicate_fields] raise UniqueConstraintError( f"以下字段的值已存在: {', '.join(messages)}", fields=duplicate_fields, ) def _get_field_labels(self, field_names: List[str]) -> Dict[str, str]: """获取字段名到标签的映射""" labels = {} form_config = self._form_config items = form_config.get("items", []) def traverse(item_list: List[Dict]): for item in item_list: field = item.get("field", "") label = item.get("label", "") if field and label and field in field_names: labels[field] = label if item.get("columns"): for col in item["columns"]: traverse(col.get("children", [])) if item.get("items"): for sub_item in item["items"]: traverse(sub_item.get("children", [])) if item.get("children"): traverse(item["children"]) traverse(items) return labels async def check_unique( self, db: AsyncSession, field_name: str, value: str, exclude_id: str = None ) -> bool: """ 检查字段值在主表中是否唯一 Args: db: 数据库会话 field_name: 字段名 value: 字段值 exclude_id: 排除的记录ID（编辑时排除自身） Returns: True 表示唯一（可用），False 表示已存在 """ table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # 验证字段是否存在 valid_field = await self._validate_sort_field(db, table, schema, database, field_name) if not valid_field: raise ColumnNotFoundError(column=field_name, table=table) # 构建查询 table_name = self.sql_builder.build_table_name(table, schema, database) quoted_field = self.sql_builder.quote_identifier(field_name) where_clause = f"{quoted_field} = :value" params: Dict[str, Any] = {"value": value} # 编辑时排除自身 if exclude_id: quoted_id = self.sql_builder.quote_identifier("id") where_clause += f" AND {quoted_id} != :exclude_id" params["exclude_id"] = exclude_id # 检查 is_deleted 字段 has_is_deleted = await self._check_column_exists(db, table, schema, database, "is_deleted") if has_is_deleted: where_clause += self.sql_builder.is_deleted_and_clause() sql = f""" SELECT COUNT(*) as cnt FROM {table_name} WHERE {where_clause} """ result = await self._execute_query(db, sql, params) count = result[0]["cnt"] if result else 0 return count == 0 def _get_field_options(self, field_name: str) -> Optional[List[Dict]]: """从表单配置中获取字段的选项列表""" form_config = self._form_config items = form_config.get("items", []) def find_field(item_list: List[Dict]) -> Optional[List[Dict]]: for item in item_list: if item.get("field") == field_name: return item.get("options") # 递归查找子项 for key in ["children", "columns", "items"]: if key in item: sub_items = item[key] if isinstance(sub_items, list): if key == "columns": for col in sub_items: if "children" in col: result = find_field(col["children"]) if result is not None: return result else: result = find_field(sub_items) if result is not None: return result return None return find_field(items) async def get(self, db: AsyncSession, pk: Any) -> Dict[str, Any]: """获取单条数据（含子表）""" table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # 查询主表 sql, params = self.sql_builder.build_select( table=table, schema=schema, database=database, where={"id": pk} ) rows = await self._execute_query(db, sql, params) if not rows: raise RecordNotFoundException(pk, table=table) result = self._serialize_row(rows[0]) # 查询子表数据 result["sub_tables"] = {} for sub_table in self.sub_tables: sub_data = await self._query_sub_table_data(db, sub_table, pk) result["sub_tables"][sub_table.table_name] = sub_data # 填充关联字段的显示名称和虚拟字段 relation_fields = self._get_relation_fields() all_relation_fields = self._get_all_selector_relation_fields(relation_fields) if all_relation_fields: await self._fill_relation_display_names(db, [result], all_relation_fields) filled = await self._fill_virtual_fields(db, [result], relation_fields, context="form") if filled: result = filled[0] # 应用字段权限过滤 result = await self.apply_field_permissions(result, db) return result async def _query_sub_table_data( self, db: AsyncSession, sub_table: FormSubTable, main_pk: Any ) -> List[Dict[str, Any]]: """查询子表数据""" sql, params = self.sql_builder.build_select( table=sub_table.table_name, schema=sub_table.table_schema or None, database=sub_table.table_database or None, where={sub_table.foreign_key: main_pk} ) sub_db = sub_table.table_database or None rows = await self._execute_query( db, sql, params, database=self._table_exec_database(sub_db) ) return [self._serialize_row(row) for row in rows] def _is_uuid_like(self, value: str) -> bool: """检查值是否像 UUID（用于判断是 ID 还是名称）""" if not isinstance(value, str): return False # UUID 格式: 8-4-4-4-12 或 32位无连字符 import re uuid_pattern = re.compile( r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$|^[0-9a-f]{32}$', re.IGNORECASE ) return bool(uuid_pattern.match(value)) def _is_date_string(self, value: str) -> bool: """检查值是否为日期字符串格式 YYYY-MM-DD""" if not isinstance(value, str): return False import re return bool(re.match(r'^\d{4}-\d{2}-\d{2}$', value)) def _is_datetime_string(self, value: str) -> bool: """检查值是否为日期时间字符串格式 YYYY-MM-DD HH:MM:SS 或 ISO 格式""" if not isinstance(value, str): return False import re # 支持多种格式：YYYY-MM-DD HH:MM:SS, YYYY-MM-DDTHH:MM:SS, 带毫秒和时区 return bool(re.match(r'^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}', value)) def _parse_datetime_string(self, value: str) -> Optional[datetime]: """解析日期时间字符串为 datetime 对象""" if not isinstance(value, str): return None # 尝试多种格式 formats = [ "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%d", ] # 处理带时区的格式 clean_value = value if value.endswith('Z'): clean_value = value[:-1] elif '+' in value[-6:] or (value.count('-') > 2 and ':' in value[-6:]): # 移除时区信息 clean_value = value.rsplit('+', 1)[0].rsplit('-', 1)[0] if '+' in value[-6:] else value for fmt in formats: try: return datetime.strptime(clean_value, fmt) except ValueError: continue return None @staticmethod def _convert_bool_string(value: str): """将字符串 'true'/'false' 转换为 Python bool，非布尔字符串返回 None""" if value.lower() == 'true': return True if value.lower() == 'false': return False return None def _convert_date_strings(self, filters: Dict[str, Any]) -> Dict[str, Any]: """将过滤器中的日期/日期时间/布尔字符串转换为原生类型（PostgreSQL asyncpg 需要）""" if not filters: return filters converted = {} for field, value in filters.items(): if isinstance(value, dict): filter_type = value.get("type") filter_value = value.get("value") # 处理布尔字符串 if filter_type in ("eq",) and isinstance(filter_value, str): bool_val = self._convert_bool_string(filter_value) if bool_val is not None: converted[field] = {"type": filter_type, "value": bool_val} continue # 处理 gte, lte, gt, lt, eq 等类型的日期/日期时间值 # 注意：如果 case_sensitive 明确设为 false，说明是文本比较，不转换日期 case_sensitive = value.get("case_sensitive", True) if filter_type in ("gte", "lte", "gt", "lt", "eq") and isinstance(filter_value, str) and case_sensitive: if self._is_datetime_string(filter_value): # 优先尝试解析为 datetime parsed = self._parse_datetime_string(filter_value) if parsed: converted[field] = {"type": filter_type, "value": parsed} else: converted[field] = value elif self._is_date_string(filter_value): # 对于 datetime 字段使用日期格式查询时，需要特殊处理 # lte/lt: 使用当天的 23:59:59.999999（包含当天所有时间） # gte/gt: 使用当天的 00:00:00 try: parsed_date = date.fromisoformat(filter_value) if filter_type == "lte": # 小于等于：转换为当天结束时间 end_of_day = datetime.combine(parsed_date, datetime.max.time()) converted[field] = {"type": filter_type, "value": end_of_day} elif filter_type == "lt": # 小于：转换为当天开始时间（不包含当天） start_of_day = datetime.combine(parsed_date, datetime.min.time()) converted[field] = {"type": filter_type, "value": start_of_day} elif filter_type == "gte": # 大于等于：转换为当天开始时间 start_of_day = datetime.combine(parsed_date, datetime.min.time()) converted[field] = {"type": filter_type, "value": start_of_day} elif filter_type == "gt": # 大于：转换为当天结束时间（不包含当天） end_of_day = datetime.combine(parsed_date, datetime.max.time()) converted[field] = {"type": filter_type, "value": end_of_day} elif filter_type == "eq": # 精确匹配日期：转换为当天范围查询（00:00:00 - 23:59:59） # 这样 2026-01-14 可以匹配 2026-01-14 16:16:09 start_of_day = datetime.combine(parsed_date, datetime.min.time()) end_of_day = datetime.combine(parsed_date, datetime.max.time()) converted[field] = {"type": "range", "value": [start_of_day, end_of_day]} else: converted[field] = {"type": filter_type, "value": parsed_date} except ValueError: converted[field] = value else: converted[field] = value # 处理 in 类型的日期值列表 elif filter_type == "in" and isinstance(filter_value, list): converted_values = [] for v in filter_value: if isinstance(v, str): if self._is_datetime_string(v): parsed = self._parse_datetime_string(v) converted_values.append(parsed if parsed else v) elif self._is_date_string(v): try: converted_values.append(date.fromisoformat(v)) except ValueError: converted_values.append(v) else: converted_values.append(v) else: converted_values.append(v) converted[field] = {"type": filter_type, "value": converted_values} # 处理 range 类型 elif filter_type == "range" and isinstance(filter_value, list) and len(filter_value) == 2: converted_values = [] for idx, v in enumerate(filter_value): if isinstance(v, str): if self._is_datetime_string(v): parsed = self._parse_datetime_string(v) converted_values.append(parsed if parsed else v) elif self._is_date_string(v): # 对于 range 查询，第一个值（开始）使用当天开始，第二个值（结束）使用当天结束 try: parsed_date = date.fromisoformat(v) if idx == 0: # 开始日期：当天 00:00:00 start_of_day = datetime.combine(parsed_date, datetime.min.time()) converted_values.append(start_of_day) else: # 结束日期：当天 23:59:59.999999 end_of_day = datetime.combine(parsed_date, datetime.max.time()) converted_values.append(end_of_day) except ValueError: converted_values.append(v) else: converted_values.append(v) else: converted_values.append(v) converted[field] = {"type": filter_type, "value": converted_values} else: converted[field] = value elif isinstance(value, str): # 简单等值条件：布尔字符串 bool_val = self._convert_bool_string(value) if bool_val is not None: converted[field] = bool_val # 简单等值条件的日期/日期时间字符串 elif self._is_datetime_string(value): parsed = self._parse_datetime_string(value) converted[field] = parsed if parsed else value elif self._is_date_string(value): try: converted[field] = date.fromisoformat(value) except ValueError: converted[field] = value else: converted[field] = value else: converted[field] = value return converted async def _convert_form_data_filter_to_ids( self, db: AsyncSession, relation_config: Dict[str, str], value: Any, ) -> Any: """将表单选择器字段的显示名筛选转为关联表主键 ID（支持第三方库）。""" form_code = relation_config.get("form_code") display_column = relation_config.get("display_column", "name") relation_key = relation_config.get("relation_key", "id") if not form_code: return value is_list, parsed = self._parse_list_value(value) if is_list and parsed and all(self._is_uuid_like(v) for v in parsed): if len(parsed) == 1: return parsed[0] return {"type": "in", "value": parsed} if isinstance(value, list) and all(self._is_uuid_like(v) for v in value): if len(value) == 1: return value[0] return {"type": "in", "value": value} if isinstance(value, str) and self._is_uuid_like(value): return value from online_dev.form_manager.model import FormMeta as RelatedFormMeta stmt = select(RelatedFormMeta).where( RelatedFormMeta.code == form_code, RelatedFormMeta.is_deleted == False, ) result = await db.execute(stmt) related_form = result.scalar_one_or_none() if not related_form: return value rel_adapter, rel_builder = await resolve_form_sql_context( db, related_form, adapter_cache=self._adapter_cache, builder_cache=self._sql_builder_cache, ) table = related_form.main_table schema = await self._resolve_schema(db, related_form.main_table_schema) or None rel_database = related_form.main_table_database or None table_name = rel_builder.build_table_name(table, schema, rel_database) key_col = rel_builder.quote_identifier(relation_key) display_col = rel_builder.quote_identifier(display_column) rel_exec_db = self._table_exec_database(rel_database) has_is_deleted = await self._check_column_exists( db, table, related_form.main_table_schema, related_form.main_table_database, "is_deleted", sql_builder=rel_builder, db_adapter=rel_adapter, ) deleted_clause = ( f" AND {rel_builder.is_deleted_predicate()}" if has_is_deleted else "" ) if isinstance(value, dict) and "type" in value: filter_type = value.get("type") filter_value = value.get("value") if filter_type in ("space_like_and", "space_like_or", "space_eq_and", "space_eq_or"): return value if filter_type == "in" and isinstance(filter_value, list): if all(self._is_uuid_like(v) for v in filter_value): return value if filter_type == "like": search_sql = ( f"SELECT {key_col} as id FROM {table_name} " f"WHERE {display_col} LIKE :search_value{deleted_clause}" ) params = {"search_value": f"%{filter_value}%"} else: case_sensitive = value.get("case_sensitive", True) if case_sensitive: where_clause = f"{display_col} = :search_value" elif rel_builder.db_type == "postgresql": where_clause = ( f"CAST({display_col} AS TEXT) ILIKE :search_value" ) else: where_clause = ( f"LOWER(CAST({display_col} AS VARCHAR(4000))) " f"= LOWER(:search_value)" ) search_sql = ( f"SELECT {key_col} as id FROM {table_name} " f"WHERE {where_clause}{deleted_clause}" ) params = {"search_value": filter_value} else: if rel_builder.db_type == "postgresql": where_clause = f"CAST({display_col} AS TEXT) ILIKE :search_value" else: where_clause = f"{display_col} LIKE :search_value" search_sql = ( f"SELECT {key_col} as id FROM {table_name} " f"WHERE {where_clause}{deleted_clause}" ) params = {"search_value": f"%{value}%"} try: rows = await rel_adapter.execute_query( search_sql, params, database=rel_exec_db ) except Exception as e: logger.error("表单关联字段名称转 ID 失败 [%s]: %s", form_code, e) return value if not rows: return "00000000-0000-0000-0000-000000000000" ids = [row["id"] for row in rows] if len(ids) == 1: return ids[0] return {"type": "in", "value": ids} async def _convert_relation_name_to_id(self, db: AsyncSession, filters: Dict[str, Any]) -> Dict[str, Any]: """将关联字段的名称搜索转换为 ID 搜索，或直接使用 ID 列表""" if not filters: return filters # 获取关联字段配置 relation_fields = self._get_relation_fields() if not relation_fields: return filters converted_filters = {} for field, value in filters.items(): # 检查是否是关联字段 if field in relation_fields: relation_config = relation_fields[field] relation_type = relation_config.get("relation_type", "") if relation_type == "form_data": try: converted_filters[field] = await self._convert_form_data_filter_to_ids( db, relation_config, value ) except Exception as e: logger.error(f"转换表单关联字段 {field} 搜索失败: {e}") converted_filters[field] = value continue relation_table = relation_config.get("relation_table") display_column = relation_config.get("display_column", "name") if not relation_table: converted_filters[field] = value continue try: is_list, parsed = self._parse_list_value(value) if is_list and parsed and all(self._is_uuid_like(v) for v in parsed): if len(parsed) == 1: converted_filters[field] = parsed[0] else: converted_filters[field] = {"type": "in", "value": parsed} continue if isinstance(value, list) and all(self._is_uuid_like(v) for v in value): if len(value) == 1: converted_filters[field] = value[0] else: converted_filters[field] = {"type": "in", "value": value} continue # 检查是否是单个 UUID（直接传递的 ID） if isinstance(value, str) and self._is_uuid_like(value): converted_filters[field] = value continue # 根据名称查询平台关联表获取 ID（始终在平台 PG） pb = self._platform_sql_builder table_name = pb.build_table_name(relation_table, schema=PLATFORM_SCHEMA) # 处理不同的过滤类型 if isinstance(value, dict) and "type" in value: filter_type = value.get("type") filter_value = value.get("value") # 空格搜索类型是纯文本搜索，不需要转换关联字段 if filter_type in ("space_like_and", "space_like_or", "space_eq_and", "space_eq_or"): converted_filters[field] = value continue # 如果 filter_value 是 ID 列表，直接使用 if filter_type == "in" and isinstance(filter_value, list): if all(self._is_uuid_like(v) for v in filter_value): converted_filters[field] = value continue if filter_type == "like": query_sql = f""" SELECT {pb.quote_identifier("id")} FROM {table_name} WHERE {pb.quote_identifier(display_column)} LIKE :search_value AND {pb.is_deleted_predicate()} """ params = {"search_value": f"%{filter_value}%"} else: case_sensitive = value.get("case_sensitive", True) if case_sensitive: where_clause = f"{pb.quote_identifier(display_column)} = :search_value" else: where_clause = ( f"LOWER({pb.quote_identifier(display_column)}) " f"= LOWER(:search_value)" ) query_sql = f""" SELECT {pb.quote_identifier("id")} FROM {table_name} WHERE {where_clause} AND {pb.is_deleted_predicate()} """ params = {"search_value": filter_value} else: query_sql = f""" SELECT {pb.quote_identifier("id")} FROM {table_name} WHERE {pb.quote_identifier(display_column)} LIKE :search_value AND {pb.is_deleted_predicate()} """ params = {"search_value": f"%{value}%"} result = await self._execute_platform_query(db, query_sql, params) if result: # 找到匹配的 ID，使用 ID 进行搜索 ids = [row["id"] for row in result] if len(ids) == 1: converted_filters[field] = ids[0] else: # 多个匹配，使用 IN 查询 converted_filters[field] = {"type": "in", "value": ids} else: # 没有找到匹配的名称，返回一个不存在的 ID（确保搜索结果为空） converted_filters[field] = "00000000-0000-0000-0000-000000000000" except Exception as e: logger.error(f"转换关联字段 {field} 搜索失败: {e}") converted_filters[field] = value else: # 非关联字段，保持原样 converted_filters[field] = value return converted_filters def _get_relation_fields(self) -> Dict[str, Dict[str, str]]: """从表单配置中提取需要关联查询的字段""" relation_fields = {} list_config = self._list_config columns = list_config.get("columns", []) form_config = self._form_config logger.debug(f"表单配置 form_config 类型: {type(form_config)}") logger.debug(f"表单配置内容: {form_config}") logger.debug(f"表单配置 items 数量: {len(form_config.get('items', []))}") logger.debug(f"列表配置 columns 数量: {len(columns)}") # 构建字段到组件配置的映射 field_to_component = {} def traverse_items(items): """递归遍历表单项，构建字段映射""" if not items: return for item in items: item_type = item.get("type", "") field = item.get("field", "") # 容器类型，递归处理子项 if item_type == "grid": for col in item.get("columns", []): traverse_items(col.get("children", [])) elif item_type in ("collapse", "steps"): for panel in item.get("items", []): traverse_items(panel.get("children", [])) elif item_type == "tabs": for panel in item.get("children", []): traverse_items(panel.get("children", [])) elif item_type == "sub-table": traverse_items(item.get("children", [])) elif field: # 保存字段对应的组件配置 field_to_component[field] = item logger.debug(f"找到字段: {field}, 类型: {item_type}") traverse_items(form_config.get("items", [])) logger.debug(f"构建的字段映射: {list(field_to_component.keys())}") # 遍历列表配置，查找需要关联查询的字段 for col in columns: field = col.get("field") display_field = col.get("displayField") if not field or not display_field: continue logger.debug(f"检查字段 {field}，displayField: {display_field}") # 从表单配置中找到对应的组件 component = field_to_component.get(field) if not component: # 组件可能在未遍历到的容器中，尝试从列配置的保存信息中恢复 if col.get("isFormDataSelector") and col.get("formCode"): display_field_name = col.get("displayFieldName", "") form_code = col.get("formCode", "") relation_key = col.get("valueField", "id") or "id" display_column = display_field_name or col.get("labelField", "name") or "name" relation_fields[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": relation_key, "display_column": display_column, } logger.debug(f"字段 {field} 组件未找到，从列配置恢复: formCode={form_code}, displayColumn={display_column}") else: logger.warning(f"字段 {field} 在表单配置中未找到对应组件") continue # 获取组件的 props props = component.get("props", {}) logger.debug(f"字段 {field} 的组件类型: {component.get('type')}, props: {props}") # 从 props 中提取关联表信息 relation_table = props.get("relationTable") relation_key = props.get("relationKey", "id") display_column = props.get("displayColumn", "name") # 如果 props 中没有配置，尝试根据组件类型和字段名推断 if not relation_table: component_type = component.get("type", "") # 根据组件类型推断 type_mapping = { "department-selector": "core_dept", "dept-selector": "core_dept", "dept-select": "core_dept", "user-selector": "core_user", "user-select": "core_user", "position-selector": "core_post", "post-selector": "core_post", "role-selector": "core_role", "org-selector": "core_dept", "region-selector": "core_region", } relation_table = type_mapping.get(component_type) # 如果还是没有，根据字段名推断 if not relation_table: field_mapping = { "dept_id": "core_dept", "department_id": "core_dept", "user_id": "core_user", "manger_id": "core_user", "manager_id": "core_user", "position_id": "core_post", "positon_id": "core_post", } relation_table = field_mapping.get(field) if relation_table: relation_fields[field] = { "display_field": display_field, "relation_table": relation_table, "relation_key": relation_key, "display_column": display_column, } logger.debug(f"字段 {field} 配置了关联表: {relation_table}") else: # 检查是否为表单数据选择器（select 或 table-selector 组件，数据源类型为 formData） # 或者是新的 form-selector 组件 component_type = component.get("type", "") # form-selector 组件的配置存储在 formSelectorConfig 中 form_selector_config = component.get("formSelectorConfig") or {} # 数据源配置存储在 dataSource 中，而不是 props 中 data_source = component.get("dataSource") or {} data_source_type = data_source.get("type", "") if data_source else "" if not data_source_type: data_source_type = props.get("dataSourceType", "") # 获取 formCode：优先从 formSelectorConfig 获取，其次从 dataSource 获取 form_code = form_selector_config.get("formCode", "") or data_source.get("formCode", "") or props.get("formCode", "") display_field_name = col.get("displayFieldName", "") # 从列表配置中获取要显示的字段名 # form-selector 组件直接处理 if component_type == "form-selector" and form_code: relation_key = form_selector_config.get("valueField", "id") or "id" label_field = form_selector_config.get("labelField", "name") or "name" relation_fields[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": relation_key, "display_column": display_field_name or label_field, } logger.debug(f"字段 {field} 配置了表单选择器关联: formCode={form_code}, relationKey={relation_key}, displayColumn={display_field_name or label_field}") elif component_type in ["select", "tree-select", "radio", "checkbox", "cascader", "table-selector"] and data_source_type == "formData" and form_code: relation_key = data_source.get("formValueField") or data_source.get("valueField", "id") or "id" label_field = data_source.get("formLabelField") or data_source.get("labelField", "name") or "name" relation_fields[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": relation_key, "display_column": display_field_name or label_field, } logger.debug(f"字段 {field} 配置了表单数据关联: formCode={form_code}, relationKey={relation_key}, displayColumn={display_field_name or label_field}") else: logger.warning(f"字段 {field} 未配置 relationTable 且无法自动推断") # 添加系统字段的关联配置（这些字段不在表单配置中，但需要关联查询） system_field_mapping = { "sys_creator_id": {"relation_table": "core_user", "display_column": "name"}, "sys_modifier_id": {"relation_table": "core_user", "display_column": "name"}, "sys_dept_id": {"relation_table": "core_dept", "display_column": "name"}, } for col in columns: field = col.get("field") display_field = col.get("displayField") if field in system_field_mapping and display_field: config = system_field_mapping[field] relation_fields[field] = { "display_field": display_field, "relation_table": config["relation_table"], "relation_key": "id", "display_column": config["display_column"], } logger.debug(f"系统字段 {field} 配置了关联表: {config['relation_table']}") # 识别没有 displayField 的数据源字段（字典 / 表单数据），原地替换 value 为 label column_fields = {col.get("field") for col in columns if col.get("field")} for field, component in field_to_component.items(): if field in relation_fields or field not in column_fields: continue data_source = component.get("dataSource") or {} ds_type = data_source.get("type", "") if ds_type == "dict" and data_source.get("dictCode"): display_field = f"{field}_label" relation_fields[field] = { "display_field": display_field, "relation_type": "dict", "dict_code": data_source["dictCode"], } logger.debug(f"字段 {field} 配置了字典数据源: dictCode={data_source['dictCode']}, displayField={display_field}") elif ds_type == "formData" and data_source.get("formCode"): component_type = component.get("type", "") if component_type in ["select", "tree-select", "radio", "checkbox", "cascader", "table-selector"]: form_code = data_source["formCode"] relation_key = data_source.get("formValueField") or data_source.get("valueField", "id") or "id" label_field = data_source.get("formLabelField") or data_source.get("labelField", "name") or "name" display_field = f"{field}_name" relation_fields[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": relation_key, "display_column": label_field, } logger.debug(f"字段 {field} 无 displayField，自动补充表单数据关联: formCode={form_code}, displayField={display_field}") elif ds_type == "dataSource" and data_source.get("dataSourceCode"): display_field = f"{field}_label" relation_fields[field] = { "display_field": display_field, "relation_type": "data_source", "data_source_code": data_source["dataSourceCode"], "value_field": data_source.get("valueField") or "value", "label_field": data_source.get("labelField") or "label", } # 静态选项（select/radio/checkbox/cascader/tree-select 等组件的硬编码 options） elif component.get("type") in ["select", "radio", "checkbox", "cascader", "tree-select"] and component.get("options"): if not ds_type or ds_type == "static": display_field = f"{field}_label" relation_fields[field] = { "display_field": display_field, "relation_type": "static", "options": component.get("options", []), } logger.debug(f"字段 {field} 配置了静态选项: displayField={display_field}, 选项数={len(component.get('options', []))}") return relation_fields def _get_all_selector_relation_fields(self, existing: Dict[str, Dict[str, str]]) -> Dict[str, Dict[str, str]]: """从 form_config 中提取所有选择器字段的关联配置，补充 _get_relation_fields 可能遗漏的字段。 _get_relation_fields 依赖 list_config.columns，不在列表中的选择器字段不会被识别。 此方法直接从 form_config.items 遍历所有选择器组件，确保 get() 上下文下也能填充显示名。 """ result = dict(existing) form_config = self._form_config system_type_mapping = { "dept-selector": "core_dept", "user-selector": "core_user", "role-selector": "core_role", "post-selector": "core_post", "region-selector": "core_region", } def traverse(items): if not items: return for item in items: t = item.get("type", "") field = item.get("field", "") if t == "grid": for col in item.get("columns", []): traverse(col.get("children", [])) elif t in ("collapse", "steps"): for panel in item.get("items", []): traverse(panel.get("children", [])) elif t == "tabs": for panel in item.get("children", []): traverse(panel.get("children", [])) elif t == "sub-table": traverse(item.get("children", [])) elif field and field not in result: if t in system_type_mapping: display_field = field.replace("_id", "_name") if field.endswith("_id") else f"{field}_name" result[field] = { "display_field": display_field, "relation_table": system_type_mapping[t], "relation_key": "id", "display_column": "name", } elif t == "form-selector": fsc = item.get("formSelectorConfig") or {} form_code = fsc.get("formCode") or item.get("props", {}).get("formCode") or "" if form_code: label_field = fsc.get("labelField") or item.get("props", {}).get("labelField") or "name" value_field = fsc.get("valueField") or "id" if label_field not in ("name", "label"): display_field = f"{field}_{label_field}" elif field.endswith("_id"): display_field = field.replace("_id", "_name") else: display_field = f"{field}_name" result[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": value_field, "display_column": label_field, } elif t == "table-selector": ds = item.get("dataSource") or {} form_code = ds.get("formCode") or item.get("props", {}).get("formCode") or "" if form_code: label_field = ds.get("formLabelField") or item.get("props", {}).get("labelField") or "label" value_field = ds.get("formValueField") or ds.get("valueField") or "id" if label_field not in ("name", "label"): display_field = f"{field}_{label_field}" elif field.endswith("_id"): display_field = field.replace("_id", "_name") else: display_field = f"{field}_name" result[field] = { "display_field": display_field, "relation_type": "form_data", "form_code": form_code, "relation_key": value_field, "display_column": label_field, } # 静态选项（select/radio/checkbox/cascader/tree-select 的硬编码 options） elif t in ["select", "radio", "checkbox", "cascader", "tree-select"]: ds = item.get("dataSource") or {} ds_type = ds.get("type", "") options = item.get("options", []) if options and (not ds_type or ds_type == "static"): display_field = f"{field}_label" if not field.endswith("_id") else field.replace("_id", "_label") result[field] = { "display_field": display_field, "relation_type": "static", "options": options, } traverse(form_config.get("items", [])) return result async def _fill_relation_display_names( self, db: AsyncSession, items: List[Dict[str, Any]], relation_fields: Dict[str, Dict[str, str]] ) -> List[Dict[str, Any]]: """填充关联字段的显示名称""" for field, config in relation_fields.items(): relation_type = config.get("relation_type", "") relation_table = config.get("relation_table", "") # 表单数据选择器特殊处理 if relation_type == "form_data": try: await self._fill_form_data_display_field(db, items, field, config) except Exception as e: logger.warning(f"填充表单数据字段 {field} 失败: {str(e)}") try: await db.rollback() except Exception: pass continue if relation_type == "data_source": code = config.get("data_source_code") value_field = config.get("value_field", "value") label_field = config.get("label_field", "label") display_field = config.get("display_field", f"{field}_label") if not code: continue try: from core.data_source.service import DataSourceService rows = await DataSourceService.execute(db, code, {}) if not isinstance(rows, list): rows = [] value_to_label = {} for row in rows: if isinstance(row, dict): vk = row.get(value_field) if vk is not None: value_to_label[str(vk)] = row.get(label_field, vk) for item in items: v = item.get(field) if v is not None: is_list, parsed = self._parse_list_value(v) if is_list: item[display_field] = [ value_to_label.get(str(x), x) for x in parsed ] else: item[display_field] = value_to_label.get(str(v), v) except Exception as e: logger.warning(f"填充平台数据源字段 {field} 失败: {str(e)}") continue # 字典数据源：批量查询 dict_item，将 label 写入显示字段（不覆盖原值） if relation_type == "dict": dict_code = config.get("dict_code") if not dict_code: continue try: values = set() for item in items: v = item.get(field) if v is not None: is_list, parsed = self._parse_list_value(v) if is_list: values.update(str(x) for x in parsed if x is not None) else: values.add(str(v)) if not values: continue value_to_label = await self._query_dict_labels(db, dict_code, list(values)) # 确定显示字段名：优先使用配置的 display_field，否则自动生成 {field}_label display_field = config.get("display_field", f"{field}_label") for item in items: v = item.get(field) if v is not None: is_list, parsed = self._parse_list_value(v) if is_list: item[display_field] = [value_to_label.get(str(x), x) for x in parsed] else: item[display_field] = value_to_label.get(str(v), v) logger.debug(f"字典字段 {field} 翻译完成: dictCode={dict_code}, 映射数={len(value_to_label)}, displayField={display_field}") except Exception as e: logger.warning(f"填充字典字段 {field} 失败: {str(e)}") continue # 静态选项：从 options 中查找 label if relation_type == "static": options = config.get("options", []) if not options: continue display_field = config.get("display_field", f"{field}_label") try: value_to_label = {} for opt in options: opt_value = opt.get("value") if isinstance(opt_value, bool) or opt_value is not None: value_to_label[str(opt_value)] = opt.get("label", str(opt_value)) for item in items: v = item.get(field) if v is not None: is_list, parsed = self._parse_list_value(v) if is_list: item[display_field] = [value_to_label.get(str(x), x) for x in parsed] else: item[display_field] = value_to_label.get(str(v), v) logger.debug(f"静态选项字段 {field} 翻译完成: displayField={display_field}, 选项数={len(options)}") except Exception as e: logger.warning(f"填充静态选项字段 {field} 失败: {str(e)}") continue # 如果没有关联表，跳过 if not relation_table: logger.warning(f"字段 {field} 没有配置关联表，跳过") continue # 省市区组件特殊处理 if relation_table == "core_region": try: codes = set() for item in items: is_list, parsed = self._parse_list_value(item.get(field)) if is_list: codes.update(str(v) for v in parsed if v) if not codes: continue code_to_name = await self._query_region_names(db, list(codes)) display_field = config["display_field"] for item in items: is_list, parsed = self._parse_list_value(item.get(field)) if is_list: names = [ code_to_name.get(str(c), str(c)) for c in parsed if c ] item[display_field] = " / ".join(names) except Exception as e: logger.warning(f"填充省市区字段 {field} 失败: {str(e)}") continue elif relation_table: # 其他关联组件的处理逻辑（用户、部门等） ids = self._collect_field_ids(items, field) if not ids: continue logger.info(f"准备查询关联字段 {field}，关联表: {relation_table}，IDs: {ids}") try: # 批量查询关联数据 relation_data = await self._query_relation_data( db, config["relation_table"], config["relation_key"], config["display_column"], list(ids) ) # 构建 ID 到名称的映射 id_to_name = {str(row["id"]): row["name"] for row in relation_data} display_field = config["display_field"] logger.info(f"字段 {field} 的 ID 到名称映射: {id_to_name}") for item in items: self._set_display_from_id_map(item, field, display_field, id_to_name) except Exception as e: logger.warning(f"填充关联字段 {field} 失败: {str(e)}") # 回滚事务，避免影响后续查询 try: await db.rollback() except Exception: pass # 填充失败不影响主流程，继续处理其他字段 continue return items @staticmethod def _parse_list_value(v: Any) -> Tuple[bool, list]: """将字段值解析为列表。 级联/多选组件的值可能是 Python list（json/jsonb 列）、JSON 字符串（varchar/text 列）， 或历史数据中 Python str(list) 产生的单引号格式（如 "['a', 'b']"）。 返回 (is_list, parsed_list)；非列表值返回 (False, [])。 """ if isinstance(v, list): return True, v if isinstance(v, str) and v.startswith("["): try: parsed = json.loads(v) if isinstance(parsed, list): return True, parsed except (json.JSONDecodeError, ValueError): pass # 兼容历史数据：str(list) 产生的单引号格式，如 "['a', 'b']" try: import ast parsed = ast.literal_eval(v) if isinstance(parsed, list): return True, parsed except (ValueError, SyntaxError): pass return False, [] @staticmethod def _collect_field_ids(items: List[Dict[str, Any]], field: str) -> set: """从列表行中收集关联字段 ID（兼容 JSON 字符串形式的多选值）。""" ids: set = set() for item in items: v = item.get(field) if v is None: continue is_list, parsed = FormDataService._parse_list_value(v) if is_list: ids.update(str(x) for x in parsed if x is not None) else: ids.add(str(v)) return ids @staticmethod def _set_display_from_id_map( item: Dict[str, Any], field: str, display_field: str, id_to_name: Dict[str, str], ) -> None: """将 ID（或 ID 列表）映射为显示名称写入 display_field。""" value = item.get(field) if not value: return is_list, parsed = FormDataService._parse_list_value(value) if is_list: names = [id_to_name.get(str(v), str(v)) for v in parsed if v] item[display_field] = ", ".join(names) else: item[display_field] = id_to_name.get(str(value), str(value)) async def _query_dict_labels(self, db: AsyncSession, dict_code: str, values: List[str]) -> Dict[str, str]: """批量查询字典项的 value->label 映射""" from core.dict.model import Dict as DictModel from core.dict_item.model import DictItem stmt = ( select(DictItem.value, DictItem.label) .join(DictModel, DictItem.dict_id == DictModel.id) .where( DictModel.code == dict_code, DictItem.value.in_(values), DictItem.is_deleted == False, ) ) result = await db.execute(stmt) return {row.value: row.label for row in result} async def _query_form_data_labels( self, db: AsyncSession, field: str, config: Dict[str, Any], items: List[Dict[str, Any]] ) -> Dict[str, str]: """查询关联表单数据，返回 value->label 映射（用于无 displayField 的表单数据源字段）""" form_code = config.get("form_code") relation_key = config.get("relation_key", "id") display_column = config.get("display_column", "name") # 收集所有不重复的 value（兼容 JSON 字符串数组） ids = set() for item in items: v = item.get(field) if v is not None: is_list, parsed = self._parse_list_value(v) if is_list: ids.update(str(x) for x in parsed if x is not None) else: ids.add(str(v)) if not ids or not form_code: return {} from online_dev.form_manager.model import FormMeta as RelatedFormMeta stmt = select(RelatedFormMeta).where( RelatedFormMeta.code == form_code, RelatedFormMeta.is_deleted == False ) result = await db.execute(stmt) related_form = result.scalar_one_or_none() if not related_form: logger.warning(f"关联表单 {form_code} 不存在") return {} rel_adapter, rel_builder = await resolve_form_sql_context( db, related_form, adapter_cache=self._adapter_cache, builder_cache=self._sql_builder_cache, ) table = related_form.main_table schema = await self._resolve_schema(db, related_form.main_table_schema) or None database = related_form.main_table_database or None table_name = rel_builder.build_table_name(table, schema, database) key_col = rel_builder.quote_identifier(relation_key) display_col = rel_builder.quote_identifier(display_column) has_is_deleted = await self._check_column_exists( db, table, related_form.main_table_schema, related_form.main_table_database, "is_deleted", sql_builder=rel_builder, db_adapter=rel_adapter, ) ids_list = list(ids) placeholders = ", ".join([f":{i}" for i in range(len(ids_list))]) is_deleted_clause = ( f"\n AND {rel_builder.is_deleted_predicate()}" if has_is_deleted else "" ) sql = f""" SELECT {key_col} as id, {display_col} as name FROM {table_name} WHERE {key_col} IN ({placeholders}){is_deleted_clause} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} rel_exec_db = (database or "").strip() or None rows = await rel_adapter.execute_query(sql, params, database=rel_exec_db) return {str(row["id"]): str(row["name"]) for row in rows} async def _fill_virtual_fields( self, db: AsyncSession, items: List[Dict[str, Any]], relation_fields: Dict[str, Dict[str, str]], context: str = "list" ) -> List[Dict[str, Any]]: """填充虚拟字段的值 虚拟字段通过值关联配置（isVirtualField=true），从源字段的关联数据中 提取指定属性填充到结果中，不对应数据库列。 Args: db: 数据库会话 items: 数据列表 relation_fields: 已识别的关联字段配置（来自 _get_relation_fields） context: 调用上下文，'list' 列表接口（受 showVirtualValue 开关控制），'form' 表单详情（始终填充） """ if not items: return items # 从 form_config 中提取虚拟字段配置 virtual_fields = self._get_virtual_fields() if not virtual_fields: return items # 构建 list_config 中虚拟字段的 showVirtualValue 映射 list_config = self._list_config columns = list_config.get("columns", []) col_show_map = {} for col in columns: if col.get("isVirtualField"): col_show_map[col.get("field", "")] = col.get("showVirtualValue", True) logger.info(f"识别到的虚拟字段: {virtual_fields}") for vf in virtual_fields: vf_field = vf["field"] # 虚拟字段名 source_field = vf["valueSourceField"] # 源字段名 display_field = vf["valueDisplayField"] # 要提取的属性名 # 仅在列表上下文中检查 showVirtualValue 开关，表单详情始终填充 if context == "list" and not col_show_map.get(vf_field, True): logger.info(f"虚拟字段 {vf_field} 的显示关联值已关闭，跳过") continue # 从 relation_fields 中获取源字段的关联配置 source_config = relation_fields.get(source_field) # 如果源字段不在 relation_fields 中（可能未添加到列表列），从 form_config 推断 if not source_config: source_config = self._infer_relation_config(source_field) if not source_config: logger.warning(f"虚拟字段 {vf_field} 的源字段 {source_field} 没有关联配置，跳过") continue relation_type = source_config.get("relation_type", "") relation_table = source_config.get("relation_table", "") relation_key = source_config.get("relation_key", "id") # 省市区组件不支持虚拟字段 if relation_table == "core_region": logger.warning(f"虚拟字段 {vf_field} 的源字段 {source_field} 是省市区组件，不支持虚拟字段") continue ids = self._collect_field_ids(items, source_field) if not ids: continue try: id_to_value = {} if relation_type == "form_data": # 表单数据选择器：查询关联表单的数据表 form_code = source_config.get("form_code") if not form_code: continue from online_dev.form_manager.model import FormMeta as RelatedFormMeta stmt = select(RelatedFormMeta).where( RelatedFormMeta.code == form_code, RelatedFormMeta.is_deleted == False ) result = await db.execute(stmt) related_form = result.scalar_one_or_none() if not related_form: logger.warning(f"虚拟字段 {vf_field} 关联表单 {form_code} 不存在") continue rel_adapter, rel_builder = await resolve_form_sql_context( db, related_form, adapter_cache=self._adapter_cache, builder_cache=self._sql_builder_cache, ) table = related_form.main_table schema = await self._resolve_schema(db, related_form.main_table_schema) or None database = related_form.main_table_database or None table_name = rel_builder.build_table_name(table, schema, database) key_col = rel_builder.quote_identifier(relation_key) val_col = rel_builder.quote_identifier(display_field) has_is_deleted = await self._check_column_exists( db, table, related_form.main_table_schema, related_form.main_table_database, "is_deleted", sql_builder=rel_builder, db_adapter=rel_adapter, ) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) ids_list = list(ids) is_deleted_clause = ( f"\n AND {rel_builder.is_deleted_predicate()}" if has_is_deleted else "" ) sql = f""" SELECT {key_col} as id, {val_col} as val FROM {table_name} WHERE {key_col} IN ({placeholders}){is_deleted_clause} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} rel_exec_db = (database or "").strip() or None rows = await rel_adapter.execute_query( sql, params, database=rel_exec_db ) id_to_value = {str(row["id"]): row["val"] for row in rows} else: if not relation_table: continue pb = self._platform_sql_builder table_name = pb.build_table_name(relation_table, schema=PLATFORM_SCHEMA) key_col = pb.quote_identifier(relation_key) val_col = pb.quote_identifier(display_field) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) ids_list = list(ids) sql = f""" SELECT {key_col} as id, {val_col} as val FROM {table_name} WHERE {key_col} IN ({placeholders}) AND {pb.is_deleted_predicate()} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} async with db.begin_nested(): rows = await self._execute_platform_query(db, sql, params) id_to_value = {str(row["id"]): row["val"] for row in rows} for item in items: source_value = item.get(source_field) if not source_value: item[vf_field] = "" continue is_list, parsed = self._parse_list_value(source_value) if is_list: vals = [ id_to_value.get(str(v), "") for v in parsed if v ] item[vf_field] = ", ".join(str(v) for v in vals if v) else: item[vf_field] = id_to_value.get(str(source_value), "") except Exception as e: logger.warning(f"填充虚拟字段 {vf_field} 失败: {str(e)}") continue return items def _get_virtual_fields(self) -> List[Dict[str, str]]: """从 form_config 中提取虚拟字段配置 Returns: 虚拟字段配置列表，每项包含: - field: 虚拟字段名 - valueSourceField: 源字段名 - valueDisplayField: 要提取的属性名 """ virtual_fields = [] form_config = self._form_config def traverse(items): if not items: return for item in items: item_type = item.get("type", "") field = item.get("field", "") props = item.get("props", {}) # 检查是否为虚拟字段 if (field and props.get("isVirtualField") and props.get("enableValueLink") and props.get("valueSourceField") and props.get("valueDisplayField")): virtual_fields.append({ "field": field, "valueSourceField": props["valueSourceField"], "valueDisplayField": props["valueDisplayField"], }) # 递归处理容器 if item_type == "grid": for col in item.get("columns", []): traverse(col.get("children", [])) elif item_type in ("collapse", "steps"): for panel in item.get("items", []): traverse(panel.get("children", [])) elif item_type == "tabs": for panel in item.get("children", []): traverse(panel.get("children", [])) elif item_type == "sub-table": traverse(item.get("children", [])) traverse(form_config.get("items", [])) return virtual_fields def _infer_relation_config(self, source_field: str) -> Optional[Dict[str, str]]: """从 form_config 中推断源字段的关联表配置 当源字段未出现在 list_config.columns 中时，需要从 form_config 的组件类型推断关联信息。 Args: source_field: 源字段名 Returns: 关联配置字典，包含 relation_table, relation_key, display_column 等，或 None """ form_config = self._form_config # 递归查找源字段的组件配置 def find_component(items): if not items: return None for item in items: item_type = item.get("type", "") field = item.get("field", "") if field == source_field: return item if item_type == "grid": for col in item.get("columns", []): found = find_component(col.get("children", [])) if found: return found elif item_type in ("collapse", "steps"): for panel in item.get("items", []): found = find_component(panel.get("children", [])) if found: return found elif item_type == "tabs": for panel in item.get("children", []): found = find_component(panel.get("children", [])) if found: return found elif item_type == "sub-table": found = find_component(item.get("children", [])) if found: return found return None component = find_component(form_config.get("items", [])) if not component: return None component_type = component.get("type", "") props = component.get("props", {}) # 根据组件类型推断关联表 type_mapping = { "department-selector": "core_dept", "dept-selector": "core_dept", "dept-select": "core_dept", "user-selector": "core_user", "user-select": "core_user", "position-selector": "core_post", "post-selector": "core_post", "role-selector": "core_role", "org-selector": "core_dept", "region-selector": "core_region", } relation_table = props.get("relationTable") or type_mapping.get(component_type) if relation_table: return { "relation_table": relation_table, "relation_key": props.get("relationKey", "id"), "display_column": "name", } # form-selector 组件 if component_type == "form-selector": form_selector_config = component.get("formSelectorConfig") or {} form_code = form_selector_config.get("formCode", "") or props.get("formCode", "") if form_code: return { "relation_type": "form_data", "form_code": form_code, "relation_key": form_selector_config.get("valueField", "id") or "id", "display_column": form_selector_config.get("labelField", "name") or "name", } # select/table-selector 组件，数据源为 formData data_source = component.get("dataSource") or {} data_source_type = data_source.get("type", "") or props.get("dataSourceType", "") form_code = data_source.get("formCode", "") or props.get("formCode", "") if component_type in ["select", "tree-select", "radio", "checkbox", "cascader", "table-selector"] and data_source_type == "formData" and form_code: return { "relation_type": "form_data", "form_code": form_code, "relation_key": data_source.get("formValueField") or data_source.get("valueField", "id") or "id", "display_column": data_source.get("formLabelField") or data_source.get("labelField", "name") or "name", } # 根据字段名推断 field_mapping = { "dept_id": "core_dept", "department_id": "core_dept", "user_id": "core_user", "manger_id": "core_user", "manager_id": "core_user", "position_id": "core_post", "positon_id": "core_post", } relation_table = field_mapping.get(source_field) if relation_table: return { "relation_table": relation_table, "relation_key": "id", "display_column": "name", } return None def _get_non_virtual_linked_fields(self) -> List[Dict[str, str]]: """从 form_config 中提取非虚拟的值关联字段配置 非虚拟关联字段（enableValueLink=true, isVirtualField=false 或未设置） 对应数据库列，需要在保存时将关联值写入数据库。 Returns: 配置列表，每项包含: - field: 字段名（对应数据库列） - valueSourceField: 源字段名 - valueDisplayField: 要从关联数据中提取的属性名 """ linked_fields = [] form_config = self._form_config def traverse(items): if not items: return for item in items: item_type = item.get("type", "") field = item.get("field", "") props = item.get("props", {}) if (field and not props.get("isVirtualField") and props.get("enableValueLink") and props.get("valueSourceField") and props.get("valueDisplayField")): linked_fields.append({ "field": field, "valueSourceField": props["valueSourceField"], "valueDisplayField": props["valueDisplayField"], }) if item_type == "grid": for col in item.get("columns", []): traverse(col.get("children", [])) elif item_type in ("collapse", "steps"): for panel in item.get("items", []): traverse(panel.get("children", [])) elif item_type == "tabs": for panel in item.get("children", []): traverse(panel.get("children", [])) elif item_type == "sub-table": traverse(item.get("children", [])) traverse(form_config.get("items", [])) return linked_fields async def _fill_linked_field_values( self, db: AsyncSession, data: Dict[str, Any], table_type: str = "main" ) -> Dict[str, Any]: """保存前填充非虚拟关联字段的值 根据值关联配置，从源字段的值反查关联表，将结果写入对应的数据库列。 Args: db: 数据库会话 data: 待保存的数据 table_type: "main" 或 "sub" """ linked_fields = self._get_non_virtual_linked_fields() if not linked_fields: return data relation_fields = self._get_relation_fields() for lf in linked_fields: target_field = lf["field"] source_field = lf["valueSourceField"] display_field = lf["valueDisplayField"] source_value = data.get(source_field) if not source_value: continue source_config = relation_fields.get(source_field) if not source_config: source_config = self._infer_relation_config(source_field) if not source_config: logger.warning(f"非虚拟关联字段 {target_field} 的源字段 {source_field} 没有关联配置，跳过") continue relation_type = source_config.get("relation_type", "") relation_table = source_config.get("relation_table", "") relation_key = source_config.get("relation_key", "id") if relation_table == "core_region": continue ids = set() if isinstance(source_value, list): ids.update(str(v) for v in source_value if v) else: ids.add(str(source_value)) if not ids: continue try: id_to_value = {} if relation_type == "form_data": form_code = source_config.get("form_code") if not form_code: continue from online_dev.form_manager.model import FormMeta as RelatedFormMeta stmt = select(RelatedFormMeta).where( RelatedFormMeta.code == form_code, RelatedFormMeta.is_deleted == False ) result = await db.execute(stmt) related_form = result.scalar_one_or_none() if not related_form: logger.warning(f"关联字段 {target_field} 的关联表单 {form_code} 不存在") continue rel_adapter, rel_builder = await resolve_form_sql_context( db, related_form, adapter_cache=self._adapter_cache, builder_cache=self._sql_builder_cache, ) table = related_form.main_table schema = await self._resolve_schema(db, related_form.main_table_schema) or None database = related_form.main_table_database or None table_name = rel_builder.build_table_name(table, schema, database) key_col = rel_builder.quote_identifier(relation_key) val_col = rel_builder.quote_identifier(display_field) has_is_deleted = await self._check_column_exists( db, table, related_form.main_table_schema, related_form.main_table_database, "is_deleted", sql_builder=rel_builder, db_adapter=rel_adapter, ) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) ids_list = list(ids) is_deleted_clause = ( f"\n AND {rel_builder.is_deleted_predicate()}" if has_is_deleted else "" ) sql = f""" SELECT {key_col} as id, {val_col} as val FROM {table_name} WHERE {key_col} IN ({placeholders}){is_deleted_clause} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} rel_exec_db = (database or "").strip() or None rows = await rel_adapter.execute_query( sql, params, database=rel_exec_db ) id_to_value = {str(row["id"]): row["val"] for row in rows} else: if not relation_table: continue pb = self._platform_sql_builder table_name = pb.build_table_name(relation_table, schema=PLATFORM_SCHEMA) key_col = pb.quote_identifier(relation_key) val_col = pb.quote_identifier(display_field) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) ids_list = list(ids) sql = f""" SELECT {key_col} as id, {val_col} as val FROM {table_name} WHERE {key_col} IN ({placeholders}) AND {pb.is_deleted_predicate()} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} rows = await self._execute_platform_query(db, sql, params) id_to_value = {str(row["id"]): row["val"] for row in rows} if id_to_value: if isinstance(source_value, list): vals = [id_to_value.get(str(v), "") for v in source_value if v] data[target_field] = ", ".join(str(v) for v in vals if v) else: resolved = id_to_value.get(str(source_value), "") if resolved: data[target_field] = resolved logger.debug(f"填充非虚拟关联字段 {target_field}: source={source_field}, value={data.get(target_field)}") except Exception as e: logger.warning(f"填充非虚拟关联字段 {target_field} 失败: {str(e)}") continue return data async def _fill_form_data_display_field( self, db: AsyncSession, items: List[Dict[str, Any]], field: str, config: Dict[str, Any] ) -> None: """填充表单数据选择器的显示字段 Args: db: 数据库会话 items: 数据列表 field: ID 字段名（如 customer_id） config: 配置信息，包含 form_code, display_field, display_column, relation_key """ form_code = config.get("form_code") display_field = config.get("display_field") display_column = config.get("display_column", "name") relation_key = config.get("relation_key", "id") if not form_code or not display_field: return ids = self._collect_field_ids(items, field) if not ids: return try: # 获取关联表单的元数据 from online_dev.form_manager.model import FormMeta as RelatedFormMeta from sqlalchemy import select stmt = select(RelatedFormMeta).where( RelatedFormMeta.code == form_code, RelatedFormMeta.is_deleted == False ) result = await db.execute(stmt) related_form = result.scalar_one_or_none() if not related_form: logger.warning(f"关联表单 {form_code} 不存在") return # 获取关联表单的表信息 table = related_form.main_table schema = await self._resolve_schema(db, related_form.main_table_schema) or None database = related_form.main_table_database or None rel_adapter, rel_builder = await resolve_form_sql_context( db, related_form, adapter_cache=self._adapter_cache, builder_cache=self._sql_builder_cache, ) table_name = rel_builder.build_table_name(table, schema, database) key_col = rel_builder.quote_identifier(relation_key) display_col = rel_builder.quote_identifier(display_column) has_is_deleted = await self._check_column_exists( db, table, related_form.main_table_schema, related_form.main_table_database, "is_deleted", sql_builder=rel_builder, db_adapter=rel_adapter, ) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) ids_list = list(ids) is_deleted_clause = ( f"\n AND {rel_builder.is_deleted_predicate()}" if has_is_deleted else "" ) sql = f""" SELECT {key_col} as id, {display_col} as name FROM {table_name} WHERE {key_col} IN ({placeholders}){is_deleted_clause} """ params = {str(i): ids_list[i] for i in range(len(ids_list))} rel_exec_db = (database or "").strip() or None rows = await rel_adapter.execute_query(sql, params, database=rel_exec_db) logger.debug(f"查询表单数据 {form_code}.{table}，找到 {len(rows)} 条记录") # 构建 ID 到名称的映射 id_to_name = {str(row["id"]): row["name"] for row in rows} for item in items: self._set_display_from_id_map(item, field, display_field, id_to_name) except Exception as e: logger.error(f"填充表单数据字段 {field} 失败: {str(e)}") raise async def _query_region_names( self, db: AsyncSession, codes: List[str] ) -> Dict[str, str]: """批量查询省市区名称 省市区数据分散在 5 个表中： - core_province: 2位代码（如 "14"） - core_city: 4位代码（如 "1404"） - core_area: 6位代码（如 "140406"） - core_street: 9位代码（如 "140406001"） - core_village: 12位代码（如 "140406001001"） Args: db: 数据库会话 codes: 行政区划代码列表 Returns: code 到 name 的映射字典 """ if not codes: return {} result = {} try: # 按代码长度分组 codes_by_length = {} for code in codes: length = len(str(code)) if length not in codes_by_length: codes_by_length[length] = [] codes_by_length[length].append(str(code)) # 定义表名映射（代码长度 -> 表名） table_mapping = { 2: "core_province", 4: "core_city", 6: "core_area", 9: "core_street", 12: "core_village", } # 分别查询每个表 for length, code_list in codes_by_length.items(): table_name = table_mapping.get(length) if not table_name: logger.warning(f"未知的代码长度: {length}，代码: {code_list}") continue pb = self._platform_sql_builder full_table_name = pb.build_table_name(table_name, schema=PLATFORM_SCHEMA) code_col = pb.quote_identifier("code") name_col = pb.quote_identifier("name") # 使用参数化查询 placeholders = ", ".join([f":{i}" for i in range(len(code_list))]) sql = f""" SELECT {code_col} as code, {name_col} as name FROM {full_table_name} WHERE {code_col} IN ({placeholders}) """ # 构建参数字典 params = {str(i): code_list[i] for i in range(len(code_list))} rows = await self._execute_platform_query(db, sql, params) logger.info(f"查询 {table_name}，找到 {len(rows)} 条记录") # 添加到结果映射 for row in rows: result[str(row["code"])] = row["name"] return result except Exception as e: logger.error(f"查询省市区名称失败: {str(e)}") return {} async def _query_relation_data( self, db: AsyncSession, table: str, key_column: str, display_column: str, ids: List[str] ) -> List[Dict[str, Any]]: """批量查询关联表数据""" if not ids: return [] pb = self._platform_sql_builder table_name = pb.build_table_name(table, schema=PLATFORM_SCHEMA) key_col = pb.quote_identifier(key_column) display_col = pb.quote_identifier(display_column) placeholders = ", ".join([f":{i}" for i in range(len(ids))]) sql = f""" SELECT {key_col} as id, {display_col} as name FROM {table_name} WHERE {key_col} IN ({placeholders}) AND {pb.is_deleted_predicate()} """ params = {str(i): ids[i] for i in range(len(ids))} try: logger.info(f"查询关联表 {table}，SQL: {sql}, params: {params}") rows = await self._execute_platform_query(db, sql, params) logger.info(f"查询关联表 {table}，找到 {len(rows)} 条记录，结果: {rows}") return [dict(row) for row in rows] except Exception as e: logger.error(f"查询关联表 {table} 失败: {str(e)}, SQL: {sql}, params: {params}") return [] # ============ 写入操作 ============ def _convert_data_types(self, data: Dict[str, Any]) -> Dict[str, Any]: """转换数据类型，将字符串日期时间转换为 Python datetime 对象，处理空字符串""" converted = {} # 获取字段类型映射（用于判断数字类型字段） field_types = self._get_field_types() # 调试日志 import logging logger = logging.getLogger(__name__) # logger.info(f"字段类型映射: {field_types}") # logger.info(f"待转换数据: {data}") for key, value in data.items(): if value is None: converted[key] = value continue # 处理空字符串 if isinstance(value, str) and value.strip() == '': # 获取字段类型 field_type = field_types.get(key, '').lower() # 数字类型字段的空字符串转为 None if field_type in ('int', 'integer', 'bigint', 'smallint', 'decimal', 'numeric', 'float', 'double', 'real'): converted[key] = None continue # 其他类型保持空字符串或转为 None converted[key] = None continue # 获取字段类型 field_type = field_types.get(key, '').lower() # 处理非字符串类型到字符串的转换 # 支持 PostgreSQL 的类型名：character varying, character, text 等 is_string_type = any(t in field_type for t in ['varchar', 'text', 'char', 'string']) if not isinstance(value, str) and is_string_type: if isinstance(value, datetime): if value.tzinfo is not None: value = value.astimezone(APP_TIMEZONE).replace(tzinfo=None) converted[key] = value.isoformat() logger.info(f"字段 {key} 从 datetime 转换为 str: {value} -> {converted[key]}") elif isinstance(value, (list, dict)): converted[key] = json.dumps(value, ensure_ascii=False) logger.info(f"字段 {key} 从 {type(value).__name__} 序列化为 JSON: {converted[key]}") else: converted[key] = str(value) logger.info(f"字段 {key} 从 {type(value).__name__} 转换为 str: {value} -> {str(value)}") continue # 处理已经是 datetime 对象的情况 if isinstance(value, datetime): # 如果是 offset-aware datetime，先转换为配置的时区再移除时区信息 if value.tzinfo is not None: value = value.astimezone(APP_TIMEZONE).replace(tzinfo=None) # 检查字段类型是否是 timestamp/datetime 类型 is_datetime_type = any(t in field_type for t in ('date', 'datetime', 'timestamp', 'time')) if is_datetime_type: # 保持为 datetime 对象 converted[key] = value else: # 字段类型未知或是字符串类型，转换为 ISO 格式字符串 converted[key] = value.isoformat() logger.info(f"字段 {key} 从 datetime 转换为 str (字段类型: {field_type}): {value} -> {converted[key]}") continue # 转换字符串类型的值 if isinstance(value, str): # 1. 尝试转换布尔类型 if field_type in ('bool', 'boolean'): # 支持多种布尔值表示 value_lower = value.lower().strip() if value_lower in ('true', '1', 'yes', 'y', 't', '是', '真'): converted[key] = True continue elif value_lower in ('false', '0', 'no', 'n', 'f', '否', '假'): converted[key] = False continue # 其他值保持原样 # 2. 尝试转换整数类型 if field_type in ('int', 'integer', 'bigint', 'smallint'): try: # 先去除空格，支持 "25 " 这样的输入 converted[key] = int(value.strip()) continue except (ValueError, TypeError): pass # 3. 尝试转换浮点数类型 elif field_type in ('decimal', 'numeric', 'float', 'double', 'real'): try: converted[key] = float(value.strip()) continue except (ValueError, TypeError): pass # 4. 尝试解析日期时间（支持 'timestamp without time zone' 等完整类型名） is_datetime_type = any(t in field_type for t in ('date', 'datetime', 'timestamp', 'time')) if is_datetime_type: try: from datetime import time as time_type # 时间格式 if field_type == 'time': for fmt in ["%H:%M:%S", "%H:%M:%S.%f", "%H:%M"]: try: t = datetime.strptime(value, fmt).time() converted[key] = t break except ValueError: continue else: converted[key] = value continue # 先尝试使用 fromisoformat 解析（支持更多 ISO 格式） parsed_dt = None try: # 处理 Z 后缀（UTC 时区标识） iso_value = value.replace('Z', '+00:00') if value.endswith('Z') else value parsed_dt = datetime.fromisoformat(iso_value) except ValueError: # fromisoformat 失败，尝试其他格式 for fmt in [ "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%d", ]: try: parsed_dt = datetime.strptime(value, fmt) break except ValueError: continue if parsed_dt is not None: # 如果是日期格式（没有时间部分），只保留日期 if len(value) == 10 and '-' in value: converted[key] = parsed_dt.date() else: # 将 UTC 时间转换为本地时间后再移除时区信息 if parsed_dt.tzinfo is not None: # 转换为配置的时区 parsed_dt = parsed_dt.astimezone(APP_TIMEZONE) # 移除时区信息 parsed_dt = parsed_dt.replace(tzinfo=None) converted[key] = parsed_dt else: # 无法解析，保持原值 converted[key] = value continue except Exception: converted[key] = value continue # 5. 尝试解析 JSON 类型（json, jsonb） if field_type in ('json', 'jsonb'): try: converted[key] = json.loads(value) continue except (json.JSONDecodeError, TypeError): # 如果解析失败，保持原字符串 pass # 6. 数组类型（PostgreSQL array） if field_type.endswith('[]') or field_type in ('array', 'text[]', 'varchar[]', 'integer[]'): try: parsed = json.loads(value) if isinstance(parsed, list): converted[key] = parsed continue except (json.JSONDecodeError, TypeError): # 尝试按逗号分隔 if ',' in value: converted[key] = [item.strip() for item in value.split(',')] continue # 默认保持原值 converted[key] = value else: converted[key] = value return converted def _get_field_types(self) -> Dict[str, str]: """获取字段名到类型的映射""" field_types = {} # 从 form_config 中获取字段类型 form_config = self._form_config table_configs = form_config.get('tableConfigs', []) for table_config in table_configs: fields = table_config.get('fields', []) for field in fields: field_name = field.get('name', '') field_type = field.get('type', '') if field_name and field_type: field_types[field_name] = field_type return field_types async def create(self, db: AsyncSession, data: Dict[str, Any]) -> Dict[str, Any]: """新增数据（含子表，事务）""" main_data = data.get("main") or {} sub_tables_data = data.get("sub_tables") or {} field_perms = await self._get_merged_field_permissions(db) # 过滤主表字段 allowed_main_fields = self._get_allowed_fields("main") filtered_main = self._filter_writable_fields(main_data, allowed_main_fields, field_perms) # 转换数据类型 filtered_main = self._convert_data_types(filtered_main) # 移除 id 字段，生成新的 UUID filtered_main.pop("id", None) generated_id = str(uuid.uuid4()) filtered_main["id"] = generated_id # 填充系统字段（创建时间、创建人、部门等） filtered_main = self._fill_system_fields_for_create(filtered_main) # 填充非虚拟关联字段的值（从源字段反查关联表写入数据库列） filtered_main = await self._fill_linked_field_values(db, filtered_main, table_type="main") if not filtered_main: raise FormDataValidationError("主表数据不能为空") # 唯一性校验（新增时不需要排除ID） await self._validate_unique_fields(db, filtered_main) # 1. 插入主表 table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None sql, params = self.sql_builder.build_insert( table=table, data=filtered_main, schema=schema, database=database, return_id=False ) main_pk = generated_id async with self._business_transaction(database=self._table_exec_database(database)): await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) for sub_table in self.sub_tables: sub_data_list = sub_tables_data.get(sub_table.table_name, []) if not sub_data_list: continue allowed_sub_fields = self._get_allowed_fields("sub", sub_table.table_name) for sub_item in sub_data_list: filtered_sub = self._filter_writable_fields( sub_item, allowed_sub_fields, field_perms, always_allow={sub_table.foreign_key}, ) filtered_sub = self._convert_data_types(filtered_sub) filtered_sub.pop("id", None) filtered_sub["id"] = str(uuid.uuid4()) filtered_sub[sub_table.foreign_key] = main_pk filtered_sub = self._fill_system_fields_for_create(filtered_sub) filtered_sub = await self._fill_linked_field_values( db, filtered_sub, table_type="sub" ) if filtered_sub: sub_sql, sub_params = self.sql_builder.build_insert( table=sub_table.table_name, data=filtered_sub, schema=sub_table.table_schema or None, database=sub_table.table_database or None, return_id=False, ) sub_db = sub_table.table_database or None await self._execute_command( db, sub_sql, sub_params, database=self._table_exec_database(sub_db), ) await db.commit() logger.info(f"表单数据创建成功: form={self._form_code}, pk={main_pk}") return await self.get(db, main_pk) async def update(self, db: AsyncSession, pk: Any, data: Dict[str, Any]) -> Dict[str, Any]: """更新数据（含子表，事务）""" main_data = data.get("main") or {} sub_tables_data = data.get("sub_tables") or {} field_perms = await self._get_merged_field_permissions(db) await self.get(db, pk) main_schema_db = self.form_meta.main_table_database or None async with self._business_transaction( database=self._table_exec_database(main_schema_db) ): if main_data: allowed_main_fields = self._get_allowed_fields("main") filtered_main = self._filter_writable_fields( main_data, allowed_main_fields, field_perms ) filtered_main = self._convert_data_types(filtered_main) filtered_main.pop("id", None) filtered_main = self._fill_system_fields_for_update(filtered_main) filtered_main = await self._fill_linked_field_values( db, filtered_main, table_type="main" ) await self._validate_unique_fields(db, filtered_main, exclude_id=str(pk)) if filtered_main: table = self.form_meta.main_table schema = await self._resolve_schema( db, self.form_meta.main_table_schema ) or None database = self.form_meta.main_table_database or None sql, params = self.sql_builder.build_update( table=table, data=filtered_main, pk_field="id", pk_value=pk, schema=schema, database=database, ) await self._execute_command( db, sql, params, database=self._table_exec_database(database), ) for sub_table in self.sub_tables: if sub_table.table_name not in sub_tables_data: continue new_sub_data = sub_tables_data[sub_table.table_name] await self._handle_sub_table_update( db, sub_table, pk, new_sub_data, field_perms ) await db.commit() logger.info(f"表单数据更新成功: form={self._form_code}, pk={pk}") return await self.get(db, pk) async def _handle_sub_table_update( self, db: AsyncSession, sub_table: FormSubTable, main_pk: Any, new_data: List[Dict[str, Any]], field_perms: Optional[Dict[str, Dict]] = None, ): """处理子表更新（差异对比：新增/更新/删除）""" table_name = sub_table.table_name schema = sub_table.table_schema or None database = sub_table.table_database or None foreign_key = sub_table.foreign_key # 获取现有数据 existing = await self._query_sub_table_data(db, sub_table, main_pk) existing_map = {item["id"]: item for item in existing if "id" in item} allowed_fields = self._get_allowed_fields("sub", table_name) field_perms = field_perms or {} writable_allow = {foreign_key} # 分类处理 new_ids = set() to_insert = [] to_update = [] for item in new_data: item_id = item.get("id") if item_id and item_id in existing_map: new_ids.add(item_id) to_update.append(item) elif not item_id: to_insert.append(item) else: to_insert.append(item) # 找出需要删除的 to_delete = [eid for eid in existing_map.keys() if eid not in new_ids] # 执行删除 for del_id in to_delete: sql, params = self.sql_builder.build_delete( table=table_name, pk_field="id", pk_value=del_id, schema=schema, database=database ) await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) # 执行更新 for item in to_update: filtered = self._filter_writable_fields(item, allowed_fields, field_perms) filtered = self._convert_data_types(filtered) item_id = filtered.pop("id", None) # 填充子表系统字段（更新） filtered = self._fill_system_fields_for_update(filtered) # 填充子表非虚拟关联字段的值 filtered = await self._fill_linked_field_values(db, filtered, table_type="sub") if filtered and item_id: sql, params = self.sql_builder.build_update( table=table_name, data=filtered, pk_field="id", pk_value=item_id, schema=schema, database=database ) await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) # 执行新增 for item in to_insert: filtered = self._filter_writable_fields( item, allowed_fields, field_perms, always_allow=writable_allow ) filtered = self._convert_data_types(filtered) if "id" in filtered: del filtered["id"] filtered["id"] = str(uuid.uuid4()) if foreign_key in filtered: del filtered[foreign_key] filtered[foreign_key] = main_pk # 填充子表系统字段（新增） filtered = self._fill_system_fields_for_create(filtered) # 填充子表非虚拟关联字段的值 filtered = await self._fill_linked_field_values(db, filtered, table_type="sub") if filtered and len(filtered) > 2: sql, params = self.sql_builder.build_insert( table=table_name, data=filtered, schema=schema, database=database, return_id=False ) await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) async def delete(self, db: AsyncSession, pk: Any) -> bool: """删除数据（含子表，事务）""" # 验证数据存在（不存在时 get 会抛出 RecordNotFoundException） await self.get(db, pk) main_database = self.form_meta.main_table_database or None async with self._business_transaction( database=self._table_exec_database(main_database) ): for sub_table in self.sub_tables: sub_db = sub_table.table_database or None sql, params = self.sql_builder.build_delete_by_foreign_key( table=sub_table.table_name, fk_field=sub_table.foreign_key, fk_value=pk, schema=sub_table.table_schema or None, database=sub_db, ) await self._execute_command( db, sql, params, database=self._table_exec_database(sub_db), ) table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None sql, params = self.sql_builder.build_delete( table=table, pk_field="id", pk_value=pk, schema=schema, database=database, ) affected = await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) await db.commit() logger.info(f"表单数据删除成功: form={self._form_code}, pk={pk}") return affected > 0 async def batch_delete(self, db: AsyncSession, pks: List[Any]) -> int: """ 批量删除（优化版：使用 IN 子句批量删除，单次事务提交） Args: db: 数据库会话 pks: 主键列表 Returns: 成功删除的数量 """ if not pks: return 0 async with self._business_transaction(): affected = await self._batch_delete_in_transaction(db, pks) await db.commit() logger.info(f"批量删除成功: form={self._form_code}, count={affected}") return affected async def _batch_delete_in_transaction(self, db: AsyncSession, pks: List[Any]) -> int: for sub_table in self.sub_tables: schema = await self._resolve_schema(db, sub_table.table_schema) if sub_table.table_schema else None database = sub_table.table_database or None # 构建批量删除 SQL: DELETE FROM table WHERE fk_field IN (:pk0, :pk1, ...) full_table = self.sql_builder.build_table_name(sub_table.table_name, schema, database) fk_field = self.sql_builder.quote_identifier(sub_table.foreign_key) placeholders = ", ".join(f":pk{i}" for i in range(len(pks))) sql = f"DELETE FROM {full_table} WHERE {fk_field} IN ({placeholders})" params = {f"pk{i}": pk for i, pk in enumerate(pks)} await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) # 2. 批量删除主表数据（使用 IN 子句） table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None full_table = self.sql_builder.build_table_name(table, schema, database) id_field = self.sql_builder.quote_identifier("id") placeholders = ", ".join(f":pk{i}" for i in range(len(pks))) sql = f"DELETE FROM {full_table} WHERE {id_field} IN ({placeholders})" params = {f"pk{i}": pk for i, pk in enumerate(pks)} return await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) # ============ 工具方法 ============ def _serialize_row(self, row: Dict[str, Any]) -> Dict[str, Any]: """序列化行数据（处理特殊类型）""" result = {} for key, value in row.items(): if isinstance(value, datetime): result[key] = value.strftime("%Y-%m-%d %H:%M:%S") elif isinstance(value, date): result[key] = value.strftime("%Y-%m-%d") elif isinstance(value, Decimal): result[key] = float(value) elif isinstance(value, uuid.UUID): result[key] = str(value) elif isinstance(value, bytes): result[key] = value.decode("utf-8", errors="ignore") elif isinstance(value, str): result[key] = value.strip() else: result[key] = value return result # ============ 导入导出 ============ async def export_to_excel_streaming( self, db: AsyncSession, selected_fields: List[str] = None, include_sub_tables: bool = False, batch_size: int = 1000, data_scope: Dict[str, Any] = None, filters: Dict[str, Any] = None, sort_list: List[Dict[str, str]] = None, search: str = None, search_fields: List[str] = None, on_progress: Any = None ) -> BytesIO: """ 流式导出数据到 Excel（分批查询，避免内存溢出） Args: db: 数据库会话 selected_fields: 选中的字段列表 include_sub_tables: 是否导出子表 batch_size: 每批查询的数据量 data_scope: 数据权限过滤配置 filters: 过滤条件（与列表查询一致） sort_list: 排序列表 search: 搜索关键词 search_fields: 搜索字段列表 on_progress: 进度回调函数 async fn(processed, total)，可选 Returns: Excel 文件的 BytesIO 对象 """ wb = Workbook() ws = wb.active ws.title = "主表数据" # 从列表配置中获取列定义 list_config = self._list_config columns = list_config.get("columns", []) # 如果指定了选择的字段，只导出这些字段 if selected_fields and columns: columns = [col for col in columns if col.get("field") in selected_fields] # 确保 selected_fields 中的字段都被包含 existing_fields = {col.get("field") for col in columns} for field in selected_fields: if field not in existing_fields: columns.append({"field": field, "label": field.upper()}) if columns: headers = [col.get("field") for col in columns] column_labels = [col.get("label", col.get("field")) for col in columns] display_fields = { col.get("field"): col.get("displayField") for col in columns if col.get("displayField") } else: # 如果没有列配置，先查询一条数据获取字段 first_result = await self.list(db=db, page=1, page_size=1, data_scope=data_scope) if first_result["items"]: headers = list(first_result["items"][0].keys()) else: headers = [] column_labels = headers display_fields = {} # 从 relation_fields 补充缺少 displayField 的关联字段映射 relation_fields = self._get_relation_fields() for field_name, config in relation_fields.items(): if field_name not in display_fields and config.get("display_field"): display_fields[field_name] = config["display_field"] # 先查询总数（用于进度计算） total_count = 0 if on_progress: count_result = await self.list( db=db, page=1, page_size=1, data_scope=data_scope, filters=filters, sort_list=sort_list, search=search, search_fields=search_fields ) total_count = min(count_result.get("total", 0), MAX_IMPORT_EXPORT_ROWS) await on_progress(0, total_count, "querying") # 定义样式 header_font = Font(name='微软雅黑', size=11, bold=True, color='FFFFFF') header_fill = PatternFill(start_color='4472C4', end_color='4472C4', fill_type='solid') header_alignment = Alignment(horizontal='center', vertical='center') data_alignment = Alignment(horizontal='left', vertical='center') thin_border = Border( left=Side(style='thin', color='D0D0D0'), right=Side(style='thin', color='D0D0D0'), top=Side(style='thin', color='D0D0D0'), bottom=Side(style='thin', color='D0D0D0') ) # 写入表头 ws.append(column_labels) for col_idx, cell in enumerate(ws[1], 1): cell.font = header_font cell.fill = header_fill cell.alignment = header_alignment cell.border = thin_border column_letter = cell.column_letter header_length = len(str(cell.value)) ws.column_dimensions[column_letter].width = max(12, header_length + 4) # 子表 sheet 初始化（延迟到第一批有数据时创建表头） sub_sheets: Dict[str, Any] = {} sub_headers_map: Dict[str, List[str]] = {} # 分批查询并写入数据（主表 + 子表同步处理） page = 1 total_rows = 0 while True: result = await self.list( db=db, page=page, page_size=batch_size, data_scope=data_scope, filters=filters, sort_list=sort_list, search=search, search_fields=search_fields ) items = result["items"] if not items: break if page == 1 and total_count == 0: total_count = min(result.get("total", 0), MAX_IMPORT_EXPORT_ROWS) batch_ids = [] for item in items: row = [] for header in headers: display_field = display_fields.get(header) if display_field and display_field in item: value = item.get(display_field, "") else: value = item.get(header, "") if isinstance(value, (list, dict)): value = str(value) row.append(value) ws.append(row) total_rows += 1 if include_sub_tables and item.get("id"): batch_ids.append(item.get("id")) # 当前批次的子表数据立即查询并写入 if include_sub_tables and self.sub_tables and batch_ids: for sub_table in self.sub_tables: sub_key = sub_table.alias or sub_table.table_name sql, params = self.sql_builder.build_select( table=sub_table.table_name, schema=sub_table.table_schema or None, database=sub_table.table_database or None, where={sub_table.foreign_key: {"type": "in", "value": batch_ids}} ) rows = await self._execute_query(db, sql, params) if not rows: continue sub_items = [self._serialize_row(r) for r in rows] if sub_key not in sub_sheets: ws_sub = wb.create_sheet(title=sub_key[:31]) sub_hdrs = ["主表ID"] + [k for k in sub_items[0].keys()] ws_sub.append(sub_hdrs) for col_idx, cell in enumerate(ws_sub[1], 1): cell.font = header_font cell.fill = header_fill cell.alignment = header_alignment cell.border = thin_border ws_sub.column_dimensions[cell.column_letter].width = max(12, len(str(cell.value)) + 4) sub_sheets[sub_key] = ws_sub sub_headers_map[sub_key] = sub_hdrs else: ws_sub = sub_sheets[sub_key] sub_hdrs = sub_headers_map[sub_key] for sub_item in sub_items: row = [sub_item.get(sub_table.foreign_key, "")] for h in sub_hdrs[1:]: value = sub_item.get(h, "") if isinstance(value, (list, dict)): value = str(value) row.append(value) ws_sub.append(row) if on_progress: await on_progress(total_rows, total_count, "querying") if len(items) < batch_size: break page += 1 if total_rows >= MAX_IMPORT_EXPORT_ROWS: logger.warning(f"导出数据量达到上限 {MAX_IMPORT_EXPORT_ROWS} 条（服务器内存 {SERVER_MEMORY_GB}GB），停止导出") break if on_progress: await on_progress(total_rows, total_count, "generating") ws.freeze_panes = 'A2' for ws_sub in sub_sheets.values(): ws_sub.freeze_panes = 'A2' # 保存到内存 output = BytesIO() wb.save(output) output.seek(0) logger.info(f"流式导出完成: 共 {total_rows} 条数据") return output async def export_to_excel(self, items: List[Dict[str, Any]], selected_fields: List[str] = None, sub_tables_data: Dict[str, List[Dict[str, Any]]] = None) -> BytesIO: """导出数据到 Excel（带样式，支持字段选择和子表导出）""" wb = Workbook() ws = wb.active ws.title = "主表数据" if not items: output = BytesIO() wb.save(output) output.seek(0) return output # 从列表配置中获取列定义 list_config = self._list_config columns = list_config.get("columns", []) # 如果没有列配置，使用第一行数据的所有字段 if not columns: headers = list(items[0].keys()) column_labels = headers display_fields = {} else: # 如果指定了选择的字段，只导出这些字段 if selected_fields: columns = [col for col in columns if col.get("field") in selected_fields] # 确保 selected_fields 中的字段都被包含（即使不在 columns 配置中） existing_fields = {col.get("field") for col in columns} for field in selected_fields: if field not in existing_fields: # 添加缺失的字段（如 id） columns.append({"field": field, "label": field.upper()}) headers = [col.get("field") for col in columns] column_labels = [col.get("label", col.get("field")) for col in columns] # 构建字段到 displayField 的映射 display_fields = { col.get("field"): col.get("displayField") for col in columns if col.get("displayField") } # 定义样式 # 表头样式：深蓝色背景，白色粗体文字 header_font = Font(name='微软雅黑', size=11, bold=True, color='FFFFFF') header_fill = PatternFill(start_color='4472C4', end_color='4472C4', fill_type='solid') header_alignment = Alignment(horizontal='center', vertical='center') # 数据单元格样式：居中对齐 data_alignment = Alignment(horizontal='left', vertical='center') # 边框样式 thin_border = Border( left=Side(style='thin', color='D0D0D0'), right=Side(style='thin', color='D0D0D0'), top=Side(style='thin', color='D0D0D0'), bottom=Side(style='thin', color='D0D0D0') ) # 写入表头 ws.append(column_labels) # 设置表头样式 for col_idx, cell in enumerate(ws[1], 1): cell.font = header_font cell.fill = header_fill cell.alignment = header_alignment cell.border = thin_border # 设置列宽（根据表头长度自动调整） column_letter = cell.column_letter header_length = len(str(cell.value)) ws.column_dimensions[column_letter].width = max(12, header_length + 4) # 写入数据 for item in items: row = [] for header in headers: # 优先使用 displayField 的值（如果配置了） display_field = display_fields.get(header) if display_field and display_field in item: value = item.get(display_field, "") else: value = item.get(header, "") # 处理特殊类型 if isinstance(value, (list, dict)): value = str(value) row.append(value) ws.append(row) # 设置数据行样式 for row_idx in range(2, ws.max_row + 1): for col_idx in range(1, ws.max_column + 1): cell = ws.cell(row=row_idx, column=col_idx) cell.alignment = data_alignment cell.border = thin_border # 交替行背景色（浅灰色） if row_idx % 2 == 0: cell.fill = PatternFill(start_color='F2F2F2', end_color='F2F2F2', fill_type='solid') # 冻结首行（表头） ws.freeze_panes = 'A2' # 导出子表数据 if sub_tables_data: for sub_table_name, sub_data_list in sub_tables_data.items(): if not sub_data_list: continue # 为每个子表创建新的工作表 ws_sub = wb.create_sheet(title=sub_table_name[:31]) # Excel 工作表名称最多 31 字符 # 子表表头（包含主表ID） sub_headers = ["主表ID"] + [k for k in sub_data_list[0].keys() if k != "_main_id"] ws_sub.append(sub_headers) # 设置子表表头样式 for col_idx, cell in enumerate(ws_sub[1], 1): cell.font = header_font cell.fill = header_fill cell.alignment = header_alignment cell.border = thin_border column_letter = cell.column_letter ws_sub.column_dimensions[column_letter].width = max(12, len(str(cell.value)) + 4) # 写入子表数据 for sub_item in sub_data_list: row = [sub_item.get("_main_id", "")] for header in sub_headers[1:]: value = sub_item.get(header, "") # 处理特殊类型 if isinstance(value, (list, dict)): value = str(value) row.append(value) ws_sub.append(row) # 设置子表数据行样式 for row_idx in range(2, ws_sub.max_row + 1): for col_idx in range(1, ws_sub.max_column + 1): cell = ws_sub.cell(row=row_idx, column=col_idx) cell.alignment = data_alignment cell.border = thin_border if row_idx % 2 == 0: cell.fill = PatternFill(start_color='F2F2F2', end_color='F2F2F2', fill_type='solid') ws_sub.freeze_panes = 'A2' # 保存到内存 output = BytesIO() wb.save(output) output.seek(0) return output async def get_import_template(self) -> BytesIO: """生成导入模板（带样式）""" wb = Workbook() ws = wb.active ws.title = "导入模板" # 从列表配置中获取列定义 list_config = self._list_config columns = list_config.get("columns", []) # 定义样式 header_font = Font(name='微软雅黑', size=11, bold=True, color='FFFFFF') header_fill = PatternFill(start_color='4472C4', end_color='4472C4', fill_type='solid') header_alignment = Alignment(horizontal='center', vertical='center') thin_border = Border( left=Side(style='thin', color='D0D0D0'), right=Side(style='thin', color='D0D0D0'), top=Side(style='thin', color='D0D0D0'), bottom=Side(style='thin', color='D0D0D0') ) if columns: # 使用列配置生成表头（排除系统字段和关联字段的 displayField） headers = [] for col in columns: field = col.get("field") # 排除系统字段 if field not in ["id", "created_at", "updated_at", "sys_create_datetime", "sys_update_datetime"]: # 排除 displayField（如 dept_name），只保留原始字段（如 dept_id） display_field = col.get("displayField") if not display_field or field != display_field: headers.append(col.get("label", field)) ws.append(headers) # 设置表头样式 for col_idx, cell in enumerate(ws[1], 1): cell.font = header_font cell.fill = header_fill cell.alignment = header_alignment cell.border = thin_border # 设置列宽 column_letter = cell.column_letter header_length = len(str(cell.value)) ws.column_dimensions[column_letter].width = max(15, header_length + 4) # 添加示例数据行（带浅黄色背景提示这是示例） example_row = ["示例数据，请删除此行后填写实际数据"] + [""] * (len(headers) - 1) ws.append(example_row) for col_idx in range(1, len(headers) + 1): cell = ws.cell(row=2, column=col_idx) cell.fill = PatternFill(start_color='FFF2CC', end_color='FFF2CC', fill_type='solid') cell.alignment = Alignment(horizontal='left', vertical='center') cell.border = thin_border cell.font = Font(name='微软雅黑', size=10, italic=True, color='808080') else: # 如果没有列配置，添加提示 ws.append(["请配置列表字段后重新下载模板"]) ws['A1'].font = Font(name='微软雅黑', size=12, bold=True, color='FF0000') ws.column_dimensions['A'].width = 40 # 冻结首行 ws.freeze_panes = 'A2' # 保存到内存 output = BytesIO() wb.save(output) output.seek(0) return output async def import_from_excel( self, db: AsyncSession, file_content: bytes, batch_size: int = 1000, mode: str = "append", validate_only: bool = False, data_handling: str = "insert_only", match_field: str = None, on_progress=None ) -> tuple[int, int, List[Dict[str, Any]]]: """ 从 Excel 批量导入数据 Args: db: 数据库会话 file_content: Excel 文件内容 batch_size: 批量插入的批次大小，默认 1000 mode: 导入模式，"append"（追加）或 "overwrite"（覆盖，先清空表再导入） validate_only: 是否仅验证数据，不执行实际导入 data_handling: 数据处理方式 insert_only / update_only / upsert match_field: 更新模式下用于匹配的字段名 on_progress: 可选的异步回调 (processed, total, stage, success, fail) Returns: (成功数量, 失败数量, 错误详情列表) """ from utils.context import get_current_user_info_from_context from sqlalchemy import text as sa_text file_io = BytesIO(file_content) del file_content wb = load_workbook(file_io, read_only=True) ws = wb.active # read_only 模式下先读表头 row_iter = ws.iter_rows(values_only=True) headers = list(next(row_iter, [])) # read_only 模式 ws.max_row 可能不准，用预估值（后续会动态修正） total_excel_rows = (ws.max_row - 1) if ws.max_row and ws.max_row > 1 else 0 # 从列表配置中获取字段映射 list_config = self._list_config columns = list_config.get("columns", []) label_to_field = {col.get("label"): col.get("field") for col in columns if col.get("label") and col.get("field")} # 获取允许的字段 allowed_fields = self._get_allowed_fields("main") # 系统字段（需要排除） system_fields = {"id", "created_at", "updated_at", "sys_create_datetime", "sys_update_datetime", "sys_creator_id", "sys_modifier_id", "sys_dept_id", "is_deleted", "sort"} success_count = 0 fail_count = 0 error_details = [] # 获取用户信息（用于填充系统字段） user_info = get_current_user_info_from_context() if on_progress: await on_progress(0, total_excel_rows, "parsing", 0, 0) # 根据预估总行数动态计算进度推送间隔（约推送 50~100 次，最少 50 行，最多 5000 行） progress_interval = max(50, min(5000, total_excel_rows // 100)) if total_excel_rows > 0 else 100 # 流式逐行解析（read_only 模式下 iter_rows 是惰性的） all_rows = [] parsed_count = 0 for row in row_iter: row_num = parsed_count + 2 try: # 跳过空行 if all(cell is None or (isinstance(cell, str) and cell.strip() == "") for cell in row): parsed_count += 1 continue # 构建数据字典 data = {} for idx, value in enumerate(row): if idx < len(headers) and headers[idx]: field = label_to_field.get(headers[idx], headers[idx]) if field and field not in system_fields: if field in allowed_fields: data[field] = value if data: data = self._convert_data_types(data) data = self._filter_fields(data, allowed_fields) is_update_mode = mode == "append" and data_handling in ("update_only", "upsert") if not is_update_mode: data["id"] = str(uuid.uuid4()) data = self._fill_system_fields_for_create(data) data = self._normalize_data_for_insert(data) validation_error = self._validate_data_against_schema(row_num, data) if validation_error: fail_count += 1 error_details.append({"row": row_num, "error": validation_error}) parsed_count += 1 continue all_rows.append((row_num, data)) if len(all_rows) >= MAX_IMPORT_EXPORT_ROWS: logger.warning(f"导入数据量达到上限 {MAX_IMPORT_EXPORT_ROWS} 条（服务器内存 {SERVER_MEMORY_GB}GB），截断后续数据") parsed_count += 1 break except Exception as e: logger.error(f"解析第 {row_num} 行数据失败: {e}") fail_count += 1 error_details.append({"row": row_num, "error": str(e)}) parsed_count += 1 if on_progress and parsed_count % progress_interval == 0: if parsed_count > total_excel_rows: total_excel_rows = parsed_count progress_interval = max(50, min(5000, total_excel_rows // 100)) await on_progress(parsed_count, total_excel_rows, "parsing", 0, fail_count) await asyncio.sleep(0) if parsed_count != total_excel_rows: total_excel_rows = parsed_count if on_progress: await on_progress(parsed_count, total_excel_rows, "parsing", 0, fail_count) wb.close() file_io.close() del wb, ws, row_iter, file_io if not all_rows: return success_count, fail_count, error_details # 预先解析 schema（只解析一次） table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None # **按导入模式执行不同的验证规则** if on_progress: await on_progress(0, len(all_rows), "validating", 0, fail_count) is_update_mode = mode == "append" and data_handling in ("update_only", "upsert") unique_errors: Dict[int, str] = {} existing_set: Set[str] = set() if mode == "overwrite": unique_errors = await self._batch_check_unique_fields_internal_only(all_rows) elif data_handling == "insert_only": unique_errors = await self._batch_check_unique_fields(db, all_rows) elif data_handling in ("update_only", "upsert") and match_field: match_label = self._get_field_labels([match_field]).get(match_field, match_field) for row_num, data in all_rows: val = data.get(match_field) if val is None or val == '': unique_errors[row_num] = f"匹配字段 '{match_label}' 不能为空" match_val_to_rows: Dict[str, list] = {} for row_num, data in all_rows: val = data.get(match_field) if val is not None and val != '': match_val_to_rows.setdefault(str(val), []).append(row_num) for val, rows in match_val_to_rows.items(): if len(rows) > 1: for r in rows[1:]: if r not in unique_errors: unique_errors[r] = f"匹配字段 '{match_label}' 值 '{val}' 在导入数据中重复（第 {rows[0]} 行已存在）" if on_progress: await on_progress(len(all_rows) // 3, len(all_rows), "validating", 0, fail_count) match_values = [str(data.get(match_field)) for _, data in all_rows if data.get(match_field) not in (None, '')] existing_set = await self._batch_query_existing(db, match_field, match_values) if on_progress: await on_progress(len(all_rows) * 2 // 3, len(all_rows), "validating", 0, fail_count) if data_handling == "update_only": for row_num, data in all_rows: val = data.get(match_field) if val is not None and val != '' and str(val) not in existing_set and row_num not in unique_errors: unique_errors[row_num] = f"未找到匹配记录（{match_label}={val}）" else: new_rows = [(rn, d) for rn, d in all_rows if d.get(match_field) is not None and d.get(match_field) != '' and str(d.get(match_field)) not in existing_set and rn not in unique_errors] if new_rows: insert_unique_errors = await self._batch_check_unique_fields(db, new_rows) unique_errors.update(insert_unique_errors) if on_progress: await on_progress(len(all_rows), len(all_rows), "validating", 0, fail_count) if unique_errors: filtered_rows = [] for row_num, data in all_rows: if row_num in unique_errors: fail_count += 1 error_details.append({"row": row_num, "error": unique_errors[row_num]}) else: filtered_rows.append((row_num, data)) all_rows = filtered_rows if not all_rows: logger.info(f"Excel 导入完成: 成功 {success_count} 条, 失败 {fail_count} 条（所有数据验证失败）") return success_count, fail_count, error_details[:100] # **仅验证模式**：返回验证结果，不执行实际导入 if validate_only: success_count = len(all_rows) # 计算各模式下的操作预估 validate_meta: Dict[str, Any] = {"_meta": True} if mode == "overwrite": validate_meta["will_insert"] = success_count validate_meta["will_update"] = 0 validate_meta["action"] = "overwrite" elif data_handling == "insert_only": validate_meta["will_insert"] = success_count validate_meta["will_update"] = 0 validate_meta["action"] = "insert_only" elif data_handling == "update_only": validate_meta["will_insert"] = 0 validate_meta["will_update"] = success_count validate_meta["action"] = "update_only" elif data_handling == "upsert" and match_field: will_update = sum( 1 for _, d in all_rows if d.get(match_field) is not None and d.get(match_field) != '' and str(d.get(match_field)) in existing_set ) will_insert = success_count - will_update validate_meta["will_insert"] = will_insert validate_meta["will_update"] = will_update validate_meta["action"] = "upsert" else: validate_meta["will_insert"] = success_count validate_meta["will_update"] = 0 validate_meta["action"] = data_handling truncated_errors = error_details[:100] truncated_errors.append(validate_meta) logger.info(f"Excel 数据验证完成: 通过 {success_count} 条, 失败 {fail_count} 条, " f"预计新增 {validate_meta.get('will_insert', 0)} 条, " f"预计更新 {validate_meta.get('will_update', 0)} 条") return success_count, fail_count, truncated_errors main_exec_db = self._table_exec_database(database) total_valid = len(all_rows) imported_count = 0 if all_rows: num_fields = len(all_rows[0][1]) if num_fields > 0: max_safe_batch = 32000 // num_fields batch_size = max(1, min(batch_size, max_safe_batch)) if on_progress: await on_progress(0, total_valid, "importing", success_count, fail_count) if is_update_mode and match_field: async with self._business_transaction(database=main_exec_db): success_count, fail_count, error_details = await self._import_with_update( db, table, schema, database, all_rows, match_field, data_handling, success_count, fail_count, error_details, on_progress=on_progress, ) if not self.db_adapter.is_external: await db.commit() return success_count, fail_count, error_details[:100] async def _import_append_batch(batch_rows: List[tuple]) -> None: nonlocal success_count, fail_count, imported_count batch_data_list = [item[1] for item in batch_rows] batch_row_nums = [item[0] for item in batch_rows] sp_batch = f"sp_batch_{uuid.uuid4().hex[:8]}" try: await self._create_savepoint(db, sp_batch) await self._batch_insert_raw( db, table, schema, database, batch_data_list ) await self._release_savepoint(db, sp_batch) success_count += len(batch_data_list) except Exception as batch_err: logger.warning( f"批量插入失败（第 {batch_row_nums[0]}-{batch_row_nums[-1]} 行），" f"降级逐条插入: {batch_err}" ) await self._rollback_to_savepoint(db, sp_batch) for row_num, data in batch_rows: sp_single = f"sp_{uuid.uuid4().hex[:8]}" try: await self._create_savepoint(db, sp_single) sql, params = self.sql_builder.build_insert( table=table, data=data, schema=schema, database=database, return_id=False, ) await self._execute_command( db, sql, params, enrich_fk=False, database=main_exec_db, ) await self._release_savepoint(db, sp_single) success_count += 1 except Exception as single_err: await self._rollback_to_savepoint(db, sp_single) fail_count += 1 error_details.append({ "row": row_num, "error": format_error_message(single_err), }) imported_count += len(batch_rows) if self.db_adapter.is_external: # 第三方库：每批独立 Handler 事务提交，避免整表导入失败时全部回滚 if mode == "overwrite": async with self._business_transaction(database=main_exec_db): await self._truncate_table(db, table, schema, database) logger.info(f"覆盖模式：已清空表 {table}") for batch_start in range(0, len(all_rows), batch_size): batch = all_rows[batch_start:batch_start + batch_size] async with self._business_transaction(database=main_exec_db): await _import_append_batch(batch) db.expire_all() if on_progress: await on_progress( imported_count, total_valid, "importing", success_count, fail_count, ) else: async with self._business_transaction(database=main_exec_db): if mode == "overwrite": await self._truncate_table(db, table, schema, database) logger.info(f"覆盖模式：已清空表 {table}") for batch_start in range(0, len(all_rows), batch_size): batch = all_rows[batch_start:batch_start + batch_size] await _import_append_batch(batch) await db.commit() db.expire_all() if on_progress: await on_progress( imported_count, total_valid, "importing", success_count, fail_count, ) logger.info(f"Excel 导入完成: 成功 {success_count} 条, 失败 {fail_count} 条") return success_count, fail_count, error_details[:100] async def _import_with_update( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], all_rows: List[tuple], match_field: str, data_handling: str, success_count: int, fail_count: int, error_details: List[Dict[str, Any]], on_progress=None ) -> tuple[int, int, List[Dict[str, Any]]]: """ 导入数据（更新/upsert 模式） 通过 match_field 匹配已有记录： - update_only: 匹配到则更新，未匹配到则跳过 - upsert: 匹配到则更新，未匹配到则新增 """ full_table = self.sql_builder.build_table_name(table, schema, database) quoted_match = self.sql_builder.quote_identifier(match_field) total_valid = len(all_rows) # 1. 收集所有行的匹配字段值 match_values = [] for row_num, data in all_rows: val = data.get(match_field) if val is not None: match_values.append(val) # 2. 批量查询数据库中已存在的记录（match_field -> id 的映射） existing_map = {} if match_values: for i in range(0, len(match_values), 500): batch_vals = match_values[i:i + 500] placeholders = ", ".join([f":mv_{j}" for j in range(len(batch_vals))]) quoted_id = self.sql_builder.quote_identifier("id") query_sql = ( f"SELECT {quoted_id}, {quoted_match} FROM {full_table} " f"WHERE {quoted_match} IN ({placeholders})" ) params = {f"mv_{j}": v for j, v in enumerate(batch_vals)} try: result = await self._execute_query( db, query_sql, params, database=self._table_exec_database(database), ) for row in result: existing_map[row[match_field]] = row["id"] except Exception as e: logger.error(f"查询已有数据失败: {e}") # 3. 分批处理每行数据 imported_count = 0 commit_batch_size = 500 main_exec_db = self._table_exec_database(database) import_progress_interval = max(50, min(5000, total_valid // 100)) if total_valid > 0 else 100 for row_num, data in all_rows: match_val = data.get(match_field) existing_id = existing_map.get(match_val) if match_val is not None else None sp = f"sp_{uuid.uuid4().hex[:8]}" try: await self._create_savepoint(db, sp) if existing_id: update_data = {k: v for k, v in data.items() if k not in ("id", "sys_create_datetime", "sys_creator_id", "sys_dept_id")} update_data = self._fill_system_fields_for_update(update_data) sql, params = self.sql_builder.build_update( table=table, data=update_data, pk_field="id", pk_value=existing_id, schema=schema, database=database ) await self._execute_command( db, sql, params, enrich_fk=False, database=main_exec_db, ) await self._release_savepoint(db, sp) success_count += 1 elif data_handling == "upsert": data["id"] = str(uuid.uuid4()) data = self._fill_system_fields_for_create(data) sql, params = self.sql_builder.build_insert( table=table, data=data, schema=schema, database=database, return_id=False ) await self._execute_command( db, sql, params, enrich_fk=False, database=main_exec_db, ) await self._release_savepoint(db, sp) success_count += 1 else: await self._release_savepoint(db, sp) fail_count += 1 error_details.append({"row": row_num, "error": f"未找到匹配记录（{match_field}={match_val}）"}) except Exception as e: await self._rollback_to_savepoint(db, sp) fail_count += 1 error_details.append({ "row": row_num, "error": format_error_message(e), }) imported_count += 1 if ( self.db_adapter.is_external and imported_count % commit_batch_size == 0 ): # 第三方在父级 _business_transaction 内由调用方整段提交；此处仅刷新 ORM db.expire_all() elif imported_count % commit_batch_size == 0 and not self.db_adapter.is_external: await db.commit() db.expire_all() if on_progress and imported_count % import_progress_interval == 0: await on_progress(imported_count, total_valid, "importing", success_count, fail_count) if not self.db_adapter.is_external: await db.commit() db.expire_all() if on_progress: await on_progress(imported_count, total_valid, "importing", success_count, fail_count) logger.info(f"Excel 导入完成（{data_handling}）: 成功 {success_count} 条, 失败 {fail_count} 条") return success_count, fail_count, error_details async def _truncate_table( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str] ) -> None: """ 清空表数据（用于覆盖模式） Args: db: 数据库会话 table: 表名 schema: Schema 名 database: 数据库名 """ full_table = self.sql_builder.build_table_name(table, schema, database) for sub_table in self.sub_tables: sub_schema = ( await self._resolve_schema(db, sub_table.table_schema) if sub_table.table_schema else None ) sub_database = sub_table.table_database or None sub_full_table = self.sql_builder.build_table_name( sub_table.table_name, sub_schema, sub_database ) await self._execute_command( db, f"DELETE FROM {sub_full_table}", database=self._table_exec_database(sub_database), ) await self._execute_command( db, f"DELETE FROM {full_table}", database=self._table_exec_database(database), ) async def _batch_check_unique_fields_internal_only( self, data_list: List[tuple[int, Dict[str, Any]]] ) -> Dict[int, str]: """ 仅检查导入数据内部重复（用于覆盖模式，跳过数据库检查） Args: data_list: [(row_num, data), ...] 数据列表 Returns: {row_num: error_message} 违反唯一性约束的行号和错误信息 """ unique_fields = self._get_unique_check_fields() if not unique_fields: return {} errors = {} for field_name in unique_fields: # 收集该字段的所有非空值及其行号 value_to_rows = {} # {value: [row_num1, row_num2, ...]} for row_num, data in data_list: value = data.get(field_name) if value is None or value == '': continue value_str = str(value) if value_str not in value_to_rows: value_to_rows[value_str] = [] value_to_rows[value_str].append(row_num) # 检查内部重复 for value, row_nums in value_to_rows.items(): if len(row_nums) > 1: field_label = self._get_field_labels([field_name]).get(field_name, field_name) for row_num in row_nums[1:]: errors[row_num] = f"字段 '{field_label}' 值 '{value}' 在导入数据中重复（第 {row_nums[0]} 行已存在）" return errors async def _batch_check_unique_fields( self, db: AsyncSession, data_list: List[tuple[int, Dict[str, Any]]] ) -> Dict[int, str]: """ 批量检查唯一性约束（高性能版） Args: db: 数据库会话 data_list: [(row_num, data), ...] 数据列表 Returns: {row_num: error_message} 违反唯一性约束的行号和错误信息 """ unique_fields = self._get_unique_check_fields() if not unique_fields: return {} errors = {} table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None table_name = self.sql_builder.build_table_name(table, schema, database) for field_name in unique_fields: # 收集该字段的所有非空值及其行号 value_to_rows = {} # {value: [row_num1, row_num2, ...]} for row_num, data in data_list: value = data.get(field_name) if value is None or value == '': continue value_str = str(value) if value_str not in value_to_rows: value_to_rows[value_str] = [] value_to_rows[value_str].append(row_num) if not value_to_rows: continue # 1. 检查导入数据内部重复 for value, row_nums in value_to_rows.items(): if len(row_nums) > 1: # 内部重复：除了第一行，其他行都标记为错误 field_label = self._get_field_labels([field_name]).get(field_name, field_name) for row_num in row_nums[1:]: errors[row_num] = f"字段 '{field_label}' 值 '{value}' 在导入数据中重复（第 {row_nums[0]} 行已存在）" # 2. 分批检查数据库中是否已存在（避免超出 32767 参数限制） values = list(value_to_rows.keys()) if not values: continue quoted_field = self.sql_builder.quote_identifier(field_name) has_is_deleted = await self._check_column_exists(db, table, schema, database, "is_deleted") existing_values: set = set() for batch_start in range(0, len(values), 500): batch_vals = values[batch_start:batch_start + 500] placeholders = ", ".join(f":v{i}" for i in range(len(batch_vals))) sql = f"SELECT {quoted_field} FROM {table_name} WHERE {quoted_field} IN ({placeholders})" params = {f"v{i}": v for i, v in enumerate(batch_vals)} if has_is_deleted: sql += f" AND {self.sql_builder.is_deleted_predicate()}" result = await self._execute_query(db, sql, params) existing_values.update(str(row[field_name]) for row in result) # 标记数据库中已存在的值 field_label = self._get_field_labels([field_name]).get(field_name, field_name) for value in existing_values: for row_num in value_to_rows[value]: if row_num not in errors: # 避免覆盖内部重复错误 errors[row_num] = f"字段 '{field_label}' 值 '{value}' 已存在于数据库中" return errors async def _batch_query_existing( self, db: AsyncSession, field: str, values: List[str] ) -> Set[str]: """ 批量查询数据库中某字段已存在的值集合 Args: db: 数据库会话 field: 要查询的字段名 values: 要检查的值列表 Returns: 数据库中已存在的值集合（字符串形式） """ if not values: return set() table = self.form_meta.main_table schema = await self._resolve_schema(db, self.form_meta.main_table_schema) or None database = self.form_meta.main_table_database or None table_name = self.sql_builder.build_table_name(table, schema, database) quoted_field = self.sql_builder.quote_identifier(field) existing: Set[str] = set() unique_values = list(dict.fromkeys(values)) for i in range(0, len(unique_values), 500): batch_vals = unique_values[i:i + 500] placeholders = ", ".join(f":v{j}" for j in range(len(batch_vals))) sql = f"SELECT {quoted_field} FROM {table_name} WHERE {quoted_field} IN ({placeholders})" params = {f"v{j}": v for j, v in enumerate(batch_vals)} if await self._check_column_exists(db, table, schema, database, "is_deleted"): sql += f" AND {self.sql_builder.is_deleted_predicate()}" result = await self._execute_query(db, sql, params) for row in result: existing.add(str(row[field] if field in row else list(row.values())[0])) return existing def _validate_data_against_schema(self, row_num: int, data: Dict[str, Any]) -> Optional[str]: """ 根据数据库表结构验证数据 Args: row_num: 行号 data: 数据字典 Returns: 错误信息，如果验证通过则返回 None """ form_config = self._form_config table_configs = form_config.get('tableConfigs', []) # 找到主表配置 main_table_config = None for tc in table_configs: if tc.get('type') == 'main': main_table_config = tc break if not main_table_config: return None fields = main_table_config.get('fields', []) field_map = {f.get('name'): f for f in fields if f.get('name')} # 验证每个字段 for field_name, value in data.items(): # 跳过系统字段 if field_name in {'id', 'sys_create_datetime', 'sys_update_datetime', 'sys_creator_id', 'sys_modifier_id', 'sys_dept_id', 'is_deleted', 'sort'}: continue field_config = field_map.get(field_name) if not field_config: continue # 1. NOT NULL 约束检查 nullable = field_config.get('nullable', True) if not nullable and (value is None or (isinstance(value, str) and value.strip() == '')): return f"字段 '{field_name}' 不能为空" # 跳过 NULL 值的其他检查 if value is None: continue # 2. 字符串长度检查 field_type = field_config.get('type', '').lower() max_length = field_config.get('maxLength') if max_length and isinstance(value, str): if 'varchar' in field_type or 'char' in field_type or 'text' in field_type: if len(value) > max_length: return f"字段 '{field_name}' 长度超限（最大 {max_length}，实际 {len(value)}）" # 3. 数值类型检查 if 'int' in field_type or 'integer' in field_type: if not isinstance(value, (int, float)): try: int(value) except (ValueError, TypeError): return f"字段 '{field_name}' 必须是整数" elif 'decimal' in field_type or 'numeric' in field_type or 'float' in field_type or 'double' in field_type: if not isinstance(value, (int, float)): try: float(value) except (ValueError, TypeError): return f"字段 '{field_name}' 必须是数值" # 精度检查 precision = field_config.get('precision') scale = field_config.get('scale') if precision and isinstance(value, (int, float)): value_str = str(value) if '.' in value_str: int_part, dec_part = value_str.split('.') if len(int_part) + len(dec_part) > precision: return f"字段 '{field_name}' 精度超限（最大 {precision}）" if scale and len(dec_part) > scale: return f"字段 '{field_name}' 小数位数超限（最大 {scale}）" # 4. 日期时间类型检查 elif 'date' in field_type or 'time' in field_type: if not isinstance(value, (date, datetime)): return f"字段 '{field_name}' 必须是日期时间类型" # 5. 布尔类型检查 elif 'bool' in field_type: if not isinstance(value, bool): return f"字段 '{field_name}' 必须是布尔类型" return None async def _batch_insert_raw( self, db: AsyncSession, table: str, schema: Optional[str], database: Optional[str], data_list: List[Dict[str, Any]] ) -> None: """ 真正的批量 INSERT：一条 SQL 插入多行数据 Args: db: 数据库会话 table: 表名 schema: Schema 名 database: 数据库名 data_list: 数据列表（已标准化） """ if not data_list: return columns = list(data_list[0].keys()) cols = ", ".join(self.sql_builder.quote_identifier(c) for c in columns) full_table = self.sql_builder.build_table_name(table, schema, database) # 构建多行 VALUES，使用命名参数 row_placeholders = [] params = {} for row_idx, data in enumerate(data_list): placeholders = [] for col in columns: param_name = f"p{row_idx}_{col}" placeholders.append(f":{param_name}") params[param_name] = data.get(col) row_placeholders.append(f"({', '.join(placeholders)})") sql = f"INSERT INTO {full_table} ({cols}) VALUES {', '.join(row_placeholders)}" await self._execute_command( db, sql, params, database=self._table_exec_database(database) ) def _normalize_data_for_insert(self, data: Dict[str, Any]) -> Dict[str, Any]: """ 标准化数据类型，确保所有值都是数据库可接受的类型 Args: data: 原始数据字典 Returns: 标准化后的数据字典 """ normalized = {} for key, value in data.items(): if value is None: normalized[key] = None elif isinstance(value, datetime): # datetime 对象保持不变，数据库驱动会正确处理 normalized[key] = value elif isinstance(value, date): # date 对象保持不变 normalized[key] = value elif isinstance(value, (int, float, bool)): # 基本类型保持不变 normalized[key] = value elif isinstance(value, (list, dict)): # 列表和字典转换为 JSON 字符串 normalized[key] = json.dumps(value, ensure_ascii=False) elif isinstance(value, str): # 字符串保持不变 normalized[key] = value else: # 其他类型转换为字符串 normalized[key] = str(value) return normalized # ============ 字段权限过滤 ============ async def _get_merged_field_permissions( self, db: AsyncSession, role_ids: List[str] = None ) -> Dict[str, Dict]: """获取当前用户合并后的字段权限配置""" from utils.context import get_current_user_info_from_context from core.resource_scope.field_permission.service import ResourceFieldPermissionService if not role_ids: user_info = get_current_user_info_from_context() logger.debug(f"[字段权限] 从上下文获取用户信息: {user_info}") if not user_info or not user_info.get('role_ids'): logger.warning("[字段权限] 未获取到用户角色信息，跳过字段权限过滤") return {} role_ids = user_info['role_ids'] resource_type = f"form:{self._form_code}" logger.debug(f"[字段权限] 资源类型: {resource_type}, 角色IDs: {role_ids}") configs = await ResourceFieldPermissionService.get_by_roles_and_resource( db, role_ids, resource_type ) logger.debug(f"[字段权限] 获取到的配置数量: {len(configs) if configs else 0}") if not configs: logger.debug("[字段权限] 未找到字段权限配置，跳过过滤") return {} merged_perms = await ResourceFieldPermissionService.merge_field_permissions( configs, "most_permissive" ) logger.debug(f"[字段权限] 合并后的权限: {merged_perms}") return merged_perms or {} async def apply_field_permissions( self, data: Any, db: AsyncSession, role_ids: List[str] = None ) -> Any: """ 应用字段权限过滤 Args: data: 数据（单个对象或列表） db: 数据库会话 role_ids: 角色ID列表，如果不传则从上下文获取 Returns: 过滤后的数据 """ merged_perms = await self._get_merged_field_permissions(db, role_ids) if not merged_perms: return data if isinstance(data, list): return [self._apply_field_permissions(item, merged_perms) for item in data] return self._apply_field_permissions(data, merged_perms) @staticmethod def _get_field_permission_type(field_name: str, field_perms: Dict[str, Dict]) -> str: """获取字段权限类型，未配置时默认可写""" if not field_perms: return 'write' perm = field_perms.get(field_name, {}) return perm.get('permission_type') or perm.get('permission', 'write') def _is_field_writable(self, field_name: str, field_perms: Dict[str, Dict]) -> bool: """判断字段是否可写""" return self._get_field_permission_type(field_name, field_perms) == 'write' def _filter_writable_fields( self, data: Dict[str, Any], allowed_fields: Set[str], field_perms: Dict[str, Dict], always_allow: Optional[Set[str]] = None, ) -> Dict[str, Any]: """过滤字段白名单，并剔除只读/隐藏/脱敏字段的写入""" filtered = self._filter_fields(data, allowed_fields) if not field_perms: return filtered allow = always_allow or set() return { key: value for key, value in filtered.items() if key in allow or self._is_field_writable(key, field_perms) } def _apply_field_permissions(self, item: Dict[str, Any], field_perms: Dict[str, Dict]) -> Dict[str, Any]: """ 应用字段权限过滤（隐藏、脱敏） Args: item: 数据项（字典） field_perms: 字段权限配置 Returns: 过滤后的字典 """ if not isinstance(item, dict): return item # 收集需要隐藏或脱敏的字段及其关联的 _name 字段 hidden_fields = set() masked_fields = {} # field_name -> mask_rule for field_name, perm in field_perms.items(): permission_type = perm.get('permission_type') or perm.get('permission', 'write') if permission_type == 'hidden': hidden_fields.add(field_name) # 同时隐藏关联的 _name 字段 # 支持多种命名模式： # - field -> field_name (如 居住地 -> 居住地_name, post_id -> post_id_name) # - field_id -> field_name (如 manger_id -> manger_name) hidden_fields.add(f"{field_name}_name") if field_name.endswith('_id'): base_name = field_name[:-3] # 去掉 _id hidden_fields.add(f"{base_name}_name") elif permission_type == 'masked': masked_fields[field_name] = perm.get('mask_rule') # 同时脱敏关联的 _name 字段（使用默认脱敏规则） masked_fields[f"{field_name}_name"] = 'default' if field_name.endswith('_id'): base_name = field_name[:-3] masked_fields[f"{base_name}_name"] = 'name' filtered = {} for field_name, value in item.items(): # 递归处理内嵌子表数据 if field_name == "sub_tables" and isinstance(value, dict): filtered["sub_tables"] = { table_name: [ self._apply_field_permissions(row, field_perms) if isinstance(row, dict) else row for row in rows ] if isinstance(rows, list) else rows for table_name, rows in value.items() } continue # 检查是否需要隐藏 if field_name in hidden_fields: continue # 检查是否需要脱敏 if field_name in masked_fields: filtered[field_name] = self._mask_value(value, masked_fields[field_name]) continue # 检查原始字段权限配置 perm = field_perms.get(field_name, {}) permission_type = perm.get('permission_type') or perm.get('permission', 'write') if permission_type == 'hidden': continue elif permission_type == 'masked': filtered[field_name] = self._mask_value(value, perm.get('mask_rule')) else: filtered[field_name] = value return filtered def _mask_value(self, value: Any, mask_rule: Optional[str]) -> str: """ 脱敏处理 Args: value: 原始值 mask_rule: 脱敏规则 Returns: 脱敏后的值 """ if not value: return value value_str = str(value) if mask_rule == "phone": # 手机号脱敏：138****5678 if len(value_str) == 11: return f"{value_str[:3]}****{value_str[-4:]}" elif mask_rule == "email": # 邮箱脱敏：abc***@example.com if "@" in value_str: local, domain = value_str.split("@", 1) if len(local) > 3: return f"{local[:3]}***@{domain}" return f"{local[0]}***@{domain}" elif mask_rule == "id_card": # 身份证脱敏：110***********1234 if len(value_str) >= 8: return f"{value_str[:3]}***********{value_str[-4:]}" elif mask_rule == "name": # 姓名脱敏：张* if len(value_str) > 1: return f"{value_str[0]}*" return "*" # 默认脱敏：显示前后各2个字符 if len(value_str) > 4: return f"{value_str[:2]}***{value_str[-2:]}" return "***"