feat: restore source parity and harden agent runtime
This commit is contained in:
@@ -0,0 +1,420 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
工作流任务超时处理器
|
||||
负责任务超时检测、提醒、自动处理等功能
|
||||
使用 APScheduler 实现定时检查
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT_CHECK_TASK_FUNC = (
|
||||
'online_dev.workflow.engine.task_timeout_process.execute_timeout_check'
|
||||
)
|
||||
TIMEOUT_CHECK_JOB_CODE = 'workflow_task_timeout_check'
|
||||
|
||||
|
||||
def calculate_timeout_datetime(duration: int, unit: str) -> datetime:
|
||||
"""
|
||||
计算超时时间点
|
||||
|
||||
Args:
|
||||
duration: 超时时长
|
||||
unit: 超时单位 - minute/hour/day
|
||||
|
||||
Returns:
|
||||
datetime: 超时时间点
|
||||
"""
|
||||
now = datetime.now()
|
||||
|
||||
if unit == 'minute':
|
||||
return now + timedelta(minutes=duration)
|
||||
elif unit == 'hour':
|
||||
return now + timedelta(hours=duration)
|
||||
elif unit == 'day':
|
||||
return now + timedelta(days=duration)
|
||||
else:
|
||||
# 默认按小时
|
||||
return now + timedelta(hours=duration)
|
||||
|
||||
|
||||
async def set_task_timeout(db: AsyncSession, task, timeout_config: dict) -> bool:
|
||||
"""
|
||||
为任务设置超时时间
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
task: WorkflowTask 实例
|
||||
timeout_config: 超时配置 {enabled, duration, unit, action}
|
||||
|
||||
Returns:
|
||||
bool: 是否设置成功
|
||||
"""
|
||||
if not timeout_config or not timeout_config.get('enabled'):
|
||||
return False
|
||||
|
||||
try:
|
||||
duration = timeout_config.get('duration', 24)
|
||||
unit = timeout_config.get('unit', 'hour')
|
||||
action = timeout_config.get('action', 'notify')
|
||||
|
||||
# 计算超时时间
|
||||
timeout_at = calculate_timeout_datetime(duration, unit)
|
||||
|
||||
# 更新任务
|
||||
task.timeout_at = timeout_at
|
||||
task.timeout_action = action
|
||||
task.timeout_notified = False
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
|
||||
logger.info(f"任务 {task.id} 设置超时: {timeout_at}, 操作: {action}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"设置任务超时失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def check_and_handle_timeouts(db: AsyncSession):
|
||||
"""
|
||||
检查并处理超时任务
|
||||
此函数应该由定时任务每分钟调用一次
|
||||
"""
|
||||
from online_dev.workflow.model import WorkflowTask
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
# 查找所有已超时但未处理的待办任务
|
||||
stmt = select(WorkflowTask).where(
|
||||
WorkflowTask.status == 'pending',
|
||||
WorkflowTask.timeout_at <= now,
|
||||
WorkflowTask.timeout_at.isnot(None),
|
||||
WorkflowTask.is_deleted == False,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
timeout_tasks = result.scalars().all()
|
||||
|
||||
logger.info(f"检查超时任务，发现 {len(timeout_tasks)} 个超时任务")
|
||||
|
||||
for task in timeout_tasks:
|
||||
try:
|
||||
await _handle_timeout_task(db, task)
|
||||
except Exception as e:
|
||||
logger.exception(f"处理超时任务 {task.id} 失败: {e}")
|
||||
|
||||
|
||||
async def _handle_timeout_task(db: AsyncSession, task):
|
||||
"""
|
||||
处理单个超时任务
|
||||
"""
|
||||
from online_dev.workflow.model import WorkflowInstance
|
||||
|
||||
action = task.timeout_action or 'notify'
|
||||
|
||||
# 获取实例
|
||||
stmt = select(WorkflowInstance).where(WorkflowInstance.id == task.instance_id)
|
||||
result = await db.execute(stmt)
|
||||
instance = result.scalar_one_or_none()
|
||||
|
||||
if not instance or instance.status != 'pending':
|
||||
# 流程已结束，跳过
|
||||
return
|
||||
|
||||
logger.info(f"处理超时任务: task={task.id}, action={action}")
|
||||
|
||||
if action == 'notify':
|
||||
# 发送超时提醒通知
|
||||
await _send_timeout_notification(db, task, instance)
|
||||
|
||||
elif action == 'auto_approve':
|
||||
# 自动通过
|
||||
await _auto_complete_task(db, task, instance, 'approve')
|
||||
|
||||
elif action == 'auto_reject':
|
||||
# 自动拒绝
|
||||
await _auto_complete_task(db, task, instance, 'reject')
|
||||
|
||||
|
||||
async def _send_timeout_notification(db: AsyncSession, task, instance):
|
||||
"""
|
||||
发送超时提醒通知
|
||||
"""
|
||||
from online_dev.workflow.model import WorkflowLog
|
||||
from online_dev.workflow.engine.handlers.notification_service import WorkflowNotificationService
|
||||
from core.user.model import User
|
||||
|
||||
# 检查是否已发送过通知
|
||||
if task.timeout_notified:
|
||||
return
|
||||
|
||||
# 获取处理人
|
||||
stmt = select(User).where(User.id == task.assignee_id)
|
||||
result = await db.execute(stmt)
|
||||
assignee = result.scalar_one_or_none()
|
||||
|
||||
if not assignee:
|
||||
return
|
||||
|
||||
# 获取发起人
|
||||
stmt = select(User).where(User.id == instance.initiator_id)
|
||||
result = await db.execute(stmt)
|
||||
initiator = result.scalar_one_or_none()
|
||||
|
||||
try:
|
||||
# 发送超时通知
|
||||
await WorkflowNotificationService.send_timeout_notification(
|
||||
db=db,
|
||||
task=task,
|
||||
instance=instance,
|
||||
assignee=assignee,
|
||||
)
|
||||
|
||||
# 标记已通知
|
||||
task.timeout_notified = True
|
||||
db.add(task)
|
||||
|
||||
# 记录日志
|
||||
log = WorkflowLog(
|
||||
instance_id=str(instance.id),
|
||||
node_id=task.node_id,
|
||||
node_name=task.node_name,
|
||||
action='task_timeout_notify',
|
||||
operator_id=str(initiator.id) if initiator else '',
|
||||
comment=f'任务超时提醒已发送给 {assignee.name or assignee.username}',
|
||||
extra_data={
|
||||
'task_id': str(task.id),
|
||||
'assignee_id': str(assignee.id),
|
||||
'assignee_name': assignee.name or assignee.username,
|
||||
},
|
||||
)
|
||||
db.add(log)
|
||||
await db.flush()
|
||||
|
||||
logger.info(f"已发送超时提醒: task={task.id}, assignee={assignee.id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送超时提醒失败: {e}")
|
||||
|
||||
|
||||
async def _auto_complete_task(db: AsyncSession, task, instance, action_type: str):
|
||||
"""
|
||||
自动完成超时任务
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
task: 任务
|
||||
instance: 流程实例
|
||||
action_type: 'approve' 或 'reject'
|
||||
"""
|
||||
from online_dev.workflow.model import WorkflowLog, WorkflowDefinition
|
||||
from online_dev.workflow.engine.workflow_engine import WorkflowEngine
|
||||
from online_dev.workflow.engine.base import ExecutionContext
|
||||
from core.user.model import User
|
||||
|
||||
# 获取发起人作为操作者
|
||||
stmt = select(User).where(User.id == instance.initiator_id)
|
||||
result = await db.execute(stmt)
|
||||
operator = result.scalar_one_or_none()
|
||||
|
||||
# 确定操作类型和日志动作
|
||||
if action_type == 'approve':
|
||||
log_action = 'task_auto_approve'
|
||||
status = 'approved'
|
||||
comment = '任务超时，系统自动通过'
|
||||
else:
|
||||
log_action = 'task_auto_reject'
|
||||
status = 'rejected'
|
||||
comment = '任务超时，系统自动拒绝'
|
||||
|
||||
# 更新任务状态
|
||||
task.status = status
|
||||
task.comment = comment
|
||||
task.handled_at = datetime.now()
|
||||
db.add(task)
|
||||
|
||||
# 记录日志
|
||||
log = WorkflowLog(
|
||||
instance_id=str(instance.id),
|
||||
node_id=task.node_id,
|
||||
node_name=task.node_name,
|
||||
action=log_action,
|
||||
operator_id=str(operator.id) if operator else '',
|
||||
comment=comment,
|
||||
extra_data={
|
||||
'task_id': str(task.id),
|
||||
'assignee_id': str(task.assignee_id),
|
||||
'timeout_at': task.timeout_at.isoformat() if task.timeout_at else None,
|
||||
},
|
||||
)
|
||||
db.add(log)
|
||||
await db.flush()
|
||||
|
||||
logger.info(f"任务自动{action_type}: task={task.id}")
|
||||
|
||||
# 完成该任务对应的钉钉待办
|
||||
try:
|
||||
from core.message.service import NotifyService
|
||||
await NotifyService.complete_dingtalk_todo(db, "workflow_task", str(task.id))
|
||||
except Exception as e:
|
||||
logger.warning(f"超时自动处理-清理钉钉待办失败 task={task.id}: {e}")
|
||||
|
||||
# 获取流程定义
|
||||
stmt = select(WorkflowDefinition).where(WorkflowDefinition.id == instance.workflow_id)
|
||||
result = await db.execute(stmt)
|
||||
workflow = result.scalar_one_or_none()
|
||||
|
||||
if not workflow:
|
||||
return
|
||||
|
||||
# 使用流程引擎推进流程
|
||||
engine = WorkflowEngine()
|
||||
|
||||
operator_id = str(operator.id) if operator else ''
|
||||
context = ExecutionContext(
|
||||
instance=instance,
|
||||
form_data={},
|
||||
current_user_id=operator_id,
|
||||
flow_definition=workflow.flow_definition,
|
||||
db=db,
|
||||
)
|
||||
|
||||
if action_type == 'approve':
|
||||
# 通过：按任务类型推进（办理节点走办理完成逻辑）
|
||||
if task.task_type == 'handle':
|
||||
await engine._handle_handle_completion(context, task)
|
||||
else:
|
||||
await engine._handle_approval(context, task)
|
||||
else:
|
||||
# 拒绝：结束流程
|
||||
await engine._end_instance(context, 'rejected')
|
||||
|
||||
|
||||
async def execute_timeout_check(**kwargs):
|
||||
"""
|
||||
定时任务入口函数
|
||||
由调度器每分钟调用
|
||||
"""
|
||||
from app.database import AsyncSessionLocal
|
||||
|
||||
logger.info("开始执行任务超时检查...")
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
await check_and_handle_timeouts(db)
|
||||
await db.commit()
|
||||
return "任务超时检查完成"
|
||||
except Exception as e:
|
||||
logger.exception(f"任务超时检查失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def create_timeout_check_job(db: AsyncSession) -> bool:
|
||||
"""
|
||||
创建任务超时检查的定时任务
|
||||
每分钟执行一次
|
||||
|
||||
Returns:
|
||||
bool: 是否创建成功
|
||||
"""
|
||||
try:
|
||||
from scheduler.model import SchedulerJob
|
||||
from scheduler.service import SchedulerService
|
||||
|
||||
job_code = TIMEOUT_CHECK_JOB_CODE
|
||||
|
||||
# 检查是否已存在
|
||||
stmt = select(SchedulerJob).where(SchedulerJob.code == job_code)
|
||||
result = await db.execute(stmt)
|
||||
existing = result.scalar_one_or_none()
|
||||
|
||||
scheduler_service = SchedulerService()
|
||||
|
||||
if existing:
|
||||
updated = False
|
||||
if existing.task_func != TIMEOUT_CHECK_TASK_FUNC:
|
||||
existing.task_func = TIMEOUT_CHECK_TASK_FUNC
|
||||
updated = True
|
||||
logger.warning(
|
||||
f'已修正任务超时检查 task_func: {TIMEOUT_CHECK_TASK_FUNC}'
|
||||
)
|
||||
if existing.status != 1:
|
||||
existing.status = 1
|
||||
updated = True
|
||||
if updated:
|
||||
db.add(existing)
|
||||
await db.flush()
|
||||
if scheduler_service.is_running():
|
||||
try:
|
||||
await scheduler_service.remove_job(job_code)
|
||||
except Exception:
|
||||
pass
|
||||
await scheduler_service.add_job(existing)
|
||||
logger.info(f'任务超时检查定时任务已就绪: {job_code}')
|
||||
return True
|
||||
|
||||
# 创建定时任务（每分钟执行）
|
||||
job = SchedulerJob(
|
||||
name='工作流任务超时检查',
|
||||
code=job_code,
|
||||
description='每分钟检查一次超时的工作流任务，执行超时提醒或自动处理',
|
||||
group='workflow',
|
||||
trigger_type='interval',
|
||||
interval_seconds=60, # 每60秒执行一次
|
||||
task_func=TIMEOUT_CHECK_TASK_FUNC,
|
||||
task_kwargs='{}',
|
||||
status=1, # 启用
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
|
||||
# 添加到调度器
|
||||
if scheduler_service.is_running():
|
||||
await scheduler_service.add_job(job)
|
||||
|
||||
logger.info(f"创建任务超时检查定时任务成功: {job_code}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建任务超时检查定时任务失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def remove_timeout_check_job(db: AsyncSession) -> bool:
|
||||
"""
|
||||
移除任务超时检查的定时任务
|
||||
|
||||
Returns:
|
||||
bool: 是否移除成功
|
||||
"""
|
||||
try:
|
||||
from scheduler.model import SchedulerJob
|
||||
from scheduler.service import SchedulerService
|
||||
|
||||
job_code = TIMEOUT_CHECK_JOB_CODE
|
||||
|
||||
# 从调度器移除
|
||||
scheduler_service = SchedulerService()
|
||||
if scheduler_service.is_running():
|
||||
await scheduler_service.remove_job(job_code)
|
||||
|
||||
# 删除数据库记录
|
||||
stmt = select(SchedulerJob).where(SchedulerJob.code == job_code)
|
||||
result = await db.execute(stmt)
|
||||
job = result.scalar_one_or_none()
|
||||
if job:
|
||||
await db.delete(job)
|
||||
await db.flush()
|
||||
|
||||
logger.info(f"移除任务超时检查定时任务成功: {job_code}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"移除任务超时检查定时任务失败: {e}")
|
||||
return False
|
||||
Reference in New Issue
Block a user