ai-agent-admin/backend-fastapi/online_dev/workflow/engine/task_timeout_process.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
工作流任务超时处理器
负责任务超时检测、提醒、自动处理等功能
使用 APScheduler 实现定时检查
"""
import logging
from datetime import datetime, timedelta

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

logger = logging.getLogger(__name__)

TIMEOUT_CHECK_TASK_FUNC = (
    'online_dev.workflow.engine.task_timeout_process.execute_timeout_check'
)
TIMEOUT_CHECK_JOB_CODE = 'workflow_task_timeout_check'


def calculate_timeout_datetime(duration: int, unit: str) -> datetime:
    """
    计算超时时间点
    
    Args:
        duration: 超时时长
        unit: 超时单位 - minute/hour/day
        
    Returns:
        datetime: 超时时间点
    """
    now = datetime.now()

    if unit == 'minute':
        return now + timedelta(minutes=duration)
    elif unit == 'hour':
        return now + timedelta(hours=duration)
    elif unit == 'day':
        return now + timedelta(days=duration)
    else:
        # 默认按小时
        return now + timedelta(hours=duration)


async def set_task_timeout(db: AsyncSession, task, timeout_config: dict) -> bool:
    """
    为任务设置超时时间
    
    Args:
        db: 数据库会话
        task: WorkflowTask 实例
        timeout_config: 超时配置 {enabled, duration, unit, action}
        
    Returns:
        bool: 是否设置成功
    """
    if not timeout_config or not timeout_config.get('enabled'):
        return False

    try:
        duration = timeout_config.get('duration', 24)
        unit = timeout_config.get('unit', 'hour')
        action = timeout_config.get('action', 'notify')

        # 计算超时时间
        timeout_at = calculate_timeout_datetime(duration, unit)

        # 更新任务
        task.timeout_at = timeout_at
        task.timeout_action = action
        task.timeout_notified = False
        db.add(task)
        await db.flush()

        logger.info(f"任务 {task.id} 设置超时: {timeout_at}, 操作: {action}")
        return True

    except Exception as e:
        logger.error(f"设置任务超时失败: {e}")
        return False


async def check_and_handle_timeouts(db: AsyncSession):
    """
    检查并处理超时任务
    此函数应该由定时任务每分钟调用一次
    """
    from online_dev.workflow.model import WorkflowTask

    now = datetime.now()

    # 查找所有已超时但未处理的待办任务
    stmt = select(WorkflowTask).where(
        WorkflowTask.status == 'pending',
        WorkflowTask.timeout_at <= now,
        WorkflowTask.timeout_at.isnot(None),
        WorkflowTask.is_deleted == False,
    )
    result = await db.execute(stmt)
    timeout_tasks = result.scalars().all()

    logger.info(f"检查超时任务，发现 {len(timeout_tasks)} 个超时任务")

    for task in timeout_tasks:
        try:
            await _handle_timeout_task(db, task)
        except Exception as e:
            logger.exception(f"处理超时任务 {task.id} 失败: {e}")


async def _handle_timeout_task(db: AsyncSession, task):
    """
    处理单个超时任务
    """
    from online_dev.workflow.model import WorkflowInstance

    action = task.timeout_action or 'notify'

    # 获取实例
    stmt = select(WorkflowInstance).where(WorkflowInstance.id == task.instance_id)
    result = await db.execute(stmt)
    instance = result.scalar_one_or_none()

    if not instance or instance.status != 'pending':
        # 流程已结束，跳过
        return

    logger.info(f"处理超时任务: task={task.id}, action={action}")

    if action == 'notify':
        # 发送超时提醒通知
        await _send_timeout_notification(db, task, instance)

    elif action == 'auto_approve':
        # 自动通过
        await _auto_complete_task(db, task, instance, 'approve')

    elif action == 'auto_reject':
        # 自动拒绝
        await _auto_complete_task(db, task, instance, 'reject')


async def _send_timeout_notification(db: AsyncSession, task, instance):
    """
    发送超时提醒通知
    """
    from online_dev.workflow.model import WorkflowLog
    from online_dev.workflow.engine.handlers.notification_service import WorkflowNotificationService
    from core.user.model import User

    # 检查是否已发送过通知
    if task.timeout_notified:
        return

    # 获取处理人
    stmt = select(User).where(User.id == task.assignee_id)
    result = await db.execute(stmt)
    assignee = result.scalar_one_or_none()

    if not assignee:
        return

    # 获取发起人
    stmt = select(User).where(User.id == instance.initiator_id)
    result = await db.execute(stmt)
    initiator = result.scalar_one_or_none()

    try:
        # 发送超时通知
        await WorkflowNotificationService.send_timeout_notification(
            db=db,
            task=task,
            instance=instance,
            assignee=assignee,
        )

        # 标记已通知
        task.timeout_notified = True
        db.add(task)

        # 记录日志
        log = WorkflowLog(
            instance_id=str(instance.id),
            node_id=task.node_id,
            node_name=task.node_name,
            action='task_timeout_notify',
            operator_id=str(initiator.id) if initiator else '',
            comment=f'任务超时提醒已发送给 {assignee.name or assignee.username}',
            extra_data={
                'task_id': str(task.id),
                'assignee_id': str(assignee.id),
                'assignee_name': assignee.name or assignee.username,
            },
        )
        db.add(log)
        await db.flush()

        logger.info(f"已发送超时提醒: task={task.id}, assignee={assignee.id}")

    except Exception as e:
        logger.error(f"发送超时提醒失败: {e}")


async def _auto_complete_task(db: AsyncSession, task, instance, action_type: str):
    """
    自动完成超时任务
    
    Args:
        db: 数据库会话
        task: 任务
        instance: 流程实例
        action_type: 'approve' 或 'reject'
    """
    from online_dev.workflow.model import WorkflowLog, WorkflowDefinition
    from online_dev.workflow.engine.workflow_engine import WorkflowEngine
    from online_dev.workflow.engine.base import ExecutionContext
    from core.user.model import User

    # 获取发起人作为操作者
    stmt = select(User).where(User.id == instance.initiator_id)
    result = await db.execute(stmt)
    operator = result.scalar_one_or_none()

    # 确定操作类型和日志动作
    if action_type == 'approve':
        log_action = 'task_auto_approve'
        status = 'approved'
        comment = '任务超时，系统自动通过'
    else:
        log_action = 'task_auto_reject'
        status = 'rejected'
        comment = '任务超时，系统自动拒绝'

    # 更新任务状态
    task.status = status
    task.comment = comment
    task.handled_at = datetime.now()
    db.add(task)

    # 记录日志
    log = WorkflowLog(
        instance_id=str(instance.id),
        node_id=task.node_id,
        node_name=task.node_name,
        action=log_action,
        operator_id=str(operator.id) if operator else '',
        comment=comment,
        extra_data={
            'task_id': str(task.id),
            'assignee_id': str(task.assignee_id),
            'timeout_at': task.timeout_at.isoformat() if task.timeout_at else None,
        },
    )
    db.add(log)
    await db.flush()

    logger.info(f"任务自动{action_type}: task={task.id}")

    # 完成该任务对应的钉钉待办
    try:
        from core.message.service import NotifyService
        await NotifyService.complete_dingtalk_todo(db, "workflow_task", str(task.id))
    except Exception as e:
        logger.warning(f"超时自动处理-清理钉钉待办失败 task={task.id}: {e}")

    # 获取流程定义
    stmt = select(WorkflowDefinition).where(WorkflowDefinition.id == instance.workflow_id)
    result = await db.execute(stmt)
    workflow = result.scalar_one_or_none()

    if not workflow:
        return

    # 使用流程引擎推进流程
    engine = WorkflowEngine()

    operator_id = str(operator.id) if operator else ''
    context = ExecutionContext(
        instance=instance,
        form_data={},
        current_user_id=operator_id,
        flow_definition=workflow.flow_definition,
        db=db,
    )

    if action_type == 'approve':
        # 通过：按任务类型推进（办理节点走办理完成逻辑）
        if task.task_type == 'handle':
            await engine._handle_handle_completion(context, task)
        else:
            await engine._handle_approval(context, task)
    else:
        # 拒绝：结束流程
        await engine._end_instance(context, 'rejected')


async def execute_timeout_check(**kwargs):
    """
    定时任务入口函数
    由调度器每分钟调用
    """
    from app.database import AsyncSessionLocal

    logger.info("开始执行任务超时检查...")
    try:
        async with AsyncSessionLocal() as db:
            await check_and_handle_timeouts(db)
            await db.commit()
        return "任务超时检查完成"
    except Exception as e:
        logger.exception(f"任务超时检查失败: {e}")
        raise


async def create_timeout_check_job(db: AsyncSession) -> bool:
    """
    创建任务超时检查的定时任务
    每分钟执行一次
    
    Returns:
        bool: 是否创建成功
    """
    try:
        from scheduler.model import SchedulerJob
        from scheduler.service import SchedulerService

        job_code = TIMEOUT_CHECK_JOB_CODE

        # 检查是否已存在
        stmt = select(SchedulerJob).where(SchedulerJob.code == job_code)
        result = await db.execute(stmt)
        existing = result.scalar_one_or_none()

        scheduler_service = SchedulerService()

        if existing:
            updated = False
            if existing.task_func != TIMEOUT_CHECK_TASK_FUNC:
                existing.task_func = TIMEOUT_CHECK_TASK_FUNC
                updated = True
                logger.warning(
                    f'已修正任务超时检查 task_func: {TIMEOUT_CHECK_TASK_FUNC}'
                )
            if existing.status != 1:
                existing.status = 1
                updated = True
            if updated:
                db.add(existing)
                await db.flush()
            if scheduler_service.is_running():
                try:
                    await scheduler_service.remove_job(job_code)
                except Exception:
                    pass
                await scheduler_service.add_job(existing)
            logger.info(f'任务超时检查定时任务已就绪: {job_code}')
            return True

        # 创建定时任务（每分钟执行）
        job = SchedulerJob(
            name='工作流任务超时检查',
            code=job_code,
            description='每分钟检查一次超时的工作流任务，执行超时提醒或自动处理',
            group='workflow',
            trigger_type='interval',
            interval_seconds=60,  # 每60秒执行一次
            task_func=TIMEOUT_CHECK_TASK_FUNC,
            task_kwargs='{}',
            status=1,  # 启用
            max_instances=1,
            coalesce=True,
        )
        db.add(job)
        await db.flush()

        # 添加到调度器
        if scheduler_service.is_running():
            await scheduler_service.add_job(job)

        logger.info(f"创建任务超时检查定时任务成功: {job_code}")
        return True

    except Exception as e:
        logger.error(f"创建任务超时检查定时任务失败: {e}")
        return False


async def remove_timeout_check_job(db: AsyncSession) -> bool:
    """
    移除任务超时检查的定时任务
    
    Returns:
        bool: 是否移除成功
    """
    try:
        from scheduler.model import SchedulerJob
        from scheduler.service import SchedulerService

        job_code = TIMEOUT_CHECK_JOB_CODE

        # 从调度器移除
        scheduler_service = SchedulerService()
        if scheduler_service.is_running():
            await scheduler_service.remove_job(job_code)

        # 删除数据库记录
        stmt = select(SchedulerJob).where(SchedulerJob.code == job_code)
        result = await db.execute(stmt)
        job = result.scalar_one_or_none()
        if job:
            await db.delete(job)
            await db.flush()

        logger.info(f"移除任务超时检查定时任务成功: {job_code}")
        return True

    except Exception as e:
        logger.error(f"移除任务超时检查定时任务失败: {e}")
        return False