Build lightweight AI agent admin
This commit is contained in:
Codex
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Module - 定时任务模块
|
||||
基于 APScheduler 实现的定时任务调度
|
||||
"""
|
||||
@@ -0,0 +1,714 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler API - 定时任务管理接口
|
||||
提供定时任务的 CRUD 操作和管理功能
|
||||
"""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import select, func, or_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.config import settings
|
||||
from app.base_schema import PaginatedResponse, ResponseModel
|
||||
from scheduler.model import SchedulerJob, SchedulerLog
|
||||
from scheduler.schema import (
|
||||
SchedulerJobCreate,
|
||||
SchedulerJobUpdate,
|
||||
SchedulerJobResponse,
|
||||
SchedulerJobSimple,
|
||||
SchedulerJobBatchDeleteIn,
|
||||
SchedulerJobBatchDeleteOut,
|
||||
SchedulerJobBatchUpdateStatusIn,
|
||||
SchedulerJobBatchUpdateStatusOut,
|
||||
SchedulerJobExecuteIn,
|
||||
SchedulerJobExecuteOut,
|
||||
SchedulerJobStatisticsOut,
|
||||
SchedulerJobSearchRequest,
|
||||
SchedulerLogResponse,
|
||||
SchedulerLogBatchDeleteIn,
|
||||
SchedulerLogBatchDeleteOut,
|
||||
SchedulerLogCleanIn,
|
||||
SchedulerLogCleanOut,
|
||||
SchedulerStatusOut,
|
||||
)
|
||||
from scheduler.service import scheduler_service
|
||||
|
||||
router = APIRouter(prefix="/scheduler", tags=["定时任务管理"])
|
||||
|
||||
|
||||
def _build_job_response(job: SchedulerJob) -> SchedulerJobResponse:
|
||||
"""构建任务响应"""
|
||||
return SchedulerJobResponse(
|
||||
id=job.id,
|
||||
application_id=job.application_id,
|
||||
name=job.name,
|
||||
code=job.code,
|
||||
description=job.description,
|
||||
group=job.group,
|
||||
trigger_type=job.trigger_type,
|
||||
trigger_type_display=job.get_trigger_type_display(),
|
||||
cron_expression=job.cron_expression,
|
||||
interval_seconds=job.interval_seconds,
|
||||
run_date=job.run_date,
|
||||
task_func=job.task_func,
|
||||
task_args=job.task_args,
|
||||
task_kwargs=job.task_kwargs,
|
||||
status=job.status,
|
||||
status_display=job.get_status_display(),
|
||||
priority=job.priority,
|
||||
max_instances=job.max_instances,
|
||||
max_retries=job.max_retries,
|
||||
timeout=job.timeout,
|
||||
coalesce=job.coalesce,
|
||||
allow_concurrent=job.allow_concurrent,
|
||||
total_run_count=job.total_run_count,
|
||||
success_count=job.success_count,
|
||||
failure_count=job.failure_count,
|
||||
success_rate=job.get_success_rate(),
|
||||
last_run_time=job.last_run_time,
|
||||
next_run_time=job.next_run_time,
|
||||
last_run_status=job.last_run_status,
|
||||
last_run_result=job.last_run_result,
|
||||
remark=job.remark,
|
||||
sort=job.sort,
|
||||
sys_create_datetime=job.sys_create_datetime,
|
||||
sys_update_datetime=job.sys_update_datetime,
|
||||
)
|
||||
|
||||
|
||||
def _build_log_response(log: SchedulerLog) -> SchedulerLogResponse:
|
||||
"""构建日志响应"""
|
||||
return SchedulerLogResponse(
|
||||
id=log.id,
|
||||
job_id=log.job_id,
|
||||
job_name=log.job_name,
|
||||
job_code=log.job_code,
|
||||
status=log.status,
|
||||
status_display=log.get_status_display(),
|
||||
start_time=log.start_time,
|
||||
end_time=log.end_time,
|
||||
duration=log.duration,
|
||||
result=log.result,
|
||||
exception=log.exception,
|
||||
traceback=log.traceback,
|
||||
hostname=log.hostname,
|
||||
process_id=log.process_id,
|
||||
retry_count=log.retry_count,
|
||||
sys_create_datetime=log.sys_create_datetime,
|
||||
)
|
||||
|
||||
|
||||
# ==================== SchedulerJob APIs ====================
|
||||
|
||||
@router.post("/job", response_model=SchedulerJobResponse, summary="创建定时任务")
|
||||
async def create_scheduler_job(data: SchedulerJobCreate, db: AsyncSession = Depends(get_db)):
|
||||
"""创建新的定时任务"""
|
||||
# 检查任务编码是否已存在
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(
|
||||
SchedulerJob.code == data.code,
|
||||
SchedulerJob.is_deleted == False # noqa: E712
|
||||
)
|
||||
)
|
||||
if result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail=f"任务编码已存在: {data.code}")
|
||||
|
||||
# 创建任务
|
||||
job = SchedulerJob(**data.model_dump())
|
||||
db.add(job)
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
|
||||
# 如果任务是启用状态，添加到调度器
|
||||
if job.is_enabled() and scheduler_service.is_running():
|
||||
await scheduler_service.add_job(job)
|
||||
|
||||
return _build_job_response(job)
|
||||
|
||||
|
||||
@router.get("/job/all", response_model=List[SchedulerJobSimple], summary="获取所有定时任务（简化版）")
|
||||
async def get_all_scheduler_jobs(
|
||||
application_id: str = Query(None, alias="applicationId", description="所属应用ID"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""获取所有定时任务（不分页，简化版）"""
|
||||
conditions = [SchedulerJob.is_deleted == False] # noqa: E712
|
||||
if application_id:
|
||||
conditions.append(SchedulerJob.application_id == application_id)
|
||||
else:
|
||||
conditions.append(SchedulerJob.application_id.is_(None))
|
||||
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(*conditions).order_by(SchedulerJob.priority.desc(), SchedulerJob.name)
|
||||
)
|
||||
jobs = result.scalars().all()
|
||||
return jobs
|
||||
|
||||
|
||||
@router.get("/job", response_model=PaginatedResponse[SchedulerJobResponse], summary="获取定时任务列表")
|
||||
async def get_scheduler_job_list(
|
||||
page: int = Query(default=1, ge=1, description="页码"),
|
||||
page_size: int = Query(default=settings.PAGE_SIZE, ge=1, le=settings.PAGE_MAX_SIZE, alias="pageSize", description="每页数量"),
|
||||
application_id: str = Query(None, alias="applicationId", description="所属应用ID"),
|
||||
name: Optional[str] = Query(default=None, description="任务名称"),
|
||||
code: Optional[str] = Query(default=None, description="任务编码"),
|
||||
group: Optional[str] = Query(default=None, description="任务分组"),
|
||||
trigger_type: Optional[str] = Query(default=None, description="触发器类型"),
|
||||
status: Optional[int] = Query(default=None, description="任务状态"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""获取定时任务列表（分页）"""
|
||||
filters = [SchedulerJob.is_deleted == False] # noqa: E712
|
||||
# 应用过滤
|
||||
if application_id:
|
||||
filters.append(SchedulerJob.application_id == application_id)
|
||||
else:
|
||||
filters.append(SchedulerJob.application_id.is_(None))
|
||||
if name:
|
||||
filters.append(SchedulerJob.name.ilike(f"%{name}%"))
|
||||
if code:
|
||||
filters.append(SchedulerJob.code.ilike(f"%{code}%"))
|
||||
if group:
|
||||
filters.append(SchedulerJob.group == group)
|
||||
if trigger_type:
|
||||
filters.append(SchedulerJob.trigger_type == trigger_type)
|
||||
if status is not None:
|
||||
filters.append(SchedulerJob.status == status)
|
||||
|
||||
# 查询总数
|
||||
count_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(*filters)
|
||||
)
|
||||
total = count_result.scalar()
|
||||
|
||||
# 查询数据
|
||||
offset = (page - 1) * page_size
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(*filters)
|
||||
.order_by(SchedulerJob.priority.desc(), SchedulerJob.sys_update_datetime.desc())
|
||||
.offset(offset).limit(page_size)
|
||||
)
|
||||
jobs = result.scalars().all()
|
||||
|
||||
return PaginatedResponse(
|
||||
items=[_build_job_response(job) for job in jobs],
|
||||
total=total
|
||||
)
|
||||
|
||||
|
||||
@router.post("/job/batch/delete", response_model=SchedulerJobBatchDeleteOut, summary="批量删除定时任务")
|
||||
async def batch_delete_scheduler_jobs(
|
||||
data: SchedulerJobBatchDeleteIn,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""批量删除定时任务"""
|
||||
success_count = 0
|
||||
failed_ids = []
|
||||
|
||||
for job_id in data.ids:
|
||||
try:
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id == job_id)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if job:
|
||||
# 从调度器移除
|
||||
if scheduler_service.is_running():
|
||||
await scheduler_service.remove_job(job.code)
|
||||
|
||||
# 软删除
|
||||
job.is_deleted = True
|
||||
success_count += 1
|
||||
else:
|
||||
failed_ids.append(job_id)
|
||||
except Exception:
|
||||
failed_ids.append(job_id)
|
||||
|
||||
await db.commit()
|
||||
return SchedulerJobBatchDeleteOut(count=success_count, failed_ids=failed_ids)
|
||||
|
||||
|
||||
@router.post("/job/batch/update_status", response_model=SchedulerJobBatchUpdateStatusOut, summary="批量更新任务状态")
|
||||
async def batch_update_scheduler_job_status(
|
||||
data: SchedulerJobBatchUpdateStatusIn,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""批量启用、禁用或暂停任务"""
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id.in_(data.ids))
|
||||
)
|
||||
jobs = result.scalars().all()
|
||||
|
||||
count = 0
|
||||
for job in jobs:
|
||||
job.status = data.status
|
||||
count += 1
|
||||
|
||||
# 同步更新调度器
|
||||
if scheduler_service.is_running():
|
||||
if job.is_enabled():
|
||||
await scheduler_service.add_job(job)
|
||||
elif job.is_paused():
|
||||
await scheduler_service.pause_job(job.code)
|
||||
else:
|
||||
await scheduler_service.remove_job(job.code)
|
||||
|
||||
await db.commit()
|
||||
return SchedulerJobBatchUpdateStatusOut(count=count)
|
||||
|
||||
|
||||
@router.post("/job/execute", response_model=SchedulerJobExecuteOut, summary="立即执行任务")
|
||||
async def execute_scheduler_job(
|
||||
data: SchedulerJobExecuteIn,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""立即执行指定任务（不影响正常调度）"""
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id == data.job_id)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
|
||||
if not scheduler_service.is_running():
|
||||
raise HTTPException(status_code=400, detail="调度器未运行")
|
||||
|
||||
# 立即执行任务
|
||||
success = await scheduler_service.run_job_now(job.code)
|
||||
|
||||
if success:
|
||||
return SchedulerJobExecuteOut(
|
||||
success=True,
|
||||
message=f"任务 {job.name} 将立即执行"
|
||||
)
|
||||
else:
|
||||
return SchedulerJobExecuteOut(
|
||||
success=False,
|
||||
message=f"任务 {job.name} 执行失败，可能任务未在调度器中"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/job/search", response_model=PaginatedResponse[SchedulerJobResponse], summary="搜索定时任务")
|
||||
async def search_scheduler_jobs(
|
||||
data: SchedulerJobSearchRequest,
|
||||
page: int = Query(default=1, ge=1, description="页码"),
|
||||
page_size: int = Query(default=settings.PAGE_SIZE, ge=1, le=settings.PAGE_MAX_SIZE, alias="pageSize", description="每页数量"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""搜索定时任务"""
|
||||
keyword = data.keyword
|
||||
filters = [
|
||||
SchedulerJob.is_deleted == False, # noqa: E712
|
||||
or_(
|
||||
SchedulerJob.name.ilike(f"%{keyword}%"),
|
||||
SchedulerJob.code.ilike(f"%{keyword}%"),
|
||||
SchedulerJob.description.ilike(f"%{keyword}%"),
|
||||
)
|
||||
]
|
||||
|
||||
# 查询总数
|
||||
count_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(*filters)
|
||||
)
|
||||
total = count_result.scalar()
|
||||
|
||||
# 查询数据
|
||||
offset = (page - 1) * page_size
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(*filters)
|
||||
.order_by(SchedulerJob.priority.desc())
|
||||
.offset(offset).limit(page_size)
|
||||
)
|
||||
jobs = result.scalars().all()
|
||||
|
||||
return PaginatedResponse(
|
||||
items=[_build_job_response(job) for job in jobs],
|
||||
total=total
|
||||
)
|
||||
|
||||
|
||||
@router.get("/job/statistics/data", response_model=SchedulerJobStatisticsOut, summary="获取任务统计信息")
|
||||
async def get_scheduler_job_statistics(db: AsyncSession = Depends(get_db)):
|
||||
"""获取任务统计信息"""
|
||||
# 任务统计
|
||||
total_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(SchedulerJob.is_deleted == False) # noqa: E712
|
||||
)
|
||||
total_jobs = total_result.scalar() or 0
|
||||
|
||||
enabled_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(
|
||||
SchedulerJob.is_deleted == False, # noqa: E712
|
||||
SchedulerJob.status == 1
|
||||
)
|
||||
)
|
||||
enabled_jobs = enabled_result.scalar() or 0
|
||||
|
||||
disabled_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(
|
||||
SchedulerJob.is_deleted == False, # noqa: E712
|
||||
SchedulerJob.status == 0
|
||||
)
|
||||
)
|
||||
disabled_jobs = disabled_result.scalar() or 0
|
||||
|
||||
paused_result = await db.execute(
|
||||
select(func.count(SchedulerJob.id)).where(
|
||||
SchedulerJob.is_deleted == False, # noqa: E712
|
||||
SchedulerJob.status == 2
|
||||
)
|
||||
)
|
||||
paused_jobs = paused_result.scalar() or 0
|
||||
|
||||
# 执行统计
|
||||
total_exec_result = await db.execute(
|
||||
select(func.count(SchedulerLog.id))
|
||||
)
|
||||
total_executions = total_exec_result.scalar() or 0
|
||||
|
||||
success_exec_result = await db.execute(
|
||||
select(func.count(SchedulerLog.id)).where(SchedulerLog.status == 'success')
|
||||
)
|
||||
success_executions = success_exec_result.scalar() or 0
|
||||
|
||||
failed_exec_result = await db.execute(
|
||||
select(func.count(SchedulerLog.id)).where(SchedulerLog.status == 'failed')
|
||||
)
|
||||
failed_executions = failed_exec_result.scalar() or 0
|
||||
|
||||
# 计算成功率
|
||||
success_rate = round(success_executions / total_executions * 100, 2) if total_executions > 0 else 0
|
||||
|
||||
return SchedulerJobStatisticsOut(
|
||||
total_jobs=total_jobs,
|
||||
enabled_jobs=enabled_jobs,
|
||||
disabled_jobs=disabled_jobs,
|
||||
paused_jobs=paused_jobs,
|
||||
total_executions=total_executions,
|
||||
success_executions=success_executions,
|
||||
failed_executions=failed_executions,
|
||||
success_rate=success_rate,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/job/{job_id}", response_model=SchedulerJobResponse, summary="获取定时任务详情")
|
||||
async def get_scheduler_job(job_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""获取单个定时任务的详细信息"""
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id == job_id)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
|
||||
return _build_job_response(job)
|
||||
|
||||
|
||||
@router.put("/job/{job_id}", response_model=SchedulerJobResponse, summary="更新定时任务")
|
||||
async def update_scheduler_job(
|
||||
job_id: str,
|
||||
data: SchedulerJobUpdate,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""更新定时任务"""
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id == job_id)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
|
||||
# 检查任务编码是否已存在（排除自身）
|
||||
if data.code:
|
||||
code_result = await db.execute(
|
||||
select(SchedulerJob).where(
|
||||
SchedulerJob.code == data.code,
|
||||
SchedulerJob.id != job_id,
|
||||
SchedulerJob.is_deleted == False # noqa: E712
|
||||
)
|
||||
)
|
||||
if code_result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail=f"任务编码已存在: {data.code}")
|
||||
|
||||
# 更新字段
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
for key, value in update_data.items():
|
||||
setattr(job, key, value)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(job)
|
||||
|
||||
# 同步更新调度器
|
||||
if scheduler_service.is_running():
|
||||
await scheduler_service.modify_job(job)
|
||||
|
||||
return _build_job_response(job)
|
||||
|
||||
|
||||
@router.delete("/job/{job_id}", response_model=ResponseModel, summary="删除定时任务")
|
||||
async def delete_scheduler_job(
|
||||
job_id: str,
|
||||
hard: bool = Query(default=False, description="是否物理删除"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""删除定时任务"""
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.id == job_id)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
|
||||
# 从调度器移除
|
||||
if scheduler_service.is_running():
|
||||
await scheduler_service.remove_job(job.code)
|
||||
|
||||
if hard:
|
||||
await db.delete(job)
|
||||
else:
|
||||
job.is_deleted = True
|
||||
|
||||
await db.commit()
|
||||
return ResponseModel(message="删除成功")
|
||||
|
||||
|
||||
# ==================== SchedulerLog APIs ====================
|
||||
|
||||
@router.get("/log", response_model=PaginatedResponse[SchedulerLogResponse], summary="获取任务执行日志列表")
|
||||
async def get_scheduler_log_list(
|
||||
page: int = Query(default=1, ge=1, description="页码"),
|
||||
page_size: int = Query(default=settings.PAGE_SIZE, ge=1, le=settings.PAGE_MAX_SIZE, alias="pageSize", description="每页数量"),
|
||||
job_id: Optional[str] = Query(default=None, description="任务ID"),
|
||||
job_code: Optional[str] = Query(default=None, description="任务编码"),
|
||||
job_name: Optional[str] = Query(default=None, description="任务名称"),
|
||||
status: Optional[str] = Query(default=None, description="执行状态"),
|
||||
start_time_gte: Optional[datetime] = Query(default=None, alias="startTimeGte", description="开始时间>="),
|
||||
start_time_lte: Optional[datetime] = Query(default=None, alias="startTimeLte", description="开始时间<="),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""获取任务执行日志列表（分页）"""
|
||||
filters = []
|
||||
if job_id:
|
||||
filters.append(SchedulerLog.job_id == job_id)
|
||||
if job_code:
|
||||
filters.append(SchedulerLog.job_code.ilike(f"%{job_code}%"))
|
||||
if job_name:
|
||||
filters.append(SchedulerLog.job_name.ilike(f"%{job_name}%"))
|
||||
if status:
|
||||
filters.append(SchedulerLog.status == status)
|
||||
if start_time_gte:
|
||||
filters.append(SchedulerLog.start_time >= start_time_gte)
|
||||
if start_time_lte:
|
||||
filters.append(SchedulerLog.start_time <= start_time_lte)
|
||||
|
||||
# 查询总数
|
||||
count_result = await db.execute(
|
||||
select(func.count(SchedulerLog.id)).where(*filters) if filters else select(func.count(SchedulerLog.id))
|
||||
)
|
||||
total = count_result.scalar()
|
||||
|
||||
# 查询数据
|
||||
offset = (page - 1) * page_size
|
||||
query = select(SchedulerLog).order_by(SchedulerLog.start_time.desc()).offset(offset).limit(page_size)
|
||||
if filters:
|
||||
query = select(SchedulerLog).where(*filters).order_by(SchedulerLog.start_time.desc()).offset(offset).limit(page_size)
|
||||
|
||||
result = await db.execute(query)
|
||||
logs = result.scalars().all()
|
||||
|
||||
return PaginatedResponse(
|
||||
items=[_build_log_response(log) for log in logs],
|
||||
total=total
|
||||
)
|
||||
|
||||
|
||||
@router.get("/log/by/job/{job_id}", response_model=PaginatedResponse[SchedulerLogResponse], summary="获取指定任务的执行日志")
|
||||
async def get_scheduler_logs_by_job(
|
||||
job_id: str,
|
||||
page: int = Query(default=1, ge=1, description="页码"),
|
||||
page_size: int = Query(default=settings.PAGE_SIZE, ge=1, le=settings.PAGE_MAX_SIZE, alias="pageSize", description="每页数量"),
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""获取指定任务的所有执行日志"""
|
||||
# 查询总数
|
||||
count_result = await db.execute(
|
||||
select(func.count(SchedulerLog.id)).where(SchedulerLog.job_id == job_id)
|
||||
)
|
||||
total = count_result.scalar()
|
||||
|
||||
# 查询数据
|
||||
offset = (page - 1) * page_size
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.job_id == job_id)
|
||||
.order_by(SchedulerLog.start_time.desc())
|
||||
.offset(offset).limit(page_size)
|
||||
)
|
||||
logs = result.scalars().all()
|
||||
|
||||
return PaginatedResponse(
|
||||
items=[_build_log_response(log) for log in logs],
|
||||
total=total
|
||||
)
|
||||
|
||||
|
||||
@router.get("/log/{log_id}", response_model=SchedulerLogResponse, summary="获取任务执行日志详情")
|
||||
async def get_scheduler_log(log_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""获取单个任务执行日志的详细信息"""
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.id == log_id)
|
||||
)
|
||||
log = result.scalar_one_or_none()
|
||||
|
||||
if not log:
|
||||
raise HTTPException(status_code=404, detail="日志不存在")
|
||||
|
||||
return _build_log_response(log)
|
||||
|
||||
|
||||
@router.delete("/log/{log_id}", response_model=ResponseModel, summary="删除任务执行日志")
|
||||
async def delete_scheduler_log(log_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""删除任务执行日志"""
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.id == log_id)
|
||||
)
|
||||
log = result.scalar_one_or_none()
|
||||
|
||||
if not log:
|
||||
raise HTTPException(status_code=404, detail="日志不存在")
|
||||
|
||||
await db.delete(log)
|
||||
await db.commit()
|
||||
return ResponseModel(message="删除成功")
|
||||
|
||||
|
||||
@router.post("/log/batch/delete", response_model=SchedulerLogBatchDeleteOut, summary="批量删除任务执行日志")
|
||||
async def batch_delete_scheduler_logs(
|
||||
data: SchedulerLogBatchDeleteIn,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""批量删除任务执行日志"""
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.id.in_(data.ids))
|
||||
)
|
||||
logs = result.scalars().all()
|
||||
|
||||
count = len(logs)
|
||||
for log in logs:
|
||||
await db.delete(log)
|
||||
|
||||
await db.commit()
|
||||
return SchedulerLogBatchDeleteOut(count=count)
|
||||
|
||||
|
||||
@router.post("/log/clean", response_model=SchedulerLogCleanOut, summary="清理旧日志")
|
||||
async def clean_scheduler_logs(
|
||||
data: SchedulerLogCleanIn,
|
||||
db: AsyncSession = Depends(get_db)
|
||||
):
|
||||
"""清理旧日志"""
|
||||
cutoff_date = datetime.now() - timedelta(days=data.days)
|
||||
|
||||
filters = [SchedulerLog.start_time < cutoff_date]
|
||||
if data.status:
|
||||
filters.append(SchedulerLog.status == data.status)
|
||||
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(*filters)
|
||||
)
|
||||
logs = result.scalars().all()
|
||||
|
||||
count = len(logs)
|
||||
for log in logs:
|
||||
await db.delete(log)
|
||||
|
||||
await db.commit()
|
||||
return SchedulerLogCleanOut(count=count)
|
||||
|
||||
|
||||
# ==================== Scheduler Control APIs ====================
|
||||
|
||||
@router.post("/start", response_model=ResponseModel, summary="启动调度器")
|
||||
async def start_scheduler():
|
||||
"""启动调度器（注意：调度器应在应用启动时自动启动）"""
|
||||
if scheduler_service.is_running():
|
||||
raise HTTPException(status_code=400, detail="调度器已在运行中")
|
||||
|
||||
# APScheduler 4.x 中调度器应在 lifespan 中启动
|
||||
raise HTTPException(status_code=400, detail="请重启应用以启动调度器")
|
||||
|
||||
|
||||
@router.post("/shutdown", response_model=ResponseModel, summary="关闭调度器")
|
||||
async def shutdown_scheduler():
|
||||
"""关闭调度器（注意：调度器会在应用关闭时自动关闭）"""
|
||||
if not scheduler_service.is_running():
|
||||
raise HTTPException(status_code=400, detail="调度器未运行")
|
||||
|
||||
# APScheduler 4.x 中调度器应在 lifespan 中关闭
|
||||
raise HTTPException(status_code=400, detail="请关闭应用以停止调度器")
|
||||
|
||||
|
||||
@router.post("/pause", response_model=ResponseModel, summary="暂停调度器")
|
||||
async def pause_scheduler():
|
||||
"""暂停调度器（暂不支持）"""
|
||||
raise HTTPException(status_code=400, detail="APScheduler 4.x 暂不支持暂停整个调度器")
|
||||
|
||||
|
||||
@router.post("/resume", response_model=ResponseModel, summary="恢复调度器")
|
||||
async def resume_scheduler():
|
||||
"""恢复调度器（暂不支持）"""
|
||||
raise HTTPException(status_code=400, detail="APScheduler 4.x 暂不支持恢复整个调度器")
|
||||
|
||||
|
||||
@router.get("/status", response_model=SchedulerStatusOut, summary="获取调度器状态")
|
||||
async def get_scheduler_status():
|
||||
"""获取调度器状态"""
|
||||
is_running = scheduler_service.is_running()
|
||||
jobs = await scheduler_service.get_all_jobs() if is_running else []
|
||||
|
||||
return SchedulerStatusOut(
|
||||
is_running=is_running,
|
||||
job_count=len(jobs),
|
||||
jobs=jobs,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/log/{log_id}/stream", summary="实时日志流")
|
||||
async def stream_task_log(log_id: str):
|
||||
"""
|
||||
通过 SSE 实时推送任务执行日志
|
||||
|
||||
Args:
|
||||
log_id: 日志记录 ID
|
||||
|
||||
Returns:
|
||||
SSE 事件流，包含任务执行过程中的日志消息
|
||||
"""
|
||||
from fastapi.responses import StreamingResponse
|
||||
from scheduler.task_log_service import TaskLogService
|
||||
import json
|
||||
|
||||
async def event_generator():
|
||||
try:
|
||||
async for log_entry in TaskLogService.subscribe(log_id):
|
||||
yield f"data: {json.dumps(log_entry, ensure_ascii=False)}\n\n"
|
||||
except Exception as e:
|
||||
yield f"data: {json.dumps({'level': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Model - 定时任务模型
|
||||
用于管理定时任务和执行记录
|
||||
"""
|
||||
from sqlalchemy import Column, String, Integer, Boolean, Text, DateTime, Float, Index
|
||||
|
||||
from app.base_model import BaseModel
|
||||
|
||||
|
||||
class SchedulerJob(BaseModel):
|
||||
"""
|
||||
定时任务模型 - 用于管理定时任务配置
|
||||
|
||||
功能特点：
|
||||
1. 支持多种触发器类型（cron、interval、date）
|
||||
2. 支持任务启用/禁用
|
||||
3. 支持任务分组管理
|
||||
4. 支持任务优先级
|
||||
5. 记录任务执行统计信息
|
||||
6. 支持任务参数配置
|
||||
"""
|
||||
__tablename__ = "core_scheduler_job"
|
||||
|
||||
# 所属应用（逻辑外键关联 core_application）
|
||||
application_id = Column(String(21), nullable=True, index=True, comment="所属应用ID")
|
||||
|
||||
# 任务类型选择
|
||||
TRIGGER_TYPE_CHOICES = {
|
||||
'cron': 'Cron表达式',
|
||||
'interval': '间隔执行',
|
||||
'date': '指定时间',
|
||||
}
|
||||
|
||||
# 任务状态选择
|
||||
STATUS_CHOICES = {
|
||||
0: '禁用',
|
||||
1: '启用',
|
||||
2: '暂停',
|
||||
}
|
||||
|
||||
# 任务名称
|
||||
name = Column(String(128), nullable=False, index=True, comment="任务名称")
|
||||
|
||||
# 任务编码（唯一标识）
|
||||
code = Column(String(128), unique=True, nullable=False, index=True, comment="任务编码")
|
||||
|
||||
# 任务描述
|
||||
description = Column(Text, nullable=True, comment="任务描述")
|
||||
|
||||
# 任务分组
|
||||
group = Column(String(64), default='default', index=True, comment="任务分组")
|
||||
|
||||
# 触发器类型
|
||||
trigger_type = Column(String(20), default='cron', index=True, comment="触发器类型")
|
||||
|
||||
# Cron 表达式（用于 cron 类型）
|
||||
cron_expression = Column(String(128), nullable=True, comment="Cron表达式")
|
||||
|
||||
# 间隔时间（秒，用于 interval 类型）
|
||||
interval_seconds = Column(Integer, nullable=True, comment="间隔时间（秒）")
|
||||
|
||||
# 指定执行时间（用于 date 类型）
|
||||
run_date = Column(DateTime, nullable=True, comment="指定执行时间")
|
||||
|
||||
# 任务函数路径（如：scheduler.tasks.test_task）
|
||||
task_func = Column(String(256), nullable=False, comment="任务函数路径")
|
||||
|
||||
# 任务参数（JSON格式）
|
||||
task_args = Column(Text, nullable=True, comment="任务位置参数（JSON数组格式）")
|
||||
|
||||
# 任务关键字参数（JSON格式）
|
||||
task_kwargs = Column(Text, nullable=True, comment="任务关键字参数（JSON对象格式）")
|
||||
|
||||
# 任务状态
|
||||
status = Column(Integer, default=0, index=True, comment="任务状态（0-禁用，1-启用，2-暂停）")
|
||||
|
||||
# 任务优先级（数字越大优先级越高）
|
||||
priority = Column(Integer, default=0, index=True, comment="任务优先级")
|
||||
|
||||
# 最大实例数（同时运行的任务实例数）
|
||||
max_instances = Column(Integer, default=1, comment="最大实例数")
|
||||
|
||||
# 错误重试次数
|
||||
max_retries = Column(Integer, default=0, comment="错误重试次数")
|
||||
|
||||
# 超时时间（秒）
|
||||
timeout = Column(Integer, nullable=True, comment="超时时间（秒）")
|
||||
|
||||
# 是否合并执行（如果上次未执行完，是否跳过本次）
|
||||
coalesce = Column(Boolean, default=True, comment="是否合并执行")
|
||||
|
||||
# 是否允许并发执行
|
||||
allow_concurrent = Column(Boolean, default=False, comment="是否允许并发执行")
|
||||
|
||||
# 执行统计
|
||||
total_run_count = Column(Integer, default=0, comment="总执行次数")
|
||||
success_count = Column(Integer, default=0, comment="成功次数")
|
||||
failure_count = Column(Integer, default=0, comment="失败次数")
|
||||
|
||||
# 最后执行时间
|
||||
last_run_time = Column(DateTime, nullable=True, comment="最后执行时间")
|
||||
|
||||
# 下次执行时间
|
||||
next_run_time = Column(DateTime, nullable=True, comment="下次执行时间")
|
||||
|
||||
# 最后执行状态
|
||||
last_run_status = Column(String(20), nullable=True, comment="最后执行状态")
|
||||
|
||||
# 最后执行结果
|
||||
last_run_result = Column(Text, nullable=True, comment="最后执行结果")
|
||||
|
||||
# 备注
|
||||
remark = Column(Text, nullable=True, comment="备注信息")
|
||||
|
||||
# 复合索引
|
||||
__table_args__ = (
|
||||
Index('ix_scheduler_job_status_trigger', 'status', 'trigger_type'),
|
||||
Index('ix_scheduler_job_group_status', 'group', 'status'),
|
||||
Index('ix_scheduler_job_priority_status', 'priority', 'status'),
|
||||
Index('ix_scheduler_job_next_run_status', 'next_run_time', 'status'),
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name} ({self.code})"
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""判断任务是否启用"""
|
||||
return self.status == 1
|
||||
|
||||
def is_paused(self) -> bool:
|
||||
"""判断任务是否暂停"""
|
||||
return self.status == 2
|
||||
|
||||
def is_disabled(self) -> bool:
|
||||
"""判断任务是否禁用"""
|
||||
return self.status == 0
|
||||
|
||||
def get_status_display(self) -> str:
|
||||
"""获取状态的显示名称"""
|
||||
return self.STATUS_CHOICES.get(self.status, '未知')
|
||||
|
||||
def get_trigger_type_display(self) -> str:
|
||||
"""获取触发器类型的显示名称"""
|
||||
return self.TRIGGER_TYPE_CHOICES.get(self.trigger_type, '未知')
|
||||
|
||||
def get_success_rate(self) -> float:
|
||||
"""获取成功率"""
|
||||
if self.total_run_count == 0:
|
||||
return 0.0
|
||||
return round(self.success_count / self.total_run_count * 100, 2)
|
||||
|
||||
|
||||
class SchedulerLog(BaseModel):
|
||||
"""
|
||||
定时任务执行日志模型
|
||||
|
||||
功能特点：
|
||||
1. 记录每次任务执行的详细信息
|
||||
2. 记录执行时间、状态、结果
|
||||
3. 记录异常信息
|
||||
4. 支持日志查询和统计
|
||||
"""
|
||||
__tablename__ = "core_scheduler_log"
|
||||
|
||||
# 执行状态选择
|
||||
STATUS_CHOICES = {
|
||||
'pending': '等待执行',
|
||||
'running': '执行中',
|
||||
'success': '执行成功',
|
||||
'failed': '执行失败',
|
||||
'timeout': '执行超时',
|
||||
'skipped': '跳过执行',
|
||||
}
|
||||
|
||||
# 关联的任务ID（逻辑外键）
|
||||
job_id = Column(String(36), nullable=False, index=True, comment="任务ID")
|
||||
|
||||
# 任务名称（冗余字段，便于查询）
|
||||
job_name = Column(String(128), nullable=False, index=True, comment="任务名称")
|
||||
|
||||
# 任务编码（冗余字段，便于查询）
|
||||
job_code = Column(String(128), nullable=False, index=True, comment="任务编码")
|
||||
|
||||
# 执行状态
|
||||
status = Column(String(20), default='pending', index=True, comment="执行状态")
|
||||
|
||||
# 开始时间
|
||||
start_time = Column(DateTime, nullable=False, index=True, comment="开始时间")
|
||||
|
||||
# 结束时间
|
||||
end_time = Column(DateTime, nullable=True, comment="结束时间")
|
||||
|
||||
# 执行耗时（秒）
|
||||
duration = Column(Float, nullable=True, comment="执行耗时（秒）")
|
||||
|
||||
# 执行结果
|
||||
result = Column(Text, nullable=True, comment="执行结果")
|
||||
|
||||
# 异常信息
|
||||
exception = Column(Text, nullable=True, comment="异常信息")
|
||||
|
||||
# 异常堆栈
|
||||
traceback = Column(Text, nullable=True, comment="异常堆栈")
|
||||
|
||||
# 执行主机
|
||||
hostname = Column(String(128), nullable=True, comment="执行主机")
|
||||
|
||||
# 进程ID
|
||||
process_id = Column(Integer, nullable=True, comment="进程ID")
|
||||
|
||||
# 重试次数
|
||||
retry_count = Column(Integer, default=0, comment="重试次数")
|
||||
|
||||
# 复合索引
|
||||
__table_args__ = (
|
||||
Index('ix_scheduler_log_job_status', 'job_id', 'status'),
|
||||
Index('ix_scheduler_log_status_start', 'status', 'start_time'),
|
||||
Index('ix_scheduler_log_code_start', 'job_code', 'start_time'),
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.job_name} - {self.status} - {self.start_time}"
|
||||
|
||||
def is_success(self) -> bool:
|
||||
"""判断是否执行成功"""
|
||||
return self.status == 'success'
|
||||
|
||||
def is_failed(self) -> bool:
|
||||
"""判断是否执行失败"""
|
||||
return self.status == 'failed'
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""判断是否正在执行"""
|
||||
return self.status == 'running'
|
||||
|
||||
def get_status_display(self) -> str:
|
||||
"""获取状态的显示名称"""
|
||||
return self.STATUS_CHOICES.get(self.status, '未知')
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Router - 定时任务模块路由
|
||||
"""
|
||||
from fastapi import APIRouter
|
||||
|
||||
from scheduler.api import router as scheduler_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# 注册调度器路由
|
||||
router.include_router(scheduler_router)
|
||||
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Schema - 定时任务数据验证和序列化
|
||||
"""
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from app.base_schema import CSTDatetime
|
||||
|
||||
|
||||
# ==================== SchedulerJob Schemas ====================
|
||||
|
||||
class SchedulerJobBase(BaseModel):
|
||||
"""定时任务基础Schema"""
|
||||
application_id: Optional[str] = Field(None, description="所属应用ID")
|
||||
name: str = Field(..., min_length=1, max_length=128, description="任务名称")
|
||||
code: str = Field(..., min_length=1, max_length=128, description="任务编码")
|
||||
description: Optional[str] = Field(None, description="任务描述")
|
||||
group: str = Field(default="default", max_length=64, description="任务分组")
|
||||
trigger_type: str = Field(..., description="触发器类型：cron/interval/date")
|
||||
cron_expression: Optional[str] = Field(None, max_length=128, description="Cron表达式")
|
||||
interval_seconds: Optional[int] = Field(None, ge=1, description="间隔时间（秒）")
|
||||
run_date: Optional[datetime] = Field(None, description="指定执行时间")
|
||||
task_func: str = Field(..., max_length=256, description="任务函数路径")
|
||||
task_args: Optional[str] = Field(None, description="任务位置参数（JSON）")
|
||||
task_kwargs: Optional[str] = Field(None, description="任务关键字参数（JSON）")
|
||||
status: int = Field(default=0, description="任务状态：0-禁用，1-启用，2-暂停")
|
||||
priority: int = Field(default=0, description="任务优先级")
|
||||
max_instances: int = Field(default=1, ge=1, description="最大实例数")
|
||||
max_retries: int = Field(default=0, ge=0, description="错误重试次数")
|
||||
timeout: Optional[int] = Field(None, ge=1, description="超时时间（秒）")
|
||||
coalesce: bool = Field(default=True, description="是否合并执行")
|
||||
allow_concurrent: bool = Field(default=False, description="是否允许并发执行")
|
||||
remark: Optional[str] = Field(None, description="备注信息")
|
||||
sort: int = Field(default=0, description="排序")
|
||||
|
||||
@field_validator('trigger_type')
|
||||
@classmethod
|
||||
def validate_trigger_type(cls, v):
|
||||
"""验证触发器类型"""
|
||||
if v not in ['cron', 'interval', 'date']:
|
||||
raise ValueError('触发器类型必须是 cron、interval 或 date')
|
||||
return v
|
||||
|
||||
@field_validator('status')
|
||||
@classmethod
|
||||
def validate_status(cls, v):
|
||||
"""验证状态"""
|
||||
if v not in [0, 1, 2]:
|
||||
raise ValueError('状态必须是 0（禁用）、1（启用）或 2（暂停）')
|
||||
return v
|
||||
|
||||
@field_validator('code')
|
||||
@classmethod
|
||||
def validate_code(cls, v):
|
||||
"""验证任务编码格式"""
|
||||
if not v:
|
||||
raise ValueError('任务编码不能为空')
|
||||
if not v.replace('_', '').isalnum():
|
||||
raise ValueError('任务编码只能包含字母、数字和下划线')
|
||||
return v
|
||||
|
||||
|
||||
class SchedulerJobCreate(SchedulerJobBase):
|
||||
"""定时任务创建Schema"""
|
||||
|
||||
@field_validator('cron_expression')
|
||||
@classmethod
|
||||
def validate_cron_expression(cls, v, info):
|
||||
"""验证 Cron 表达式"""
|
||||
if info.data.get('trigger_type') == 'cron' and not v:
|
||||
raise ValueError('Cron 类型任务必须提供 cron_expression')
|
||||
return v
|
||||
|
||||
@field_validator('interval_seconds')
|
||||
@classmethod
|
||||
def validate_interval_seconds(cls, v, info):
|
||||
"""验证间隔时间"""
|
||||
if info.data.get('trigger_type') == 'interval' and not v:
|
||||
raise ValueError('Interval 类型任务必须提供 interval_seconds')
|
||||
return v
|
||||
|
||||
@field_validator('run_date')
|
||||
@classmethod
|
||||
def validate_run_date(cls, v, info):
|
||||
"""验证指定时间"""
|
||||
if info.data.get('trigger_type') == 'date' and not v:
|
||||
raise ValueError('Date 类型任务必须提供 run_date')
|
||||
return v
|
||||
|
||||
|
||||
class SchedulerJobUpdate(BaseModel):
|
||||
"""定时任务更新Schema - 所有字段可选"""
|
||||
application_id: Optional[str] = Field(None, description="所属应用ID")
|
||||
name: Optional[str] = Field(None, min_length=1, max_length=128, description="任务名称")
|
||||
code: Optional[str] = Field(None, min_length=1, max_length=128, description="任务编码")
|
||||
description: Optional[str] = Field(None, description="任务描述")
|
||||
group: Optional[str] = Field(None, max_length=64, description="任务分组")
|
||||
trigger_type: Optional[str] = Field(None, description="触发器类型")
|
||||
cron_expression: Optional[str] = Field(None, max_length=128, description="Cron表达式")
|
||||
interval_seconds: Optional[int] = Field(None, ge=1, description="间隔时间（秒）")
|
||||
run_date: Optional[datetime] = Field(None, description="指定执行时间")
|
||||
task_func: Optional[str] = Field(None, max_length=256, description="任务函数路径")
|
||||
task_args: Optional[str] = Field(None, description="任务位置参数（JSON）")
|
||||
task_kwargs: Optional[str] = Field(None, description="任务关键字参数（JSON）")
|
||||
status: Optional[int] = Field(None, description="任务状态")
|
||||
priority: Optional[int] = Field(None, description="任务优先级")
|
||||
max_instances: Optional[int] = Field(None, ge=1, description="最大实例数")
|
||||
max_retries: Optional[int] = Field(None, ge=0, description="错误重试次数")
|
||||
timeout: Optional[int] = Field(None, ge=1, description="超时时间（秒）")
|
||||
coalesce: Optional[bool] = Field(None, description="是否合并执行")
|
||||
allow_concurrent: Optional[bool] = Field(None, description="是否允许并发执行")
|
||||
remark: Optional[str] = Field(None, description="备注信息")
|
||||
sort: Optional[int] = Field(None, description="排序")
|
||||
|
||||
|
||||
class SchedulerJobResponse(BaseModel):
|
||||
"""定时任务响应Schema"""
|
||||
id: str
|
||||
application_id: Optional[str] = None
|
||||
name: str
|
||||
code: str
|
||||
description: Optional[str] = None
|
||||
group: str
|
||||
trigger_type: str
|
||||
trigger_type_display: Optional[str] = None
|
||||
cron_expression: Optional[str] = None
|
||||
interval_seconds: Optional[int] = None
|
||||
run_date: Optional[CSTDatetime] = None
|
||||
task_func: str
|
||||
task_args: Optional[str] = None
|
||||
task_kwargs: Optional[str] = None
|
||||
status: int
|
||||
status_display: Optional[str] = None
|
||||
priority: int
|
||||
max_instances: int
|
||||
max_retries: int
|
||||
timeout: Optional[int] = None
|
||||
coalesce: bool
|
||||
allow_concurrent: bool
|
||||
total_run_count: int
|
||||
success_count: int
|
||||
failure_count: int
|
||||
success_rate: Optional[float] = None
|
||||
last_run_time: Optional[CSTDatetime] = None
|
||||
next_run_time: Optional[CSTDatetime] = None
|
||||
last_run_status: Optional[str] = None
|
||||
last_run_result: Optional[str] = None
|
||||
remark: Optional[str] = None
|
||||
sort: int = 0
|
||||
sys_create_datetime: Optional[CSTDatetime] = None
|
||||
sys_update_datetime: Optional[CSTDatetime] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class SchedulerJobSimple(BaseModel):
|
||||
"""定时任务简化输出（用于选择器）"""
|
||||
id: str
|
||||
application_id: Optional[str] = None
|
||||
name: str
|
||||
code: str
|
||||
group: str
|
||||
status: int
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class SchedulerJobBatchDeleteIn(BaseModel):
|
||||
"""批量删除输入"""
|
||||
ids: List[str] = Field(..., description="任务ID列表")
|
||||
|
||||
|
||||
class SchedulerJobBatchDeleteOut(BaseModel):
|
||||
"""批量删除输出"""
|
||||
count: int = Field(..., description="删除成功数量")
|
||||
failed_ids: List[str] = Field(default=[], description="删除失败的ID列表")
|
||||
|
||||
|
||||
class SchedulerJobBatchUpdateStatusIn(BaseModel):
|
||||
"""批量更新状态输入"""
|
||||
ids: List[str] = Field(..., description="任务ID列表")
|
||||
status: int = Field(..., description="目标状态：0-禁用，1-启用，2-暂停")
|
||||
|
||||
|
||||
class SchedulerJobBatchUpdateStatusOut(BaseModel):
|
||||
"""批量更新状态输出"""
|
||||
count: int = Field(..., description="更新成功数量")
|
||||
|
||||
|
||||
class SchedulerJobExecuteIn(BaseModel):
|
||||
"""立即执行任务输入"""
|
||||
job_id: str = Field(..., description="任务ID")
|
||||
|
||||
|
||||
class SchedulerJobExecuteOut(BaseModel):
|
||||
"""立即执行任务输出"""
|
||||
success: bool = Field(..., description="是否成功")
|
||||
message: str = Field(..., description="消息")
|
||||
log_id: Optional[str] = Field(None, description="日志ID")
|
||||
|
||||
|
||||
class SchedulerJobStatisticsOut(BaseModel):
|
||||
"""任务统计输出"""
|
||||
total_jobs: int = Field(..., description="总任务数")
|
||||
enabled_jobs: int = Field(..., description="启用任务数")
|
||||
disabled_jobs: int = Field(..., description="禁用任务数")
|
||||
paused_jobs: int = Field(..., description="暂停任务数")
|
||||
total_executions: int = Field(..., description="总执行次数")
|
||||
success_executions: int = Field(..., description="成功执行次数")
|
||||
failed_executions: int = Field(..., description="失败执行次数")
|
||||
success_rate: float = Field(..., description="成功率")
|
||||
|
||||
|
||||
class SchedulerJobSearchRequest(BaseModel):
|
||||
"""搜索任务请求"""
|
||||
keyword: str = Field(..., description="搜索关键词")
|
||||
|
||||
|
||||
# ==================== SchedulerLog Schemas ====================
|
||||
|
||||
class SchedulerLogResponse(BaseModel):
|
||||
"""定时任务日志响应Schema"""
|
||||
id: str
|
||||
job_id: str
|
||||
job_name: str
|
||||
job_code: str
|
||||
status: str
|
||||
status_display: Optional[str] = None
|
||||
start_time: CSTDatetime
|
||||
end_time: Optional[CSTDatetime] = None
|
||||
duration: Optional[float] = None
|
||||
result: Optional[str] = None
|
||||
exception: Optional[str] = None
|
||||
traceback: Optional[str] = None
|
||||
hostname: Optional[str] = None
|
||||
process_id: Optional[int] = None
|
||||
retry_count: int
|
||||
sys_create_datetime: Optional[CSTDatetime] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class SchedulerLogBatchDeleteIn(BaseModel):
|
||||
"""批量删除日志输入"""
|
||||
ids: List[str] = Field(..., description="日志ID列表")
|
||||
|
||||
|
||||
class SchedulerLogBatchDeleteOut(BaseModel):
|
||||
"""批量删除日志输出"""
|
||||
count: int = Field(..., description="删除成功数量")
|
||||
|
||||
|
||||
class SchedulerLogCleanIn(BaseModel):
|
||||
"""清理日志输入"""
|
||||
days: int = Field(..., ge=1, description="保留最近N天的日志")
|
||||
status: Optional[str] = Field(None, description="只清理指定状态的日志")
|
||||
|
||||
|
||||
class SchedulerLogCleanOut(BaseModel):
|
||||
"""清理日志输出"""
|
||||
count: int = Field(..., description="清理数量")
|
||||
|
||||
|
||||
# ==================== Scheduler Status Schemas ====================
|
||||
|
||||
class SchedulerStatusOut(BaseModel):
|
||||
"""调度器状态输出"""
|
||||
is_running: bool = Field(..., description="是否运行中")
|
||||
job_count: int = Field(..., description="任务数量")
|
||||
jobs: List[dict] = Field(default=[], description="任务列表")
|
||||
@@ -0,0 +1,554 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Service - APScheduler 4.x 调度服务
|
||||
基于 APScheduler 4.x 实现的定时任务调度核心服务
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
from apscheduler import AsyncScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.date import DateTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
"""
|
||||
定时任务调度服务 (APScheduler 4.x)
|
||||
|
||||
功能特点：
|
||||
1. 使用 APScheduler 4.x 的 AsyncScheduler
|
||||
2. 支持多种触发器类型（cron、interval、date）
|
||||
3. 自动从数据库加载任务
|
||||
4. 通过事件订阅监听任务执行
|
||||
5. 自动更新任务状态和记录日志
|
||||
"""
|
||||
|
||||
_instance = None
|
||||
_scheduler: Optional[AsyncScheduler] = None
|
||||
_running: bool = False
|
||||
|
||||
def __new__(cls):
|
||||
"""单例模式"""
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def get_scheduler(self) -> Optional[AsyncScheduler]:
|
||||
"""获取调度器实例"""
|
||||
return self._scheduler
|
||||
|
||||
def set_scheduler(self, scheduler: AsyncScheduler):
|
||||
"""设置调度器实例"""
|
||||
self._scheduler = scheduler
|
||||
self._running = True
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""判断调度器是否运行中"""
|
||||
return self._running and self._scheduler is not None
|
||||
|
||||
def set_running(self, running: bool):
|
||||
"""设置运行状态"""
|
||||
self._running = running
|
||||
|
||||
async def load_jobs_from_db(self):
|
||||
"""从数据库加载所有启用的任务"""
|
||||
logger.info("开始从数据库加载任务...")
|
||||
|
||||
if not self._scheduler:
|
||||
logger.warning("调度器未初始化，无法加载任务")
|
||||
return
|
||||
|
||||
try:
|
||||
from sqlalchemy import select, update
|
||||
from app.database import AsyncSessionLocal
|
||||
from scheduler.model import SchedulerJob, SchedulerLog
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 注意：启动时处理 running 日志的逻辑已移除
|
||||
# 因为可能会因为数据库锁而阻塞启动
|
||||
# running 状态的日志会在前端显示为"执行中"，用户可手动处理
|
||||
|
||||
# 获取所有启用的任务
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(
|
||||
SchedulerJob.status == 1,
|
||||
SchedulerJob.is_deleted == False # noqa: E712
|
||||
)
|
||||
)
|
||||
jobs = result.scalars().all()
|
||||
|
||||
logger.info(f"查询到 {len(jobs)} 个启用的任务")
|
||||
|
||||
for job in jobs:
|
||||
try:
|
||||
success = await self.add_job(job)
|
||||
if success:
|
||||
logger.info(f"加载任务成功: {job.code}")
|
||||
else:
|
||||
logger.warning(f"加载任务返回失败: {job.code}")
|
||||
except Exception as e:
|
||||
logger.error(f"加载任务失败 {job.code}: {str(e)}")
|
||||
|
||||
# 启动定期清理任务
|
||||
await self._start_cleanup_job()
|
||||
|
||||
logger.info(f"任务加载完成，共加载 {len(jobs)} 个任务")
|
||||
except Exception as e:
|
||||
logger.error(f"从数据库加载任务失败: {str(e)}")
|
||||
|
||||
async def _start_cleanup_job(self):
|
||||
"""启动定期清理过期任务的定时任务"""
|
||||
if not self._scheduler:
|
||||
return
|
||||
|
||||
cleanup_job_id = '_scheduler_cleanup'
|
||||
|
||||
# 先注册任务
|
||||
await self._scheduler.configure_task(cleanup_job_id, func=self._cleanup_expired_jobs_wrapper)
|
||||
|
||||
# 每天凌晨3点清理过期任务
|
||||
await self._scheduler.add_schedule(
|
||||
func_or_task_id=cleanup_job_id,
|
||||
trigger=CronTrigger(hour=3, minute=0),
|
||||
id=cleanup_job_id,
|
||||
)
|
||||
logger.info("定期清理任务已启动（每天凌晨3点）")
|
||||
|
||||
async def _cleanup_expired_jobs_wrapper(self):
|
||||
"""清理过期任务的包装函数"""
|
||||
await self.cleanup_expired_jobs(days=7)
|
||||
|
||||
async def add_job(self, job_obj) -> bool:
|
||||
"""添加任务到调度器"""
|
||||
if not self._scheduler:
|
||||
logger.error("调度器未初始化")
|
||||
return False
|
||||
|
||||
try:
|
||||
# 构建触发器
|
||||
trigger = self._build_trigger(job_obj)
|
||||
if not trigger:
|
||||
logger.error(f"无法为任务 {job_obj.code} 构建触发器")
|
||||
return False
|
||||
|
||||
# 导入任务函数
|
||||
task_func = self._import_task_func(job_obj.task_func)
|
||||
if not task_func:
|
||||
logger.error(f"无法导入任务函数: {job_obj.task_func}")
|
||||
return False
|
||||
|
||||
# 解析任务参数
|
||||
args = json.loads(job_obj.task_args) if job_obj.task_args else []
|
||||
kwargs = json.loads(job_obj.task_kwargs) if job_obj.task_kwargs else {}
|
||||
kwargs['job_code'] = job_obj.code
|
||||
|
||||
# 创建带日志记录的包装函数
|
||||
wrapper_func = self._create_job_wrapper(task_func, job_obj.code, args, kwargs)
|
||||
|
||||
# APScheduler 4.x: 需要先使用 configure_task 注册任务
|
||||
task_id = job_obj.code
|
||||
await self._scheduler.configure_task(task_id, func=wrapper_func)
|
||||
|
||||
# 添加任务调度
|
||||
await self._scheduler.add_schedule(
|
||||
func_or_task_id=task_id,
|
||||
trigger=trigger,
|
||||
id=job_obj.code,
|
||||
)
|
||||
|
||||
logger.info(f"任务 {job_obj.code} 已添加到调度器")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"添加任务失败 {job_obj.code}: {str(e)}")
|
||||
return False
|
||||
|
||||
def _create_job_wrapper(self, task_func, job_code: str, args: list, kwargs: dict):
|
||||
"""创建带日志记录的任务包装函数"""
|
||||
async def wrapper():
|
||||
return await self._execute_job(task_func, job_code, args, kwargs)
|
||||
return wrapper
|
||||
|
||||
async def _execute_job(self, task_func, job_code: str, args: list, kwargs: dict):
|
||||
"""执行任务并记录日志"""
|
||||
from app.database import AsyncSessionLocal
|
||||
from scheduler.model import SchedulerJob, SchedulerLog
|
||||
from scheduler.task_log_service import TaskLogService, TaskLogger
|
||||
from sqlalchemy import select
|
||||
|
||||
start_time = datetime.now()
|
||||
exception_info = None
|
||||
result = None
|
||||
log_id = None
|
||||
job_name = job_code
|
||||
|
||||
# 在任务开始执行时创建日志记录（状态为 running）
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 获取任务
|
||||
query_result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.code == job_code)
|
||||
)
|
||||
job_obj = query_result.scalar_one_or_none()
|
||||
|
||||
if job_obj:
|
||||
job_name = job_obj.name
|
||||
# 创建执行日志（状态为 running）
|
||||
log = SchedulerLog(
|
||||
job_id=job_obj.id,
|
||||
job_name=job_obj.name,
|
||||
job_code=job_obj.code,
|
||||
start_time=start_time,
|
||||
status='running',
|
||||
hostname=socket.gethostname(),
|
||||
process_id=os.getpid(),
|
||||
)
|
||||
|
||||
# 更新任务状态为执行中
|
||||
job_obj.last_run_status = 'running'
|
||||
|
||||
db.add(log)
|
||||
await db.commit()
|
||||
await db.refresh(log)
|
||||
log_id = log.id
|
||||
|
||||
# 发布任务开始事件
|
||||
await TaskLogService.publish_start(log_id, job_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建任务执行日志失败: {str(e)}")
|
||||
|
||||
# 创建 TaskLogger 并注入到 kwargs
|
||||
task_logger = None
|
||||
if log_id:
|
||||
task_logger = TaskLogger(str(log_id), job_name)
|
||||
kwargs['task_logger'] = task_logger
|
||||
|
||||
# 执行任务
|
||||
try:
|
||||
if args:
|
||||
result = await task_func(*args, **kwargs)
|
||||
else:
|
||||
result = await task_func(**kwargs)
|
||||
except Exception as e:
|
||||
exception_info = e
|
||||
logger.error(f"任务 {job_code} 执行失败: {str(e)}")
|
||||
|
||||
end_time = datetime.now()
|
||||
|
||||
# 发布完成/错误事件
|
||||
if log_id:
|
||||
if exception_info:
|
||||
await TaskLogService.publish_error(str(log_id), job_name, str(exception_info))
|
||||
else:
|
||||
await TaskLogService.publish_complete(str(log_id), job_name, str(result) if result else None)
|
||||
|
||||
# 更新日志和任务状态
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 获取任务
|
||||
query_result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.code == job_code)
|
||||
)
|
||||
job_obj = query_result.scalar_one_or_none()
|
||||
|
||||
if job_obj:
|
||||
# 更新执行日志
|
||||
if log_id:
|
||||
log_result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.id == log_id)
|
||||
)
|
||||
log = log_result.scalar_one_or_none()
|
||||
|
||||
if log:
|
||||
log.end_time = end_time
|
||||
log.duration = (end_time - start_time).total_seconds()
|
||||
|
||||
if exception_info:
|
||||
log.status = 'failed'
|
||||
log.exception = str(exception_info)
|
||||
import traceback
|
||||
log.traceback = traceback.format_exc()
|
||||
else:
|
||||
log.status = 'success'
|
||||
log.result = str(result) if result else None
|
||||
|
||||
if exception_info:
|
||||
# 执行失败
|
||||
job_obj.last_run_status = 'failed'
|
||||
job_obj.last_run_result = str(exception_info)
|
||||
job_obj.failure_count += 1
|
||||
else:
|
||||
# 执行成功
|
||||
job_obj.last_run_status = 'success'
|
||||
job_obj.last_run_result = str(result) if result else None
|
||||
job_obj.success_count += 1
|
||||
|
||||
job_obj.total_run_count += 1
|
||||
job_obj.last_run_time = end_time
|
||||
|
||||
# 一次性任务（date 类型）执行后更新状态为禁用，但不删除
|
||||
if job_obj.trigger_type == 'date':
|
||||
job_obj.status = 0 # 禁用
|
||||
job_obj.next_run_time = None
|
||||
|
||||
await db.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新任务执行日志失败: {str(e)}")
|
||||
|
||||
if exception_info:
|
||||
raise exception_info
|
||||
|
||||
return result
|
||||
|
||||
async def remove_job(self, job_code: str) -> bool:
|
||||
"""从调度器移除任务"""
|
||||
if not self._scheduler:
|
||||
return False
|
||||
|
||||
try:
|
||||
await self._scheduler.remove_schedule(job_code)
|
||||
logger.info(f"任务 {job_code} 已从调度器移除")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"移除任务失败 {job_code}: {str(e)}")
|
||||
return False
|
||||
|
||||
async def pause_job(self, job_code: str) -> bool:
|
||||
"""暂停任务"""
|
||||
if not self._scheduler:
|
||||
return False
|
||||
|
||||
try:
|
||||
await self._scheduler.pause_schedule(job_code)
|
||||
logger.info(f"任务 {job_code} 已暂停")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"暂停任务失败 {job_code}: {str(e)}")
|
||||
return False
|
||||
|
||||
async def resume_job(self, job_code: str) -> bool:
|
||||
"""恢复任务"""
|
||||
if not self._scheduler:
|
||||
return False
|
||||
|
||||
try:
|
||||
await self._scheduler.unpause_schedule(job_code)
|
||||
logger.info(f"任务 {job_code} 已恢复")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"恢复任务失败 {job_code}: {str(e)}")
|
||||
return False
|
||||
|
||||
async def modify_job(self, job_obj) -> bool:
|
||||
"""修改任务"""
|
||||
try:
|
||||
# 先移除旧任务
|
||||
await self.remove_job(job_obj.code)
|
||||
|
||||
# 如果任务是启用状态，重新添加
|
||||
if job_obj.is_enabled():
|
||||
return await self.add_job(job_obj)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"修改任务失败 {job_obj.code}: {str(e)}")
|
||||
return False
|
||||
|
||||
async def run_job_now(self, job_code: str) -> bool:
|
||||
"""立即执行任务"""
|
||||
if not self._scheduler:
|
||||
return False
|
||||
|
||||
try:
|
||||
# 在 APScheduler 4.x 中，使用 run_job 立即执行
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from scheduler.model import SchedulerJob
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(SchedulerJob.code == job_code)
|
||||
)
|
||||
job_obj = result.scalar_one_or_none()
|
||||
|
||||
if job_obj:
|
||||
# 解析任务参数
|
||||
args = json.loads(job_obj.task_args) if job_obj.task_args else []
|
||||
kwargs = json.loads(job_obj.task_kwargs) if job_obj.task_kwargs else {}
|
||||
kwargs['job_code'] = job_obj.code
|
||||
|
||||
# 导入并执行任务函数
|
||||
task_func = self._import_task_func(job_obj.task_func)
|
||||
if task_func:
|
||||
await self._execute_job(task_func, job_code, args, kwargs)
|
||||
logger.info(f"任务 {job_code} 已立即执行")
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"立即执行任务失败 {job_code}: {str(e)}")
|
||||
return False
|
||||
|
||||
async def get_job_info(self, job_code: str) -> Optional[Dict[str, Any]]:
|
||||
"""获取任务信息"""
|
||||
if not self._scheduler:
|
||||
return None
|
||||
|
||||
try:
|
||||
schedules = await self._scheduler.get_schedules()
|
||||
for schedule in schedules:
|
||||
if schedule.id == job_code:
|
||||
return {
|
||||
'id': schedule.id,
|
||||
'next_run_time': schedule.next_fire_time.isoformat() if schedule.next_fire_time else None,
|
||||
'trigger': str(schedule.trigger),
|
||||
}
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"获取任务信息失败 {job_code}: {str(e)}")
|
||||
return None
|
||||
|
||||
async def get_all_jobs(self) -> List[Dict[str, Any]]:
|
||||
"""获取所有任务"""
|
||||
if not self._scheduler:
|
||||
return []
|
||||
|
||||
try:
|
||||
schedules = await self._scheduler.get_schedules()
|
||||
return [
|
||||
{
|
||||
'id': schedule.id,
|
||||
'next_run_time': schedule.next_fire_time.isoformat() if schedule.next_fire_time else None,
|
||||
'trigger': str(schedule.trigger),
|
||||
}
|
||||
for schedule in schedules
|
||||
]
|
||||
except Exception as e:
|
||||
logger.error(f"获取所有任务失败: {str(e)}")
|
||||
return []
|
||||
|
||||
def _build_trigger(self, job_obj):
|
||||
"""构建触发器"""
|
||||
try:
|
||||
if job_obj.trigger_type == 'cron':
|
||||
# Cron 触发器
|
||||
parts = job_obj.cron_expression.split()
|
||||
if len(parts) != 5:
|
||||
logger.error(f"Cron 表达式格式错误: {job_obj.cron_expression}")
|
||||
return None
|
||||
|
||||
return CronTrigger(
|
||||
minute=parts[0],
|
||||
hour=parts[1],
|
||||
day=parts[2],
|
||||
month=parts[3],
|
||||
day_of_week=parts[4],
|
||||
)
|
||||
|
||||
elif job_obj.trigger_type == 'interval':
|
||||
# 间隔触发器
|
||||
return IntervalTrigger(seconds=job_obj.interval_seconds)
|
||||
|
||||
elif job_obj.trigger_type == 'date':
|
||||
# 指定时间触发器
|
||||
return DateTrigger(run_time=job_obj.run_date)
|
||||
|
||||
else:
|
||||
logger.error(f"不支持的触发器类型: {job_obj.trigger_type}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"构建触发器失败: {str(e)}")
|
||||
return None
|
||||
|
||||
def _import_task_func(self, task_path: str):
|
||||
"""动态导入任务函数"""
|
||||
try:
|
||||
module_path, func_name = task_path.rsplit('.', 1)
|
||||
module = __import__(module_path, fromlist=[func_name])
|
||||
return getattr(module, func_name)
|
||||
except Exception as e:
|
||||
logger.error(f"导入任务函数失败 {task_path}: {str(e)}")
|
||||
return None
|
||||
|
||||
async def _cleanup_one_time_job(self, db, job_obj):
|
||||
"""清理一次性任务"""
|
||||
try:
|
||||
job_code = job_obj.code
|
||||
|
||||
# 从调度器移除
|
||||
try:
|
||||
await self._scheduler.remove_schedule(job_code)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 软删除数据库记录
|
||||
job_obj.is_deleted = True
|
||||
await db.commit()
|
||||
|
||||
logger.debug(f"一次性任务已清理: {job_code}")
|
||||
except Exception as e:
|
||||
logger.error(f"清理一次性任务失败 {job_obj.code}: {str(e)}")
|
||||
|
||||
async def cleanup_expired_jobs(self, days: int = 7) -> int:
|
||||
"""
|
||||
清理过期的一次性任务
|
||||
|
||||
Args:
|
||||
days: 清理多少天前的任务，默认7天
|
||||
"""
|
||||
try:
|
||||
from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
from app.database import AsyncSessionLocal
|
||||
from scheduler.model import SchedulerJob
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 查找过期的一次性任务
|
||||
result = await db.execute(
|
||||
select(SchedulerJob).where(
|
||||
SchedulerJob.trigger_type == 'date',
|
||||
SchedulerJob.run_date < cutoff,
|
||||
SchedulerJob.is_deleted == False # noqa: E712
|
||||
)
|
||||
)
|
||||
expired_jobs = result.scalars().all()
|
||||
|
||||
count = len(expired_jobs)
|
||||
if count > 0:
|
||||
for job in expired_jobs:
|
||||
# 从调度器移除
|
||||
try:
|
||||
await self._scheduler.remove_schedule(job.code)
|
||||
except Exception:
|
||||
pass
|
||||
# 软删除
|
||||
job.is_deleted = True
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"清理了 {count} 个过期的一次性任务")
|
||||
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"清理过期任务失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
|
||||
# 全局调度器服务实例
|
||||
scheduler_service = SchedulerService()
|
||||
@@ -0,0 +1,320 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
任务执行日志服务
|
||||
通过 Redis Pub/Sub 实现任务执行过程中的实时日志推送
|
||||
"""
|
||||
import json
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Optional, AsyncGenerator
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
|
||||
from utils.redis import RedisClient
|
||||
|
||||
|
||||
class LogLevel(str, Enum):
|
||||
"""日志级别"""
|
||||
DEBUG = "debug"
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
ERROR = "error"
|
||||
SUCCESS = "success"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskLogEntry:
|
||||
"""任务日志条目"""
|
||||
timestamp: str
|
||||
level: str
|
||||
message: str
|
||||
step: Optional[str] = None
|
||||
progress: Optional[int] = None # 0-100
|
||||
data: Optional[dict] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
result = asdict(self)
|
||||
# 移除 None 值
|
||||
return {k: v for k, v in result.items() if v is not None}
|
||||
|
||||
|
||||
class TaskLogService:
|
||||
"""
|
||||
任务日志服务
|
||||
|
||||
用于在任务执行过程中发布实时日志，前端通过 SSE 订阅
|
||||
使用 Redis List 存储历史日志 + Pub/Sub 推送新消息
|
||||
"""
|
||||
|
||||
# Redis 频道前缀
|
||||
CHANNEL_PREFIX = "scheduler:task_log:"
|
||||
# Redis List 前缀（存储历史日志）
|
||||
LIST_PREFIX = "scheduler:task_log_history:"
|
||||
# 日志过期时间（秒）- 1小时
|
||||
LOG_EXPIRE_SECONDS = 3600
|
||||
|
||||
@classmethod
|
||||
def _get_channel(cls, log_id: str) -> str:
|
||||
"""获取 Redis Pub/Sub 频道名称"""
|
||||
return f"{cls.CHANNEL_PREFIX}{log_id}"
|
||||
|
||||
@classmethod
|
||||
def _get_list_key(cls, log_id: str) -> str:
|
||||
"""获取 Redis List 键名"""
|
||||
return f"{cls.LIST_PREFIX}{log_id}"
|
||||
|
||||
@classmethod
|
||||
async def publish(
|
||||
cls,
|
||||
log_id: str,
|
||||
message: str,
|
||||
level: LogLevel = LogLevel.INFO,
|
||||
step: Optional[str] = None,
|
||||
progress: Optional[int] = None,
|
||||
data: Optional[dict] = None
|
||||
):
|
||||
"""
|
||||
发布日志消息
|
||||
|
||||
同时存储到 Redis List（历史）和发布到 Pub/Sub（实时）
|
||||
|
||||
Args:
|
||||
log_id: 日志记录 ID
|
||||
message: 日志消息
|
||||
level: 日志级别
|
||||
step: 当前步骤名称
|
||||
progress: 进度百分比 (0-100)
|
||||
data: 附加数据
|
||||
"""
|
||||
try:
|
||||
redis = await RedisClient.get_client()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
entry = TaskLogEntry(
|
||||
timestamp=datetime.now().isoformat(),
|
||||
level=level.value,
|
||||
message=message,
|
||||
step=step,
|
||||
progress=progress,
|
||||
data=data
|
||||
)
|
||||
|
||||
entry_json = json.dumps(entry.to_dict(), ensure_ascii=False)
|
||||
list_key = cls._get_list_key(log_id)
|
||||
channel = cls._get_channel(log_id)
|
||||
|
||||
# 存储到 List（历史日志）
|
||||
await redis.rpush(list_key, entry_json)
|
||||
# 设置过期时间
|
||||
await redis.expire(list_key, cls.LOG_EXPIRE_SECONDS)
|
||||
# 发布到 Pub/Sub（实时推送）
|
||||
await redis.publish(channel, entry_json)
|
||||
|
||||
@classmethod
|
||||
async def publish_start(cls, log_id: str, job_name: str):
|
||||
"""发布任务开始事件"""
|
||||
await cls.publish(
|
||||
log_id=log_id,
|
||||
message=f"任务 {job_name} 开始执行",
|
||||
level=LogLevel.INFO,
|
||||
step="start",
|
||||
progress=0
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def publish_complete(cls, log_id: str, job_name: str, result: Optional[str] = None):
|
||||
"""发布任务完成事件"""
|
||||
await cls.publish(
|
||||
log_id=log_id,
|
||||
message=f"任务 {job_name} 执行完成",
|
||||
level=LogLevel.SUCCESS,
|
||||
step="complete",
|
||||
progress=100,
|
||||
data={"result": result} if result else None
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def publish_error(cls, log_id: str, job_name: str, error: str):
|
||||
"""发布任务错误事件"""
|
||||
await cls.publish(
|
||||
log_id=log_id,
|
||||
message=f"任务 {job_name} 执行失败: {error}",
|
||||
level=LogLevel.ERROR,
|
||||
step="error",
|
||||
data={"error": error}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def get_history(cls, log_id: str) -> list:
|
||||
"""
|
||||
获取历史日志
|
||||
|
||||
Args:
|
||||
log_id: 日志记录 ID
|
||||
|
||||
Returns:
|
||||
日志条目列表
|
||||
"""
|
||||
try:
|
||||
redis = await RedisClient.get_client()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
list_key = cls._get_list_key(log_id)
|
||||
entries = await redis.lrange(list_key, 0, -1)
|
||||
|
||||
result = []
|
||||
for entry_json in entries:
|
||||
try:
|
||||
result.append(json.loads(entry_json))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
async def subscribe(cls, log_id: str) -> AsyncGenerator[dict, None]:
|
||||
"""
|
||||
订阅日志消息
|
||||
|
||||
先返回历史日志，然后订阅新消息
|
||||
|
||||
Args:
|
||||
log_id: 日志记录 ID
|
||||
|
||||
Yields:
|
||||
日志条目字典
|
||||
"""
|
||||
try:
|
||||
redis = await RedisClient.get_client()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
list_key = cls._get_list_key(log_id)
|
||||
channel = cls._get_channel(log_id)
|
||||
|
||||
# 先获取历史日志
|
||||
history = await redis.lrange(list_key, 0, -1)
|
||||
|
||||
# 返回历史日志
|
||||
for entry_json in history:
|
||||
try:
|
||||
data = json.loads(entry_json)
|
||||
yield data
|
||||
|
||||
# 如果历史日志中已经有完成或错误事件，直接结束
|
||||
if data.get("step") in ("complete", "error"):
|
||||
return
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# 订阅新消息
|
||||
pubsub = redis.pubsub()
|
||||
await pubsub.subscribe(channel)
|
||||
|
||||
try:
|
||||
async for message in pubsub.listen():
|
||||
if message["type"] == "message":
|
||||
data = json.loads(message["data"])
|
||||
yield data
|
||||
|
||||
# 如果是完成或错误事件，结束订阅
|
||||
if data.get("step") in ("complete", "error"):
|
||||
break
|
||||
finally:
|
||||
await pubsub.unsubscribe(channel)
|
||||
await pubsub.close()
|
||||
|
||||
|
||||
class TaskLogger:
|
||||
"""
|
||||
任务日志记录器
|
||||
|
||||
在任务函数中使用，用于记录执行过程中的日志
|
||||
"""
|
||||
|
||||
def __init__(self, log_id: str, job_name: str = ""):
|
||||
self.log_id = log_id
|
||||
self.job_name = job_name
|
||||
self._current_step = None
|
||||
self._progress = 0
|
||||
|
||||
async def debug(self, message: str, **kwargs):
|
||||
"""记录调试日志"""
|
||||
await TaskLogService.publish(
|
||||
log_id=self.log_id,
|
||||
message=message,
|
||||
level=LogLevel.DEBUG,
|
||||
step=self._current_step,
|
||||
progress=self._progress,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def info(self, message: str, **kwargs):
|
||||
"""记录信息日志"""
|
||||
await TaskLogService.publish(
|
||||
log_id=self.log_id,
|
||||
message=message,
|
||||
level=LogLevel.INFO,
|
||||
step=self._current_step,
|
||||
progress=self._progress,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def warning(self, message: str, **kwargs):
|
||||
"""记录警告日志"""
|
||||
await TaskLogService.publish(
|
||||
log_id=self.log_id,
|
||||
message=message,
|
||||
level=LogLevel.WARNING,
|
||||
step=self._current_step,
|
||||
progress=self._progress,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def error(self, message: str, **kwargs):
|
||||
"""记录错误日志"""
|
||||
await TaskLogService.publish(
|
||||
log_id=self.log_id,
|
||||
message=message,
|
||||
level=LogLevel.ERROR,
|
||||
step=self._current_step,
|
||||
progress=self._progress,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
async def success(self, message: str, **kwargs):
|
||||
"""记录成功日志"""
|
||||
await TaskLogService.publish(
|
||||
log_id=self.log_id,
|
||||
message=message,
|
||||
level=LogLevel.SUCCESS,
|
||||
step=self._current_step,
|
||||
progress=self._progress,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def set_step(self, step: str):
|
||||
"""设置当前步骤"""
|
||||
self._current_step = step
|
||||
|
||||
def set_progress(self, progress: int):
|
||||
"""设置进度 (0-100)"""
|
||||
self._progress = max(0, min(100, progress))
|
||||
|
||||
async def step(self, step: str, message: str, progress: Optional[int] = None):
|
||||
"""
|
||||
记录步骤日志
|
||||
|
||||
Args:
|
||||
step: 步骤名称
|
||||
message: 日志消息
|
||||
progress: 进度百分比
|
||||
"""
|
||||
self._current_step = step
|
||||
if progress is not None:
|
||||
self._progress = progress
|
||||
|
||||
await self.info(message, step=step, progress=self._progress)
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Task Utils - 定时任务工具函数
|
||||
提供任务执行中常用的工具函数
|
||||
"""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TaskLoggerWrapper:
|
||||
"""
|
||||
任务日志包装器
|
||||
|
||||
同时输出日志到控制台和实时推送到前端
|
||||
"""
|
||||
|
||||
def __init__(self, job_code: str, task_logger=None):
|
||||
"""
|
||||
初始化日志包装器
|
||||
|
||||
Args:
|
||||
job_code: 任务编码
|
||||
task_logger: TaskLogger 实例（可选）
|
||||
"""
|
||||
self.job_code = job_code
|
||||
self.task_logger = task_logger
|
||||
|
||||
async def info(self, message: str):
|
||||
"""记录信息日志"""
|
||||
logger.info(f"[{self.job_code}] {message}")
|
||||
if self.task_logger:
|
||||
await self.task_logger.info(message)
|
||||
|
||||
async def warning(self, message: str):
|
||||
"""记录警告日志"""
|
||||
logger.warning(f"[{self.job_code}] {message}")
|
||||
if self.task_logger:
|
||||
await self.task_logger.warning(message)
|
||||
|
||||
async def error(self, message: str):
|
||||
"""记录错误日志"""
|
||||
logger.error(f"[{self.job_code}] {message}")
|
||||
if self.task_logger:
|
||||
await self.task_logger.error(message)
|
||||
|
||||
async def debug(self, message: str):
|
||||
"""记录调试日志"""
|
||||
logger.debug(f"[{self.job_code}] {message}")
|
||||
if self.task_logger:
|
||||
await self.task_logger.debug(message)
|
||||
@@ -0,0 +1,477 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Scheduler Tasks - 定时任务函数示例
|
||||
定义可被调度器调用的任务函数
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def test_task(job_code: str = None, word: str = None, **kwargs):
|
||||
"""
|
||||
测试任务
|
||||
|
||||
这是一个简单的测试任务，用于验证调度器是否正常工作。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
**kwargs: 其他参数
|
||||
"""
|
||||
logger.info(f"[{job_code}-{word}] 测试任务执行开始: {datetime.now()}")
|
||||
|
||||
# 模拟任务执行
|
||||
import asyncio
|
||||
await asyncio.sleep(10)
|
||||
|
||||
logger.info(f"[{job_code}] 测试任务执行完成: {datetime.now()}")
|
||||
return f"测试任务执行成功: {datetime.now()}"
|
||||
|
||||
|
||||
async def cleanup_task(job_code: str = None, days: int = 30, **kwargs):
|
||||
"""
|
||||
清理任务
|
||||
|
||||
清理过期的日志数据。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
days: 保留最近N天的数据
|
||||
**kwargs: 其他参数
|
||||
"""
|
||||
logger.info(f"[{job_code}] 清理任务执行开始，保留最近 {days} 天数据")
|
||||
|
||||
try:
|
||||
from datetime import timedelta
|
||||
from sqlalchemy import select, delete
|
||||
from app.database import AsyncSessionLocal
|
||||
from scheduler.model import SchedulerLog
|
||||
|
||||
cutoff = datetime.now() - timedelta(days=days)
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 删除过期日志
|
||||
result = await db.execute(
|
||||
select(SchedulerLog).where(SchedulerLog.start_time < cutoff)
|
||||
)
|
||||
logs = result.scalars().all()
|
||||
count = len(logs)
|
||||
|
||||
for log in logs:
|
||||
await db.delete(log)
|
||||
|
||||
await db.commit()
|
||||
|
||||
logger.info(f"[{job_code}] 清理任务执行完成，删除了 {count} 条日志")
|
||||
return f"清理了 {count} 条过期日志"
|
||||
except Exception as e:
|
||||
logger.error(f"[{job_code}] 清理任务执行失败: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
def sync_test_task(job_code: str = None, **kwargs):
|
||||
"""
|
||||
同步测试任务
|
||||
|
||||
这是一个同步任务示例，用于演示同步任务的使用。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
**kwargs: 其他参数
|
||||
"""
|
||||
import time
|
||||
|
||||
logger.info(f"[{job_code}] 同步测试任务执行开始: {datetime.now()}")
|
||||
|
||||
# 模拟任务执行
|
||||
time.sleep(1)
|
||||
|
||||
logger.info(f"[{job_code}] 同步测试任务执行完成: {datetime.now()}")
|
||||
return f"同步测试任务执行成功: {datetime.now()}"
|
||||
|
||||
|
||||
async def restore_database_task(
|
||||
job_code: str = None,
|
||||
file_path: str = './db_init.json',
|
||||
app_name: str = None,
|
||||
clear_before_restore: bool = True,
|
||||
exclude_tables: list = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
数据库数据恢复任务
|
||||
|
||||
从 JSON 文件恢复数据库数据，类似 Django 的 loaddata 命令。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
file_path: JSON 数据文件路径（必填）
|
||||
app_name: 应用名称过滤（可选），如 core、scheduler，只恢复指定应用的数据
|
||||
clear_before_restore: 恢复前是否清空目标表（默认 True）
|
||||
exclude_tables: 排除的表名列表（可选），如 ['scheduler_log', 'core_operation_log']
|
||||
**kwargs: 其他参数（包含 task_logger）
|
||||
|
||||
Returns:
|
||||
str: 恢复结果摘要
|
||||
|
||||
Raises:
|
||||
ValueError: 文件路径未指定或文件不存在
|
||||
Exception: 数据恢复过程中的错误
|
||||
"""
|
||||
exclude_tables = exclude_tables or ['core_city', 'core_street', 'core_village', 'core_area','core_province', 'core_scheduler_job', 'core_scheduler_log']
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import delete
|
||||
from app.database import AsyncSessionLocal, Base
|
||||
from scheduler.task_utils import TaskLoggerWrapper
|
||||
|
||||
# 创建日志包装器：同时输出到控制台和实时日志
|
||||
log = TaskLoggerWrapper(job_code, kwargs.get('task_logger'))
|
||||
|
||||
await log.info(f"数据库恢复任务开始执行")
|
||||
|
||||
# 验证文件路径
|
||||
if not file_path:
|
||||
raise ValueError("file_path 参数不能为空")
|
||||
|
||||
data_file = Path(file_path)
|
||||
if not data_file.exists():
|
||||
raise ValueError(f"数据文件不存在: {file_path}")
|
||||
|
||||
if not data_file.suffix.lower() == '.json':
|
||||
raise ValueError(f"只支持 JSON 格式的数据文件: {file_path}")
|
||||
|
||||
await log.info(f"从文件加载数据: {file_path}")
|
||||
|
||||
def parse_datetime(value):
|
||||
"""解析日期时间字符串"""
|
||||
if isinstance(value, str):
|
||||
# 尝试解析 ISO 格式的日期时间字符串
|
||||
try:
|
||||
# 支持多种格式
|
||||
if 'T' in value or ' ' in value:
|
||||
# 包含时间部分
|
||||
return datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
elif len(value) == 10 and value.count('-') == 2:
|
||||
# 只有日期部分 YYYY-MM-DD
|
||||
from datetime import date
|
||||
return date.fromisoformat(value)
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
return value
|
||||
|
||||
async def auto_import_models():
|
||||
"""自动加载所有模型类定义"""
|
||||
import importlib
|
||||
project_root = Path(__file__).parent.parent
|
||||
|
||||
scan_dirs = ["zq_demo", "core", "scheduler", "online_dev", "ai_platform"]
|
||||
loaded_count = 0
|
||||
|
||||
for scan_dir in scan_dirs:
|
||||
scan_path = project_root / scan_dir
|
||||
if not scan_path.exists():
|
||||
continue
|
||||
|
||||
for model_file in scan_path.rglob("*model.py"):
|
||||
relative_path = model_file.relative_to(project_root)
|
||||
module_path = str(relative_path.with_suffix("")).replace("/", ".").replace("\\", ".")
|
||||
|
||||
try:
|
||||
importlib.import_module(module_path)
|
||||
loaded_count += 1
|
||||
except ImportError as e:
|
||||
logger.warning(f"[{job_code}] 加载模型定义失败 {module_path}: {e}")
|
||||
|
||||
await log.info(f"已加载 {loaded_count} 个模型定义文件")
|
||||
|
||||
try:
|
||||
# 自动加载所有模型类定义（不是导入数据）
|
||||
await log.info(f"开始加载模型定义...")
|
||||
await auto_import_models()
|
||||
|
||||
# 读取 JSON 文件
|
||||
with open(data_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
await log.info(f"读取到 {len(data)} 条记录")
|
||||
|
||||
# 构建模型映射
|
||||
model_map: Dict[str, Any] = {}
|
||||
for mapper in Base.registry.mappers:
|
||||
model_class = mapper.class_
|
||||
model_key = f"{model_class.__module__}.{model_class.__name__}"
|
||||
model_map[model_key] = model_class
|
||||
|
||||
# 如果指定了 app_name，过滤数据
|
||||
if app_name:
|
||||
data = [item for item in data if item.get("model", "").startswith(app_name)]
|
||||
await log.info(f"过滤后剩余 {len(data)} 条记录（应用: {app_name})")
|
||||
|
||||
# 排除指定的表
|
||||
if exclude_tables:
|
||||
original_count = len(data)
|
||||
data = [
|
||||
item for item in data
|
||||
if not (model_map.get(item.get("model")) and
|
||||
model_map[item.get("model")].__tablename__ in exclude_tables)
|
||||
]
|
||||
excluded_count = original_count - len(data)
|
||||
if excluded_count > 0:
|
||||
await log.info(f"排除 {excluded_count} 条记录（表: {', '.join(exclude_tables)})")
|
||||
|
||||
success_count = 0
|
||||
error_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
# 阶段1：清空表（每个表独立事务，避免长时间持有锁）
|
||||
if clear_before_restore:
|
||||
tables_to_clear = set()
|
||||
for item in data:
|
||||
model_name = item.get("model")
|
||||
if model_name in model_map:
|
||||
tables_to_clear.add(model_name)
|
||||
|
||||
# 按表名排序，便于追踪进度
|
||||
sorted_tables = sorted(tables_to_clear, key=lambda x: model_map[x].__tablename__)
|
||||
total_tables = len(sorted_tables)
|
||||
await log.info(f"准备清空 {total_tables} 个表...")
|
||||
|
||||
# 每个表独立事务清空
|
||||
for idx, model_name in enumerate(sorted_tables, 1):
|
||||
model_class = model_map[model_name]
|
||||
table_name = model_class.__tablename__
|
||||
await log.info(f"[{idx}/{total_tables}] 清空表: {table_name}")
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
# 使用 TRUNCATE CASCADE 处理外键依赖
|
||||
await session.execute(text(f'TRUNCATE TABLE "{table_name}" CASCADE'))
|
||||
await session.commit()
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
# 如果 TRUNCATE 失败，尝试 DELETE
|
||||
await log.warning(f"TRUNCATE 失败，使用 DELETE: {e}")
|
||||
try:
|
||||
await session.execute(delete(model_class))
|
||||
await session.commit()
|
||||
except Exception as e2:
|
||||
await log.error(f"DELETE 也失败: {e2}")
|
||||
await session.rollback()
|
||||
|
||||
await log.info(f"已清空 {total_tables} 个表")
|
||||
|
||||
# 阶段2：导入数据（分批提交，每批独立事务）
|
||||
total_records = len(data)
|
||||
await log.info(f"开始导入 {total_records} 条记录...")
|
||||
|
||||
batch_size = 500
|
||||
batch = []
|
||||
|
||||
for idx, item in enumerate(data, 1):
|
||||
model_name = item.get("model")
|
||||
fields = item.get("fields", {})
|
||||
|
||||
if model_name not in model_map:
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
model_class = model_map[model_name]
|
||||
|
||||
# 转换日期时间字段（对所有字段值尝试转换）
|
||||
for key, value in fields.items():
|
||||
fields[key] = parse_datetime(value)
|
||||
|
||||
batch.append((model_class, fields))
|
||||
|
||||
# 每 batch_size 条提交一次
|
||||
if len(batch) >= batch_size:
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
for model_class, fields in batch:
|
||||
instance = model_class(**fields)
|
||||
session.add(instance)
|
||||
await session.commit()
|
||||
success_count += len(batch)
|
||||
except Exception as e:
|
||||
await log.error(f"批量导入失败: {e}")
|
||||
error_count += len(batch)
|
||||
await session.rollback()
|
||||
|
||||
batch = []
|
||||
progress = round(idx / total_records * 100, 1)
|
||||
await log.info(f"进度: {progress}% ({idx}/{total_records}) - 成功: {success_count}, 失败: {error_count}, 跳过: {skipped_count}")
|
||||
|
||||
# 提交剩余的数据
|
||||
if batch:
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
for model_class, fields in batch:
|
||||
instance = model_class(**fields)
|
||||
session.add(instance)
|
||||
await session.commit()
|
||||
success_count += len(batch)
|
||||
except Exception as e:
|
||||
await log.error(f"最后批次导入失败: {e}")
|
||||
error_count += len(batch)
|
||||
await session.rollback()
|
||||
|
||||
result = f"恢复完成: 成功 {success_count} 条, 失败 {error_count} 条, 跳过 {skipped_count} 条"
|
||||
await log.info(f"{result}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
await log.error(f"数据库恢复任务执行失败: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
async def backup_database_task(
|
||||
job_code: str = None,
|
||||
output_path: str = None,
|
||||
app_name: str = None,
|
||||
exclude_tables: list = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
数据库数据备份任务
|
||||
|
||||
将数据库数据导出到 JSON 文件，类似 Django 的 dumpdata 命令。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
output_path: 输出文件路径（可选），默认为 backups/backup_YYYYMMDD_HHMMSS.json
|
||||
app_name: 应用名称过滤（可选），如 core、scheduler，只备份指定应用的数据
|
||||
exclude_tables: 排除的表名列表（可选），如 ['scheduler_log', 'core_operation_log']
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
str: 备份结果摘要，包含文件路径和记录数
|
||||
"""
|
||||
exclude_tables = exclude_tables or []
|
||||
import json
|
||||
from pathlib import Path
|
||||
from decimal import Decimal
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from app.database import AsyncSessionLocal, Base
|
||||
|
||||
logger.info(f"[{job_code}] 数据库备份任务开始执行")
|
||||
|
||||
class DateTimeEncoder(json.JSONEncoder):
|
||||
"""自定义 JSON 编码器"""
|
||||
def default(self, obj):
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, date):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
return super().default(obj)
|
||||
|
||||
def auto_import_models():
|
||||
"""自动导入所有模型"""
|
||||
import importlib
|
||||
project_root = Path(__file__).parent.parent
|
||||
|
||||
scan_dirs = ["zq_demo", "core", "scheduler", "online_dev", "ai_platform"]
|
||||
|
||||
for scan_dir in scan_dirs:
|
||||
scan_path = project_root / scan_dir
|
||||
if not scan_path.exists():
|
||||
continue
|
||||
|
||||
for model_file in scan_path.rglob("*model.py"):
|
||||
relative_path = model_file.relative_to(project_root)
|
||||
module_path = str(relative_path.with_suffix("")).replace("/", ".").replace("\\", ".")
|
||||
|
||||
try:
|
||||
importlib.import_module(module_path)
|
||||
except ImportError as e:
|
||||
logger.warning(f"[{job_code}] 导入模型失败 {module_path}: {e}")
|
||||
|
||||
try:
|
||||
# 自动导入所有模型
|
||||
auto_import_models()
|
||||
|
||||
# 确定输出路径
|
||||
if output_path:
|
||||
output_file = Path(output_path)
|
||||
else:
|
||||
project_root = Path(__file__).parent.parent
|
||||
backups_dir = project_root / "backups"
|
||||
backups_dir.mkdir(exist_ok=True)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"backup_{app_name}_{timestamp}.json" if app_name else f"backup_{timestamp}.json"
|
||||
output_file = backups_dir / filename
|
||||
|
||||
# 确保输出目录存在
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
all_data = []
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
# 获取所有模型
|
||||
models = []
|
||||
for mapper in Base.registry.mappers:
|
||||
model_class = mapper.class_
|
||||
|
||||
# 如果指定了 app_name，只导出该应用的模型
|
||||
if app_name:
|
||||
module_name = model_class.__module__
|
||||
if not module_name.startswith(app_name):
|
||||
continue
|
||||
|
||||
# 排除指定的表
|
||||
if model_class.__tablename__ in exclude_tables:
|
||||
logger.info(f"[{job_code}] 跳过表: {model_class.__tablename__}")
|
||||
continue
|
||||
|
||||
models.append(model_class)
|
||||
|
||||
# 按表名排序
|
||||
models.sort(key=lambda m: m.__tablename__)
|
||||
|
||||
logger.info(f"[{job_code}] 准备备份 {len(models)} 个表")
|
||||
|
||||
# 导出每个表
|
||||
from sqlalchemy import select
|
||||
|
||||
for model_class in models:
|
||||
logger.info(f"[{job_code}] 备份表: {model_class.__tablename__}")
|
||||
|
||||
result = await session.execute(select(model_class))
|
||||
items = result.scalars().all()
|
||||
|
||||
for item in items:
|
||||
item_dict = {}
|
||||
for column in inspect(model_class).columns:
|
||||
value = getattr(item, column.name)
|
||||
item_dict[column.name] = value
|
||||
|
||||
all_data.append({
|
||||
"model": f"{model_class.__module__}.{model_class.__name__}",
|
||||
"pk": item.id if hasattr(item, 'id') else None,
|
||||
"fields": item_dict
|
||||
})
|
||||
|
||||
logger.info(f"[{job_code}] - 备份 {len(items)} 条记录")
|
||||
|
||||
# 写入文件
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_data, f, ensure_ascii=False, indent=2, cls=DateTimeEncoder)
|
||||
|
||||
result = f"备份完成: 共 {len(all_data)} 条记录，保存到 {output_file}"
|
||||
logger.info(f"[{job_code}] {result}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{job_code}] 数据库备份任务执行失败: {str(e)}")
|
||||
raise
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
工作流执行任务 - 供定时任务调度器调用
|
||||
"""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def execute_workflow_task(
|
||||
job_code: str = None,
|
||||
workflow_code: str = None,
|
||||
inputs: dict = None,
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
执行工作流任务
|
||||
|
||||
在定时任务中调用已发布的数据处理/自动化工作流。
|
||||
|
||||
Args:
|
||||
job_code: 任务编码（由调度器自动传入）
|
||||
workflow_code: 要执行的工作流编码（必填）
|
||||
inputs: 工作流输入变量
|
||||
**kwargs: 其他参数（包含 task_logger）
|
||||
|
||||
Returns:
|
||||
dict: 执行结果
|
||||
"""
|
||||
if not workflow_code:
|
||||
raise ValueError("workflow_code 参数不能为空，请在 task_kwargs 中配置")
|
||||
|
||||
inputs = inputs or {}
|
||||
inputs['job_code'] = job_code
|
||||
|
||||
from app.database import AsyncSessionLocal
|
||||
from ai_platform.services.workflow_service import AIWorkflowService
|
||||
from ai_platform.models.workflow import AIWorkflow
|
||||
from sqlalchemy import select
|
||||
from scheduler.task_utils import TaskLoggerWrapper
|
||||
|
||||
log = TaskLoggerWrapper(job_code, kwargs.get('task_logger'))
|
||||
await log.info(f"开始执行工作流: {workflow_code}")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(AIWorkflow).where(
|
||||
AIWorkflow.code == workflow_code,
|
||||
AIWorkflow.is_deleted == False
|
||||
)
|
||||
)
|
||||
workflow = result.scalar_one_or_none()
|
||||
|
||||
if not workflow:
|
||||
raise ValueError(f"工作流不存在: {workflow_code}")
|
||||
|
||||
if workflow.workflow_type not in ('data_process', 'automation'):
|
||||
raise ValueError(
|
||||
f"工作流类型不支持定时执行: {workflow.workflow_type}，"
|
||||
f"仅支持 data_process 和 automation 类型"
|
||||
)
|
||||
|
||||
if workflow.status != 'published':
|
||||
raise ValueError(f"工作流未发布: {workflow_code}")
|
||||
|
||||
await log.info(
|
||||
f"工作流类型: {workflow.workflow_type}, "
|
||||
f"工作流名称: {workflow.name}, "
|
||||
f"输入变量: {list(inputs.keys())}"
|
||||
)
|
||||
|
||||
service = AIWorkflowService(db)
|
||||
run = await service.run_workflow(
|
||||
workflow_id=str(workflow.id),
|
||||
inputs=inputs,
|
||||
trigger_type='api',
|
||||
)
|
||||
|
||||
await log.info(
|
||||
f"工作流执行完成: status={run.status}, "
|
||||
f"elapsed_time={run.elapsed_time}ms, "
|
||||
f"total_steps={run.total_steps}"
|
||||
)
|
||||
|
||||
if run.status == 'failed':
|
||||
raise RuntimeError(f"工作流执行失败: {run.error_message}")
|
||||
|
||||
return {
|
||||
'run_id': str(run.id),
|
||||
'workflow_code': workflow_code,
|
||||
'workflow_name': workflow.name,
|
||||
'status': run.status,
|
||||
'elapsed_time': run.elapsed_time,
|
||||
'total_steps': run.total_steps,
|
||||
'total_tokens': run.total_tokens,
|
||||
'outputs': run.outputs,
|
||||
}
|
||||
Reference in New Issue
Block a user