forked from xiaohei/taiji-AI-PAD
940 lines
39 KiB
Python
940 lines
39 KiB
Python
"""
|
||
平台监控模块
|
||
提供系统健康检查、性能指标、资源使用等监控功能
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import psutil
|
||
import time
|
||
import httpx
|
||
from datetime import datetime, timedelta
|
||
from typing import Dict, List, Optional, Any
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from sqlalchemy import text, func
|
||
from sqlalchemy.orm import selectinload
|
||
|
||
from models import Agent, Execution, User, Tool, Session
|
||
from database import AsyncSessionLocal
|
||
from config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class SystemMonitor:
|
||
"""系统监控类"""
|
||
|
||
def __init__(self):
|
||
self.start_time = datetime.utcnow()
|
||
# 服务健康检查端点配置
|
||
self._service_endpoints = {
|
||
"mcp_server": "http://localhost:8000/health", # 本服务
|
||
"data_ingestion": "http://data-ingestion:8000/health", # Docker 内部网络
|
||
"model_gateway": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/health/liveliness", # LiteLLM 模型网关
|
||
"agent_manager": "http://10.0.0.4:8000", # Agent Manager 服务（本地）
|
||
}
|
||
# LiteLLM API Key - 从配置读取
|
||
self._litellm_api_key = settings.litellm_master_key
|
||
|
||
async def get_system_health(self) -> Dict[str, Any]:
|
||
"""获取系统健康状态，包括三个微服务的状态"""
|
||
health = {
|
||
"status": "healthy",
|
||
"score": 100,
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
"uptime_seconds": (datetime.utcnow() - self.start_time).total_seconds(),
|
||
"services": {}
|
||
}
|
||
|
||
# 检查数据库（内部使用，不对外暴露）
|
||
try:
|
||
async with AsyncSessionLocal() as session:
|
||
await session.execute(text("SELECT 1"))
|
||
# 数据库健康检查成功，但不添加到 services 中
|
||
except Exception as e:
|
||
# 数据库不健康会影响整体状态
|
||
health["status"] = "degraded"
|
||
health["score"] -= 25
|
||
|
||
# 检查三个微服务状态
|
||
service_health = await self._check_microservices_health()
|
||
health["services"].update(service_health)
|
||
|
||
# 计算整体健康分数
|
||
unhealthy_count = sum(1 for s in health["services"].values()
|
||
if s.get("status") not in ["healthy", "running"])
|
||
if unhealthy_count > 0:
|
||
health["score"] = max(0, 100 - (unhealthy_count * 20))
|
||
if unhealthy_count >= 2:
|
||
health["status"] = "degraded"
|
||
if unhealthy_count >= 3:
|
||
health["status"] = "unhealthy"
|
||
|
||
return health
|
||
|
||
async def _check_microservices_health(self) -> Dict[str, Dict[str, Any]]:
|
||
"""
|
||
检查四个微服务的健康状态
|
||
|
||
返回格式：
|
||
{
|
||
"mcp_server": { "status": "healthy", "latency": 45 },
|
||
"data_ingestion": { "status": "healthy", "latency": 32 },
|
||
"model_gateway": { "status": "healthy", "latency": 28 },
|
||
"agent_manager": { "status": "healthy", "latency": 15 }
|
||
}
|
||
|
||
状态值：
|
||
- healthy/running: 运行中
|
||
- maintenance: 维护中
|
||
- error: 错误
|
||
"""
|
||
results = {}
|
||
|
||
async def check_service(name: str, url: str, headers: Dict[str, str] = None) -> Dict[str, Any]:
|
||
"""检查单个服务的健康状态"""
|
||
start_time = time.time()
|
||
try:
|
||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||
response = await client.get(url, headers=headers)
|
||
latency = int((time.time() - start_time) * 1000)
|
||
|
||
if response.status_code == 200:
|
||
return {"status": "healthy", "latency": latency}
|
||
elif response.status_code == 503:
|
||
return {"status": "maintenance", "latency": latency}
|
||
else:
|
||
return {"status": "error", "latency": latency, "code": response.status_code}
|
||
except httpx.TimeoutException:
|
||
return {"status": "error", "latency": 5000, "error": "timeout"}
|
||
except httpx.ConnectError:
|
||
# 服务不可达，可能是维护中或未启动
|
||
return {"status": "maintenance", "latency": 0, "error": "connection_refused"}
|
||
except Exception as e:
|
||
return {"status": "error", "latency": 0, "error": str(e)}
|
||
|
||
# mcp_server 本服务始终是健康的（因为能响应请求）
|
||
results["mcp_server"] = {"status": "healthy", "latency": 1}
|
||
|
||
# 检查 data_ingestion 服务
|
||
results["data_ingestion"] = await check_service(
|
||
"data_ingestion",
|
||
self._service_endpoints["data_ingestion"]
|
||
)
|
||
|
||
# 检查 model_gateway (LiteLLM) - 需要带上 API Key
|
||
results["model_gateway"] = await check_service(
|
||
"model_gateway",
|
||
self._service_endpoints["model_gateway"],
|
||
headers={"Authorization": f"Bearer {self._litellm_api_key}"}
|
||
)
|
||
|
||
# 检查 agent_manager 服务
|
||
results["agent_manager"] = await check_service(
|
||
"agent_manager",
|
||
self._service_endpoints["agent_manager"]
|
||
)
|
||
|
||
return results
|
||
|
||
async def get_system_metrics(self) -> Dict[str, Any]:
|
||
"""获取系统性能指标"""
|
||
try:
|
||
# 系统资源使用
|
||
cpu_percent = psutil.cpu_percent(interval=1)
|
||
memory = psutil.virtual_memory()
|
||
disk = psutil.disk_usage('/')
|
||
|
||
# 数据库统计
|
||
async with AsyncSessionLocal() as session:
|
||
# Agent统计
|
||
agent_result = await session.execute(
|
||
text("SELECT COUNT(*) FROM agents WHERE status = 'active'")
|
||
)
|
||
active_agents = agent_result.scalar() or 0
|
||
|
||
# 执行统计
|
||
execution_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
AVG(execution_time) as avg_time,
|
||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END)::float / NULLIF(COUNT(*), 0) * 100 as success_rate
|
||
FROM executions
|
||
WHERE started_at > NOW() - INTERVAL '24 hours'
|
||
""")
|
||
)
|
||
exec_stats = execution_result.fetchone()
|
||
total_executions = exec_stats[0] or 0 if exec_stats else 0
|
||
avg_execution_time = float(exec_stats[1] or 0) if exec_stats and exec_stats[1] else 0.0
|
||
success_rate = float(exec_stats[2] or 0) if exec_stats and exec_stats[2] else 0.0
|
||
|
||
# 用户统计
|
||
user_result = await session.execute(
|
||
text("""
|
||
SELECT COUNT(DISTINCT user_id)
|
||
FROM sessions
|
||
WHERE created_at > NOW() - INTERVAL '24 hours'
|
||
""")
|
||
)
|
||
daily_active_users = user_result.scalar() or 0
|
||
|
||
# EU消耗统计
|
||
eu_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
SUM(eu_consumed) as total_eu,
|
||
SUM(cost) as total_cost
|
||
FROM billing
|
||
WHERE created_at > NOW() - INTERVAL '24 hours'
|
||
""")
|
||
)
|
||
eu_stats = eu_result.fetchone()
|
||
total_eu = float(eu_stats[0] or 0) if eu_stats and eu_stats[0] else 0.0
|
||
total_cost = float(eu_stats[1] or 0) if eu_stats and eu_stats[1] else 0.0
|
||
|
||
return {
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
"system": {
|
||
"cpu_usage_percent": cpu_percent,
|
||
"memory_usage_percent": memory.percent,
|
||
"memory_used_mb": memory.used / 1024 / 1024,
|
||
"memory_total_mb": memory.total / 1024 / 1024,
|
||
"disk_usage_percent": disk.percent,
|
||
"disk_used_gb": disk.used / 1024 / 1024 / 1024,
|
||
"disk_total_gb": disk.total / 1024 / 1024 / 1024,
|
||
},
|
||
"services": {
|
||
"active_agents": active_agents,
|
||
"total_executions_24h": total_executions,
|
||
"success_rate_percent": round(success_rate, 2),
|
||
"avg_execution_time_ms": round(avg_execution_time, 2),
|
||
"daily_active_users": daily_active_users,
|
||
},
|
||
"billing": {
|
||
"total_eu_consumed_24h": round(total_eu, 4),
|
||
"total_cost_24h": round(total_cost, 4),
|
||
}
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取系统指标失败: {e}")
|
||
raise
|
||
|
||
async def get_service_stats(self, service: str = "all") -> Dict[str, Any]:
|
||
"""获取服务统计信息"""
|
||
try:
|
||
async with AsyncSessionLocal() as session:
|
||
stats = {}
|
||
|
||
if service == "all" or service == "agents":
|
||
# Agent统计
|
||
agent_stats = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN status = 'active' THEN 1 END) as active,
|
||
COUNT(CASE WHEN status = 'inactive' THEN 1 END) as inactive,
|
||
AVG(total_executions) as avg_executions,
|
||
AVG(success_rate) as avg_success_rate
|
||
FROM agents
|
||
""")
|
||
)
|
||
row = agent_stats.fetchone()
|
||
if row:
|
||
stats["agents"] = {
|
||
"total": row[0] or 0,
|
||
"active": row[1] or 0,
|
||
"inactive": row[2] or 0,
|
||
"avg_executions": float(row[3] or 0),
|
||
"avg_success_rate": float(row[4] or 0),
|
||
}
|
||
|
||
if service == "all" or service == "executions":
|
||
# 执行统计
|
||
exec_stats = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN status = 'completed' THEN 1 END) as completed,
|
||
COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed,
|
||
COUNT(CASE WHEN status = 'running' THEN 1 END) as running,
|
||
AVG(execution_time) as avg_time,
|
||
SUM(eu_consumed) as total_eu
|
||
FROM executions
|
||
WHERE started_at > NOW() - INTERVAL '7 days'
|
||
""")
|
||
)
|
||
row = exec_stats.fetchone()
|
||
if row:
|
||
stats["executions"] = {
|
||
"total_7d": row[0] or 0,
|
||
"completed": row[1] or 0,
|
||
"failed": row[2] or 0,
|
||
"running": row[3] or 0,
|
||
"avg_time_ms": float(row[4] or 0),
|
||
"total_eu": float(row[5] or 0),
|
||
}
|
||
|
||
if service == "all" or service == "tools":
|
||
# 工具统计
|
||
tool_stats = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN is_active = true THEN 1 END) as active,
|
||
SUM(total_calls) as total_calls,
|
||
AVG(success_rate) as avg_success_rate,
|
||
AVG(avg_response_time) as avg_response_time
|
||
FROM tools
|
||
""")
|
||
)
|
||
row = tool_stats.fetchone()
|
||
if row:
|
||
stats["tools"] = {
|
||
"total": row[0] or 0,
|
||
"active": row[1] or 0,
|
||
"total_calls": row[2] or 0,
|
||
"avg_success_rate": float(row[3] or 0),
|
||
"avg_response_time_ms": float(row[4] or 0),
|
||
}
|
||
|
||
if service == "all" or service == "users":
|
||
# 用户统计
|
||
user_stats = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN is_active = true THEN 1 END) as active,
|
||
COUNT(CASE WHEN is_admin = true THEN 1 END) as admins
|
||
FROM users
|
||
""")
|
||
)
|
||
row = user_stats.fetchone()
|
||
if row:
|
||
stats["users"] = {
|
||
"total": row[0] or 0,
|
||
"active": row[1] or 0,
|
||
"admins": row[2] or 0,
|
||
}
|
||
|
||
return {
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
"stats": stats
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取服务统计失败: {e}")
|
||
raise
|
||
|
||
async def get_performance_trends(
|
||
self,
|
||
metric: str = "executions",
|
||
period: str = "24h",
|
||
interval: str = "1h"
|
||
) -> Dict[str, Any]:
|
||
"""获取性能趋势数据"""
|
||
try:
|
||
async with AsyncSessionLocal() as session:
|
||
# 计算时间范围
|
||
if period == "24h":
|
||
hours = 24
|
||
elif period == "7d":
|
||
hours = 168
|
||
elif period == "30d":
|
||
hours = 720
|
||
else:
|
||
hours = 24
|
||
|
||
if interval == "1h":
|
||
interval_sql = "1 hour"
|
||
elif interval == "6h":
|
||
interval_sql = "6 hours"
|
||
elif interval == "1d":
|
||
interval_sql = "1 day"
|
||
else:
|
||
interval_sql = "1 hour"
|
||
|
||
if metric == "executions":
|
||
result = await session.execute(
|
||
text(f"""
|
||
SELECT
|
||
DATE_TRUNC('hour', started_at) as time_bucket,
|
||
COUNT(*) as count,
|
||
AVG(execution_time) as avg_time,
|
||
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END)::float / NULLIF(COUNT(*), 0) * 100 as success_rate
|
||
FROM executions
|
||
WHERE started_at > NOW() - INTERVAL '{hours} hours'
|
||
GROUP BY time_bucket
|
||
ORDER BY time_bucket
|
||
""")
|
||
)
|
||
rows = result.fetchall()
|
||
return {
|
||
"metric": metric,
|
||
"period": period,
|
||
"interval": interval,
|
||
"data": [
|
||
{
|
||
"timestamp": row[0].isoformat() if row[0] else None,
|
||
"count": row[1] or 0,
|
||
"avg_time_ms": float(row[2] or 0),
|
||
"success_rate": float(row[3] or 0),
|
||
}
|
||
for row in rows
|
||
]
|
||
}
|
||
elif metric == "eu_consumption":
|
||
result = await session.execute(
|
||
text(f"""
|
||
SELECT
|
||
DATE_TRUNC('hour', created_at) as time_bucket,
|
||
SUM(eu_consumed) as total_eu,
|
||
SUM(cost) as total_cost
|
||
FROM billing
|
||
WHERE created_at > NOW() - INTERVAL '{hours} hours'
|
||
GROUP BY time_bucket
|
||
ORDER BY time_bucket
|
||
""")
|
||
)
|
||
rows = result.fetchall()
|
||
return {
|
||
"metric": metric,
|
||
"period": period,
|
||
"interval": interval,
|
||
"data": [
|
||
{
|
||
"timestamp": row[0].isoformat() if row[0] else None,
|
||
"eu_consumed": float(row[1] or 0),
|
||
"cost": float(row[2] or 0),
|
||
}
|
||
for row in rows
|
||
]
|
||
}
|
||
else:
|
||
return {
|
||
"metric": metric,
|
||
"period": period,
|
||
"interval": interval,
|
||
"data": []
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取性能趋势失败: {e}")
|
||
raise
|
||
|
||
async def get_alerts(self, severity: Optional[str] = None) -> List[Dict[str, Any]]:
|
||
"""获取系统告警"""
|
||
alerts = []
|
||
|
||
try:
|
||
# 检查系统资源
|
||
cpu_percent = psutil.cpu_percent(interval=1)
|
||
memory = psutil.virtual_memory()
|
||
disk = psutil.disk_usage('/')
|
||
|
||
if cpu_percent > 80:
|
||
alerts.append({
|
||
"severity": "warning",
|
||
"type": "high_cpu",
|
||
"message": f"CPU使用率过高: {cpu_percent:.1f}%",
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
})
|
||
|
||
if memory.percent > 85:
|
||
alerts.append({
|
||
"severity": "warning",
|
||
"type": "high_memory",
|
||
"message": f"内存使用率过高: {memory.percent:.1f}%",
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
})
|
||
|
||
if disk.percent > 90:
|
||
alerts.append({
|
||
"severity": "critical",
|
||
"type": "low_disk",
|
||
"message": f"磁盘空间不足: {disk.percent:.1f}%",
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
})
|
||
|
||
# 检查服务健康
|
||
try:
|
||
async with AsyncSessionLocal() as session:
|
||
# 检查失败的执行
|
||
failed_result = await session.execute(
|
||
text("""
|
||
SELECT COUNT(*)
|
||
FROM executions
|
||
WHERE status = 'failed'
|
||
AND started_at > NOW() - INTERVAL '1 hour'
|
||
""")
|
||
)
|
||
failed_count = failed_result.scalar() or 0
|
||
|
||
if failed_count > 10:
|
||
alerts.append({
|
||
"severity": "warning",
|
||
"type": "high_failure_rate",
|
||
"message": f"过去1小时内有 {failed_count} 次执行失败",
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
})
|
||
except Exception as db_error:
|
||
# 如果executions表不存在或其他数据库错误，仅记录日志不中断
|
||
logger.warning(f"检查执行失败率时出错: {db_error}")
|
||
|
||
# 过滤严重程度
|
||
if severity:
|
||
alerts = [a for a in alerts if a["severity"] == severity]
|
||
|
||
return alerts
|
||
except Exception as e:
|
||
logger.error(f"获取告警失败: {e}")
|
||
return []
|
||
|
||
async def get_global_api_calls(self) -> Dict[str, Any]:
|
||
"""
|
||
获取全局API调用统计
|
||
|
||
返回：
|
||
- totalApiCalls: 历史总调用次数
|
||
- todayApiCalls: 今日调用次数
|
||
- todaySuccess: 今日成功次数
|
||
- todayFailed: 今日失败次数
|
||
"""
|
||
try:
|
||
async with AsyncSessionLocal() as session:
|
||
# 今日开始时间（UTC 00:00:00）
|
||
today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||
|
||
# 尝试从多个表统计API调用
|
||
total_api_calls = 0
|
||
today_api_calls = 0
|
||
today_success = 0
|
||
today_failed = 0
|
||
|
||
# 1. 从 model_billing_records 统计（模型API调用）
|
||
try:
|
||
# 总调用数
|
||
total_result = await session.execute(
|
||
text("SELECT COUNT(*) FROM model_billing_records")
|
||
)
|
||
total_api_calls += total_result.scalar() or 0
|
||
|
||
# 今日调用数
|
||
today_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN status = 'success' THEN 1 END) as success,
|
||
COUNT(CASE WHEN status != 'success' THEN 1 END) as failed
|
||
FROM model_billing_records
|
||
WHERE created_at >= :today_start
|
||
"""),
|
||
{"today_start": today_start}
|
||
)
|
||
row = today_result.fetchone()
|
||
if row:
|
||
today_api_calls += int(row[0] or 0)
|
||
today_success += int(row[1] or 0)
|
||
today_failed += int(row[2] or 0)
|
||
except Exception as e:
|
||
logger.warning(f"统计model_billing_records失败: {e}")
|
||
await session.rollback()
|
||
|
||
# 2. 从 agent_billing_records 统计（Agent运行时调用）
|
||
try:
|
||
# 总调用数
|
||
agent_total_result = await session.execute(
|
||
text("SELECT COUNT(*) FROM agent_billing_records")
|
||
)
|
||
total_api_calls += agent_total_result.scalar() or 0
|
||
|
||
# 今日调用数
|
||
agent_today_result = await session.execute(
|
||
text("""
|
||
SELECT COUNT(*)
|
||
FROM agent_billing_records
|
||
WHERE start_time >= :today_start
|
||
"""),
|
||
{"today_start": today_start}
|
||
)
|
||
agent_today = agent_today_result.scalar() or 0
|
||
today_api_calls += agent_today
|
||
today_success += agent_today # Agent计费记录默认成功
|
||
except Exception as e:
|
||
logger.warning(f"统计agent_billing_records失败: {e}")
|
||
await session.rollback()
|
||
|
||
# 3. 从 executions 统计（执行记录）
|
||
try:
|
||
exec_total_result = await session.execute(
|
||
text("SELECT COUNT(*) FROM executions")
|
||
)
|
||
total_api_calls += exec_total_result.scalar() or 0
|
||
|
||
exec_today_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN status = 'completed' THEN 1 END) as success,
|
||
COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed
|
||
FROM executions
|
||
WHERE started_at >= :today_start
|
||
"""),
|
||
{"today_start": today_start}
|
||
)
|
||
exec_row = exec_today_result.fetchone()
|
||
if exec_row:
|
||
today_api_calls += int(exec_row[0] or 0)
|
||
today_success += int(exec_row[1] or 0)
|
||
today_failed += int(exec_row[2] or 0)
|
||
except Exception as e:
|
||
logger.warning(f"统计executions失败: {e}")
|
||
await session.rollback()
|
||
|
||
return {
|
||
"totalApiCalls": total_api_calls,
|
||
"todayApiCalls": today_api_calls,
|
||
"todaySuccess": today_success,
|
||
"todayFailed": today_failed,
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取全局API调用统计失败: {e}")
|
||
return {
|
||
"totalApiCalls": 0,
|
||
"todayApiCalls": 0,
|
||
"todaySuccess": 0,
|
||
"todayFailed": 0,
|
||
}
|
||
|
||
async def get_tenant_dashboard(self, tenant_id: str) -> Dict[str, Any]:
|
||
"""
|
||
获取租户级别的监控仪表盘数据
|
||
|
||
包含：
|
||
1. 当前租户EU消耗过去24小时的消耗情况
|
||
2. 使用的模型和模型的使用情况、当前调用次数
|
||
3. 每周API调用的次数，7天的每一天调用多少次，包括成功和失败
|
||
4. 三个微服务的状态
|
||
|
||
✅ 优化：使用单个数据库会话，避免连接池耗尽
|
||
"""
|
||
try:
|
||
# 使用单个数据库会话执行所有查询，减少连接数
|
||
async with AsyncSessionLocal() as session:
|
||
try:
|
||
# 1. 获取租户过去24小时的EU消耗
|
||
eu_consumption = await self._get_tenant_eu_consumption_24h(session, tenant_id)
|
||
|
||
# 2. 获取模型使用情况
|
||
model_usage = await self._get_tenant_model_usage(session, tenant_id)
|
||
|
||
# 3. 获取每周API调用统计（7天每天成功/失败）
|
||
weekly_api_calls = await self._get_tenant_weekly_api_calls(session, tenant_id)
|
||
|
||
# 提交所有查询
|
||
await session.commit()
|
||
except Exception as db_error:
|
||
# 数据库错误时回滚并返回默认值
|
||
logger.error(f"查询租户仪表盘数据失败: {db_error}")
|
||
await session.rollback()
|
||
eu_consumption = {"total": 0, "totalCost": 0, "totalCalls": 0, "hourlyData": []}
|
||
model_usage = {"models": [], "totalCalls": 0}
|
||
weekly_api_calls = {"dailyData": [], "totalSuccess": 0, "totalFailed": 0, "total": 0}
|
||
|
||
# 4. 获取微服务状态（不使用数据库连接）
|
||
services_health = await self._check_microservices_health()
|
||
|
||
return {
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
"tenantId": tenant_id,
|
||
"euConsumption24h": eu_consumption,
|
||
"modelUsage": model_usage,
|
||
"weeklyApiCalls": weekly_api_calls,
|
||
"health": {
|
||
"score": 100, # 将在下面计算
|
||
"services": services_health
|
||
}
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取租户仪表盘数据失败: {e}")
|
||
# 返回默认值而不是抛出异常
|
||
return {
|
||
"timestamp": datetime.utcnow().isoformat(),
|
||
"tenantId": tenant_id,
|
||
"euConsumption24h": {"total": 0, "totalCost": 0, "totalCalls": 0, "hourlyData": []},
|
||
"modelUsage": {"models": [], "totalCalls": 0},
|
||
"weeklyApiCalls": {"dailyData": [], "totalSuccess": 0, "totalFailed": 0, "total": 0},
|
||
"health": {
|
||
"score": 100,
|
||
"services": {}
|
||
}
|
||
}
|
||
|
||
async def _get_tenant_eu_consumption_24h(
|
||
self,
|
||
session: AsyncSession,
|
||
tenant_id: str
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
获取租户过去24小时的EU消耗情况
|
||
|
||
✅ 修复：改为查询 agent_billing_records 表
|
||
✅ 增强：合并 model_billing_records 表（LiteLLM Token计费）
|
||
|
||
返回：
|
||
- total: 总EU消耗
|
||
- hourlyData: 每小时的消耗数据
|
||
- agentEU: Agent运行时长计费的EU
|
||
- modelEU: 模型Token计费的EU
|
||
"""
|
||
# 初始化默认值
|
||
agent_row = None
|
||
model_row = None
|
||
hourly_rows = []
|
||
|
||
try:
|
||
# Agent运行时长计费 - 独立捕获异常
|
||
# 注意：agent_billing_records 表使用 cost 列，没有 eu_consumed 列
|
||
try:
|
||
agent_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COALESCE(SUM(cost), 0) as total_cost,
|
||
COUNT(*) as total_calls
|
||
FROM agent_billing_records
|
||
WHERE user_id = :tenant_id
|
||
AND start_time > NOW() - INTERVAL '24 hours'
|
||
"""),
|
||
{"tenant_id": tenant_id}
|
||
)
|
||
agent_row = agent_result.fetchone()
|
||
except Exception as agent_error:
|
||
logger.warning(f"查询agent_billing_records失败: {agent_error}")
|
||
# 不要回滚，继续使用同一个session
|
||
|
||
# 模型Token计费 - 独立捕获异常
|
||
try:
|
||
model_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COALESCE(SUM(eu_consumed), 0) as total_eu,
|
||
COALESCE(SUM(total_cost), 0) as total_cost,
|
||
COUNT(*) as total_calls
|
||
FROM model_billing_records
|
||
WHERE tenant_id = :tenant_id
|
||
AND created_at > NOW() - INTERVAL '24 hours'
|
||
"""),
|
||
{"tenant_id": tenant_id}
|
||
)
|
||
model_row = model_result.fetchone()
|
||
except Exception as model_error:
|
||
logger.warning(f"查询model_billing_records失败: {model_error}")
|
||
# 不要回滚，继续使用同一个session
|
||
|
||
# 每小时消耗 - 仅查询存在的表
|
||
try:
|
||
# 尝试查询model_billing_records的每小时数据
|
||
hourly_result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
DATE_TRUNC('hour', created_at) as hour,
|
||
COALESCE(SUM(eu_consumed), 0) as eu,
|
||
COALESCE(SUM(total_cost), 0) as cost,
|
||
COUNT(*) as calls
|
||
FROM model_billing_records
|
||
WHERE tenant_id = :tenant_id
|
||
AND created_at > NOW() - INTERVAL '24 hours'
|
||
GROUP BY DATE_TRUNC('hour', created_at)
|
||
ORDER BY hour
|
||
"""),
|
||
{"tenant_id": tenant_id}
|
||
)
|
||
hourly_rows = hourly_result.fetchall()
|
||
except Exception as hourly_error:
|
||
logger.warning(f"查询每小时数据失败: {hourly_error}")
|
||
# 不要回滚，继续使用同一个session
|
||
|
||
# 合并数据
|
||
# agent_row: (total_cost, total_calls)
|
||
# model_row: (total_eu, total_cost, total_calls)
|
||
agent_cost = float(agent_row[0]) if agent_row else 0
|
||
agent_calls = int(agent_row[1]) if agent_row else 0
|
||
|
||
model_eu = float(model_row[0]) if model_row else 0
|
||
model_cost = float(model_row[1]) if model_row else 0
|
||
model_calls = int(model_row[2]) if model_row else 0
|
||
|
||
# Agent计费使用cost换算EU（假设1 EU = 1 cost单位）
|
||
agent_eu = agent_cost
|
||
total_eu = agent_eu + model_eu
|
||
total_cost = agent_cost + model_cost
|
||
total_calls = agent_calls + model_calls
|
||
|
||
return {
|
||
"total": total_eu,
|
||
"totalCost": total_cost,
|
||
"totalCalls": total_calls,
|
||
"agentEU": agent_eu,
|
||
"modelEU": model_eu,
|
||
"hourlyData": [
|
||
{
|
||
"timestamp": row[0].isoformat() if row[0] else None,
|
||
"value": float(row[1]),
|
||
"cost": float(row[2]),
|
||
"calls": int(row[3])
|
||
}
|
||
for row in hourly_rows
|
||
]
|
||
}
|
||
except Exception as e:
|
||
logger.warning(f"获取租户EU消耗失败: {e}")
|
||
return {"total": 0, "totalCost": 0, "totalCalls": 0, "agentEU": 0, "modelEU": 0, "hourlyData": []}
|
||
|
||
async def _get_tenant_model_usage(
|
||
self,
|
||
session: AsyncSession,
|
||
tenant_id: str
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
获取租户的模型使用情况
|
||
|
||
✅ 修复：改为查询 model_billing_records 表（LiteLLM Token计费）
|
||
|
||
返回：
|
||
- models: 使用的模型列表及其调用次数
|
||
- totalCalls: 总调用次数
|
||
"""
|
||
try:
|
||
# 查询模型使用统计 - 使用model_billing_records表
|
||
result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
COALESCE(model_name, 'unknown') as model,
|
||
COUNT(*) as calls,
|
||
COALESCE(SUM(eu_consumed), 0) as eu_consumed,
|
||
COALESCE(SUM(total_cost), 0) as cost,
|
||
COALESCE(SUM(total_tokens), 0) as total_tokens
|
||
FROM model_billing_records
|
||
WHERE tenant_id = :tenant_id
|
||
AND created_at > NOW() - INTERVAL '30 days'
|
||
GROUP BY model_name
|
||
ORDER BY calls DESC
|
||
"""),
|
||
{"tenant_id": tenant_id}
|
||
)
|
||
rows = result.fetchall()
|
||
await session.commit() # 确保提交查询
|
||
|
||
models = []
|
||
total_calls = 0
|
||
for row in rows:
|
||
calls = int(row[1])
|
||
total_calls += calls
|
||
models.append({
|
||
"name": row[0],
|
||
"calls": calls,
|
||
"euConsumed": float(row[2]),
|
||
"cost": float(row[3]),
|
||
"totalTokens": int(row[4])
|
||
})
|
||
|
||
return {
|
||
"models": models,
|
||
"totalCalls": total_calls
|
||
}
|
||
except Exception as e:
|
||
logger.warning(f"获取租户模型使用情况失败: {e}")
|
||
# 不要回滚，返回空数据
|
||
return {"models": [], "totalCalls": 0}
|
||
|
||
async def _get_tenant_weekly_api_calls(
|
||
self,
|
||
session: AsyncSession,
|
||
tenant_id: str
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
获取租户每周API调用统计（7天每天成功/失败）
|
||
|
||
✅ 修复：改为查询 model_billing_records 表（模型调用统计）
|
||
|
||
返回：
|
||
- dailyData: 7天每天的调用数据
|
||
- totalSuccess: 总成功次数
|
||
- totalFailed: 总失败次数
|
||
"""
|
||
try:
|
||
# 查询7天每天的模型调用统计
|
||
result = await session.execute(
|
||
text("""
|
||
SELECT
|
||
DATE(created_at) as date,
|
||
COUNT(*) as total,
|
||
COUNT(CASE WHEN status = 'success' THEN 1 END) as success,
|
||
COUNT(CASE WHEN status != 'success' THEN 1 END) as failed
|
||
FROM model_billing_records
|
||
WHERE tenant_id = :tenant_id
|
||
AND created_at > NOW() - INTERVAL '7 days'
|
||
GROUP BY DATE(created_at)
|
||
ORDER BY date
|
||
"""),
|
||
{"tenant_id": tenant_id}
|
||
)
|
||
rows = result.fetchall()
|
||
await session.commit() # 确保提交查询
|
||
|
||
daily_data = []
|
||
total_success = 0
|
||
total_failed = 0
|
||
|
||
# 生成过去7天的日期列表
|
||
today = datetime.utcnow().date()
|
||
date_map = {}
|
||
for row in rows:
|
||
if row[0]:
|
||
date_map[row[0]] = {
|
||
"total": int(row[1]),
|
||
"success": int(row[2]),
|
||
"failed": int(row[3])
|
||
}
|
||
total_success += int(row[2])
|
||
total_failed += int(row[3])
|
||
|
||
# 填充7天数据（包括没有数据的日期）
|
||
for i in range(6, -1, -1):
|
||
date = today - timedelta(days=i)
|
||
if date in date_map:
|
||
daily_data.append({
|
||
"date": date.isoformat(),
|
||
"total": date_map[date]["total"],
|
||
"success": date_map[date]["success"],
|
||
"failed": date_map[date]["failed"]
|
||
})
|
||
else:
|
||
daily_data.append({
|
||
"date": date.isoformat(),
|
||
"total": 0,
|
||
"success": 0,
|
||
"failed": 0
|
||
})
|
||
|
||
return {
|
||
"dailyData": daily_data,
|
||
"totalSuccess": total_success,
|
||
"totalFailed": total_failed,
|
||
"total": total_success + total_failed
|
||
}
|
||
except Exception as e:
|
||
logger.warning(f"获取租户每周API调用统计失败: {e}")
|
||
# 不要回滚，返回空数据
|
||
# 返回空的7天数据
|
||
today = datetime.utcnow().date()
|
||
daily_data = [
|
||
{"date": (today - timedelta(days=i)).isoformat(), "total": 0, "success": 0, "failed": 0}
|
||
for i in range(6, -1, -1)
|
||
]
|
||
return {"dailyData": daily_data, "totalSuccess": 0, "totalFailed": 0, "total": 0}
|
||
|
||
|
||
# 全局监控实例
|
||
system_monitor = SystemMonitor()
|
||
|