forked from xiaohei/taiji-AI-PAD
729 lines
29 KiB
Python
729 lines
29 KiB
Python
"""
|
|
平台监控模块
|
|
提供系统健康检查、性能指标、资源使用等监控功能
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import psutil
|
|
import time
|
|
import httpx
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Optional, Any
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import text, func
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from models import Agent, Execution, User, Tool, Session
|
|
from database import AsyncSessionLocal
|
|
from config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class SystemMonitor:
|
|
"""系统监控类"""
|
|
|
|
def __init__(self):
|
|
self.start_time = datetime.utcnow()
|
|
# 服务健康检查端点配置
|
|
self._service_endpoints = {
|
|
"mcp_server": "http://localhost:8000/health", # 本服务
|
|
"data_ingestion": "http://data-ingestion:8000/health", # Docker 内部网络
|
|
"model_gateway": "https://litellm.graystone-fb459c5d.southeastasia.azurecontainerapps.io/health", # LiteLLM 模型网关
|
|
"agent_manager": "http://127.0.0.1:8000/health", # Agent Manager 服务（本地）
|
|
}
|
|
# LiteLLM API Key
|
|
self._litellm_api_key = "sk-taiji-prod-2026"
|
|
|
|
async def get_system_health(self) -> Dict[str, Any]:
|
|
"""获取系统健康状态，包括三个微服务的状态"""
|
|
health = {
|
|
"status": "healthy",
|
|
"score": 100,
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"uptime_seconds": (datetime.utcnow() - self.start_time).total_seconds(),
|
|
"services": {}
|
|
}
|
|
|
|
# 检查数据库（内部使用，不对外暴露）
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
await session.execute(text("SELECT 1"))
|
|
# 数据库健康检查成功，但不添加到 services 中
|
|
except Exception as e:
|
|
# 数据库不健康会影响整体状态
|
|
health["status"] = "degraded"
|
|
health["score"] -= 25
|
|
|
|
# 检查三个微服务状态
|
|
service_health = await self._check_microservices_health()
|
|
health["services"].update(service_health)
|
|
|
|
# 计算整体健康分数
|
|
unhealthy_count = sum(1 for s in health["services"].values()
|
|
if s.get("status") not in ["healthy", "running"])
|
|
if unhealthy_count > 0:
|
|
health["score"] = max(0, 100 - (unhealthy_count * 20))
|
|
if unhealthy_count >= 2:
|
|
health["status"] = "degraded"
|
|
if unhealthy_count >= 3:
|
|
health["status"] = "unhealthy"
|
|
|
|
return health
|
|
|
|
async def _check_microservices_health(self) -> Dict[str, Dict[str, Any]]:
|
|
"""
|
|
检查四个微服务的健康状态
|
|
|
|
返回格式：
|
|
{
|
|
"mcp_server": { "status": "healthy", "latency": 45 },
|
|
"data_ingestion": { "status": "healthy", "latency": 32 },
|
|
"model_gateway": { "status": "healthy", "latency": 28 },
|
|
"agent_manager": { "status": "healthy", "latency": 15 }
|
|
}
|
|
|
|
状态值：
|
|
- healthy/running: 运行中
|
|
- maintenance: 维护中
|
|
- error: 错误
|
|
"""
|
|
results = {}
|
|
|
|
async def check_service(name: str, url: str, headers: Dict[str, str] = None) -> Dict[str, Any]:
|
|
"""检查单个服务的健康状态"""
|
|
start_time = time.time()
|
|
try:
|
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
|
response = await client.get(url, headers=headers)
|
|
latency = int((time.time() - start_time) * 1000)
|
|
|
|
if response.status_code == 200:
|
|
return {"status": "healthy", "latency": latency}
|
|
elif response.status_code == 503:
|
|
return {"status": "maintenance", "latency": latency}
|
|
else:
|
|
return {"status": "error", "latency": latency, "code": response.status_code}
|
|
except httpx.TimeoutException:
|
|
return {"status": "error", "latency": 5000, "error": "timeout"}
|
|
except httpx.ConnectError:
|
|
# 服务不可达，可能是维护中或未启动
|
|
return {"status": "maintenance", "latency": 0, "error": "connection_refused"}
|
|
except Exception as e:
|
|
return {"status": "error", "latency": 0, "error": str(e)}
|
|
|
|
# mcp_server 本服务始终是健康的（因为能响应请求）
|
|
results["mcp_server"] = {"status": "healthy", "latency": 1}
|
|
|
|
# 检查 data_ingestion 服务
|
|
results["data_ingestion"] = await check_service(
|
|
"data_ingestion",
|
|
self._service_endpoints["data_ingestion"]
|
|
)
|
|
|
|
# 检查 model_gateway (LiteLLM) - 需要带上 API Key
|
|
results["model_gateway"] = await check_service(
|
|
"model_gateway",
|
|
self._service_endpoints["model_gateway"],
|
|
headers={"Authorization": f"Bearer {self._litellm_api_key}"}
|
|
)
|
|
|
|
# 检查 agent_manager 服务
|
|
results["agent_manager"] = await check_service(
|
|
"agent_manager",
|
|
self._service_endpoints["agent_manager"]
|
|
)
|
|
|
|
return results
|
|
|
|
async def get_system_metrics(self) -> Dict[str, Any]:
|
|
"""获取系统性能指标"""
|
|
try:
|
|
# 系统资源使用
|
|
cpu_percent = psutil.cpu_percent(interval=1)
|
|
memory = psutil.virtual_memory()
|
|
disk = psutil.disk_usage('/')
|
|
|
|
# 数据库统计
|
|
async with AsyncSessionLocal() as session:
|
|
# Agent统计
|
|
agent_result = await session.execute(
|
|
text("SELECT COUNT(*) FROM agents WHERE status = 'active'")
|
|
)
|
|
active_agents = agent_result.scalar() or 0
|
|
|
|
# 执行统计
|
|
execution_result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
AVG(execution_time) as avg_time,
|
|
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END)::float / NULLIF(COUNT(*), 0) * 100 as success_rate
|
|
FROM executions
|
|
WHERE started_at > NOW() - INTERVAL '24 hours'
|
|
""")
|
|
)
|
|
exec_stats = execution_result.fetchone()
|
|
total_executions = exec_stats[0] or 0 if exec_stats else 0
|
|
avg_execution_time = float(exec_stats[1] or 0) if exec_stats and exec_stats[1] else 0.0
|
|
success_rate = float(exec_stats[2] or 0) if exec_stats and exec_stats[2] else 0.0
|
|
|
|
# 用户统计
|
|
user_result = await session.execute(
|
|
text("""
|
|
SELECT COUNT(DISTINCT user_id)
|
|
FROM sessions
|
|
WHERE created_at > NOW() - INTERVAL '24 hours'
|
|
""")
|
|
)
|
|
daily_active_users = user_result.scalar() or 0
|
|
|
|
# EU消耗统计
|
|
eu_result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
SUM(eu_consumed) as total_eu,
|
|
SUM(cost) as total_cost
|
|
FROM billing
|
|
WHERE created_at > NOW() - INTERVAL '24 hours'
|
|
""")
|
|
)
|
|
eu_stats = eu_result.fetchone()
|
|
total_eu = float(eu_stats[0] or 0) if eu_stats and eu_stats[0] else 0.0
|
|
total_cost = float(eu_stats[1] or 0) if eu_stats and eu_stats[1] else 0.0
|
|
|
|
return {
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"system": {
|
|
"cpu_usage_percent": cpu_percent,
|
|
"memory_usage_percent": memory.percent,
|
|
"memory_used_mb": memory.used / 1024 / 1024,
|
|
"memory_total_mb": memory.total / 1024 / 1024,
|
|
"disk_usage_percent": disk.percent,
|
|
"disk_used_gb": disk.used / 1024 / 1024 / 1024,
|
|
"disk_total_gb": disk.total / 1024 / 1024 / 1024,
|
|
},
|
|
"services": {
|
|
"active_agents": active_agents,
|
|
"total_executions_24h": total_executions,
|
|
"success_rate_percent": round(success_rate, 2),
|
|
"avg_execution_time_ms": round(avg_execution_time, 2),
|
|
"daily_active_users": daily_active_users,
|
|
},
|
|
"billing": {
|
|
"total_eu_consumed_24h": round(total_eu, 4),
|
|
"total_cost_24h": round(total_cost, 4),
|
|
}
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"获取系统指标失败: {e}")
|
|
raise
|
|
|
|
async def get_service_stats(self, service: str = "all") -> Dict[str, Any]:
|
|
"""获取服务统计信息"""
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
stats = {}
|
|
|
|
if service == "all" or service == "agents":
|
|
# Agent统计
|
|
agent_stats = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNT(CASE WHEN status = 'active' THEN 1 END) as active,
|
|
COUNT(CASE WHEN status = 'inactive' THEN 1 END) as inactive,
|
|
AVG(total_executions) as avg_executions,
|
|
AVG(success_rate) as avg_success_rate
|
|
FROM agents
|
|
""")
|
|
)
|
|
row = agent_stats.fetchone()
|
|
if row:
|
|
stats["agents"] = {
|
|
"total": row[0] or 0,
|
|
"active": row[1] or 0,
|
|
"inactive": row[2] or 0,
|
|
"avg_executions": float(row[3] or 0),
|
|
"avg_success_rate": float(row[4] or 0),
|
|
}
|
|
|
|
if service == "all" or service == "executions":
|
|
# 执行统计
|
|
exec_stats = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNT(CASE WHEN status = 'completed' THEN 1 END) as completed,
|
|
COUNT(CASE WHEN status = 'failed' THEN 1 END) as failed,
|
|
COUNT(CASE WHEN status = 'running' THEN 1 END) as running,
|
|
AVG(execution_time) as avg_time,
|
|
SUM(eu_consumed) as total_eu
|
|
FROM executions
|
|
WHERE started_at > NOW() - INTERVAL '7 days'
|
|
""")
|
|
)
|
|
row = exec_stats.fetchone()
|
|
if row:
|
|
stats["executions"] = {
|
|
"total_7d": row[0] or 0,
|
|
"completed": row[1] or 0,
|
|
"failed": row[2] or 0,
|
|
"running": row[3] or 0,
|
|
"avg_time_ms": float(row[4] or 0),
|
|
"total_eu": float(row[5] or 0),
|
|
}
|
|
|
|
if service == "all" or service == "tools":
|
|
# 工具统计
|
|
tool_stats = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNT(CASE WHEN is_active = true THEN 1 END) as active,
|
|
SUM(total_calls) as total_calls,
|
|
AVG(success_rate) as avg_success_rate,
|
|
AVG(avg_response_time) as avg_response_time
|
|
FROM tools
|
|
""")
|
|
)
|
|
row = tool_stats.fetchone()
|
|
if row:
|
|
stats["tools"] = {
|
|
"total": row[0] or 0,
|
|
"active": row[1] or 0,
|
|
"total_calls": row[2] or 0,
|
|
"avg_success_rate": float(row[3] or 0),
|
|
"avg_response_time_ms": float(row[4] or 0),
|
|
}
|
|
|
|
if service == "all" or service == "users":
|
|
# 用户统计
|
|
user_stats = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNT(CASE WHEN is_active = true THEN 1 END) as active,
|
|
COUNT(CASE WHEN is_admin = true THEN 1 END) as admins
|
|
FROM users
|
|
""")
|
|
)
|
|
row = user_stats.fetchone()
|
|
if row:
|
|
stats["users"] = {
|
|
"total": row[0] or 0,
|
|
"active": row[1] or 0,
|
|
"admins": row[2] or 0,
|
|
}
|
|
|
|
return {
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"stats": stats
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"获取服务统计失败: {e}")
|
|
raise
|
|
|
|
async def get_performance_trends(
|
|
self,
|
|
metric: str = "executions",
|
|
period: str = "24h",
|
|
interval: str = "1h"
|
|
) -> Dict[str, Any]:
|
|
"""获取性能趋势数据"""
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
# 计算时间范围
|
|
if period == "24h":
|
|
hours = 24
|
|
elif period == "7d":
|
|
hours = 168
|
|
elif period == "30d":
|
|
hours = 720
|
|
else:
|
|
hours = 24
|
|
|
|
if interval == "1h":
|
|
interval_sql = "1 hour"
|
|
elif interval == "6h":
|
|
interval_sql = "6 hours"
|
|
elif interval == "1d":
|
|
interval_sql = "1 day"
|
|
else:
|
|
interval_sql = "1 hour"
|
|
|
|
if metric == "executions":
|
|
result = await session.execute(
|
|
text(f"""
|
|
SELECT
|
|
DATE_TRUNC('hour', started_at) as time_bucket,
|
|
COUNT(*) as count,
|
|
AVG(execution_time) as avg_time,
|
|
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END)::float / NULLIF(COUNT(*), 0) * 100 as success_rate
|
|
FROM executions
|
|
WHERE started_at > NOW() - INTERVAL '{hours} hours'
|
|
GROUP BY time_bucket
|
|
ORDER BY time_bucket
|
|
""")
|
|
)
|
|
rows = result.fetchall()
|
|
return {
|
|
"metric": metric,
|
|
"period": period,
|
|
"interval": interval,
|
|
"data": [
|
|
{
|
|
"timestamp": row[0].isoformat() if row[0] else None,
|
|
"count": row[1] or 0,
|
|
"avg_time_ms": float(row[2] or 0),
|
|
"success_rate": float(row[3] or 0),
|
|
}
|
|
for row in rows
|
|
]
|
|
}
|
|
elif metric == "eu_consumption":
|
|
result = await session.execute(
|
|
text(f"""
|
|
SELECT
|
|
DATE_TRUNC('hour', created_at) as time_bucket,
|
|
SUM(eu_consumed) as total_eu,
|
|
SUM(cost) as total_cost
|
|
FROM billing
|
|
WHERE created_at > NOW() - INTERVAL '{hours} hours'
|
|
GROUP BY time_bucket
|
|
ORDER BY time_bucket
|
|
""")
|
|
)
|
|
rows = result.fetchall()
|
|
return {
|
|
"metric": metric,
|
|
"period": period,
|
|
"interval": interval,
|
|
"data": [
|
|
{
|
|
"timestamp": row[0].isoformat() if row[0] else None,
|
|
"eu_consumed": float(row[1] or 0),
|
|
"cost": float(row[2] or 0),
|
|
}
|
|
for row in rows
|
|
]
|
|
}
|
|
else:
|
|
return {
|
|
"metric": metric,
|
|
"period": period,
|
|
"interval": interval,
|
|
"data": []
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"获取性能趋势失败: {e}")
|
|
raise
|
|
|
|
async def get_alerts(self, severity: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
"""获取系统告警"""
|
|
alerts = []
|
|
|
|
try:
|
|
# 检查系统资源
|
|
cpu_percent = psutil.cpu_percent(interval=1)
|
|
memory = psutil.virtual_memory()
|
|
disk = psutil.disk_usage('/')
|
|
|
|
if cpu_percent > 80:
|
|
alerts.append({
|
|
"severity": "warning",
|
|
"type": "high_cpu",
|
|
"message": f"CPU使用率过高: {cpu_percent:.1f}%",
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
})
|
|
|
|
if memory.percent > 85:
|
|
alerts.append({
|
|
"severity": "warning",
|
|
"type": "high_memory",
|
|
"message": f"内存使用率过高: {memory.percent:.1f}%",
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
})
|
|
|
|
if disk.percent > 90:
|
|
alerts.append({
|
|
"severity": "critical",
|
|
"type": "low_disk",
|
|
"message": f"磁盘空间不足: {disk.percent:.1f}%",
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
})
|
|
|
|
# 检查服务健康
|
|
async with AsyncSessionLocal() as session:
|
|
# 检查失败的执行
|
|
failed_result = await session.execute(
|
|
text("""
|
|
SELECT COUNT(*)
|
|
FROM executions
|
|
WHERE status = 'failed'
|
|
AND started_at > NOW() - INTERVAL '1 hour'
|
|
""")
|
|
)
|
|
failed_count = failed_result.scalar() or 0
|
|
|
|
if failed_count > 10:
|
|
alerts.append({
|
|
"severity": "warning",
|
|
"type": "high_failure_rate",
|
|
"message": f"过去1小时内有 {failed_count} 次执行失败",
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
})
|
|
|
|
# 过滤严重程度
|
|
if severity:
|
|
alerts = [a for a in alerts if a["severity"] == severity]
|
|
|
|
return alerts
|
|
except Exception as e:
|
|
logger.error(f"获取告警失败: {e}")
|
|
return []
|
|
|
|
async def get_tenant_dashboard(self, tenant_id: str) -> Dict[str, Any]:
|
|
"""
|
|
获取租户级别的监控仪表盘数据
|
|
|
|
包含：
|
|
1. 当前租户EU消耗过去24小时的消耗情况
|
|
2. 使用的模型和模型的使用情况、当前调用次数
|
|
3. 每周API调用的次数，7天的每一天调用多少次，包括成功和失败
|
|
4. 三个微服务的状态
|
|
"""
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
# 1. 获取租户过去24小时的EU消耗
|
|
eu_consumption = await self._get_tenant_eu_consumption_24h(session, tenant_id)
|
|
|
|
# 2. 获取模型使用情况
|
|
model_usage = await self._get_tenant_model_usage(session, tenant_id)
|
|
|
|
# 3. 获取每周API调用统计（7天每天成功/失败）
|
|
weekly_api_calls = await self._get_tenant_weekly_api_calls(session, tenant_id)
|
|
|
|
# 4. 获取微服务状态
|
|
services_health = await self._check_microservices_health()
|
|
|
|
return {
|
|
"timestamp": datetime.utcnow().isoformat(),
|
|
"tenantId": tenant_id,
|
|
"euConsumption24h": eu_consumption,
|
|
"modelUsage": model_usage,
|
|
"weeklyApiCalls": weekly_api_calls,
|
|
"health": {
|
|
"score": 100, # 将在下面计算
|
|
"services": services_health
|
|
}
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"获取租户仪表盘数据失败: {e}")
|
|
raise
|
|
|
|
async def _get_tenant_eu_consumption_24h(
|
|
self,
|
|
session: AsyncSession,
|
|
tenant_id: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
获取租户过去24小时的EU消耗情况
|
|
|
|
返回：
|
|
- total: 总EU消耗
|
|
- hourlyData: 每小时的消耗数据
|
|
"""
|
|
try:
|
|
# 总消耗
|
|
total_result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COALESCE(SUM(eu), 0) as total_eu,
|
|
COALESCE(SUM(cost), 0) as total_cost,
|
|
COUNT(*) as total_calls
|
|
FROM billing_records
|
|
WHERE tenant_id = :tenant_id
|
|
AND timestamp > NOW() - INTERVAL '24 hours'
|
|
"""),
|
|
{"tenant_id": tenant_id}
|
|
)
|
|
total_row = total_result.fetchone()
|
|
|
|
# 每小时消耗
|
|
hourly_result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
DATE_TRUNC('hour', timestamp) as hour,
|
|
COALESCE(SUM(eu), 0) as eu,
|
|
COALESCE(SUM(cost), 0) as cost,
|
|
COUNT(*) as calls
|
|
FROM billing_records
|
|
WHERE tenant_id = :tenant_id
|
|
AND timestamp > NOW() - INTERVAL '24 hours'
|
|
GROUP BY DATE_TRUNC('hour', timestamp)
|
|
ORDER BY hour
|
|
"""),
|
|
{"tenant_id": tenant_id}
|
|
)
|
|
hourly_rows = hourly_result.fetchall()
|
|
|
|
return {
|
|
"total": float(total_row[0]) if total_row else 0,
|
|
"totalCost": float(total_row[1]) if total_row else 0,
|
|
"totalCalls": int(total_row[2]) if total_row else 0,
|
|
"hourlyData": [
|
|
{
|
|
"timestamp": row[0].isoformat() if row[0] else None,
|
|
"value": float(row[1]),
|
|
"cost": float(row[2]),
|
|
"calls": int(row[3])
|
|
}
|
|
for row in hourly_rows
|
|
]
|
|
}
|
|
except Exception as e:
|
|
logger.warning(f"获取租户EU消耗失败: {e}")
|
|
return {"total": 0, "totalCost": 0, "totalCalls": 0, "hourlyData": []}
|
|
|
|
async def _get_tenant_model_usage(
|
|
self,
|
|
session: AsyncSession,
|
|
tenant_id: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
获取租户的模型使用情况
|
|
|
|
返回：
|
|
- models: 使用的模型列表及其调用次数
|
|
- totalCalls: 总调用次数
|
|
"""
|
|
try:
|
|
# 查询模型使用统计
|
|
result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
COALESCE(model_name, 'unknown') as model,
|
|
COUNT(*) as calls,
|
|
COALESCE(SUM(eu), 0) as eu_consumed,
|
|
COALESCE(SUM(cost), 0) as cost
|
|
FROM billing_records
|
|
WHERE tenant_id = :tenant_id
|
|
AND timestamp > NOW() - INTERVAL '30 days'
|
|
GROUP BY model_name
|
|
ORDER BY calls DESC
|
|
"""),
|
|
{"tenant_id": tenant_id}
|
|
)
|
|
rows = result.fetchall()
|
|
|
|
models = []
|
|
total_calls = 0
|
|
for row in rows:
|
|
calls = int(row[1])
|
|
total_calls += calls
|
|
models.append({
|
|
"name": row[0],
|
|
"calls": calls,
|
|
"euConsumed": float(row[2]),
|
|
"cost": float(row[3])
|
|
})
|
|
|
|
return {
|
|
"models": models,
|
|
"totalCalls": total_calls
|
|
}
|
|
except Exception as e:
|
|
logger.warning(f"获取租户模型使用情况失败: {e}")
|
|
return {"models": [], "totalCalls": 0}
|
|
|
|
async def _get_tenant_weekly_api_calls(
|
|
self,
|
|
session: AsyncSession,
|
|
tenant_id: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
获取租户每周API调用统计（7天每天成功/失败）
|
|
|
|
返回：
|
|
- dailyData: 7天每天的调用数据
|
|
- totalSuccess: 总成功次数
|
|
- totalFailed: 总失败次数
|
|
"""
|
|
try:
|
|
# 查询7天每天的调用统计
|
|
result = await session.execute(
|
|
text("""
|
|
SELECT
|
|
DATE(timestamp) as date,
|
|
COUNT(*) as total,
|
|
COUNT(CASE WHEN status = 'success' OR status = 'completed' THEN 1 END) as success,
|
|
COUNT(CASE WHEN status = 'failed' OR status = 'error' THEN 1 END) as failed
|
|
FROM billing_records
|
|
WHERE tenant_id = :tenant_id
|
|
AND timestamp > NOW() - INTERVAL '7 days'
|
|
GROUP BY DATE(timestamp)
|
|
ORDER BY date
|
|
"""),
|
|
{"tenant_id": tenant_id}
|
|
)
|
|
rows = result.fetchall()
|
|
|
|
daily_data = []
|
|
total_success = 0
|
|
total_failed = 0
|
|
|
|
# 生成过去7天的日期列表
|
|
today = datetime.utcnow().date()
|
|
date_map = {}
|
|
for row in rows:
|
|
if row[0]:
|
|
date_map[row[0]] = {
|
|
"total": int(row[1]),
|
|
"success": int(row[2]),
|
|
"failed": int(row[3])
|
|
}
|
|
total_success += int(row[2])
|
|
total_failed += int(row[3])
|
|
|
|
# 填充7天数据（包括没有数据的日期）
|
|
for i in range(6, -1, -1):
|
|
date = today - timedelta(days=i)
|
|
if date in date_map:
|
|
daily_data.append({
|
|
"date": date.isoformat(),
|
|
"total": date_map[date]["total"],
|
|
"success": date_map[date]["success"],
|
|
"failed": date_map[date]["failed"]
|
|
})
|
|
else:
|
|
daily_data.append({
|
|
"date": date.isoformat(),
|
|
"total": 0,
|
|
"success": 0,
|
|
"failed": 0
|
|
})
|
|
|
|
return {
|
|
"dailyData": daily_data,
|
|
"totalSuccess": total_success,
|
|
"totalFailed": total_failed,
|
|
"total": total_success + total_failed
|
|
}
|
|
except Exception as e:
|
|
logger.warning(f"获取租户每周API调用统计失败: {e}")
|
|
# 返回空的7天数据
|
|
today = datetime.utcnow().date()
|
|
daily_data = [
|
|
{"date": (today - timedelta(days=i)).isoformat(), "total": 0, "success": 0, "failed": 0}
|
|
for i in range(6, -1, -1)
|
|
]
|
|
return {"dailyData": daily_data, "totalSuccess": 0, "totalFailed": 0, "total": 0}
|
|
|
|
|
|
# 全局监控实例
|
|
system_monitor = SystemMonitor()
|
|
|