feat: 完成taiji-AI-PAD核心架构和代码实现

✨ 新功能:
- 实现五层技术架构的容器化部署
- 完成MCP Server核心服务 (Python/FastAPI)
- 实现数据接入服务框架 (RapidAPI/APILLAMA)
- 配置LiteLLM模型网关 (100+模型支持)
- 设置完整的监控体系 (Prometheus/Grafana)

🏗️ 基础设施:
- Docker Compose编排配置
- Nginx API网关
- PostgreSQL + Redis + NATS
- 完整的启动/停止/测试脚本

📚 文档:
- 工程排期计划
- 任务拆分与分工
- 系统运作流程图
- 项目开发状态文档

🔧 开发工具:
- 容器化开发环境
- 自动化测试脚本
- API文档自动生成
- 健康检查机制

该提交包含了约60%的项目核心功能实现,可以进行基础的Agent管理和MCP协议通信。
This commit is contained in:
2025-12-20 05:31:03 +00:00
parent dd4417e34c
commit cd0719e1cb
26 changed files with 5928 additions and 0 deletions
+106
View File
@@ -0,0 +1,106 @@
-- taiji-AI-PAD 数据库初始化脚本
-- 创建扩展
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "pg_trgm";
-- 创建数据库(如果不存在)
-- 注意:在Docker初始化脚本中,数据库已经存在
-- 设置时区
SET timezone = 'UTC';
-- 创建一些基础索引(如果表已存在的话,模型会自动创建)
-- 这里可以添加一些额外的性能优化索引
-- 创建全文搜索配置
CREATE TEXT SEARCH CONFIGURATION IF NOT EXISTS simple_english (COPY = english);
-- 创建一些有用的函数
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
-- 日志表(用于审计和调试)
CREATE TABLE IF NOT EXISTS system_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
level VARCHAR(20) NOT NULL,
service VARCHAR(50) NOT NULL,
message TEXT NOT NULL,
context JSONB,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_system_logs_service ON system_logs(service);
CREATE INDEX IF NOT EXISTS idx_system_logs_level ON system_logs(level);
CREATE INDEX IF NOT EXISTS idx_system_logs_created ON system_logs(created_at);
-- 配置表
CREATE TABLE IF NOT EXISTS system_config (
key VARCHAR(100) PRIMARY KEY,
value JSONB NOT NULL,
description TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
-- 创建触发器
CREATE TRIGGER update_system_config_updated_at
BEFORE UPDATE ON system_config
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
-- 插入初始配置
INSERT INTO system_config (key, value, description) VALUES
('app_version', '"1.0.0"', 'Application version'),
('maintenance_mode', 'false', 'Maintenance mode flag'),
('max_api_calls_per_minute', '1000', 'Maximum API calls per minute'),
('default_timeout', '30', 'Default timeout in seconds')
ON CONFLICT (key) DO NOTHING;
-- 性能优化设置
-- 注意:这些设置可能需要根据实际硬件调整
ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements';
ALTER SYSTEM SET max_connections = 200;
ALTER SYSTEM SET shared_buffers = '256MB';
ALTER SYSTEM SET effective_cache_size = '1GB';
ALTER SYSTEM SET maintenance_work_mem = '64MB';
ALTER SYSTEM SET checkpoint_completion_target = 0.9;
ALTER SYSTEM SET wal_buffers = '16MB';
ALTER SYSTEM SET default_statistics_target = 100;
ALTER SYSTEM SET random_page_cost = 1.1;
ALTER SYSTEM SET effective_io_concurrency = 200;
ALTER SYSTEM SET work_mem = '4MB';
ALTER SYSTEM SET min_wal_size = '1GB';
ALTER SYSTEM SET max_wal_size = '4GB';
-- 创建监控视图
CREATE OR REPLACE VIEW system_stats AS
SELECT
schemaname,
tablename,
attname,
n_distinct,
correlation
FROM pg_stats
WHERE schemaname = 'public';
-- 创建连接监控视图
CREATE OR REPLACE VIEW connection_stats AS
SELECT
datname,
numbackends,
xact_commit,
xact_rollback,
blks_read,
blks_hit,
tup_returned,
tup_fetched,
tup_inserted,
tup_updated,
tup_deleted
FROM pg_stat_database
WHERE datname = current_database();
+168
View File
@@ -0,0 +1,168 @@
#!/bin/bash
# taiji-AI-PAD 启动脚本
set -e
echo "🚀 启动 taiji-AI-PAD 平台..."
# 检查Docker是否运行
if ! docker info >/dev/null 2>&1; then
echo "❌ Docker 未运行,请先启动Docker"
exit 1
fi
# 检查Docker Compose是否可用
if ! command -v docker-compose >/dev/null 2>&1; then
echo "❌ Docker Compose 未找到,请安装Docker Compose"
exit 1
fi
# 创建必要的目录
echo "📁 创建必要的目录..."
mkdir -p logs
mkdir -p config/ssl
mkdir -p services/model-gateway/config
mkdir -p services/data-ingestion/models
mkdir -p services/data-ingestion/cache
# 设置环境变量(如果.env文件不存在)
if [ ! -f .env ]; then
echo "⚙️ 创建环境配置文件..."
cat > .env << EOF
# 环境设置
ENVIRONMENT=development
# 数据库设置
POSTGRES_DB=taiji_db
POSTGRES_USER=taiji_user
POSTGRES_PASSWORD=taiji_pass
DATABASE_URL=postgresql+asyncpg://taiji_user:taiji_pass@postgres:5432/taiji_db
# Redis设置
REDIS_URL=redis://redis:6379
# NATS设置
NATS_URL=nats://nats:4222
# LiteLLM设置
LITELLM_MASTER_KEY=sk-taiji-master-key
LITELLM_URL=http://litellm-gateway:4000
# RapidAPI设置(需要实际的API Key)
RAPIDAPI_KEY=your-rapidapi-key-here
RAPIDAPI_HOST=rapidapi.com
# APILLAMA模型设置
APILLAMA_MODEL_PATH=/app/models/llama-3-8b-instruct
APILLAMA_DEVICE=cpu
EOF
echo "✅ 环境配置文件已创建,请根据需要修改 .env 文件"
fi
# 检查必要的配置文件
if [ ! -f config/nginx.conf ]; then
echo "❌ Nginx配置文件未找到:config/nginx.conf"
exit 1
fi
if [ ! -f scripts/init.sql ]; then
echo "❌ 数据库初始化脚本未找到:scripts/init.sql"
exit 1
fi
# 拉取基础镜像
echo "⬇️ 拉取基础镜像..."
docker-compose pull postgres redis nats prometheus grafana nginx
# 构建服务镜像
echo "🏗️ 构建服务镜像..."
docker-compose build
# 启动基础设施服务
echo "🗄️ 启动基础设施服务..."
docker-compose up -d postgres redis nats
# 等待数据库就绪
echo "⏳ 等待数据库就绪..."
sleep 10
# 检查数据库连接
echo "🔍 检查数据库连接..."
until docker-compose exec -T postgres pg_isready -U taiji_user -d taiji_db; do
echo "等待数据库..."
sleep 2
done
# 启动应用服务
echo "🚀 启动应用服务..."
docker-compose up -d
# 等待服务启动
echo "⏳ 等待服务启动..."
sleep 15
# 检查服务状态
echo "🔍 检查服务状态..."
docker-compose ps
# 健康检查
echo "🏥 执行健康检查..."
services=("mcp-server:8002" "data-ingestion:8001" "agent-registry:8003" "billing-engine:8004")
for service in "${services[@]}"; do
service_name=$(echo $service | cut -d':' -f1)
port=$(echo $service | cut -d':' -f2)
echo "检查 $service_name..."
if curl -f -s http://localhost:$port/health > /dev/null; then
echo "✅ $service_name 健康"
else
echo "⚠️ $service_name 可能未就绪"
fi
done
# 显示访问信息
echo ""
echo "🎉 taiji-AI-PAD 启动完成!"
echo ""
echo "📊 服务访问地址:"
echo " • API网关: http://localhost"
echo " • MCP服务器: http://localhost:8002"
echo " • 数据接入服务: http://localhost:8001"
echo " • Agent注册中心: http://localhost:8003"
echo " • 计费引擎: http://localhost:8004"
echo " • LiteLLM网关: http://localhost:4000"
echo ""
echo "📈 监控服务:"
echo " • Grafana: http://localhost:3000 (admin/admin)"
echo " • Prometheus: http://localhost:9090"
echo ""
echo "🗄️ 数据库服务:"
echo " • PostgreSQL: localhost:5432"
echo " • Redis: localhost:6379"
echo " • NATS: localhost:4222"
echo ""
echo "📚 API文档:"
echo " • MCP服务器: http://localhost:8002/docs"
echo " • 数据接入服务: http://localhost:8001/docs"
echo ""
echo "🔧 管理命令:"
echo " • 查看日志: docker-compose logs -f [服务名]"
echo " • 停止服务: docker-compose down"
echo " • 重启服务: docker-compose restart [服务名]"
echo ""
# 开发环境提示
if [ "$ENVIRONMENT" = "development" ]; then
echo "🔧 开发环境提示:"
echo " • 代码变更会自动重载"
echo " • 日志级别设置为DEBUG"
echo " • 请确保修改.env文件中的API密钥"
echo ""
fi
echo "🎯 接下来您可以:"
echo " 1. 访问 http://localhost:8002/docs 查看MCP API文档"
echo " 2. 使用 scripts/test.sh 运行测试"
echo " 3. 查看 docs/ 目录了解更多使用方法"
+57
View File
@@ -0,0 +1,57 @@
#!/bin/bash
# taiji-AI-PAD 停止脚本
set -e
echo "🛑 停止 taiji-AI-PAD 平台..."
# 检查Docker Compose是否可用
if ! command -v docker-compose >/dev/null 2>&1; then
echo "❌ Docker Compose 未找到"
exit 1
fi
# 显示当前运行的服务
echo "📋 当前运行的服务:"
docker-compose ps
# 停止所有服务
echo "⏹️ 停止所有服务..."
docker-compose down
# 可选:清理数据卷(谨慎使用)
if [ "$1" = "--clean" ]; then
echo "🧹 清理数据卷..."
read -p "⚠️ 这将删除所有数据,是否继续? (y/N): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
docker-compose down -v
docker system prune -f
echo "✅ 数据卷已清理"
else
echo "❌ 已取消清理操作"
fi
fi
# 可选:清理镜像
if [ "$1" = "--clean-all" ]; then
echo "🧹 清理镜像和数据..."
read -p "⚠️ 这将删除所有镜像和数据,是否继续? (y/N): " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
docker-compose down -v --rmi all
docker system prune -a -f
echo "✅ 镜像和数据已清理"
else
echo "❌ 已取消清理操作"
fi
fi
echo ""
echo "✅ taiji-AI-PAD 已停止"
echo ""
echo "💡 清理选项:"
echo " • 清理数据卷: ./scripts/stop.sh --clean"
echo " • 清理所有数据: ./scripts/stop.sh --clean-all"
echo " • 重新启动: ./scripts/start.sh"
+242
View File
@@ -0,0 +1,242 @@
#!/bin/bash
# taiji-AI-PAD 测试脚本
set -e
echo "🧪 开始测试 taiji-AI-PAD 平台..."
# 检查服务是否运行
check_service() {
local service_name=$1
local url=$2
local expected_status=${3:-200}
echo "🔍 检查 $service_name..."
if curl -s -o /dev/null -w "%{http_code}" "$url" | grep -q "$expected_status"; then
echo "✅ $service_name 正常运行"
return 0
else
echo "❌ $service_name 无响应"
return 1
fi
}
# 测试API端点
test_api_endpoint() {
local name=$1
local url=$2
local method=${3:-GET}
local data=${4:-""}
echo "🧪 测试 $name..."
if [ -n "$data" ]; then
response=$(curl -s -X "$method" -H "Content-Type: application/json" -d "$data" "$url" 2>/dev/null || echo "ERROR")
else
response=$(curl -s -X "$method" "$url" 2>/dev/null || echo "ERROR")
fi
if [ "$response" = "ERROR" ]; then
echo "❌ $name 测试失败"
return 1
else
echo "✅ $name 测试通过"
if command -v jq >/dev/null 2>&1; then
echo " 响应: $(echo "$response" | jq -c . 2>/dev/null || echo "$response")"
else
echo " 响应: $response"
fi
return 0
fi
}
# 等待服务启动
wait_for_services() {
echo "⏳ 等待服务启动..."
sleep 5
local max_attempts=30
local attempt=1
while [ $attempt -le $max_attempts ]; do
if curl -s http://localhost:8002/health > /dev/null 2>&1; then
echo "✅ 服务已就绪"
break
fi
echo "等待中... ($attempt/$max_attempts)"
sleep 2
((attempt++))
done
if [ $attempt -gt $max_attempts ]; then
echo "❌ 服务启动超时"
exit 1
fi
}
# 主测试流程
main() {
echo "🚀 taiji-AI-PAD 平台测试"
echo "========================"
# 等待服务启动
wait_for_services
# 基础健康检查
echo ""
echo "📋 基础健康检查"
echo "----------------"
local services=(
"MCP服务器:http://localhost:8002/health"
"数据接入服务:http://localhost:8001/health"
"API网关:http://localhost/health"
)
local failed_services=0
for service_info in "${services[@]}"; do
IFS=':' read -r name url <<< "$service_info"
if ! check_service "$name" "$url"; then
((failed_services++))
fi
done
# API功能测试
echo ""
echo "🔧 API功能测试"
echo "---------------"
local api_tests=(
"MCP服务器健康检查:http://localhost:8002/health:GET"
"数据接入服务健康检查:http://localhost:8001/health:GET"
"MCP工具列表:http://localhost:8002/tools:GET"
"数据接入统计:http://localhost:8001/stats:GET"
)
local failed_tests=0
for test_info in "${api_tests[@]}"; do
IFS=':' read -r name url method <<< "$test_info"
if ! test_api_endpoint "$name" "$url" "$method"; then
((failed_tests++))
fi
done
# Agent创建测试
echo ""
echo "🤖 Agent创建测试"
echo "----------------"
local agent_data='{
"name": "test-agent",
"description": "测试Agent",
"role": "测试助手",
"goal": "执行测试任务",
"tools": ["web_search"],
"config": {}
}'
if test_api_endpoint "创建Agent" "http://localhost:8002/agents" "POST" "$agent_data"; then
echo "🎉 Agent创建测试通过"
else
echo "❌ Agent创建测试失败"
((failed_tests++))
fi
# MCP协议测试
echo ""
echo "🔗 MCP协议测试"
echo "--------------"
local mcp_request='{
"jsonrpc": "2.0",
"id": "test-1",
"method": "tools/list",
"params": {}
}'
if test_api_endpoint "MCP工具列表" "http://localhost:8002/agents/test-agent/execute" "POST" "$mcp_request"; then
echo "🎉 MCP协议测试通过"
else
echo "❌ MCP协议测试失败"
((failed_tests++))
fi
# 性能测试
echo ""
echo "⚡ 简单性能测试"
echo "---------------"
echo "🔄 并发请求测试..."
local start_time=$(date +%s%N)
for i in {1..10}; do
curl -s http://localhost:8002/health > /dev/null &
done
wait
local end_time=$(date +%s%N)
local duration=$((($end_time - $start_time) / 1000000))
echo "✅ 10个并发请求耗时: ${duration}ms"
# 负载测试(如果安装了ab)
if command -v ab >/dev/null 2>&1; then
echo "🚀 负载测试 (100个请求,并发10)..."
ab -n 100 -c 10 -q http://localhost:8002/health | grep -E "(Requests per second|Time per request)"
else
echo "💡 提示: 安装 apache2-utils 可进行更详细的性能测试"
fi
# 测试报告
echo ""
echo "📊 测试报告"
echo "==========="
local total_services=${#services[@]}
local total_tests=$((${#api_tests[@]} + 2)) # API测试 + Agent创建 + MCP协议
echo "服务检查: $((total_services - failed_services))/$total_services 通过"
echo "功能测试: $((total_tests - failed_tests))/$total_tests 通过"
if [ $failed_services -eq 0 ] && [ $failed_tests -eq 0 ]; then
echo ""
echo "🎉 所有测试通过!taiji-AI-PAD 运行正常"
echo ""
echo "🔗 快速访问链接:"
echo " • MCP API文档: http://localhost:8002/docs"
echo " • 数据接入API: http://localhost:8001/docs"
echo " • Grafana监控: http://localhost:3000"
echo ""
return 0
else
echo ""
echo "❌ 部分测试失败,请检查服务状态"
echo ""
echo "🔧 故障排除:"
echo " • 查看日志: docker-compose logs"
echo " • 检查服务状态: docker-compose ps"
echo " • 重启服务: docker-compose restart"
echo ""
return 1
fi
}
# 清理函数
cleanup() {
echo ""
echo "🧹 测试清理..."
# 删除测试创建的Agent(如果存在)
curl -s -X DELETE http://localhost:8002/agents/test-agent > /dev/null 2>&1 || true
echo "✅ 清理完成"
}
# 设置清理陷阱
trap cleanup EXIT
# 运行测试
main "$@"