chenchen/pingtai_agent
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

备份

Browse Source
This commit is contained in:
zhanggangyong
2026-03-15 15:34:11 +00:00
parent e54c91edbf
commit 09d45476ef
1234 changed files with 238240 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent_templates/README.md
+12
View File
@@ -48,6 +48,9 @@ docker build -t your-agent:latest .
```
your_agent/
├── Dockerfile
├── common/
│ ├── __init__.py
│ └── agent_callback_utils.py # callback 工具
├── requirements.txt
├── run_api_server.py # 启动脚本
└── src/
@@ -65,3 +68,12 @@ your_agent/
| LITELLM_GATEWAY_URL | 是 | LiteLLM Gateway URL |
| LITELLM_MODEL | 否 | 模型名称，默认 taiji/gpt-4o-mini |
| API_PORT | 否 | 端口，默认 8000 |
| POD_NAME | 否 | Agent 名称，用于 callback 中的 `agentName` |
| USER_ID | 否 | 用户 ID，用于 callback 中的 `userId` |
| AGENT_CALLBACK_URL | 否 | 回调地址，默认指向 Agent Manager 计费回调接口 |
## Callback 模板说明
- 模板已内置 `common/agent_callback_utils.py`
- `src/server/api_server.py` 已示范在 `tools/call` 和业务 API 中使用 `CallbackContextManager`
- 以后新增业务接口时，优先复用 `run_with_callback(...)` 来包裹真实工具调用
agent_templates/common/__init__.py
+1
View File
@@ -0,0 +1 @@
agent_templates/common/agent_callback_utils.py
+151
View File
@@ -0,0 +1,151 @@
"""
Agent回调工具 - 用于向Agent Manager回调运行时长记录
"""
import os
import time
import logging
import requests
from typing import Optional, List
from datetime import datetime, timezone
logger = logging.getLogger(__name__)
class AgentCallbackHandler:
"""Agent回调处理器"""
def __init__(
self,
agent_name: Optional[str] = None,
user_id: Optional[str] = None,
callback_url: Optional[str] = None
):
self.agent_name = agent_name or os.getenv("POD_NAME", "unknown-agent")
self.user_id = user_id or os.getenv("USER_ID", "")
self.callback_url = callback_url or os.getenv(
"AGENT_CALLBACK_URL",
"http://mcp-server.taiji-ai.svc.cluster.local:8000/api/v1/billing/agent-callback"
)
self.start_time: Optional[datetime] = None
self.tools_used: List[str] = []
self.request_id: Optional[str] = None
logger.info(
"AgentCallbackHandler initialized: agent=%s, callback_url=%s",
self.agent_name,
self.callback_url,
)
def start_request(self, request_id: Optional[str] = None, user_id: Optional[str] = None):
self.start_time = datetime.now(timezone.utc)
self.tools_used = []
self.request_id = request_id or f"req-{int(time.time())}"
if user_id:
self.user_id = user_id
logger.info("Request started: request_id=%s, user_id=%s", self.request_id, self.user_id)
def add_tool_used(self, tool_name: str):
if tool_name not in self.tools_used:
self.tools_used.append(tool_name)
logger.debug("Tool used: %s", tool_name)
def end_request(self, tools_used: Optional[List[str]] = None) -> bool:
if not self.start_time:
logger.warning("Cannot end request: no start time recorded")
return False
if not self.user_id:
logger.warning("Cannot send callback: user_id not set")
return False
end_time = datetime.now(timezone.utc)
running_time = (end_time - self.start_time).total_seconds()
final_tools_used = tools_used if tools_used is not None else self.tools_used
success = self._send_callback(
running_time_seconds=int(running_time),
start_time=self.start_time,
end_time=end_time,
tools_used=final_tools_used
)
self.start_time = None
self.tools_used = []
self.request_id = None
return success
def _send_callback(
self,
running_time_seconds: int,
start_time: datetime,
end_time: datetime,
tools_used: List[str]
) -> bool:
try:
payload = {
"agentName": self.agent_name,
"userId": self.user_id,
"podRunningTimeSeconds": running_time_seconds,
"toolsUsed": tools_used,
"startTime": start_time.isoformat(),
"endTime": end_time.isoformat(),
"requestId": self.request_id
}
logger.info("Sending callback: %s", payload)
response = requests.post(
self.callback_url,
json=payload,
timeout=5
)
if response.status_code == 200:
logger.info("Callback sent successfully: %s", response.json())
return True
logger.error("Callback failed with status %s: %s", response.status_code, response.text)
return False
except requests.exceptions.RequestException as e:
logger.error("Failed to send callback: %s", str(e))
return False
except Exception as e:
logger.error("Unexpected error sending callback: %s", str(e))
return False
class CallbackContextManager:
"""回调上下文管理器 - 使用with语句自动处理开始和结束"""
def __init__(
self,
handler: AgentCallbackHandler,
request_id: Optional[str] = None,
user_id: Optional[str] = None,
tools_used: Optional[List[str]] = None
):
self.handler = handler
self.request_id = request_id
self.user_id = user_id
self.tools_used = tools_used or []
def __enter__(self):
self.handler.start_request(
request_id=self.request_id,
user_id=self.user_id
)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.handler.end_request(tools_used=self.tools_used)
return False
def add_tool(self, tool_name: str):
self.handler.add_tool_used(tool_name)
if tool_name not in self.tools_used:
self.tools_used.append(tool_name)
agent_templates/requirements.txt
+1
View File
@@ -11,3 +11,4 @@ uvicorn[standard]>=0.27.0
# HTTP Client
aiohttp>=3.9.0
requests>=2.31.0
agent_templates/src/server/api_server.py
+48 -4
View File
@@ -14,18 +14,24 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, JSONResponse
from pydantic import BaseModel, Field
from common.agent_callback_utils import AgentCallbackHandler, CallbackContextManager
from .mcp_server import TOOL_MAP, TOOL_LIST
# ==================== 配置 ====================
SERVER_NAME = "Your Agent API" # 修改为你的 Agent 名称
POD_NAME = os.getenv("POD_NAME", "your-agent")
USER_ID = os.getenv("USER_ID", "")
callback_handler: Optional[AgentCallbackHandler] = None
# ==================== FastAPI 应用 ====================
@asynccontextmanager
async def lifespan(app: FastAPI):
global callback_handler
print(f"🚀 {SERVER_NAME} 启动")
callback_handler = AgentCallbackHandler(agent_name=POD_NAME, user_id=USER_ID)
yield
print(f"🛑 {SERVER_NAME} 关闭")
@@ -79,13 +85,18 @@ async def root():
return {
"service": SERVER_NAME,
"status": "running",
"tools": list(TOOL_MAP.keys())
"tools": list(TOOL_MAP.keys()),
"callback_enabled": callback_handler is not None
}
@app.get("/health")
async def health():
return {"status": "healthy", "service": SERVER_NAME}
return {
"status": "healthy",
"service": SERVER_NAME,
"callback_enabled": callback_handler is not None
}
# ==================== MCP 端点 ====================
@@ -93,6 +104,27 @@ async def health():
sessions: Dict[str, Dict] = {}
async def run_with_callback(
tool_name: str,
func,
*args,
user_id: Optional[str] = None,
request_id: Optional[str] = None,
**kwargs
):
"""统一包装 callback 逻辑，便于后续新 Agent 直接复用。"""
if not callback_handler:
return await func(*args, **kwargs)
with CallbackContextManager(
handler=callback_handler,
user_id=user_id or USER_ID,
request_id=request_id or f"{tool_name}-{uuid.uuid4().hex}"
) as ctx:
ctx.add_tool(tool_name)
return await func(*args, **kwargs)
async def handle_mcp_request(data: Dict, session_id: str = None, api_key: str = None) -> Dict:
"""处理 MCP JSON-RPC 请求"""
method = data.get("method")
@@ -132,7 +164,13 @@ async def handle_mcp_request(data: Dict, session_id: str = None, api_key: str =
os.environ['OPENAI_API_KEY'] = api_key
try:
result = await TOOL_MAP[tool_name](**args)
result = await run_with_callback(
tool_name,
TOOL_MAP[tool_name],
user_id=args.get("user_id"),
request_id=req_id or f"mcp-{tool_name}-{uuid.uuid4().hex}",
**args
)
finally:
if old_key:
os.environ['OPENAI_API_KEY'] = old_key
@@ -223,7 +261,13 @@ async def api_query(request: QueryRequest, api_key: str = Depends(verify_api_key
os.environ['OPENAI_API_KEY'] = api_key
try:
result = await TOOL_MAP['your_tool'](query=request.query, option=request.option)
result = await run_with_callback(
"your_tool",
TOOL_MAP['your_tool'],
query=request.query,
option=request.option,
request_id=f"api-your-tool-{uuid.uuid4().hex}"
)
return QueryResponse(success=True, result=result)
finally:
if old_key:
gcp_batch_agent/Dockerfile
+20
View File
@@ -0,0 +1,20 @@
FROM python:3.12-slim
WORKDIR /app
ENV PYTHONUNBUFFERED=1
ENV PYTHONDONTWRITEBYTECODE=1
RUN apt-get update && apt-get install -y gcc curl && rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8000/health || exit 1
CMD ["python", "run_api_server.py"]
gcp_batch_agent/README.md
+169
View File
@@ -0,0 +1,169 @@
# GCP Batch Agent
基于 Vertex AI 的批量推理 Agent，支持 Flex PayGo 模式。用户传入 Cloud Storage 地址，Agent 自动提交批量作业并监控状态。
## 功能特性
- **提交批量作业** - 支持 Gemini 系列模型的批量推理
- **监控作业状态** - 实时查询作业进度和状态
- **列出作业** - 查看所有批量作业
- **取消作业** - 取消正在运行的作业
## 快速开始
### 1. 环境配置
```bash
# 必需环境变量
export GCP_PROJECT_ID="your-project-id"
export GCP_LOCATION="us-central1"
# 服务账号凭证（二选一）
export GCP_SERVICE_ACCOUNT_JSON='{"type":"service_account",...}' # JSON 字符串
# 或
export GCP_SERVICE_ACCOUNT_FILE="/path/to/service-account.json" # 文件路径
# 可选配置
export DEFAULT_MODEL="gemini-2.0-flash"
export API_PORT="8000"
```
### 2. 用户授权存储桶
用户需要给 Agent 服务账号授权访问其 Cloud Storage 存储桶：
```bash
# 获取 Agent 服务账号邮箱
AGENT_SA="your-agent-sa@your-project.iam.gserviceaccount.com"
# 授权存储桶访问权限
gsutil iam ch serviceAccount:${AGENT_SA}:objectViewer gs://your-bucket
gsutil iam ch serviceAccount:${AGENT_SA}:objectCreator gs://your-bucket
```
### 3. 准备输入文件
创建 JSONL 格式的输入文件：
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"What is AI?"}]}]}}
{"request":{"contents":[{"role":"user","parts":[{"text":"Explain quantum computing."}]}]}}
```
上传到 Cloud Storage：
```bash
gsutil cp input.jsonl gs://your-bucket/input.jsonl
```
### 4. 启动服务
```bash
# 本地运行
pip install -r requirements.txt
python run_api_server.py
# Docker 运行
docker build -t gcp-batch-agent .
docker run -p 8000:8000 \
-e GCP_PROJECT_ID=your-project \
-e GCP_SERVICE_ACCOUNT_FILE=/app/sa.json \
-v /path/to/sa.json:/app/sa.json \
gcp-batch-agent
```
## API 使用
### 提交批量作业
```bash
curl -X POST http://localhost:8000/api/v1/batch/submit \
-H "Content-Type: application/json" \
-H "api-key: your-api-key" \
-d '{
"input_uri": "gs://your-bucket/input.jsonl",
"output_uri": "gs://your-bucket/output/",
"model": "gemini-2.0-flash"
}'
```
### 查询作业状态
```bash
curl http://localhost:8000/api/v1/batch/status/123456 \
-H "api-key: your-api-key"
```
### 列出作业
```bash
curl "http://localhost:8000/api/v1/batch/list?page_size=10" \
-H "api-key: your-api-key"
```
### 取消作业
```bash
curl -X POST http://localhost:8000/api/v1/batch/cancel/123456 \
-H "api-key: your-api-key"
```
## MCP 工具
| 工具 | 说明 |
|------|------|
| submit_batch_job | 提交批量推理作业 |
| get_job_status | 查询作业状态 |
| list_jobs | 列出批量作业 |
| cancel_job | 取消作业 |
## 支持的模型
- gemini-2.0-flash（默认）
- gemini-2.0-flash-lite
- gemini-1.5-flash
- gemini-1.5-pro
- gemini-1.0-pro
## 服务端点
| 端点 | 方法 | 说明 |
|------|------|------|
| / | GET | 服务状态 |
| /health | GET | 健康检查 |
| /mcp | POST | MCP JSON-RPC |
| /mcp/sse | GET/POST | MCP SSE 流式 |
| /api/v1/batch/submit | POST | 提交批量作业 |
| /api/v1/batch/status/{job_id} | GET | 查询作业状态 |
| /api/v1/batch/list | GET | 列出作业 |
| /api/v1/batch/cancel/{job_id} | POST | 取消作业 |
## 项目结构
```
gcp_batch_agent/
├── Dockerfile
├── README.md
├── requirements.txt
├── run_api_server.py
├── USAGE.md
└── src/
├── __init__.py
└── server/
├── __init__.py
├── api_server.py # FastAPI + MCP HTTP 端点
├── mcp_server.py # MCP 工具定义
└── core/
├── __init__.py
├── config.py # 配置管理
├── credentials.py # GCP 凭证管理
├── batch_client.py # Vertex AI 批量推理客户端
└── schemas.py # 数据模型
```
## 注意事项
1. **Flex PayGo 模式**：作业可能需要排队等待资源
2. **存储桶权限**：用户必须提前授权 Agent 服务账号
3. **文件格式**：输入必须是有效的 JSONL 格式
4. **计费**：按实际处理的行数计费，中途取消只收取已完成部分
gcp_batch_agent/USAGE.md
+468
View File
@@ -0,0 +1,468 @@
# GCP 批量推理 Agent 使用指南
## 概述
GCP Batch Agent 是一个基于 Vertex AI 的批量推理服务，支持 Flex PayGo 模式。您只需传入 Cloud Storage 地址，Agent 会自动提交批量作业、监控状态并返回结果。
**默认模型**: `gemini-2.5-flash`
---
## 快速开始
### 第一步：授权存储桶访问
在使用本服务前，您需要给 Agent 服务账号授权访问您的 Cloud Storage 存储桶。
**Agent 服务账号邮箱**：
```
taijiclound@gemini-20251105-b.iam.gserviceaccount.com
```
**方式一：使用 gsutil 命令**
```bash
# 授权读取权限（用于读取输入文件）
gsutil iam ch serviceAccount:taijiclound@gemini-20251105-b.iam.gserviceaccount.com:objectViewer gs://您的存储桶名称
# 授权写入权限（用于写入输出结果）
gsutil iam ch serviceAccount:taijiclound@gemini-20251105-b.iam.gserviceaccount.com:objectCreator gs://您的存储桶名称
```
**方式二：在 GCP 控制台操作**
1. 进入 [Cloud Storage 控制台](https://console.cloud.google.com/storage/browser)
2. 选择您的存储桶 → 点击"权限"标签
3. 点击"授予访问权限"
4. 添加主账号：`taijiclound@gemini-20251105-b.iam.gserviceaccount.com`
5. 选择角色：
- `Storage Object Viewer`（读取）
- `Storage Object Creator`（写入）
### 第二步：准备输入文件
输入文件必须是 **JSONL 格式**（每行一个 JSON 请求）或 **JSON 格式**。
**文本请求示例** (`input.jsonl`)：
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"什么是人工智能？"}]}]}}
{"request":{"contents":[{"role":"user","parts":[{"text":"用简单的话解释机器学习。"}]}]}}
{"request":{"contents":[{"role":"user","parts":[{"text":"深度学习有哪些应用？"}]}]}}
```
**带系统提示的请求**：
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"翻译成英文：你好世界"}]}],"systemInstruction":{"parts":[{"text":"你是一个专业的翻译。"}]}}}
```
**多模态请求（图片）**：
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"描述这张图片"},{"fileData":{"mimeType":"image/jpeg","fileUri":"gs://您的存储桶/images/photo1.jpg"}}]}]}}
```
**多模态请求（视频）**：
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"总结这个视频"},{"fileData":{"mimeType":"video/mp4","fileUri":"gs://您的存储桶/videos/video1.mp4"}}]}]}}
```
### 第三步：上传输入文件
```bash
gsutil cp input.jsonl gs://您的存储桶/batch-input/input.jsonl
```
### 第四步：提交批量作业
```bash
curl -X POST http://服务地址/api/v1/batch/submit \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"input_uri": "gs://您的存储桶/batch-input/input.jsonl",
"output_uri": "gs://您的存储桶/batch-output/"
}'
```
---
## API 接口说明
### 1. 提交批量作业
**请求**：
```
POST /api/v1/batch/submit
```
**请求参数**：
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| input_uri | string | 是 | 输入文件的 GCS 路径，如 `gs://bucket/input.jsonl` |
| output_uri | string | 是 | 输出目录的 GCS 路径，如 `gs://bucket/output/` |
| model | string | 否 | 模型名称，默认 `gemini-2.5-flash` |
| display_name | string | 否 | 作业显示名称 |
**请求示例**：
```bash
curl -X POST http://服务地址/api/v1/batch/submit \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"input_uri": "gs://my-bucket/input.jsonl",
"output_uri": "gs://my-bucket/output/",
"model": "gemini-2.5-flash",
"display_name": "my-batch-job"
}'
```
**响应示例**：
```json
{
"success": true,
"message": "批量作业已提交",
"job": {
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456789",
"job_name": "my-batch-job",
"state": "JOB_STATE_PENDING",
"input_uri": "gs://my-bucket/input.jsonl",
"output_uri": "gs://my-bucket/output/",
"model": "publishers/google/models/gemini-2.5-flash",
"create_time": "2026-03-05T07:30:00.000000Z"
}
}
```
---
### 2. 查询作业状态
**请求**：
```
GET /api/v1/batch/status/{job_id}
```
**请求示例**：
```bash
# 使用简短 ID
curl "http://服务地址/api/v1/batch/status/123456789" \
-H "api-key: 您的API密钥"
# 使用完整 job_id
curl "http://服务地址/api/v1/batch/status/projects/xxx/locations/us-central1/batchPredictionJobs/123456789" \
-H "api-key: 您的API密钥"
```
**响应示例**：
```json
{
"success": true,
"job": {
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456789",
"job_name": "my-batch-job",
"state": "JOB_STATE_RUNNING",
"input_uri": "gs://my-bucket/input.jsonl",
"output_uri": "gs://my-bucket/output/",
"model": "publishers/google/models/gemini-2.5-flash",
"create_time": "2026-03-05T07:30:00.000000Z",
"start_time": "2026-03-05T07:31:00.000000Z",
"progress": {
"total_count": 1000,
"succeeded_count": 450,
"failed_count": 2
}
}
}
```
---
### 3. 列出作业
**请求**：
```
GET /api/v1/batch/list
```
**请求参数**：
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| page_size | int | 否 | 每页数量，默认 10 |
| filter_str | string | 否 | 过滤条件 |
**请求示例**：
```bash
curl "http://服务地址/api/v1/batch/list?page_size=10" \
-H "api-key: 您的API密钥"
```
**响应示例**：
```json
{
"success": true,
"total": 2,
"jobs": [
{
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456789",
"job_name": "my-batch-job-001",
"state": "JOB_STATE_SUCCEEDED",
"create_time": "2026-03-05T07:30:00.000000Z"
},
{
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456788",
"job_name": "my-batch-job-002",
"state": "JOB_STATE_RUNNING",
"create_time": "2026-03-05T08:00:00.000000Z"
}
],
"next_page_token": null
}
```
---
### 4. 取消作业
**请求**：
```
POST /api/v1/batch/cancel/{job_id}
```
**请求示例**：
```bash
curl -X POST "http://服务地址/api/v1/batch/cancel/123456789" \
-H "api-key: 您的API密钥"
```
**响应示例**：
```json
{
"success": true,
"message": "作业取消请求已发送",
"job": {
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456789",
"job_name": "my-batch-job",
"state": "JOB_STATE_CANCELLING"
}
}
```
---
## 作业状态说明
| 状态 | 说明 |
|------|------|
| JOB_STATE_QUEUED | 作业已排队，等待资源 |
| JOB_STATE_PENDING | 作业待处理 |
| JOB_STATE_RUNNING | 作业正在运行 |
| JOB_STATE_SUCCEEDED | 作业成功完成 |
| JOB_STATE_FAILED | 作业失败 |
| JOB_STATE_CANCELLING | 作业正在取消 |
| JOB_STATE_CANCELLED | 作业已取消 |
| JOB_STATE_PARTIALLY_SUCCEEDED | 作业部分成功 |
---
## 获取输出结果
作业完成后，结果会自动保存到您指定的输出目录。
**输出文件格式**：
```jsonl
{"status":"","response":{"candidates":[{"content":{"role":"model","parts":[{"text":"人工智能是..."}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":150}}}
{"status":"","response":{"candidates":[{"content":{"role":"model","parts":[{"text":"机器学习是人工智能的一个子集..."}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":12,"candidatesTokenCount":200}}}
```
**下载输出结果**：
```bash
gsutil cp -r gs://您的存储桶/batch-output/ ./local-output/
```
---
## 支持的模型
| 模型 | 说明 |
|------|------|
| gemini-2.5-flash | 默认模型，快速响应 |
| gemini-2.5-pro | 高性能模型 |
| gemini-2.0-flash | 上一代快速模型 |
| gemini-1.5-flash | 稳定版快速模型 |
| gemini-1.5-pro | 稳定版高性能模型 |
---
## 常见问题
### 1. 403 Permission Denied
**原因**：Agent 服务账号无权访问您的存储桶。
**解决方案**：按照"第一步：授权存储桶访问"的说明，给服务账号授权。
### 2. 404 Not Found
**原因**：输入文件不存在。
**解决方案**：检查 GCS 路径是否正确，确保文件已上传。
### 3. 400 Invalid Request
**原因**：输入文件格式错误。
**解决方案**：确保输入文件是有效的 JSONL 格式，每行一个完整的 JSON 对象。
### 4. 作业长时间处于 PENDING 状态
**原因**：Flex PayGo 模式下，作业可能需要排队等待资源。
**解决方案**：耐心等待，大型作业可能需要较长时间。
---
## 计费说明
- **Flex PayGo 模式**：按实际处理的请求数计费
- **中途取消**：只收取已完成部分的费用
- **失败请求**：不收费
详细定价请参考 [Vertex AI 定价页面](https://cloud.google.com/vertex-ai/pricing)。
---
## 最佳实践
1. **批量大小**：建议每个 JSONL 文件包含 100-10000 个请求
2. **文件组织**：使用有意义的目录结构，如 `gs://bucket/batch-jobs/2026-03-05/input.jsonl`
3. **监控频率**：对于大型作业，建议每 1-5 分钟查询一次状态
4. **错误处理**：检查输出文件中的 `status` 字段，处理失败的请求
---
## MCP 接口
本服务同时支持 MCP (Model Context Protocol) 接口，可用于 AI Agent 集成。
### MCP 端点
| 端点 | 方法 | 说明 |
|------|------|------|
| /mcp | POST | MCP JSON-RPC 端点 |
| /mcp/sse | GET | MCP SSE 连接端点 |
| /mcp/sse | POST | MCP SSE 请求端点 |
### 获取工具列表
```bash
curl -X POST http://服务地址/mcp \
-H "Content-Type: application/json" \
-d '{
"jsonrpc": "2.0",
"id": 1,
"method": "tools/list"
}'
```
### 调用工具
**提交批量作业**：
```bash
curl -X POST http://服务地址/mcp \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
"name": "submit_batch_job",
"arguments": {
"input_uri": "gs://您的存储桶/input.jsonl",
"output_uri": "gs://您的存储桶/output/"
}
}
}'
```
**查询作业状态**：
```bash
curl -X POST http://服务地址/mcp \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {
"name": "get_job_status",
"arguments": {
"job_id": "123456789"
}
}
}'
```
**列出作业**：
```bash
curl -X POST http://服务地址/mcp \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"jsonrpc": "2.0",
"id": 3,
"method": "tools/call",
"params": {
"name": "list_jobs",
"arguments": {
"page_size": 10
}
}
}'
```
**取消作业**：
```bash
curl -X POST http://服务地址/mcp \
-H "Content-Type: application/json" \
-H "api-key: 您的API密钥" \
-d '{
"jsonrpc": "2.0",
"id": 4,
"method": "tools/call",
"params": {
"name": "cancel_job",
"arguments": {
"job_id": "123456789"
}
}
}'
```
### MCP 工具列表
| 工具名称 | 说明 |
|----------|------|
| submit_batch_job | 提交批量推理作业 |
| get_job_status | 查询作业状态 |
| list_jobs | 列出批量作业 |
| cancel_job | 取消作业 |
gcp_batch_agent/build.log
+31
View File
@@ -0,0 +1,31 @@
#0 building with "multiarch-builder" instance using docker-container driver
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 476B done
#1 DONE 0.0s
#2 [internal] load metadata for docker.io/library/python:3.12-slim
#2 DONE 1.1s
#3 [internal] load .dockerignore
#3 transferring context: 2B done
#3 DONE 0.0s
#4 [internal] load build context
#4 transferring context: 1.01kB done
#4 DONE 0.0s
#5 [1/6] FROM docker.io/library/python:3.12-slim@sha256:ccc7089399c8bb65dd1fb3ed6d55efa538a3f5e7fca3f5988ac3b5b87e593bf0
#5 resolve docker.io/library/python:3.12-slim@sha256:ccc7089399c8bb65dd1fb3ed6d55efa538a3f5e7fca3f5988ac3b5b87e593bf0 0.0s done
#5 DONE 0.0s
#6 [2/6] WORKDIR /app
#6 CACHED
#7 [3/6] RUN apt-get update && apt-get install -y gcc curl && rm -rf /var/lib/apt/lists/*
#7 CACHED
#8 [4/6] COPY requirements.txt .
#8 CACHED
#9 [5/6] RUN pip install --no-cache-dir -r requirements.txt
gcp_batch_agent/gemini-20251105-b-ba3214d839ad.json
+13
View File
@@ -0,0 +1,13 @@
{
"type": "service_account",
"project_id": "gemini-20251105-b",
"private_key_id": "ba3214d839ad28ab6e700558ea43b64918b7b656",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDa8S1aaS9kzmaB\nwIzRrA+Bvya0RxtrXn2WsCyyg/mIJPd0Rdv4/pcGF39ahjK51+3ZMlwdm0cOck4m\nzOqOkBZ//OfQCnYFlZ1TcFLwq8X2iEjns2+R3rPgK64J/3nS4yJFxwevXGX5D8oh\nkVm4h7nWuTBE44T0uQFhEd4VDDiPCwEomm6+rclmDyWkHac07F9CB4OMquHVo7eK\nxyYezMZ0sPEMcdgjpYshlvr/67RB75HkqHSi1sJO7G89qPPITPzXhR9HfZu61/28\ntqswxFPo2AE2w9Dwvp6CwTO2adk8MYhCypLTqqnwsxuyuCX5fcuWUyyRDUlCU+rn\nkrNa350fAgMBAAECggEAZoVIl21po+qCVX+0cPoAyOCZXxFs5Xn5lPYp9+2UiwvE\nU83ik1Wjolaea+UrG5rg5TOPrs1nGHyt271YmCkgYA4s/l3npfXJFakjLU9CA48N\nlHGRbNy7ndzWQhg/E2EIS2RHVPYPSfD61X0aZi4n90QANkHToESBQNL9Cx4N3eyL\ngaJBbCTeyjpneZUJcIvKw7ms7rs1pAaEjzHlhtrfGcyytU8DDylc5Y9EDEXpLLvY\nX4ZfY0DnY2oLWQe5yCKAle1lGu7JZmh4ygra2xwVU8tNgfCQ/TRoIWH6KJnvQRND\nmNMvOXrjfjY6JrnfiZtImbiv4ceDodAwSEFqzh1NAQKBgQDvx6D+/p2EMLlkm4ap\nugpbZgieJTkERIrOQRR0amo3qX3mGMHJ7Gkusi32uiNiVKq8Pa1YlpCBeZqKFdbG\nE7GQ/Zj2j4JvdI3790leNBEQoOYa3VDoky17XmLk9svOY70oPgDDCEF07qR8qNdN\naaffP49iF4ol43zy8ys1DH2oHQKBgQDpwLFWAODuDvOjPGf7kJhgoG4Stu0oUVRe\nS6Vw5cQ1uozmGMdX6GIqqLOuWoVnqSf2QhYUfkHks4oY6sxqb8Qw62yZ1XIxtB8J\nF1k9MAd68KYTnpqGSCqWA3Mqn2qXCHvPgDGWB2OgYJuhu8NuBZn5xyNG6sTC7PS3\nJnE2Gf5tawKBgQCSA8oJnjmDtzwehQsjLlSCRgc3bslizO6OHVl8bxURolg1l5vi\n/+Epe328AqvrhE2YZiK6kK2c+tVeA2CPrIx0E8pjUw8GGj8gUf1OrXw62RBnCaDs\nkDuhJeApOBTyMRCOgftqyQs7TtvBwfuie5WrwIPrIxSQcBC1zdjg7CNVDQKBgGlM\nBhZp2ukiDxpPUb/+yMVuP2dqw7ZPvKOli+cpeZOCb4oPS20MH3x79kTqPgIOLxQ9\nfNjKb0BoqCLFUkP4CPbIdmltvz0omHpt7CMmCZiV4xofbhhjVduxviVLHqbeiXdV\nlgCxX22VV0Yp4MUk/NGJpRiDFROBzlLanFD0sKn1AoGBAMgU1u5Ec/X0gM4vR77R\nWXCWPSRVCtsAKexcGJZm7GxB5qDWDBjCK/Akbzf2PjcnR5uBZAtRmowc7/rs0SXZ\nObcEBbAUptHoD0NpSbZYPnUYro+RjAW9sieYgIrgRhEWTKdUmSWgULTPXo2bUAO5\nzrMIl+YXy9KeT4WBE/B1IZhc\n-----END PRIVATE KEY-----\n",
"client_email": "taijiclound@gemini-20251105-b.iam.gserviceaccount.com",
"client_id": "101810691980030950016",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/taijiclound%40gemini-20251105-b.iam.gserviceaccount.com",
"universe_domain": "googleapis.com"
}
gcp_batch_agent/requirements.txt
+18
View File
@@ -0,0 +1,18 @@
# Pydantic AI
pydantic-ai>=0.0.14
# MCP
mcp>=0.9.0
fastmcp>=0.1.0
# FastAPI
fastapi>=0.109.0
uvicorn[standard]>=0.27.0
# HTTP Client
aiohttp>=3.9.0
# GCP
google-cloud-aiplatform>=1.38.0
google-auth>=2.23.0
google-genai>=1.0.0
gcp_batch_agent/run_api_server.py
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python
"""启动 API 服务器"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
if __name__ == '__main__':
from src.server.api_server import app
import uvicorn
import os
host = os.getenv('API_HOST', '0.0.0.0')
port = int(os.getenv('API_PORT', '8000'))
print(f"🚀 启动 GCP Batch Agent API: http://{host}:{port}")
uvicorn.run(app, host=host, port=port, log_level="info")
gcp_batch_agent/src/__init__.py
+1
View File
@@ -0,0 +1 @@
"""GCP Batch Agent Package"""
gcp_batch_agent/src/server/__init__.py
+1
View File
@@ -0,0 +1 @@
"""GCP Batch Agent Server Package"""
gcp_batch_agent/src/server/api_server.py
+293
View File
@@ -0,0 +1,293 @@
"""
GCP Batch Agent HTTP API 服务器
提供 REST API 和 MCP HTTP/SSE 端点。
"""
import json
import uuid
import os
from typing import Optional, Dict, Any, AsyncGenerator
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request, Header, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, JSONResponse
from pydantic import BaseModel, Field
from .mcp_server import TOOL_MAP, TOOL_LIST
# ==================== 配置 ====================
SERVER_NAME = "GCP Batch Agent API"
# ==================== FastAPI 应用 ====================
@asynccontextmanager
async def lifespan(app: FastAPI):
print(f"🚀 {SERVER_NAME} 启动")
yield
print(f"🛑 {SERVER_NAME} 关闭")
app = FastAPI(
title=SERVER_NAME,
description="GCP Vertex AI 批量推理服务，支持提交、监控和管理批量作业",
version="1.0.0",
lifespan=lifespan
)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# ==================== API Key 验证 ====================
async def verify_api_key(
api_key: Optional[str] = Header(None, alias="api-key"),
authorization: Optional[str] = Header(None)
) -> str:
"""验证 API Key"""
if api_key and api_key.strip() and api_key.strip() != "sk":
return api_key.strip()
if authorization:
key = authorization[7:].strip() if authorization.startswith("Bearer ") else authorization.strip()
if key and key != "sk":
return key
raise HTTPException(status_code=401, detail="缺少 API Key")
def get_api_key_from_request(request: Request) -> Optional[str]:
"""从请求头提取 API Key（不验证）"""
api_key = request.headers.get("api-key") or request.headers.get("api_key")
if not api_key:
auth = request.headers.get("Authorization")
if auth:
api_key = auth[7:] if auth.startswith("Bearer ") else auth
return api_key
# ==================== 健康检查 ====================
@app.get("/")
async def root():
return {
"service": SERVER_NAME,
"status": "running",
"tools": list(TOOL_MAP.keys())
}
@app.get("/health")
async def health():
return {"status": "healthy", "service": SERVER_NAME}
# ==================== MCP 端点 ====================
sessions: Dict[str, Dict] = {}
async def handle_mcp_request(data: Dict, session_id: str = None, api_key: str = None) -> Dict:
"""处理 MCP JSON-RPC 请求"""
method = data.get("method")
params = data.get("params", {})
req_id = data.get("id")
# tools/call 需要验证 API Key
if method == "tools/call" and (not api_key or api_key == "sk"):
return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32001, "message": "缺少 API Key"}}
try:
if method == "initialize":
session_id = session_id or str(uuid.uuid4())
sessions[session_id] = {"initialized": True}
return {
"jsonrpc": "2.0", "id": req_id,
"result": {
"protocolVersion": "2024-11-05",
"capabilities": {"tools": {}},
"serverInfo": {"name": SERVER_NAME, "version": "1.0.0"}
}
}
elif method == "tools/list":
return {"jsonrpc": "2.0", "id": req_id, "result": {"tools": TOOL_LIST}}
elif method == "tools/call":
tool_name = params.get("name")
args = params.get("arguments", {})
if tool_name not in TOOL_MAP:
raise ValueError(f"Unknown tool: {tool_name}")
result = await TOOL_MAP[tool_name](**args)
return {
"jsonrpc": "2.0", "id": req_id,
"result": {"content": [{"type": "text", "text": str(result)}]}
}
elif method == "ping":
return {"jsonrpc": "2.0", "id": req_id, "result": {}}
else:
raise ValueError(f"Unknown method: {method}")
except Exception as e:
return {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32603, "message": str(e)}}
@app.post("/mcp")
async def mcp_endpoint(request: Request):
"""MCP HTTP 端点"""
try:
body = await request.json()
session_id = request.headers.get("x-mcp-session-id")
api_key = get_api_key_from_request(request)
response = await handle_mcp_request(body, session_id, api_key)
return JSONResponse(content=response, headers={"x-mcp-session-id": session_id or ""})
except Exception as e:
return JSONResponse(status_code=400, content={"jsonrpc": "2.0", "error": {"code": -32700, "message": str(e)}})
@app.get("/mcp/sse")
async def mcp_sse(request: Request):
"""MCP SSE 端点"""
session_id = request.headers.get("x-mcp-session-id") or str(uuid.uuid4())
async def stream() -> AsyncGenerator[str, None]:
yield f"data: {json.dumps({'type': 'connection', 'sessionId': session_id})}\n\n"
import asyncio
while True:
await asyncio.sleep(30)
yield f"data: {json.dumps({'type': 'ping'})}\n\n"
return StreamingResponse(stream(), media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "x-mcp-session-id": session_id})
@app.post("/mcp/sse")
async def mcp_sse_post(request: Request):
"""MCP SSE POST 端点"""
try:
body = await request.json()
session_id = request.headers.get("x-mcp-session-id") or str(uuid.uuid4())
api_key = get_api_key_from_request(request)
async def stream() -> AsyncGenerator[str, None]:
response = await handle_mcp_request(body, session_id, api_key)
yield f"data: {json.dumps(response)}\n\n"
return StreamingResponse(stream(), media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "x-mcp-session-id": session_id})
except Exception as e:
return JSONResponse(status_code=400, content={"jsonrpc": "2.0", "error": {"code": -32700, "message": str(e)}})
# ==================== REST API 端点 ====================
class SubmitJobRequest(BaseModel):
"""提交作业请求"""
input_uri: str = Field(..., description="输入文件的 GCS 路径，格式: gs://bucket/path/input.jsonl")
output_uri: str = Field(..., description="输出目录的 GCS 路径，格式: gs://bucket/path/output/")
model: Optional[str] = Field(None, description="模型名称，默认 gemini-2.0-flash")
display_name: Optional[str] = Field(None, description="作业显示名称")
class ListJobsRequest(BaseModel):
"""列出作业请求"""
filter_str: Optional[str] = Field(None, description="过滤条件")
page_size: int = Field(10, description="每页数量", ge=1, le=100)
@app.post("/api/v1/batch/submit")
async def api_submit_job(request: SubmitJobRequest, api_key: str = Depends(verify_api_key)):
"""
提交批量推理作业
用户需要：
1. 将 JSONL 格式的输入文件上传到 Cloud Storage
2. 给 Agent 服务账号授权访问存储桶
"""
try:
result = await TOOL_MAP['submit_batch_job'](
input_uri=request.input_uri,
output_uri=request.output_uri,
model=request.model,
display_name=request.display_name
)
return json.loads(result)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/api/v1/batch/status/{job_id:path}")
async def api_get_job_status(job_id: str, api_key: str = Depends(verify_api_key)):
"""
查询作业状态
job_id 可以是完整路径或简短 ID
"""
try:
result = await TOOL_MAP['get_job_status'](job_id=job_id)
return json.loads(result)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/api/v1/batch/list")
async def api_list_jobs(
filter_str: Optional[str] = None,
page_size: int = 10,
api_key: str = Depends(verify_api_key)
):
"""
列出批量作业
"""
try:
result = await TOOL_MAP['list_jobs'](
filter_str=filter_str,
page_size=page_size
)
return json.loads(result)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/api/v1/batch/cancel/{job_id:path}")
async def api_cancel_job(job_id: str, api_key: str = Depends(verify_api_key)):
"""
取消批量作业
job_id 可以是完整路径或简短 ID
"""
try:
result = await TOOL_MAP['cancel_job'](job_id=job_id)
return json.loads(result)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/api/v1/tools")
async def api_tools():
"""获取可用工具列表"""
return {
"tools": {name: tool.__doc__.split('\n')[0].strip() if tool.__doc__ else "MCP 工具" for name, tool in TOOL_MAP.items()},
"count": len(TOOL_MAP)
}
# 导出
__all__ = ['app']
if __name__ == '__main__':
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
gcp_batch_agent/src/server/core/__init__.py
+1
View File
@@ -0,0 +1 @@
"""GCP Batch Agent Core Package"""
gcp_batch_agent/src/server/core/batch_client.py
+303
View File
@@ -0,0 +1,303 @@
"""
Vertex AI 批量推理客户端
封装 Vertex AI Batch Prediction API 调用。
"""
import logging
from typing import Optional, List
from datetime import datetime
import uuid
from google.cloud import aiplatform
from google.cloud.aiplatform_v1 import (
BatchPredictionJob,
JobServiceClient,
CreateBatchPredictionJobRequest,
GetBatchPredictionJobRequest,
ListBatchPredictionJobsRequest,
CancelBatchPredictionJobRequest,
GcsSource,
GcsDestination,
)
from google.protobuf import timestamp_pb2
from .config import Config, get_config
from .credentials import CredentialsManager, get_credentials_manager
from .schemas import (
BatchJobRequest,
BatchJobInfo,
BatchJobListResponse,
JobState,
JobProgress
)
logger = logging.getLogger(__name__)
class BatchClient:
"""Vertex AI 批量推理客户端"""
def __init__(
self,
config: Optional[Config] = None,
credentials_manager: Optional[CredentialsManager] = None
):
self.config = config or get_config()
self.credentials_manager = credentials_manager or get_credentials_manager()
self._client: Optional[JobServiceClient] = None
@property
def client(self) -> JobServiceClient:
"""获取 Job Service 客户端"""
if self._client is None:
credentials = self.credentials_manager.get_credentials()
# 构建 API 端点
api_endpoint = f"{self.config.location}-aiplatform.googleapis.com"
self._client = JobServiceClient(
credentials=credentials,
client_options={"api_endpoint": api_endpoint}
)
return self._client
@property
def parent(self) -> str:
"""获取父资源路径"""
return f"projects/{self.config.project_id}/locations/{self.config.location}"
def _get_model_resource_name(self, model: str) -> str:
"""获取模型资源名称"""
# 支持的模型列表
supported_models = [
"gemini-2.5-flash",
"gemini-2.5-pro",
"gemini-2.0-flash",
"gemini-2.0-flash-lite",
"gemini-1.5-flash",
"gemini-1.5-pro",
"gemini-1.0-pro",
]
# 如果是简短名称，转换为完整路径
if model in supported_models:
return f"publishers/google/models/{model}"
# 如果已经是完整路径，直接返回
if model.startswith("publishers/") or model.startswith("projects/"):
return model
# 默认添加 publishers/google/models/ 前缀
return f"publishers/google/models/{model}"
def _parse_timestamp(self, ts) -> Optional[datetime]:
"""解析时间戳"""
if ts is None:
return None
if isinstance(ts, datetime):
return ts
if hasattr(ts, 'seconds'):
return datetime.fromtimestamp(ts.seconds + ts.nanos / 1e9)
return None
def _parse_job_state(self, state) -> JobState:
"""解析作业状态"""
state_str = str(state).split('.')[-1] if '.' in str(state) else str(state)
try:
return JobState(state_str)
except ValueError:
return JobState.JOB_STATE_UNSPECIFIED
def _job_to_info(self, job: BatchPredictionJob) -> BatchJobInfo:
"""将 BatchPredictionJob 转换为 BatchJobInfo"""
# 解析进度信息
progress = None
if hasattr(job, 'completion_stats') and job.completion_stats:
stats = job.completion_stats
progress = JobProgress(
total_count=getattr(stats, 'total_count', 0) or 0,
succeeded_count=getattr(stats, 'successful_count', 0) or 0,
failed_count=getattr(stats, 'failed_count', 0) or 0
)
# 解析输入输出 URI
input_uri = ""
if hasattr(job, 'input_config') and job.input_config:
if hasattr(job.input_config, 'gcs_source') and job.input_config.gcs_source:
uris = list(job.input_config.gcs_source.uris)
input_uri = uris[0] if uris else ""
output_uri = ""
if hasattr(job, 'output_config') and job.output_config:
if hasattr(job.output_config, 'gcs_destination') and job.output_config.gcs_destination:
output_uri = job.output_config.gcs_destination.output_uri_prefix or ""
# 解析错误信息
error_message = None
if hasattr(job, 'error') and job.error:
error_message = job.error.message
return BatchJobInfo(
job_id=job.name,
job_name=job.display_name,
state=self._parse_job_state(job.state),
input_uri=input_uri,
output_uri=output_uri,
model=job.model if hasattr(job, 'model') else "",
create_time=self._parse_timestamp(job.create_time),
start_time=self._parse_timestamp(job.start_time),
end_time=self._parse_timestamp(job.end_time),
progress=progress,
error_message=error_message
)
async def submit_job(self, request: BatchJobRequest) -> BatchJobInfo:
"""
提交批量推理作业
Args:
request: 批量作业请求
Returns:
BatchJobInfo: 作业信息
"""
# 验证请求
request.validate()
# 生成作业名称
display_name = request.display_name or f"batch-job-{uuid.uuid4().hex[:8]}"
# 获取模型资源名称
model_name = self._get_model_resource_name(request.model)
logger.info(f"提交批量作业: {display_name}, 模型: {model_name}")
logger.info(f"输入: {request.input_uri}, 输出: {request.output_uri}")
# 构建批量预测作业
batch_prediction_job = BatchPredictionJob(
display_name=display_name,
model=model_name,
input_config={
"instances_format": "jsonl",
"gcs_source": {
"uris": [request.input_uri]
}
},
output_config={
"predictions_format": "jsonl",
"gcs_destination": {
"output_uri_prefix": request.output_uri
}
}
)
# 创建请求
create_request = CreateBatchPredictionJobRequest(
parent=self.parent,
batch_prediction_job=batch_prediction_job
)
# 提交作业
job = self.client.create_batch_prediction_job(request=create_request)
logger.info(f"作业已提交: {job.name}, 状态: {job.state}")
return self._job_to_info(job)
async def get_job_status(self, job_id: str) -> BatchJobInfo:
"""
获取作业状态
Args:
job_id: 作业 ID（完整资源名称）
Returns:
BatchJobInfo: 作业信息
"""
# 如果只传入了作业 ID 数字，构建完整路径
if not job_id.startswith("projects/"):
job_id = f"{self.parent}/batchPredictionJobs/{job_id}"
logger.info(f"查询作业状态: {job_id}")
request = GetBatchPredictionJobRequest(name=job_id)
job = self.client.get_batch_prediction_job(request=request)
return self._job_to_info(job)
async def list_jobs(
self,
filter_str: Optional[str] = None,
page_size: int = 10,
page_token: Optional[str] = None
) -> BatchJobListResponse:
"""
列出批量作业
Args:
filter_str: 过滤条件
page_size: 每页数量
page_token: 分页令牌
Returns:
BatchJobListResponse: 作业列表响应
"""
logger.info(f"列出批量作业, filter: {filter_str}, page_size: {page_size}")
request = ListBatchPredictionJobsRequest(
parent=self.parent,
filter=filter_str or "",
page_size=page_size,
page_token=page_token or ""
)
response = self.client.list_batch_prediction_jobs(request=request)
jobs = []
for job in response.batch_prediction_jobs:
jobs.append(self._job_to_info(job))
return BatchJobListResponse(
jobs=jobs,
next_page_token=response.next_page_token if response.next_page_token else None
)
async def cancel_job(self, job_id: str) -> BatchJobInfo:
"""
取消批量作业
Args:
job_id: 作业 ID（完整资源名称）
Returns:
BatchJobInfo: 作业信息
"""
# 如果只传入了作业 ID 数字，构建完整路径
if not job_id.startswith("projects/"):
job_id = f"{self.parent}/batchPredictionJobs/{job_id}"
logger.info(f"取消作业: {job_id}")
request = CancelBatchPredictionJobRequest(name=job_id)
self.client.cancel_batch_prediction_job(request=request)
# 获取更新后的状态
return await self.get_job_status(job_id)
# 全局客户端实例
_batch_client: Optional[BatchClient] = None
def get_batch_client() -> BatchClient:
"""获取全局批量推理客户端实例"""
global _batch_client
if _batch_client is None:
_batch_client = BatchClient()
return _batch_client
def reset_batch_client():
"""重置客户端（用于测试）"""
global _batch_client
_batch_client = None
gcp_batch_agent/src/server/core/config.py
+117
View File
@@ -0,0 +1,117 @@
"""
GCP Batch Agent 配置管理
管理 GCP 项目配置和服务账号凭证。
"""
import os
import json
import base64
from dataclasses import dataclass
from typing import Optional
from pathlib import Path
# 默认配置值（硬编码）
DEFAULT_PROJECT_ID = "gemini-20251105-b"
DEFAULT_LOCATION = "us-central1"
DEFAULT_MODEL = "gemini-2.5-flash"
DEFAULT_SERVICE_ACCOUNT_FILE = "gemini-20251105-b-ba3214d839ad.json"
@dataclass
class Config:
"""GCP Batch Agent 配置"""
# GCP 项目配置
project_id: str
location: str
# 服务账号凭证
service_account_json: Optional[str] = None
service_account_file: Optional[str] = None
# 默认模型
default_model: str = DEFAULT_MODEL
# API 配置
api_port: int = 8000
@classmethod
def from_env(cls) -> "Config":
"""从环境变量加载配置，如果未设置则使用默认值"""
# 项目 ID：优先使用环境变量，否则使用默认值
project_id = os.getenv("GCP_PROJECT_ID", DEFAULT_PROJECT_ID)
# 区域：优先使用环境变量，否则使用默认值
location = os.getenv("GCP_LOCATION", DEFAULT_LOCATION)
# 服务账号凭证：支持 Base64 编码的 JSON 或文件路径
sa_json = os.getenv("GCP_SERVICE_ACCOUNT_JSON")
sa_file = os.getenv("GCP_SERVICE_ACCOUNT_FILE")
# 如果没有指定凭证，自动查找默认的服务账号文件
if not sa_json and not sa_file:
sa_file = cls._find_service_account_file()
return cls(
project_id=project_id,
location=location,
service_account_json=sa_json,
service_account_file=sa_file,
default_model=os.getenv("DEFAULT_MODEL", DEFAULT_MODEL),
api_port=int(os.getenv("API_PORT", "8000"))
)
@staticmethod
def _find_service_account_file() -> Optional[str]:
"""自动查找服务账号文件"""
# 查找路径列表
search_paths = [
# 当前工作目录
Path.cwd() / DEFAULT_SERVICE_ACCOUNT_FILE,
# /app 目录（Docker 容器内）
Path("/app") / DEFAULT_SERVICE_ACCOUNT_FILE,
# 脚本所在目录
Path(__file__).parent.parent.parent.parent / DEFAULT_SERVICE_ACCOUNT_FILE,
]
for path in search_paths:
if path.exists():
return str(path)
return None
def get_credentials_info(self) -> Optional[dict]:
"""获取服务账号凭证信息"""
if self.service_account_json:
# 尝试 Base64 解码
try:
decoded = base64.b64decode(self.service_account_json)
return json.loads(decoded)
except Exception:
# 直接作为 JSON 字符串解析
return json.loads(self.service_account_json)
if self.service_account_file:
with open(self.service_account_file, 'r') as f:
return json.load(f)
return None
# 全局配置实例
_config: Optional[Config] = None
def get_config() -> Config:
"""获取全局配置实例"""
global _config
if _config is None:
_config = Config.from_env()
return _config
def reset_config():
"""重置配置（用于测试）"""
global _config
_config = None
gcp_batch_agent/src/server/core/credentials.py
+65
View File
@@ -0,0 +1,65 @@
"""
GCP 凭证管理
管理 GCP 服务账号凭证的创建和验证。
"""
from typing import Optional
from google.oauth2 import service_account
from google.auth import default as default_credentials
from .config import Config, get_config
class CredentialsManager:
"""GCP 凭证管理器"""
SCOPES = ["https://www.googleapis.com/auth/cloud-platform"]
def __init__(self, config: Optional[Config] = None):
self.config = config or get_config()
self._credentials = None
def get_credentials(self) -> service_account.Credentials:
"""获取 GCP 凭证"""
if self._credentials is not None:
return self._credentials
creds_info = self.config.get_credentials_info()
if creds_info:
# 使用服务账号凭证
self._credentials = service_account.Credentials.from_service_account_info(
creds_info,
scopes=self.SCOPES
)
else:
# 使用应用默认凭证 (ADC)
self._credentials, _ = default_credentials(scopes=self.SCOPES)
return self._credentials
def get_project_id(self) -> str:
"""获取项目 ID"""
return self.config.project_id
def get_location(self) -> str:
"""获取区域"""
return self.config.location
# 全局凭证管理器实例
_credentials_manager: Optional[CredentialsManager] = None
def get_credentials_manager() -> CredentialsManager:
"""获取全局凭证管理器实例"""
global _credentials_manager
if _credentials_manager is None:
_credentials_manager = CredentialsManager()
return _credentials_manager
def reset_credentials_manager():
"""重置凭证管理器（用于测试）"""
global _credentials_manager
_credentials_manager = None
gcp_batch_agent/src/server/core/schemas.py
+113
View File
@@ -0,0 +1,113 @@
"""
GCP Batch Agent 数据模型
定义批量作业相关的数据结构。
"""
from dataclasses import dataclass, field
from typing import Optional, List
from datetime import datetime
from enum import Enum
class JobState(str, Enum):
"""批量作业状态"""
JOB_STATE_UNSPECIFIED = "JOB_STATE_UNSPECIFIED"
JOB_STATE_QUEUED = "JOB_STATE_QUEUED"
JOB_STATE_PENDING = "JOB_STATE_PENDING"
JOB_STATE_RUNNING = "JOB_STATE_RUNNING"
JOB_STATE_SUCCEEDED = "JOB_STATE_SUCCEEDED"
JOB_STATE_FAILED = "JOB_STATE_FAILED"
JOB_STATE_CANCELLING = "JOB_STATE_CANCELLING"
JOB_STATE_CANCELLED = "JOB_STATE_CANCELLED"
JOB_STATE_PAUSED = "JOB_STATE_PAUSED"
JOB_STATE_EXPIRED = "JOB_STATE_EXPIRED"
JOB_STATE_UPDATING = "JOB_STATE_UPDATING"
JOB_STATE_PARTIALLY_SUCCEEDED = "JOB_STATE_PARTIALLY_SUCCEEDED"
@dataclass
class JobProgress:
"""作业进度"""
total_count: int = 0
succeeded_count: int = 0
failed_count: int = 0
def to_dict(self) -> dict:
return {
"total_count": self.total_count,
"succeeded_count": self.succeeded_count,
"failed_count": self.failed_count
}
@dataclass
class BatchJobRequest:
"""批量作业请求"""
input_uri: str
output_uri: str
model: str = "gemini-2.0-flash"
display_name: Optional[str] = None
def validate(self):
"""验证请求参数"""
if not self.input_uri.startswith("gs://"):
raise ValueError(f"input_uri 必须是 GCS 路径，格式: gs://bucket/path，当前值: {self.input_uri}")
if not self.output_uri.startswith("gs://"):
raise ValueError(f"output_uri 必须是 GCS 路径，格式: gs://bucket/path，当前值: {self.output_uri}")
# 支持 .jsonl 和 .json 文件
if not (self.input_uri.endswith(".jsonl") or self.input_uri.endswith(".json")):
raise ValueError(f"input_uri 必须是 JSONL 或 JSON 文件，当前值: {self.input_uri}")
@dataclass
class BatchJobInfo:
"""批量作业信息"""
job_id: str
job_name: str
state: JobState
input_uri: str
output_uri: str
model: str
create_time: Optional[datetime] = None
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
progress: Optional[JobProgress] = None
error_message: Optional[str] = None
def to_dict(self) -> dict:
result = {
"job_id": self.job_id,
"job_name": self.job_name,
"state": self.state.value if isinstance(self.state, JobState) else self.state,
"input_uri": self.input_uri,
"output_uri": self.output_uri,
"model": self.model
}
if self.create_time:
result["create_time"] = self.create_time.isoformat()
if self.start_time:
result["start_time"] = self.start_time.isoformat()
if self.end_time:
result["end_time"] = self.end_time.isoformat()
if self.progress:
result["progress"] = self.progress.to_dict()
if self.error_message:
result["error_message"] = self.error_message
return result
@dataclass
class BatchJobListResponse:
"""批量作业列表响应"""
jobs: List[BatchJobInfo] = field(default_factory=list)
next_page_token: Optional[str] = None
def to_dict(self) -> dict:
return {
"jobs": [job.to_dict() for job in self.jobs],
"next_page_token": self.next_page_token
}
gcp_batch_agent/src/server/mcp_server.py
+286
View File
@@ -0,0 +1,286 @@
"""
GCP Batch Agent MCP 服务器
提供批量推理相关的 MCP 工具。
"""
import os
import json
import logging
from typing import Optional
from mcp.server.fastmcp import FastMCP
from .core.config import get_config
from .core.batch_client import get_batch_client, BatchClient
from .core.schemas import BatchJobRequest
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# ==================== MCP 服务器 ====================
server = FastMCP('GCP Batch Agent')
# 系统提示词
SYSTEM_PROMPT = '''你是一个 GCP 批量推理助手。
能够帮助用户提交 Vertex AI 批量推理作业、监控作业状态、列出作业和取消作业。
核心能力：
1. 提交批量作业 - 用户提供 GCS 输入文件和输出目录
2. 查询作业状态 - 获取作业的当前状态和进度
3. 列出作业 - 查看所有批量作业
4. 取消作业 - 取消正在运行的作业
使用前提：
- 用户需要将输入文件（JSONL 格式）上传到 Cloud Storage
- 用户需要给 Agent 服务账号授权访问其存储桶
'''
# ==================== MCP 工具定义 ====================
@server.tool()
async def submit_batch_job(
input_uri: str,
output_uri: str,
model: Optional[str] = None,
display_name: Optional[str] = None
) -> str:
"""
提交 GCP Vertex AI 批量推理作业
Args:
input_uri: 输入文件的 GCS 路径，格式: gs://bucket/path/input.jsonl
output_uri: 输出目录的 GCS 路径，格式: gs://bucket/path/output/
model: 模型名称，默认 gemini-2.0-flash。支持: gemini-2.0-flash, gemini-1.5-flash, gemini-1.5-pro
display_name: 作业显示名称，可选
Returns:
JSON 格式的作业信息，包含 job_id、state 等
"""
try:
config = get_config()
client = get_batch_client()
# 使用默认模型
if not model:
model = config.default_model
# 创建请求
request = BatchJobRequest(
input_uri=input_uri,
output_uri=output_uri,
model=model,
display_name=display_name
)
# 提交作业
job_info = await client.submit_job(request)
return json.dumps({
"success": True,
"message": "批量作业已提交",
"job": job_info.to_dict()
}, ensure_ascii=False, indent=2)
except ValueError as e:
return json.dumps({
"success": False,
"error": f"参数错误: {str(e)}"
}, ensure_ascii=False)
except Exception as e:
logger.exception("提交批量作业失败")
return json.dumps({
"success": False,
"error": str(e)
}, ensure_ascii=False)
@server.tool()
async def get_job_status(job_id: str) -> str:
"""
查询批量作业状态
Args:
job_id: 作业 ID，可以是完整路径或简短 ID
Returns:
JSON 格式的作业状态信息
"""
try:
client = get_batch_client()
job_info = await client.get_job_status(job_id)
return json.dumps({
"success": True,
"job": job_info.to_dict()
}, ensure_ascii=False, indent=2)
except Exception as e:
logger.exception("查询作业状态失败")
return json.dumps({
"success": False,
"error": str(e)
}, ensure_ascii=False)
@server.tool()
async def list_jobs(
filter_str: Optional[str] = None,
page_size: int = 10
) -> str:
"""
列出批量作业
Args:
filter_str: 过滤条件，可选。例如: state="JOB_STATE_RUNNING"
page_size: 每页数量，默认 10
Returns:
JSON 格式的作业列表
"""
try:
client = get_batch_client()
response = await client.list_jobs(
filter_str=filter_str,
page_size=page_size
)
return json.dumps({
"success": True,
"total": len(response.jobs),
"jobs": [job.to_dict() for job in response.jobs],
"next_page_token": response.next_page_token
}, ensure_ascii=False, indent=2)
except Exception as e:
logger.exception("列出作业失败")
return json.dumps({
"success": False,
"error": str(e)
}, ensure_ascii=False)
@server.tool()
async def cancel_job(job_id: str) -> str:
"""
取消批量作业
Args:
job_id: 作业 ID，可以是完整路径或简短 ID
Returns:
JSON 格式的取消结果
"""
try:
client = get_batch_client()
job_info = await client.cancel_job(job_id)
return json.dumps({
"success": True,
"message": "作业取消请求已发送",
"job": job_info.to_dict()
}, ensure_ascii=False, indent=2)
except Exception as e:
logger.exception("取消作业失败")
return json.dumps({
"success": False,
"error": str(e)
}, ensure_ascii=False)
# ==================== 工具映射（供 API 使用）====================
TOOL_MAP = {
'submit_batch_job': submit_batch_job,
'get_job_status': get_job_status,
'list_jobs': list_jobs,
'cancel_job': cancel_job,
}
TOOL_LIST = [
{
"name": "submit_batch_job",
"description": "提交 GCP Vertex AI 批量推理作业",
"inputSchema": {
"type": "object",
"properties": {
"input_uri": {
"type": "string",
"description": "输入文件的 GCS 路径，格式: gs://bucket/path/input.jsonl"
},
"output_uri": {
"type": "string",
"description": "输出目录的 GCS 路径，格式: gs://bucket/path/output/"
},
"model": {
"type": "string",
"description": "模型名称，默认 gemini-2.0-flash",
"default": "gemini-2.0-flash"
},
"display_name": {
"type": "string",
"description": "作业显示名称，可选"
}
},
"required": ["input_uri", "output_uri"]
}
},
{
"name": "get_job_status",
"description": "查询批量作业状态",
"inputSchema": {
"type": "object",
"properties": {
"job_id": {
"type": "string",
"description": "作业 ID，可以是完整路径或简短 ID"
}
},
"required": ["job_id"]
}
},
{
"name": "list_jobs",
"description": "列出批量作业",
"inputSchema": {
"type": "object",
"properties": {
"filter_str": {
"type": "string",
"description": "过滤条件，可选"
},
"page_size": {
"type": "integer",
"description": "每页数量，默认 10",
"default": 10
}
},
"required": []
}
},
{
"name": "cancel_job",
"description": "取消批量作业",
"inputSchema": {
"type": "object",
"properties": {
"job_id": {
"type": "string",
"description": "作业 ID，可以是完整路径或简短 ID"
}
},
"required": ["job_id"]
}
}
]
# 导出
__all__ = ['server', 'SYSTEM_PROMPT', 'TOOL_MAP', 'TOOL_LIST']
if __name__ == '__main__':
server.run()
plans/gcp_batch_inference_agent_plan.md
+351
View File
@@ -0,0 +1,351 @@
# GCP 批量推理 Agent 设计方案
## 概述
基于 Vertex AI Flex PayGo 模式的批量推理 Agent，用户传入 Cloud Storage 地址，Agent 自动提交批量作业并监控状态。
---
## 架构设计
### 认证方案
```
┌─────────────────────────────────────────────────────────────────┐
│ 用户操作流程 │
├─────────────────────────────────────────────────────────────────┤
│ 1. 用户在 GCP 控制台给 Agent 服务账号授权存储桶访问权限 │
│ 2. 用户准备 JSONL 输入文件并上传到自己的 Cloud Storage │
│ 3. 用户调用 Agent API，传入 GCS 输入/输出路径 │
│ 4. Agent 使用固定服务账号提交批量作业 │
│ 5. Agent 监控作业状态并返回结果 │
└─────────────────────────────────────────────────────────────────┘
```
### 系统架构
```mermaid
flowchart TB
subgraph User[用户侧]
U1[准备 JSONL 文件]
U2[上传到 GCS]
U3[授权 Agent 服务账号]
end
subgraph Agent[GCP Batch Agent]
A1[API Server]
A2[MCP Server]
A3[Batch Job Manager]
A4[Job Status Monitor]
A5[Service Account Credentials]
end
subgraph GCP[Google Cloud Platform]
G1[Cloud Storage - 输入]
G2[Vertex AI Batch API]
G3[Gemini Model]
G4[Cloud Storage - 输出]
end
U1 --> U2
U2 --> G1
U3 --> A5
A1 --> A2
A2 --> A3
A3 --> G2
G2 --> G3
G3 --> G4
A4 --> G2
A5 --> A3
```
---
## 核心功能
### 1. 提交批量作业 - submit_batch_job
**输入参数：**
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| input_uri | string | 是 | 输入文件 GCS 路径，如 gs://bucket/input.jsonl |
| output_uri | string | 是 | 输出目录 GCS 路径，如 gs://bucket/output/ |
| model | string | 否 | 模型名称，默认 gemini-2.0-flash |
| display_name | string | 否 | 作业显示名称 |
**输出：**
```json
{
"success": true,
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456",
"job_name": "batch-job-20240305-123456",
"state": "JOB_STATE_PENDING",
"create_time": "2024-03-05T07:30:00Z"
}
```
### 2. 查询作业状态 - get_job_status
**输入参数：**
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| job_id | string | 是 | 批量作业 ID |
**输出：**
```json
{
"success": true,
"job_id": "projects/xxx/locations/us-central1/batchPredictionJobs/123456",
"state": "JOB_STATE_SUCCEEDED",
"progress": {
"total_count": 1000,
"succeeded_count": 998,
"failed_count": 2
},
"output_uri": "gs://bucket/output/",
"create_time": "2024-03-05T07:30:00Z",
"end_time": "2024-03-05T08:15:00Z"
}
```
### 3. 列出作业 - list_jobs
**输入参数：**
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| filter | string | 否 | 过滤条件 |
| page_size | int | 否 | 每页数量，默认 10 |
### 4. 取消作业 - cancel_job
**输入参数：**
| 参数 | 类型 | 必填 | 说明 |
|------|------|------|------|
| job_id | string | 是 | 批量作业 ID |
---
## 输入文件格式
用户需要准备符合 Vertex AI 批量推理格式的 JSONL 文件：
### 文本请求示例
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"What is the capital of France?"}]}]}}
{"request":{"contents":[{"role":"user","parts":[{"text":"Explain quantum computing in simple terms."}]}]}}
```
### 多模态请求示例（图片）
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"Describe this image"},{"fileData":{"mimeType":"image/jpeg","fileUri":"gs://bucket/image1.jpg"}}]}]}}
```
### 多模态请求示例（视频）
```jsonl
{"request":{"contents":[{"role":"user","parts":[{"text":"Summarize this video"},{"fileData":{"mimeType":"video/mp4","fileUri":"gs://bucket/video1.mp4"}}]}]}}
```
---
## 项目结构
```
gcp_batch_agent/
├── Dockerfile
├── README.md
├── requirements.txt
├── run_api_server.py
├── USAGE.md
└── src/
├── __init__.py
└── server/
├── __init__.py
├── api_server.py # FastAPI + MCP HTTP 端点
├── mcp_server.py # MCP 工具定义
└── core/
├── __init__.py
├── config.py # 配置管理
├── credentials.py # GCP 凭证管理
├── batch_client.py # Vertex AI 批量推理客户端
└── schemas.py # 数据模型
```
---
## 环境变量配置
| 变量 | 必需 | 说明 |
|------|------|------|
| GCP_PROJECT_ID | 是 | GCP 项目 ID |
| GCP_LOCATION | 否 | GCP 区域，默认 us-central1 |
| GCP_SERVICE_ACCOUNT_JSON | 是 | 服务账号 JSON 凭证（Base64 编码或文件路径） |
| DEFAULT_MODEL | 否 | 默认模型，默认 gemini-2.0-flash |
| API_PORT | 否 | API 端口，默认 8000 |
---
## API 端点
| 端点 | 方法 | 说明 |
|------|------|------|
| / | GET | 服务状态 |
| /health | GET | 健康检查 |
| /mcp | POST | MCP JSON-RPC |
| /mcp/sse | GET/POST | MCP SSE 流式 |
| /api/v1/batch/submit | POST | 提交批量作业 |
| /api/v1/batch/status/{job_id} | GET | 查询作业状态 |
| /api/v1/batch/list | GET | 列出作业 |
| /api/v1/batch/cancel/{job_id} | POST | 取消作业 |
---
## 用户使用前提
### 1. 授权 Agent 服务账号访问存储桶
用户需要在 GCP 控制台执行以下操作：
```bash
# 获取 Agent 服务账号邮箱（由 Agent 管理员提供）
AGENT_SA="batch-agent@your-project.iam.gserviceaccount.com"
# 授权存储桶访问权限
gsutil iam ch serviceAccount:${AGENT_SA}:objectViewer gs://your-bucket
gsutil iam ch serviceAccount:${AGENT_SA}:objectCreator gs://your-bucket
```
或在 GCP 控制台：
1. 进入 Cloud Storage > 存储桶 > 权限
2. 添加主账号：Agent 服务账号邮箱
3. 角色：Storage Object Viewer + Storage Object Creator
### 2. 准备输入文件
按照上述 JSONL 格式准备输入文件并上传到 GCS。
---
## 依赖包
```txt
# Pydantic AI
pydantic-ai>=0.0.14
# MCP
mcp>=0.9.0
fastmcp>=0.1.0
# FastAPI
fastapi>=0.109.0
uvicorn[standard]>=0.27.0
# HTTP Client
aiohttp>=3.9.0
# GCP
google-cloud-aiplatform>=1.38.0
google-auth>=2.23.0
```
---
## 实现计划
### Phase 1: 基础框架 ✅
- [x] 创建项目目录结构
- [x] 配置 Dockerfile 和 requirements.txt
- [x] 实现 GCP 凭证管理模块
### Phase 2: 核心功能 ✅
- [x] 实现 Vertex AI 批量推理客户端
- [x] 实现 submit_batch_job 工具
- [x] 实现 get_job_status 工具
- [x] 实现 list_jobs 工具
- [x] 实现 cancel_job 工具
### Phase 3: API 集成 ✅
- [x] 实现 MCP Server 工具注册
- [x] 实现 REST API 端点
- [x] 添加错误处理和日志
### Phase 4: 文档和测试 ✅
- [x] 编写 README.md
- [x] 编写 USAGE.md 使用文档
- [ ] 本地测试验证（需要 GCP 凭证）
---
## 调用示例
### REST API 调用
```bash
# 提交批量作业
curl -X POST http://localhost:8000/api/v1/batch/submit \
-H "Content-Type: application/json" \
-H "api-key: your-api-key" \
-d '{
"input_uri": "gs://my-bucket/input.jsonl",
"output_uri": "gs://my-bucket/output/",
"model": "gemini-2.0-flash",
"display_name": "my-batch-job"
}'
# 查询作业状态
curl http://localhost:8000/api/v1/batch/status/123456 \
-H "api-key: your-api-key"
```
### MCP 调用
```json
{
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
"name": "submit_batch_job",
"arguments": {
"input_uri": "gs://my-bucket/input.jsonl",
"output_uri": "gs://my-bucket/output/",
"model": "gemini-2.0-flash"
}
}
}
```
---
## 安全考虑
1. **服务账号权限最小化**：Agent 服务账号只需要 Vertex AI User 和 Storage Object Admin 权限
2. **用户存储桶隔离**：每个用户的数据存储在自己的存储桶中
3. **API Key 验证**：所有 API 调用需要有效的 API Key
4. **日志审计**：记录所有批量作业的提交和状态变更
---
## 注意事项
1. **Flex PayGo 限制**：
- 作业可能需要排队等待资源
- 大型作业可能需要较长时间完成
2. **存储桶权限**：
- 用户必须提前授权 Agent 服务账号
- 输入文件必须存在且可读
- 输出目录必须可写
3. **文件格式**：
- 输入必须是有效的 JSONL 格式
- 每行必须符合 Vertex AI 批量推理的请求格式
test_agent/.venv/bin/Activate.ps1
+247
View File
@@ -0,0 +1,247 @@
<#
.Synopsis
Activate a Python virtual environment for the current PowerShell session.
.Description
Pushes the python executable for a virtual environment to the front of the
$Env:PATH environment variable and sets the prompt to signify that you are
in a Python virtual environment. Makes use of the command line switches as
well as the `pyvenv.cfg` file values present in the virtual environment.
.Parameter VenvDir
Path to the directory that contains the virtual environment to activate. The
default value for this is the parent of the directory that the Activate.ps1
script is located within.
.Parameter Prompt
The prompt prefix to display when this virtual environment is activated. By
default, this prompt is the name of the virtual environment folder (VenvDir)
surrounded by parentheses and followed by a single space (ie. '(.venv) ').
.Example
Activate.ps1
Activates the Python virtual environment that contains the Activate.ps1 script.
.Example
Activate.ps1 -Verbose
Activates the Python virtual environment that contains the Activate.ps1 script,
and shows extra information about the activation as it executes.
.Example
Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
Activates the Python virtual environment located in the specified location.
.Example
Activate.ps1 -Prompt "MyPython"
Activates the Python virtual environment that contains the Activate.ps1 script,
and prefixes the current prompt with the specified string (surrounded in
parentheses) while the virtual environment is active.
.Notes
On Windows, it may be required to enable this Activate.ps1 script by setting the
execution policy for the user. You can do this by issuing the following PowerShell
command:
PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
For more information on Execution Policies:
https://go.microsoft.com/fwlink/?LinkID=135170
#>
Param(
[Parameter(Mandatory = $false)]
[String]
$VenvDir,
[Parameter(Mandatory = $false)]
[String]
$Prompt
)
<# Function declarations --------------------------------------------------- #>
<#
.Synopsis
Remove all shell session elements added by the Activate script, including the
addition of the virtual environment's Python executable from the beginning of
the PATH variable.
.Parameter NonDestructive
If present, do not remove this function from the global namespace for the
session.
#>
function global:deactivate ([switch]$NonDestructive) {
# Revert to original values
# The prior prompt:
if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
}
# The prior PYTHONHOME:
if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
}
# The prior PATH:
if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
Remove-Item -Path Env:_OLD_VIRTUAL_PATH
}
# Just remove the VIRTUAL_ENV altogether:
if (Test-Path -Path Env:VIRTUAL_ENV) {
Remove-Item -Path env:VIRTUAL_ENV
}
# Just remove VIRTUAL_ENV_PROMPT altogether.
if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) {
Remove-Item -Path env:VIRTUAL_ENV_PROMPT
}
# Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
}
# Leave deactivate function in the global namespace if requested:
if (-not $NonDestructive) {
Remove-Item -Path function:deactivate
}
}
<#
.Description
Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
given folder, and returns them in a map.
For each line in the pyvenv.cfg file, if that line can be parsed into exactly
two strings separated by `=` (with any amount of whitespace surrounding the =)
then it is considered a `key = value` line. The left hand string is the key,
the right hand is the value.
If the value starts with a `'` or a `"` then the first and last character is
stripped from the value before being captured.
.Parameter ConfigDir
Path to the directory that contains the `pyvenv.cfg` file.
#>
function Get-PyVenvConfig(
[String]
$ConfigDir
) {
Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
# Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
$pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
# An empty map will be returned if no config file is found.
$pyvenvConfig = @{ }
if ($pyvenvConfigPath) {
Write-Verbose "File exists, parse `key = value` lines"
$pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
$pyvenvConfigContent | ForEach-Object {
$keyval = $PSItem -split "\s*=\s*", 2
if ($keyval[0] -and $keyval[1]) {
$val = $keyval[1]
# Remove extraneous quotations around a string value.
if ("'""".Contains($val.Substring(0, 1))) {
$val = $val.Substring(1, $val.Length - 2)
}
$pyvenvConfig[$keyval[0]] = $val
Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
}
}
}
return $pyvenvConfig
}
<# Begin Activate script --------------------------------------------------- #>
# Determine the containing directory of this script
$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
$VenvExecDir = Get-Item -Path $VenvExecPath
Write-Verbose "Activation script is located in path: '$VenvExecPath'"
Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
# Set values required in priority: CmdLine, ConfigFile, Default
# First, get the location of the virtual environment, it might not be
# VenvExecDir if specified on the command line.
if ($VenvDir) {
Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
}
else {
Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
$VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
Write-Verbose "VenvDir=$VenvDir"
}
# Next, read the `pyvenv.cfg` file to determine any required value such
# as `prompt`.
$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
# Next, set the prompt from the command line, or the config file, or
# just use the name of the virtual environment folder.
if ($Prompt) {
Write-Verbose "Prompt specified as argument, using '$Prompt'"
}
else {
Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
$Prompt = $pyvenvCfg['prompt'];
}
else {
Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
$Prompt = Split-Path -Path $venvDir -Leaf
}
}
Write-Verbose "Prompt = '$Prompt'"
Write-Verbose "VenvDir='$VenvDir'"
# Deactivate any currently active virtual environment, but leave the
# deactivate function in place.
deactivate -nondestructive
# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
# that there is an activated venv.
$env:VIRTUAL_ENV = $VenvDir
if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
Write-Verbose "Setting prompt to '$Prompt'"
# Set the prompt to include the env name
# Make sure _OLD_VIRTUAL_PROMPT is global
function global:_OLD_VIRTUAL_PROMPT { "" }
Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
function global:prompt {
Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
_OLD_VIRTUAL_PROMPT
}
$env:VIRTUAL_ENV_PROMPT = $Prompt
}
# Clear PYTHONHOME
if (Test-Path -Path Env:PYTHONHOME) {
Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
Remove-Item -Path Env:PYTHONHOME
}
# Add the venv to the PATH
Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"
test_agent/.venv/bin/activate
+70
View File
@@ -0,0 +1,70 @@
# This file must be used with "source bin/activate" *from bash*
# You cannot run it directly
deactivate () {
# reset old environment variables
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
PATH="${_OLD_VIRTUAL_PATH:-}"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# Call hash to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
hash -r 2> /dev/null
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
unset VIRTUAL_ENV_PROMPT
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
# on Windows, a path can contain colons and backslashes and has to be converted:
if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then
# transform D:\path\to\venv to /d/path/to/venv on MSYS
# and to /cygdrive/d/path/to/venv on Cygwin
export VIRTUAL_ENV=$(cygpath /home/taiji/tools/pingtai_agent/test_agent/.venv)
else
# use the path as-is
export VIRTUAL_ENV=/home/taiji/tools/pingtai_agent/test_agent/.venv
fi
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/"bin":$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
PS1='(.venv) '"${PS1:-}"
export PS1
VIRTUAL_ENV_PROMPT='(.venv) '
export VIRTUAL_ENV_PROMPT
fi
# Call hash to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
hash -r 2> /dev/null
test_agent/.venv/bin/activate.csh
+27
View File
@@ -0,0 +1,27 @@
# This file must be used with "source bin/activate.csh" *from csh*.
# You cannot run it directly.
# Created by Davide Di Blasi <davidedb@gmail.com>.
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate'
# Unset irrelevant variables.
deactivate nondestructive
setenv VIRTUAL_ENV /home/taiji/tools/pingtai_agent/test_agent/.venv
set _OLD_VIRTUAL_PATH="$PATH"
setenv PATH "$VIRTUAL_ENV/"bin":$PATH"
set _OLD_VIRTUAL_PROMPT="$prompt"
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
set prompt = '(.venv) '"$prompt"
setenv VIRTUAL_ENV_PROMPT '(.venv) '
endif
alias pydoc python -m pydoc
rehash
test_agent/.venv/bin/activate.fish
+69
View File
@@ -0,0 +1,69 @@
# This file must be used with "source <venv>/bin/activate.fish" *from fish*
# (https://fishshell.com/). You cannot run it directly.
function deactivate -d "Exit virtual environment and return to normal shell environment"
# reset old environment variables
if test -n "$_OLD_VIRTUAL_PATH"
set -gx PATH $_OLD_VIRTUAL_PATH
set -e _OLD_VIRTUAL_PATH
end
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
set -e _OLD_VIRTUAL_PYTHONHOME
end
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
set -e _OLD_FISH_PROMPT_OVERRIDE
# prevents error when using nested fish instances (Issue #93858)
if functions -q _old_fish_prompt
functions -e fish_prompt
functions -c _old_fish_prompt fish_prompt
functions -e _old_fish_prompt
end
end
set -e VIRTUAL_ENV
set -e VIRTUAL_ENV_PROMPT
if test "$argv[1]" != "nondestructive"
# Self-destruct!
functions -e deactivate
end
end
# Unset irrelevant variables.
deactivate nondestructive
set -gx VIRTUAL_ENV /home/taiji/tools/pingtai_agent/test_agent/.venv
set -gx _OLD_VIRTUAL_PATH $PATH
set -gx PATH "$VIRTUAL_ENV/"bin $PATH
# Unset PYTHONHOME if set.
if set -q PYTHONHOME
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
set -e PYTHONHOME
end
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
# fish uses a function instead of an env var to generate the prompt.
# Save the current fish_prompt function as the function _old_fish_prompt.
functions -c fish_prompt _old_fish_prompt
# With the original prompt function renamed, we can override with our own.
function fish_prompt
# Save the return status of the last command.
set -l old_status $status
# Output the venv prompt; color taken from the blue of the Python logo.
printf "%s%s%s" (set_color 4B8BBE) '(.venv) ' (set_color normal)
# Restore the return status of the previous command.
echo "exit $old_status" | .
# Output the original/"old" prompt.
_old_fish_prompt
end
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
set -gx VIRTUAL_ENV_PROMPT '(.venv) '
end
test_agent/.venv/bin/normalizer
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/home/taiji/tools/pingtai_agent/test_agent/.venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from charset_normalizer.cli import cli_detect
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(cli_detect())
test_agent/.venv/bin/pip
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/home/taiji/tools/pingtai_agent/test_agent/.venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
test_agent/.venv/bin/pip3
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/home/taiji/tools/pingtai_agent/test_agent/.venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
test_agent/.venv/bin/pip3.12
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/home/taiji/tools/pingtai_agent/test_agent/.venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
test_agent/.venv/bin/python
Symlink
+1
View File
@@ -0,0 +1 @@
python3
test_agent/.venv/bin/python3
Symlink
+1
View File
@@ -0,0 +1 @@
/usr/bin/python3
test_agent/.venv/bin/python3.12
Symlink
+1
View File
@@ -0,0 +1 @@
python3
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/INSTALLER
+1
View File
@@ -0,0 +1 @@
pip
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/METADATA
+78
View File
@@ -0,0 +1,78 @@
Metadata-Version: 2.4
Name: certifi
Version: 2026.2.25
Summary: Python package for providing Mozilla's CA Bundle.
Home-page: https://github.com/certifi/python-certifi
Author: Kenneth Reitz
Author-email: me@kennethreitz.com
License: MPL-2.0
Project-URL: Source, https://github.com/certifi/python-certifi
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
Classifier: Natural Language :: English
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Requires-Python: >=3.7
License-File: LICENSE
Dynamic: author
Dynamic: author-email
Dynamic: classifier
Dynamic: description
Dynamic: home-page
Dynamic: license
Dynamic: license-file
Dynamic: project-url
Dynamic: requires-python
Dynamic: summary
Certifi: Python SSL Certificates
================================
Certifi provides Mozilla's carefully curated collection of Root Certificates for
validating the trustworthiness of SSL certificates while verifying the identity
of TLS hosts. It has been extracted from the `Requests`_ project.
Installation
------------
``certifi`` is available on PyPI. Simply install it with ``pip``::
$ pip install certifi
Usage
-----
To reference the installed certificate authority (CA) bundle, you can use the
built-in function::
>>> import certifi
>>> certifi.where()
'/usr/local/lib/python3.7/site-packages/certifi/cacert.pem'
Or from the command line::
$ python -m certifi
/usr/local/lib/python3.7/site-packages/certifi/cacert.pem
Enjoy!
.. _`Requests`: https://requests.readthedocs.io/en/master/
Addition/Removal of Certificates
--------------------------------
Certifi does not support any addition/removal or other modification of the
CA trust store content. This project is intended to provide a reliable and
highly portable root of trust to python deployments. Look to upstream projects
for methods to use alternate trust.
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/RECORD
+14
View File
@@ -0,0 +1,14 @@
certifi-2026.2.25.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
certifi-2026.2.25.dist-info/METADATA,sha256=4NMuGXdg_hBiRA3paKVXYcDmE3VXEBWxTvCL2xlDyPU,2474
certifi-2026.2.25.dist-info/RECORD,,
certifi-2026.2.25.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
certifi-2026.2.25.dist-info/licenses/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989
certifi-2026.2.25.dist-info/top_level.txt,sha256=KMu4vUCfsjLrkPbSNdgdekS-pVJzBAJFO__nI8NF6-U,8
certifi/__init__.py,sha256=c9eaYufv1pSLl0Q8QNcMiMLLH4WquDcxdPyKjmI4opY,94
certifi/__main__.py,sha256=xBBoj905TUWBLRGANOcf7oi6e-3dMP4cEoG9OyMs11g,243
certifi/__pycache__/__init__.cpython-312.pyc,,
certifi/__pycache__/__main__.cpython-312.pyc,,
certifi/__pycache__/core.cpython-312.pyc,,
certifi/cacert.pem,sha256=_JFloSQDJj5-v72te-ej6sD6XTJdPHBGXyjTaQByyig,272441
certifi/core.py,sha256=XFXycndG5pf37ayeF8N32HUuDafsyhkVMbO4BAPWHa0,3394
certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/WHEEL
+5
View File
@@ -0,0 +1,5 @@
Wheel-Version: 1.0
Generator: setuptools (82.0.0)
Root-Is-Purelib: true
Tag: py3-none-any
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/licenses/LICENSE
+20
View File
@@ -0,0 +1,20 @@
This package contains a modified version of ca-bundle.crt:
ca-bundle.crt -- Bundle of CA Root Certificates
This is a bundle of X.509 certificates of public Certificate Authorities
(CA). These were automatically extracted from Mozilla's root certificates
file (certdata.txt). This file can be found in the mozilla source tree:
https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt
It contains the certificates in PEM format and therefore
can be directly used with curl / libcurl / php_curl, or with
an Apache+mod_ssl webserver for SSL client authentication.
Just configure this file as the SSLCACertificateFile.#
***** BEGIN LICENSE BLOCK *****
This Source Code Form is subject to the terms of the Mozilla Public License,
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
one at http://mozilla.org/MPL/2.0/.
***** END LICENSE BLOCK *****
@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $
test_agent/.venv/lib/python3.12/site-packages/certifi-2026.2.25.dist-info/top_level.txt
+1
View File
@@ -0,0 +1 @@
certifi
test_agent/.venv/lib/python3.12/site-packages/certifi/__init__.py
+4
View File
@@ -0,0 +1,4 @@
from .core import contents, where
__all__ = ["contents", "where"]
__version__ = "2026.02.25"
test_agent/.venv/lib/python3.12/site-packages/certifi/__main__.py
+12
View File
@@ -0,0 +1,12 @@
import argparse
from certifi import contents, where
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--contents", action="store_true")
args = parser.parse_args()
if args.contents:
print(contents())
else:
print(where())
test_agent/.venv/lib/python3.12/site-packages/certifi/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/certifi/__pycache__/__main__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/certifi/__pycache__/core.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/certifi/cacert.pem
+4494
View File
File diff suppressed because it is too large Load Diff
test_agent/.venv/lib/python3.12/site-packages/certifi/core.py
+83
View File
@@ -0,0 +1,83 @@
"""
certifi.py
~~~~~~~~~~
This module returns the installation location of cacert.pem or its contents.
"""
import sys
import atexit
def exit_cacert_ctx() -> None:
_CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr]
if sys.version_info >= (3, 11):
from importlib.resources import as_file, files
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the file
# in cases where we're inside of a zipimport situation until someone
# actually calls where(), but we don't want to re-extract the file
# on every call of where(), so we'll do it once then store it in a
# global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you to
# manage the cleanup of this file, so it doesn't actually return a
# path, it returns a context manager that will give you the path
# when you enter it and will do any cleanup when you leave it. In
# the common case of not needing a temporary file, it will just
# return the file system location and the __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem"))
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii")
else:
from importlib.resources import path as get_path, read_text
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the
# file in cases where we're inside of a zipimport situation until
# someone actually calls where(), but we don't want to re-extract
# the file on every call of where(), so we'll do it once then store
# it in a global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you
# to manage the cleanup of this file, so it doesn't actually
# return a path, it returns a context manager that will give
# you the path when you enter it and will do any cleanup when
# you leave it. In the common case of not needing a temporary
# file, it will just return the file system location and the
# __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = get_path("certifi", "cacert.pem")
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return read_text("certifi", "cacert.pem", encoding="ascii")
test_agent/.venv/lib/python3.12/site-packages/certifi/py.typed
View File
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/INSTALLER
+1
View File
@@ -0,0 +1 @@
pip
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/METADATA
+764
View File
@@ -0,0 +1,764 @@
Metadata-Version: 2.4
Name: charset-normalizer
Version: 3.4.4
Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet.
Author-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
Maintainer-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
License: MIT
Project-URL: Changelog, https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md
Project-URL: Documentation, https://charset-normalizer.readthedocs.io/
Project-URL: Code, https://github.com/jawah/charset_normalizer
Project-URL: Issue tracker, https://github.com/jawah/charset_normalizer/issues
Keywords: encoding,charset,charset-detector,detector,normalization,unicode,chardet,detect
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Text Processing :: Linguistic
Classifier: Topic :: Utilities
Classifier: Typing :: Typed
Requires-Python: >=3.7
Description-Content-Type: text/markdown
License-File: LICENSE
Provides-Extra: unicode-backport
Dynamic: license-file
<h1 align="center">Charset Detection, for Everyone 👋</h1>
<p align="center">
<sup>The Real First Universal Charset Detector</sup><br>
<a href="https://pypi.org/project/charset-normalizer">
<img src="https://img.shields.io/pypi/pyversions/charset_normalizer.svg?orange=blue" />
</a>
<a href="https://pepy.tech/project/charset-normalizer/">
<img alt="Download Count Total" src="https://static.pepy.tech/badge/charset-normalizer/month" />
</a>
<a href="https://bestpractices.coreinfrastructure.org/projects/7297">
<img src="https://bestpractices.coreinfrastructure.org/projects/7297/badge">
</a>
</p>
<p align="center">
<sup><i>Featured Packages</i></sup><br>
<a href="https://github.com/jawah/niquests">
<img alt="Static Badge" src="https://img.shields.io/badge/Niquests-Most_Advanced_HTTP_Client-cyan">
</a>
<a href="https://github.com/jawah/wassima">
<img alt="Static Badge" src="https://img.shields.io/badge/Wassima-Certifi_Replacement-cyan">
</a>
</p>
<p align="center">
<sup><i>In other language (unofficial port - by the community)</i></sup><br>
<a href="https://github.com/nickspring/charset-normalizer-rs">
<img alt="Static Badge" src="https://img.shields.io/badge/Rust-red">
</a>
</p>
> A library that helps you read text from an unknown charset encoding.<br /> Motivated by `chardet`,
> I'm trying to resolve the issue by taking a new approach.
> All IANA character set names for which the Python core library provides codecs are supported.
<p align="center">
>>>>> <a href="https://charsetnormalizerweb.ousret.now.sh" target="_blank">👉 Try Me Online Now, Then Adopt Me 👈 </a> <<<<<
</p>
This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**.
| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) |
|--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:|
| `Fast` | ❌ | ✅ | ✅ |
| `Universal**` | ❌ | ✅ | ❌ |
| `Reliable` **without** distinguishable standards | ❌ | ✅ | ✅ |
| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ |
| `License` | LGPL-2.1<br>_restrictive_ | MIT | MPL-1.1<br>_restrictive_ |
| `Native Python` | ✅ | ✅ | ❌ |
| `Detect spoken language` | ❌ | ✅ | N/A |
| `UnicodeDecodeError Safety` | ❌ | ✅ | ❌ |
| `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB |
| `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 |
<p align="center">
<img src="https://i.imgflip.com/373iay.gif" alt="Reading Normalized Text" width="226"/><img src="https://media.tenor.com/images/c0180f70732a18b4965448d33adba3d0/tenor.gif" alt="Cat Reading Text" width="200"/>
</p>
*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*<br>
## ⚡ Performance
This package offer better performance than its counterpart Chardet. Here are some numbers.
| Package | Accuracy | Mean per file (ms) | File per sec (est) |
|-----------------------------------------------|:--------:|:------------------:|:------------------:|
| [chardet](https://github.com/chardet/chardet) | 86 % | 63 ms | 16 file/sec |
| charset-normalizer | **98 %** | **10 ms** | 100 file/sec |
| Package | 99th percentile | 95th percentile | 50th percentile |
|-----------------------------------------------|:---------------:|:---------------:|:---------------:|
| [chardet](https://github.com/chardet/chardet) | 265 ms | 71 ms | 7 ms |
| charset-normalizer | 100 ms | 50 ms | 5 ms |
_updated as of december 2024 using CPython 3.12_
Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload.
> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows.
> And yes, these results might change at any time. The dataset can be updated to include more files.
> The actual delays heavily depends on your CPU capabilities. The factors should remain the same.
> Keep in mind that the stats are generous and that Chardet accuracy vs our is measured using Chardet initial capability
> (e.g. Supported Encoding) Challenge-them if you want.
## ✨ Installation
Using pip:
```sh
pip install charset-normalizer -U
```
## 🚀 Basic Usage
### CLI
This package comes with a CLI.
```
usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD]
file [file ...]
The Real First Universal Charset Detector. Discover originating encoding used
on text file. Normalize text to unicode.
positional arguments:
files File(s) to be analysed
optional arguments:
-h, --help show this help message and exit
-v, --verbose Display complementary information about file if any.
Stdout will contain logs about the detection process.
-a, --with-alternative
Output complementary possibilities if any. Top-level
JSON WILL be a list.
-n, --normalize Permit to normalize input file. If not set, program
does not write anything.
-m, --minimal Only output the charset detected to STDOUT. Disabling
JSON output.
-r, --replace Replace file when trying to normalize it instead of
creating a new one.
-f, --force Replace file without asking if you are sure, use this
flag with caution.
-t THRESHOLD, --threshold THRESHOLD
Define a custom maximum amount of chaos allowed in
decoded content. 0. <= chaos <= 1.
--version Show version information and exit.
```
```bash
normalizer ./data/sample.1.fr.srt
```
or
```bash
python -m charset_normalizer ./data/sample.1.fr.srt
```
🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format.
```json
{
"path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt",
"encoding": "cp1252",
"encoding_aliases": [
"1252",
"windows_1252"
],
"alternative_encodings": [
"cp1254",
"cp1256",
"cp1258",
"iso8859_14",
"iso8859_15",
"iso8859_16",
"iso8859_3",
"iso8859_9",
"latin_1",
"mbcs"
],
"language": "French",
"alphabets": [
"Basic Latin",
"Latin-1 Supplement"
],
"has_sig_or_bom": false,
"chaos": 0.149,
"coherence": 97.152,
"unicode_path": null,
"is_preferred": true
}
```
### Python
*Just print out normalized text*
```python
from charset_normalizer import from_path
results = from_path('./my_subtitle.srt')
print(str(results.best()))
```
*Upgrade your code without effort*
```python
from charset_normalizer import detect
```
The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible.
See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/)
## 😇 Why
When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a
reliable alternative using a completely different method. Also! I never back down on a good challenge!
I **don't care** about the **originating charset** encoding, because **two different tables** can
produce **two identical rendered string.**
What I want is to get readable text, the best I can.
In a way, **I'm brute forcing text decoding.** How cool is that ? 😎
Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair Unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode.
## 🍰 How
- Discard all charset encoding table that could not fit the binary content.
- Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding.
- Extract matches with the lowest mess detected.
- Additionally, we measure coherence / probe for a language.
**Wait a minute**, what is noise/mess and coherence according to **YOU ?**
*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then
**I established** some ground rules about **what is obvious** when **it seems like** a mess (aka. defining noise in rendered text).
I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to
improve or rewrite it.
*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought
that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design.
## ⚡ Known limitations
- Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters))
- Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content.
## ⚠️ About Python EOLs
**If you are running:**
- Python >=2.7,<3.5: Unsupported
- Python 3.5: charset-normalizer < 2.1
- Python 3.6: charset-normalizer < 3.1
- Python 3.7: charset-normalizer < 4.0
Upgrade your Python interpreter as soon as possible.
## 👤 Contributing
Contributions, issues and feature requests are very much welcome.<br />
Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute.
## 📝 License
Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).<br />
This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed.
Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/)
## 💼 For Enterprise
Professional support for charset-normalizer is available as part of the [Tidelift
Subscription][1]. Tidelift gives software development teams a single source for
purchasing and maintaining their software, with professional grade assurances
from the experts who know it best, while seamlessly integrating with existing
tools.
[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/7297/badge)](https://www.bestpractices.dev/projects/7297)
# Changelog
All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [3.4.4](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.4) (2025-10-13)
### Changed
- Bound `setuptools` to a specific constraint `setuptools>=68,<=81`.
- Raised upper bound of mypyc for the optional pre-built extension to v1.18.2
### Removed
- `setuptools-scm` as a build dependency.
### Misc
- Enforced hashes in `dev-requirements.txt` and created `ci-requirements.txt` for security purposes.
- Additional pre-built wheels for riscv64, s390x, and armv7l architectures.
- Restore ` multiple.intoto.jsonl` in GitHub releases in addition to individual attestation file per wheel.
## [3.4.3](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.3) (2025-08-09)
### Changed
- mypy(c) is no longer a required dependency at build time if `CHARSET_NORMALIZER_USE_MYPYC` isn't set to `1`. (#595) (#583)
- automatically lower confidence on small bytes samples that are not Unicode in `detect` output legacy function. (#391)
### Added
- Custom build backend to overcome inability to mark mypy as an optional dependency in the build phase.
- Support for Python 3.14
### Fixed
- sdist archive contained useless directories.
- automatically fallback on valid UTF-16 or UTF-32 even if the md says it's noisy. (#633)
### Misc
- SBOM are automatically published to the relevant GitHub release to comply with regulatory changes.
Each published wheel comes with its SBOM. We choose CycloneDX as the format.
- Prebuilt optimized wheel are no longer distributed by default for CPython 3.7 due to a change in cibuildwheel.
## [3.4.2](https://github.com/Ousret/charset_normalizer/compare/3.4.1...3.4.2) (2025-05-02)
### Fixed
- Addressed the DeprecationWarning in our CLI regarding `argparse.FileType` by backporting the target class into the package. (#591)
- Improved the overall reliability of the detector with CJK Ideographs. (#605) (#587)
### Changed
- Optional mypyc compilation upgraded to version 1.15 for Python >= 3.8
## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...3.4.1) (2024-12-24)
### Changed
- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend.
- Enforce annotation delayed loading for a simpler and consistent types in the project.
- Optional mypyc compilation upgraded to version 1.14 for Python >= 3.8
### Added
- pre-commit configuration.
- noxfile.
### Removed
- `build-requirements.txt` as per using `pyproject.toml` native build configuration.
- `bin/integration.py` and `bin/serve.py` in favor of downstream integration test (see noxfile).
- `setup.cfg` in favor of `pyproject.toml` metadata configuration.
- Unused `utils.range_scan` function.
### Fixed
- Converting content to Unicode bytes may insert `utf_8` instead of preferred `utf-8`. (#572)
- Deprecation warning "'count' is passed as positional argument" when converting to Unicode bytes on Python 3.13+
## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08)
### Added
- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints.
- Support for Python 3.13 (#512)
### Fixed
- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch.
- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537)
- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381)
## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31)
### Fixed
- Unintentional memory usage regression when using large payload that match several encoding (#376)
- Regression on some detection case showcased in the documentation (#371)
### Added
- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife)
## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22)
### Changed
- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8
- Improved the general detection reliability based on reports from the community
## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30)
### Added
- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer`
- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323)
### Removed
- (internal) Redundant utils.is_ascii function and unused function is_private_use_only
- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant
### Changed
- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection
- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8
### Fixed
- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350)
## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07)
### Changed
- Typehint for function `from_path` no longer enforce `PathLike` as its first argument
- Minor improvement over the global detection reliability
### Added
- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries
- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True)
- Explicit support for Python 3.12
### Fixed
- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289)
## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06)
### Added
- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262)
### Removed
- Support for Python 3.6 (PR #260)
### Changed
- Optional speedup provided by mypy/c 1.0.1
## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18)
### Fixed
- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233)
### Changed
- Speedup provided by mypy/c 0.990 on Python >= 3.7
## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20)
### Added
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
### Changed
- Build with static metadata using 'build' frontend
- Make the language detection stricter
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
### Fixed
- CLI with opt --normalize fail when using full path for files
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
- Sphinx warnings when generating the documentation
### Removed
- Coherence detector no longer return 'Simple English' instead return 'English'
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
- Breaking: Method `first()` and `best()` from CharsetMatch
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
- Breaking: Top-level function `normalize`
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
- Support for the backport `unicodedata2`
## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18)
### Added
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
### Changed
- Build with static metadata using 'build' frontend
- Make the language detection stricter
### Fixed
- CLI with opt --normalize fail when using full path for files
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
### Removed
- Coherence detector no longer return 'Simple English' instead return 'English'
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21)
### Added
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
### Removed
- Breaking: Method `first()` and `best()` from CharsetMatch
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
### Fixed
- Sphinx warnings when generating the documentation
## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15)
### Changed
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
### Removed
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
- Breaking: Top-level function `normalize`
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
- Support for the backport `unicodedata2`
## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19)
### Deprecated
- Function `normalize` scheduled for removal in 3.0
### Changed
- Removed useless call to decode in fn is_unprintable (#206)
### Fixed
- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204)
## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19)
### Added
- Output the Unicode table version when running the CLI with `--version` (PR #194)
### Changed
- Re-use decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175)
- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183)
### Fixed
- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175)
- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181)
### Removed
- Support for Python 3.5 (PR #192)
### Deprecated
- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194)
## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12)
### Fixed
- ASCII miss-detection on rare cases (PR #170)
## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30)
### Added
- Explicit support for Python 3.11 (PR #164)
### Changed
- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165)
## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04)
### Fixed
- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154)
### Changed
- Skipping the language-detection (CD) on ASCII (PR #155)
## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03)
### Changed
- Moderating the logging impact (since 2.0.8) for specific environments (PR #147)
### Fixed
- Wrong logging level applied when setting kwarg `explain` to True (PR #146)
## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24)
### Changed
- Improvement over Vietnamese detection (PR #126)
- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124)
- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122)
- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129)
- Code style as refactored by Sourcery-AI (PR #131)
- Minor adjustment on the MD around european words (PR #133)
- Remove and replace SRTs from assets / tests (PR #139)
- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135)
- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135)
### Fixed
- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137)
- Avoid using too insignificant chunk (PR #137)
### Added
- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135)
- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141)
## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11)
### Added
- Add support for Kazakh (Cyrillic) language detection (PR #109)
### Changed
- Further, improve inferring the language from a given single-byte code page (PR #112)
- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116)
- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113)
- Various detection improvement (MD+CD) (PR #117)
### Removed
- Remove redundant logging entry about detected language(s) (PR #115)
### Fixed
- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102)
## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18)
### Fixed
- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100)
- Fix CLI crash when using --minimal output in certain cases (PR #103)
### Changed
- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101)
## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14)
### Changed
- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81)
- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82)
- The Unicode detection is slightly improved (PR #93)
- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91)
### Removed
- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92)
### Fixed
- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95)
- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96)
- The MANIFEST.in was not exhaustive (PR #78)
## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30)
### Fixed
- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70)
- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68)
- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72)
- Submatch factoring could be wrong in rare edge cases (PR #72)
- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72)
- Fix line endings from CRLF to LF for certain project files (PR #67)
### Changed
- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76)
- Allow fallback on specified encoding if any (PR #71)
## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16)
### Changed
- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63)
- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64)
## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15)
### Fixed
- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59)
### Changed
- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57)
## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13)
### Fixed
- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55)
- Using explain=False permanently disable the verbose output in the current runtime (PR #47)
- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47)
- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52)
### Changed
- Public function normalize default args values were not aligned with from_bytes (PR #53)
### Added
- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47)
## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02)
### Changed
- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet.
- Accent has been made on UTF-8 detection, should perform rather instantaneous.
- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible.
- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time)
- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+
- utf_7 detection has been reinstated.
### Removed
- This package no longer require anything when used with Python 3.5 (Dropped cached_property)
- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian.
- The exception hook on UnicodeDecodeError has been removed.
### Deprecated
- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0
### Fixed
- The CLI output used the relative path of the file(s). Should be absolute.
## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28)
### Fixed
- Logger configuration/usage no longer conflict with others (PR #44)
## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21)
### Removed
- Using standard logging instead of using the package loguru.
- Dropping nose test framework in favor of the maintained pytest.
- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text.
- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version.
- Stop support for UTF-7 that does not contain a SIG.
- Dropping PrettyTable, replaced with pure JSON output in CLI.
### Fixed
- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process.
- Not searching properly for the BOM when trying utf32/16 parent codec.
### Changed
- Improving the package final size by compressing frequencies.json.
- Huge improvement over the larges payload.
### Added
- CLI now produces JSON consumable output.
- Return ASCII if given sequences fit. Given reasonable confidence.
## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13)
### Fixed
- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40)
## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12)
### Fixed
- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39)
## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12)
### Fixed
- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38)
## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09)
### Changed
- Amend the previous release to allow prettytable 2.0 (PR #35)
## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08)
### Fixed
- Fix error while using the package with a python pre-release interpreter (PR #33)
### Changed
- Dependencies refactoring, constraints revised.
### Added
- Add python 3.9 and 3.10 to the supported interpreters
MIT License
Copyright (c) 2025 TAHRI Ahmed R.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/RECORD
+35
View File
@@ -0,0 +1,35 @@
../../../bin/normalizer,sha256=ae-OfnFR0skdznlS0a0XdzIDKXxxZ4zYMp1Z1b49kBI,277
charset_normalizer-3.4.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
charset_normalizer-3.4.4.dist-info/METADATA,sha256=jVuUFBti8dav19YLvWissTihVdF2ozUY4KKMw7jdkBQ,37303
charset_normalizer-3.4.4.dist-info/RECORD,,
charset_normalizer-3.4.4.dist-info/WHEEL,sha256=DxRnWQz-Kp9-4a4hdDHsSv0KUC3H7sN9Nbef3-8RjXU,190
charset_normalizer-3.4.4.dist-info/entry_points.txt,sha256=ADSTKrkXZ3hhdOVFi6DcUEHQRS0xfxDIE_pEz4wLIXA,65
charset_normalizer-3.4.4.dist-info/licenses/LICENSE,sha256=bQ1Bv-FwrGx9wkjJpj4lTQ-0WmDVCoJX0K-SxuJJuIc,1071
charset_normalizer-3.4.4.dist-info/top_level.txt,sha256=7ASyzePr8_xuZWJsnqJjIBtyV8vhEo0wBCv1MPRRi3Q,19
charset_normalizer/__init__.py,sha256=OKRxRv2Zhnqk00tqkN0c1BtJjm165fWXLydE52IKuHc,1590
charset_normalizer/__main__.py,sha256=yzYxMR-IhKRHYwcSlavEv8oGdwxsR89mr2X09qXGdps,109
charset_normalizer/__pycache__/__init__.cpython-312.pyc,,
charset_normalizer/__pycache__/__main__.cpython-312.pyc,,
charset_normalizer/__pycache__/api.cpython-312.pyc,,
charset_normalizer/__pycache__/cd.cpython-312.pyc,,
charset_normalizer/__pycache__/constant.cpython-312.pyc,,
charset_normalizer/__pycache__/legacy.cpython-312.pyc,,
charset_normalizer/__pycache__/md.cpython-312.pyc,,
charset_normalizer/__pycache__/models.cpython-312.pyc,,
charset_normalizer/__pycache__/utils.cpython-312.pyc,,
charset_normalizer/__pycache__/version.cpython-312.pyc,,
charset_normalizer/api.py,sha256=V07i8aVeCD8T2fSia3C-fn0i9t8qQguEBhsqszg32Ns,22668
charset_normalizer/cd.py,sha256=WKTo1HDb-H9HfCDc3Bfwq5jzS25Ziy9SE2a74SgTq88,12522
charset_normalizer/cli/__init__.py,sha256=D8I86lFk2-py45JvqxniTirSj_sFyE6sjaY_0-G1shc,136
charset_normalizer/cli/__main__.py,sha256=dMaXG6IJXRvqq8z2tig7Qb83-BpWTln55ooiku5_uvg,12646
charset_normalizer/cli/__pycache__/__init__.cpython-312.pyc,,
charset_normalizer/cli/__pycache__/__main__.cpython-312.pyc,,
charset_normalizer/constant.py,sha256=7UVY4ldYhmQMHUdgQ_sgZmzcQ0xxYxpBunqSZ-XJZ8U,42713
charset_normalizer/legacy.py,sha256=sYBzSpzsRrg_wF4LP536pG64BItw7Tqtc3SMQAHvFLM,2731
charset_normalizer/md.cpython-312-x86_64-linux-gnu.so,sha256=sZ7umtJLjKfA83NFJ7npkiDyr06zDT8cWtl6uIx2MsM,15912
charset_normalizer/md.py,sha256=-_oN3h3_X99nkFfqamD3yu45DC_wfk5odH0Tr_CQiXs,20145
charset_normalizer/md__mypyc.cpython-312-x86_64-linux-gnu.so,sha256=J2WWgLBQiO8sqdFsENp9u5V9uEH0tTwvTLszPdqhsv0,290584
charset_normalizer/models.py,sha256=lKXhOnIPtiakbK3i__J9wpOfzx3JDTKj7Dn3Rg0VaRI,12394
charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
charset_normalizer/utils.py,sha256=sTejPgrdlNsKNucZfJCxJ95lMTLA0ShHLLE3n5wpT9Q,12170
charset_normalizer/version.py,sha256=nKE4qBNk5WA4LIJ_yIH_aSDfvtsyizkWMg-PUG-UZVk,115
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/WHEEL
+7
View File
@@ -0,0 +1,7 @@
Wheel-Version: 1.0
Generator: setuptools (80.9.0)
Root-Is-Purelib: false
Tag: cp312-cp312-manylinux_2_17_x86_64
Tag: cp312-cp312-manylinux2014_x86_64
Tag: cp312-cp312-manylinux_2_28_x86_64
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/entry_points.txt
+2
View File
@@ -0,0 +1,2 @@
[console_scripts]
normalizer = charset_normalizer.cli:cli_detect
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/licenses/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 TAHRI Ahmed R.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/top_level.txt
+1
View File
@@ -0,0 +1 @@
charset_normalizer
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__init__.py
+48
View File
@@ -0,0 +1,48 @@
"""
Charset-Normalizer
~~~~~~~~~~~~~~
The Real First Universal Charset Detector.
A library that helps you read text from an unknown charset encoding.
Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
All IANA character set names for which the Python core library provides codecs are supported.
Basic usage:
>>> from charset_normalizer import from_bytes
>>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
>>> best_guess = results.best()
>>> str(best_guess)
'Bсеки човек има право на образование. Oбразованието!'
Others methods and usages are available - see the full documentation
at <https://github.com/Ousret/charset_normalizer>.
:copyright: (c) 2021 by Ahmed TAHRI
:license: MIT, see LICENSE for more details.
"""
from __future__ import annotations
import logging
from .api import from_bytes, from_fp, from_path, is_binary
from .legacy import detect
from .models import CharsetMatch, CharsetMatches
from .utils import set_logging_handler
from .version import VERSION, __version__
__all__ = (
"from_fp",
"from_path",
"from_bytes",
"is_binary",
"detect",
"CharsetMatch",
"CharsetMatches",
"__version__",
"VERSION",
"set_logging_handler",
)
# Attach a NullHandler to the top level logger by default
# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__main__.py
+6
View File
@@ -0,0 +1,6 @@
from __future__ import annotations
from .cli import cli_detect
if __name__ == "__main__":
cli_detect()
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/__main__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/api.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/cd.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/constant.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/legacy.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/md.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/models.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/utils.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/version.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/api.py
+669
View File
@@ -0,0 +1,669 @@
from __future__ import annotations
import logging
from os import PathLike
from typing import BinaryIO
from .cd import (
coherence_ratio,
encoding_languages,
mb_encoding_languages,
merge_coherence_ratios,
)
from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE
from .md import mess_ratio
from .models import CharsetMatch, CharsetMatches
from .utils import (
any_specified_encoding,
cut_sequence_chunks,
iana_name,
identify_sig_or_bom,
is_cp_similar,
is_multi_byte_encoding,
should_strip_sig_or_bom,
)
logger = logging.getLogger("charset_normalizer")
explain_handler = logging.StreamHandler()
explain_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
def from_bytes(
sequences: bytes | bytearray,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.2,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Given a raw bytes sequence, return the best possibles charset usable to render str objects.
If there is no results, it is a strong indicator that the source is binary/not text.
By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
but never take it for granted. Can improve the performance.
You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
purpose.
This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
Custom logging format and handler can be set manually.
"""
if not isinstance(sequences, (bytearray, bytes)):
raise TypeError(
"Expected object of type bytes or bytearray, got: {}".format(
type(sequences)
)
)
if explain:
previous_logger_level: int = logger.level
logger.addHandler(explain_handler)
logger.setLevel(TRACE)
length: int = len(sequences)
if length == 0:
logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level or logging.WARNING)
return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
if cp_isolation is not None:
logger.log(
TRACE,
"cp_isolation is set. use this flag for debugging purpose. "
"limited list of encoding allowed : %s.",
", ".join(cp_isolation),
)
cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
else:
cp_isolation = []
if cp_exclusion is not None:
logger.log(
TRACE,
"cp_exclusion is set. use this flag for debugging purpose. "
"limited list of encoding excluded : %s.",
", ".join(cp_exclusion),
)
cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
else:
cp_exclusion = []
if length <= (chunk_size * steps):
logger.log(
TRACE,
"override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
steps,
chunk_size,
length,
)
steps = 1
chunk_size = length
if steps > 1 and length / steps < chunk_size:
chunk_size = int(length / steps)
is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
if is_too_small_sequence:
logger.log(
TRACE,
"Trying to detect encoding from a tiny portion of ({}) byte(s).".format(
length
),
)
elif is_too_large_sequence:
logger.log(
TRACE,
"Using lazy str decoding because the payload is quite large, ({}) byte(s).".format(
length
),
)
prioritized_encodings: list[str] = []
specified_encoding: str | None = (
any_specified_encoding(sequences) if preemptive_behaviour else None
)
if specified_encoding is not None:
prioritized_encodings.append(specified_encoding)
logger.log(
TRACE,
"Detected declarative mark in sequence. Priority +1 given for %s.",
specified_encoding,
)
tested: set[str] = set()
tested_but_hard_failure: list[str] = []
tested_but_soft_failure: list[str] = []
fallback_ascii: CharsetMatch | None = None
fallback_u8: CharsetMatch | None = None
fallback_specified: CharsetMatch | None = None
results: CharsetMatches = CharsetMatches()
early_stop_results: CharsetMatches = CharsetMatches()
sig_encoding, sig_payload = identify_sig_or_bom(sequences)
if sig_encoding is not None:
prioritized_encodings.append(sig_encoding)
logger.log(
TRACE,
"Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
len(sig_payload),
sig_encoding,
)
prioritized_encodings.append("ascii")
if "utf_8" not in prioritized_encodings:
prioritized_encodings.append("utf_8")
for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
if cp_isolation and encoding_iana not in cp_isolation:
continue
if cp_exclusion and encoding_iana in cp_exclusion:
continue
if encoding_iana in tested:
continue
tested.add(encoding_iana)
decoded_payload: str | None = None
bom_or_sig_available: bool = sig_encoding == encoding_iana
strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
encoding_iana
)
if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
encoding_iana,
)
continue
if encoding_iana in {"utf_7"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.",
encoding_iana,
)
continue
try:
is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
except (ModuleNotFoundError, ImportError):
logger.log(
TRACE,
"Encoding %s does not provide an IncrementalDecoder",
encoding_iana,
)
continue
try:
if is_too_large_sequence and is_multi_byte_decoder is False:
str(
(
sequences[: int(50e4)]
if strip_sig_or_bom is False
else sequences[len(sig_payload) : int(50e4)]
),
encoding=encoding_iana,
)
else:
decoded_payload = str(
(
sequences
if strip_sig_or_bom is False
else sequences[len(sig_payload) :]
),
encoding=encoding_iana,
)
except (UnicodeDecodeError, LookupError) as e:
if not isinstance(e, LookupError):
logger.log(
TRACE,
"Code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
similar_soft_failure_test: bool = False
for encoding_soft_failed in tested_but_soft_failure:
if is_cp_similar(encoding_iana, encoding_soft_failed):
similar_soft_failure_test = True
break
if similar_soft_failure_test:
logger.log(
TRACE,
"%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
encoding_iana,
encoding_soft_failed,
)
continue
r_ = range(
0 if not bom_or_sig_available else len(sig_payload),
length,
int(length / steps),
)
multi_byte_bonus: bool = (
is_multi_byte_decoder
and decoded_payload is not None
and len(decoded_payload) < length
)
if multi_byte_bonus:
logger.log(
TRACE,
"Code page %s is a multi byte encoding table and it appear that at least one character "
"was encoded using n-bytes.",
encoding_iana,
)
max_chunk_gave_up: int = int(len(r_) / 4)
max_chunk_gave_up = max(max_chunk_gave_up, 2)
early_stop_count: int = 0
lazy_str_hard_failure = False
md_chunks: list[str] = []
md_ratios = []
try:
for chunk in cut_sequence_chunks(
sequences,
encoding_iana,
r_,
chunk_size,
bom_or_sig_available,
strip_sig_or_bom,
sig_payload,
is_multi_byte_decoder,
decoded_payload,
):
md_chunks.append(chunk)
md_ratios.append(
mess_ratio(
chunk,
threshold,
explain is True and 1 <= len(cp_isolation) <= 2,
)
)
if md_ratios[-1] >= threshold:
early_stop_count += 1
if (early_stop_count >= max_chunk_gave_up) or (
bom_or_sig_available and strip_sig_or_bom is False
):
break
except (
UnicodeDecodeError
) as e: # Lazy str loading may have missed something there
logger.log(
TRACE,
"LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
early_stop_count = max_chunk_gave_up
lazy_str_hard_failure = True
# We might want to check the sequence again with the whole content
# Only if initial MD tests passes
if (
not lazy_str_hard_failure
and is_too_large_sequence
and not is_multi_byte_decoder
):
try:
sequences[int(50e3) :].decode(encoding_iana, errors="strict")
except UnicodeDecodeError as e:
logger.log(
TRACE,
"LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
tested_but_soft_failure.append(encoding_iana)
logger.log(
TRACE,
"%s was excluded because of initial chaos probing. Gave up %i time(s). "
"Computed mean chaos is %f %%.",
encoding_iana,
early_stop_count,
round(mean_mess_ratio * 100, ndigits=3),
)
# Preparing those fallbacks in case we got nothing.
if (
enable_fallback
and encoding_iana
in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"]
and not lazy_str_hard_failure
):
fallback_entry = CharsetMatch(
sequences,
encoding_iana,
threshold,
bom_or_sig_available,
[],
decoded_payload,
preemptive_declaration=specified_encoding,
)
if encoding_iana == specified_encoding:
fallback_specified = fallback_entry
elif encoding_iana == "ascii":
fallback_ascii = fallback_entry
else:
fallback_u8 = fallback_entry
continue
logger.log(
TRACE,
"%s passed initial chaos probing. Mean measured chaos is %f %%",
encoding_iana,
round(mean_mess_ratio * 100, ndigits=3),
)
if not is_multi_byte_decoder:
target_languages: list[str] = encoding_languages(encoding_iana)
else:
target_languages = mb_encoding_languages(encoding_iana)
if target_languages:
logger.log(
TRACE,
"{} should target any language(s) of {}".format(
encoding_iana, str(target_languages)
),
)
cd_ratios = []
# We shall skip the CD when its about ASCII
# Most of the time its not relevant to run "language-detection" on it.
if encoding_iana != "ascii":
for chunk in md_chunks:
chunk_languages = coherence_ratio(
chunk,
language_threshold,
",".join(target_languages) if target_languages else None,
)
cd_ratios.append(chunk_languages)
cd_ratios_merged = merge_coherence_ratios(cd_ratios)
if cd_ratios_merged:
logger.log(
TRACE,
"We detected language {} using {}".format(
cd_ratios_merged, encoding_iana
),
)
current_match = CharsetMatch(
sequences,
encoding_iana,
mean_mess_ratio,
bom_or_sig_available,
cd_ratios_merged,
(
decoded_payload
if (
is_too_large_sequence is False
or encoding_iana in [specified_encoding, "ascii", "utf_8"]
)
else None
),
preemptive_declaration=specified_encoding,
)
results.append(current_match)
if (
encoding_iana in [specified_encoding, "ascii", "utf_8"]
and mean_mess_ratio < 0.1
):
# If md says nothing to worry about, then... stop immediately!
if mean_mess_ratio == 0.0:
logger.debug(
"Encoding detection: %s is most likely the one.",
current_match.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([current_match])
early_stop_results.append(current_match)
if (
len(early_stop_results)
and (specified_encoding is None or specified_encoding in tested)
and "ascii" in tested
and "utf_8" in tested
):
probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment]
logger.debug(
"Encoding detection: %s is most likely the one.",
probable_result.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([probable_result])
if encoding_iana == sig_encoding:
logger.debug(
"Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
"the beginning of the sequence.",
encoding_iana,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([results[encoding_iana]])
if len(results) == 0:
if fallback_u8 or fallback_ascii or fallback_specified:
logger.log(
TRACE,
"Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
)
if fallback_specified:
logger.debug(
"Encoding detection: %s will be used as a fallback match",
fallback_specified.encoding,
)
results.append(fallback_specified)
elif (
(fallback_u8 and fallback_ascii is None)
or (
fallback_u8
and fallback_ascii
and fallback_u8.fingerprint != fallback_ascii.fingerprint
)
or (fallback_u8 is not None)
):
logger.debug("Encoding detection: utf_8 will be used as a fallback match")
results.append(fallback_u8)
elif fallback_ascii:
logger.debug("Encoding detection: ascii will be used as a fallback match")
results.append(fallback_ascii)
if results:
logger.debug(
"Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
results.best().encoding, # type: ignore
len(results) - 1,
)
else:
logger.debug("Encoding detection: Unable to determine any suitable charset.")
if explain:
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return results
def from_fp(
fp: BinaryIO,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but using a file pointer that is already ready.
Will not close the file pointer.
"""
return from_bytes(
fp.read(),
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def from_path(
path: str | bytes | PathLike, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
Can raise IOError.
"""
with open(path, "rb") as fp:
return from_fp(
fp,
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def is_binary(
fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = False,
) -> bool:
"""
Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
are disabled to be stricter around ASCII-compatible but unlikely to be a string.
"""
if isinstance(fp_or_path_or_payload, (str, PathLike)):
guesses = from_path(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
elif isinstance(
fp_or_path_or_payload,
(
bytes,
bytearray,
),
):
guesses = from_bytes(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
else:
guesses = from_fp(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
return not guesses
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/cd.py
+395
View File
@@ -0,0 +1,395 @@
from __future__ import annotations
import importlib
from codecs import IncrementalDecoder
from collections import Counter
from functools import lru_cache
from typing import Counter as TypeCounter
from .constant import (
FREQUENCIES,
KO_NAMES,
LANGUAGE_SUPPORTED_COUNT,
TOO_SMALL_SEQUENCE,
ZH_NAMES,
)
from .md import is_suspiciously_successive_range
from .models import CoherenceMatches
from .utils import (
is_accentuated,
is_latin,
is_multi_byte_encoding,
is_unicode_range_secondary,
unicode_range,
)
def encoding_unicode_range(iana_name: str) -> list[str]:
"""
Return associated unicode ranges in a single byte code page.
"""
if is_multi_byte_encoding(iana_name):
raise OSError("Function not supported on multi-byte code page")
decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
p: IncrementalDecoder = decoder(errors="ignore")
seen_ranges: dict[str, int] = {}
character_count: int = 0
for i in range(0x40, 0xFF):
chunk: str = p.decode(bytes([i]))
if chunk:
character_range: str | None = unicode_range(chunk)
if character_range is None:
continue
if is_unicode_range_secondary(character_range) is False:
if character_range not in seen_ranges:
seen_ranges[character_range] = 0
seen_ranges[character_range] += 1
character_count += 1
return sorted(
[
character_range
for character_range in seen_ranges
if seen_ranges[character_range] / character_count >= 0.15
]
)
def unicode_range_languages(primary_range: str) -> list[str]:
"""
Return inferred languages used with a unicode range.
"""
languages: list[str] = []
for language, characters in FREQUENCIES.items():
for character in characters:
if unicode_range(character) == primary_range:
languages.append(language)
break
return languages
@lru_cache()
def encoding_languages(iana_name: str) -> list[str]:
"""
Single-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
unicode_ranges: list[str] = encoding_unicode_range(iana_name)
primary_range: str | None = None
for specified_range in unicode_ranges:
if "Latin" not in specified_range:
primary_range = specified_range
break
if primary_range is None:
return ["Latin Based"]
return unicode_range_languages(primary_range)
@lru_cache()
def mb_encoding_languages(iana_name: str) -> list[str]:
"""
Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
if (
iana_name.startswith("shift_")
or iana_name.startswith("iso2022_jp")
or iana_name.startswith("euc_j")
or iana_name == "cp932"
):
return ["Japanese"]
if iana_name.startswith("gb") or iana_name in ZH_NAMES:
return ["Chinese"]
if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
return ["Korean"]
return []
@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
def get_target_features(language: str) -> tuple[bool, bool]:
"""
Determine main aspects from a supported language if it contains accents and if is pure Latin.
"""
target_have_accents: bool = False
target_pure_latin: bool = True
for character in FREQUENCIES[language]:
if not target_have_accents and is_accentuated(character):
target_have_accents = True
if target_pure_latin and is_latin(character) is False:
target_pure_latin = False
return target_have_accents, target_pure_latin
def alphabet_languages(
characters: list[str], ignore_non_latin: bool = False
) -> list[str]:
"""
Return associated languages associated to given characters.
"""
languages: list[tuple[str, float]] = []
source_have_accents = any(is_accentuated(character) for character in characters)
for language, language_characters in FREQUENCIES.items():
target_have_accents, target_pure_latin = get_target_features(language)
if ignore_non_latin and target_pure_latin is False:
continue
if target_have_accents is False and source_have_accents:
continue
character_count: int = len(language_characters)
character_match_count: int = len(
[c for c in language_characters if c in characters]
)
ratio: float = character_match_count / character_count
if ratio >= 0.2:
languages.append((language, ratio))
languages = sorted(languages, key=lambda x: x[1], reverse=True)
return [compatible_language[0] for compatible_language in languages]
def characters_popularity_compare(
language: str, ordered_characters: list[str]
) -> float:
"""
Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
"""
if language not in FREQUENCIES:
raise ValueError(f"{language} not available")
character_approved_count: int = 0
FREQUENCIES_language_set = set(FREQUENCIES[language])
ordered_characters_count: int = len(ordered_characters)
target_language_characters_count: int = len(FREQUENCIES[language])
large_alphabet: bool = target_language_characters_count > 26
for character, character_rank in zip(
ordered_characters, range(0, ordered_characters_count)
):
if character not in FREQUENCIES_language_set:
continue
character_rank_in_language: int = FREQUENCIES[language].index(character)
expected_projection_ratio: float = (
target_language_characters_count / ordered_characters_count
)
character_rank_projection: int = int(character_rank * expected_projection_ratio)
if (
large_alphabet is False
and abs(character_rank_projection - character_rank_in_language) > 4
):
continue
if (
large_alphabet is True
and abs(character_rank_projection - character_rank_in_language)
< target_language_characters_count / 3
):
character_approved_count += 1
continue
characters_before_source: list[str] = FREQUENCIES[language][
0:character_rank_in_language
]
characters_after_source: list[str] = FREQUENCIES[language][
character_rank_in_language:
]
characters_before: list[str] = ordered_characters[0:character_rank]
characters_after: list[str] = ordered_characters[character_rank:]
before_match_count: int = len(
set(characters_before) & set(characters_before_source)
)
after_match_count: int = len(
set(characters_after) & set(characters_after_source)
)
if len(characters_before_source) == 0 and before_match_count <= 4:
character_approved_count += 1
continue
if len(characters_after_source) == 0 and after_match_count <= 4:
character_approved_count += 1
continue
if (
before_match_count / len(characters_before_source) >= 0.4
or after_match_count / len(characters_after_source) >= 0.4
):
character_approved_count += 1
continue
return character_approved_count / len(ordered_characters)
def alpha_unicode_split(decoded_sequence: str) -> list[str]:
"""
Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
One containing the latin letters and the other hebrew.
"""
layers: dict[str, str] = {}
for character in decoded_sequence:
if character.isalpha() is False:
continue
character_range: str | None = unicode_range(character)
if character_range is None:
continue
layer_target_range: str | None = None
for discovered_range in layers:
if (
is_suspiciously_successive_range(discovered_range, character_range)
is False
):
layer_target_range = discovered_range
break
if layer_target_range is None:
layer_target_range = character_range
if layer_target_range not in layers:
layers[layer_target_range] = character.lower()
continue
layers[layer_target_range] += character.lower()
return list(layers.values())
def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
"""
This function merge results previously given by the function coherence_ratio.
The return type is the same as coherence_ratio.
"""
per_language_ratios: dict[str, list[float]] = {}
for result in results:
for sub_result in result:
language, ratio = sub_result
if language not in per_language_ratios:
per_language_ratios[language] = [ratio]
continue
per_language_ratios[language].append(ratio)
merge = [
(
language,
round(
sum(per_language_ratios[language]) / len(per_language_ratios[language]),
4,
),
)
for language in per_language_ratios
]
return sorted(merge, key=lambda x: x[1], reverse=True)
def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
"""
We shall NOT return "English—" in CoherenceMatches because it is an alternative
of "English". This function only keeps the best match and remove the em-dash in it.
"""
index_results: dict[str, list[float]] = dict()
for result in results:
language, ratio = result
no_em_name: str = language.replace("—", "")
if no_em_name not in index_results:
index_results[no_em_name] = []
index_results[no_em_name].append(ratio)
if any(len(index_results[e]) > 1 for e in index_results):
filtered_results: CoherenceMatches = []
for language in index_results:
filtered_results.append((language, max(index_results[language])))
return filtered_results
return results
@lru_cache(maxsize=2048)
def coherence_ratio(
decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
) -> CoherenceMatches:
"""
Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
A layer = Character extraction by alphabets/ranges.
"""
results: list[tuple[str, float]] = []
ignore_non_latin: bool = False
sufficient_match_count: int = 0
lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
if "Latin Based" in lg_inclusion_list:
ignore_non_latin = True
lg_inclusion_list.remove("Latin Based")
for layer in alpha_unicode_split(decoded_sequence):
sequence_frequencies: TypeCounter[str] = Counter(layer)
most_common = sequence_frequencies.most_common()
character_count: int = sum(o for c, o in most_common)
if character_count <= TOO_SMALL_SEQUENCE:
continue
popular_character_ordered: list[str] = [c for c, o in most_common]
for language in lg_inclusion_list or alphabet_languages(
popular_character_ordered, ignore_non_latin
):
ratio: float = characters_popularity_compare(
language, popular_character_ordered
)
if ratio < threshold:
continue
elif ratio >= 0.8:
sufficient_match_count += 1
results.append((language, round(ratio, 4)))
if sufficient_match_count >= 3:
break
return sorted(
filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
)
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/cli/__init__.py
+8
View File
@@ -0,0 +1,8 @@
from __future__ import annotations
from .__main__ import cli_detect, query_yes_no
__all__ = (
"cli_detect",
"query_yes_no",
)
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/cli/__main__.py
+381
View File
@@ -0,0 +1,381 @@
from __future__ import annotations
import argparse
import sys
import typing
from json import dumps
from os.path import abspath, basename, dirname, join, realpath
from platform import python_version
from unicodedata import unidata_version
import charset_normalizer.md as md_module
from charset_normalizer import from_fp
from charset_normalizer.models import CliDetectionResult
from charset_normalizer.version import __version__
def query_yes_no(question: str, default: str = "yes") -> bool:
"""Ask a yes/no question via input() and return their answer.
"question" is a string that is presented to the user.
"default" is the presumed answer if the user just hits <Enter>.
It must be "yes" (the default), "no" or None (meaning
an answer is required of the user).
The "answer" return value is True for "yes" or False for "no".
Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input
"""
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
if default is None:
prompt = " [y/n] "
elif default == "yes":
prompt = " [Y/n] "
elif default == "no":
prompt = " [y/N] "
else:
raise ValueError("invalid default answer: '%s'" % default)
while True:
sys.stdout.write(question + prompt)
choice = input().lower()
if default is not None and choice == "":
return valid[default]
elif choice in valid:
return valid[choice]
else:
sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")
class FileType:
"""Factory for creating file object types
Instances of FileType are typically passed as type= arguments to the
ArgumentParser add_argument() method.
Keyword Arguments:
- mode -- A string indicating how the file is to be opened. Accepts the
same values as the builtin open() function.
- bufsize -- The file's desired buffer size. Accepts the same values as
the builtin open() function.
- encoding -- The file's encoding. Accepts the same values as the
builtin open() function.
- errors -- A string indicating how encoding and decoding errors are to
be handled. Accepts the same value as the builtin open() function.
Backported from CPython 3.12
"""
def __init__(
self,
mode: str = "r",
bufsize: int = -1,
encoding: str | None = None,
errors: str | None = None,
):
self._mode = mode
self._bufsize = bufsize
self._encoding = encoding
self._errors = errors
def __call__(self, string: str) -> typing.IO: # type: ignore[type-arg]
# the special argument "-" means sys.std{in,out}
if string == "-":
if "r" in self._mode:
return sys.stdin.buffer if "b" in self._mode else sys.stdin
elif any(c in self._mode for c in "wax"):
return sys.stdout.buffer if "b" in self._mode else sys.stdout
else:
msg = f'argument "-" with mode {self._mode}'
raise ValueError(msg)
# all other arguments are used as file names
try:
return open(string, self._mode, self._bufsize, self._encoding, self._errors)
except OSError as e:
message = f"can't open '{string}': {e}"
raise argparse.ArgumentTypeError(message)
def __repr__(self) -> str:
args = self._mode, self._bufsize
kwargs = [("encoding", self._encoding), ("errors", self._errors)]
args_str = ", ".join(
[repr(arg) for arg in args if arg != -1]
+ [f"{kw}={arg!r}" for kw, arg in kwargs if arg is not None]
)
return f"{type(self).__name__}({args_str})"
def cli_detect(argv: list[str] | None = None) -> int:
"""
CLI assistant using ARGV and ArgumentParser
:param argv:
:return: 0 if everything is fine, anything else equal trouble
"""
parser = argparse.ArgumentParser(
description="The Real First Universal Charset Detector. "
"Discover originating encoding used on text file. "
"Normalize text to unicode."
)
parser.add_argument(
"files", type=FileType("rb"), nargs="+", help="File(s) to be analysed"
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
default=False,
dest="verbose",
help="Display complementary information about file if any. "
"Stdout will contain logs about the detection process.",
)
parser.add_argument(
"-a",
"--with-alternative",
action="store_true",
default=False,
dest="alternatives",
help="Output complementary possibilities if any. Top-level JSON WILL be a list.",
)
parser.add_argument(
"-n",
"--normalize",
action="store_true",
default=False,
dest="normalize",
help="Permit to normalize input file. If not set, program does not write anything.",
)
parser.add_argument(
"-m",
"--minimal",
action="store_true",
default=False,
dest="minimal",
help="Only output the charset detected to STDOUT. Disabling JSON output.",
)
parser.add_argument(
"-r",
"--replace",
action="store_true",
default=False,
dest="replace",
help="Replace file when trying to normalize it instead of creating a new one.",
)
parser.add_argument(
"-f",
"--force",
action="store_true",
default=False,
dest="force",
help="Replace file without asking if you are sure, use this flag with caution.",
)
parser.add_argument(
"-i",
"--no-preemptive",
action="store_true",
default=False,
dest="no_preemptive",
help="Disable looking at a charset declaration to hint the detector.",
)
parser.add_argument(
"-t",
"--threshold",
action="store",
default=0.2,
type=float,
dest="threshold",
help="Define a custom maximum amount of noise allowed in decoded content. 0. <= noise <= 1.",
)
parser.add_argument(
"--version",
action="version",
version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format(
__version__,
python_version(),
unidata_version,
"OFF" if md_module.__file__.lower().endswith(".py") else "ON",
),
help="Show version information and exit.",
)
args = parser.parse_args(argv)
if args.replace is True and args.normalize is False:
if args.files:
for my_file in args.files:
my_file.close()
print("Use --replace in addition of --normalize only.", file=sys.stderr)
return 1
if args.force is True and args.replace is False:
if args.files:
for my_file in args.files:
my_file.close()
print("Use --force in addition of --replace only.", file=sys.stderr)
return 1
if args.threshold < 0.0 or args.threshold > 1.0:
if args.files:
for my_file in args.files:
my_file.close()
print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr)
return 1
x_ = []
for my_file in args.files:
matches = from_fp(
my_file,
threshold=args.threshold,
explain=args.verbose,
preemptive_behaviour=args.no_preemptive is False,
)
best_guess = matches.best()
if best_guess is None:
print(
'Unable to identify originating encoding for "{}". {}'.format(
my_file.name,
(
"Maybe try increasing maximum amount of chaos."
if args.threshold < 1.0
else ""
),
),
file=sys.stderr,
)
x_.append(
CliDetectionResult(
abspath(my_file.name),
None,
[],
[],
"Unknown",
[],
False,
1.0,
0.0,
None,
True,
)
)
else:
x_.append(
CliDetectionResult(
abspath(my_file.name),
best_guess.encoding,
best_guess.encoding_aliases,
[
cp
for cp in best_guess.could_be_from_charset
if cp != best_guess.encoding
],
best_guess.language,
best_guess.alphabets,
best_guess.bom,
best_guess.percent_chaos,
best_guess.percent_coherence,
None,
True,
)
)
if len(matches) > 1 and args.alternatives:
for el in matches:
if el != best_guess:
x_.append(
CliDetectionResult(
abspath(my_file.name),
el.encoding,
el.encoding_aliases,
[
cp
for cp in el.could_be_from_charset
if cp != el.encoding
],
el.language,
el.alphabets,
el.bom,
el.percent_chaos,
el.percent_coherence,
None,
False,
)
)
if args.normalize is True:
if best_guess.encoding.startswith("utf") is True:
print(
'"{}" file does not need to be normalized, as it already came from unicode.'.format(
my_file.name
),
file=sys.stderr,
)
if my_file.closed is False:
my_file.close()
continue
dir_path = dirname(realpath(my_file.name))
file_name = basename(realpath(my_file.name))
o_: list[str] = file_name.split(".")
if args.replace is False:
o_.insert(-1, best_guess.encoding)
if my_file.closed is False:
my_file.close()
elif (
args.force is False
and query_yes_no(
'Are you sure to normalize "{}" by replacing it ?'.format(
my_file.name
),
"no",
)
is False
):
if my_file.closed is False:
my_file.close()
continue
try:
x_[0].unicode_path = join(dir_path, ".".join(o_))
with open(x_[0].unicode_path, "wb") as fp:
fp.write(best_guess.output())
except OSError as e:
print(str(e), file=sys.stderr)
if my_file.closed is False:
my_file.close()
return 2
if my_file.closed is False:
my_file.close()
if args.minimal is False:
print(
dumps(
[el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__,
ensure_ascii=True,
indent=4,
)
)
else:
for my_file in args.files:
print(
", ".join(
[
el.encoding or "undefined"
for el in x_
if el.path == abspath(my_file.name)
]
)
)
return 0
if __name__ == "__main__":
cli_detect()
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/cli/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/cli/__pycache__/__main__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/constant.py
+2015
View File
File diff suppressed because it is too large Load Diff
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/legacy.py
+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from warnings import warn
from .api import from_bytes
from .constant import CHARDET_CORRESPONDENCE, TOO_SMALL_SEQUENCE
# TODO: remove this check when dropping Python 3.7 support
if TYPE_CHECKING:
from typing_extensions import TypedDict
class ResultDict(TypedDict):
encoding: str | None
language: str
confidence: float | None
def detect(
byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any
) -> ResultDict:
"""
chardet legacy method
Detect the encoding of the given byte string. It should be mostly backward-compatible.
Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it)
This function is deprecated and should be used to migrate your project easily, consult the documentation for
further information. Not planned for removal.
:param byte_str: The byte sequence to examine.
:param should_rename_legacy: Should we rename legacy encodings
to their more modern equivalents?
"""
if len(kwargs):
warn(
f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()"
)
if not isinstance(byte_str, (bytearray, bytes)):
raise TypeError( # pragma: nocover
f"Expected object of type bytes or bytearray, got: {type(byte_str)}"
)
if isinstance(byte_str, bytearray):
byte_str = bytes(byte_str)
r = from_bytes(byte_str).best()
encoding = r.encoding if r is not None else None
language = r.language if r is not None and r.language != "Unknown" else ""
confidence = 1.0 - r.chaos if r is not None else None
# automatically lower confidence
# on small bytes samples.
# https://github.com/jawah/charset_normalizer/issues/391
if (
confidence is not None
and confidence >= 0.9
and encoding
not in {
"utf_8",
"ascii",
}
and r.bom is False # type: ignore[union-attr]
and len(byte_str) < TOO_SMALL_SEQUENCE
):
confidence -= 0.2
# Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process
# but chardet does return 'utf-8-sig' and it is a valid codec name.
if r is not None and encoding == "utf_8" and r.bom:
encoding += "_sig"
if should_rename_legacy is False and encoding in CHARDET_CORRESPONDENCE:
encoding = CHARDET_CORRESPONDENCE[encoding]
return {
"encoding": encoding,
"language": language,
"confidence": confidence,
}
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/md.cpython-312-x86_64-linux-gnu.so
Executable
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/md.py
+635
View File
@@ -0,0 +1,635 @@
from __future__ import annotations
from functools import lru_cache
from logging import getLogger
from .constant import (
COMMON_SAFE_ASCII_CHARACTERS,
TRACE,
UNICODE_SECONDARY_RANGE_KEYWORD,
)
from .utils import (
is_accentuated,
is_arabic,
is_arabic_isolated_form,
is_case_variable,
is_cjk,
is_emoticon,
is_hangul,
is_hiragana,
is_katakana,
is_latin,
is_punctuation,
is_separator,
is_symbol,
is_thai,
is_unprintable,
remove_accent,
unicode_range,
is_cjk_uncommon,
)
class MessDetectorPlugin:
"""
Base abstract class used for mess detection plugins.
All detectors MUST extend and implement given methods.
"""
def eligible(self, character: str) -> bool:
"""
Determine if given character should be fed in.
"""
raise NotImplementedError # pragma: nocover
def feed(self, character: str) -> None:
"""
The main routine to be executed upon character.
Insert the logic in witch the text would be considered chaotic.
"""
raise NotImplementedError # pragma: nocover
def reset(self) -> None: # pragma: no cover
"""
Permit to reset the plugin to the initial state.
"""
raise NotImplementedError
@property
def ratio(self) -> float:
"""
Compute the chaos ratio based on what your feed() has seen.
Must NOT be lower than 0.; No restriction gt 0.
"""
raise NotImplementedError # pragma: nocover
class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._punctuation_count: int = 0
self._symbol_count: int = 0
self._character_count: int = 0
self._last_printable_char: str | None = None
self._frenzy_symbol_in_word: bool = False
def eligible(self, character: str) -> bool:
return character.isprintable()
def feed(self, character: str) -> None:
self._character_count += 1
if (
character != self._last_printable_char
and character not in COMMON_SAFE_ASCII_CHARACTERS
):
if is_punctuation(character):
self._punctuation_count += 1
elif (
character.isdigit() is False
and is_symbol(character)
and is_emoticon(character) is False
):
self._symbol_count += 2
self._last_printable_char = character
def reset(self) -> None: # Abstract
self._punctuation_count = 0
self._character_count = 0
self._symbol_count = 0
@property
def ratio(self) -> float:
if self._character_count == 0:
return 0.0
ratio_of_punctuation: float = (
self._punctuation_count + self._symbol_count
) / self._character_count
return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0
class TooManyAccentuatedPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._character_count: int = 0
self._accentuated_count: int = 0
def eligible(self, character: str) -> bool:
return character.isalpha()
def feed(self, character: str) -> None:
self._character_count += 1
if is_accentuated(character):
self._accentuated_count += 1
def reset(self) -> None: # Abstract
self._character_count = 0
self._accentuated_count = 0
@property
def ratio(self) -> float:
if self._character_count < 8:
return 0.0
ratio_of_accentuation: float = self._accentuated_count / self._character_count
return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0
class UnprintablePlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._unprintable_count: int = 0
self._character_count: int = 0
def eligible(self, character: str) -> bool:
return True
def feed(self, character: str) -> None:
if is_unprintable(character):
self._unprintable_count += 1
self._character_count += 1
def reset(self) -> None: # Abstract
self._unprintable_count = 0
@property
def ratio(self) -> float:
if self._character_count == 0:
return 0.0
return (self._unprintable_count * 8) / self._character_count
class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._successive_count: int = 0
self._character_count: int = 0
self._last_latin_character: str | None = None
def eligible(self, character: str) -> bool:
return character.isalpha() and is_latin(character)
def feed(self, character: str) -> None:
self._character_count += 1
if (
self._last_latin_character is not None
and is_accentuated(character)
and is_accentuated(self._last_latin_character)
):
if character.isupper() and self._last_latin_character.isupper():
self._successive_count += 1
# Worse if its the same char duplicated with different accent.
if remove_accent(character) == remove_accent(self._last_latin_character):
self._successive_count += 1
self._last_latin_character = character
def reset(self) -> None: # Abstract
self._successive_count = 0
self._character_count = 0
self._last_latin_character = None
@property
def ratio(self) -> float:
if self._character_count == 0:
return 0.0
return (self._successive_count * 2) / self._character_count
class SuspiciousRange(MessDetectorPlugin):
def __init__(self) -> None:
self._suspicious_successive_range_count: int = 0
self._character_count: int = 0
self._last_printable_seen: str | None = None
def eligible(self, character: str) -> bool:
return character.isprintable()
def feed(self, character: str) -> None:
self._character_count += 1
if (
character.isspace()
or is_punctuation(character)
or character in COMMON_SAFE_ASCII_CHARACTERS
):
self._last_printable_seen = None
return
if self._last_printable_seen is None:
self._last_printable_seen = character
return
unicode_range_a: str | None = unicode_range(self._last_printable_seen)
unicode_range_b: str | None = unicode_range(character)
if is_suspiciously_successive_range(unicode_range_a, unicode_range_b):
self._suspicious_successive_range_count += 1
self._last_printable_seen = character
def reset(self) -> None: # Abstract
self._character_count = 0
self._suspicious_successive_range_count = 0
self._last_printable_seen = None
@property
def ratio(self) -> float:
if self._character_count <= 13:
return 0.0
ratio_of_suspicious_range_usage: float = (
self._suspicious_successive_range_count * 2
) / self._character_count
return ratio_of_suspicious_range_usage
class SuperWeirdWordPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._word_count: int = 0
self._bad_word_count: int = 0
self._foreign_long_count: int = 0
self._is_current_word_bad: bool = False
self._foreign_long_watch: bool = False
self._character_count: int = 0
self._bad_character_count: int = 0
self._buffer: str = ""
self._buffer_accent_count: int = 0
self._buffer_glyph_count: int = 0
def eligible(self, character: str) -> bool:
return True
def feed(self, character: str) -> None:
if character.isalpha():
self._buffer += character
if is_accentuated(character):
self._buffer_accent_count += 1
if (
self._foreign_long_watch is False
and (is_latin(character) is False or is_accentuated(character))
and is_cjk(character) is False
and is_hangul(character) is False
and is_katakana(character) is False
and is_hiragana(character) is False
and is_thai(character) is False
):
self._foreign_long_watch = True
if (
is_cjk(character)
or is_hangul(character)
or is_katakana(character)
or is_hiragana(character)
or is_thai(character)
):
self._buffer_glyph_count += 1
return
if not self._buffer:
return
if (
character.isspace() or is_punctuation(character) or is_separator(character)
) and self._buffer:
self._word_count += 1
buffer_length: int = len(self._buffer)
self._character_count += buffer_length
if buffer_length >= 4:
if self._buffer_accent_count / buffer_length >= 0.5:
self._is_current_word_bad = True
# Word/Buffer ending with an upper case accentuated letter are so rare,
# that we will consider them all as suspicious. Same weight as foreign_long suspicious.
elif (
is_accentuated(self._buffer[-1])
and self._buffer[-1].isupper()
and all(_.isupper() for _ in self._buffer) is False
):
self._foreign_long_count += 1
self._is_current_word_bad = True
elif self._buffer_glyph_count == 1:
self._is_current_word_bad = True
self._foreign_long_count += 1
if buffer_length >= 24 and self._foreign_long_watch:
camel_case_dst = [
i
for c, i in zip(self._buffer, range(0, buffer_length))
if c.isupper()
]
probable_camel_cased: bool = False
if camel_case_dst and (len(camel_case_dst) / buffer_length <= 0.3):
probable_camel_cased = True
if not probable_camel_cased:
self._foreign_long_count += 1
self._is_current_word_bad = True
if self._is_current_word_bad:
self._bad_word_count += 1
self._bad_character_count += len(self._buffer)
self._is_current_word_bad = False
self._foreign_long_watch = False
self._buffer = ""
self._buffer_accent_count = 0
self._buffer_glyph_count = 0
elif (
character not in {"<", ">", "-", "=", "~", "|", "_"}
and character.isdigit() is False
and is_symbol(character)
):
self._is_current_word_bad = True
self._buffer += character
def reset(self) -> None: # Abstract
self._buffer = ""
self._is_current_word_bad = False
self._foreign_long_watch = False
self._bad_word_count = 0
self._word_count = 0
self._character_count = 0
self._bad_character_count = 0
self._foreign_long_count = 0
@property
def ratio(self) -> float:
if self._word_count <= 10 and self._foreign_long_count == 0:
return 0.0
return self._bad_character_count / self._character_count
class CjkUncommonPlugin(MessDetectorPlugin):
"""
Detect messy CJK text that probably means nothing.
"""
def __init__(self) -> None:
self._character_count: int = 0
self._uncommon_count: int = 0
def eligible(self, character: str) -> bool:
return is_cjk(character)
def feed(self, character: str) -> None:
self._character_count += 1
if is_cjk_uncommon(character):
self._uncommon_count += 1
return
def reset(self) -> None: # Abstract
self._character_count = 0
self._uncommon_count = 0
@property
def ratio(self) -> float:
if self._character_count < 8:
return 0.0
uncommon_form_usage: float = self._uncommon_count / self._character_count
# we can be pretty sure it's garbage when uncommon characters are widely
# used. otherwise it could just be traditional chinese for example.
return uncommon_form_usage / 10 if uncommon_form_usage > 0.5 else 0.0
class ArchaicUpperLowerPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._buf: bool = False
self._character_count_since_last_sep: int = 0
self._successive_upper_lower_count: int = 0
self._successive_upper_lower_count_final: int = 0
self._character_count: int = 0
self._last_alpha_seen: str | None = None
self._current_ascii_only: bool = True
def eligible(self, character: str) -> bool:
return True
def feed(self, character: str) -> None:
is_concerned = character.isalpha() and is_case_variable(character)
chunk_sep = is_concerned is False
if chunk_sep and self._character_count_since_last_sep > 0:
if (
self._character_count_since_last_sep <= 64
and character.isdigit() is False
and self._current_ascii_only is False
):
self._successive_upper_lower_count_final += (
self._successive_upper_lower_count
)
self._successive_upper_lower_count = 0
self._character_count_since_last_sep = 0
self._last_alpha_seen = None
self._buf = False
self._character_count += 1
self._current_ascii_only = True
return
if self._current_ascii_only is True and character.isascii() is False:
self._current_ascii_only = False
if self._last_alpha_seen is not None:
if (character.isupper() and self._last_alpha_seen.islower()) or (
character.islower() and self._last_alpha_seen.isupper()
):
if self._buf is True:
self._successive_upper_lower_count += 2
self._buf = False
else:
self._buf = True
else:
self._buf = False
self._character_count += 1
self._character_count_since_last_sep += 1
self._last_alpha_seen = character
def reset(self) -> None: # Abstract
self._character_count = 0
self._character_count_since_last_sep = 0
self._successive_upper_lower_count = 0
self._successive_upper_lower_count_final = 0
self._last_alpha_seen = None
self._buf = False
self._current_ascii_only = True
@property
def ratio(self) -> float:
if self._character_count == 0:
return 0.0
return self._successive_upper_lower_count_final / self._character_count
class ArabicIsolatedFormPlugin(MessDetectorPlugin):
def __init__(self) -> None:
self._character_count: int = 0
self._isolated_form_count: int = 0
def reset(self) -> None: # Abstract
self._character_count = 0
self._isolated_form_count = 0
def eligible(self, character: str) -> bool:
return is_arabic(character)
def feed(self, character: str) -> None:
self._character_count += 1
if is_arabic_isolated_form(character):
self._isolated_form_count += 1
@property
def ratio(self) -> float:
if self._character_count < 8:
return 0.0
isolated_form_usage: float = self._isolated_form_count / self._character_count
return isolated_form_usage
@lru_cache(maxsize=1024)
def is_suspiciously_successive_range(
unicode_range_a: str | None, unicode_range_b: str | None
) -> bool:
"""
Determine if two Unicode range seen next to each other can be considered as suspicious.
"""
if unicode_range_a is None or unicode_range_b is None:
return True
if unicode_range_a == unicode_range_b:
return False
if "Latin" in unicode_range_a and "Latin" in unicode_range_b:
return False
if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b:
return False
# Latin characters can be accompanied with a combining diacritical mark
# eg. Vietnamese.
if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and (
"Combining" in unicode_range_a or "Combining" in unicode_range_b
):
return False
keywords_range_a, keywords_range_b = (
unicode_range_a.split(" "),
unicode_range_b.split(" "),
)
for el in keywords_range_a:
if el in UNICODE_SECONDARY_RANGE_KEYWORD:
continue
if el in keywords_range_b:
return False
# Japanese Exception
range_a_jp_chars, range_b_jp_chars = (
unicode_range_a
in (
"Hiragana",
"Katakana",
),
unicode_range_b in ("Hiragana", "Katakana"),
)
if (range_a_jp_chars or range_b_jp_chars) and (
"CJK" in unicode_range_a or "CJK" in unicode_range_b
):
return False
if range_a_jp_chars and range_b_jp_chars:
return False
if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b:
if "CJK" in unicode_range_a or "CJK" in unicode_range_b:
return False
if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
return False
# Chinese/Japanese use dedicated range for punctuation and/or separators.
if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or (
unicode_range_a in ["Katakana", "Hiragana"]
and unicode_range_b in ["Katakana", "Hiragana"]
):
if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b:
return False
if "Forms" in unicode_range_a or "Forms" in unicode_range_b:
return False
if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin":
return False
return True
@lru_cache(maxsize=2048)
def mess_ratio(
decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False
) -> float:
"""
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
"""
detectors: list[MessDetectorPlugin] = [
md_class() for md_class in MessDetectorPlugin.__subclasses__()
]
length: int = len(decoded_sequence) + 1
mean_mess_ratio: float = 0.0
if length < 512:
intermediary_mean_mess_ratio_calc: int = 32
elif length <= 1024:
intermediary_mean_mess_ratio_calc = 64
else:
intermediary_mean_mess_ratio_calc = 128
for character, index in zip(decoded_sequence + "\n", range(length)):
for detector in detectors:
if detector.eligible(character):
detector.feed(character)
if (
index > 0 and index % intermediary_mean_mess_ratio_calc == 0
) or index == length - 1:
mean_mess_ratio = sum(dt.ratio for dt in detectors)
if mean_mess_ratio >= maximum_threshold:
break
if debug:
logger = getLogger("charset_normalizer")
logger.log(
TRACE,
"Mess-detector extended-analysis start. "
f"intermediary_mean_mess_ratio_calc={intermediary_mean_mess_ratio_calc} mean_mess_ratio={mean_mess_ratio} "
f"maximum_threshold={maximum_threshold}",
)
if len(decoded_sequence) > 16:
logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}")
logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}")
for dt in detectors:
logger.log(TRACE, f"{dt.__class__}: {dt.ratio}")
return round(mean_mess_ratio, 3)
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/md__mypyc.cpython-312-x86_64-linux-gnu.so
Executable
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/models.py
+360
View File
@@ -0,0 +1,360 @@
from __future__ import annotations
from encodings.aliases import aliases
from hashlib import sha256
from json import dumps
from re import sub
from typing import Any, Iterator, List, Tuple
from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE
from .utils import iana_name, is_multi_byte_encoding, unicode_range
class CharsetMatch:
def __init__(
self,
payload: bytes,
guessed_encoding: str,
mean_mess_ratio: float,
has_sig_or_bom: bool,
languages: CoherenceMatches,
decoded_payload: str | None = None,
preemptive_declaration: str | None = None,
):
self._payload: bytes = payload
self._encoding: str = guessed_encoding
self._mean_mess_ratio: float = mean_mess_ratio
self._languages: CoherenceMatches = languages
self._has_sig_or_bom: bool = has_sig_or_bom
self._unicode_ranges: list[str] | None = None
self._leaves: list[CharsetMatch] = []
self._mean_coherence_ratio: float = 0.0
self._output_payload: bytes | None = None
self._output_encoding: str | None = None
self._string: str | None = decoded_payload
self._preemptive_declaration: str | None = preemptive_declaration
def __eq__(self, other: object) -> bool:
if not isinstance(other, CharsetMatch):
if isinstance(other, str):
return iana_name(other) == self.encoding
return False
return self.encoding == other.encoding and self.fingerprint == other.fingerprint
def __lt__(self, other: object) -> bool:
"""
Implemented to make sorted available upon CharsetMatches items.
"""
if not isinstance(other, CharsetMatch):
raise ValueError
chaos_difference: float = abs(self.chaos - other.chaos)
coherence_difference: float = abs(self.coherence - other.coherence)
# Below 1% difference --> Use Coherence
if chaos_difference < 0.01 and coherence_difference > 0.02:
return self.coherence > other.coherence
elif chaos_difference < 0.01 and coherence_difference <= 0.02:
# When having a difficult decision, use the result that decoded as many multi-byte as possible.
# preserve RAM usage!
if len(self._payload) >= TOO_BIG_SEQUENCE:
return self.chaos < other.chaos
return self.multi_byte_usage > other.multi_byte_usage
return self.chaos < other.chaos
@property
def multi_byte_usage(self) -> float:
return 1.0 - (len(str(self)) / len(self.raw))
def __str__(self) -> str:
# Lazy Str Loading
if self._string is None:
self._string = str(self._payload, self._encoding, "strict")
return self._string
def __repr__(self) -> str:
return f"<CharsetMatch '{self.encoding}' bytes({self.fingerprint})>"
def add_submatch(self, other: CharsetMatch) -> None:
if not isinstance(other, CharsetMatch) or other == self:
raise ValueError(
"Unable to add instance <{}> as a submatch of a CharsetMatch".format(
other.__class__
)
)
other._string = None # Unload RAM usage; dirty trick.
self._leaves.append(other)
@property
def encoding(self) -> str:
return self._encoding
@property
def encoding_aliases(self) -> list[str]:
"""
Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855.
"""
also_known_as: list[str] = []
for u, p in aliases.items():
if self.encoding == u:
also_known_as.append(p)
elif self.encoding == p:
also_known_as.append(u)
return also_known_as
@property
def bom(self) -> bool:
return self._has_sig_or_bom
@property
def byte_order_mark(self) -> bool:
return self._has_sig_or_bom
@property
def languages(self) -> list[str]:
"""
Return the complete list of possible languages found in decoded sequence.
Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'.
"""
return [e[0] for e in self._languages]
@property
def language(self) -> str:
"""
Most probable language found in decoded sequence. If none were detected or inferred, the property will return
"Unknown".
"""
if not self._languages:
# Trying to infer the language based on the given encoding
# Its either English or we should not pronounce ourselves in certain cases.
if "ascii" in self.could_be_from_charset:
return "English"
# doing it there to avoid circular import
from charset_normalizer.cd import encoding_languages, mb_encoding_languages
languages = (
mb_encoding_languages(self.encoding)
if is_multi_byte_encoding(self.encoding)
else encoding_languages(self.encoding)
)
if len(languages) == 0 or "Latin Based" in languages:
return "Unknown"
return languages[0]
return self._languages[0][0]
@property
def chaos(self) -> float:
return self._mean_mess_ratio
@property
def coherence(self) -> float:
if not self._languages:
return 0.0
return self._languages[0][1]
@property
def percent_chaos(self) -> float:
return round(self.chaos * 100, ndigits=3)
@property
def percent_coherence(self) -> float:
return round(self.coherence * 100, ndigits=3)
@property
def raw(self) -> bytes:
"""
Original untouched bytes.
"""
return self._payload
@property
def submatch(self) -> list[CharsetMatch]:
return self._leaves
@property
def has_submatch(self) -> bool:
return len(self._leaves) > 0
@property
def alphabets(self) -> list[str]:
if self._unicode_ranges is not None:
return self._unicode_ranges
# list detected ranges
detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)]
# filter and sort
self._unicode_ranges = sorted(list({r for r in detected_ranges if r}))
return self._unicode_ranges
@property
def could_be_from_charset(self) -> list[str]:
"""
The complete list of encoding that output the exact SAME str result and therefore could be the originating
encoding.
This list does include the encoding available in property 'encoding'.
"""
return [self._encoding] + [m.encoding for m in self._leaves]
def output(self, encoding: str = "utf_8") -> bytes:
"""
Method to get re-encoded bytes payload using given target encoding. Default to UTF-8.
Any errors will be simply ignored by the encoder NOT replaced.
"""
if self._output_encoding is None or self._output_encoding != encoding:
self._output_encoding = encoding
decoded_string = str(self)
if (
self._preemptive_declaration is not None
and self._preemptive_declaration.lower()
not in ["utf-8", "utf8", "utf_8"]
):
patched_header = sub(
RE_POSSIBLE_ENCODING_INDICATION,
lambda m: m.string[m.span()[0] : m.span()[1]].replace(
m.groups()[0],
iana_name(self._output_encoding).replace("_", "-"), # type: ignore[arg-type]
),
decoded_string[:8192],
count=1,
)
decoded_string = patched_header + decoded_string[8192:]
self._output_payload = decoded_string.encode(encoding, "replace")
return self._output_payload # type: ignore
@property
def fingerprint(self) -> str:
"""
Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one.
"""
return sha256(self.output()).hexdigest()
class CharsetMatches:
"""
Container with every CharsetMatch items ordered by default from most probable to the less one.
Act like a list(iterable) but does not implements all related methods.
"""
def __init__(self, results: list[CharsetMatch] | None = None):
self._results: list[CharsetMatch] = sorted(results) if results else []
def __iter__(self) -> Iterator[CharsetMatch]:
yield from self._results
def __getitem__(self, item: int | str) -> CharsetMatch:
"""
Retrieve a single item either by its position or encoding name (alias may be used here).
Raise KeyError upon invalid index or encoding not present in results.
"""
if isinstance(item, int):
return self._results[item]
if isinstance(item, str):
item = iana_name(item, False)
for result in self._results:
if item in result.could_be_from_charset:
return result
raise KeyError
def __len__(self) -> int:
return len(self._results)
def __bool__(self) -> bool:
return len(self._results) > 0
def append(self, item: CharsetMatch) -> None:
"""
Insert a single match. Will be inserted accordingly to preserve sort.
Can be inserted as a submatch.
"""
if not isinstance(item, CharsetMatch):
raise ValueError(
"Cannot append instance '{}' to CharsetMatches".format(
str(item.__class__)
)
)
# We should disable the submatch factoring when the input file is too heavy (conserve RAM usage)
if len(item.raw) < TOO_BIG_SEQUENCE:
for match in self._results:
if match.fingerprint == item.fingerprint and match.chaos == item.chaos:
match.add_submatch(item)
return
self._results.append(item)
self._results = sorted(self._results)
def best(self) -> CharsetMatch | None:
"""
Simply return the first match. Strict equivalent to matches[0].
"""
if not self._results:
return None
return self._results[0]
def first(self) -> CharsetMatch | None:
"""
Redundant method, call the method best(). Kept for BC reasons.
"""
return self.best()
CoherenceMatch = Tuple[str, float]
CoherenceMatches = List[CoherenceMatch]
class CliDetectionResult:
def __init__(
self,
path: str,
encoding: str | None,
encoding_aliases: list[str],
alternative_encodings: list[str],
language: str,
alphabets: list[str],
has_sig_or_bom: bool,
chaos: float,
coherence: float,
unicode_path: str | None,
is_preferred: bool,
):
self.path: str = path
self.unicode_path: str | None = unicode_path
self.encoding: str | None = encoding
self.encoding_aliases: list[str] = encoding_aliases
self.alternative_encodings: list[str] = alternative_encodings
self.language: str = language
self.alphabets: list[str] = alphabets
self.has_sig_or_bom: bool = has_sig_or_bom
self.chaos: float = chaos
self.coherence: float = coherence
self.is_preferred: bool = is_preferred
@property
def __dict__(self) -> dict[str, Any]: # type: ignore
return {
"path": self.path,
"encoding": self.encoding,
"encoding_aliases": self.encoding_aliases,
"alternative_encodings": self.alternative_encodings,
"language": self.language,
"alphabets": self.alphabets,
"has_sig_or_bom": self.has_sig_or_bom,
"chaos": self.chaos,
"coherence": self.coherence,
"unicode_path": self.unicode_path,
"is_preferred": self.is_preferred,
}
def to_json(self) -> str:
return dumps(self.__dict__, ensure_ascii=True, indent=4)
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/py.typed
View File
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/utils.py
+414
View File
@@ -0,0 +1,414 @@
from __future__ import annotations
import importlib
import logging
import unicodedata
from codecs import IncrementalDecoder
from encodings.aliases import aliases
from functools import lru_cache
from re import findall
from typing import Generator
from _multibytecodec import ( # type: ignore[import-not-found,import]
MultibyteIncrementalDecoder,
)
from .constant import (
ENCODING_MARKS,
IANA_SUPPORTED_SIMILAR,
RE_POSSIBLE_ENCODING_INDICATION,
UNICODE_RANGES_COMBINED,
UNICODE_SECONDARY_RANGE_KEYWORD,
UTF8_MAXIMAL_ALLOCATION,
COMMON_CJK_CHARACTERS,
)
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_accentuated(character: str) -> bool:
try:
description: str = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return (
"WITH GRAVE" in description
or "WITH ACUTE" in description
or "WITH CEDILLA" in description
or "WITH DIAERESIS" in description
or "WITH CIRCUMFLEX" in description
or "WITH TILDE" in description
or "WITH MACRON" in description
or "WITH RING ABOVE" in description
)
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def remove_accent(character: str) -> str:
decomposed: str = unicodedata.decomposition(character)
if not decomposed:
return character
codes: list[str] = decomposed.split(" ")
return chr(int(codes[0], 16))
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def unicode_range(character: str) -> str | None:
"""
Retrieve the Unicode range official name from a single character.
"""
character_ord: int = ord(character)
for range_name, ord_range in UNICODE_RANGES_COMBINED.items():
if character_ord in ord_range:
return range_name
return None
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_latin(character: str) -> bool:
try:
description: str = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "LATIN" in description
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_punctuation(character: str) -> bool:
character_category: str = unicodedata.category(character)
if "P" in character_category:
return True
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Punctuation" in character_range
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_symbol(character: str) -> bool:
character_category: str = unicodedata.category(character)
if "S" in character_category or "N" in character_category:
return True
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Forms" in character_range and character_category != "Lo"
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_emoticon(character: str) -> bool:
character_range: str | None = unicode_range(character)
if character_range is None:
return False
return "Emoticons" in character_range or "Pictographs" in character_range
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_separator(character: str) -> bool:
if character.isspace() or character in {"｜", "+", "<", ">"}:
return True
character_category: str = unicodedata.category(character)
return "Z" in character_category or character_category in {"Po", "Pd", "Pc"}
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_case_variable(character: str) -> bool:
return character.islower() != character.isupper()
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_cjk(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "CJK" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_hiragana(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "HIRAGANA" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_katakana(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "KATAKANA" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_hangul(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "HANGUL" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_thai(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "THAI" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_arabic(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "ARABIC" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_arabic_isolated_form(character: str) -> bool:
try:
character_name = unicodedata.name(character)
except ValueError: # Defensive: unicode database outdated?
return False
return "ARABIC" in character_name and "ISOLATED FORM" in character_name
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_cjk_uncommon(character: str) -> bool:
return character not in COMMON_CJK_CHARACTERS
@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED))
def is_unicode_range_secondary(range_name: str) -> bool:
return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD)
@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION)
def is_unprintable(character: str) -> bool:
return (
character.isspace() is False # includes \n \t \r \v
and character.isprintable() is False
and character != "\x1a" # Why? Its the ASCII substitute character.
and character != "\ufeff" # bug discovered in Python,
# Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space.
)
def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> str | None:
"""
Extract using ASCII-only decoder any specified encoding in the first n-bytes.
"""
if not isinstance(sequence, bytes):
raise TypeError
seq_len: int = len(sequence)
results: list[str] = findall(
RE_POSSIBLE_ENCODING_INDICATION,
sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"),
)
if len(results) == 0:
return None
for specified_encoding in results:
specified_encoding = specified_encoding.lower().replace("-", "_")
encoding_alias: str
encoding_iana: str
for encoding_alias, encoding_iana in aliases.items():
if encoding_alias == specified_encoding:
return encoding_iana
if encoding_iana == specified_encoding:
return encoding_iana
return None
@lru_cache(maxsize=128)
def is_multi_byte_encoding(name: str) -> bool:
"""
Verify is a specific encoding is a multi byte one based on it IANA name
"""
return name in {
"utf_8",
"utf_8_sig",
"utf_16",
"utf_16_be",
"utf_16_le",
"utf_32",
"utf_32_le",
"utf_32_be",
"utf_7",
} or issubclass(
importlib.import_module(f"encodings.{name}").IncrementalDecoder,
MultibyteIncrementalDecoder,
)
def identify_sig_or_bom(sequence: bytes) -> tuple[str | None, bytes]:
"""
Identify and extract SIG/BOM in given sequence.
"""
for iana_encoding in ENCODING_MARKS:
marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding]
if isinstance(marks, bytes):
marks = [marks]
for mark in marks:
if sequence.startswith(mark):
return iana_encoding, mark
return None, b""
def should_strip_sig_or_bom(iana_encoding: str) -> bool:
return iana_encoding not in {"utf_16", "utf_32"}
def iana_name(cp_name: str, strict: bool = True) -> str:
"""Returns the Python normalized encoding name (Not the IANA official name)."""
cp_name = cp_name.lower().replace("-", "_")
encoding_alias: str
encoding_iana: str
for encoding_alias, encoding_iana in aliases.items():
if cp_name in [encoding_alias, encoding_iana]:
return encoding_iana
if strict:
raise ValueError(f"Unable to retrieve IANA for '{cp_name}'")
return cp_name
def cp_similarity(iana_name_a: str, iana_name_b: str) -> float:
if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b):
return 0.0
decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder
decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder
id_a: IncrementalDecoder = decoder_a(errors="ignore")
id_b: IncrementalDecoder = decoder_b(errors="ignore")
character_match_count: int = 0
for i in range(255):
to_be_decoded: bytes = bytes([i])
if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded):
character_match_count += 1
return character_match_count / 254
def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool:
"""
Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using
the function cp_similarity.
"""
return (
iana_name_a in IANA_SUPPORTED_SIMILAR
and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a]
)
def set_logging_handler(
name: str = "charset_normalizer",
level: int = logging.INFO,
format_string: str = "%(asctime)s | %(levelname)s | %(message)s",
) -> None:
logger = logging.getLogger(name)
logger.setLevel(level)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(format_string))
logger.addHandler(handler)
def cut_sequence_chunks(
sequences: bytes,
encoding_iana: str,
offsets: range,
chunk_size: int,
bom_or_sig_available: bool,
strip_sig_or_bom: bool,
sig_payload: bytes,
is_multi_byte_decoder: bool,
decoded_payload: str | None = None,
) -> Generator[str, None, None]:
if decoded_payload and is_multi_byte_decoder is False:
for i in offsets:
chunk = decoded_payload[i : i + chunk_size]
if not chunk:
break
yield chunk
else:
for i in offsets:
chunk_end = i + chunk_size
if chunk_end > len(sequences) + 8:
continue
cut_sequence = sequences[i : i + chunk_size]
if bom_or_sig_available and strip_sig_or_bom is False:
cut_sequence = sig_payload + cut_sequence
chunk = cut_sequence.decode(
encoding_iana,
errors="ignore" if is_multi_byte_decoder else "strict",
)
# multi-byte bad cutting detector and adjustment
# not the cleanest way to perform that fix but clever enough for now.
if is_multi_byte_decoder and i > 0:
chunk_partial_size_chk: int = min(chunk_size, 16)
if (
decoded_payload
and chunk[:chunk_partial_size_chk] not in decoded_payload
):
for j in range(i, i - 4, -1):
cut_sequence = sequences[j:chunk_end]
if bom_or_sig_available and strip_sig_or_bom is False:
cut_sequence = sig_payload + cut_sequence
chunk = cut_sequence.decode(encoding_iana, errors="ignore")
if chunk[:chunk_partial_size_chk] in decoded_payload:
break
yield chunk
test_agent/.venv/lib/python3.12/site-packages/charset_normalizer/version.py
+8
View File
@@ -0,0 +1,8 @@
"""
Expose version
"""
from __future__ import annotations
__version__ = "3.4.4"
VERSION = __version__.split(".")
test_agent/.venv/lib/python3.12/site-packages/idna-3.11.dist-info/INSTALLER
+1
View File
@@ -0,0 +1 @@
pip
test_agent/.venv/lib/python3.12/site-packages/idna-3.11.dist-info/METADATA
+209
View File
@@ -0,0 +1,209 @@
Metadata-Version: 2.4
Name: idna
Version: 3.11
Summary: Internationalized Domain Names in Applications (IDNA)
Author-email: Kim Davies <kim+pypi@gumleaf.org>
Requires-Python: >=3.8
Description-Content-Type: text/x-rst
License-Expression: BSD-3-Clause
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: System Administrators
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Internet :: Name Service (DNS)
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Classifier: Topic :: Utilities
License-File: LICENSE.md
Requires-Dist: ruff >= 0.6.2 ; extra == "all"
Requires-Dist: mypy >= 1.11.2 ; extra == "all"
Requires-Dist: pytest >= 8.3.2 ; extra == "all"
Requires-Dist: flake8 >= 7.1.1 ; extra == "all"
Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.rst
Project-URL: Issue tracker, https://github.com/kjd/idna/issues
Project-URL: Source, https://github.com/kjd/idna
Provides-Extra: all
Internationalized Domain Names in Applications (IDNA)
=====================================================
Support for `Internationalized Domain Names in
Applications (IDNA) <https://tools.ietf.org/html/rfc5891>`_
and `Unicode IDNA Compatibility Processing
<https://unicode.org/reports/tr46/>`_.
The latest versions of these standards supplied here provide
more comprehensive language coverage and reduce the potential of
allowing domains with known security vulnerabilities. This library
is a suitable replacement for the “encodings.idna”
module that comes with the Python standard library, but which
only supports an older superseded IDNA specification from 2003.
Basic functions are simply executed:
.. code-block:: pycon
>>> import idna
>>> idna.encode('ドメイン.テスト')
b'xn--eckwd4c7c.xn--zckzah'
>>> print(idna.decode('xn--eckwd4c7c.xn--zckzah'))
ドメイン.テスト
Installation
------------
This package is available for installation from PyPI via the
typical mechanisms, such as:
.. code-block:: bash
$ python3 -m pip install idna
Usage
-----
For typical usage, the ``encode`` and ``decode`` functions will take a
domain name argument and perform a conversion to ASCII compatible encoding
(known as A-labels), or to Unicode strings (known as U-labels)
respectively.
.. code-block:: pycon
>>> import idna
>>> idna.encode('ドメイン.テスト')
b'xn--eckwd4c7c.xn--zckzah'
>>> print(idna.decode('xn--eckwd4c7c.xn--zckzah'))
ドメイン.テスト
Conversions can be applied at a per-label basis using the ``ulabel`` or
``alabel`` functions if necessary:
.. code-block:: pycon
>>> idna.alabel('测试')
b'xn--0zwm56d'
Compatibility Mapping (UTS #46)
+++++++++++++++++++++++++++++++
This library provides support for `Unicode IDNA Compatibility
Processing <https://unicode.org/reports/tr46/>`_ which normalizes input from
different potential ways a user may input a domain prior to performing the IDNA
conversion operations. This functionality, known as a
`mapping <https://tools.ietf.org/html/rfc5895>`_, is considered by the
specification to be a local user-interface issue distinct from IDNA
conversion functionality.
For example, “Königsgäßchen” is not a permissible label as *LATIN
CAPITAL LETTER K* is not allowed (nor are capital letters in general).
UTS 46 will convert this into lower case prior to applying the IDNA
conversion.
.. code-block:: pycon
>>> import idna
>>> idna.encode('Königsgäßchen')
...
idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed
>>> idna.encode('Königsgäßchen', uts46=True)
b'xn--knigsgchen-b4a3dun'
>>> print(idna.decode('xn--knigsgchen-b4a3dun'))
königsgäßchen
Exceptions
----------
All errors raised during the conversion following the specification
should raise an exception derived from the ``idna.IDNAError`` base
class.
More specific exceptions that may be generated as ``idna.IDNABidiError``
when the error reflects an illegal combination of left-to-right and
right-to-left characters in a label; ``idna.InvalidCodepoint`` when
a specific codepoint is an illegal character in an IDN label (i.e.
INVALID); and ``idna.InvalidCodepointContext`` when the codepoint is
illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ
but the contextual requirements are not satisfied.)
Building and Diagnostics
------------------------
The IDNA and UTS 46 functionality relies upon pre-calculated lookup
tables for performance. These tables are derived from computing against
eligibility criteria in the respective standards using the command-line
script ``tools/idna-data``.
This tool will fetch relevant codepoint data from the Unicode repository
and perform the required calculations to identify eligibility. There are
three main modes:
* ``idna-data make-libdata``. Generates ``idnadata.py`` and
``uts46data.py``, the pre-calculated lookup tables used for IDNA and
UTS 46 conversions. Implementers who wish to track this library against
a different Unicode version may use this tool to manually generate a
different version of the ``idnadata.py`` and ``uts46data.py`` files.
* ``idna-data make-table``. Generate a table of the IDNA disposition
(e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix
B.1 of RFC 5892 and the pre-computed tables published by `IANA
<https://www.iana.org/>`_.
* ``idna-data U+0061``. Prints debugging output on the various
properties associated with an individual Unicode codepoint (in this
case, U+0061), that are used to assess the IDNA and UTS 46 status of a
codepoint. This is helpful in debugging or analysis.
The tool accepts a number of arguments, described using ``idna-data
-h``. Most notably, the ``--version`` argument allows the specification
of the version of Unicode to be used in computing the table data. For
example, ``idna-data --version 9.0.0 make-libdata`` will generate
library data against Unicode 9.0.0.
Additional Notes
----------------
* **Packages**. The latest tagged release version is published in the
`Python Package Index <https://pypi.org/project/idna/>`_.
* **Version support**. This library supports Python 3.8 and higher.
As this library serves as a low-level toolkit for a variety of
applications, many of which strive for broad compatibility with older
Python versions, there is no rush to remove older interpreter support.
Support for older versions are likely to be removed from new releases
as automated tests can no longer easily be run, i.e. once the Python
version is officially end-of-life.
* **Testing**. The library has a test suite based on each rule of the
IDNA specification, as well as tests that are provided as part of the
Unicode Technical Standard 46, `Unicode IDNA Compatibility Processing
<https://unicode.org/reports/tr46/>`_.
* **Emoji**. It is an occasional request to support emoji domains in
this library. Encoding of symbols like emoji is expressly prohibited by
the technical standard IDNA 2008 and emoji domains are broadly phased
out across the domain industry due to associated security risks. For
now, applications that need to support these non-compliant labels
may wish to consider trying the encode/decode operation in this library
first, and then falling back to using `encodings.idna`. See `the Github
project <https://github.com/kjd/idna/issues/18>`_ for more discussion.
* **Transitional processing**. Unicode 16.0.0 removed transitional
processing so the `transitional` argument for the encode() method
no longer has any effect and will be removed at a later date.
test_agent/.venv/lib/python3.12/site-packages/idna-3.11.dist-info/RECORD
+22
View File
@@ -0,0 +1,22 @@
idna-3.11.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
idna-3.11.dist-info/METADATA,sha256=fCwSww9SuiN8TIHllFSASUQCW55hAs8dzKnr9RaEEbA,8378
idna-3.11.dist-info/RECORD,,
idna-3.11.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
idna-3.11.dist-info/licenses/LICENSE.md,sha256=t6M2q_OwThgOwGXN0W5wXQeeHMehT5EKpukYfza5zYc,1541
idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868
idna/__pycache__/__init__.cpython-312.pyc,,
idna/__pycache__/codec.cpython-312.pyc,,
idna/__pycache__/compat.cpython-312.pyc,,
idna/__pycache__/core.cpython-312.pyc,,
idna/__pycache__/idnadata.cpython-312.pyc,,
idna/__pycache__/intranges.cpython-312.pyc,,
idna/__pycache__/package_data.cpython-312.pyc,,
idna/__pycache__/uts46data.cpython-312.pyc,,
idna/codec.py,sha256=M2SGWN7cs_6B32QmKTyTN6xQGZeYQgQ2wiX3_DR6loE,3438
idna/compat.py,sha256=RzLy6QQCdl9784aFhb2EX9EKGCJjg0P3PilGdeXXcx8,316
idna/core.py,sha256=P26_XVycuMTZ1R2mNK1ZREVzM5mvTzdabBXfyZVU1Lc,13246
idna/idnadata.py,sha256=SG8jhaGE53iiD6B49pt2pwTv_UvClciWE-N54oR2p4U,79623
idna/intranges.py,sha256=amUtkdhYcQG8Zr-CoMM_kVRacxkivC1WgxN1b63KKdU,1898
idna/package_data.py,sha256=_CUavOxobnbyNG2FLyHoN8QHP3QM9W1tKuw7eq9QwBk,21
idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
idna/uts46data.py,sha256=H9J35VkD0F9L9mKOqjeNGd2A-Va6FlPoz6Jz4K7h-ps,243725
test_agent/.venv/lib/python3.12/site-packages/idna-3.11.dist-info/WHEEL
+4
View File
@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: flit 3.12.0
Root-Is-Purelib: true
Tag: py3-none-any
test_agent/.venv/lib/python3.12/site-packages/idna-3.11.dist-info/licenses/LICENSE.md
+31
View File
@@ -0,0 +1,31 @@
BSD 3-Clause License
Copyright (c) 2013-2025, Kim Davies and contributors.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
test_agent/.venv/lib/python3.12/site-packages/idna/__init__.py
+45
View File
@@ -0,0 +1,45 @@
from .core import (
IDNABidiError,
IDNAError,
InvalidCodepoint,
InvalidCodepointContext,
alabel,
check_bidi,
check_hyphen_ok,
check_initial_combiner,
check_label,
check_nfc,
decode,
encode,
ulabel,
uts46_remap,
valid_contextj,
valid_contexto,
valid_label_length,
valid_string_length,
)
from .intranges import intranges_contain
from .package_data import __version__
__all__ = [
"__version__",
"IDNABidiError",
"IDNAError",
"InvalidCodepoint",
"InvalidCodepointContext",
"alabel",
"check_bidi",
"check_hyphen_ok",
"check_initial_combiner",
"check_label",
"check_nfc",
"decode",
"encode",
"intranges_contain",
"ulabel",
"uts46_remap",
"valid_contextj",
"valid_contexto",
"valid_label_length",
"valid_string_length",
]
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/__init__.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/codec.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/compat.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/core.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/idnadata.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/intranges.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/package_data.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/__pycache__/uts46data.cpython-312.pyc
BIN
View File
Binary file not shown.
test_agent/.venv/lib/python3.12/site-packages/idna/codec.py
+122
View File
@@ -0,0 +1,122 @@
import codecs
import re
from typing import Any, Optional, Tuple
from .core import IDNAError, alabel, decode, encode, ulabel
_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]")
class Codec(codecs.Codec):
def encode(self, data: str, errors: str = "strict") -> Tuple[bytes, int]:
if errors != "strict":
raise IDNAError('Unsupported error handling "{}"'.format(errors))
if not data:
return b"", 0
return encode(data), len(data)
def decode(self, data: bytes, errors: str = "strict") -> Tuple[str, int]:
if errors != "strict":
raise IDNAError('Unsupported error handling "{}"'.format(errors))
if not data:
return "", 0
return decode(data), len(data)
class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[bytes, int]:
if errors != "strict":
raise IDNAError('Unsupported error handling "{}"'.format(errors))
if not data:
return b"", 0
labels = _unicode_dots_re.split(data)
trailing_dot = b""
if labels:
if not labels[-1]:
trailing_dot = b"."
del labels[-1]
elif not final:
# Keep potentially unfinished label until the next call
del labels[-1]
if labels:
trailing_dot = b"."
result = []
size = 0
for label in labels:
result.append(alabel(label))
if size:
size += 1
size += len(label)
# Join with U+002E
result_bytes = b".".join(result) + trailing_dot
size += len(trailing_dot)
return result_bytes, size
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
def _buffer_decode(self, data: Any, errors: str, final: bool) -> Tuple[str, int]:
if errors != "strict":
raise IDNAError('Unsupported error handling "{}"'.format(errors))
if not data:
return ("", 0)
if not isinstance(data, str):
data = str(data, "ascii")
labels = _unicode_dots_re.split(data)
trailing_dot = ""
if labels:
if not labels[-1]:
trailing_dot = "."
del labels[-1]
elif not final:
# Keep potentially unfinished label until the next call
del labels[-1]
if labels:
trailing_dot = "."
result = []
size = 0
for label in labels:
result.append(ulabel(label))
if size:
size += 1
size += len(label)
result_str = ".".join(result) + trailing_dot
size += len(trailing_dot)
return (result_str, size)
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
def search_function(name: str) -> Optional[codecs.CodecInfo]:
if name != "idna2008":
return None
return codecs.CodecInfo(
name=name,
encode=Codec().encode,
decode=Codec().decode, # type: ignore
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
)
codecs.register(search_function)
test_agent/.venv/lib/python3.12/site-packages/idna/compat.py
+15
View File
@@ -0,0 +1,15 @@
from typing import Any, Union
from .core import decode, encode
def ToASCII(label: str) -> bytes:
return encode(label)
def ToUnicode(label: Union[bytes, bytearray]) -> str:
return decode(label)
def nameprep(s: Any) -> None:
raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol")
test_agent/.venv/lib/python3.12/site-packages/idna/core.py
+437
View File
@@ -0,0 +1,437 @@
import bisect
import re
import unicodedata
from typing import Optional, Union
from . import idnadata
from .intranges import intranges_contain
_virama_combining_class = 9
_alabel_prefix = b"xn--"
_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]")
class IDNAError(UnicodeError):
"""Base exception for all IDNA-encoding related problems"""
pass
class IDNABidiError(IDNAError):
"""Exception when bidirectional requirements are not satisfied"""
pass
class InvalidCodepoint(IDNAError):
"""Exception when a disallowed or unallocated codepoint is used"""
pass
class InvalidCodepointContext(IDNAError):
"""Exception when the codepoint is not valid in the context it is used"""
pass
def _combining_class(cp: int) -> int:
v = unicodedata.combining(chr(cp))
if v == 0:
if not unicodedata.name(chr(cp)):
raise ValueError("Unknown character in unicodedata")
return v
def _is_script(cp: str, script: str) -> bool:
return intranges_contain(ord(cp), idnadata.scripts[script])
def _punycode(s: str) -> bytes:
return s.encode("punycode")
def _unot(s: int) -> str:
return "U+{:04X}".format(s)
def valid_label_length(label: Union[bytes, str]) -> bool:
if len(label) > 63:
return False
return True
def valid_string_length(label: Union[bytes, str], trailing_dot: bool) -> bool:
if len(label) > (254 if trailing_dot else 253):
return False
return True
def check_bidi(label: str, check_ltr: bool = False) -> bool:
# Bidi rules should only be applied if string contains RTL characters
bidi_label = False
for idx, cp in enumerate(label, 1):
direction = unicodedata.bidirectional(cp)
if direction == "":
# String likely comes from a newer version of Unicode
raise IDNABidiError("Unknown directionality in label {} at position {}".format(repr(label), idx))
if direction in ["R", "AL", "AN"]:
bidi_label = True
if not bidi_label and not check_ltr:
return True
# Bidi rule 1
direction = unicodedata.bidirectional(label[0])
if direction in ["R", "AL"]:
rtl = True
elif direction == "L":
rtl = False
else:
raise IDNABidiError("First codepoint in label {} must be directionality L, R or AL".format(repr(label)))
valid_ending = False
number_type: Optional[str] = None
for idx, cp in enumerate(label, 1):
direction = unicodedata.bidirectional(cp)
if rtl:
# Bidi rule 2
if direction not in [
"R",
"AL",
"AN",
"EN",
"ES",
"CS",
"ET",
"ON",
"BN",
"NSM",
]:
raise IDNABidiError("Invalid direction for codepoint at position {} in a right-to-left label".format(idx))
# Bidi rule 3
if direction in ["R", "AL", "EN", "AN"]:
valid_ending = True
elif direction != "NSM":
valid_ending = False
# Bidi rule 4
if direction in ["AN", "EN"]:
if not number_type:
number_type = direction
else:
if number_type != direction:
raise IDNABidiError("Can not mix numeral types in a right-to-left label")
else:
# Bidi rule 5
if direction not in ["L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"]:
raise IDNABidiError("Invalid direction for codepoint at position {} in a left-to-right label".format(idx))
# Bidi rule 6
if direction in ["L", "EN"]:
valid_ending = True
elif direction != "NSM":
valid_ending = False
if not valid_ending:
raise IDNABidiError("Label ends with illegal codepoint directionality")
return True
def check_initial_combiner(label: str) -> bool:
if unicodedata.category(label[0])[0] == "M":
raise IDNAError("Label begins with an illegal combining character")
return True
def check_hyphen_ok(label: str) -> bool:
if label[2:4] == "--":
raise IDNAError("Label has disallowed hyphens in 3rd and 4th position")
if label[0] == "-" or label[-1] == "-":
raise IDNAError("Label must not start or end with a hyphen")
return True
def check_nfc(label: str) -> None:
if unicodedata.normalize("NFC", label) != label:
raise IDNAError("Label must be in Normalization Form C")
def valid_contextj(label: str, pos: int) -> bool:
cp_value = ord(label[pos])
if cp_value == 0x200C:
if pos > 0:
if _combining_class(ord(label[pos - 1])) == _virama_combining_class:
return True
ok = False
for i in range(pos - 1, -1, -1):
joining_type = idnadata.joining_types.get(ord(label[i]))
if joining_type == ord("T"):
continue
elif joining_type in [ord("L"), ord("D")]:
ok = True
break
else:
break
if not ok:
return False
ok = False
for i in range(pos + 1, len(label)):
joining_type = idnadata.joining_types.get(ord(label[i]))
if joining_type == ord("T"):
continue
elif joining_type in [ord("R"), ord("D")]:
ok = True
break
else:
break
return ok
if cp_value == 0x200D:
if pos > 0:
if _combining_class(ord(label[pos - 1])) == _virama_combining_class:
return True
return False
else:
return False
def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
cp_value = ord(label[pos])
if cp_value == 0x00B7:
if 0 < pos < len(label) - 1:
if ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C:
return True
return False
elif cp_value == 0x0375:
if pos < len(label) - 1 and len(label) > 1:
return _is_script(label[pos + 1], "Greek")
return False
elif cp_value == 0x05F3 or cp_value == 0x05F4:
if pos > 0:
return _is_script(label[pos - 1], "Hebrew")
return False
elif cp_value == 0x30FB:
for cp in label:
if cp == "\u30fb":
continue
if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"):
return True
return False
elif 0x660 <= cp_value <= 0x669:
for cp in label:
if 0x6F0 <= ord(cp) <= 0x06F9:
return False
return True
elif 0x6F0 <= cp_value <= 0x6F9:
for cp in label:
if 0x660 <= ord(cp) <= 0x0669:
return False
return True
return False
def check_label(label: Union[str, bytes, bytearray]) -> None:
if isinstance(label, (bytes, bytearray)):
label = label.decode("utf-8")
if len(label) == 0:
raise IDNAError("Empty Label")
check_nfc(label)
check_hyphen_ok(label)
check_initial_combiner(label)
for pos, cp in enumerate(label):
cp_value = ord(cp)
if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]):
continue
elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]):
try:
if not valid_contextj(label, pos):
raise InvalidCodepointContext(
"Joiner {} not allowed at position {} in {}".format(_unot(cp_value), pos + 1, repr(label))
)
except ValueError:
raise IDNAError(
"Unknown codepoint adjacent to joiner {} at position {} in {}".format(
_unot(cp_value), pos + 1, repr(label)
)
)
elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]):
if not valid_contexto(label, pos):
raise InvalidCodepointContext(
"Codepoint {} not allowed at position {} in {}".format(_unot(cp_value), pos + 1, repr(label))
)
else:
raise InvalidCodepoint(
"Codepoint {} at position {} of {} not allowed".format(_unot(cp_value), pos + 1, repr(label))
)
check_bidi(label)
def alabel(label: str) -> bytes:
try:
label_bytes = label.encode("ascii")
ulabel(label_bytes)
if not valid_label_length(label_bytes):
raise IDNAError("Label too long")
return label_bytes
except UnicodeEncodeError:
pass
check_label(label)
label_bytes = _alabel_prefix + _punycode(label)
if not valid_label_length(label_bytes):
raise IDNAError("Label too long")
return label_bytes
def ulabel(label: Union[str, bytes, bytearray]) -> str:
if not isinstance(label, (bytes, bytearray)):
try:
label_bytes = label.encode("ascii")
except UnicodeEncodeError:
check_label(label)
return label
else:
label_bytes = bytes(label)
label_bytes = label_bytes.lower()
if label_bytes.startswith(_alabel_prefix):
label_bytes = label_bytes[len(_alabel_prefix) :]
if not label_bytes:
raise IDNAError("Malformed A-label, no Punycode eligible content found")
if label_bytes.decode("ascii")[-1] == "-":
raise IDNAError("A-label must not end with a hyphen")
else:
check_label(label_bytes)
return label_bytes.decode("ascii")
try:
label = label_bytes.decode("punycode")
except UnicodeError:
raise IDNAError("Invalid A-label")
check_label(label)
return label
def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:
"""Re-map the characters in the string according to UTS46 processing."""
from .uts46data import uts46data
output = ""
for pos, char in enumerate(domain):
code_point = ord(char)
try:
uts46row = uts46data[code_point if code_point < 256 else bisect.bisect_left(uts46data, (code_point, "Z")) - 1]
status = uts46row[1]
replacement: Optional[str] = None
if len(uts46row) == 3:
replacement = uts46row[2]
if (
status == "V"
or (status == "D" and not transitional)
or (status == "3" and not std3_rules and replacement is None)
):
output += char
elif replacement is not None and (
status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional)
):
output += replacement
elif status != "I":
raise IndexError()
except IndexError:
raise InvalidCodepoint(
"Codepoint {} not allowed at position {} in {}".format(_unot(code_point), pos + 1, repr(domain))
)
return unicodedata.normalize("NFC", output)
def encode(
s: Union[str, bytes, bytearray],
strict: bool = False,
uts46: bool = False,
std3_rules: bool = False,
transitional: bool = False,
) -> bytes:
if not isinstance(s, str):
try:
s = str(s, "ascii")
except UnicodeDecodeError:
raise IDNAError("should pass a unicode string to the function rather than a byte string.")
if uts46:
s = uts46_remap(s, std3_rules, transitional)
trailing_dot = False
result = []
if strict:
labels = s.split(".")
else:
labels = _unicode_dots_re.split(s)
if not labels or labels == [""]:
raise IDNAError("Empty domain")
if labels[-1] == "":
del labels[-1]
trailing_dot = True
for label in labels:
s = alabel(label)
if s:
result.append(s)
else:
raise IDNAError("Empty label")
if trailing_dot:
result.append(b"")
s = b".".join(result)
if not valid_string_length(s, trailing_dot):
raise IDNAError("Domain too long")
return s
def decode(
s: Union[str, bytes, bytearray],
strict: bool = False,
uts46: bool = False,
std3_rules: bool = False,
) -> str:
try:
if not isinstance(s, str):
s = str(s, "ascii")
except UnicodeDecodeError:
raise IDNAError("Invalid ASCII in A-label")
if uts46:
s = uts46_remap(s, std3_rules, False)
trailing_dot = False
result = []
if not strict:
labels = _unicode_dots_re.split(s)
else:
labels = s.split(".")
if not labels or labels == [""]:
raise IDNAError("Empty domain")
if not labels[-1]:
del labels[-1]
trailing_dot = True
for label in labels:
s = ulabel(label)
if s:
result.append(s)
else:
raise IDNAError("Empty label")
if trailing_dot:
result.append("")
return ".".join(result)
test_agent/.venv/lib/python3.12/site-packages/idna/idnadata.py
+4309
View File
File diff suppressed because it is too large Load Diff
test_agent/.venv/lib/python3.12/site-packages/idna/intranges.py
+57
View File
@@ -0,0 +1,57 @@
"""
Given a list of integers, made up of (hopefully) a small number of long runs
of consecutive integers, compute a representation of the form
((start1, end1), (start2, end2) ...). Then answer the question "was x present
in the original list?" in time O(log(# runs)).
"""
import bisect
from typing import List, Tuple
def intranges_from_list(list_: List[int]) -> Tuple[int, ...]:
"""Represent a list of integers as a sequence of ranges:
((start_0, end_0), (start_1, end_1), ...), such that the original
integers are exactly those x such that start_i <= x < end_i for some i.
Ranges are encoded as single integers (start << 32 | end), not as tuples.
"""
sorted_list = sorted(list_)
ranges = []
last_write = -1
for i in range(len(sorted_list)):
if i + 1 < len(sorted_list):
if sorted_list[i] == sorted_list[i + 1] - 1:
continue
current_range = sorted_list[last_write + 1 : i + 1]
ranges.append(_encode_range(current_range[0], current_range[-1] + 1))
last_write = i
return tuple(ranges)
def _encode_range(start: int, end: int) -> int:
return (start << 32) | end
def _decode_range(r: int) -> Tuple[int, int]:
return (r >> 32), (r & ((1 << 32) - 1))
def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool:
"""Determine if `int_` falls into one of the ranges in `ranges`."""
tuple_ = _encode_range(int_, 0)
pos = bisect.bisect_left(ranges, tuple_)
# we could be immediately ahead of a tuple (start, end)
# with start < int_ <= end
if pos > 0:
left, right = _decode_range(ranges[pos - 1])
if left <= int_ < right:
return True
# or we could be immediately behind a tuple (int_, end)
if pos < len(ranges):
left, _ = _decode_range(ranges[pos])
if left == int_:
return True
return False

Some files were not shown because too many files have changed in this diff Show More