102 lines
3.4 KiB
Python
102 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import httpx
|
|
|
|
from .config import Settings
|
|
from .models import PageContent, SearchResult
|
|
from .parsers import trim_text
|
|
|
|
|
|
class JinaClient:
|
|
def __init__(self, settings: Settings):
|
|
self.settings = settings
|
|
self._semaphore = asyncio.Semaphore(settings.max_concurrency)
|
|
|
|
@property
|
|
def _headers(self) -> dict[str, str]:
|
|
return {
|
|
"Authorization": f"Bearer {self.settings.jina_api_key}",
|
|
"X-Respond-With": "readerlm-v2",
|
|
"X-Retain-Images": "none",
|
|
"X-Retain-Links": "gpt-oss",
|
|
}
|
|
|
|
async def read_pages(self, results: list[SearchResult], max_page_chars: int) -> list[PageContent]:
|
|
tasks = [self.read_page(result.url, result.title, max_page_chars) for result in results]
|
|
return await asyncio.gather(*tasks)
|
|
|
|
async def read_page(self, url: str, fallback_title: str, max_page_chars: int) -> PageContent:
|
|
reader_url = f"{self.settings.reader_base_url}{url}"
|
|
try:
|
|
text = await self._get_text(
|
|
reader_url,
|
|
timeout=self.settings.reader_timeout,
|
|
extra_headers={"Accept": "text/plain"},
|
|
use_reader_headers=False,
|
|
)
|
|
title = ""
|
|
first_line = text.splitlines()[0].strip() if text else ""
|
|
if first_line.startswith("#"):
|
|
title = first_line.lstrip("#").strip()
|
|
|
|
return PageContent(
|
|
url=url,
|
|
title=title or fallback_title,
|
|
content=trim_text(text.strip(), max_page_chars),
|
|
fetched=True,
|
|
usable=self._is_usable_content(text),
|
|
)
|
|
except Exception as exc:
|
|
return PageContent(
|
|
url=url,
|
|
title=fallback_title,
|
|
content="",
|
|
fetched=False,
|
|
usable=False,
|
|
error=str(exc),
|
|
)
|
|
|
|
def _is_usable_content(self, text: str) -> bool:
|
|
normalized = text.strip()
|
|
if len(normalized) < 300:
|
|
return False
|
|
|
|
blocked_markers = [
|
|
"403: Forbidden",
|
|
"451 Unavailable For Legal Reasons",
|
|
"Target URL returned error 403",
|
|
"Target URL returned error 451",
|
|
"enable javascript",
|
|
"captcha",
|
|
]
|
|
lowered = normalized.lower()
|
|
return not any(marker.lower() in lowered for marker in blocked_markers)
|
|
|
|
async def _get_text(
|
|
self,
|
|
url: str,
|
|
timeout: float,
|
|
extra_headers: dict[str, str] | None = None,
|
|
use_reader_headers: bool = True,
|
|
) -> str:
|
|
headers = dict(self._headers) if use_reader_headers else {}
|
|
if extra_headers:
|
|
headers.update(extra_headers)
|
|
|
|
last_error: Exception | None = None
|
|
for _ in range(self.settings.retry_attempts + 1):
|
|
try:
|
|
async with self._semaphore:
|
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
|
response = await client.get(url, headers=headers)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except Exception as exc:
|
|
last_error = exc
|
|
await asyncio.sleep(0.5)
|
|
|
|
assert last_error is not None
|
|
raise last_error
|