54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .config import Settings
|
|
from .models import SearchResult
|
|
class SearchClient:
|
|
def __init__(self, settings: Settings):
|
|
self.settings = settings
|
|
|
|
async def search(self, query: str) -> tuple[str, list[SearchResult]]:
|
|
return await self._bing_search(query)
|
|
|
|
async def _bing_search(self, query: str) -> tuple[str, list[SearchResult]]:
|
|
url = f"{self.settings.bing_search_url}?q={quote(query)}"
|
|
async with httpx.AsyncClient(timeout=self.settings.search_timeout, follow_redirects=True) as client:
|
|
response = await client.get(
|
|
url,
|
|
headers={
|
|
"User-Agent": "Mozilla/5.0",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
html = response.text
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
results: list[SearchResult] = []
|
|
for rank, item in enumerate(soup.select("li.b_algo"), start=1):
|
|
link = item.select_one("h2 a")
|
|
if not link:
|
|
continue
|
|
href = (link.get("href") or "").strip()
|
|
if not href.startswith("http"):
|
|
continue
|
|
snippet_node = item.select_one(".b_caption p")
|
|
results.append(
|
|
SearchResult(
|
|
title=link.get_text(" ", strip=True),
|
|
url=href,
|
|
snippet=snippet_node.get_text(" ", strip=True) if snippet_node else "",
|
|
rank=rank,
|
|
)
|
|
)
|
|
if len(results) >= self.settings.top_k_pages * 2:
|
|
break
|
|
|
|
if not results:
|
|
raise ValueError("No Bing HTML search results parsed")
|
|
return html, results
|