94 lines
2.4 KiB
Python
94 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from urllib.parse import urlparse
|
|
|
|
from .models import SearchResult
|
|
|
|
_DOMAIN_TRUST: dict[str, float] = {
|
|
"arxiv.org": 0.95,
|
|
"docs.python.org": 0.95,
|
|
"developer.mozilla.org": 0.95,
|
|
"pypi.org": 0.90,
|
|
"github.com": 0.90,
|
|
"wikipedia.org": 0.85,
|
|
"readthedocs.io": 0.85,
|
|
"microsoft.com": 0.85,
|
|
"azure.microsoft.com": 0.85,
|
|
"cloud.google.com": 0.85,
|
|
"aws.amazon.com": 0.85,
|
|
"stackoverflow.com": 0.80,
|
|
"npmjs.com": 0.80,
|
|
"medium.com": 0.55,
|
|
"zhihu.com": 0.55,
|
|
"csdn.net": 0.50,
|
|
"juejin.cn": 0.55,
|
|
}
|
|
|
|
|
|
def get_domain_trust(url: str) -> float:
|
|
"""Return a 0-1 trust score based on the URL's domain."""
|
|
try:
|
|
host = urlparse(url).hostname or ""
|
|
for domain, score in _DOMAIN_TRUST.items():
|
|
if host == domain or host.endswith(f".{domain}"):
|
|
return score
|
|
except Exception:
|
|
pass
|
|
return 0.60
|
|
|
|
MARKDOWN_LINK_RE = re.compile(r"\[(?P<title>[^\]]+)\]\((?P<url>https?://[^\s)]+)\)")
|
|
URL_RE = re.compile(r"https?://[^\s)>\]]+")
|
|
|
|
|
|
def extract_markdown_links(text: str) -> list[SearchResult]:
|
|
results: list[SearchResult] = []
|
|
seen: set[str] = set()
|
|
|
|
for rank, match in enumerate(MARKDOWN_LINK_RE.finditer(text), start=1):
|
|
url = match.group("url").strip()
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
start = max(0, match.start() - 160)
|
|
end = min(len(text), match.end() + 160)
|
|
snippet = " ".join(text[start:end].split())
|
|
results.append(
|
|
SearchResult(
|
|
title=match.group("title").strip(),
|
|
url=url,
|
|
snippet=snippet,
|
|
rank=rank,
|
|
)
|
|
)
|
|
|
|
return results
|
|
|
|
|
|
def extract_urls(text: str) -> list[str]:
|
|
urls: list[str] = []
|
|
seen: set[str] = set()
|
|
|
|
for match in URL_RE.finditer(text):
|
|
url = match.group(0).rstrip(".,")
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
urls.append(url)
|
|
|
|
return urls
|
|
|
|
|
|
def trim_text(text: str, limit: int) -> str:
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[:limit].rstrip() + "\n...[truncated]"
|
|
|
|
|
|
def extract_json_object(text: str) -> str | None:
|
|
start = text.find("{")
|
|
end = text.rfind("}")
|
|
if start == -1 or end == -1 or end <= start:
|
|
return None
|
|
return text[start : end + 1]
|