更新: 13 个文件 - 2026-03-18 09:44:57

scripts/intel/http_client.py

@@ -1,24 +1,36 @@
 from __future__ import annotations
 
+import time
 from typing import Any, Dict
 
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from intel.config import DEFAULT_HEALTH_POLICY, DEFAULT_REQUEST_POLICY
+
 
 DEFAULT_TIMEOUT = 30
 DEFAULT_USER_AGENT = "websafe-intel"
 
 
-def build_session() -> requests.Session:
+def _request_policy(source: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    return {**DEFAULT_REQUEST_POLICY, **((source or {}).get("request_policy") or {})}
+
+
+def _health_policy(source: Dict[str, Any] | None = None) -> Dict[str, Any]:
+    return {**DEFAULT_HEALTH_POLICY, **((source or {}).get("health_policy") or {})}
+
+
+def build_session(source: Dict[str, Any] | None = None) -> requests.Session:
+    health_policy = _health_policy(source)
     session = requests.Session()
     retry = Retry(
-        total=3,
-        connect=3,
-        read=3,
-        status=3,
-        backoff_factor=0.5,
+        total=int(health_policy.get("retries") or 3),
+        connect=int(health_policy.get("retries") or 3),
+        read=int(health_policy.get("retries") or 3),
+        status=int(health_policy.get("retries") or 3),
+        backoff_factor=float(health_policy.get("backoff_seconds") or 0.5),
         allowed_methods=frozenset(["GET", "POST"]),
         status_forcelist=[429, 500, 502, 503, 504],
         raise_on_status=False,
@@ -26,7 +38,11 @@ def build_session() -> requests.Session:
     adapter = HTTPAdapter(max_retries=retry)
     session.mount("https://", adapter)
     session.mount("http://", adapter)
-    session.headers.update({"User-Agent": DEFAULT_USER_AGENT})
+    request_policy = _request_policy(source)
+    headers = {"User-Agent": request_policy.get("user_agent") or DEFAULT_USER_AGENT}
+    if request_policy.get("accept"):
+        headers["Accept"] = request_policy["accept"]
+    session.headers.update(headers)
     return session
 
 
@@ -34,16 +50,60 @@ def request(
     method: str,
     url: str,
     *,
+    source: Dict[str, Any] | None = None,
     session: requests.Session | None = None,
     timeout: int = DEFAULT_TIMEOUT,
     **kwargs: Any,
 ) -> requests.Response:
-    client = session or build_session()
+    request_policy = _request_policy(source)
+    health_policy = _health_policy(source)
+    client = session or build_session(source)
     headers = dict(kwargs.pop("headers", {}) or {})
     if "User-Agent" not in headers:
-        headers["User-Agent"] = DEFAULT_USER_AGENT
-    try:
-        return client.request(method, url, headers=headers, timeout=timeout, **kwargs)
-    except requests.exceptions.SSLError:
-        return client.request(method, url, headers=headers, timeout=timeout, verify=False, **kwargs)
+        headers["User-Agent"] = request_policy.get("user_agent") or DEFAULT_USER_AGENT
+    if request_policy.get("accept") and "Accept" not in headers:
+        headers["Accept"] = request_policy["accept"]
+    if request_policy.get("http_version") == "1.1" and "Connection" not in headers:
+        headers["Connection"] = "close"
+    timeout_value = timeout if timeout != DEFAULT_TIMEOUT else int(request_policy.get("timeout_seconds") or DEFAULT_TIMEOUT)
+    allow_redirects = kwargs.pop("allow_redirects", bool(request_policy.get("follow_redirects", True)))
+    verify = kwargs.pop("verify", bool(request_policy.get("verify_tls", True)))
+    status_retries = max(1, int(health_policy.get("retries") or 1))
+    backoff_seconds = float(health_policy.get("backoff_seconds") or 0.5)
 
+    last_error: Exception | None = None
+    for attempt in range(1, status_retries + 1):
+        try:
+            return client.request(
+                method,
+                url,
+                headers=headers,
+                timeout=timeout_value,
+                allow_redirects=allow_redirects,
+                verify=verify,
+                **kwargs,
+            )
+        except requests.exceptions.SSLError as exc:
+            last_error = exc
+            if verify:
+                try:
+                    return client.request(
+                        method,
+                        url,
+                        headers=headers,
+                        timeout=timeout_value,
+                        allow_redirects=allow_redirects,
+                        verify=False,
+                        **kwargs,
+                    )
+                except requests.exceptions.RequestException as fallback_error:
+                    last_error = fallback_error
+            if attempt < status_retries:
+                time.sleep(backoff_seconds * attempt)
+        except requests.exceptions.RequestException as exc:
+            last_error = exc
+            if attempt < status_retries:
+                time.sleep(backoff_seconds * attempt)
+    if last_error is not None:
+        raise last_error
+    raise RuntimeError(f"request failed without an exception for {method} {url}")