更新: 11 个文件 - 2026-03-17 21:30:02

scripts/intel/main.py

@@ -17,7 +17,7 @@ from intel.normalize import normalize_candidates  # noqa: E402
 from intel.pr import open_pr  # noqa: E402
 from intel.render import render_case_pages, render_generated, render_registry, render_secure_code, render_system_scaffolding  # noqa: E402
 from intel.route import route_advisories  # noqa: E402
-from intel.sources.runner import collect_candidates  # noqa: E402
+from intel.sources.runner import collect_candidates, probe_sources  # noqa: E402
 from intel.utils import isoformat, load_all_json, now_utc, parse_since, read_json, write_json  # noqa: E402
 from intel.validators import validate  # noqa: E402
 
@@ -167,19 +167,11 @@ def cmd_render(args) -> int:
 def cmd_source_health(args) -> int:
     full_source_map = load_source_map()
     source_map = _filter_source_map(full_source_map, args.system)
-    since_dt = parse_since(args.since, default_days=30)
-    candidates, failures = collect_candidates(
-        source_map,
-        since_dt=since_dt,
-        tier=args.tier,
-        include_undated=args.include_undated,
-    )
+    probes, failures = probe_sources(source_map, tier=args.tier)
     render_map, advisories, triage = _load_existing_selection(full_source_map, source_map)
     existing_summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
     render_generated(render_map, advisories, triage, failures, existing_summary)
-    print(
-        f"Source health checked {len(candidates)} candidates across {len(source_map['systems'])} systems; failures {len(failures)}"
-    )
+    print(f"Source health checked {len(probes)} sources across {len(source_map['systems'])} systems; failures {len(failures)}")
     for failure in failures:
         print(f"- {failure}")
     return 0 if not failures else 1
@@ -303,9 +295,7 @@ def main() -> int:
     render.set_defaults(func=cmd_render)
 
     source_health = subparsers.add_parser("source-health", help="Check source adapter health without mutating registry advisories")
-    source_health.add_argument("--since", default="30d")
     source_health.add_argument("--tier", choices=["history-full", "rolling-24m"])
-    source_health.add_argument("--include-undated", action="store_true")
     source_health.add_argument("--system", action="append")
     source_health.set_defaults(func=cmd_source_health)
 

scripts/intel/sources/runner.py

@@ -1,8 +1,14 @@
 from __future__ import annotations
 
+import os
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple
 
+import requests
+
+from intel.http_client import request
 from intel.models import Candidate
 from intel.utils import parse_dt
 
@@ -19,6 +25,76 @@ HANDLERS = {
 }
 
 
+def _probe_source(system: Dict[str, Any], source: Dict[str, Any]) -> Dict[str, Any]:
+    kind = source["kind"]
+    if kind == "ghsa-global":
+        headers = {"Accept": "application/vnd.github+json", "User-Agent": "websafe-intel"}
+        token = os.environ.get("GITHUB_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        response = request(
+            "GET",
+            github_global.API_URL,
+            headers=headers,
+            params={"per_page": 1, "page": 1, "ecosystem": source.get("ecosystem")},
+        )
+        if response.status_code == 403 and "rate limit" in response.text.lower():
+            raise requests.HTTPError("GitHub advisory rate limit exceeded; set GITHUB_TOKEN for higher quota", response=response)
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, list):
+            raise ValueError("GitHub advisory probe returned non-list payload")
+        return {"kind": kind, "items_seen": len(payload)}
+    if kind == "osv-batch":
+        packages = system.get("package_names", [])
+        if not packages:
+            return {"kind": kind, "items_seen": 0}
+        response = request(
+            "POST",
+            osv_api.QUERY_BATCH_URL,
+            json={"queries": [{"package": {"name": packages[0]["name"], "ecosystem": packages[0]["ecosystem"]}}]},
+            headers={"User-Agent": "websafe-intel"},
+        )
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("OSV probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("results", []))}
+    if kind == "kev-json":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("KEV probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("vulnerabilities", []))}
+    if kind == "nvd-search":
+        params = {
+            "keywordSearch": source.get("keyword") or system["display_name"],
+            "resultsPerPage": 1,
+        }
+        headers = {"User-Agent": "websafe-intel"}
+        api_key = os.environ.get("NVD_API_KEY")
+        if api_key:
+            headers["apiKey"] = api_key
+        response = request("GET", nvd_api.API_URL, headers=headers, params=params)
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("NVD probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("vulnerabilities", []))}
+    if kind == "rss-feed":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        root = ET.fromstring(response.content)
+        return {"kind": kind, "items_seen": len(root.findall(".//item"))}
+    if kind == "html-links":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        html = response.text
+        return {"kind": kind, "items_seen": len(html_links.ANCHOR_RE.findall(html))}
+    raise ValueError(f"Unsupported source kind {kind}")
+
+
 def _passes_since(candidate: Candidate, since_dt: Optional[datetime], include_undated: bool) -> bool:
     if since_dt is None:
         return True
@@ -55,3 +131,38 @@ def collect_candidates(
                 except Exception as exc:
                     failures.append(f"{system['system_id']}::{source['name']}::{exc.__class__.__name__}")
     return all_candidates, failures
+
+
+def probe_sources(
+    source_map: Dict[str, Any],
+    tier: Optional[str] = None,
+) -> Tuple[List[Dict[str, Any]], List[str]]:
+    jobs: List[Tuple[Dict[str, Any], Dict[str, Any]]] = []
+    probes: List[Dict[str, Any]] = []
+    failures: List[str] = []
+
+    for system in source_map["systems"]:
+        if tier and system.get("tier") != tier:
+            continue
+        for bucket_name in ("official_sources", "ecosystem_sources", "research_sources"):
+            for source in system.get(bucket_name, []):
+                jobs.append((system, source))
+
+    max_workers = min(16, max(4, len(jobs) or 1))
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_map = {executor.submit(_probe_source, system, source): (system, source) for system, source in jobs}
+        for future in as_completed(future_map):
+            system, source = future_map[future]
+            try:
+                result = future.result()
+                probes.append(
+                    {
+                        "system_id": system["system_id"],
+                        "source_name": source["name"],
+                        "source_kind": source["kind"],
+                        **result,
+                    }
+                )
+            except Exception as exc:
+                failures.append(f"{system['system_id']}::{source['name']}::{exc.__class__.__name__}")
+    return probes, failures

scripts/lab/render.py

@@ -941,8 +941,8 @@ def _render_section_dashboard_shells() -> None:
     source_index = LOVART_TEMPLATE_DIR / "index.html"
     for section in SECTION_ROUTE_DIRS:
         section_dir = DASHBOARD_DIR / section
-        if section == "runs":
-            # Preserve existing /runs/<run-id>/ bundles; only refresh the section shell.
+        if section in {"runs", "data"}:
+            # Preserve existing section artifacts; only refresh the shell entrypoint.
             ensure_dir(section_dir)
             index_path = section_dir / "index.html"
             if index_path.exists():