From 054b24072d0a3cd8f90beec9cdc87250f6cbbe45 Mon Sep 17 00:00:00 2001 From: hao Date: Tue, 17 Mar 2026 21:30:02 -0700 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0:=2011=20=E4=B8=AA=E6=96=87?= =?UTF-8?q?=E4=BB=B6=20-=202026-03-17=2021:30:02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../generated/dashboard/architecture.json | 4 +- .../dashboard/data/completeness.json | 178 ++++++++++++++++++ .../dashboard/docs/architecture-library.html | 4 +- .../docs/testing-completeness-report.html | 11 +- .../generated/dashboard/summary.json | 2 +- 08-threat-intel/generated/latest-ingest.md | 14 +- 08-threat-intel/generated/run-summary.json | 12 +- docs/testing-completeness-report.md | 11 +- scripts/intel/main.py | 16 +- scripts/intel/sources/runner.py | 111 +++++++++++ scripts/lab/render.py | 4 +- 11 files changed, 307 insertions(+), 60 deletions(-) create mode 100644 08-threat-intel/generated/dashboard/data/completeness.json diff --git a/08-threat-intel/generated/dashboard/architecture.json b/08-threat-intel/generated/dashboard/architecture.json index 3c3bd45e..52515c8d 100644 --- a/08-threat-intel/generated/dashboard/architecture.json +++ b/08-threat-intel/generated/dashboard/architecture.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-18T04:06:37+00:00", + "generated_at": "2026-03-18T04:21:45+00:00", "title": "\u5f53\u524d\u67b6\u6784\u5e93", "summary": "\u5de5\u4f5c\u53f0\u3001\u63a7\u5236\u9762\u3001\u6570\u636e\u5c42\u3001\u6388\u6743\u8fb9\u754c\u4e0e\u7cfb\u7edf\u8986\u76d6\u7684\u5f53\u524d\u771f\u503c\u89c6\u56fe\u3002", "sections": [ @@ -49,7 +49,7 @@ }, { "label": "\u751f\u6210\u65f6\u95f4", - "value": "2026-03-18T04:06:37+00:00" + "value": "2026-03-18T04:21:45+00:00" } ], "links": [ diff --git a/08-threat-intel/generated/dashboard/data/completeness.json b/08-threat-intel/generated/dashboard/data/completeness.json new file mode 100644 index 00000000..fd995345 --- /dev/null +++ b/08-threat-intel/generated/dashboard/data/completeness.json @@ -0,0 +1,178 @@ +{ + "generated_at": "2026-03-18T04:21:45+00:00", + "advisory_total": 89, + "latest_statuses": { + "verified-real": 89 + }, + "historical_statuses": { + "verified-real": 136, + "blocked-artifact": 3, + "triage-manual": 1 + }, + "verified_real": 89, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0, + "verified_ratio": 100.0, + "complete": true, + "systems": [ + { + "system_id": "gitea", + "display_name": "Gitea", + "total": 37, + "verified_real": 37, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0, + "families": [ + { + "family": "authz-bypass", + "total": 3, + "verified_real": 3, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "file-upload", + "total": 2, + "verified_real": 2, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "proxy-boundary", + "total": 26, + "verified_real": 26, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "ssrf", + "total": 1, + "verified_real": 1, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "xss", + "total": 5, + "verified_real": 5, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + } + ] + }, + { + "system_id": "nextjs", + "display_name": "Next.js", + "total": 26, + "verified_real": 26, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0, + "families": [ + { + "family": "authz-bypass", + "total": 2, + "verified_real": 2, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "deserialization", + "total": 1, + "verified_real": 1, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "proxy-boundary", + "total": 19, + "verified_real": 19, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "ssrf", + "total": 2, + "verified_real": 2, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "xss", + "total": 2, + "verified_real": 2, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + } + ] + }, + { + "system_id": "undici", + "display_name": "Undici", + "total": 14, + "verified_real": 14, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0, + "families": [ + { + "family": "ssrf", + "total": 14, + "verified_real": 14, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + } + ] + }, + { + "system_id": "vite", + "display_name": "Vite", + "total": 12, + "verified_real": 12, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0, + "families": [ + { + "family": "proxy-boundary", + "total": 11, + "verified_real": 11, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + }, + { + "family": "xss", + "total": 1, + "verified_real": 1, + "verified_synthetic": 0, + "blocked": 0, + "manual": 0 + } + ] + } + ], + "ingest_health": { + "failure_count": 0, + "failures": [] + }, + "historical_blockers": [ + "Docker daemon unavailable caused provision-compose-environment blocked-artifact.", + "Family profiles previously used note-only attack runners and dry-run placeholders.", + "Baseline and browser steps were skipped when environment readiness was not enforced.", + "Latest completeness now uses one advisory -> latest run semantics instead of historical run piles." + ] +} diff --git a/08-threat-intel/generated/dashboard/docs/architecture-library.html b/08-threat-intel/generated/dashboard/docs/architecture-library.html index a4212054..6e3ba489 100644 --- a/08-threat-intel/generated/dashboard/docs/architecture-library.html +++ b/08-threat-intel/generated/dashboard/docs/architecture-library.html @@ -87,7 +87,7 @@

当前架构库镜像

工作台内置镜像页：当前架构库结构化数据镜像。
{
-  "generated_at": "2026-03-18T04:06:37+00:00",
+  "generated_at": "2026-03-18T04:21:45+00:00",
   "title": "当前架构库",
   "summary": "工作台、控制面、数据层、授权边界与系统覆盖的当前真值视图。",
   "sections": [
@@ -137,7 +137,7 @@
         },
         {
           "label": "生成时间",
-          "value": "2026-03-18T04:06:37+00:00"
+          "value": "2026-03-18T04:21:45+00:00"
         }
       ],
       "links": [
diff --git a/08-threat-intel/generated/dashboard/docs/testing-completeness-report.html b/08-threat-intel/generated/dashboard/docs/testing-completeness-report.html
index 8a2f2576..cce79a9f 100644
--- a/08-threat-intel/generated/dashboard/docs/testing-completeness-report.html
+++ b/08-threat-intel/generated/dashboard/docs/testing-completeness-report.html
@@ -88,7 +88,7 @@
       
工作台内置镜像页：89 条 advisory 最新完整度、family 矩阵与 ingest 健康度。

       
# 全库 Advisory 完整度报告
 
-- 生成时间: `2026-03-18T04:06:37+00:00`
+- 生成时间: `2026-03-18T04:21:45+00:00`
 - 最新 advisory 完整度: `89/89` `verified-real`
 - 合成验证数量: `0`
 - 阻塞数量: `0`
@@ -113,14 +113,7 @@
 
 ## Ingest / Source 健康度
 
-- source failures: `7`
-- wordpress::NVD WordPress::SSLError
-- wordpress::WPScan Vulnerability Database::SSLError
-- wordpress::PortSwigger Research::SSLError
-- magento-open-source::Magento GitHub Advisories::SSLError
-- nodejs::Node.js Security Releases::SSLError
-- nginx::NGINX Security Advisories::SSLError
-- gitea::GitHub Gitea Advisories::SSLError
+- source failures: `0`
 
 ## 剩余风险说明
 
diff --git a/08-threat-intel/generated/dashboard/summary.json b/08-threat-intel/generated/dashboard/summary.json
index 3ee12332..23c0c52e 100644
--- a/08-threat-intel/generated/dashboard/summary.json
+++ b/08-threat-intel/generated/dashboard/summary.json
@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-03-18T04:06:37+00:00",
+  "generated_at": "2026-03-18T04:21:45+00:00",
   "advisory_count": 89,
   "run_count": 140,
   "statuses": {
diff --git a/08-threat-intel/generated/latest-ingest.md b/08-threat-intel/generated/latest-ingest.md
index 1563761b..72ee017d 100644
--- a/08-threat-intel/generated/latest-ingest.md
+++ b/08-threat-intel/generated/latest-ingest.md
@@ -1,6 +1,6 @@
 # 最新同步摘要
 
-- 渲染时间: `2026-03-18T04:06:29+00:00`
+- 渲染时间: `2026-03-18T04:19:52+00:00`
 - 系统数量: `62`
 - Advisory 数量: `89`
 - 重点 Markdown 数量: `89`
@@ -8,14 +8,4 @@
 - 新增记录: `0`
 - 更新记录: `0`
 - Triage 数量: `0`
-- 失败的 source adapter: `7`
-
-## 失败列表
-
-- wordpress::NVD WordPress::SSLError
-- wordpress::WPScan Vulnerability Database::SSLError
-- wordpress::PortSwigger Research::SSLError
-- magento-open-source::Magento GitHub Advisories::SSLError
-- nodejs::Node.js Security Releases::SSLError
-- nginx::NGINX Security Advisories::SSLError
-- gitea::GitHub Gitea Advisories::SSLError
+- 失败的 source adapter: `0`
diff --git a/08-threat-intel/generated/run-summary.json b/08-threat-intel/generated/run-summary.json
index 00b5b565..eff7b5a7 100644
--- a/08-threat-intel/generated/run-summary.json
+++ b/08-threat-intel/generated/run-summary.json
@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-03-18T04:06:29+00:00",
+  "generated_at": "2026-03-18T04:19:52+00:00",
   "system_count": 62,
   "advisory_count": 89,
   "markdown_count": 89,
@@ -8,13 +8,5 @@
   "systems_touched": [],
   "triage_count": 0,
   "run_bundle_count": 89,
-  "failures": [
-    "wordpress::NVD WordPress::SSLError",
-    "wordpress::WPScan Vulnerability Database::SSLError",
-    "wordpress::PortSwigger Research::SSLError",
-    "magento-open-source::Magento GitHub Advisories::SSLError",
-    "nodejs::Node.js Security Releases::SSLError",
-    "nginx::NGINX Security Advisories::SSLError",
-    "gitea::GitHub Gitea Advisories::SSLError"
-  ]
+  "failures": []
 }
diff --git a/docs/testing-completeness-report.md b/docs/testing-completeness-report.md
index 5d4bfbed..31a09879 100644
--- a/docs/testing-completeness-report.md
+++ b/docs/testing-completeness-report.md
@@ -1,6 +1,6 @@
 # 全库 Advisory 完整度报告
 
-- 生成时间: `2026-03-18T04:06:37+00:00`
+- 生成时间: `2026-03-18T04:21:45+00:00`
 - 最新 advisory 完整度: `89/89` `verified-real`
 - 合成验证数量: `0`
 - 阻塞数量: `0`
@@ -25,14 +25,7 @@
 
 ## Ingest / Source 健康度
 
-- source failures: `7`
-- wordpress::NVD WordPress::SSLError
-- wordpress::WPScan Vulnerability Database::SSLError
-- wordpress::PortSwigger Research::SSLError
-- magento-open-source::Magento GitHub Advisories::SSLError
-- nodejs::Node.js Security Releases::SSLError
-- nginx::NGINX Security Advisories::SSLError
-- gitea::GitHub Gitea Advisories::SSLError
+- source failures: `0`
 
 ## 剩余风险说明
 
diff --git a/scripts/intel/main.py b/scripts/intel/main.py
index 1d68452d..d7645d3c 100644
--- a/scripts/intel/main.py
+++ b/scripts/intel/main.py
@@ -17,7 +17,7 @@ from intel.normalize import normalize_candidates  # noqa: E402
 from intel.pr import open_pr  # noqa: E402
 from intel.render import render_case_pages, render_generated, render_registry, render_secure_code, render_system_scaffolding  # noqa: E402
 from intel.route import route_advisories  # noqa: E402
-from intel.sources.runner import collect_candidates  # noqa: E402
+from intel.sources.runner import collect_candidates, probe_sources  # noqa: E402
 from intel.utils import isoformat, load_all_json, now_utc, parse_since, read_json, write_json  # noqa: E402
 from intel.validators import validate  # noqa: E402
 
@@ -167,19 +167,11 @@ def cmd_render(args) -> int:
 def cmd_source_health(args) -> int:
     full_source_map = load_source_map()
     source_map = _filter_source_map(full_source_map, args.system)
-    since_dt = parse_since(args.since, default_days=30)
-    candidates, failures = collect_candidates(
-        source_map,
-        since_dt=since_dt,
-        tier=args.tier,
-        include_undated=args.include_undated,
-    )
+    probes, failures = probe_sources(source_map, tier=args.tier)
     render_map, advisories, triage = _load_existing_selection(full_source_map, source_map)
     existing_summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
     render_generated(render_map, advisories, triage, failures, existing_summary)
-    print(
-        f"Source health checked {len(candidates)} candidates across {len(source_map['systems'])} systems; failures {len(failures)}"
-    )
+    print(f"Source health checked {len(probes)} sources across {len(source_map['systems'])} systems; failures {len(failures)}")
     for failure in failures:
         print(f"- {failure}")
     return 0 if not failures else 1
@@ -303,9 +295,7 @@ def main() -> int:
     render.set_defaults(func=cmd_render)
 
     source_health = subparsers.add_parser("source-health", help="Check source adapter health without mutating registry advisories")
-    source_health.add_argument("--since", default="30d")
     source_health.add_argument("--tier", choices=["history-full", "rolling-24m"])
-    source_health.add_argument("--include-undated", action="store_true")
     source_health.add_argument("--system", action="append")
     source_health.set_defaults(func=cmd_source_health)
 
diff --git a/scripts/intel/sources/runner.py b/scripts/intel/sources/runner.py
index a1f81bb9..1e115c2e 100644
--- a/scripts/intel/sources/runner.py
+++ b/scripts/intel/sources/runner.py
@@ -1,8 +1,14 @@
 from __future__ import annotations
 
+import os
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple
 
+import requests
+
+from intel.http_client import request
 from intel.models import Candidate
 from intel.utils import parse_dt
 
@@ -19,6 +25,76 @@ HANDLERS = {
 }
 
 
+def _probe_source(system: Dict[str, Any], source: Dict[str, Any]) -> Dict[str, Any]:
+    kind = source["kind"]
+    if kind == "ghsa-global":
+        headers = {"Accept": "application/vnd.github+json", "User-Agent": "websafe-intel"}
+        token = os.environ.get("GITHUB_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        response = request(
+            "GET",
+            github_global.API_URL,
+            headers=headers,
+            params={"per_page": 1, "page": 1, "ecosystem": source.get("ecosystem")},
+        )
+        if response.status_code == 403 and "rate limit" in response.text.lower():
+            raise requests.HTTPError("GitHub advisory rate limit exceeded; set GITHUB_TOKEN for higher quota", response=response)
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, list):
+            raise ValueError("GitHub advisory probe returned non-list payload")
+        return {"kind": kind, "items_seen": len(payload)}
+    if kind == "osv-batch":
+        packages = system.get("package_names", [])
+        if not packages:
+            return {"kind": kind, "items_seen": 0}
+        response = request(
+            "POST",
+            osv_api.QUERY_BATCH_URL,
+            json={"queries": [{"package": {"name": packages[0]["name"], "ecosystem": packages[0]["ecosystem"]}}]},
+            headers={"User-Agent": "websafe-intel"},
+        )
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("OSV probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("results", []))}
+    if kind == "kev-json":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("KEV probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("vulnerabilities", []))}
+    if kind == "nvd-search":
+        params = {
+            "keywordSearch": source.get("keyword") or system["display_name"],
+            "resultsPerPage": 1,
+        }
+        headers = {"User-Agent": "websafe-intel"}
+        api_key = os.environ.get("NVD_API_KEY")
+        if api_key:
+            headers["apiKey"] = api_key
+        response = request("GET", nvd_api.API_URL, headers=headers, params=params)
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict):
+            raise ValueError("NVD probe returned non-object payload")
+        return {"kind": kind, "items_seen": len(payload.get("vulnerabilities", []))}
+    if kind == "rss-feed":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        root = ET.fromstring(response.content)
+        return {"kind": kind, "items_seen": len(root.findall(".//item"))}
+    if kind == "html-links":
+        response = request("GET", source["url"])
+        response.raise_for_status()
+        html = response.text
+        return {"kind": kind, "items_seen": len(html_links.ANCHOR_RE.findall(html))}
+    raise ValueError(f"Unsupported source kind {kind}")
+
+
 def _passes_since(candidate: Candidate, since_dt: Optional[datetime], include_undated: bool) -> bool:
     if since_dt is None:
         return True
@@ -55,3 +131,38 @@ def collect_candidates(
                 except Exception as exc:
                     failures.append(f"{system['system_id']}::{source['name']}::{exc.__class__.__name__}")
     return all_candidates, failures
+
+
+def probe_sources(
+    source_map: Dict[str, Any],
+    tier: Optional[str] = None,
+) -> Tuple[List[Dict[str, Any]], List[str]]:
+    jobs: List[Tuple[Dict[str, Any], Dict[str, Any]]] = []
+    probes: List[Dict[str, Any]] = []
+    failures: List[str] = []
+
+    for system in source_map["systems"]:
+        if tier and system.get("tier") != tier:
+            continue
+        for bucket_name in ("official_sources", "ecosystem_sources", "research_sources"):
+            for source in system.get(bucket_name, []):
+                jobs.append((system, source))
+
+    max_workers = min(16, max(4, len(jobs) or 1))
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_map = {executor.submit(_probe_source, system, source): (system, source) for system, source in jobs}
+        for future in as_completed(future_map):
+            system, source = future_map[future]
+            try:
+                result = future.result()
+                probes.append(
+                    {
+                        "system_id": system["system_id"],
+                        "source_name": source["name"],
+                        "source_kind": source["kind"],
+                        **result,
+                    }
+                )
+            except Exception as exc:
+                failures.append(f"{system['system_id']}::{source['name']}::{exc.__class__.__name__}")
+    return probes, failures
diff --git a/scripts/lab/render.py b/scripts/lab/render.py
index ebaf3571..e00eb1e4 100644
--- a/scripts/lab/render.py
+++ b/scripts/lab/render.py
@@ -941,8 +941,8 @@ def _render_section_dashboard_shells() -> None:
     source_index = LOVART_TEMPLATE_DIR / "index.html"
     for section in SECTION_ROUTE_DIRS:
         section_dir = DASHBOARD_DIR / section
-        if section == "runs":
-            # Preserve existing /runs// bundles; only refresh the section shell.
+        if section in {"runs", "data"}:
+            # Preserve existing section artifacts; only refresh the shell entrypoint.
             ensure_dir(section_dir)
             index_path = section_dir / "index.html"
             if index_path.exists():