更新: 4 个文件 - 2026-03-21 17:54:57

scripts/intel/main.py

@@ -31,6 +31,7 @@ from intel.route import route_advisories  # noqa: E402
 from intel.sources.runner import build_failure, collect_candidates, failure_summary, find_source, probe_source, probe_sources  # noqa: E402
 from intel.utils import isoformat, load_all_json, now_utc, parse_since, read_json, write_json  # noqa: E402
 from intel.validators import validate  # noqa: E402
+from intel.versioning import discover_entities, sync_versions, write_entity_registry, write_version_registry  # noqa: E402
 
 
 def _load_existing_advisories() -> List[AdvisoryRecord]:
@@ -146,6 +147,35 @@ def _summarize_changes(advisories: List[AdvisoryRecord]) -> Dict[str, Any]:
     }
 
 
+def _selected_system_ids(source_map: Dict[str, Any]) -> set[str]:
+    return {system["system_id"] for system in source_map.get("systems", []) or []}
+
+
+def _apply_discovery_and_version_sync(
+    source_map: Dict[str, Any],
+    advisories: List[AdvisoryRecord],
+    *,
+    deep: bool = False,
+    enqueue_lab: bool = False,
+) -> tuple[List[AdvisoryRecord], Dict[str, Any], Dict[str, Any]]:
+    selected_system_ids = _selected_system_ids(source_map)
+    advisory_rows = [item.to_dict() for item in advisories]
+    discovery = discover_entities(source_map, advisory_rows, write_registry=False)
+    write_entity_registry(discovery["entities"], selected_system_ids=selected_system_ids)
+    version_state = sync_versions(
+        source_map,
+        advisory_rows,
+        entity_records=discovery["entities"],
+        deep=deep,
+        enqueue_lab=enqueue_lab,
+        write_registry=False,
+    )
+    write_entity_registry(version_state["entities"], selected_system_ids=selected_system_ids)
+    write_version_registry(version_state["versions"], selected_system_ids=selected_system_ids)
+    synced = route_advisories(source_map, [AdvisoryRecord(**item) for item in version_state["advisories"]])
+    return synced, discovery["summary"], version_state["summary"]
+
+
 def _select_hotlane(
     advisories: List[AdvisoryRecord],
     triage: List[Dict[str, Any]],
@@ -239,6 +269,8 @@ def pipeline(
     tier: str | None,
     include_undated: bool,
     hotlane_only: bool = False,
+    deep_version_sync: bool = False,
+    enqueue_lab: bool = False,
 ) -> tuple[list[AdvisoryRecord], list[Dict[str, Any]], list[str], Dict[str, Any]]:
     if tier == "history-full":
         since_dt = None
@@ -252,7 +284,16 @@ def pipeline(
     if hotlane_only:
         advisories, triage = _select_hotlane(advisories, triage)
     advisories, triage = _merge_existing_registry(advisories, triage)
+    advisories = route_advisories(source_map, advisories)
+    advisories, discovery_summary, version_summary = _apply_discovery_and_version_sync(
+        source_map,
+        advisories,
+        deep=deep_version_sync,
+        enqueue_lab=enqueue_lab,
+    )
     change_summary = _summarize_changes(advisories)
+    change_summary["auto_promoted_entity_count"] = discovery_summary.get("auto_promoted_count", 0)
+    change_summary["version_sync"] = version_summary
     render_map = source_map
     selected_system_ids = None
     if len(source_map["systems"]) != len(full_source_map["systems"]):
@@ -332,7 +373,14 @@ def cmd_ingest(args) -> int:
     if since == "last-success":
         state = read_json(STATE_PATH, default={}) or {}
         since = state.get("last_success", "30d")
-    advisories, triage, failures, summary = pipeline(full_source_map, source_map, since, None, include_undated=False)
+    advisories, triage, failures, summary = pipeline(
+        full_source_map,
+        source_map,
+        since,
+        None,
+        include_undated=False,
+        enqueue_lab=True,
+    )
     _write_state("success" if not failures else "degraded", record_success=not failures)
     print(
         f"Ingested {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
@@ -343,7 +391,15 @@ def cmd_ingest(args) -> int:
 def cmd_hotlane(args) -> int:
     full_source_map = load_source_map()
     source_map = _filter_source_map(full_source_map, args.system)
-    advisories, triage, failures, summary = pipeline(full_source_map, source_map, "1d", None, include_undated=False, hotlane_only=True)
+    advisories, triage, failures, summary = pipeline(
+        full_source_map,
+        source_map,
+        "1d",
+        None,
+        include_undated=False,
+        hotlane_only=True,
+        enqueue_lab=True,
+    )
     _write_state("success" if not failures else "degraded", record_success=not failures)
     print(
         f"Hotlane synced {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
@@ -354,7 +410,15 @@ def cmd_hotlane(args) -> int:
 def cmd_reconcile(args) -> int:
     full_source_map = load_source_map()
     source_map = _filter_source_map(full_source_map, args.system)
-    advisories, triage, failures, summary = pipeline(full_source_map, source_map, "30d", None, include_undated=False)
+    advisories, triage, failures, summary = pipeline(
+        full_source_map,
+        source_map,
+        "30d",
+        None,
+        include_undated=False,
+        deep_version_sync=True,
+        enqueue_lab=True,
+    )
     _write_state("success" if not failures else "degraded", record_success=not failures)
     print(
         f"Reconciled {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
@@ -382,6 +446,8 @@ def cmd_backfill(args) -> int:
         args.tier,
         include_undated=True,
         hotlane_only=args.hotlane_only,
+        deep_version_sync=args.tier == "history-full",
+        enqueue_lab=True,
     )
     print(
         f"Backfilled {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
@@ -389,6 +455,45 @@ def cmd_backfill(args) -> int:
     return 0 if not failures else 1
 
 
+def cmd_discover_entities(args) -> int:
+    full_source_map = load_source_map()
+    source_map = _filter_source_map(full_source_map, args.system)
+    advisories = [item for item in _load_existing_advisories() if item.system_id in _selected_system_ids(source_map)]
+    discovery = discover_entities(source_map, advisories, write_registry=False)
+    write_entity_registry(discovery["entities"], selected_system_ids=_selected_system_ids(source_map))
+    _refresh_render_state(full_source_map, source_map)
+    print(
+        f"Discovered cataloged_entities={discovery['summary'].get('cataloged_entity_total', 0)} "
+        f"candidate_backlog={discovery['summary'].get('candidate_entity_total', 0)} "
+        f"auto_promoted={discovery['summary'].get('auto_promoted_count', 0)}"
+    )
+    return 0
+
+
+def cmd_sync_versions(args) -> int:
+    full_source_map = load_source_map()
+    source_map = _filter_source_map(full_source_map, args.system)
+    selected_ids = _selected_system_ids(source_map)
+    advisories = [item for item in _load_existing_advisories() if item.system_id in selected_ids]
+    synced, discovery_summary, version_summary = _apply_discovery_and_version_sync(
+        source_map,
+        route_advisories(source_map, advisories),
+        deep=args.deep,
+        enqueue_lab=True,
+    )
+    _refresh_render_state(full_source_map, source_map)
+    print(
+        "Version sync completed: "
+        f"cataloged_entities={discovery_summary.get('cataloged_entity_total', 0)} "
+        f"auto_promoted={discovery_summary.get('auto_promoted_count', 0)} "
+        f"latest_synced={version_summary.get('latest_version_synced_count', 0)} "
+        f"source_gap={version_summary.get('source_gap_count', 0)} "
+        f"security_versions={version_summary.get('security_version_total', 0)} "
+        f"lab_enqueued={version_summary.get('lab_enqueued_count', 0)}"
+    )
+    return 0
+
+
 def cmd_monitor(args) -> int:
     full_source_map = load_source_map()
     source_map = _filter_source_map(full_source_map, args.system)
@@ -399,6 +504,9 @@ def cmd_monitor(args) -> int:
 
     audit = write_source_catalog_audit(source_map)
 
+    existing_advisories = [item for item in _load_existing_advisories() if item.system_id in _selected_system_ids(source_map)]
+    _apply_discovery_and_version_sync(source_map, route_advisories(source_map, existing_advisories), deep=False, enqueue_lab=False)
+
     probes, failures = probe_sources(source_map)
     retried_probes, remaining_failures, retries_performed = _retry_degraded_sources(source_map, failures)
     if retried_probes:
@@ -419,7 +527,14 @@ def cmd_monitor(args) -> int:
 
     state = read_json(STATE_PATH, default={}) or {}
     since = state.get("last_success", "30d")
-    advisories, triage, ingest_failures, summary = pipeline(full_source_map, source_map, since, None, include_undated=False)
+    advisories, triage, ingest_failures, summary = pipeline(
+        full_source_map,
+        source_map,
+        since,
+        None,
+        include_undated=False,
+        enqueue_lab=True,
+    )
     alerts = build_alerts(
         source_health.get("failures", []),
         previous_alerts=previous_alerts,
@@ -495,6 +610,15 @@ def main() -> int:
     render.add_argument("--system", action="append")
     render.set_defaults(func=cmd_render)
 
+    discover = subparsers.add_parser("discover-entities", help="Discover and auto-catalog stable security-related entities")
+    discover.add_argument("--system", action="append")
+    discover.set_defaults(func=cmd_discover_entities)
+
+    sync_versions_parser = subparsers.add_parser("sync-versions", help="Refresh latest versions and security-related version history")
+    sync_versions_parser.add_argument("--system", action="append")
+    sync_versions_parser.add_argument("--deep", action="store_true")
+    sync_versions_parser.set_defaults(func=cmd_sync_versions)
+
     source_health = subparsers.add_parser("source-health", help="Check source adapter health without mutating registry advisories")
     source_health.add_argument("--tier", choices=["history-full", "rolling-24m"])
     source_health.add_argument("--system", action="append")