更新: 5 个文件 - 2026-03-18 09:50:04
这个提交包含在：
hao
@@ -12,12 +12,23 @@ if str(SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from intel.config import ADVISORIES_DIR, GENERATED_DIR, STATE_DIR, STATE_PATH, TRIAGE_DIR, load_source_map # noqa: E402
|
||||
from intel.monitoring import ( # noqa: E402
|
||||
build_alerts,
|
||||
build_source_health_snapshot,
|
||||
normalize_failures,
|
||||
read_previous_alerts,
|
||||
read_previous_source_health,
|
||||
write_alerts,
|
||||
write_monitoring_state,
|
||||
write_source_catalog_audit,
|
||||
write_source_health,
|
||||
)
|
||||
from intel.models import AdvisoryRecord # noqa: E402
|
||||
from intel.normalize import normalize_candidates # noqa: E402
|
||||
from intel.pr import open_pr # noqa: E402
|
||||
from intel.render import render_case_pages, render_generated, render_registry, render_secure_code, render_system_scaffolding # noqa: E402
|
||||
from intel.route import route_advisories # noqa: E402
|
||||
from intel.sources.runner import collect_candidates, probe_sources # noqa: E402
|
||||
from intel.sources.runner import build_failure, collect_candidates, failure_summary, find_source, probe_source, probe_sources # noqa: E402
|
||||
from intel.utils import isoformat, load_all_json, now_utc, parse_since, read_json, write_json # noqa: E402
|
||||
from intel.validators import validate # noqa: E402
|
||||
|
||||
@@ -131,6 +142,52 @@ def _write_outputs(
|
||||
render_generated(source_map, advisories, triage, failures, change_summary)
|
||||
|
||||
|
||||
def _refresh_render_state(
|
||||
full_source_map: Dict[str, Any],
|
||||
source_map: Dict[str, Any],
|
||||
) -> None:
|
||||
render_map, advisories, triage = _load_existing_selection(full_source_map, source_map)
|
||||
summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
|
||||
_write_outputs(render_map, advisories, triage, summary.get("failures", []), summary)
|
||||
|
||||
|
||||
def _retry_degraded_sources(
|
||||
source_map: Dict[str, Any],
|
||||
failures: List[Dict[str, Any]],
|
||||
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]:
|
||||
recovered_probes: List[Dict[str, Any]] = []
|
||||
remaining_failures: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
retries_performed = 0
|
||||
|
||||
for failure in normalize_failures(failures):
|
||||
key = (failure.get("system_id"), failure.get("source_name"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
match = find_source(source_map, failure.get("system_id", ""), failure.get("source_name", ""))
|
||||
if match is None:
|
||||
remaining_failures.append(failure)
|
||||
continue
|
||||
system, source = match
|
||||
if source.get("status") == "retired":
|
||||
continue
|
||||
retries_performed += 1
|
||||
try:
|
||||
result = probe_source(system, source)
|
||||
recovered_probes.append(
|
||||
{
|
||||
"system_id": system["system_id"],
|
||||
"source_name": source["name"],
|
||||
"source_kind": source["kind"],
|
||||
**result,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
remaining_failures.append(build_failure(system, source, exc))
|
||||
return recovered_probes, remaining_failures, retries_performed
|
||||
|
||||
|
||||
def pipeline(
|
||||
full_source_map: Dict[str, Any],
|
||||
source_map: Dict[str, Any],
|
||||
@@ -167,14 +224,33 @@ def cmd_render(args) -> int:
|
||||
def cmd_source_health(args) -> int:
|
||||
full_source_map = load_source_map()
|
||||
source_map = _filter_source_map(full_source_map, args.system)
|
||||
previous_source_health = read_previous_source_health()
|
||||
probes, failures = probe_sources(source_map, tier=args.tier)
|
||||
retried_probes, remaining_failures, retries_performed = _retry_degraded_sources(source_map, failures)
|
||||
if retried_probes:
|
||||
probe_map = {(item["system_id"], item["source_name"]): item for item in probes}
|
||||
for item in retried_probes:
|
||||
probe_map[(item["system_id"], item["source_name"])] = item
|
||||
probes = sorted(probe_map.values(), key=lambda item: (item["system_id"], item["source_name"]))
|
||||
else:
|
||||
remaining_failures = normalize_failures(failures)
|
||||
snapshot = build_source_health_snapshot(
|
||||
source_map,
|
||||
probes,
|
||||
remaining_failures,
|
||||
previous=previous_source_health,
|
||||
retries_performed=retries_performed,
|
||||
)
|
||||
write_source_health(snapshot)
|
||||
render_map, advisories, triage = _load_existing_selection(full_source_map, source_map)
|
||||
existing_summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
|
||||
render_generated(render_map, advisories, triage, failures, existing_summary)
|
||||
print(f"Source health checked {len(probes)} sources across {len(source_map['systems'])} systems; failures {len(failures)}")
|
||||
for failure in failures:
|
||||
print(f"- {failure}")
|
||||
return 0 if not failures else 1
|
||||
render_generated(render_map, advisories, triage, snapshot.get("failures", []), existing_summary)
|
||||
print(
|
||||
f"Source health checked {len(probes)} active sources across {len(source_map['systems'])} systems; failures {snapshot['failure_count']}; retries {retries_performed}"
|
||||
)
|
||||
for failure in snapshot["failures"]:
|
||||
print(f"- {failure_summary(failure)}")
|
||||
return 0 if not snapshot["failures"] else 1
|
||||
|
||||
|
||||
def cmd_validate(args) -> int:
|
||||
@@ -189,11 +265,12 @@ def cmd_validate(args) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _write_state(status: str) -> None:
|
||||
def _write_state(status: str, *, record_success: bool = True) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
state = read_json(STATE_PATH, default={}) or {}
|
||||
state["last_success"] = isoformat(now_utc())
|
||||
state["status"] = status
|
||||
if record_success:
|
||||
state["last_success"] = isoformat(now_utc())
|
||||
write_json(STATE_PATH, state)
|
||||
|
||||
|
||||
@@ -205,33 +282,33 @@ def cmd_ingest(args) -> int:
|
||||
state = read_json(STATE_PATH, default={}) or {}
|
||||
since = state.get("last_success", "30d")
|
||||
advisories, triage, failures, summary = pipeline(full_source_map, source_map, since, None, include_undated=False)
|
||||
_write_state("success")
|
||||
_write_state("success" if not failures else "degraded", record_success=not failures)
|
||||
print(
|
||||
f"Ingested {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
|
||||
)
|
||||
return 0
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
def cmd_hotlane(args) -> int:
|
||||
full_source_map = load_source_map()
|
||||
source_map = _filter_source_map(full_source_map, args.system)
|
||||
advisories, triage, failures, summary = pipeline(full_source_map, source_map, "1d", None, include_undated=False, hotlane_only=True)
|
||||
_write_state("success")
|
||||
_write_state("success" if not failures else "degraded", record_success=not failures)
|
||||
print(
|
||||
f"Hotlane synced {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
|
||||
)
|
||||
return 0
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
def cmd_reconcile(args) -> int:
|
||||
full_source_map = load_source_map()
|
||||
source_map = _filter_source_map(full_source_map, args.system)
|
||||
advisories, triage, failures, summary = pipeline(full_source_map, source_map, "30d", None, include_undated=False)
|
||||
_write_state("success")
|
||||
_write_state("success" if not failures else "degraded", record_success=not failures)
|
||||
print(
|
||||
f"Reconciled {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
|
||||
)
|
||||
return 0
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
def cmd_backfill(args) -> int:
|
||||
@@ -258,7 +335,73 @@ def cmd_backfill(args) -> int:
|
||||
print(
|
||||
f"Backfilled {len(advisories)} advisories, new {summary['new_count']}, updated {summary['updated_count']}, triage {len(triage)}, failures {len(failures)}"
|
||||
)
|
||||
return 0
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
def cmd_monitor(args) -> int:
|
||||
full_source_map = load_source_map()
|
||||
source_map = _filter_source_map(full_source_map, args.system)
|
||||
existing_run_summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
|
||||
previous_source_health = read_previous_source_health()
|
||||
previous_alerts = read_previous_alerts()
|
||||
bootstrap_failures = previous_source_health.get("failures") or existing_run_summary.get("failures", [])
|
||||
|
||||
audit = write_source_catalog_audit(source_map)
|
||||
|
||||
probes, failures = probe_sources(source_map)
|
||||
retried_probes, remaining_failures, retries_performed = _retry_degraded_sources(source_map, failures)
|
||||
if retried_probes:
|
||||
probe_map = {(item["system_id"], item["source_name"]): item for item in probes}
|
||||
for item in retried_probes:
|
||||
probe_map[(item["system_id"], item["source_name"])] = item
|
||||
probes = sorted(probe_map.values(), key=lambda item: (item["system_id"], item["source_name"]))
|
||||
else:
|
||||
remaining_failures = normalize_failures(failures)
|
||||
source_health = build_source_health_snapshot(
|
||||
source_map,
|
||||
probes,
|
||||
remaining_failures,
|
||||
previous=previous_source_health,
|
||||
retries_performed=retries_performed,
|
||||
)
|
||||
write_source_health(source_health)
|
||||
|
||||
state = read_json(STATE_PATH, default={}) or {}
|
||||
since = state.get("last_success", "30d")
|
||||
advisories, triage, ingest_failures, summary = pipeline(full_source_map, source_map, since, None, include_undated=False)
|
||||
alerts = build_alerts(
|
||||
source_health.get("failures", []),
|
||||
previous_alerts=previous_alerts,
|
||||
bootstrap_failures=bootstrap_failures,
|
||||
generated_at=source_health.get("generated_at"),
|
||||
)
|
||||
write_alerts(alerts)
|
||||
|
||||
validation_errors = validate(source_map)
|
||||
write_monitoring_state(
|
||||
audit=audit,
|
||||
source_health=source_health,
|
||||
alerts=alerts,
|
||||
ingest_summary={**summary, "failures": ingest_failures},
|
||||
validation_errors=validation_errors,
|
||||
)
|
||||
_refresh_render_state(full_source_map, source_map)
|
||||
|
||||
passed = not source_health.get("failures") and not ingest_failures and not validation_errors
|
||||
_write_state("success" if passed else "degraded", record_success=passed)
|
||||
print(
|
||||
"Monitor completed: "
|
||||
f"active_sources={source_health.get('active_source_count', 0)} "
|
||||
f"green_sources={source_health.get('green_source_count', 0)} "
|
||||
f"open_alerts={len([item for item in alerts if item.get('status') == 'open'])} "
|
||||
f"ingest_failures={len(ingest_failures)} "
|
||||
f"validation_errors={len(validation_errors)}"
|
||||
)
|
||||
for failure in source_health.get("failures", []):
|
||||
print(f"- {failure_summary(failure)}")
|
||||
for error in validation_errors:
|
||||
print(f"- validate::{error}")
|
||||
return 0 if passed else 1
|
||||
|
||||
|
||||
def cmd_open_pr(args) -> int:
|
||||
@@ -299,6 +442,10 @@ def main() -> int:
|
||||
source_health.add_argument("--system", action="append")
|
||||
source_health.set_defaults(func=cmd_source_health)
|
||||
|
||||
monitor = subparsers.add_parser("monitor", help="Run source audit, health, ingest, render and monitoring state persistence")
|
||||
monitor.add_argument("--system", action="append")
|
||||
monitor.set_defaults(func=cmd_monitor)
|
||||
|
||||
validate_parser = subparsers.add_parser("validate", help="Validate generated content")
|
||||
validate_parser.add_argument("--system", action="append")
|
||||
validate_parser.set_defaults(func=cmd_validate)
|
||||
|
||||
在新工单中引用
屏蔽一个用户