更新: 109 个文件 - 2026-03-18 10:55:52
这个提交包含在:
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PLIST_SOURCE="/Users/x/websafe/ops/launchd/com.hao.websafe.intel-monitor.plist"
|
||||
PLIST_TARGET="$HOME/Library/LaunchAgents/com.hao.websafe.intel-monitor.plist"
|
||||
LABEL="com.hao.websafe.intel-monitor"
|
||||
GUI_DOMAIN="gui/$(id -u)"
|
||||
|
||||
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
|
||||
cp "$PLIST_SOURCE" "$PLIST_TARGET"
|
||||
|
||||
launchctl bootout "$GUI_DOMAIN" "$PLIST_TARGET" >/dev/null 2>&1 || true
|
||||
launchctl bootstrap "$GUI_DOMAIN" "$PLIST_TARGET"
|
||||
launchctl enable "$GUI_DOMAIN/$LABEL"
|
||||
|
||||
echo "Installed $LABEL"
|
||||
echo "Plist: $PLIST_TARGET"
|
||||
@@ -33,7 +33,7 @@ SOURCE_BUCKETS = ("official_sources", "ecosystem_sources", "research_sources")
|
||||
MACHINE_READABLE_SOURCE_KINDS = {"ghsa-global", "osv-batch", "nvd-search", "kev-json", "json-feed", "rss-feed", "atom-feed"}
|
||||
|
||||
DEFAULT_REQUEST_POLICY = {
|
||||
"user_agent": "websafe-intel",
|
||||
"user_agent": "python-requests/2.31.0",
|
||||
"accept": "",
|
||||
"timeout_seconds": 30,
|
||||
"verify_tls": True,
|
||||
|
||||
@@ -4,14 +4,12 @@ import time
|
||||
from typing import Any, Dict
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from intel.config import DEFAULT_HEALTH_POLICY, DEFAULT_REQUEST_POLICY
|
||||
|
||||
|
||||
DEFAULT_TIMEOUT = 30
|
||||
DEFAULT_USER_AGENT = "websafe-intel"
|
||||
DEFAULT_USER_AGENT = "python-requests/2.31.0"
|
||||
|
||||
|
||||
def _request_policy(source: Dict[str, Any] | None = None) -> Dict[str, Any]:
|
||||
@@ -23,21 +21,8 @@ def _health_policy(source: Dict[str, Any] | None = None) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def build_session(source: Dict[str, Any] | None = None) -> requests.Session:
|
||||
health_policy = _health_policy(source)
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=int(health_policy.get("retries") or 3),
|
||||
connect=int(health_policy.get("retries") or 3),
|
||||
read=int(health_policy.get("retries") or 3),
|
||||
status=int(health_policy.get("retries") or 3),
|
||||
backoff_factor=float(health_policy.get("backoff_seconds") or 0.5),
|
||||
allowed_methods=frozenset(["GET", "POST"]),
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
session.trust_env = True
|
||||
request_policy = _request_policy(source)
|
||||
headers = {"User-Agent": request_policy.get("user_agent") or DEFAULT_USER_AGENT}
|
||||
if request_policy.get("accept"):
|
||||
@@ -63,8 +48,6 @@ def request(
|
||||
headers["User-Agent"] = request_policy.get("user_agent") or DEFAULT_USER_AGENT
|
||||
if request_policy.get("accept") and "Accept" not in headers:
|
||||
headers["Accept"] = request_policy["accept"]
|
||||
if request_policy.get("http_version") == "1.1" and "Connection" not in headers:
|
||||
headers["Connection"] = "close"
|
||||
timeout_value = timeout if timeout != DEFAULT_TIMEOUT else int(request_policy.get("timeout_seconds") or DEFAULT_TIMEOUT)
|
||||
allow_redirects = kwargs.pop("allow_redirects", bool(request_policy.get("follow_redirects", True)))
|
||||
verify = kwargs.pop("verify", bool(request_policy.get("verify_tls", True)))
|
||||
|
||||
@@ -242,9 +242,6 @@ def cmd_source_health(args) -> int:
|
||||
retries_performed=retries_performed,
|
||||
)
|
||||
write_source_health(snapshot)
|
||||
render_map, advisories, triage = _load_existing_selection(full_source_map, source_map)
|
||||
existing_summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
|
||||
render_generated(render_map, advisories, triage, snapshot.get("failures", []), existing_summary)
|
||||
print(
|
||||
f"Source health checked {len(probes)} active sources across {len(source_map['systems'])} systems; failures {snapshot['failure_count']}; retries {retries_performed}"
|
||||
)
|
||||
@@ -377,6 +374,14 @@ def cmd_monitor(args) -> int:
|
||||
)
|
||||
write_alerts(alerts)
|
||||
|
||||
write_monitoring_state(
|
||||
audit=audit,
|
||||
source_health=source_health,
|
||||
alerts=alerts,
|
||||
ingest_summary={**summary, "failures": ingest_failures},
|
||||
validation_errors=[],
|
||||
)
|
||||
_refresh_render_state(full_source_map, source_map)
|
||||
validation_errors = validate(source_map)
|
||||
write_monitoring_state(
|
||||
audit=audit,
|
||||
@@ -385,7 +390,6 @@ def cmd_monitor(args) -> int:
|
||||
ingest_summary={**summary, "failures": ingest_failures},
|
||||
validation_errors=validation_errors,
|
||||
)
|
||||
_refresh_render_state(full_source_map, source_map)
|
||||
|
||||
passed = not source_health.get("failures") and not ingest_failures and not validation_errors
|
||||
_write_state("success" if passed else "degraded", record_success=passed)
|
||||
|
||||
@@ -313,8 +313,8 @@ def _prune_monitoring_history(now_value: str) -> None:
|
||||
return
|
||||
cutoff = current_dt - timedelta(days=90)
|
||||
for path in sorted(MONITORING_DIR.glob("*.json")):
|
||||
stem = path.stem.replace("-", ":", 2)
|
||||
snapshot_dt = parse_dt(stem)
|
||||
snapshot = read_json(path, default={}) or {}
|
||||
snapshot_dt = parse_dt(snapshot.get("generated_at"))
|
||||
if snapshot_dt is None:
|
||||
continue
|
||||
if snapshot_dt < cutoff:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
@@ -11,6 +13,30 @@ from intel.utils import unique
|
||||
|
||||
|
||||
API_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0"
|
||||
PUBLIC_INTERVAL_SECONDS = 7.0
|
||||
_NVD_RATE_LOCK = threading.Lock()
|
||||
_NVD_LAST_REQUEST = 0.0
|
||||
|
||||
|
||||
def _wait_for_slot() -> None:
|
||||
global _NVD_LAST_REQUEST
|
||||
if os.environ.get("NVD_API_KEY"):
|
||||
return
|
||||
with _NVD_RATE_LOCK:
|
||||
elapsed = time.monotonic() - _NVD_LAST_REQUEST
|
||||
if elapsed < PUBLIC_INTERVAL_SECONDS:
|
||||
time.sleep(PUBLIC_INTERVAL_SECONDS - elapsed)
|
||||
_NVD_LAST_REQUEST = time.monotonic()
|
||||
|
||||
|
||||
def request_nvd(source: Dict[str, Any], headers: Dict[str, Any], params: Dict[str, Any]) -> requests.Response:
|
||||
_wait_for_slot()
|
||||
response = request("GET", API_URL, source=source, headers=headers, params=params)
|
||||
if response.status_code == 429 and not os.environ.get("NVD_API_KEY"):
|
||||
time.sleep(PUBLIC_INTERVAL_SECONDS)
|
||||
_wait_for_slot()
|
||||
response = request("GET", API_URL, source=source, headers=headers, params=params)
|
||||
return response
|
||||
|
||||
|
||||
def fetch(system: Dict[str, Any], source: Dict[str, Any]) -> List[Candidate]:
|
||||
@@ -23,7 +49,7 @@ def fetch(system: Dict[str, Any], source: Dict[str, Any]) -> List[Candidate]:
|
||||
if api_key:
|
||||
headers["apiKey"] = api_key
|
||||
|
||||
response = request("GET", API_URL, source=source, headers=headers, params=params)
|
||||
response = request_nvd(source, headers, params)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ def probe_source(system: Dict[str, Any], source: Dict[str, Any]) -> Dict[str, An
|
||||
api_key = os.environ.get("NVD_API_KEY")
|
||||
if api_key:
|
||||
headers["apiKey"] = api_key
|
||||
response = request("GET", nvd_api.API_URL, source=source, headers=headers, params=params)
|
||||
response = nvd_api.request_nvd(source, headers, params)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
@@ -160,7 +160,7 @@ def probe_source(system: Dict[str, Any], source: Dict[str, Any]) -> Dict[str, An
|
||||
response = request("GET", source["url"], source=source)
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
return {"kind": kind, "items_seen": len(html_links.ANCHOR_RE.findall(html))}
|
||||
return {"kind": kind, "items_seen": len(vendor_index.extract_links(html))}
|
||||
raise ValueError(f"Unsupported source kind {kind}")
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import urljoin
|
||||
|
||||
@@ -9,7 +10,42 @@ from intel.http_client import request
|
||||
from intel.models import Candidate
|
||||
from intel.utils import unique
|
||||
|
||||
from .html_links import ANCHOR_RE, TAG_RE, canonicalize_url
|
||||
from .html_links import canonicalize_url
|
||||
|
||||
|
||||
class _AnchorCollector(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.links: List[tuple[str, str]] = []
|
||||
self._href: str | None = None
|
||||
self._chunks: List[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs) -> None:
|
||||
if tag.lower() != "a":
|
||||
return
|
||||
href = dict(attrs).get("href")
|
||||
if href:
|
||||
self._href = href
|
||||
self._chunks = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._href is not None:
|
||||
self._chunks.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() != "a" or self._href is None:
|
||||
return
|
||||
text = unescape(" ".join(self._chunks)).strip()
|
||||
self.links.append((self._href, text))
|
||||
self._href = None
|
||||
self._chunks = []
|
||||
|
||||
|
||||
def extract_links(html: str) -> List[tuple[str, str]]:
|
||||
parser = _AnchorCollector()
|
||||
parser.feed(html)
|
||||
parser.close()
|
||||
return parser.links
|
||||
|
||||
|
||||
def _matches(value: str, patterns: List[str]) -> bool:
|
||||
@@ -29,9 +65,9 @@ def fetch(system: Dict[str, Any], source: Dict[str, Any]) -> List[Candidate]:
|
||||
|
||||
candidates: List[Candidate] = []
|
||||
seen = set()
|
||||
for href, text in ANCHOR_RE.findall(html):
|
||||
for href, text in extract_links(html):
|
||||
absolute = canonicalize_url(urljoin(source["url"], href))
|
||||
title = unescape(TAG_RE.sub(" ", text)).strip()
|
||||
title = unescape(text).strip()
|
||||
if not title:
|
||||
continue
|
||||
haystack = " ".join(filter(None, [absolute, title])).lower()
|
||||
|
||||
@@ -34,6 +34,18 @@ REQUIRED_SYSTEM_FIELDS = {
|
||||
"render_policy",
|
||||
}
|
||||
|
||||
REQUIRED_SOURCE_FIELDS = {
|
||||
"name",
|
||||
"kind",
|
||||
"confidence",
|
||||
"status",
|
||||
"retired_reason",
|
||||
"replacement_sources",
|
||||
"request_policy",
|
||||
"health_policy",
|
||||
"parser_hints",
|
||||
}
|
||||
|
||||
FORBIDDEN_RUNTIME_PATTERNS = [
|
||||
"assets-persist.lovart.ai",
|
||||
"cdnjs.cloudflare.com",
|
||||
@@ -73,6 +85,11 @@ def validate(source_map: Dict[str, Any]) -> List[str]:
|
||||
errors.append(f"system INDEX missing: {system_root / 'INDEX.md'}")
|
||||
if not (SYSTEMS_DIR / f"{system_id}.json").exists():
|
||||
errors.append(f"system registry summary missing: {SYSTEMS_DIR / f'{system_id}.json'}")
|
||||
for bucket_name in ("official_sources", "ecosystem_sources", "research_sources"):
|
||||
for source in system.get(bucket_name, []):
|
||||
missing_source_fields = REQUIRED_SOURCE_FIELDS - set(source.keys())
|
||||
if missing_source_fields:
|
||||
errors.append(f"source missing required fields: {system_id}/{source.get('name', 'unknown')} -> {sorted(missing_source_fields)}")
|
||||
|
||||
if not (FRAMEWORK_ROOT / "README.md").exists():
|
||||
errors.append(f"framework root README missing: {FRAMEWORK_ROOT / 'README.md'}")
|
||||
@@ -89,6 +106,12 @@ def validate(source_map: Dict[str, Any]) -> List[str]:
|
||||
GENERATED_DIR / "coverage-matrix.md",
|
||||
GENERATED_DIR / "latest-ingest.md",
|
||||
GENERATED_DIR / "run-summary.json",
|
||||
GENERATED_DIR / "source-health.json",
|
||||
GENERATED_DIR / "alerts.json",
|
||||
GENERATED_DIR / "monitor-summary.json",
|
||||
GENERATED_DIR / "source-catalog-audit.json",
|
||||
GENERATED_DIR / "source-catalog-audit.md",
|
||||
GENERATED_DIR / "retired-sources.json",
|
||||
GENERATED_DIR / "dashboard" / "index.html",
|
||||
GENERATED_DIR / "dashboard" / "overview" / "index.html",
|
||||
GENERATED_DIR / "dashboard" / "runs" / "index.html",
|
||||
@@ -115,17 +138,27 @@ def validate(source_map: Dict[str, Any]) -> List[str]:
|
||||
GENERATED_DIR / "dashboard" / "docs" / "root-readme.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "authorization-model.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "source-map.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "source-catalog-audit.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "retired-sources.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "repro-map.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "coverage-matrix.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "design-source.html",
|
||||
GENERATED_DIR / "dashboard" / "docs" / "architecture-library.html",
|
||||
GENERATED_DIR / "dashboard" / "data" / "completeness.json",
|
||||
GENERATED_DIR / "dashboard" / "data" / "source-health.json",
|
||||
GENERATED_DIR / "dashboard" / "data" / "alerts.json",
|
||||
GENERATED_DIR / "dashboard" / "data" / "monitor-summary.json",
|
||||
GENERATED_DIR / "dashboard" / "data" / "source-catalog-audit.json",
|
||||
ROOT / "docs" / "testing-completeness-report.md",
|
||||
ROOT / "08-threat-intel" / "registry" / "source-confidence.md",
|
||||
]:
|
||||
if not path.exists():
|
||||
errors.append(f"generated artifact missing: {path}")
|
||||
|
||||
monitoring_files = sorted((REGISTRY_ROOT / "monitoring").glob("*.json"))
|
||||
if not monitoring_files:
|
||||
errors.append(f"monitoring history missing: {REGISTRY_ROOT / 'monitoring'}")
|
||||
|
||||
runtime_files = [
|
||||
GENERATED_DIR / "dashboard" / "index.html",
|
||||
GENERATED_DIR / "dashboard" / "overview" / "index.html",
|
||||
|
||||
@@ -774,6 +774,7 @@ function renderPanel(panelKey, title, meta, iconName, content) {
|
||||
|
||||
function renderCompletenessPanel(panelKey, compact = false) {
|
||||
const completeness = state.completeness || state.summary?.completeness || {};
|
||||
const sourceHealth = state.sourceHealth || completeness.source_health || {};
|
||||
const systems = (state.completeness?.systems || []).map((system) => `
|
||||
<article class="plan-card">
|
||||
<span class="plan-label">${escapeHtml(system.system_id)}</span>
|
||||
@@ -807,12 +808,21 @@ function renderCompletenessPanel(panelKey, compact = false) {
|
||||
<strong>ingest failures</strong>
|
||||
<span>${escapeHtml(state.completeness?.ingest_health?.failure_count || 0)}</span>
|
||||
</article>
|
||||
<article class="detail-stat">
|
||||
<strong>active sources</strong>
|
||||
<span>${escapeHtml(sourceHealth.active_source_count || 0)}</span>
|
||||
</article>
|
||||
<article class="detail-stat">
|
||||
<strong>open alerts</strong>
|
||||
<span>${escapeHtml(sourceHealth.open_alert_count || 0)}</span>
|
||||
</article>
|
||||
</div>
|
||||
<div class="plan-grid" style="margin-top:16px;">${systems || `<div class="empty-state">暂无系统完整度数据。</div>`}</div>
|
||||
${compact ? "" : `
|
||||
<div class="detail-actions" style="margin-top:16px;">
|
||||
<a class="button button-secondary" href="/docs/testing-completeness-report.html" target="_blank" rel="noreferrer">${icon("docs")}<span>打开中文报告</span></a>
|
||||
<a class="button button-secondary" href="/data/completeness.json" target="_blank" rel="noreferrer">${icon("json")}<span>打开 completeness.json</span></a>
|
||||
<a class="button button-secondary" href="/data/source-health.json" target="_blank" rel="noreferrer">${icon("json")}<span>打开 source-health.json</span></a>
|
||||
</div>
|
||||
${failures.length ? `<div class="callout" style="margin-top:16px;"><strong>Ingest 未清零</strong><div class="plan-copy">${escapeHtml(failures.join(" | "))}</div></div>` : ""}
|
||||
`}
|
||||
@@ -820,6 +830,66 @@ function renderCompletenessPanel(panelKey, compact = false) {
|
||||
);
|
||||
}
|
||||
|
||||
function renderSourceHealthPanel(panelKey, compact = false) {
|
||||
const sourceHealth = state.sourceHealth || {};
|
||||
const alerts = state.alerts || [];
|
||||
const failures = (sourceHealth.failures || []).slice(0, 6);
|
||||
const openAlertItems = alerts.filter((item) => item.status === "open");
|
||||
const openAlerts = openAlertItems.slice(0, 6);
|
||||
const failureCards = failures.length
|
||||
? failures.map((item) => `
|
||||
<article class="plan-card">
|
||||
<span class="plan-label">${escapeHtml(item.system_id || "-")} · ${escapeHtml(item.source_name || "-")}</span>
|
||||
<div class="plan-copy">${escapeHtml(item.category || "unknown")} · ${escapeHtml(item.message || item.summary || "-")}</div>
|
||||
</article>
|
||||
`).join("")
|
||||
: `<div class="empty-state">当前 active source 集合全绿。</div>`;
|
||||
const alertCards = openAlerts.length
|
||||
? openAlerts.map((item) => `
|
||||
<article class="plan-card">
|
||||
<span class="plan-label">${escapeHtml(item.system_id || "-")} · ${escapeHtml(item.source_name || "-")}</span>
|
||||
<div class="plan-copy">streak ${escapeHtml(item.failure_streak || 0)} · ${escapeHtml(item.last_category || "-")}</div>
|
||||
</article>
|
||||
`).join("")
|
||||
: `<div class="empty-state">当前没有 open alert。</div>`;
|
||||
return renderPanel(
|
||||
panelKey,
|
||||
"Source Health 与告警",
|
||||
`${escapeHtml(sourceHealth.green_source_count || 0)}/${escapeHtml(sourceHealth.active_source_count || 0)}`,
|
||||
"shield",
|
||||
`
|
||||
<div class="detail-stat-grid">
|
||||
<article class="detail-stat">
|
||||
<strong>green</strong>
|
||||
<span>${escapeHtml(sourceHealth.green_source_count || 0)}</span>
|
||||
</article>
|
||||
<article class="detail-stat">
|
||||
<strong>failures</strong>
|
||||
<span>${escapeHtml(sourceHealth.failure_count || 0)}</span>
|
||||
</article>
|
||||
<article class="detail-stat">
|
||||
<strong>open alerts</strong>
|
||||
<span>${escapeHtml(openAlertItems.length)}</span>
|
||||
</article>
|
||||
<article class="detail-stat">
|
||||
<strong>last fully green</strong>
|
||||
<span>${escapeHtml(sourceHealth.last_fully_green_run ? formatDateTime(sourceHealth.last_fully_green_run) : "-")}</span>
|
||||
</article>
|
||||
</div>
|
||||
${compact ? "" : `
|
||||
<div class="detail-actions" style="margin-top:16px;">
|
||||
<a class="button button-secondary" href="/data/source-health.json" target="_blank" rel="noreferrer">${icon("json")}<span>source-health.json</span></a>
|
||||
<a class="button button-secondary" href="/data/alerts.json" target="_blank" rel="noreferrer">${icon("json")}<span>alerts.json</span></a>
|
||||
<a class="button button-secondary" href="/data/monitor-summary.json" target="_blank" rel="noreferrer">${icon("json")}<span>monitor-summary.json</span></a>
|
||||
<a class="button button-secondary" href="/docs/source-catalog-audit.html" target="_blank" rel="noreferrer">${icon("docs")}<span>source catalog audit</span></a>
|
||||
</div>
|
||||
`}
|
||||
<div class="plan-grid" style="margin-top:16px;">${failureCards}</div>
|
||||
<div class="plan-grid" style="margin-top:16px;">${alertCards}</div>
|
||||
`
|
||||
);
|
||||
}
|
||||
|
||||
function renderArchitectureFields(fields = []) {
|
||||
if (!fields.length) return "";
|
||||
return `
|
||||
@@ -1197,6 +1267,7 @@ function renderOverviewWorkspace() {
|
||||
<div class="detail-subtitle">根入口保留为概览页,同时新增运行、系统、架构、文档和数据的独立 URL。顶部菜单负责分类切换,搜索与筛选会同步到地址栏。</div>
|
||||
</section>
|
||||
${renderCompletenessPanel("overview_completeness")}
|
||||
${renderSourceHealthPanel("overview_source_health")}
|
||||
${renderPanel("overview_runs", "最新运行", `${escapeHtml(runs.length)} 条`, "queue", renderRunList(runs, "暂无运行数据。"))}
|
||||
${renderPanel("overview_systems", "系统覆盖概览", `${escapeHtml(systems.length)} 个系统`, "systems", `<div class="system-grid">${renderSystemCards(systems)}</div>`)}
|
||||
${renderArchitecturePanel()}
|
||||
@@ -1263,6 +1334,7 @@ function renderDocsWorkspace() {
|
||||
<div class="detail-subtitle">不再把所有入口混在首页链接堆里。这里按说明、设计、真值镜像和 secure-code 索引集中展示。</div>
|
||||
</section>
|
||||
${renderCompletenessPanel("docs_completeness", true)}
|
||||
${renderSourceHealthPanel("docs_source_health", true)}
|
||||
${renderPanel("docs_hub", "文档与镜像页", `${escapeHtml(DOC_HUB_ITEMS.length)} 个入口`, "docs", renderHubCards(DOC_HUB_ITEMS))}
|
||||
</div>
|
||||
`;
|
||||
@@ -1284,6 +1356,7 @@ function renderDataWorkspace() {
|
||||
<div class="detail-subtitle">summary、runs、systems、advisories、profiles、architecture 已单独归入数据中心,避免和文档、运行详情混在一个地址里。</div>
|
||||
</section>
|
||||
${renderCompletenessPanel("data_completeness", true)}
|
||||
${renderSourceHealthPanel("data_source_health")}
|
||||
${renderPanel("data_hub", "JSON 与生成数据", `${escapeHtml(DATA_HUB_ITEMS.length)} 个入口`, "json", renderHubCards(DATA_HUB_ITEMS))}
|
||||
</div>
|
||||
`;
|
||||
@@ -1485,14 +1558,17 @@ async function loadData(preserveSelection = true) {
|
||||
renderSyncState("loading", "刷新中", `本地时间 ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}`);
|
||||
|
||||
try {
|
||||
const [summary, runs, systems, advisories, profiles, architecture, completeness] = await Promise.all([
|
||||
const [summary, runs, systems, advisories, profiles, architecture, completeness, sourceHealth, alerts, monitorSummary] = await Promise.all([
|
||||
fetchJson("/summary.json"),
|
||||
fetchJson("/runs.json"),
|
||||
fetchJson("/systems.json"),
|
||||
fetchJson("/advisories.json"),
|
||||
fetchJson("/profiles.json"),
|
||||
fetchJson("/architecture.json"),
|
||||
fetchJson("/data/completeness.json")
|
||||
fetchJson("/data/completeness.json"),
|
||||
fetchJson("/data/source-health.json"),
|
||||
fetchJson("/data/alerts.json"),
|
||||
fetchJson("/data/monitor-summary.json")
|
||||
]);
|
||||
|
||||
state.summary = summary;
|
||||
@@ -1502,6 +1578,9 @@ async function loadData(preserveSelection = true) {
|
||||
state.profiles = profiles;
|
||||
state.architecture = architecture;
|
||||
state.completeness = completeness;
|
||||
state.sourceHealth = sourceHealth;
|
||||
state.alerts = alerts;
|
||||
state.monitorSummary = monitorSummary;
|
||||
|
||||
const filtered = filteredRuns();
|
||||
const candidate = preserveSelection ? (state.selectedRunId || previousRunId) : state.selectedRunId;
|
||||
|
||||
@@ -236,6 +236,79 @@ def _latest_advisories(advisories: List[Dict[str, Any]]) -> List[Dict[str, Any]]
|
||||
return [annotate_with_latest_run(item, run_map.get(item.get("canonical_id"))) for item in advisories]
|
||||
|
||||
|
||||
def _latest_run_map(runs: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
|
||||
latest: Dict[str, Dict[str, Any]] = {}
|
||||
for item in runs:
|
||||
advisory_id = item.get("advisory_id")
|
||||
if not advisory_id:
|
||||
continue
|
||||
previous = latest.get(advisory_id)
|
||||
if previous is None or (item.get("finished_at") or "") >= (previous.get("finished_at") or ""):
|
||||
latest[advisory_id] = item
|
||||
return latest
|
||||
|
||||
|
||||
def _synthetic_advisory_from_run(run: Dict[str, Any], source_system_map: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
|
||||
system_id = run.get("system_id") or "unknown"
|
||||
system_meta = source_system_map.get(system_id, {})
|
||||
browser_evidence = run.get("browser_evidence") or {
|
||||
"required": False,
|
||||
"present": bool(run.get("browser_refs")),
|
||||
"refs": run.get("browser_refs", []),
|
||||
}
|
||||
return {
|
||||
"canonical_id": run.get("advisory_id"),
|
||||
"system_id": system_id,
|
||||
"display_name": system_meta.get("display_name", system_id),
|
||||
"title": run.get("advisory_title") or run.get("advisory_id") or run.get("run_id"),
|
||||
"summary": run.get("blocked_reason") or f"Derived from latest run {run.get('run_id')}",
|
||||
"category": system_meta.get("category"),
|
||||
"aliases": [],
|
||||
"secure_code_topics": system_meta.get("secure_code_topics", []),
|
||||
"verification_status": run.get("verification_status"),
|
||||
"verification_mode": run.get("verification_mode"),
|
||||
"last_verified_at": run.get("finished_at"),
|
||||
"last_run_id": run.get("run_id"),
|
||||
"browser_evidence": browser_evidence,
|
||||
"repro_profile_id": run.get("repro_profile_id"),
|
||||
"artifact_mode": run.get("artifact_mode"),
|
||||
"blocked_reason": run.get("blocked_reason"),
|
||||
"published_at": run.get("started_at") or run.get("finished_at"),
|
||||
"updated_at": run.get("finished_at") or run.get("started_at"),
|
||||
"official_source_url": "",
|
||||
"secondary_source_urls": [],
|
||||
}
|
||||
|
||||
|
||||
def _merge_latest_advisories(
|
||||
advisories: List[Dict[str, Any]],
|
||||
runs: List[Dict[str, Any]],
|
||||
source_system_map: Dict[str, Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
run_map = _latest_run_map(runs)
|
||||
merged: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for item in advisories:
|
||||
canonical_id = item.get("canonical_id")
|
||||
if not canonical_id:
|
||||
continue
|
||||
merged[canonical_id] = annotate_with_latest_run(item, run_map.get(canonical_id))
|
||||
|
||||
for advisory_id, run in run_map.items():
|
||||
if advisory_id in merged:
|
||||
continue
|
||||
merged[advisory_id] = annotate_with_latest_run(_synthetic_advisory_from_run(run, source_system_map), run)
|
||||
|
||||
return sorted(
|
||||
merged.values(),
|
||||
key=lambda item: (
|
||||
item.get("updated_at") or item.get("published_at") or "",
|
||||
item.get("canonical_id") or "",
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_completeness(
|
||||
advisories: List[Dict[str, Any]],
|
||||
runs: List[Dict[str, Any]],
|
||||
@@ -1202,7 +1275,7 @@ def render_dashboard() -> Dict[str, str]:
|
||||
source_map = read_yaml(SOURCE_MAP_PATH, default={}) or {}
|
||||
repro_map = read_yaml(REPRO_MAP_PATH, default={}) or {}
|
||||
source_system_map = {item["system_id"]: item for item in source_map.get("systems", []) if item.get("system_id")}
|
||||
merged_advisories = _latest_advisories(advisory_records)
|
||||
merged_advisories = _merge_latest_advisories(advisory_records, runs, source_system_map)
|
||||
advisory_map = {item["canonical_id"]: item for item in merged_advisories if item.get("canonical_id")}
|
||||
profile_map = load_profiles()
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
# ./sync-gitea.sh --init # 初始化仓库
|
||||
# ./sync-gitea.sh --commit # 仅提交
|
||||
# ./sync-gitea.sh --push # 仅推送
|
||||
# ./sync-gitea.sh --monitor-sync # 运行监控、提交监控产物并推送
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -302,6 +303,30 @@ auto_sync() {
|
||||
full_sync
|
||||
}
|
||||
|
||||
monitor_sync() {
|
||||
acquire_lock || return 0
|
||||
init_repo
|
||||
|
||||
local monitor_status=0
|
||||
local timestamp
|
||||
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
if python3 "$REPO_DIR/scripts/intel/main.py" monitor; then
|
||||
log_success "监控流水线执行完成"
|
||||
else
|
||||
monitor_status=$?
|
||||
log_warning "监控流水线返回非零,将继续提交和推送最新监控产物"
|
||||
fi
|
||||
|
||||
SKIP_VALIDATE=1 commit_changes "监控更新: ${timestamp}"
|
||||
if needs_push; then
|
||||
push_changes
|
||||
else
|
||||
log_info "没有需要推送的监控提交"
|
||||
fi
|
||||
return "$monitor_status"
|
||||
}
|
||||
|
||||
# 显示帮助
|
||||
show_help() {
|
||||
echo "用法: $0 [选项]"
|
||||
@@ -311,6 +336,7 @@ show_help() {
|
||||
echo " --commit 仅提交更改"
|
||||
echo " --push 仅推送到远程"
|
||||
echo " --autosync 定时任务模式: 无并发锁 + 校验 + 提交 + 推送"
|
||||
echo " --monitor-sync 运行监控、提交监控快照、推送后按监控结果退出"
|
||||
echo " --ensure 检查远程仓库;不存在则创建"
|
||||
echo " --status 显示仓库状态"
|
||||
echo " --help 显示此帮助"
|
||||
@@ -352,6 +378,9 @@ case "${1:-}" in
|
||||
--autosync)
|
||||
auto_sync
|
||||
;;
|
||||
--monitor-sync)
|
||||
monitor_sync
|
||||
;;
|
||||
--ensure)
|
||||
init_repo
|
||||
ensure_remote_repo
|
||||
|
||||
在新工单中引用
屏蔽一个用户