hao/websafe-kb
1
0
派生 0
代码 工单 合并请求 1 工作流 软件包 项目 发布 百科 活动

实现分层实体漏洞知识库与实体级完整度监控

浏览代码
这个提交包含在：
hao
2026-03-19 17:57:45 -07:00
父节点 49fe46ab89
当前提交 1e81279e32
共修改 2712 个文件，包含 434447 行新增和 2774 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

57
scripts/lab/dashboard_templates/lovart/assets/app.js
查看文件

@@ -45,6 +45,8 @@ const DOC_HUB_ITEMS = [
{ title: "source-map 镜像", href: "/docs/source-map.html", description: "系统覆盖、来源、输出目录和 secure-code 主题真值。", badge: "source-map" },
{ title: "source catalog audit", href: "/docs/source-catalog-audit.html", description: "active/retired source、replacement map 与覆盖摘要。", badge: "audit" },
{ title: "retired sources", href: "/docs/retired-sources.html", description: "退役源、退役原因和 replacement_sources 真值。", badge: "retired" },
{ title: "entity catalog report", href: "/docs/entity-catalog-report.html", description: "分层实体覆盖、history-full 完整度与 workflow 指标。", badge: "entities" },
{ title: "entity discovery backlog", href: "/docs/entity-discovery-backlog.html", description: "待编目 repo / 插件 / 包 backlog 与等待原因。", badge: "backlog" },
{ title: "repro-map 镜像", href: "/docs/repro-map.html", description: "默认漏洞家族、浏览器要求和日志策略真值。", badge: "repro-map" },
{ title: "覆盖矩阵镜像", href: "/docs/coverage-matrix.html", description: "当前全库覆盖矩阵的本地镜像。", badge: "coverage" },
{ title: "安全编码索引", href: "/docs/secure-code-index.html", description: "secure-code 修复主题索引镜像。", badge: "secure-code" },
@@ -58,6 +60,9 @@ const DATA_HUB_ITEMS = [
{ title: "alerts.json", href: "/data/alerts.json", description: "source 告警状态机、failure streak 与 resolved 记录。", badge: "json" },
{ title: "monitor-summary.json", href: "/data/monitor-summary.json", description: "每日监控摘要、open alerts 与最近全绿时间。", badge: "json" },
{ title: "source-catalog-audit.json", href: "/data/source-catalog-audit.json", description: "source catalog 审计真值与 retired/replacement 关系。", badge: "json" },
{ title: "entity-completeness.json", href: "/data/entity-completeness.json", description: "实体级 catalog 完整度、版本映射和 workflow 覆盖。", badge: "json" },
{ title: "entity-discovery-backlog.json", href: "/data/entity-discovery-backlog.json", description: "发现但尚未正式编目的 repo / 插件 / 包 backlog。", badge: "json" },
{ title: "entity-queues.json", href: "/data/entity-queues.json", description: "discovery/history/latest/workflow 四类队列摘要。", badge: "json" },
{ title: "runs.json", href: "/runs.json", description: "最近运行的结构化详情，可用于 UI 和调试。", badge: "json" },
{ title: "systems.json", href: "/systems.json", description: "系统级覆盖、分类、更新时间和浏览器证据统计。", badge: "json" },
{ title: "advisories.json", href: "/advisories.json", description: "漏洞条目元数据、来源和 secure-code 主题。", badge: "json" },
@@ -93,6 +98,7 @@ const state = {
profiles: {},
architecture: null,
completeness: null,
entityCompleteness: null,
sourceHealth: null,
alerts: [],
monitorSummary: null,
@@ -288,9 +294,12 @@ function familyOptions() {
function metricCards() {
const completeness = state.completeness || state.summary?.completeness || {};
const entityCoverage = state.entityCompleteness || state.summary?.entity_coverage || completeness.entity_coverage || {};
const monitoring = state.monitorSummary || state.summary?.monitoring || {};
const advisoryTotal = Number(completeness.advisory_total || state.summary?.advisory_count || 0);
const advisorySuccess = Number(completeness.verified_real || 0);
const catalogedEntities = Number(entityCoverage.cataloged_entity_total || 0);
const candidateEntities = Number(entityCoverage.candidate_entity_total || 0);
const activeSources = Number(monitoring.active_source_count || state.sourceHealth?.active_source_count || 0);
const greenSources = Number(monitoring.green_source_count || state.sourceHealth?.green_source_count || 0);
const openAlerts = Number(monitoring.open_alert_count || state.sourceHealth?.open_alert_count || 0);
@@ -304,6 +313,13 @@ function metricCards() {
color: "var(--accent-green)",
iconName: "report"
},
{
label: "分层实体",
value: catalogedEntities,
note: `backlog ${candidateEntities}`,
color: "var(--accent-yellow)",
iconName: "systems"
},
{
label: "active sources",
value: activeSources,
@@ -774,6 +790,7 @@ function renderPanel(panelKey, title, meta, iconName, content) {
function renderCompletenessPanel(panelKey, compact = false) {
const completeness = state.completeness || state.summary?.completeness || {};
const entityCoverage = state.entityCompleteness || state.summary?.entity_coverage || completeness.entity_coverage || {};
const sourceHealth = state.sourceHealth || completeness.source_health || {};
const systems = (state.completeness?.systems || []).map((system) => `
<article class="plan-card">
@@ -816,12 +833,30 @@ function renderCompletenessPanel(panelKey, compact = false) {
<strong>open alerts</strong>
<span>${escapeHtml(sourceHealth.open_alert_count || 0)}</span>
</article>
<article class="detail-stat">
<strong>cataloged entities</strong>
<span>${escapeHtml(entityCoverage.cataloged_entity_total || 0)}</span>
</article>
<article class="detail-stat">
<strong>candidate backlog</strong>
<span>${escapeHtml(entityCoverage.candidate_entity_total || 0)}</span>
</article>
<article class="detail-stat">
<strong>workflow complete</strong>
<span>${escapeHtml(entityCoverage.workflow_complete_count || 0)}</span>
</article>
<article class="detail-stat">
<strong>version mapped</strong>
<span>${escapeHtml(entityCoverage.version_mapped_count || 0)}</span>
</article>
</div>
<div class="plan-grid" style="margin-top:16px;">${systems || `<div class="empty-state">暂无系统完整度数据。</div>`}</div>
${compact ? "" : `
<div class="detail-actions" style="margin-top:16px;">
<a class="button button-secondary" href="/docs/testing-completeness-report.html" target="_blank" rel="noreferrer">${icon("docs")}<span>打开中文报告</span></a>
<a class="button button-secondary" href="/docs/entity-catalog-report.html" target="_blank" rel="noreferrer">${icon("docs")}<span>打开实体报告</span></a>
<a class="button button-secondary" href="/data/completeness.json" target="_blank" rel="noreferrer">${icon("json")}<span>打开 completeness.json</span></a>
<a class="button button-secondary" href="/data/entity-completeness.json" target="_blank" rel="noreferrer">${icon("json")}<span>打开 entity-completeness.json</span></a>
<a class="button button-secondary" href="/data/source-health.json" target="_blank" rel="noreferrer">${icon("json")}<span>打开 source-health.json</span></a>
</div>
${failures.length ? `<div class="callout" style="margin-top:16px;"><strong>Ingest 未清零</strong><div class="plan-copy">${escapeHtml(failures.join(" | "))}</div></div>` : ""}
@@ -1114,7 +1149,8 @@ function renderRunWorkspace() {
{ label: "概要", copy: advisory.summary || "当前漏洞条目没有摘要。" },
{ label: "成功判据", copy: (profile.success_criteria || []).join(" | ") || "当前复现档案没有定义成功判据。" },
{ label: "Seed / 攻击思路", copy: (run.reasoning_lines || []).join("\n\n") || "当前运行没有记录思路说明。" },
{ label: "允许目标", copy: (profile.allowed_target_types || []).join(", ") || "当前复现档案没有声明允许目标类型。" }
{ label: "允许目标", copy: (profile.allowed_target_types || []).join(", ") || "当前复现档案没有声明允许目标类型。" },
{ label: "版本与对象", copy: `${advisory.advisory_scope || "core"} · ${advisory.version_confidence || "unknown"} · ${(advisory.affected_version_ranges || []).join(", ") || "版本待补齐"}` }
];
const evidenceContent = `
@@ -1239,6 +1275,21 @@ function renderRunWorkspace() {
<div class="tag-row">
${(advisory.aliases || []).map((alias) => `<span class="tag">${escapeHtml(alias)}</span>`).join("")}
${(advisory.secure_code_topics || []).map((topic) => `<a class="tag" href="/docs/secure-code-index.html" target="_blank" rel="noreferrer">${escapeHtml(topic)}</a>`).join("")}
${(advisory.entity_refs || []).map((item) => `<span class="tag">${escapeHtml(item.entity_type || "entity")} · ${escapeHtml(item.entity_id || "-")}</span>`).join("")}
</div>
<div class="plan-grid" style="margin-bottom:16px;">
<article class="plan-card">
<span class="plan-label">影响对象</span>
<div class="plan-copy">${escapeHtml((advisory.affected_components || []).map((item) => item.name).join(" | ") || "未定义")}</div>
</article>
<article class="plan-card">
<span class="plan-label">版本映射</span>
<div class="plan-copy">${escapeHtml((advisory.affected_version_ranges || []).join(" | ") || advisory.version_gap_reason || "未定义")}</div>
</article>
<article class="plan-card">
<span class="plan-label">Workflow</span>
<div class="plan-copy">${escapeHtml(advisory.workflow?.vuln_family || "unknown")} · ${escapeHtml(advisory.workflow?.entry_surface || "-")}</div>
</article>
</div>
<div class="source-links">${sourceLinks}</div>
`)}
@@ -1558,7 +1609,7 @@ async function loadData(preserveSelection = true) {
renderSyncState("loading", "刷新中", `本地时间 ${new Date().toLocaleTimeString("zh-CN", { hour12: false })}`);
try {
const [summary, runs, systems, advisories, profiles, architecture, completeness, sourceHealth, alerts, monitorSummary] = await Promise.all([
const [summary, runs, systems, advisories, profiles, architecture, completeness, entityCompleteness, sourceHealth, alerts, monitorSummary] = await Promise.all([
fetchJson("/summary.json"),
fetchJson("/runs.json"),
fetchJson("/systems.json"),
@@ -1566,6 +1617,7 @@ async function loadData(preserveSelection = true) {
fetchJson("/profiles.json"),
fetchJson("/architecture.json"),
fetchJson("/data/completeness.json"),
fetchJson("/data/entity-completeness.json"),
fetchJson("/data/source-health.json"),
fetchJson("/data/alerts.json"),
fetchJson("/data/monitor-summary.json")
@@ -1578,6 +1630,7 @@ async function loadData(preserveSelection = true) {
state.profiles = profiles;
state.architecture = architecture;
state.completeness = completeness;
state.entityCompleteness = entityCompleteness;
state.sourceHealth = sourceHealth;
state.alerts = alerts;
state.monitorSummary = monitorSummary;

92
scripts/lab/render.py
查看文件

@@ -154,6 +154,17 @@ def _advisory_meta(advisory: Dict[str, Any]) -> Dict[str, Any]:
"artifact_mode": advisory.get("artifact_mode"),
"blocked_reason": advisory.get("blocked_reason"),
"browser_evidence": advisory.get("browser_evidence", {}),
"entity_refs": advisory.get("entity_refs", []),
"affected_components": advisory.get("affected_components", []),
"affected_version_ranges": advisory.get("affected_version_ranges", []),
"fixed_version_ranges": advisory.get("fixed_version_ranges", []),
"introduced_version": advisory.get("introduced_version"),
"patched_version": advisory.get("patched_version"),
"version_confidence": advisory.get("version_confidence"),
"version_gap_reason": advisory.get("version_gap_reason"),
"version_resolution_needed": advisory.get("version_resolution_needed"),
"advisory_scope": advisory.get("advisory_scope"),
"workflow": advisory.get("workflow", {}),
}
@@ -277,6 +288,37 @@ def _synthetic_advisory_from_run(run: Dict[str, Any], source_system_map: Dict[st
"updated_at": run.get("finished_at") or run.get("started_at"),
"official_source_url": "",
"secondary_source_urls": [],
"entity_refs": [
{
"entity_id": system_id,
"entity_type": "system",
"relation": "root-system",
"root_system_id": system_id,
"official": True,
}
],
"affected_components": [
{
"name": system_meta.get("display_name", system_id),
"entity_id": system_id,
"scope": "core",
"package_name": None,
"official": True,
}
],
"affected_version_ranges": [],
"fixed_version_ranges": [],
"introduced_version": None,
"patched_version": None,
"version_confidence": "low",
"version_gap_reason": "derived from stored run without a persisted advisory record",
"version_resolution_needed": True,
"advisory_scope": "core",
"workflow": {
"workflow_id": f"{run.get('advisory_id')}--derived-workflow",
"vuln_family": run.get("repro_profile_id") or "unknown",
"entry_surface": "stored-run-derived",
},
}
@@ -317,6 +359,7 @@ def _build_completeness(
source_health: Dict[str, Any],
alerts: List[Dict[str, Any]],
monitor_summary: Dict[str, Any],
entity_completeness: Dict[str, Any],
) -> Dict[str, Any]:
tracked_advisories = [item for item in advisories if item.get("last_run_id")] or advisories
latest_statuses: Dict[str, int] = {}
@@ -402,6 +445,7 @@ def _build_completeness(
"open_alert_count": open_alert_count,
"resolved_alert_count": len([item for item in alerts if item.get("status") == "resolved"]),
},
"entity_coverage": entity_completeness or {},
"monitor_summary": monitor_summary or {},
"historical_blockers": [
"Docker daemon unavailable caused provision-compose-environment blocked-artifact.",
@@ -409,6 +453,7 @@ def _build_completeness(
"Baseline and browser steps were skipped when environment readiness was not enforced.",
"Latest completeness now uses one advisory -> latest run semantics instead of historical run piles.",
"Source health now counts only status=active sources; retired sources are audited separately with replacement links.",
"Entity coverage now separates cataloged entities from discovery backlog so infinite internet scope no longer pollutes completion numbers.",
],
}
@@ -426,6 +471,8 @@ def _write_testing_completeness_report(completeness: Dict[str, Any]) -> None:
f"- active source 全绿: `{completeness['source_health']['green_source_count']}/{completeness['source_health']['active_source_count']}`",
f"- source open alerts: `{completeness['source_health']['open_alert_count']}`",
f"- 最近一次 source 全绿: `{completeness['source_health'].get('last_fully_green_run') or '-'}`",
f"- 已编目实体: `{completeness.get('entity_coverage', {}).get('cataloged_entity_total', 0)}`",
f"- 待编目 backlog: `{completeness.get('entity_coverage', {}).get('candidate_entity_total', 0)}`",
"",
"## 系统覆盖矩阵",
"",
@@ -441,6 +488,15 @@ def _write_testing_completeness_report(completeness: Dict[str, Any]) -> None:
)
lines.extend(
[
"",
"## 分层实体完整度",
"",
f"- history-full 已完成: `{completeness.get('entity_coverage', {}).get('history_full_complete_count', 0)}`",
f"- latest green: `{completeness.get('entity_coverage', {}).get('latest_green_count', 0)}`",
f"- workflow 完整: `{completeness.get('entity_coverage', {}).get('workflow_complete_count', 0)}`",
f"- 版本映射完整: `{completeness.get('entity_coverage', {}).get('version_mapped_count', 0)}`",
f"- 官方源覆盖: `{completeness.get('entity_coverage', {}).get('official_source_covered_count', 0)}`",
f"- 插件 history-full 已完成: `{completeness.get('entity_coverage', {}).get('plugin_history_full_count', 0)}`",
"",
"## 历史阻塞项修复纪要",
"",
@@ -495,6 +551,8 @@ def _build_architecture_data(summary: Dict[str, Any], source_map: Dict[str, Any]
_link("source-map 真值", "/docs/source-map.html", "系统覆盖、来源和输出目录真值。"),
_link("source catalog audit", "/docs/source-catalog-audit.html", "active/retired source 审计、替代关系与覆盖摘要。"),
_link("retired sources", "/docs/retired-sources.html", "退役源、退役原因与 replacement map。"),
_link("entity catalog report", "/docs/entity-catalog-report.html", "分层实体覆盖、history-full 完整度与 workflow 指标。"),
_link("entity discovery backlog", "/docs/entity-discovery-backlog.html", "待编目 repo / 插件 / 包 backlog 与等待原因。"),
_link("repro-map 真值", "/docs/repro-map.html", "复现族路由、浏览器要求和日志策略。"),
_link("覆盖矩阵", "/docs/coverage-matrix.html", "自动生成覆盖摘要的本地镜像。"),
_link("设计来源清单", "/docs/design-source.html", "Lovart 模板本地 vendor manifest。"),
@@ -507,6 +565,9 @@ def _build_architecture_data(summary: Dict[str, Any], source_map: Dict[str, Any]
_link("source-health.json", "/data/source-health.json", "active source 健康度、系统分布与失败分类。"),
_link("alerts.json", "/data/alerts.json", "source 告警状态机、failure streak 与 resolved 记录。"),
_link("monitor-summary.json", "/data/monitor-summary.json", "每日监控摘要、open alerts 与最近全绿时间。"),
_link("entity-completeness.json", "/data/entity-completeness.json", "实体级 catalog 完整度、版本映射与 workflow 覆盖。"),
_link("entity-discovery-backlog.json", "/data/entity-discovery-backlog.json", "发现但尚未正式编目的 repo / 插件 / 包 backlog。"),
_link("entity-queues.json", "/data/entity-queues.json", "discovery/history/latest/workflow 四类队列摘要。"),
_link("runs.json", "/runs.json", "最近 run 的结构化详情。"),
_link("systems.json", "/systems.json", "系统级覆盖与浏览器证据摘要。"),
_link("advisories.json", "/advisories.json", "漏洞条目元数据与来源。"),
@@ -650,6 +711,8 @@ def _build_architecture_data(summary: Dict[str, Any], source_map: Dict[str, Any]
_stat("纳管系统", len(source_systems)),
_stat("历史全量系统", tier_counts.get("history-full", 0)),
_stat("近两年全量系统", tier_counts.get("rolling-24m", 0)),
_stat("已编目实体", (summary.get("entity_coverage") or {}).get("cataloged_entity_total", 0)),
_stat("发现 backlog", (summary.get("entity_coverage") or {}).get("candidate_entity_total", 0)),
_stat("当前运行", summary.get("run_count", 0)),
_stat("当前漏洞条目", summary.get("advisory_count", 0)),
],
@@ -756,6 +819,7 @@ def _build_architecture_data(summary: Dict[str, Any], source_map: Dict[str, Any]
"fields": [
_field("漏洞条目 Registry", "08-threat-intel/registry/advisories/*.json"),
_field("系统 Registry", "08-threat-intel/registry/systems/*.json"),
_field("实体 Registry", "08-threat-intel/registry/entities/*.json"),
_field("运行 Registry", "08-threat-intel/registry/runs/*.json"),
_field("source-map 真值", "08-threat-intel/source-map.yaml"),
_field("repro-map 真值", "08-threat-intel/repro-map.yaml"),
@@ -769,6 +833,9 @@ def _build_architecture_data(summary: Dict[str, Any], source_map: Dict[str, Any]
"fields": [
_field("工作台根目录", "08-threat-intel/generated/dashboard/"),
_field("运行归档根目录", "06-case-studies/generated-runs/<run-id>/"),
_field("实体完整度", "/data/entity-completeness.json"),
_field("发现 backlog", "/data/entity-discovery-backlog.json"),
_field("实体队列", "/data/entity-queues.json"),
_field("默认入口", "/index.html"),
_field("总览入口", "/overview/index.html"),
_field("运行入口", "/runs/index.html"),
@@ -1013,6 +1080,18 @@ def _write_dashboard_docs(architecture: Dict[str, Any]) -> None:
REPRO_MAP_PATH.read_text(encoding="utf-8"),
"工作台内置镜像页：默认漏洞家族、浏览器要求和日志策略真值。",
),
(
"entity-catalog-report.html",
"分层实体覆盖报告",
_safe_read_text(ROOT / "08-threat-intel" / "generated" / "entity-catalog-report.md", "entity catalog report has not been generated yet."),
"工作台内置镜像页：分层实体覆盖、history-full 完整度和 workflow 统计。",
),
(
"entity-discovery-backlog.html",
"分层实体发现 Backlog",
_safe_read_text(ROOT / "08-threat-intel" / "generated" / "entity-discovery-backlog.md", "entity discovery backlog has not been generated yet."),
"工作台内置镜像页：待编目 repo / 插件 / 包 backlog 与等待原因。",
),
(
"coverage-matrix.html",
"覆盖矩阵镜像",
@@ -1281,6 +1360,9 @@ def render_dashboard(
alerts = read_json(ROOT / "08-threat-intel" / "generated" / "alerts.json", default=[]) or []
monitor_summary = read_json(ROOT / "08-threat-intel" / "generated" / "monitor-summary.json", default={}) or {}
source_catalog_audit = read_json(ROOT / "08-threat-intel" / "generated" / "source-catalog-audit.json", default={}) or {}
entity_completeness = read_json(ROOT / "08-threat-intel" / "generated" / "entity-completeness.json", default={}) or {}
entity_backlog = read_json(ROOT / "08-threat-intel" / "generated" / "entity-discovery-backlog.json", default=[]) or []
entity_queues = read_json(ROOT / "08-threat-intel" / "generated" / "entity-queues.json", default={}) or {}
source_map = source_map_data if source_map_data is not None else (read_yaml(SOURCE_MAP_PATH, default={}) or {})
repro_map = repro_map_data if repro_map_data is not None else (read_yaml(REPRO_MAP_PATH, default={}) or {})
source_system_map = {item["system_id"]: item for item in source_map.get("systems", []) if item.get("system_id")}
@@ -1400,6 +1482,7 @@ def render_dashboard(
"open_alert_count": len([item for item in alerts if item.get("status") == "open"]),
"last_fully_green_run": source_health.get("last_fully_green_run"),
},
"entity_coverage": entity_completeness,
}
for item in merged_advisories:
status = item.get("verification_status", "triage-manual")
@@ -1428,7 +1511,7 @@ def render_dashboard(
for item in sorted(merged_advisories, key=lambda value: value.get("updated_at") or value.get("published_at") or "", reverse=True)
if item.get("verification_status") in {"triage-manual", "blocked-artifact", "blocked-destructive"}
][:20]
completeness = _build_completeness(merged_advisories, runs, profile_map, run_summary, source_health, alerts, monitor_summary)
completeness = _build_completeness(merged_advisories, runs, profile_map, run_summary, source_health, alerts, monitor_summary, entity_completeness)
summary["completeness"] = {
"advisory_total": completeness["advisory_total"],
"verified_real": completeness["verified_real"],
@@ -1440,6 +1523,10 @@ def render_dashboard(
"source_failure_count": completeness["source_health"]["failure_count"],
"active_source_count": completeness["source_health"]["active_source_count"],
"open_alert_count": completeness["source_health"]["open_alert_count"],
"cataloged_entity_total": entity_completeness.get("cataloged_entity_total", 0),
"candidate_entity_total": entity_completeness.get("candidate_entity_total", 0),
"workflow_complete_count": entity_completeness.get("workflow_complete_count", 0),
"version_mapped_count": entity_completeness.get("version_mapped_count", 0),
}
write_json(DASHBOARD_DIR / "summary.json", summary)
@@ -1452,6 +1539,9 @@ def render_dashboard(
write_json(DASHBOARD_DIR / "data" / "alerts.json", alerts)
write_json(DASHBOARD_DIR / "data" / "monitor-summary.json", monitor_summary)
write_json(DASHBOARD_DIR / "data" / "source-catalog-audit.json", source_catalog_audit)
write_json(DASHBOARD_DIR / "data" / "entity-completeness.json", entity_completeness)
write_json(DASHBOARD_DIR / "data" / "entity-discovery-backlog.json", entity_backlog)
write_json(DASHBOARD_DIR / "data" / "entity-queues.json", entity_queues)
_write_testing_completeness_report(completeness)
architecture = _build_architecture_data(summary, source_map, repro_map)
write_json(DASHBOARD_DIR / "architecture.json", architecture)