hao/websafe-kb
1
0
派生 0
代码 工单 合并请求 1 工作流 软件包 项目 发布 百科 活动

更新: 21 个文件 - 2026-03-17 00:00:00

浏览代码
这个提交包含在：
hao
2026-03-17 00:00:00 -07:00
父节点 17a26fa7d0
当前提交 dddbe19df8
共修改 21 个文件，包含 787 行新增和 144 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

19
scripts/lab/browser.py
查看文件

@@ -25,8 +25,12 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any]
dom_path = assets_dir / f"{prefix}-dom.html"
console_path = run_dir / "logs" / f"{prefix}-console.json"
network_path = run_dir / "logs" / f"{prefix}-network.json"
page_path = run_dir / "logs" / f"{prefix}-page.json"
console_messages: List[Dict[str, Any]] = []
requests_seen: List[Dict[str, Any]] = []
page_title = ""
page_body_excerpt = ""
final_url = url
try:
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
@@ -36,6 +40,9 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any]
page.goto(url, wait_until="networkidle", timeout=20000)
page.screenshot(path=str(screenshot_path), full_page=True)
dom_path.write_text(page.content(), encoding="utf-8")
final_url = page.url
page_title = page.title()
page_body_excerpt = (page.text_content("body") or "")[:600]
browser.close()
except Exception as exc:
payload["reason"] = str(exc)
@@ -43,10 +50,20 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any]
return payload
write_json(console_path, console_messages)
write_json(network_path, requests_seen)
write_json(
page_path,
{
"url": final_url,
"title": page_title,
"body_excerpt": page_body_excerpt,
},
)
payload = {
"required": True,
"present": True,
"refs": [str(screenshot_path), str(dom_path), str(console_path), str(network_path)],
"page_title": page_title,
"page_url": final_url,
"refs": [str(screenshot_path), str(dom_path), str(console_path), str(network_path), str(page_path)],
}
write_json(run_dir / "logs" / f"{prefix}-browser.json", payload)
return payload

266
scripts/lab/main.py
查看文件

@@ -35,6 +35,32 @@ def _compose_run_id(advisory: Dict[str, Any]) -> str:
return f"{advisory['system_id']}-{advisory['canonical_id']}-{now_utc().strftime('%Y%m%d%H%M%S')}"
def _existing_refs(*paths: Path) -> List[str]:
return [str(path) for path in paths if path.exists()]
def _timeline_event(timeline: List[Dict[str, Any]], step: str, status: str, detail: str = "") -> None:
timeline.append(
{
"at": isoformat(now_utc()),
"step": step,
"status": status,
"detail": detail,
}
)
def _sync_registry_outputs() -> None:
from intel.config import GENERATED_DIR, load_source_map # noqa: E402
from intel.main import _load_existing_advisories, _load_existing_triage, _write_outputs # noqa: E402
source_map = load_source_map()
advisories = _load_existing_advisories()
triage = _load_existing_triage()
summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {}
_write_outputs(source_map, advisories, triage, summary.get("failures", []), summary)
def _resolve_profile(advisory: Dict[str, Any]) -> Dict[str, Any]:
profile = repro.resolve_profile(advisory["canonical_id"], advisory)
current_profile = read_yaml(ENV_PROFILES_DIR / "core" / advisory["system_id"] / "current.yaml", default={}) or {}
@@ -70,6 +96,11 @@ def _build_run_bundle(
browser_refs: List[str],
container_log_refs: List[str],
request_log_refs: List[str],
compose_refs: List[str],
browser_evidence: Dict[str, Any],
timeline: List[Dict[str, Any]],
started_at: str,
finished_at: str,
blocked_reason: str | None,
) -> Dict[str, Any]:
return {
@@ -85,15 +116,174 @@ def _build_run_bundle(
"baseline_refs": baseline_refs,
"attack_steps": attack_steps,
"browser_refs": browser_refs,
"browser_evidence": browser_evidence,
"container_log_refs": container_log_refs,
"request_log_refs": request_log_refs,
"timeline": [],
"started_at": isoformat(now_utc()),
"finished_at": isoformat(now_utc()),
"compose_refs": compose_refs,
"timeline": timeline,
"started_at": started_at,
"finished_at": finished_at,
"blocked_reason": blocked_reason,
}
def _dry_run_case_plan(advisory: Dict[str, Any], profile: Dict[str, Any], run_id: str) -> Dict[str, Any]:
provision_result = provision.prepare(profile, CASE_RUNS_DIR / run_id, dry_run=True)
return {
"run_id": run_id,
"system_id": advisory["system_id"],
"advisory_id": advisory["canonical_id"],
"repro_profile_id": profile["profile_id"],
"verification_mode": profile.get("verification_mode", "synthetic"),
"artifact_mode": profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic")),
"browser_required": bool(profile.get("browser_assertions", {}).get("required")),
"baseline_urls": profile.get("baseline_urls", []),
"compose_services": sorted(profile.get("services", {}).keys()),
"seed_actions": profile.get("seed_actions", []),
"attack_actions": profile.get("attack_actions", []),
"compose_preview": provision_result.get("compose_preview", {}),
"note": "dry-run only; no bundle, report, compose file, or registry update was written",
}
def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool = False, sync_outputs: bool = True) -> Dict[str, Any]:
advisory = _load_advisory(canonical_id)
profile = _resolve_profile(advisory)
resolved_run_id = run_id or _compose_run_id(advisory)
if dry_run:
return _dry_run_case_plan(advisory, profile, resolved_run_id)
started_at = isoformat(now_utc())
timeline: List[Dict[str, Any]] = []
_timeline_event(timeline, "select-advisory", "completed", advisory["canonical_id"])
_timeline_event(timeline, "resolve-repro-profile", "completed", profile["profile_id"])
run_dir = _run_dir(resolved_run_id)
provision_result = provision.prepare(profile, run_dir, dry_run=False)
_timeline_event(
timeline,
"provision-compose-environment",
provision_result.get("status", "unknown"),
provision_result.get("blocked_reason", ""),
)
allow_runtime_steps = provision_result.get("status") not in {"blocked-artifact"}
browser_required = bool(profile.get("browser_assertions", {}).get("required"))
baseline_payload = {"observations": []}
if profile.get("baseline_urls") and allow_runtime_steps:
baseline_payload = baseline.collect(profile, run_dir)
_timeline_event(timeline, "baseline-snapshot", "completed", f"urls={len(profile.get('baseline_urls', []))}")
else:
_timeline_event(timeline, "baseline-snapshot", "skipped", "no baseline urls or provisioning blocked")
baseline_browser = {"required": browser_required, "present": False, "refs": []}
if browser_required and allow_runtime_steps and profile.get("baseline_urls"):
baseline_browser = browser.capture(profile["baseline_urls"][0], run_dir, prefix="baseline")
_timeline_event(
timeline,
"browser-replay-before-attack",
"completed" if baseline_browser.get("present") else "failed",
baseline_browser.get("reason", ""),
)
elif browser_required:
_timeline_event(timeline, "browser-replay-before-attack", "skipped", "baseline browser capture unavailable")
attack_payload = {"steps": []}
if allow_runtime_steps:
attack_payload = attack.run_attack(profile, advisory, run_dir, dry_run=False)
attack_failed = any(step.get("status") == "failed" for step in attack_payload.get("steps", []))
_timeline_event(
timeline,
"controlled-attack-chain",
"failed" if attack_failed else "completed",
f"steps={len(attack_payload.get('steps', []))}",
)
else:
_timeline_event(timeline, "controlled-attack-chain", "skipped", "provisioning blocked")
proof_browser = {"required": browser_required, "present": False, "refs": []}
if browser_required and allow_runtime_steps and profile.get("baseline_urls"):
proof_browser = browser.capture(profile["baseline_urls"][0], run_dir, prefix="proof")
_timeline_event(
timeline,
"browser-replay-after-attack",
"completed" if proof_browser.get("present") else "failed",
proof_browser.get("reason", ""),
)
elif browser_required:
_timeline_event(timeline, "browser-replay-after-attack", "skipped", "proof browser capture unavailable")
compose_path = Path(provision_result["compose_path"])
container_logs = evidence.collect_container_logs(run_dir, compose_path) if compose_path.exists() and allow_runtime_steps else []
_timeline_event(
timeline,
"collect-logs-and-evidence",
"completed" if allow_runtime_steps else "skipped",
f"container_logs={len(container_logs)}",
)
browser_present = bool(baseline_browser.get("present")) and bool(proof_browser.get("present"))
browser_payload = {
"required": browser_required,
"present": browser_present,
"refs": baseline_browser.get("refs", []) + proof_browser.get("refs", []),
"baseline_refs": baseline_browser.get("refs", []),
"proof_refs": proof_browser.get("refs", []),
"baseline_title": baseline_browser.get("page_title"),
"proof_title": proof_browser.get("page_title"),
}
blocked_reason = provision_result.get("blocked_reason")
if browser_required and not browser_present:
blocked_reason = blocked_reason or baseline_browser.get("reason") or proof_browser.get("reason") or "browser evidence incomplete"
verification_mode = profile.get("verification_mode", "synthetic")
artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic"))
verification_status = "triage-manual"
if provision_result.get("status") == "blocked-artifact":
verification_status = "blocked-artifact"
elif browser_required and not browser_present:
verification_status = "triage-manual"
elif any(step.get("status") == "failed" for step in attack_payload.get("steps", [])):
verification_status = "triage-manual"
elif artifact_mode == "synthetic":
verification_status = "verified-synthetic"
else:
verification_status = "verified-real"
finished_at = isoformat(now_utc())
bundle = _build_run_bundle(
advisory=advisory,
profile=profile,
run_id=resolved_run_id,
verification_status=verification_status,
verification_mode=verification_mode,
artifact_mode=artifact_mode,
baseline_refs=_existing_refs(run_dir / "logs" / "baseline.json"),
attack_steps=attack_payload.get("steps", []),
browser_refs=browser_payload["refs"],
container_log_refs=container_logs,
request_log_refs=_existing_refs(run_dir / "logs" / "attack.json", run_dir / "logs" / "baseline.json"),
compose_refs=[str(compose_path)] if compose_path.exists() else [],
browser_evidence=browser_payload,
timeline=timeline,
started_at=started_at,
finished_at=finished_at,
blocked_reason=blocked_reason,
)
_timeline_event(bundle["timeline"], "update-registry-and-reports", "completed", resolved_run_id)
report_refs = render.render_run(bundle)
bundle["report_refs"] = report_refs
evidence.write_run_bundle(run_dir, bundle)
ensure_dir(RUNS_DIR)
write_json(RUNS_DIR / f"{resolved_run_id}.json", bundle)
if sync_outputs:
_sync_registry_outputs()
else:
render.render_dashboard()
return bundle
def cmd_catalog_sync(args) -> int:
summary = catalog.sync_catalog(write_profiles=True, write_repro_map=True)
print(summary)
@@ -158,66 +348,8 @@ def cmd_verify(args) -> int:
def cmd_run_case(args) -> int:
advisory = _load_advisory(args.case)
profile = _resolve_profile(advisory)
run_id = args.run_id or _compose_run_id(advisory)
run_dir = _run_dir(run_id)
provision_result = provision.prepare(profile, run_dir, dry_run=args.dry_run)
allow_runtime_steps = provision_result.get("status") not in {"blocked-artifact"}
baseline_payload = (
baseline.collect(profile, run_dir) if profile.get("baseline_urls") and allow_runtime_steps else {"observations": []}
)
attack_payload = (
attack.run_attack(profile, advisory, run_dir, dry_run=args.dry_run) if allow_runtime_steps else {"steps": []}
)
browser_payload = {"required": bool(profile.get("browser_assertions", {}).get("required")), "present": False, "refs": []}
blocked_reason = provision_result.get("blocked_reason")
if browser_payload["required"] and not args.dry_run and profile.get("baseline_urls") and allow_runtime_steps:
browser_payload = browser.capture(profile["baseline_urls"][0], run_dir, prefix="proof")
if not browser_payload.get("present"):
blocked_reason = blocked_reason or browser_payload.get("reason")
compose_path = Path(provision_result["compose_path"])
container_logs = evidence.collect_container_logs(run_dir, compose_path) if compose_path.exists() and allow_runtime_steps else []
verification_status = "triage-manual"
verification_mode = profile.get("verification_mode", "synthetic")
artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic"))
if args.dry_run:
verification_status = "triage-manual"
blocked_reason = blocked_reason or "dry-run only"
elif provision_result.get("status") == "blocked-artifact":
verification_status = "blocked-artifact"
elif browser_payload.get("required") and not browser_payload.get("present"):
verification_status = "triage-manual"
elif artifact_mode == "synthetic":
verification_status = "verified-synthetic"
else:
verification_status = "verified-real"
bundle = _build_run_bundle(
advisory=advisory,
profile=profile,
run_id=run_id,
verification_status=verification_status,
verification_mode=verification_mode,
artifact_mode=artifact_mode,
baseline_refs=[str(run_dir / "logs" / "baseline.json")] if baseline_payload.get("observations") else [],
attack_steps=attack_payload.get("steps", []),
browser_refs=browser_payload.get("refs", []),
container_log_refs=container_logs,
request_log_refs=[str(run_dir / "logs" / "attack.json"), str(run_dir / "logs" / "baseline.json")],
blocked_reason=blocked_reason,
)
report_refs = render.render_run(bundle)
bundle["report_refs"] = report_refs
evidence.write_run_bundle(run_dir, bundle)
ensure_dir(RUNS_DIR)
write_json(RUNS_DIR / f"{run_id}.json", bundle)
render.render_dashboard()
print(bundle)
result = _execute_case(args.case, run_id=args.run_id, dry_run=args.dry_run, sync_outputs=not args.dry_run)
print(result)
return 0
@@ -225,7 +357,9 @@ def cmd_run_system(args) -> int:
advisories = [item for item in load_json_dir(ADVISORIES_DIR) if item.get("system_id") == args.system]
selected = advisories[: args.limit]
for advisory in selected:
cmd_run_case(argparse.Namespace(case=advisory["canonical_id"], run_id=None, dry_run=args.dry_run))
_execute_case(advisory["canonical_id"], run_id=None, dry_run=args.dry_run, sync_outputs=False)
if selected and not args.dry_run:
_sync_registry_outputs()
print({"system": args.system, "count": len(selected)})
return 0
@@ -237,7 +371,9 @@ def cmd_run_batch(args) -> int:
task_queue.enqueue_from_registry(only_hotlane=args.only_hotlane, limit=args.limit)
items = task_queue.dequeue(limit=args.limit)
for item in items:
cmd_run_case(argparse.Namespace(case=item["advisory_id"], run_id=None, dry_run=args.dry_run))
_execute_case(item["advisory_id"], run_id=None, dry_run=args.dry_run, sync_outputs=False)
if items and not args.dry_run:
_sync_registry_outputs()
print({"processed": len(items)})
return 0

7
scripts/lab/provision.py
查看文件

@@ -3,15 +3,17 @@ from __future__ import annotations
from pathlib import Path
from typing import Any, Dict
from lab.compose import generate_compose
from lab.compose import compose_payload, generate_compose
from lab.utils import command_available, run
def prepare(profile: Dict[str, Any], run_dir: Path, dry_run: bool = False) -> Dict[str, Any]:
compose_path, payload = generate_compose(profile, run_dir)
payload = compose_payload(profile)
compose_path = run_dir / "compose" / "compose.yaml"
result = {
"compose_path": str(compose_path),
"service_count": len(payload.get("services", {})),
"compose_preview": payload,
"docker_available": command_available("docker"),
"status": "ready",
}
@@ -19,6 +21,7 @@ def prepare(profile: Dict[str, Any], run_dir: Path, dry_run: bool = False) -> Di
result["status"] = "planned"
return result
compose_path, payload = generate_compose(profile, run_dir)
if not result["docker_available"]:
result["status"] = "blocked-artifact"
result["blocked_reason"] = "docker unavailable on this machine"

303
scripts/lab/render.py
查看文件

@@ -1,11 +1,12 @@
from __future__ import annotations
import html
import os
from pathlib import Path
from typing import Any, Dict, List
from lab.config import CASE_RUNS_DIR, DASHBOARD_DIR, RUNS_DIR
from lab.utils import ensure_dir, load_json_dir, read_json, write_json, write_text
from lab.config import ADVISORIES_DIR, CASE_RUNS_DIR, DASHBOARD_DIR, RUNS_DIR
from lab.utils import ensure_dir, isoformat, load_json_dir, now_utc, write_json, write_text
def mermaid_from_steps(run: Dict[str, Any]) -> str:
@@ -24,11 +25,29 @@ def mermaid_from_steps(run: Dict[str, Any]) -> str:
return "\n".join(lines)
def _relative_ref(run_dir: Path, ref: str) -> str:
try:
return str(Path(ref).resolve().relative_to(run_dir.resolve()))
except ValueError:
return ref
def _dashboard_ref(run: Dict[str, Any], ref: str) -> str:
try:
bundle_dir = Path(run["report_refs"]["bundle_dir"]).resolve()
relative = Path(ref).resolve().relative_to(bundle_dir)
return f"./runs/{run['run_id']}/{relative.as_posix()}"
except Exception:
return ref
def render_run(run: Dict[str, Any]) -> Dict[str, str]:
run_dir = CASE_RUNS_DIR / run["run_id"]
ensure_dir(run_dir / "assets")
timeline_path = run_dir / "timeline.mmd"
write_text(timeline_path, mermaid_from_steps(run))
screenshot_refs = [ref for ref in run.get("browser_refs", []) if ref.endswith((".png", ".jpg", ".jpeg"))]
relative_screenshots = [_relative_ref(run_dir, ref) for ref in screenshot_refs]
md_lines = [
f"# Run {run['run_id']}",
@@ -44,42 +63,92 @@ def render_run(run: Dict[str, Any]) -> Dict[str, str]:
f"- 启动时间: `{run['started_at']}`",
f"- 完成时间: `{run['finished_at']}`",
f"- 阻塞原因: `{run.get('blocked_reason') or '-'}`",
f"- Compose 服务: `{', '.join(run.get('compose_services', [])) or '-'}`",
"",
"## 运行时间线",
"",
f"- Mermaid: [{timeline_path.name}]({timeline_path})",
"",
"## 证据摘要",
"",
f"- Baseline: `{len(run.get('baseline_refs', []))}`",
f"- 攻击步骤: `{len(run.get('attack_steps', []))}`",
f"- 浏览器证据: `{len(run.get('browser_refs', []))}`",
f"- 容器日志: `{len(run.get('container_log_refs', []))}`",
f"- 请求日志: `{len(run.get('request_log_refs', []))}`",
"",
"## 最小化验证说明",
"",
"- 仅限自有资产、本地靶场或已授权实验目标。",
"- 默认执行 minimal-proof；不会把破坏性或不可回滚动作作为默认路径。",
"",
"| 时间 | 步骤 | 状态 | 说明 |",
"|------|------|------|------|",
]
if run.get("timeline"):
for item in run["timeline"]:
md_lines.append(
f"| `{item.get('at', '')}` | `{item.get('step', '')}` | `{item.get('status', '')}` | {item.get('detail', '') or '-'} |"
)
else:
md_lines.append("| `-` | `-` | `-` | 无时间线 |")
md_lines.extend(
[
"",
"## Compose 拓扑",
"",
f"- Compose 文件: `{', '.join(run.get('compose_refs', [])) or '-'}`",
f"- 服务列表: `{', '.join(run.get('compose_services', [])) or '-'}`",
"",
"## 攻击步骤",
"",
"| 工具/步骤 | 状态 | 结果 |",
"|-----------|------|------|",
]
)
if run.get("attack_steps"):
for step in run["attack_steps"]:
outcome = step.get("result_path") or step.get("detail") or "-"
md_lines.append(f"| `{step.get('tool') or step.get('kind')}` | `{step.get('status', '-')}` | `{outcome}` |")
else:
md_lines.append("| `-` | `skipped` | `no attack steps` |")
md_lines.extend(
[
"",
"## 证据摘要",
"",
f"- Baseline: `{len(run.get('baseline_refs', []))}`",
f"- 攻击步骤: `{len(run.get('attack_steps', []))}`",
f"- 浏览器证据: `{len(run.get('browser_refs', []))}`",
f"- 容器日志: `{len(run.get('container_log_refs', []))}`",
f"- 请求日志: `{len(run.get('request_log_refs', []))}`",
"",
]
)
if relative_screenshots:
md_lines.extend(["## 浏览器截图", ""])
for ref in relative_screenshots:
md_lines.append(f"![{Path(ref).stem}]({ref})")
md_lines.append("")
if run.get("browser_refs"):
md_lines.extend(["## 浏览器证据", ""])
for ref in run["browser_refs"]:
md_lines.append(f"- {ref}")
md_lines.append(f"- `{_relative_ref(run_dir, ref)}`")
md_lines.append("")
if run.get("container_log_refs"):
md_lines.extend(["## 容器日志", ""])
for ref in run["container_log_refs"]:
md_lines.append(f"- {ref}")
md_lines.append(f"- `{_relative_ref(run_dir, ref)}`")
md_lines.append("")
if run.get("request_log_refs"):
md_lines.extend(["## 请求与基线日志", ""])
for ref in run["request_log_refs"]:
md_lines.append(f"- `{_relative_ref(run_dir, ref)}`")
md_lines.append("")
md_lines.extend(
[
"## 最小化验证说明",
"",
"- 仅限自有资产、本地靶场或已授权实验目标。",
"- 默认执行 minimal-proof；不会把破坏性或不可回滚动作作为默认路径。",
"- 若浏览器证据缺失，前端类案例不会被标为 `verified-*`。",
"",
]
)
report_md = run_dir / "report.md"
write_text(report_md, "\n".join(md_lines))
html_body = [
"<!doctype html>",
"<html><head><meta charset='utf-8'><title>websafe run report</title>",
"<style>body{font-family:ui-monospace,Menlo,monospace;margin:2rem;line-height:1.5;} code,pre{background:#f5f5f5;padding:.2rem .4rem;} .grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem;} .card{border:1px solid #ddd;padding:1rem;border-radius:.5rem;}</style>",
"<style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;line-height:1.55;background:#f8fafc;color:#0f172a;} code,pre{background:#e2e8f0;padding:.2rem .4rem;border-radius:.3rem;} pre{white-space:pre-wrap;} .grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:1rem;} .card{border:1px solid #cbd5e1;padding:1rem;border-radius:.75rem;background:#fff;} table{width:100%;border-collapse:collapse;background:#fff;border:1px solid #cbd5e1;border-radius:.75rem;overflow:hidden;} th,td{padding:.75rem;border-bottom:1px solid #e2e8f0;text-align:left;vertical-align:top;} img{max-width:100%;border:1px solid #cbd5e1;border-radius:.5rem;} .gallery{display:grid;grid-template-columns:repeat(auto-fit,minmax(320px,1fr));gap:1rem;}</style>",
"</head><body>",
f"<h1>Run {html.escape(run['run_id'])}</h1>",
"<div class='grid'>",
@@ -90,10 +159,42 @@ def render_run(run: Dict[str, Any]) -> Dict[str, str]:
"</div>",
"<h2>Mermaid Timeline</h2>",
f"<pre>{html.escape(mermaid_from_steps(run))}</pre>",
"<h2>Evidence</h2><ul>",
"<h2>Timeline</h2>",
"<table><thead><tr><th>Time</th><th>Step</th><th>Status</th><th>Detail</th></tr></thead><tbody>",
]
for ref in run.get("browser_refs", []) + run.get("container_log_refs", []) + run.get("request_log_refs", []):
html_body.append(f"<li><code>{html.escape(ref)}</code></li>")
if run.get("timeline"):
for item in run["timeline"]:
html_body.append(
"<tr>"
f"<td><code>{html.escape(item.get('at', ''))}</code></td>"
f"<td><code>{html.escape(item.get('step', ''))}</code></td>"
f"<td><code>{html.escape(item.get('status', ''))}</code></td>"
f"<td>{html.escape(item.get('detail', '') or '-')}</td>"
"</tr>"
)
html_body.extend(["</tbody></table>", "<h2>Attack Steps</h2>", "<table><thead><tr><th>Tool</th><th>Status</th><th>Output</th></tr></thead><tbody>"])
if run.get("attack_steps"):
for step in run["attack_steps"]:
html_body.append(
"<tr>"
f"<td><code>{html.escape(step.get('tool') or step.get('kind') or '-')}</code></td>"
f"<td><code>{html.escape(step.get('status', '-'))}</code></td>"
f"<td><code>{html.escape(step.get('result_path') or '-')}</code></td>"
"</tr>"
)
else:
html_body.append("<tr><td><code>-</code></td><td><code>skipped</code></td><td><code>no attack steps</code></td></tr>")
html_body.extend(["</tbody></table>"])
if relative_screenshots:
html_body.extend(["<h2>Browser Screenshots</h2>", "<div class='gallery'>"])
for ref in relative_screenshots:
html_body.append(
f"<figure><img src='{html.escape(ref)}' alt='{html.escape(Path(ref).stem)}'><figcaption><code>{html.escape(ref)}</code></figcaption></figure>"
)
html_body.append("</div>")
html_body.extend(["<h2>Evidence</h2><ul>"])
for ref in run.get("compose_refs", []) + run.get("browser_refs", []) + run.get("container_log_refs", []) + run.get("request_log_refs", []):
html_body.append(f"<li><code>{html.escape(_relative_ref(run_dir, ref))}</code></li>")
html_body.extend(["</ul>", "</body></html>"])
report_html = run_dir / "report.html"
write_text(report_html, "\n".join(html_body))
@@ -102,17 +203,102 @@ def render_run(run: Dict[str, Any]) -> Dict[str, str]:
def render_dashboard() -> Dict[str, str]:
ensure_dir(DASHBOARD_DIR)
advisory_records = load_json_dir(ADVISORIES_DIR)
runs = load_json_dir(RUNS_DIR)
runs_dir = DASHBOARD_DIR / "runs"
ensure_dir(runs_dir)
for item in runs:
bundle_dir = Path(item.get("report_refs", {}).get("bundle_dir", ""))
if not bundle_dir.exists():
continue
symlink_path = runs_dir / item["run_id"]
try:
if symlink_path.is_symlink() or symlink_path.exists():
if symlink_path.is_symlink() and symlink_path.resolve() == bundle_dir.resolve():
pass
else:
symlink_path.unlink()
os.symlink(bundle_dir, symlink_path, target_is_directory=True)
else:
os.symlink(bundle_dir, symlink_path, target_is_directory=True)
except OSError:
continue
systems: Dict[str, Dict[str, Any]] = {}
for advisory in advisory_records:
system = systems.setdefault(
advisory["system_id"],
{
"system_id": advisory["system_id"],
"display_name": advisory.get("display_name", advisory["system_id"]),
"total": 0,
"verified_real": 0,
"verified_synthetic": 0,
"blocked": 0,
"manual": 0,
"browser_required": 0,
"browser_present": 0,
"latest_update": "",
},
)
system["total"] += 1
status = advisory.get("verification_status", "triage-manual")
if status == "verified-real":
system["verified_real"] += 1
elif status == "verified-synthetic":
system["verified_synthetic"] += 1
elif status.startswith("blocked-"):
system["blocked"] += 1
else:
system["manual"] += 1
browser = advisory.get("browser_evidence", {})
if browser.get("required"):
system["browser_required"] += 1
if browser.get("present"):
system["browser_present"] += 1
latest = advisory.get("updated_at") or advisory.get("published_at") or ""
if latest > system["latest_update"]:
system["latest_update"] = latest
recent_runs = sorted(runs, key=lambda item: item.get("finished_at") or "", reverse=True)[:100]
decorated_runs: List[Dict[str, Any]] = []
for item in recent_runs:
cloned = dict(item)
cloned["dashboard_refs"] = {
"report_html": f"./runs/{item['run_id']}/report.html",
"report_md": f"./runs/{item['run_id']}/report.md",
"timeline": f"./runs/{item['run_id']}/timeline.mmd",
"bundle": f"./runs/{item['run_id']}/run.json",
}
cloned["browser_links"] = [_dashboard_ref(item, ref) for ref in item.get("browser_refs", [])]
cloned["container_links"] = [_dashboard_ref(item, ref) for ref in item.get("container_log_refs", [])]
cloned["request_links"] = [_dashboard_ref(item, ref) for ref in item.get("request_log_refs", [])]
decorated_runs.append(cloned)
summary = {
"generated_at": isoformat(now_utc()),
"advisory_count": len(advisory_records),
"run_count": len(runs),
"statuses": {},
"recent_runs": sorted(runs, key=lambda item: item.get("finished_at") or "", reverse=True)[:50],
"recent_failures": [],
}
for item in runs:
status = item.get("verification_status", "triage-manual")
summary["statuses"][status] = summary["statuses"].get(status, 0) + 1
summary["systems"] = sorted(systems.values(), key=lambda item: (-item["total"], item["system_id"]))
summary["recent_failures"] = [
{
"run_id": item["run_id"],
"advisory_id": item["advisory_id"],
"status": item.get("verification_status"),
"blocked_reason": item.get("blocked_reason"),
}
for item in decorated_runs
if item.get("verification_status") in {"triage-manual", "blocked-artifact", "blocked-destructive"}
][:20]
write_json(DASHBOARD_DIR / "summary.json", summary)
write_json(DASHBOARD_DIR / "runs.json", summary["recent_runs"])
write_json(DASHBOARD_DIR / "runs.json", decorated_runs)
write_json(DASHBOARD_DIR / "systems.json", summary["systems"])
html_page = """<!doctype html>
<html>
@@ -124,35 +310,86 @@ def render_dashboard() -> Dict[str, str]:
h1, h2 { margin-bottom: .5rem; }
.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 1rem; margin: 1rem 0 2rem; }
.card { background: white; border: 1px solid #cbd5e1; border-radius: 14px; padding: 1rem; box-shadow: 0 4px 18px rgba(15,23,42,.06); }
table { width: 100%%; border-collapse: collapse; background: white; border-radius: 12px; overflow: hidden; }
.filters { display:flex; flex-wrap:wrap; gap:.75rem; margin: 1rem 0; }
input, select { padding: .6rem .75rem; border: 1px solid #cbd5e1; border-radius: 10px; background: white; }
table { width: 100%%; border-collapse: collapse; background: white; border-radius: 12px; overflow: hidden; margin-bottom: 2rem; }
th, td { padding: .75rem; border-bottom: 1px solid #e2e8f0; text-align: left; font-size: .92rem; }
code { background: #e2e8f0; padding: .1rem .35rem; border-radius: 6px; }
.muted { color: #475569; }
</style>
</head>
<body>
<h1>websafe Local Lab Dashboard</h1>
<p>LAB ONLY | AUTHORIZED TARGETS ONLY | 本地静态看板</p>
<div id="summary" class="cards"></div>
<h2>Recent Runs</h2>
<h2>System Coverage</h2>
<table>
<thead><tr><th>Run</th><th>Advisory</th><th>Status</th><th>Mode</th><th>Finished</th><th>Report</th></tr></thead>
<thead><tr><th>System</th><th>Total</th><th>Verified Real</th><th>Verified Synthetic</th><th>Blocked</th><th>Manual</th><th>Browser</th><th>Latest</th></tr></thead>
<tbody id="systemRows"></tbody>
</table>
<h2>Recent Runs</h2>
<div class="filters">
<input id="search" placeholder="Search advisory or run id">
<select id="systemFilter"><option value="">All systems</option></select>
<select id="statusFilter"><option value="">All statuses</option></select>
<select id="familyFilter"><option value="">All profiles</option></select>
</div>
<table>
<thead><tr><th>Run</th><th>System</th><th>Advisory</th><th>Status</th><th>Mode</th><th>Profile</th><th>Finished</th><th>Artifacts</th></tr></thead>
<tbody id="rows"></tbody>
</table>
<script>
async function main() {
const summary = await fetch('./summary.json').then(r => r.json());
const runs = await fetch('./runs.json').then(r => r.json());
const [summary, runs, systems] = await Promise.all([
fetch('./summary.json').then(r => r.json()),
fetch('./runs.json').then(r => r.json()),
fetch('./systems.json').then(r => r.json())
]);
const summaryRoot = document.getElementById('summary');
const cards = [{label: 'Run Count', value: summary.run_count}];
const cards = [{label: 'Advisories', value: summary.advisory_count}, {label: 'Run Count', value: summary.run_count}];
for (const [key, value] of Object.entries(summary.statuses)) {
cards.push({label: key, value});
}
summaryRoot.innerHTML = cards.map(item => `<div class="card"><strong>${item.label}</strong><div style="font-size:2rem;margin-top:.5rem;">${item.value}</div></div>`).join('');
const systemRows = document.getElementById('systemRows');
systemRows.innerHTML = systems.map(item => `<tr><td><code>${item.system_id}</code></td><td>${item.total}</td><td>${item.verified_real}</td><td>${item.verified_synthetic}</td><td>${item.blocked}</td><td>${item.manual}</td><td>${item.browser_present}/${item.browser_required}</td><td>${item.latest_update || ''}</td></tr>`).join('');
const systemFilter = document.getElementById('systemFilter');
const statusFilter = document.getElementById('statusFilter');
const familyFilter = document.getElementById('familyFilter');
const search = document.getElementById('search');
const distinct = (values) => Array.from(new Set(values.filter(Boolean))).sort();
systemFilter.innerHTML += distinct(runs.map(item => item.system_id)).map(value => `<option value="${value}">${value}</option>`).join('');
statusFilter.innerHTML += distinct(runs.map(item => item.verification_status)).map(value => `<option value="${value}">${value}</option>`).join('');
familyFilter.innerHTML += distinct(runs.map(item => item.repro_profile_id)).map(value => `<option value="${value}">${value}</option>`).join('');
const rows = document.getElementById('rows');
rows.innerHTML = runs.map(item => {
const report = item.report_refs && item.report_refs.report_html ? item.report_refs.report_html : '';
return `<tr><td><code>${item.run_id}</code></td><td><code>${item.advisory_id}</code></td><td>${item.verification_status}</td><td>${item.verification_mode}</td><td>${item.finished_at || ''}</td><td>${report ? `<a href="../../../../${report.replace('/Users/x/websafe/', '')}">open</a>` : '-'}</td></tr>`;
}).join('');
function renderRows() {
const query = search.value.trim().toLowerCase();
const filtered = runs.filter(item => {
if (systemFilter.value && item.system_id !== systemFilter.value) return false;
if (statusFilter.value && item.verification_status !== statusFilter.value) return false;
if (familyFilter.value && item.repro_profile_id !== familyFilter.value) return false;
if (query) {
const haystack = `${item.run_id} ${item.advisory_id} ${item.system_id} ${item.repro_profile_id}`.toLowerCase();
if (!haystack.includes(query)) return false;
}
return true;
});
rows.innerHTML = filtered.map(item => {
const links = [];
if (item.dashboard_refs && item.dashboard_refs.report_html) links.push(`<a href="${item.dashboard_refs.report_html}">report</a>`);
if (item.dashboard_refs && item.dashboard_refs.timeline) links.push(`<a href="${item.dashboard_refs.timeline}">timeline</a>`);
if (item.dashboard_refs && item.dashboard_refs.bundle) links.push(`<a href="${item.dashboard_refs.bundle}">bundle</a>`);
if (item.browser_links && item.browser_links.length) links.push(`<a href="${item.browser_links[0]}">browser</a>`);
if (item.container_links && item.container_links.length) links.push(`<a href="${item.container_links[0]}">logs</a>`);
const reason = item.blocked_reason ? `<div class="muted">${item.blocked_reason}</div>` : '';
return `<tr><td><code>${item.run_id}</code>${reason}</td><td><code>${item.system_id}</code></td><td><code>${item.advisory_id}</code></td><td>${item.verification_status}</td><td>${item.verification_mode}</td><td><code>${item.repro_profile_id}</code></td><td>${item.finished_at || ''}</td><td>${links.join(' | ') || '-'}</td></tr>`;
}).join('');
}
[systemFilter, statusFilter, familyFilter, search].forEach(node => node.addEventListener('input', renderRows));
renderRows();
}
main();
</script>

55
scripts/lab/task_queue.py
查看文件

@@ -1,11 +1,15 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any, Dict, List
from lab.config import ADVISORIES_DIR, QUEUE_PATH
from lab.utils import load_json_dir, read_json, write_json
UTC = timezone.utc
def load_queue() -> Dict[str, Any]:
return read_json(QUEUE_PATH, default={"items": []}) or {"items": []}
@@ -28,15 +32,64 @@ def enqueue_items(items: List[Dict[str, Any]]) -> Dict[str, Any]:
return {"queued": len(queue["items"]), "added": added}
def _parse_iso(value: str | None) -> datetime:
if not value:
return datetime(1970, 1, 1, tzinfo=UTC)
try:
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC)
except ValueError:
return datetime(1970, 1, 1, tzinfo=UTC)
def _priority_tuple(advisory: Dict[str, Any], only_hotlane: bool) -> tuple[int, float]:
score = 0
verification_status = advisory.get("verification_status", "triage-manual")
if verification_status == "triage-manual":
score += 500
elif verification_status.startswith("blocked-"):
score += 450
elif verification_status == "verified-synthetic":
score += 300
else:
score += 150
last_verified = _parse_iso(advisory.get("last_verified_at"))
latest_upstream = max(_parse_iso(advisory.get("updated_at")), _parse_iso(advisory.get("published_at")))
if advisory.get("last_verified_at") is None:
score += 350
elif latest_upstream > last_verified:
score += 250
exploit_status = advisory.get("exploit_status")
if exploit_status in {"known_exploited", "active_exploitation", "in_the_wild"}:
score += 1000
severity = advisory.get("severity")
if severity == "critical":
score += 250
score += int((advisory.get("cvss_score") or 0) * 10)
if only_hotlane:
score += 100
return score, latest_upstream.timestamp()
def enqueue_from_registry(only_hotlane: bool = False, limit: int = 50) -> Dict[str, Any]:
advisories = load_json_dir(ADVISORIES_DIR)
advisories = sorted(advisories, key=lambda item: _priority_tuple(item, only_hotlane), reverse=True)
items = []
for advisory in advisories:
if only_hotlane:
hot = advisory.get("exploit_status") in {"known_exploited", "active_exploitation", "in_the_wild"}
if not hot and not (advisory.get("cvss_score") or 0) >= 8.8 and advisory.get("severity") != "critical":
continue
items.append({"advisory_id": advisory["canonical_id"], "system_id": advisory["system_id"], "priority": "hotlane" if only_hotlane else "default"})
items.append(
{
"advisory_id": advisory["canonical_id"],
"system_id": advisory["system_id"],
"priority": "hotlane" if only_hotlane else "default",
}
)
return enqueue_items(items[:limit])

21
scripts/lab/validators.py
查看文件

@@ -1,10 +1,12 @@
from __future__ import annotations
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Dict, List
from lab.compose import compose_payload
from lab.config import ENV_CATALOG_DIR, REPRO_MAP_PATH, REPRO_PROFILES_DIR
from lab.utils import read_yaml
from lab.utils import command_available, read_yaml, run, write_yaml
def validate_assets() -> List[str]:
@@ -32,4 +34,21 @@ def validate_assets() -> List[str]:
]:
if field not in content:
errors.append(f"repro profile missing {field}: {path}")
docker_available = command_available("docker")
profile_roots = sorted((ENV_CATALOG_DIR.parent.parent / "profiles").rglob("*.yaml"))
for path in profile_roots:
content = read_yaml(path, default=None)
if not isinstance(content, dict):
errors.append(f"invalid environment profile yaml: {path}")
continue
for field in ["profile_id", "system_id", "services", "cleanup_policy"]:
if field not in content:
errors.append(f"environment profile missing {field}: {path}")
if docker_available:
with TemporaryDirectory() as temp_dir:
compose_path = Path(temp_dir) / "compose.yaml"
write_yaml(compose_path, compose_payload(content))
result = run(["docker", "compose", "-f", str(compose_path), "config"], cwd=compose_path.parent)
if result.returncode != 0:
errors.append(f"docker compose config failed for {path}: {result.stderr.strip() or result.stdout.strip()}")
return errors