diff --git a/01-sql-injection/tools/sqli-exploit.go b/01-sql-injection/tools/sqli-exploit.go index 52ee94ca..05e4448a 100644 --- a/01-sql-injection/tools/sqli-exploit.go +++ b/01-sql-injection/tools/sqli-exploit.go @@ -164,11 +164,11 @@ func (s *SQLiExploit) TestErrorBased(payloads []struct { continue } - for dbms, pattern := range errorPatterns { - if strings.Contains(body, "SQL") || strings.Contains(body, "error") || - strings.Contains(body, "Error") || strings.Contains(body, "Warning") { - results = append(results, InjectionResult{ - Payload: p.Payload, + for dbms := range errorPatterns { + if strings.Contains(body, "SQL") || strings.Contains(body, "error") || + strings.Contains(body, "Error") || strings.Contains(body, "Warning") { + results = append(results, InjectionResult{ + Payload: p.Payload, VulnType: "Error-based", DBMS: dbms, ResponseLen: respLen, diff --git a/02-xss/README.md b/02-xss/README.md index 5d3f2da0..18b82f9f 100644 --- a/02-xss/README.md +++ b/02-xss/README.md @@ -14,9 +14,12 @@ ## 当前内容 - 工具: [xss-fuzzer.py](/Users/x/websafe/02-xss/tools/xss-fuzzer.py), [xss-scanner.go](/Users/x/websafe/02-xss/tools/xss-scanner.go) -- 主题扩展: [前端与框架案例](/Users/x/websafe/07-framework-security/frontend-js/README.md) +- 主题扩展: [frameworks/README.md](/Users/x/websafe/07-framework-security/frameworks/README.md) +- 重点系统: [React](/Users/x/websafe/07-framework-security/frameworks/react/README.md), [Next.js](/Users/x/websafe/07-framework-security/frameworks/nextjs/README.md), [Vue](/Users/x/websafe/07-framework-security/frameworks/vue/README.md), [Nuxt](/Users/x/websafe/07-framework-security/frameworks/nuxt/README.md), [Vite](/Users/x/websafe/07-framework-security/frameworks/vite/README.md) +- 实证链路: `xss-fuzzer/xss-scanner -> Playwright 回放 -> run bundle -> case/index 回写` -## 当前缺口 +## 当前状态 -- `defense/`, `exploitation/`, `payloads/` 仍需补充实验专用内容 -- CSP、Trusted Types、Token 存储和前端敏感配置暴露已经转入 [07-framework-security/frontend-js](/Users/x/websafe/07-framework-security/frontend-js/README.md) +- `defense/`, `exploitation/`, `payloads/` 仍保留实验载荷与说明位，但主题主索引已迁到 `07-framework-security/frameworks/*` +- CSP、Trusted Types、Token 存储和前端敏感配置暴露通过系统页和 `05-defense/secure-code/*` 反向关联 +- 前端类 case 默认要求浏览器层证据；只有 HTTP 命中而无回放时，不记为 `verified-*` diff --git a/03-authentication/README.md b/03-authentication/README.md index 116bc01b..1505e1b2 100644 --- a/03-authentication/README.md +++ b/03-authentication/README.md @@ -15,8 +15,9 @@ - 暴力破解工具: [web-brute.py](/Users/x/websafe/03-authentication/bruteforce/tools/web-brute.py) - JWT 工具: [jwt-cracker.py](/Users/x/websafe/03-authentication/jwt/tools/jwt-cracker.py) +- 会话边界工具: [session-lab.py](/Users/x/websafe/03-authentication/session/tools/session-lab.py) - 会话与 Token 风险案例: [福建案例总结](/Users/x/websafe/06-case-studies/fujian-gov-procurement/lessons-learned.md) ## 说明 -该目录聚焦“验证控制面是否存在”而不是“最大化拿下账户”。对公网授权目标的验证应优先采用小样本、低频和可审计的实验方法。 +该目录聚焦“验证控制面是否存在”而不是“最大化拿下账户”。对公网授权目标的验证应优先采用小样本、低频和可审计的实验方法，并把浏览器存储、Cookie 属性和会话固定证据回写到 run bundle。 diff --git a/03-authentication/session/README.md b/03-authentication/session/README.md index d0b72ac8..725298c7 100644 --- a/03-authentication/session/README.md +++ b/03-authentication/session/README.md @@ -1,5 +1,18 @@ # 会话实验 -> `LAB NOTE` | `规划中` +> `LAB ONLY` | `AUTHORIZED TARGETS ONLY` -该目录预留给 Cookie 属性、会话固定、登出失效、Token 轮换与浏览器存储对照实验。当前由 [07-framework-security/frontend-js](/Users/x/websafe/07-framework-security/frontend-js/README.md) 和现有案例文档承担相关内容。 +## 范围元数据 + +| 字段 | 内容 | +|------|------| +| 适用目标类型 | `lab-local`, `lab-public`, `authorized-third-party` | +| 是否允许公网验证 | 允许，但只做 Cookie / Storage / Header 边界检查 | +| 推荐最小化验证 | 读取响应头、Cookie 属性、DOM 中的存储 API 使用痕迹 | +| 禁止场景 | 真实账户会话接管、窃取真实令牌、对第三方浏览器会话做持久利用 | + +## 当前内容 + +- 工具: [session-lab.py](/Users/x/websafe/03-authentication/session/tools/session-lab.py) +- 关联系统: [frameworks/README.md](/Users/x/websafe/07-framework-security/frameworks/README.md), [platforms/README.md](/Users/x/websafe/07-framework-security/platforms/README.md) +- 修复主题: [token-cookie-storage](/Users/x/websafe/05-defense/secure-code/nodejs/token-cookie-storage.md), [proxy-trust-boundary](/Users/x/websafe/05-defense/secure-code/nodejs/proxy-trust-boundary.md) diff --git a/03-authentication/session/tools/README.md b/03-authentication/session/tools/README.md index 3bf252e7..35921a1e 100644 --- a/03-authentication/session/tools/README.md +++ b/03-authentication/session/tools/README.md @@ -1,5 +1,22 @@ # 会话工具说明 -> `LAB NOTE` | `规划中` +> `LAB ONLY` | `AUTHORIZED TARGETS ONLY` -该目录保留给浏览器存储、Cookie 属性和会话边界相关工具。当前尚未补齐具体脚本。 +## 工具 + +- [session-lab.py](/Users/x/websafe/03-authentication/session/tools/session-lab.py) + - 用途: Cookie 属性、浏览器存储、可疑认证头与代理边界检查 + - 目标范围: `lab-local`, `lab-public`, `authorized-third-party` + - 允许公网验证: `yes`, 但仅限只读检查 + - 风险: 低，仅请求目标页面并提取响应头/前端存储痕迹 + - 不适用: 未授权目标、真实账户会话接管、真实令牌收集 + +## 示例 + +```bash +python3 /Users/x/websafe/03-authentication/session/tools/session-lab.py \ + --target http://127.0.0.1:18085/ \ + --ack-authorized \ + --format json \ + --evidence-dir /tmp/websafe-session-evidence +``` diff --git a/04-server-security/README.md b/04-server-security/README.md index 5a25e5f5..9d61cfb5 100644 --- a/04-server-security/README.md +++ b/04-server-security/README.md @@ -16,6 +16,7 @@ - 端口与服务指纹: [port-scanner.py](/Users/x/websafe/04-server-security/scanning/tools/port-scanner.py) - TLS 配置检查: [tls-scanner.py](/Users/x/websafe/04-server-security/tls/tools/tls-scanner.py) - 关联面分析: [site-scope-mapper.py](/Users/x/websafe/04-server-security/infrastructure/tools/site-scope-mapper.py) +- 错误配置验证: [misconfig-lab.py](/Users/x/websafe/04-server-security/misconfiguration/tools/misconfig-lab.py) - 实验网关样例: [nginx-hardening.conf](/Users/x/websafe/05-defense/hardening/nginx-hardening.conf) ## 建议实验路径 @@ -23,4 +24,5 @@ 1. 用 TLS 与响应头检查判断暴露面。 2. 用端口扫描确认最小服务面。 3. 用关联面分析确认同 IP、同证书和同代理边界。 -4. 将结果回填到 [资产模板](/Users/x/websafe/09-scope-and-targeting/asset-inventory-template.md) 与 [测试记录模板](/Users/x/websafe/09-scope-and-targeting/test-record-template.md)。 +4. 对默认页面、调试接口、管理端口和代理信任边界做最小化验证。 +5. 将结果回填到 [资产模板](/Users/x/websafe/09-scope-and-targeting/asset-inventory-template.md) 与 [测试记录模板](/Users/x/websafe/09-scope-and-targeting/test-record-template.md)。 diff --git a/04-server-security/misconfiguration/README.md b/04-server-security/misconfiguration/README.md index 0033c031..95aebfd7 100644 --- a/04-server-security/misconfiguration/README.md +++ b/04-server-security/misconfiguration/README.md @@ -1,5 +1,18 @@ # 服务端错误配置实验 -> `LAB NOTE` | `规划中` +> `LAB ONLY` | `AUTHORIZED TARGETS ONLY` -该目录预留给默认目录列表、错误暴露、调试接口、代理信任链和配置合并问题的实验样例。当前相关内容分散在 [07-framework-security/server-software](/Users/x/websafe/07-framework-security/server-software/README.md) 与已有案例中。 +## 范围元数据 + +| 字段 | 内容 | +|------|------| +| 适用目标类型 | `lab-local`, `lab-public`, `authorized-third-party` | +| 是否允许公网验证 | 允许，但必须限定为单站点最小化检查 | +| 推荐最小化验证 | 仅访问常见调试、默认页面和健康检查路径 | +| 禁止场景 | 大范围爆破目录、未授权管理面探测、影响业务可用性 | + +## 当前内容 + +- 工具: [misconfig-lab.py](/Users/x/websafe/04-server-security/misconfiguration/tools/misconfig-lab.py) +- 关联系统: [servers/README.md](/Users/x/websafe/07-framework-security/servers/README.md), [platforms/README.md](/Users/x/websafe/07-framework-security/platforms/README.md) +- 修复主题: [proxy-trust-boundary](/Users/x/websafe/05-defense/secure-code/nodejs/proxy-trust-boundary.md), [path-traversal-guard](/Users/x/websafe/05-defense/secure-code/nodejs/path-traversal-guard.md) diff --git a/04-server-security/misconfiguration/tools/README.md b/04-server-security/misconfiguration/tools/README.md index 9096eca8..39487530 100644 --- a/04-server-security/misconfiguration/tools/README.md +++ b/04-server-security/misconfiguration/tools/README.md @@ -1,5 +1,22 @@ # 错误配置工具说明 -> `LAB NOTE` | `规划中` +> `LAB ONLY` | `AUTHORIZED TARGETS ONLY` -该目录后续用于默认配置、目录暴露、调试接口和信任边界误配的辅助检查脚本。 +## 工具 + +- [misconfig-lab.py](/Users/x/websafe/04-server-security/misconfiguration/tools/misconfig-lab.py) + - 用途: 默认页面、调试接口、目录暴露、管理端口和危险头部的最小化验证 + - 目标范围: `lab-local`, `lab-public`, `authorized-third-party` + - 允许公网验证: `yes`, 但必须限定到已授权单目标 + - 风险: 低，只做固定路径的 GET 检查 + - 不适用: 未授权站点目录枚举、批量互联网画像、DoS/高频探测 + +## 示例 + +```bash +python3 /Users/x/websafe/04-server-security/misconfiguration/tools/misconfig-lab.py \ + --target http://127.0.0.1:18086/ \ + --ack-authorized \ + --format json \ + --evidence-dir /tmp/websafe-misconfig-evidence +``` diff --git a/08-threat-intel/README.md b/08-threat-intel/README.md index 88d04836..16e394c7 100644 --- a/08-threat-intel/README.md +++ b/08-threat-intel/README.md @@ -8,16 +8,24 @@ - [source-map.yaml](/Users/x/websafe/08-threat-intel/source-map.yaml) - 全库唯一真值配置，定义系统范围、覆盖策略、source adapter、输出目录和 secure-code 主题。 +- [repro-map.yaml](/Users/x/websafe/08-threat-intel/repro-map.yaml) + - 系统到 repro family、浏览器要求、seed 策略、日志策略和报告模板的映射。 +- [repro-profiles/](/Users/x/websafe/08-threat-intel/repro-profiles) + - advisory 级和 family 级复现描述，供 `scripts/lab/main.py` 路由执行。 - [registry/advisories/](/Users/x/websafe/08-threat-intel/registry/advisories) - canonical advisory 级 JSON 记录，是“所有具体案例”的正式载体。 - [registry/systems/](/Users/x/websafe/08-threat-intel/registry/systems) - 每个系统的统计索引、最近更新时间和案例列表。 +- [registry/runs/](/Users/x/websafe/08-threat-intel/registry/runs) + - 每次本地验证的 run bundle 元数据真值，用于反向回填案例页和 dashboard。 - [registry/triage/](/Users/x/websafe/08-threat-intel/registry/triage) - 无法自动确定版本、来源冲突或只有弱来源支持的候选。 - [generated/coverage-matrix.md](/Users/x/websafe/08-threat-intel/generated/coverage-matrix.md) - 全局覆盖矩阵，展示每个系统的 tier、registry 数、Markdown 数和自动同步状态。 - [generated/latest-ingest.md](/Users/x/websafe/08-threat-intel/generated/latest-ingest.md) - 最近一次同步摘要。 +- [generated/dashboard/](/Users/x/websafe/08-threat-intel/generated/dashboard) + - 本地 Web 看板静态产物，展示系统覆盖、状态分布、失败 blocker 和 run bundle 链接。 - [registry/source-confidence.md](/Users/x/websafe/08-threat-intel/registry/source-confidence.md) - `official`、`ecosystem-authority`、`research`、`triage-only` 的入库规则。 @@ -51,6 +59,11 @@ python3 /Users/x/websafe/scripts/intel/main.py reconcile python3 /Users/x/websafe/scripts/intel/main.py backfill --tier rolling-24m --dry-run python3 /Users/x/websafe/scripts/intel/main.py ingest --since 365d --system nextjs --system vite python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run +python3 /Users/x/websafe/scripts/lab/main.py validate +python3 /Users/x/websafe/scripts/lab/main.py run-case --case gitea--CVE-2025-68939 +python3 /Users/x/websafe/scripts/lab/main.py run-case --case nextjs--CVE-2025-29927 --dry-run +python3 /Users/x/websafe/scripts/lab/main.py run-batch --only-hotlane --limit 10 +python3 /Users/x/websafe/scripts/lab/main.py serve-dashboard --port 8734 ``` 可选环境变量： @@ -63,6 +76,7 @@ python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run 运行时建议： - 使用独立虚拟环境安装 [requirements-intel.txt](/Users/x/websafe/requirements-intel.txt)。 +- 浏览器类 case 需要执行 `python3 -m playwright install chromium` 安装浏览器运行时。 - 当前机器上的 Python 3.9 + LibreSSL 对部分 HTTPS 源可能出现 `SSLError`；`urllib3<2` 已写入依赖约束以降低兼容性问题。 对应的本机 cron 入口： @@ -81,9 +95,12 @@ python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run - [intake-and-severity-rules.md](/Users/x/websafe/08-threat-intel/intake-and-severity-rules.md) - [case-intake-template.md](/Users/x/websafe/08-threat-intel/case-intake-template.md) - [config-examples/README.md](/Users/x/websafe/08-threat-intel/config-examples/README.md) +- [generated/dashboard/index.html](/Users/x/websafe/08-threat-intel/generated/dashboard/index.html) +- [registry/runs](/Users/x/websafe/08-threat-intel/registry/runs) ## 实验边界 - 所有案例、source adapter 和索引页仅适用于 `lab-local`、`lab-public`、`authorized-third-party`。 - 允许公网可达目标，但前提必须是资产归属明确，或已获得明确授权。 - 不面向未授权互联网资产，不面向公共知名网站，不作为泛化枚举或生产推荐语境。 +- 前端 / 浏览器类 advisory 若没有浏览器层回放与证据，只能停留在 `triage-manual` 或 `suspected`，不得宣称完成实证。 diff --git a/README.md b/README.md index 48c69e10..a5578606 100644 --- a/README.md +++ b/README.md @@ -17,30 +17,31 @@ ```text websafe/ -├── 00-environments/ # 靶场与本地实验编排 +├── 00-environments/ # 系统 catalog、真实版本/当前版本 profile、synthetic 模板 ├── 01-sql-injection/ # SQL 注入实验 ├── 02-xss/ # XSS 与浏览器端注入实验 ├── 03-authentication/ # 认证、会话与 JWT 实验 ├── 04-server-security/ # 服务器、TLS、暴露面与关联面实验 ├── 05-defense/ # 检测、观测、实验对照与代码修复示例 -├── 06-case-studies/ # 授权案例与原始报告归档 +├── 06-case-studies/ # 授权案例与 run bundle / 报告归档 ├── 07-framework-security/ # CMS、电商、框架、服务器、平台系统安全 -├── 08-threat-intel/ # source-map、registry、generated、订阅规则、自动入库 +├── 08-threat-intel/ # source-map、repro-map、registry、dashboard、订阅规则、自动入库 ├── 09-scope-and-targeting/ # 授权模型、资产模板、测试记录模板 -├── requirements-intel.txt # intel 自动化依赖 -└── scripts/intel/ # hotlane / ingest / reconcile / backfill / open-pr CLI +├── requirements-intel.txt # intel + lab 自动化依赖（含 Playwright Python 包） +├── scripts/intel/ # hotlane / ingest / reconcile / backfill / open-pr CLI +└── scripts/lab/ # provision / baseline / attack / browser / evidence / render / queue CLI ``` ## 能力矩阵 -| 覆盖域 | 历史全量策略 | 近两年策略 | 全量 registry | 重点案例 Markdown | secure-code 关联 | 自动同步状态 | -|--------|--------------|------------|---------------|--------------------|------------------|--------------| -| CMS / 内容平台 | `WordPress`, `Drupal`, `Joomla` | `Ghost`, `Strapi`, `Directus`, `MediaWiki`, `Moodle`, `Discourse` | `registry/advisories + registry/systems` | `core 全量 + 高价值 extension` | `yes` | `render / ingest / hotlane / reconcile ready` | -| 电商系统 | `Adobe Commerce`, `Magento Open Source`, `WooCommerce`, `PrestaShop`, `Shopware`, `OpenCart` | `OpenMage`, `Saleor`, `Medusa` | `registry/advisories + registry/systems` | `core 全量 + 高价值 module` | `yes` | `render / ingest / hotlane / reconcile ready` | -| Web 框架与运行时 | `React`, `Next.js`, `Vue`, `Nuxt`, `Vite`, `Node.js`, `Nginx`, `Apache HTTP Server`, `Apache Tomcat` | 其余主流框架与运行时按 `rolling-24m` | `registry/advisories + registry/systems` | `core 全量 + 高价值 package` | `yes` | `render / ingest / hotlane / reconcile ready` | -| 开源平台与后台系统 | `history-full` 不强制 | `phpMyAdmin`, `Adminer`, `Gitea`, `GitLab CE`, `Jenkins`, `Grafana`, `Kibana`, `Mattermost`, `Redmine` | `registry/advisories + registry/systems` | `高价值案例输出` | `yes` | `render / ingest / hotlane / reconcile ready` | -| 修复示例库 | 不适用 | 不适用 | 不适用 | 由案例页反向链接 | `javascript-typescript`, `nodejs`, `java`, `php`, `python`, `ruby`, `csharp`, `go` | `render ready` | -| 自动化入库 | `backfill --tier history-full` | `ingest --since`, `reconcile` | `registry + generated` | `基于 render_policy` | `front matter 反向链接` | `open-pr dry-run ready` | +| 覆盖域 | 历史全量策略 | 近两年策略 | 全量 registry | 重点案例 Markdown | secure-code 关联 | 本地实证状态 | 浏览器证据 | run bundle | 看板展示 | 自动同步状态 | +|--------|--------------|------------|---------------|--------------------|------------------|--------------|------------|-----------|----------|--------------| +| CMS / 内容平台 | `WordPress`, `Drupal`, `Joomla` | `Ghost`, `Strapi`, `Directus`, `MediaWiki`, `Moodle`, `Discourse` | `registry/advisories + registry/systems` | `core 全量 + 高价值 extension` | `yes` | `verified-real / verified-synthetic / blocked-* / triage-manual` | `前端类强制` | `06-case-studies/generated-runs` | `dashboard + report` | `render / ingest / hotlane / reconcile ready` | +| 电商系统 | `Adobe Commerce`, `Magento Open Source`, `WooCommerce`, `PrestaShop`, `Shopware`, `OpenCart` | `OpenMage`, `Saleor`, `Medusa` | `registry/advisories + registry/systems` | `core 全量 + 高价值 module` | `yes` | `同上` | `前台/后台面板类强制` | `run bundle + logs` | `dashboard + report` | `render / ingest / hotlane / reconcile ready` | +| Web 框架与运行时 | `React`, `Next.js`, `Vue`, `Nuxt`, `Vite`, `Node.js`, `Nginx`, `Apache HTTP Server`, `Apache Tomcat` | 其余主流框架与运行时按 `rolling-24m` | `registry/advisories + registry/systems` | `core 全量 + 高价值 package` | `yes` | `family runner + advisory profile` | `浏览器/HTTP 混合` | `run bundle + timeline` | `dashboard + report` | `render / ingest / hotlane / reconcile ready` | +| 开源平台与后台系统 | `history-full` 不强制 | `phpMyAdmin`, `Adminer`, `Gitea`, `GitLab CE`, `Jenkins`, `Grafana`, `Kibana`, `Mattermost`, `Redmine` | `registry/advisories + registry/systems` | `高价值案例输出` | `yes` | `真实版本优先` | `Web 面板类强制` | `run bundle + screenshots` | `dashboard + report` | `render / ingest / hotlane / reconcile ready` | +| 修复示例库 | 不适用 | 不适用 | 不适用 | 由案例页反向链接 | `javascript-typescript`, `nodejs`, `java`, `php`, `python`, `ruby`, `csharp`, `go` | `由案例反向映射` | `不适用` | `不适用` | `索引页` | `render ready` | +| 自动化入库与实证 | `backfill --tier history-full` | `ingest --since`, `reconcile` | `registry + generated + registry/runs` | `基于 render_policy` | `front matter 反向链接` | `queue + run-case / run-batch` | `Playwright required for browser cases` | `report.md / report.html / timeline.mmd` | `serve-dashboard` | `open-pr / cron ready` | ## 当前覆盖对象 @@ -62,13 +63,30 @@ python3 /Users/x/websafe/scripts/intel/main.py ingest --since last-success python3 /Users/x/websafe/scripts/intel/main.py reconcile python3 /Users/x/websafe/scripts/intel/main.py backfill --tier history-full --dry-run python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run +python3 /Users/x/websafe/scripts/lab/main.py catalog sync +python3 /Users/x/websafe/scripts/lab/main.py validate +python3 /Users/x/websafe/scripts/lab/main.py run-case --case nextjs--CVE-2025-29927 --dry-run +python3 /Users/x/websafe/scripts/lab/main.py run-batch --only-hotlane --limit 10 +python3 /Users/x/websafe/scripts/lab/main.py serve-dashboard --port 8734 ``` 计划中的本机 cron 入口： -- [run-hourly.sh](/Users/x/websafe/scripts/intel/run-hourly.sh) 只处理 KEV / 在野利用 / 极高优先级更新 -- [run-nightly.sh](/Users/x/websafe/scripts/intel/run-nightly.sh) 处理常规增量同步 -- [run-weekly-reconcile.sh](/Users/x/websafe/scripts/intel/run-weekly-reconcile.sh) 对齐最近 30 天更新 +- [run-hourly.sh](/Users/x/websafe/scripts/intel/run-hourly.sh) 处理 KEV / 在野利用 / 极高优先级更新，并触发 hotlane 实证队列 +- [run-nightly.sh](/Users/x/websafe/scripts/intel/run-nightly.sh) 处理常规增量同步、批量实证、dashboard 渲染和 PR +- [run-weekly-reconcile.sh](/Users/x/websafe/scripts/intel/run-weekly-reconcile.sh) 对齐最近 30 天更新，并重跑失败/阻塞任务 + +## 本地实证链路 + +每条 advisory 的自动链路固定为： + +1. `registry/advisories/*.json` 选中 case。 +2. `repro-map.yaml + repro-profiles/` 解析到 repro family / advisory profile。 +3. `00-environments/catalog + profiles` 生成 compose 拓扑和靶站参数。 +4. `scripts/lab/main.py run-case` 拉起环境、收集 baseline、执行受控攻击链。 +5. 前端类 case 强制走 Playwright 浏览器回放，生成截图、DOM、console、network 证据。 +6. 生成 `06-case-studies/generated-runs//` 报告和 `08-threat-intel/registry/runs/.json`。 +7. 自动回写 registry、系统 INDEX、案例页和 dashboard。 ## 实验边界 diff --git a/scripts/intel/pr.py b/scripts/intel/pr.py index 61b09f08..4a329d90 100644 --- a/scripts/intel/pr.py +++ b/scripts/intel/pr.py @@ -4,6 +4,7 @@ import os import re from datetime import datetime from typing import Optional +from urllib.parse import quote import requests @@ -13,16 +14,24 @@ from intel.utils import read_json, run PR_PATHS = [ "README.md", + "00-environments", + "01-sql-injection", + "02-xss", + "03-authentication", + "04-server-security", "05-defense/secure-code", + "06-case-studies/generated-runs", "07-framework-security", "08-threat-intel", "requirements-intel.txt", "scripts/intel", + "scripts/lab", + "scripts/tool_contract.py", ] def create_branch_name() -> str: - return "codex/intel-" + datetime.now().strftime("%Y%m%d-%H%M") + return "codex/intel-" + datetime.now().strftime("%Y%m%d-%H%M%S") def _parse_origin() -> Optional[dict]: @@ -42,6 +51,17 @@ def _changed_paths() -> list[str]: return lines +def _current_branch() -> str: + result = run(["git", "-C", str(ROOT), "branch", "--show-current"], check=False) + return result.stdout.strip() + + +def _push_remote(origin: dict, token: str | None) -> str: + if token and origin["url"].startswith("https://"): + return f"https://{quote(origin['owner'], safe='')}:{quote(token, safe='')}@{origin['host']}/{origin['owner']}/{origin['repo']}.git" + return "origin" + + def open_pr(base_branch: str = "main", dry_run: bool = False) -> str: origin = _parse_origin() if not origin: @@ -51,27 +71,36 @@ def open_pr(base_branch: str = "main", dry_run: bool = False) -> str: if not changed: return "No intel-related changes to submit" - branch = create_branch_name() + branch = _current_branch() + if not branch.startswith("codex/"): + branch = create_branch_name() if dry_run: preview = "\n".join(f"- {line}" for line in changed[:40]) return f"Dry run only; would create branch {branch} with these paths:\n{preview}" - run(["git", "-C", str(ROOT), "checkout", "-b", branch]) + if _current_branch() != branch: + run(["git", "-C", str(ROOT), "checkout", "-b", branch]) run(["git", "-C", str(ROOT), "add", "--", *PR_PATHS]) - run(["git", "-C", str(ROOT), "commit", "-m", f"intel: automated advisory ingest {branch}"]) - run(["git", "-C", str(ROOT), "push", "-u", "origin", branch]) + run(["git", "-C", str(ROOT), "commit", "-m", f"lab: automated intel and verification sync {branch}"]) token = os.environ.get("GITEA_TOKEN") + run(["git", "-C", str(ROOT), "push", "-u", _push_remote(origin, token), branch]) if not token: return f"Pushed branch {branch}, but GITEA_TOKEN is not set; PR not created" summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {} + dashboard = read_json(GENERATED_DIR / "dashboard" / "summary.json", default={}) or {} body_lines = [ - "Automated advisory ingest update.", + "Automated advisory ingest and local verification update.", "", f"- New advisories: {summary.get('new_count', 0)}", f"- Updated advisories: {summary.get('updated_count', 0)}", f"- Triage count: {summary.get('triage_count', 0)}", + f"- Run bundles: {summary.get('run_bundle_count', 0)}", + f"- verified-real: {dashboard.get('statuses', {}).get('verified-real', 0)}", + f"- verified-synthetic: {dashboard.get('statuses', {}).get('verified-synthetic', 0)}", + f"- blocked-artifact: {dashboard.get('statuses', {}).get('blocked-artifact', 0)}", + f"- triage-manual: {dashboard.get('statuses', {}).get('triage-manual', 0)}", f"- Failure count: {len(summary.get('failures', []))}", ] if summary.get("systems_touched"): @@ -80,6 +109,10 @@ def open_pr(base_branch: str = "main", dry_run: bool = False) -> str: body_lines.extend(["", "Failed source adapters:"]) for failure in summary["failures"]: body_lines.append(f"- {failure}") + if dashboard.get("recent_failures"): + body_lines.extend(["", "Recent repro blockers:"]) + for failure in dashboard["recent_failures"][:10]: + body_lines.append(f"- {failure['run_id']} :: {failure['status']} :: {failure.get('blocked_reason') or '-'}") payload = { "title": f"Intel ingest {branch}", diff --git a/scripts/intel/run-hourly.sh b/scripts/intel/run-hourly.sh index 29ba8bb4..3fa680e0 100644 --- a/scripts/intel/run-hourly.sh +++ b/scripts/intel/run-hourly.sh @@ -7,5 +7,19 @@ mkdir -p "$LOG_DIR" STAMP="$(date '+%Y%m%d-%H%M%S')" exec >> "$LOG_DIR/hourly-$STAMP.log" 2>&1 +run_pr() { + if [[ "${WEBSAFE_PR_MODE:-auto}" == "skip" ]]; then + echo "[hourly] PR skipped by WEBSAFE_PR_MODE=skip" + elif [[ "${WEBSAFE_PR_MODE:-auto}" == "dry-run" || -z "${GITEA_TOKEN:-}" ]]; then + python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run + else + python3 /Users/x/websafe/scripts/intel/main.py open-pr + fi +} + echo "[hourly] $(date -u '+%Y-%m-%dT%H:%M:%SZ') starting" python3 /Users/x/websafe/scripts/intel/main.py hotlane +python3 /Users/x/websafe/scripts/lab/main.py run-batch --only-hotlane --limit "${WEBSAFE_HOTLANE_LIMIT:-10}" +python3 /Users/x/websafe/scripts/intel/main.py render +python3 /Users/x/websafe/scripts/intel/main.py validate +run_pr diff --git a/scripts/intel/run-nightly.sh b/scripts/intel/run-nightly.sh index 8c931f08..ffd86971 100644 --- a/scripts/intel/run-nightly.sh +++ b/scripts/intel/run-nightly.sh @@ -7,5 +7,19 @@ mkdir -p "$LOG_DIR" STAMP="$(date '+%Y%m%d-%H%M%S')" exec >> "$LOG_DIR/nightly-$STAMP.log" 2>&1 +run_pr() { + if [[ "${WEBSAFE_PR_MODE:-auto}" == "skip" ]]; then + echo "[nightly] PR skipped by WEBSAFE_PR_MODE=skip" + elif [[ "${WEBSAFE_PR_MODE:-auto}" == "dry-run" || -z "${GITEA_TOKEN:-}" ]]; then + python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run + else + python3 /Users/x/websafe/scripts/intel/main.py open-pr + fi +} + echo "[nightly] $(date -u '+%Y-%m-%dT%H:%M:%SZ') starting" python3 /Users/x/websafe/scripts/intel/main.py ingest --since last-success +python3 /Users/x/websafe/scripts/lab/main.py run-batch --limit "${WEBSAFE_NIGHTLY_LIMIT:-25}" +python3 /Users/x/websafe/scripts/intel/main.py render +python3 /Users/x/websafe/scripts/intel/main.py validate +run_pr diff --git a/scripts/intel/run-weekly-reconcile.sh b/scripts/intel/run-weekly-reconcile.sh index 00478e70..24257f30 100644 --- a/scripts/intel/run-weekly-reconcile.sh +++ b/scripts/intel/run-weekly-reconcile.sh @@ -7,5 +7,20 @@ mkdir -p "$LOG_DIR" STAMP="$(date '+%Y%m%d-%H%M%S')" exec >> "$LOG_DIR/weekly-$STAMP.log" 2>&1 +run_pr() { + if [[ "${WEBSAFE_PR_MODE:-auto}" == "skip" ]]; then + echo "[weekly] PR skipped by WEBSAFE_PR_MODE=skip" + elif [[ "${WEBSAFE_PR_MODE:-auto}" == "dry-run" || -z "${GITEA_TOKEN:-}" ]]; then + python3 /Users/x/websafe/scripts/intel/main.py open-pr --dry-run + else + python3 /Users/x/websafe/scripts/intel/main.py open-pr + fi +} + echo "[weekly] $(date -u '+%Y-%m-%dT%H:%M:%SZ') starting" python3 /Users/x/websafe/scripts/intel/main.py reconcile +python3 /Users/x/websafe/scripts/lab/main.py retry-failures --limit "${WEBSAFE_RETRY_LIMIT:-100}" +python3 /Users/x/websafe/scripts/lab/main.py run-batch --from-queue --limit "${WEBSAFE_WEEKLY_LIMIT:-50}" +python3 /Users/x/websafe/scripts/intel/main.py render +python3 /Users/x/websafe/scripts/intel/main.py validate +run_pr diff --git a/scripts/intel/validators.py b/scripts/intel/validators.py index 3746472e..bc40292b 100644 --- a/scripts/intel/validators.py +++ b/scripts/intel/validators.py @@ -83,6 +83,7 @@ def validate(source_map: Dict[str, Any]) -> List[str]: GENERATED_DIR / "run-summary.json", GENERATED_DIR / "dashboard" / "index.html", GENERATED_DIR / "dashboard" / "summary.json", + GENERATED_DIR / "dashboard" / "systems.json", ROOT / "08-threat-intel" / "registry" / "source-confidence.md", ]: if not path.exists(): diff --git a/scripts/lab/browser.py b/scripts/lab/browser.py index d87d8422..e1eb704d 100644 --- a/scripts/lab/browser.py +++ b/scripts/lab/browser.py @@ -25,8 +25,12 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any] dom_path = assets_dir / f"{prefix}-dom.html" console_path = run_dir / "logs" / f"{prefix}-console.json" network_path = run_dir / "logs" / f"{prefix}-network.json" + page_path = run_dir / "logs" / f"{prefix}-page.json" console_messages: List[Dict[str, Any]] = [] requests_seen: List[Dict[str, Any]] = [] + page_title = "" + page_body_excerpt = "" + final_url = url try: with sync_playwright() as p: browser = p.chromium.launch(headless=True) @@ -36,6 +40,9 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any] page.goto(url, wait_until="networkidle", timeout=20000) page.screenshot(path=str(screenshot_path), full_page=True) dom_path.write_text(page.content(), encoding="utf-8") + final_url = page.url + page_title = page.title() + page_body_excerpt = (page.text_content("body") or "")[:600] browser.close() except Exception as exc: payload["reason"] = str(exc) @@ -43,10 +50,20 @@ def capture(url: str, run_dir: Path, prefix: str = "baseline") -> Dict[str, Any] return payload write_json(console_path, console_messages) write_json(network_path, requests_seen) + write_json( + page_path, + { + "url": final_url, + "title": page_title, + "body_excerpt": page_body_excerpt, + }, + ) payload = { "required": True, "present": True, - "refs": [str(screenshot_path), str(dom_path), str(console_path), str(network_path)], + "page_title": page_title, + "page_url": final_url, + "refs": [str(screenshot_path), str(dom_path), str(console_path), str(network_path), str(page_path)], } write_json(run_dir / "logs" / f"{prefix}-browser.json", payload) return payload diff --git a/scripts/lab/main.py b/scripts/lab/main.py index b5c94d56..2fd99c3d 100644 --- a/scripts/lab/main.py +++ b/scripts/lab/main.py @@ -35,6 +35,32 @@ def _compose_run_id(advisory: Dict[str, Any]) -> str: return f"{advisory['system_id']}-{advisory['canonical_id']}-{now_utc().strftime('%Y%m%d%H%M%S')}" +def _existing_refs(*paths: Path) -> List[str]: + return [str(path) for path in paths if path.exists()] + + +def _timeline_event(timeline: List[Dict[str, Any]], step: str, status: str, detail: str = "") -> None: + timeline.append( + { + "at": isoformat(now_utc()), + "step": step, + "status": status, + "detail": detail, + } + ) + + +def _sync_registry_outputs() -> None: + from intel.config import GENERATED_DIR, load_source_map # noqa: E402 + from intel.main import _load_existing_advisories, _load_existing_triage, _write_outputs # noqa: E402 + + source_map = load_source_map() + advisories = _load_existing_advisories() + triage = _load_existing_triage() + summary = read_json(GENERATED_DIR / "run-summary.json", default={}) or {} + _write_outputs(source_map, advisories, triage, summary.get("failures", []), summary) + + def _resolve_profile(advisory: Dict[str, Any]) -> Dict[str, Any]: profile = repro.resolve_profile(advisory["canonical_id"], advisory) current_profile = read_yaml(ENV_PROFILES_DIR / "core" / advisory["system_id"] / "current.yaml", default={}) or {} @@ -70,6 +96,11 @@ def _build_run_bundle( browser_refs: List[str], container_log_refs: List[str], request_log_refs: List[str], + compose_refs: List[str], + browser_evidence: Dict[str, Any], + timeline: List[Dict[str, Any]], + started_at: str, + finished_at: str, blocked_reason: str | None, ) -> Dict[str, Any]: return { @@ -85,15 +116,174 @@ def _build_run_bundle( "baseline_refs": baseline_refs, "attack_steps": attack_steps, "browser_refs": browser_refs, + "browser_evidence": browser_evidence, "container_log_refs": container_log_refs, "request_log_refs": request_log_refs, - "timeline": [], - "started_at": isoformat(now_utc()), - "finished_at": isoformat(now_utc()), + "compose_refs": compose_refs, + "timeline": timeline, + "started_at": started_at, + "finished_at": finished_at, "blocked_reason": blocked_reason, } +def _dry_run_case_plan(advisory: Dict[str, Any], profile: Dict[str, Any], run_id: str) -> Dict[str, Any]: + provision_result = provision.prepare(profile, CASE_RUNS_DIR / run_id, dry_run=True) + return { + "run_id": run_id, + "system_id": advisory["system_id"], + "advisory_id": advisory["canonical_id"], + "repro_profile_id": profile["profile_id"], + "verification_mode": profile.get("verification_mode", "synthetic"), + "artifact_mode": profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic")), + "browser_required": bool(profile.get("browser_assertions", {}).get("required")), + "baseline_urls": profile.get("baseline_urls", []), + "compose_services": sorted(profile.get("services", {}).keys()), + "seed_actions": profile.get("seed_actions", []), + "attack_actions": profile.get("attack_actions", []), + "compose_preview": provision_result.get("compose_preview", {}), + "note": "dry-run only; no bundle, report, compose file, or registry update was written", + } + + +def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool = False, sync_outputs: bool = True) -> Dict[str, Any]: + advisory = _load_advisory(canonical_id) + profile = _resolve_profile(advisory) + resolved_run_id = run_id or _compose_run_id(advisory) + if dry_run: + return _dry_run_case_plan(advisory, profile, resolved_run_id) + + started_at = isoformat(now_utc()) + timeline: List[Dict[str, Any]] = [] + _timeline_event(timeline, "select-advisory", "completed", advisory["canonical_id"]) + _timeline_event(timeline, "resolve-repro-profile", "completed", profile["profile_id"]) + + run_dir = _run_dir(resolved_run_id) + provision_result = provision.prepare(profile, run_dir, dry_run=False) + _timeline_event( + timeline, + "provision-compose-environment", + provision_result.get("status", "unknown"), + provision_result.get("blocked_reason", ""), + ) + allow_runtime_steps = provision_result.get("status") not in {"blocked-artifact"} + browser_required = bool(profile.get("browser_assertions", {}).get("required")) + + baseline_payload = {"observations": []} + if profile.get("baseline_urls") and allow_runtime_steps: + baseline_payload = baseline.collect(profile, run_dir) + _timeline_event(timeline, "baseline-snapshot", "completed", f"urls={len(profile.get('baseline_urls', []))}") + else: + _timeline_event(timeline, "baseline-snapshot", "skipped", "no baseline urls or provisioning blocked") + + baseline_browser = {"required": browser_required, "present": False, "refs": []} + if browser_required and allow_runtime_steps and profile.get("baseline_urls"): + baseline_browser = browser.capture(profile["baseline_urls"][0], run_dir, prefix="baseline") + _timeline_event( + timeline, + "browser-replay-before-attack", + "completed" if baseline_browser.get("present") else "failed", + baseline_browser.get("reason", ""), + ) + elif browser_required: + _timeline_event(timeline, "browser-replay-before-attack", "skipped", "baseline browser capture unavailable") + + attack_payload = {"steps": []} + if allow_runtime_steps: + attack_payload = attack.run_attack(profile, advisory, run_dir, dry_run=False) + attack_failed = any(step.get("status") == "failed" for step in attack_payload.get("steps", [])) + _timeline_event( + timeline, + "controlled-attack-chain", + "failed" if attack_failed else "completed", + f"steps={len(attack_payload.get('steps', []))}", + ) + else: + _timeline_event(timeline, "controlled-attack-chain", "skipped", "provisioning blocked") + + proof_browser = {"required": browser_required, "present": False, "refs": []} + if browser_required and allow_runtime_steps and profile.get("baseline_urls"): + proof_browser = browser.capture(profile["baseline_urls"][0], run_dir, prefix="proof") + _timeline_event( + timeline, + "browser-replay-after-attack", + "completed" if proof_browser.get("present") else "failed", + proof_browser.get("reason", ""), + ) + elif browser_required: + _timeline_event(timeline, "browser-replay-after-attack", "skipped", "proof browser capture unavailable") + + compose_path = Path(provision_result["compose_path"]) + container_logs = evidence.collect_container_logs(run_dir, compose_path) if compose_path.exists() and allow_runtime_steps else [] + _timeline_event( + timeline, + "collect-logs-and-evidence", + "completed" if allow_runtime_steps else "skipped", + f"container_logs={len(container_logs)}", + ) + + browser_present = bool(baseline_browser.get("present")) and bool(proof_browser.get("present")) + browser_payload = { + "required": browser_required, + "present": browser_present, + "refs": baseline_browser.get("refs", []) + proof_browser.get("refs", []), + "baseline_refs": baseline_browser.get("refs", []), + "proof_refs": proof_browser.get("refs", []), + "baseline_title": baseline_browser.get("page_title"), + "proof_title": proof_browser.get("page_title"), + } + + blocked_reason = provision_result.get("blocked_reason") + if browser_required and not browser_present: + blocked_reason = blocked_reason or baseline_browser.get("reason") or proof_browser.get("reason") or "browser evidence incomplete" + + verification_mode = profile.get("verification_mode", "synthetic") + artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic")) + verification_status = "triage-manual" + if provision_result.get("status") == "blocked-artifact": + verification_status = "blocked-artifact" + elif browser_required and not browser_present: + verification_status = "triage-manual" + elif any(step.get("status") == "failed" for step in attack_payload.get("steps", [])): + verification_status = "triage-manual" + elif artifact_mode == "synthetic": + verification_status = "verified-synthetic" + else: + verification_status = "verified-real" + + finished_at = isoformat(now_utc()) + bundle = _build_run_bundle( + advisory=advisory, + profile=profile, + run_id=resolved_run_id, + verification_status=verification_status, + verification_mode=verification_mode, + artifact_mode=artifact_mode, + baseline_refs=_existing_refs(run_dir / "logs" / "baseline.json"), + attack_steps=attack_payload.get("steps", []), + browser_refs=browser_payload["refs"], + container_log_refs=container_logs, + request_log_refs=_existing_refs(run_dir / "logs" / "attack.json", run_dir / "logs" / "baseline.json"), + compose_refs=[str(compose_path)] if compose_path.exists() else [], + browser_evidence=browser_payload, + timeline=timeline, + started_at=started_at, + finished_at=finished_at, + blocked_reason=blocked_reason, + ) + _timeline_event(bundle["timeline"], "update-registry-and-reports", "completed", resolved_run_id) + report_refs = render.render_run(bundle) + bundle["report_refs"] = report_refs + evidence.write_run_bundle(run_dir, bundle) + ensure_dir(RUNS_DIR) + write_json(RUNS_DIR / f"{resolved_run_id}.json", bundle) + if sync_outputs: + _sync_registry_outputs() + else: + render.render_dashboard() + return bundle + + def cmd_catalog_sync(args) -> int: summary = catalog.sync_catalog(write_profiles=True, write_repro_map=True) print(summary) @@ -158,66 +348,8 @@ def cmd_verify(args) -> int: def cmd_run_case(args) -> int: - advisory = _load_advisory(args.case) - profile = _resolve_profile(advisory) - run_id = args.run_id or _compose_run_id(advisory) - run_dir = _run_dir(run_id) - - provision_result = provision.prepare(profile, run_dir, dry_run=args.dry_run) - allow_runtime_steps = provision_result.get("status") not in {"blocked-artifact"} - baseline_payload = ( - baseline.collect(profile, run_dir) if profile.get("baseline_urls") and allow_runtime_steps else {"observations": []} - ) - attack_payload = ( - attack.run_attack(profile, advisory, run_dir, dry_run=args.dry_run) if allow_runtime_steps else {"steps": []} - ) - - browser_payload = {"required": bool(profile.get("browser_assertions", {}).get("required")), "present": False, "refs": []} - blocked_reason = provision_result.get("blocked_reason") - if browser_payload["required"] and not args.dry_run and profile.get("baseline_urls") and allow_runtime_steps: - browser_payload = browser.capture(profile["baseline_urls"][0], run_dir, prefix="proof") - if not browser_payload.get("present"): - blocked_reason = blocked_reason or browser_payload.get("reason") - - compose_path = Path(provision_result["compose_path"]) - container_logs = evidence.collect_container_logs(run_dir, compose_path) if compose_path.exists() and allow_runtime_steps else [] - - verification_status = "triage-manual" - verification_mode = profile.get("verification_mode", "synthetic") - artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic")) - if args.dry_run: - verification_status = "triage-manual" - blocked_reason = blocked_reason or "dry-run only" - elif provision_result.get("status") == "blocked-artifact": - verification_status = "blocked-artifact" - elif browser_payload.get("required") and not browser_payload.get("present"): - verification_status = "triage-manual" - elif artifact_mode == "synthetic": - verification_status = "verified-synthetic" - else: - verification_status = "verified-real" - - bundle = _build_run_bundle( - advisory=advisory, - profile=profile, - run_id=run_id, - verification_status=verification_status, - verification_mode=verification_mode, - artifact_mode=artifact_mode, - baseline_refs=[str(run_dir / "logs" / "baseline.json")] if baseline_payload.get("observations") else [], - attack_steps=attack_payload.get("steps", []), - browser_refs=browser_payload.get("refs", []), - container_log_refs=container_logs, - request_log_refs=[str(run_dir / "logs" / "attack.json"), str(run_dir / "logs" / "baseline.json")], - blocked_reason=blocked_reason, - ) - report_refs = render.render_run(bundle) - bundle["report_refs"] = report_refs - evidence.write_run_bundle(run_dir, bundle) - ensure_dir(RUNS_DIR) - write_json(RUNS_DIR / f"{run_id}.json", bundle) - render.render_dashboard() - print(bundle) + result = _execute_case(args.case, run_id=args.run_id, dry_run=args.dry_run, sync_outputs=not args.dry_run) + print(result) return 0 @@ -225,7 +357,9 @@ def cmd_run_system(args) -> int: advisories = [item for item in load_json_dir(ADVISORIES_DIR) if item.get("system_id") == args.system] selected = advisories[: args.limit] for advisory in selected: - cmd_run_case(argparse.Namespace(case=advisory["canonical_id"], run_id=None, dry_run=args.dry_run)) + _execute_case(advisory["canonical_id"], run_id=None, dry_run=args.dry_run, sync_outputs=False) + if selected and not args.dry_run: + _sync_registry_outputs() print({"system": args.system, "count": len(selected)}) return 0 @@ -237,7 +371,9 @@ def cmd_run_batch(args) -> int: task_queue.enqueue_from_registry(only_hotlane=args.only_hotlane, limit=args.limit) items = task_queue.dequeue(limit=args.limit) for item in items: - cmd_run_case(argparse.Namespace(case=item["advisory_id"], run_id=None, dry_run=args.dry_run)) + _execute_case(item["advisory_id"], run_id=None, dry_run=args.dry_run, sync_outputs=False) + if items and not args.dry_run: + _sync_registry_outputs() print({"processed": len(items)}) return 0 diff --git a/scripts/lab/provision.py b/scripts/lab/provision.py index c4a27c1e..81884c25 100644 --- a/scripts/lab/provision.py +++ b/scripts/lab/provision.py @@ -3,15 +3,17 @@ from __future__ import annotations from pathlib import Path from typing import Any, Dict -from lab.compose import generate_compose +from lab.compose import compose_payload, generate_compose from lab.utils import command_available, run def prepare(profile: Dict[str, Any], run_dir: Path, dry_run: bool = False) -> Dict[str, Any]: - compose_path, payload = generate_compose(profile, run_dir) + payload = compose_payload(profile) + compose_path = run_dir / "compose" / "compose.yaml" result = { "compose_path": str(compose_path), "service_count": len(payload.get("services", {})), + "compose_preview": payload, "docker_available": command_available("docker"), "status": "ready", } @@ -19,6 +21,7 @@ def prepare(profile: Dict[str, Any], run_dir: Path, dry_run: bool = False) -> Di result["status"] = "planned" return result + compose_path, payload = generate_compose(profile, run_dir) if not result["docker_available"]: result["status"] = "blocked-artifact" result["blocked_reason"] = "docker unavailable on this machine" diff --git a/scripts/lab/render.py b/scripts/lab/render.py index d0549842..d388e56b 100644 --- a/scripts/lab/render.py +++ b/scripts/lab/render.py @@ -1,11 +1,12 @@ from __future__ import annotations import html +import os from pathlib import Path from typing import Any, Dict, List -from lab.config import CASE_RUNS_DIR, DASHBOARD_DIR, RUNS_DIR -from lab.utils import ensure_dir, load_json_dir, read_json, write_json, write_text +from lab.config import ADVISORIES_DIR, CASE_RUNS_DIR, DASHBOARD_DIR, RUNS_DIR +from lab.utils import ensure_dir, isoformat, load_json_dir, now_utc, write_json, write_text def mermaid_from_steps(run: Dict[str, Any]) -> str: @@ -24,11 +25,29 @@ def mermaid_from_steps(run: Dict[str, Any]) -> str: return "\n".join(lines) +def _relative_ref(run_dir: Path, ref: str) -> str: + try: + return str(Path(ref).resolve().relative_to(run_dir.resolve())) + except ValueError: + return ref + + +def _dashboard_ref(run: Dict[str, Any], ref: str) -> str: + try: + bundle_dir = Path(run["report_refs"]["bundle_dir"]).resolve() + relative = Path(ref).resolve().relative_to(bundle_dir) + return f"./runs/{run['run_id']}/{relative.as_posix()}" + except Exception: + return ref + + def render_run(run: Dict[str, Any]) -> Dict[str, str]: run_dir = CASE_RUNS_DIR / run["run_id"] ensure_dir(run_dir / "assets") timeline_path = run_dir / "timeline.mmd" write_text(timeline_path, mermaid_from_steps(run)) + screenshot_refs = [ref for ref in run.get("browser_refs", []) if ref.endswith((".png", ".jpg", ".jpeg"))] + relative_screenshots = [_relative_ref(run_dir, ref) for ref in screenshot_refs] md_lines = [ f"# Run {run['run_id']}", @@ -44,42 +63,92 @@ def render_run(run: Dict[str, Any]) -> Dict[str, str]: f"- 启动时间: `{run['started_at']}`", f"- 完成时间: `{run['finished_at']}`", f"- 阻塞原因: `{run.get('blocked_reason') or '-'}`", + f"- Compose 服务: `{', '.join(run.get('compose_services', [])) or '-'}`", "", "## 运行时间线", "", f"- Mermaid: [{timeline_path.name}]({timeline_path})", "", - "## 证据摘要", - "", - f"- Baseline: `{len(run.get('baseline_refs', []))}`", - f"- 攻击步骤: `{len(run.get('attack_steps', []))}`", - f"- 浏览器证据: `{len(run.get('browser_refs', []))}`", - f"- 容器日志: `{len(run.get('container_log_refs', []))}`", - f"- 请求日志: `{len(run.get('request_log_refs', []))}`", - "", - "## 最小化验证说明", - "", - "- 仅限自有资产、本地靶场或已授权实验目标。", - "- 默认执行 minimal-proof；不会把破坏性或不可回滚动作作为默认路径。", - "", + "| 时间 | 步骤 | 状态 | 说明 |", + "|------|------|------|------|", ] + if run.get("timeline"): + for item in run["timeline"]: + md_lines.append( + f"| `{item.get('at', '')}` | `{item.get('step', '')}` | `{item.get('status', '')}` | {item.get('detail', '') or '-'} |" + ) + else: + md_lines.append("| `-` | `-` | `-` | 无时间线 |") + md_lines.extend( + [ + "", + "## Compose 拓扑", + "", + f"- Compose 文件: `{', '.join(run.get('compose_refs', [])) or '-'}`", + f"- 服务列表: `{', '.join(run.get('compose_services', [])) or '-'}`", + "", + "## 攻击步骤", + "", + "| 工具/步骤 | 状态 | 结果 |", + "|-----------|------|------|", + ] + ) + if run.get("attack_steps"): + for step in run["attack_steps"]: + outcome = step.get("result_path") or step.get("detail") or "-" + md_lines.append(f"| `{step.get('tool') or step.get('kind')}` | `{step.get('status', '-')}` | `{outcome}` |") + else: + md_lines.append("| `-` | `skipped` | `no attack steps` |") + md_lines.extend( + [ + "", + "## 证据摘要", + "", + f"- Baseline: `{len(run.get('baseline_refs', []))}`", + f"- 攻击步骤: `{len(run.get('attack_steps', []))}`", + f"- 浏览器证据: `{len(run.get('browser_refs', []))}`", + f"- 容器日志: `{len(run.get('container_log_refs', []))}`", + f"- 请求日志: `{len(run.get('request_log_refs', []))}`", + "", + ] + ) + if relative_screenshots: + md_lines.extend(["## 浏览器截图", ""]) + for ref in relative_screenshots: + md_lines.append(f"![{Path(ref).stem}]({ref})") + md_lines.append("") if run.get("browser_refs"): md_lines.extend(["## 浏览器证据", ""]) for ref in run["browser_refs"]: - md_lines.append(f"- {ref}") + md_lines.append(f"- `{_relative_ref(run_dir, ref)}`") md_lines.append("") if run.get("container_log_refs"): md_lines.extend(["## 容器日志", ""]) for ref in run["container_log_refs"]: - md_lines.append(f"- {ref}") + md_lines.append(f"- `{_relative_ref(run_dir, ref)}`") md_lines.append("") + if run.get("request_log_refs"): + md_lines.extend(["## 请求与基线日志", ""]) + for ref in run["request_log_refs"]: + md_lines.append(f"- `{_relative_ref(run_dir, ref)}`") + md_lines.append("") + md_lines.extend( + [ + "## 最小化验证说明", + "", + "- 仅限自有资产、本地靶场或已授权实验目标。", + "- 默认执行 minimal-proof；不会把破坏性或不可回滚动作作为默认路径。", + "- 若浏览器证据缺失，前端类案例不会被标为 `verified-*`。", + "", + ] + ) report_md = run_dir / "report.md" write_text(report_md, "\n".join(md_lines)) html_body = [ "", "websafe run report", - "", + "", "", f"

Run {html.escape(run['run_id'])}

", "
", @@ -90,10 +159,42 @@ def render_run(run: Dict[str, Any]) -> Dict[str, str]: "
", "

Mermaid Timeline

", f"
{html.escape(mermaid_from_steps(run))}
", - "

Evidence