更新: 421 个文件 - 2026-03-17 18:30:02

scripts/lab/main.py

@@ -11,7 +11,7 @@ SCRIPTS_DIR = CURRENT_DIR.parent
 if str(SCRIPTS_DIR) not in sys.path:
     sys.path.insert(0, str(SCRIPTS_DIR))
 
-from lab import attack, baseline, browser, catalog, evidence, provision, render, repro, seed, task_queue, validators  # noqa: E402
+from lab import attack, baseline, browser, catalog, doctor, evaluate, evidence, provision, render, repro, seed, task_queue, validators  # noqa: E402
 from lab.config import ADVISORIES_DIR, CASE_RUNS_DIR, ENV_PROFILES_DIR, RUNS_DIR  # noqa: E402
 from lab.utils import command_available, ensure_dir, isoformat, load_json_dir, now_utc, read_json, read_yaml, write_json  # noqa: E402
 
@@ -99,6 +99,7 @@ def _build_run_bundle(
     compose_refs: List[str],
     browser_evidence: Dict[str, Any],
     timeline: List[Dict[str, Any]],
+    success_evaluation: Dict[str, Any],
     started_at: str,
     finished_at: str,
     blocked_reason: str | None,
@@ -121,6 +122,9 @@ def _build_run_bundle(
         "request_log_refs": request_log_refs,
         "compose_refs": compose_refs,
         "timeline": timeline,
+        "success_evaluation": success_evaluation,
+        "historical_status": verification_status,
+        "latest_status": verification_status,
         "started_at": started_at,
         "finished_at": finished_at,
         "blocked_reason": blocked_reason,
@@ -141,6 +145,9 @@ def _dry_run_case_plan(advisory: Dict[str, Any], profile: Dict[str, Any], run_id
         "compose_services": sorted(profile.get("services", {}).keys()),
         "seed_actions": profile.get("seed_actions", []),
         "attack_actions": profile.get("attack_actions", []),
+        "runner_id": profile.get("runner_id"),
+        "fixture_path": profile.get("fixture_path"),
+        "success_assertions": profile.get("success_assertions", []),
         "compose_preview": provision_result.get("compose_preview", {}),
         "note": "dry-run only; no bundle, report, compose file, or registry update was written",
     }
@@ -159,7 +166,23 @@ def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool =
     _timeline_event(timeline, "resolve-repro-profile", "completed", profile["profile_id"])
 
     run_dir = _run_dir(resolved_run_id)
-    provision_result = provision.prepare(profile, run_dir, dry_run=False)
+    doctor_result = doctor.run_checks([profile])
+    write_json(run_dir / "logs" / "doctor.json", doctor_result)
+    _timeline_event(
+        timeline,
+        "doctor",
+        "completed" if doctor_result.get("ok") else "failed",
+        doctor_result.get("summary", ""),
+    )
+
+    if doctor_result.get("ok"):
+        provision_result = provision.prepare(profile, run_dir, dry_run=False)
+    else:
+        provision_result = {
+            "compose_path": str(run_dir / "compose" / "compose.yaml"),
+            "status": "blocked-artifact",
+            "blocked_reason": doctor_result.get("summary", "doctor failed"),
+        }
     _timeline_event(
         timeline,
         "provision-compose-environment",
@@ -168,11 +191,39 @@ def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool =
     )
     allow_runtime_steps = provision_result.get("status") not in {"blocked-artifact"}
     browser_required = bool(profile.get("browser_assertions", {}).get("required"))
+    compose_path = Path(provision_result.get("compose_path", run_dir / "compose" / "compose.yaml"))
+
+    ready_payload = {"status": "skipped", "detail": "provisioning blocked", "observations": []}
+    if allow_runtime_steps:
+        ready_payload = provision.wait_ready(profile, run_dir, compose_path)
+        allow_runtime_steps = ready_payload.get("status") == "completed"
+        _timeline_event(timeline, "wait-ready", ready_payload.get("status", "unknown"), ready_payload.get("detail", ""))
+    else:
+        _timeline_event(timeline, "wait-ready", "skipped", "provisioning blocked")
+
+    seed_payload = {"steps": [], "seeded": False}
+    if allow_runtime_steps:
+        seed_payload = seed.run_seed(profile, advisory, run_dir, dry_run=False)
+        seed_failed = any(step.get("status") == "failed" for step in seed_payload.get("steps", []))
+        _timeline_event(
+            timeline,
+            "seed-environment",
+            "failed" if seed_failed else "completed",
+            f"steps={len(seed_payload.get('steps', []))}",
+        )
+    else:
+        _timeline_event(timeline, "seed-environment", "skipped", "runtime steps unavailable")
 
     baseline_payload = {"observations": []}
     if profile.get("baseline_urls") and allow_runtime_steps:
         baseline_payload = baseline.collect(profile, run_dir)
-        _timeline_event(timeline, "baseline-snapshot", "completed", f"urls={len(profile.get('baseline_urls', []))}")
+        baseline_failed = any(item.get("error") for item in baseline_payload.get("observations", []))
+        _timeline_event(
+            timeline,
+            "baseline-snapshot",
+            "failed" if baseline_failed else "completed",
+            f"urls={len(profile.get('baseline_urls', []))}",
+        )
     else:
         _timeline_event(timeline, "baseline-snapshot", "skipped", "no baseline urls or provisioning blocked")
 
@@ -213,7 +264,6 @@ def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool =
     elif browser_required:
         _timeline_event(timeline, "browser-replay-after-attack", "skipped", "proof browser capture unavailable")
 
-    compose_path = Path(provision_result["compose_path"])
     container_logs = evidence.collect_container_logs(run_dir, compose_path) if compose_path.exists() and allow_runtime_steps else []
     _timeline_event(
         timeline,
@@ -231,25 +281,31 @@ def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool =
         "proof_refs": proof_browser.get("refs", []),
         "baseline_title": baseline_browser.get("page_title"),
         "proof_title": proof_browser.get("page_title"),
+        "error_kind": proof_browser.get("error_kind") or baseline_browser.get("error_kind"),
+        "reason": proof_browser.get("reason") or baseline_browser.get("reason"),
     }
 
-    blocked_reason = provision_result.get("blocked_reason")
-    if browser_required and not browser_present:
-        blocked_reason = blocked_reason or baseline_browser.get("reason") or proof_browser.get("reason") or "browser evidence incomplete"
-
     verification_mode = profile.get("verification_mode", "synthetic")
     artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic"))
-    verification_status = "triage-manual"
-    if provision_result.get("status") == "blocked-artifact":
-        verification_status = "blocked-artifact"
-    elif browser_required and not browser_present:
-        verification_status = "triage-manual"
-    elif any(step.get("status") == "failed" for step in attack_payload.get("steps", [])):
-        verification_status = "triage-manual"
-    elif artifact_mode == "synthetic":
-        verification_status = "verified-synthetic"
-    else:
-        verification_status = "verified-real"
+    success_evaluation = evaluate.evaluate_run(
+        profile=profile,
+        provision_result=provision_result,
+        baseline_payload=baseline_payload,
+        attack_payload=attack_payload,
+        browser_payload=browser_payload,
+    )
+    verification_status = success_evaluation["verification_status"]
+    blocked_reason = success_evaluation.get("blocked_reason")
+
+    cleanup_payload = {"status": "skipped", "detail": "cleanup_policy not destroy"}
+    if compose_path.exists() and profile.get("cleanup_policy") == "destroy":
+        cleanup_payload = provision.teardown(run_dir, compose_path)
+    _timeline_event(
+        timeline,
+        "cleanup-compose-environment",
+        cleanup_payload.get("status", "unknown"),
+        cleanup_payload.get("detail", ""),
+    )
 
     finished_at = isoformat(now_utc())
     bundle = _build_run_bundle(
@@ -267,6 +323,7 @@ def _execute_case(canonical_id: str, run_id: str | None = None, dry_run: bool =
         compose_refs=[str(compose_path)] if compose_path.exists() else [],
         browser_evidence=browser_payload,
         timeline=timeline,
+        success_evaluation=success_evaluation,
         started_at=started_at,
         finished_at=finished_at,
         blocked_reason=blocked_reason,
@@ -311,7 +368,8 @@ def cmd_provision(args) -> int:
 def cmd_seed(args) -> int:
     advisory = _load_advisory(args.case)
     profile = _resolve_profile(advisory)
-    print({"steps": seed.run_seed(profile)})
+    run_dir = _run_dir(args.run_id or _compose_run_id(advisory))
+    print(seed.run_seed(profile, advisory, run_dir, dry_run=args.dry_run))
     return 0
 
 
@@ -355,7 +413,8 @@ def cmd_run_case(args) -> int:
 
 def cmd_run_system(args) -> int:
     advisories = [item for item in load_json_dir(ADVISORIES_DIR) if item.get("system_id") == args.system]
-    selected = advisories[: args.limit]
+    advisories = sorted(advisories, key=lambda item: item.get("canonical_id", ""))
+    selected = advisories if not args.limit or args.limit <= 0 else advisories[: args.limit]
     for advisory in selected:
         _execute_case(advisory["canonical_id"], run_id=None, dry_run=args.dry_run, sync_outputs=False)
     if selected and not args.dry_run:
@@ -442,6 +501,20 @@ def cmd_validate(args) -> int:
     return 0
 
 
+def cmd_doctor(args) -> int:
+    profiles: List[Dict[str, Any]] = []
+    if getattr(args, "case", None):
+        advisory = _load_advisory(args.case)
+        profiles.append(_resolve_profile(advisory))
+    elif getattr(args, "system", None):
+        advisories = [item for item in load_json_dir(ADVISORIES_DIR) if item.get("system_id") == args.system]
+        if advisories:
+            profiles.append(_resolve_profile(advisories[0]))
+    result = doctor.run_checks(profiles)
+    print(result)
+    return 0 if result.get("ok") else 1
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="Websafe local lab orchestrator")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -479,7 +552,7 @@ def build_parser() -> argparse.ArgumentParser:
 
     run_system = subparsers.add_parser("run-system", help="run the first N advisories for a system")
     run_system.add_argument("--system", required=True)
-    run_system.add_argument("--limit", type=int, default=5)
+    run_system.add_argument("--limit", type=int, default=0)
     run_system.add_argument("--dry-run", action="store_true")
     run_system.set_defaults(func=cmd_run_system)
 
@@ -508,6 +581,11 @@ def build_parser() -> argparse.ArgumentParser:
 
     validate = subparsers.add_parser("validate", help="validate lab assets")
     validate.set_defaults(func=cmd_validate)
+
+    doctor_cmd = subparsers.add_parser("doctor", help="run environment preflight checks")
+    doctor_cmd.add_argument("--case")
+    doctor_cmd.add_argument("--system")
+    doctor_cmd.set_defaults(func=cmd_doctor)
     return parser