101 行
3.9 KiB
Python
101 行
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any, Dict, List
|
|
|
|
|
|
def _assertion(name: str, kind: str, passed: bool, detail: str) -> Dict[str, Any]:
|
|
return {
|
|
"name": name,
|
|
"kind": kind,
|
|
"passed": passed,
|
|
"detail": detail,
|
|
}
|
|
|
|
|
|
def _baseline_ok(payload: Dict[str, Any]) -> bool:
|
|
observations = payload.get("observations", []) or []
|
|
if not observations:
|
|
return False
|
|
for item in observations:
|
|
if item.get("error"):
|
|
return False
|
|
status_code = item.get("status_code")
|
|
if status_code is None or int(status_code) >= 500:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _attack_steps_ok(payload: Dict[str, Any]) -> bool:
|
|
steps = payload.get("steps", []) or []
|
|
if payload.get("success") is True:
|
|
return True
|
|
if not steps:
|
|
return False
|
|
return not any(step.get("status") == "failed" for step in steps)
|
|
|
|
|
|
def evaluate_run(
|
|
profile: Dict[str, Any],
|
|
provision_result: Dict[str, Any],
|
|
baseline_payload: Dict[str, Any],
|
|
attack_payload: Dict[str, Any],
|
|
browser_payload: Dict[str, Any],
|
|
) -> Dict[str, Any]:
|
|
assertions: List[Dict[str, Any]] = []
|
|
configured = profile.get("success_assertions", []) or []
|
|
browser_required = bool(profile.get("browser_assertions", {}).get("required"))
|
|
if not configured:
|
|
configured = [
|
|
{"name": "baseline-ok", "type": "baseline-ok"},
|
|
{"name": "attack-steps", "type": "attack-steps-ok"},
|
|
]
|
|
if browser_required:
|
|
configured.append({"name": "browser-present", "type": "browser-present"})
|
|
|
|
for item in configured:
|
|
assertion_type = item.get("type", "")
|
|
name = item.get("name") or assertion_type or "assertion"
|
|
if assertion_type == "runner-success":
|
|
passed = bool(attack_payload.get("success"))
|
|
detail = attack_payload.get("detail") or ("runner reported success" if passed else "runner did not confirm success")
|
|
elif assertion_type == "baseline-ok":
|
|
passed = _baseline_ok(baseline_payload)
|
|
detail = "baseline URLs responded without 5xx or transport errors" if passed else "baseline checks were incomplete"
|
|
elif assertion_type == "attack-steps-ok":
|
|
passed = _attack_steps_ok(attack_payload)
|
|
detail = "attack steps completed without failures" if passed else "attack steps failed or produced no usable result"
|
|
elif assertion_type == "browser-present":
|
|
passed = bool(browser_payload.get("present"))
|
|
detail = "browser evidence captured" if passed else (browser_payload.get("reason") or "browser evidence missing")
|
|
else:
|
|
passed = False
|
|
detail = f"unsupported assertion type: {assertion_type}"
|
|
assertions.append(_assertion(name, assertion_type, passed, detail))
|
|
|
|
blocked_reason = provision_result.get("blocked_reason")
|
|
if browser_required and not browser_payload.get("present"):
|
|
blocked_reason = blocked_reason or browser_payload.get("reason") or "browser evidence incomplete"
|
|
|
|
passed = all(item["passed"] for item in assertions)
|
|
artifact_mode = profile.get("artifact_mode", profile.get("provisioning_mode", "synthetic"))
|
|
verification_status = "triage-manual"
|
|
if provision_result.get("status") == "blocked-artifact":
|
|
verification_status = "blocked-artifact"
|
|
elif not passed:
|
|
verification_status = "triage-manual"
|
|
failed = next((item for item in assertions if not item["passed"]), None)
|
|
if failed and not blocked_reason:
|
|
blocked_reason = failed["detail"]
|
|
elif artifact_mode == "synthetic":
|
|
verification_status = "verified-synthetic"
|
|
else:
|
|
verification_status = "verified-real"
|
|
|
|
return {
|
|
"passed": passed and verification_status.startswith("verified-"),
|
|
"verification_status": verification_status,
|
|
"blocked_reason": blocked_reason,
|
|
"assertions": assertions,
|
|
}
|
|
|