kb: expand authorized lab coverage and intel automation

scripts/intel/validators.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List
+
+from intel.config import FRAMEWORK_ROOT, GENERATED_DIR, REGISTRY_ROOT, ROOT, SECURE_CODE_ROOT, SOURCE_MAP_PATH, SYSTEMS_DIR
+from intel.render import LANGUAGES, TOPIC_DESCRIPTIONS
+from intel.utils import load_all_json
+
+
+REQUIRED_REGISTRY_FIELDS = {
+    "canonical_id",
+    "system_id",
+    "title",
+    "severity",
+    "source_confidence",
+    "status",
+}
+
+REQUIRED_SYSTEM_FIELDS = {
+    "system_id",
+    "display_name",
+    "category",
+    "tier",
+    "advisory_modes",
+    "official_sources",
+    "ecosystem_sources",
+    "research_sources",
+    "output_dir",
+    "secure_code_topics",
+    "render_policy",
+}
+
+
+def validate(source_map: Dict[str, Any]) -> List[str]:
+    errors: List[str] = []
+    if not SOURCE_MAP_PATH.exists():
+        errors.append("source-map.yaml is missing")
+
+    systems = source_map.get("systems", [])
+    ids = set()
+    groups = set()
+    for system in systems:
+        missing = REQUIRED_SYSTEM_FIELDS - set(system.keys())
+        if missing:
+            errors.append(f"system missing required fields: {system.get('system_id', 'unknown')} -> {sorted(missing)}")
+        system_id = system["system_id"]
+        if system_id in ids:
+            errors.append(f"duplicate system_id: {system_id}")
+        ids.add(system_id)
+        output_dir = Path(system["output_dir"])
+        if len(output_dir.parts) < 3:
+            errors.append(f"invalid output_dir for {system_id}: {system['output_dir']}")
+            continue
+        groups.add(output_dir.parts[1])
+        system_root = ROOT / output_dir
+        if not (system_root / "README.md").exists():
+            errors.append(f"system README missing: {system_root / 'README.md'}")
+        if not (system_root / "INDEX.md").exists():
+            errors.append(f"system INDEX missing: {system_root / 'INDEX.md'}")
+        if not (SYSTEMS_DIR / f"{system_id}.json").exists():
+            errors.append(f"system registry summary missing: {SYSTEMS_DIR / f'{system_id}.json'}")
+
+    if not (FRAMEWORK_ROOT / "README.md").exists():
+        errors.append(f"framework root README missing: {FRAMEWORK_ROOT / 'README.md'}")
+    for group in groups:
+        if not (FRAMEWORK_ROOT / group / "README.md").exists():
+            errors.append(f"group README missing: {FRAMEWORK_ROOT / group / 'README.md'}")
+
+    for item in load_all_json(REGISTRY_ROOT / "advisories"):
+        missing = REQUIRED_REGISTRY_FIELDS - set(item.keys())
+        if missing:
+            errors.append(f"registry advisory missing fields: {item.get('canonical_id', 'unknown')} -> {sorted(missing)}")
+
+    for path in [
+        GENERATED_DIR / "coverage-matrix.md",
+        GENERATED_DIR / "latest-ingest.md",
+        GENERATED_DIR / "run-summary.json",
+        ROOT / "08-threat-intel" / "registry" / "source-confidence.md",
+    ]:
+        if not path.exists():
+            errors.append(f"generated artifact missing: {path}")
+
+    if not (SECURE_CODE_ROOT / "README.md").exists():
+        errors.append(f"secure-code README missing: {SECURE_CODE_ROOT / 'README.md'}")
+
+    for language in LANGUAGES:
+        language_dir = SECURE_CODE_ROOT / language
+        if not (language_dir / "README.md").exists():
+            errors.append(f"language README missing: {language_dir / 'README.md'}")
+        if not (language_dir / "INDEX.md").exists():
+            errors.append(f"language INDEX missing: {language_dir / 'INDEX.md'}")
+        for topic in TOPIC_DESCRIPTIONS:
+            if not (language_dir / f"{topic}.md").exists():
+                errors.append(f"secure-code topic missing: {language_dir / f'{topic}.md'}")
+
+    return errors