#!/usr/bin/env python3
"""Classify generated files under out/ by current usage surface.
The project has accumulated runtime data, review evidence, smoke/probe reports,
and obsolete analysis artifacts in a single out/ directory. This audit is
intentionally conservative: it does not delete files and it treats known dynamic
loader outputs, such as map chunks, as active even when no literal filename
reference exists in web/docs/tools.
"""
from __future__ import annotations
import argparse
import ast
import html
import json
import os
import re
from collections import Counter, defaultdict
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT_DIR = ROOT / "out"
SCAN_DIRS = ("web", "docs", "tools", "out")
TEXT_SUFFIXES = {
".css",
".html",
".js",
".json",
".md",
".mjs",
".py",
".txt",
".ts",
}
SKIP_SCAN_DIRS = {
".git",
"__pycache__",
"node_modules",
"web/soundfonts",
}
DIAGNOSTIC_TOKENS = (
"smoke",
"probe",
"poll",
"gap",
"audit",
"trace",
"context",
"handoff",
"checklist",
"oracle",
"capture",
)
RUNTIME_CORE_NAMES = {
"audio_archive_manifest.json",
"battle_action_mapping.json",
"battle_backgrounds.js",
"battle_backgrounds.json",
"battle_enemy_candidates.js",
"battle_enemy_candidates.json",
"battle_monster_action_catalog.json",
"battle_skill_records.js",
"battle_skill_records.json",
"battle_skill_timeline_canonical.js",
"battle_skill_timeline_canonical.json",
"cns_frame_assets.js",
"cns_frame_assets.json",
"cns_rect_review_data.js",
"cns_rect_review_data.json",
"enemy_stat_table.js",
"enemy_stat_table.json",
"event_dialogue_blocks_runtime.js",
"input_keymap.js",
"input_keymap.json",
"maps.js",
"maps_runtime.js",
"map_exit_candidates_runtime.js",
"object_assets.js",
"scene_manifest.json",
"scene_script_player_data.json",
"text_tables.js",
"text_tables.json",
}
@dataclass(frozen=True)
class SourceFile:
surface: str
path: Path
text: str
def rel_posix(path: Path) -> str:
return path.relative_to(ROOT).as_posix()
def should_skip(path: Path) -> bool:
rel = path.relative_to(ROOT).as_posix()
parts = set(path.relative_to(ROOT).parts)
if parts & {".git", "__pycache__", "node_modules"}:
return True
if rel in {"out/out_usage_audit.json", "out/out_usage_audit.html"}:
return True
return any(rel == skipped or rel.startswith(skipped + "/") for skipped in SKIP_SCAN_DIRS)
def iter_text_sources() -> list[SourceFile]:
sources: list[SourceFile] = []
for surface in SCAN_DIRS:
base = ROOT / surface
if not base.exists():
continue
for path in base.rglob("*"):
if not path.is_file() or should_skip(path):
continue
if path.suffix.lower() not in TEXT_SUFFIXES:
continue
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
try:
text = path.read_text(encoding="cp949")
except UnicodeDecodeError:
continue
sources.append(SourceFile(surface=surface, path=path, text=text))
return sources
def out_files(out_dir: Path) -> list[Path]:
return sorted(path for path in out_dir.rglob("*") if path.is_file())
def reference_needles(rel_out: str) -> tuple[str, ...]:
path = Path(rel_out)
needles = [
rel_out,
"../" + rel_out,
"../../" + rel_out,
"/" + rel_out,
]
if path.parent.as_posix() == "out":
needles.append(path.name)
return tuple(needles)
def verifier_active_out_files() -> set[str]:
verifier = ROOT / "tools" / "verify_web_assets.py"
if not verifier.exists():
return set()
try:
module = ast.parse(verifier.read_text(encoding="utf-8"))
except SyntaxError:
return set()
active: set[str] = set()
for node in module.body:
if not isinstance(node, ast.Assign):
continue
if not any(isinstance(target, ast.Name) and target.id == "ACTIVE_OUT_FILES" for target in node.targets):
continue
try:
values = ast.literal_eval(node.value)
except (SyntaxError, ValueError):
continue
if isinstance(values, list):
active.update(str(value) for value in values if isinstance(value, str))
return active
def file_refs(rel_out: str, sources: list[SourceFile]) -> dict[str, list[str]]:
needles = [needle for needle in reference_needles(rel_out) if needle]
refs: dict[str, list[str]] = defaultdict(list)
for source in sources:
# Exact relative-path references are strong evidence. Basename-only is
# considered only for top-level out files, where false positives are rare
# enough to keep as weak evidence in this conservative audit.
found = False
for needle in needles:
if needle in source.text:
found = True
break
if found:
refs[source.surface].append(rel_posix(source.path))
return {key: sorted(values) for key, values in refs.items()}
PATH_REF_RE = re.compile(r"(?:\.\./\.\./|\.\./|/)?out/[A-Za-z0-9_./-]+\.[A-Za-z0-9]+")
NAME_REF_RE = re.compile(r"\b[A-Za-z0-9_][A-Za-z0-9_.-]+\.(?:html|json|js|md|txt)\b")
OUT_FAMILY_REF_RE = re.compile(r"(?:\.\./\.\./|\.\./|/)?out/([A-Za-z0-9_.-]+)\.\*")
def normalized_out_ref(token: str) -> str:
while token.startswith("../"):
token = token[3:]
if token.startswith("/"):
token = token[1:]
return token
def build_reference_index(out_paths: list[Path], sources: list[SourceFile]) -> dict[str, dict[str, list[str]]]:
rel_set = {rel_posix(path) for path in out_paths}
top_level_by_name: dict[str, list[str]] = defaultdict(list)
top_level_by_stem: dict[str, list[str]] = defaultdict(list)
for rel in rel_set:
rel_path = Path(rel)
if rel_path.parent.as_posix() == "out":
top_level_by_name[rel_path.name].append(rel)
top_level_by_stem[rel_path.stem].append(rel)
refs: dict[str, dict[str, set[str]]] = {
rel: defaultdict(set) for rel in rel_set
}
for source in sources:
source_rel = rel_posix(source.path)
found: set[str] = set()
for match in PATH_REF_RE.finditer(source.text):
rel = normalized_out_ref(match.group(0))
if rel in rel_set:
found.add(rel)
for match in NAME_REF_RE.finditer(source.text):
for rel in top_level_by_name.get(match.group(0), []):
found.add(rel)
# Cleanup notes often cite generated report families as `out/foo.*`
# when HTML/JSON/MD are meant to be retained or retired together. Treat
# that as a weak-but-deliberate reference to every tracked top-level out
# artifact with the same stem so paired JSON is not misclassified as
# tool-only while the HTML is active evidence.
for match in OUT_FAMILY_REF_RE.finditer(source.text):
found.update(top_level_by_stem.get(match.group(1), []))
for rel in found:
if rel == source_rel:
continue
refs[rel][source.surface].add(source_rel)
return {
rel: {surface: sorted(paths) for surface, paths in surface_refs.items()}
for rel, surface_refs in refs.items()
}
def dynamic_reason(rel_out: str) -> str | None:
if rel_out.startswith("out/maps_runtime_chunks/") and rel_out.endswith(".js"):
return "dynamic map chunk loaded by out/maps_runtime.js and web map loaders"
if rel_out == "out/maps_runtime.js":
return "runtime map index used to resolve dynamic map chunks"
return None
def classify(
path: Path,
refs: dict[str, list[str]],
generated_by: list[str],
active_out_files: set[str],
) -> tuple[str, list[str]]:
rel = rel_posix(path)
name = path.name
lower = name.lower()
reasons: list[str] = []
if name in {"out_usage_audit.json", "out_usage_audit.html"}:
return "audit-report", ["current generated out usage audit"]
if name in active_out_files:
reasons.append("listed in tools/verify_web_assets.py ACTIVE_OUT_FILES")
if refs.get("web"):
reasons.append(f"referenced by {len(refs['web'])} web file(s)")
return "verified-active-out", reasons
dynamic = dynamic_reason(rel)
if dynamic:
return "runtime-dynamic", [dynamic]
if name in RUNTIME_CORE_NAMES:
reasons.append("known runtime/core data surface")
if refs.get("web"):
reasons.append("referenced by web")
return "runtime-core", reasons
if refs.get("web"):
reasons.append(f"referenced by {len(refs['web'])} web file(s)")
return "web-review-or-runtime", reasons
if refs.get("docs"):
reasons.append(f"referenced by {len(refs['docs'])} docs file(s)")
if refs.get("tools"):
reasons.append(f"referenced by {len(refs['tools'])} tool file(s)")
return "docs-evidence", reasons
if refs.get("tools"):
reasons.append(f"referenced by {len(refs['tools'])} tool file(s)")
if any(token in lower for token in DIAGNOSTIC_TOKENS):
reasons.append("diagnostic/probe naming")
return "tool-diagnostic", reasons
return "tool-generated-or-input", reasons
if refs.get("out"):
reasons.append(f"referenced by {len(refs['out'])} out report/data file(s)")
return "out-report-dependency", reasons
if any(token in lower for token in DIAGNOSTIC_TOKENS):
reasons.append("no direct refs and diagnostic/probe naming")
return "cleanup-candidate", reasons
if generated_by:
reasons.append("generated by tools but no active refs found")
return "generated-unreferenced", reasons
reasons.append("no direct refs found")
return "unknown-unreferenced", reasons
def build_audit(out_dir: Path) -> dict:
sources = iter_text_sources()
active_out_files = verifier_active_out_files()
paths = out_files(out_dir)
refs_by_path = build_reference_index(paths, sources)
rows = []
for path in paths:
rel = rel_posix(path)
refs = refs_by_path.get(rel, {})
gen = refs.get("tools", [])
category, reasons = classify(path, refs, gen, active_out_files)
stat = path.stat()
rows.append(
{
"path": rel,
"name": path.name,
"extension": path.suffix.lower() or "(none)",
"sizeBytes": stat.st_size,
"category": category,
"reasons": reasons,
"refs": refs,
}
)
by_category = Counter(row["category"] for row in rows)
by_ext = Counter(row["extension"] for row in rows)
size_by_category: dict[str, int] = defaultdict(int)
for row in rows:
size_by_category[row["category"]] += int(row["sizeBytes"])
return {
"kind": "hwanse-out-usage-audit",
"source": "tools/audit_out_usage.py",
"root": str(ROOT),
"outFileCount": len(rows),
"outSizeBytes": sum(int(row["sizeBytes"]) for row in rows),
"categoryCounts": dict(sorted(by_category.items())),
"categorySizeBytes": dict(sorted(size_by_category.items())),
"extensionCounts": dict(sorted(by_ext.items())),
"scanDirs": list(SCAN_DIRS),
"dynamicRules": [
{
"pattern": "out/maps_runtime_chunks/*.js",
"reason": "loaded dynamically by map runtime, so literal refs are not required",
}
],
"verifierActiveOutFiles": sorted(active_out_files),
"rows": sorted(rows, key=lambda row: (row["category"], row["path"])),
}
def human_size(num: int) -> str:
value = float(num)
for unit in ("B", "KB", "MB", "GB"):
if value < 1024 or unit == "GB":
return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} B"
value /= 1024
return f"{num} B"
def render_html(audit: dict) -> str:
categories = audit["categoryCounts"]
rows = audit["rows"]
summary_rows = "\n".join(
""
f"| {html.escape(category)} | "
f"{count} | "
f"{human_size(audit['categorySizeBytes'].get(category, 0))} | "
"
"
for category, count in sorted(categories.items())
)
row_html = []
for row in rows:
refs = row["refs"]
ref_bits = []
for surface in ("web", "docs", "tools"):
values = refs.get(surface) or []
if values:
sample = ", ".join(values[:3])
more = "" if len(values) <= 3 else f" +{len(values) - 3}"
ref_bits.append(f"{surface}: {html.escape(sample)}{more}")
reasons = "; ".join(row["reasons"])
row_html.append(
""
f"{html.escape(row['path'])} | "
f"{html.escape(row['category'])} | "
f"{human_size(int(row['sizeBytes']))} | "
f"{html.escape(reasons)} | "
f"{'
'.join(ref_bits) if ref_bits else 'none'} | "
"
"
)
return f"""
out usage audit
out usage audit
{audit['outFileCount']} files, {human_size(audit['outSizeBytes'])}. This audit is conservative and does not delete files.
Summary
| category | files | size |
|---|
{summary_rows}
Files
| path | category | size | reason | refs |
|---|
{''.join(row_html)}