# Compara tags WD14 (scores.json) contra biblioteca PlumLoom.
# Salida: top tags por pack y lista de ausentes sin cobertura en wildcards/biblioteca.

import json
import pathlib
import re
from collections import Counter, defaultdict

BASE = pathlib.Path(r"C:\Users\Usuario\Downloads\SD")
SCORES = BASE / "Archivos" / "wd14" / "tags" / "pawchive" / "scores.json"
data = json.loads(SCORES.read_text(encoding="utf-8"))

# Normaliza: score >= 0.35 ya viene filtrado en .txt, pero scores.json trae todo;
# contamos presencia por imagen con score >= 0.35.
def pack_of(key: str) -> str:
    return key.split("/")[0] if "/" in key else "desconocido"

by_pack = defaultdict(Counter)
by_pack_n = Counter()
norm_count = Counter()  # total por tag
for key, pairs in data.items():
    pk = pack_of(key)
    by_pack_n[pk] += 1
    seen = set()
    for tag, score in pairs:
        if score >= 0.35:
            seen.add(tag)
            by_pack[pk][tag] += 1
            norm_count[tag] += 1

print("=== imágenes por pack ===")
for k, v in sorted(by_pack_n.items()):
    print(k, v)

# Biblioteca: tokens de wildcards + biblioteca unificada (todo el texto)
lib_text = ""
for p in (BASE / "Archivos" / "wildcards").glob("*.txt"):
    lib_text += p.read_text(encoding="utf-8").lower().replace("_", " ") + "\n"
for extra in [
    BASE / "Archivos/pose_bank/BIBLIOTECA_UNIFICADA.md",
    BASE / "Archivos/pose_bank/explicit1_acts_biblioteca.md",
    BASE / "Archivos/pose_bank/explicit2_positions_biblioteca.md",
    BASE / "Archivos/pose_bank/attire_biblioteca.md",
]:
    if extra.exists():
        lib_text += extra.read_text(encoding="utf-8").lower().replace("_", " ") + "\n"
lib_tokens = set(re.findall(r"[a-z0-9]+", lib_text))
# also include exact tag strings lowercased with underscores
lib_tags = set()
for p in (BASE / "Archivos" / "wildcards").glob("*.txt"):
    for line in p.read_text(encoding="utf-8").splitlines():
        for t in line.split(","):
            lib_tags.add(t.strip().lower())

def covered(tag: str) -> bool:
    t = tag.lower()
    if t in lib_tags:
        return True
    parts = set(re.findall(r"[a-z0-9]+", t.replace("_", " ")))
    # cubierto si todos los tokens aparecen en la biblioteca
    return parts and parts.issubset(lib_tokens)

TOP = 40
for pk, cnt in sorted(by_pack.items()):
    print(f"\n=== {pk} top {TOP} (de {by_pack_n[pk]} imgs) ===")
    for tag, c in cnt.most_common(TOP):
        mark = "" if covered(tag) else "  <-- AUSENTE"
        print(f"{c:4d}  {tag}{mark}")

print("\n=== TOP global ausentes (frecuencia >= 12 imágenes) ===")
missing = [(t, c) for t, c in norm_count.most_common() if not covered(t) and c >= 12]
for t, c in missing[:80]:
    print(f"{c:4d}  {t}")