import json, re, os, glob
from pathlib import Path

L = str(Path(__file__).resolve().parent)
SHUTDOWN = '20250323'
sets = json.load(open(f'{L}/_sets.json'))
rb, bl = sets['ratedbooks'], set(sets['booklooks'])

JUNK = [
    (r'mv2 (jpg|png|jpeg|gif|webp|avif)$', 'wix _mv2 image'),
    (r'\b(jpg|jpeg|png|gif|webp|avif|mp4|svg|css|js)$', 'asset extension'),
    (r'\b[0-9a-f]{16,}\b', 'hex blob'),
    (r'\b(inline|bundle|bundler|min js|widget|module|chunk|polyfill|runtime|viewer)\b', 'wix js bundle'),
    (r'^h \d+$', 'image height param'),
    (r'^(w|q|enc|blur|fill|fit|crop|quality) ?\d*$', 'image transform param'),
    (r'^\W*$|^.{0,4}$', 'too short/empty'),
    (r'^\d[\d ]*$', 'digits only'),
    (r'^copy of ', 'wix "copy of" page'),
]


def why(k):
    for pat, w in JUNK:
        if re.search(pat, k):
            return w
    return None


clean, drop = {}, {}
for k, ts in rb.items():
    w = why(k)
    (drop.setdefault(w, []).append(k) if w else clean.update({k: ts}))

print(f'raw: {len(rb)}')
for w, ks in sorted(drop.items(), key=lambda kv: -len(kv[1])):
    print(f'   -{len(ks):>4}  {w}')
print(f'CLEAN: {len(clean)}\n')

pre = {k for k, ts in clean.items() if ts < SHUTDOWN}
post = {k for k, ts in clean.items() if ts >= SHUTDOWN}
print(f'{"":<32}{"n":>6}{"∩BookLooks":>13}{"rate":>8}')
print('-' * 60)
for lbl, s in [('all clean', set(clean)), ('pre 2025-03-23', pre), ('post', post)]:
    print(f'{lbl:<32}{len(s):>6}{len(s & bl):>13}{len(s & bl)/max(1,len(s)):>7.1%}')