import json, re, os, sys
from pathlib import Path
from urllib.parse import unquote

D = str(Path(__file__).resolve().parent)
SHUTDOWN = '20250323'          # BookLooks ceased operation, [P] in the lineage memo

STOP = re.compile(r'\b(a|an|the)\b')


def norm(s):
    s = unquote(s).lower()
    s = re.sub(r'[‘’“”–—]', ' ', s)
    s = re.sub(r'[^a-z0-9 ]+', ' ', s)
    s = STOP.sub(' ', s)
    return re.sub(r'\s+', ' ', s).strip()


# ---------- BookLooks: title from the PDF filename
bl_rows = json.load(open(f'{D}/bl_cdx.json'))[1:]
bl = {}
for orig, ts in bl_rows:
    if not orig.lower().endswith('.pdf'):
        continue
    name = unquote(orig.rsplit('/', 1)[-1])[:-4]
    name = re.sub(r'\s*(slick\s*sheet|report|final|v\d+|\(\d+\))\s*$', '', name, flags=re.I)
    k = norm(name)
    if len(k) < 3:
        continue
    bl.setdefault(k, ts)
    if ts < bl[k]:
        bl[k] = ts

# ---------- RatedBooks: title from the product-page slug, minus "-by-<author>"
rb_rows = json.load(open(f'{D}/rb_cdx.json'))[1:]
rb = {}
for orig, ts in rb_rows:
    m = re.search(r'/product-page/([^/?#]+)', orig)
    if not m:
        continue
    slug = unquote(m.group(1))
    if len(slug) < 3 or slug in (')', '-'):
        continue
    title = re.split(r'-by-', slug, maxsplit=1)[0]
    title = re.sub(r'^\d{1,3}-', '', title)       # leading list numbers, e.g. "04-the-bluest-eye"
    k = norm(title.replace('-', ' '))
    if len(k) < 3:
        continue
    if k not in rb or ts < rb[k]:
        rb[k] = ts

print(f'BookLooks distinct normalised titles : {len(bl)}   (from 960 archived PDFs)')
print(f'RatedBooks distinct normalised titles: {len(rb)}   (from {len(rb_rows)} archived product pages)')
print()

# ---------- CONTROL: titles that MUST join if the normaliser works
CTRL = ['gender queer', 'lawn boy', 'looking for alaska', 'bluest eye', 'beloved']
ok = 0
for c in CTRL:
    inb, inr = c in bl, c in rb
    print(f'  CONTROL {c!r:<22} BookLooks={inb}  RatedBooks={inr}')
    if inb and inr:
        ok += 1
if ok < 3:
    print('\n⛔ CONTROL FAILED (<3 of 5 join). Normaliser is broken; refusing to print an overlap.')
    sys.exit(1)
print(f'  -> {ok}/5 join. Control PASSES.\n')

# ---------- the measurement
shared = set(bl) & set(rb)
print(f'SHARED titles: {len(shared)}')
print(f'  = {len(shared)/len(rb):.1%} of the RatedBooks set')
print(f'  = {len(shared)/len(bl):.1%} of the BookLooks set')
print()

pre = {k for k, ts in rb.items() if ts < SHUTDOWN}
post = {k for k, ts in rb.items() if ts >= SHUTDOWN}
print(f'RatedBooks titles first archived BEFORE {SHUTDOWN}: {len(pre)}')
print(f'                          ON/AFTER {SHUTDOWN}: {len(post)}')
print()
print(f'  of the PRE set,  shared with BookLooks: {len(pre & shared)}  ({len(pre & shared)/max(1,len(pre)):.1%})')
print(f'  of the POST set, shared with BookLooks: {len(post & shared)}  ({len(post & shared)/max(1,len(post)):.1%})')
print()
print('RatedBooks titles first archived by year:')
yr = {}
for k, ts in rb.items():
    yr[ts[:4]] = yr.get(ts[:4], 0) + 1
for y in sorted(yr):
    print(f'   {y}: {yr[y]:>5}')

json.dump({'booklooks': sorted(bl), 'ratedbooks': {k: v for k, v in sorted(rb.items())},
           'shared': sorted(shared)}, open(f'{D}/rebuilt-sets.json', 'w'), indent=1)