"""Measure what unit-like strings actually appear in question stems."""
import re
from collections import Counter
from app.database import SessionLocal
from sqlalchemy import text as sa_text

db = SessionLocal()
rows = db.execute(sa_text("SELECT id, question_text FROM questions")).fetchall()
print("questions:", len(rows))

# A unit token: something after a number, made of letters/slashes/%/micro signs.
unit_re = re.compile(r"(?<=[\d\s])([A-Za-zµμµ%][A-Za-zµμµ%/\.³²¹\^]{0,14})(?=[\s,;\)\.]|$)")
counts = Counter()
for qid, text in rows:
    if not text:
        continue
    for m in unit_re.finditer(text):
        counts[m.group(1)] += 1

for tok, n in counts.most_common(400):
    print(f"{n:6d}  {tok!r}")