331 lines
11 KiB
Python
331 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Analyze all JSON files in packs-src/ for text quality issues."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from html.parser import HTMLParser
|
|
|
|
BASE = Path("/home/morr/work/uberwald/fvtt-chroniques-de-l-etrange")
|
|
PACKS = BASE / "packs-src"
|
|
REGLES = BASE / "regles.txt"
|
|
|
|
# Load PDF text
|
|
pdf_lines = REGLES.read_text(encoding="utf-8").splitlines()
|
|
pdf_text = REGLES.read_text(encoding="utf-8")
|
|
|
|
issues = []
|
|
|
|
# ---------- helpers ----------
|
|
|
|
def strip_html(html):
|
|
"""Remove HTML tags and return plain text."""
|
|
return re.sub(r'<[^>]+>', '', html or '')
|
|
|
|
def check_unclosed_tags(html):
|
|
"""Returns list of unclosed/mismatched tags."""
|
|
open_tags = re.findall(r'<([a-zA-Z][a-zA-Z0-9]*)[^>]*>', html)
|
|
close_tags = re.findall(r'</([a-zA-Z][a-zA-Z0-9]*)>', html)
|
|
issues_found = []
|
|
# basic: count opens vs closes for block-level tags
|
|
for tag in ['ul', 'ol', 'li', 'p', 'div', 'strong', 'em', 'b', 'i']:
|
|
opens = open_tags.count(tag)
|
|
closes = close_tags.count(tag)
|
|
if opens != closes:
|
|
issues_found.append(f"<{tag}>: {opens} open, {closes} close")
|
|
return issues_found
|
|
|
|
def has_bad_newlines(html):
|
|
"""Check for literal \\n inside HTML strings that would render as bad breaks."""
|
|
# In JSON, \n is a newline. In HTML strings, raw newlines can be bad.
|
|
return '\n' in html
|
|
|
|
def looks_truncated(text):
|
|
"""Heuristics for truncation - text ends without proper punctuation."""
|
|
if not text:
|
|
return False
|
|
plain = strip_html(text).strip()
|
|
if not plain:
|
|
return False
|
|
# ends without sentence-ending punctuation
|
|
if plain and plain[-1] not in '.!?»)':
|
|
return True
|
|
return False
|
|
|
|
def looks_truncated_strict(text):
|
|
"""Stricter: ends mid-word or mid-sentence."""
|
|
if not text:
|
|
return False
|
|
plain = strip_html(text).strip()
|
|
if not plain:
|
|
return False
|
|
# ends mid-word (no space before end, no punctuation)
|
|
last_char = plain[-1] if plain else ''
|
|
if last_char.isalpha() or last_char in ',;:-(':
|
|
return True
|
|
return False
|
|
|
|
def get_field(data, path):
|
|
"""Get nested field value by dot-path."""
|
|
parts = path.split('.')
|
|
cur = data
|
|
for p in parts:
|
|
if isinstance(cur, dict):
|
|
cur = cur.get(p)
|
|
else:
|
|
return None
|
|
if cur is None:
|
|
return None
|
|
return cur
|
|
|
|
def search_pdf(keyword, context=300):
|
|
"""Search PDF text for a keyword and return surrounding context."""
|
|
# clean keyword for searching
|
|
kw = re.sub(r'<[^>]+>', '', keyword).strip()
|
|
if len(kw) < 10:
|
|
return None
|
|
# take last 30 chars of plain text as search key
|
|
search_key = kw[-30:].strip()
|
|
# normalize whitespace
|
|
search_key_norm = re.sub(r'\s+', ' ', search_key)
|
|
|
|
# Try to find in PDF
|
|
idx = pdf_text.find(search_key_norm)
|
|
if idx == -1:
|
|
# try shorter
|
|
search_key_norm = re.sub(r'\s+', ' ', kw[-20:].strip())
|
|
idx = pdf_text.find(search_key_norm)
|
|
if idx == -1:
|
|
# try even shorter
|
|
search_key_norm = re.sub(r'\s+', ' ', kw[-15:].strip())
|
|
idx = pdf_text.find(search_key_norm)
|
|
|
|
if idx == -1:
|
|
return None
|
|
|
|
start = max(0, idx - 50)
|
|
end = min(len(pdf_text), idx + len(search_key_norm) + context)
|
|
return pdf_text[start:end].replace('\n', ' ')
|
|
|
|
def get_all_html_fields(data, prefix=""):
|
|
"""Recursively yield (field_path, value) for all string fields containing HTML."""
|
|
if isinstance(data, dict):
|
|
for k, v in data.items():
|
|
path = f"{prefix}.{k}" if prefix else k
|
|
if isinstance(v, str) and ('<' in v or len(v) > 50):
|
|
yield path, v
|
|
elif isinstance(v, (dict, list)):
|
|
yield from get_all_html_fields(v, path)
|
|
elif isinstance(data, list):
|
|
for i, v in enumerate(data):
|
|
yield from get_all_html_fields(v, f"{prefix}[{i}]")
|
|
|
|
# ---------- fields to check ----------
|
|
|
|
IMPORTANT_FIELDS = [
|
|
"system.description",
|
|
"system.effects",
|
|
"system.examples",
|
|
"system.components",
|
|
"system.notes",
|
|
"system.style",
|
|
"system.techniques.technique1.technique",
|
|
"system.techniques.technique2.technique",
|
|
"system.techniques.technique3.technique",
|
|
]
|
|
|
|
# ---------- main scan ----------
|
|
|
|
json_files = sorted(PACKS.rglob("*.json"))
|
|
print(f"Scanning {len(json_files)} JSON files...", flush=True)
|
|
|
|
for jf in json_files:
|
|
rel = str(jf.relative_to(PACKS))
|
|
try:
|
|
data = json.loads(jf.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as e:
|
|
issues.append({
|
|
"file": rel,
|
|
"field": "(file)",
|
|
"issue": "json_parse_error",
|
|
"current_text": str(e),
|
|
"correct_continuation": None,
|
|
})
|
|
continue
|
|
|
|
item_name = data.get("name", "(unnamed)")
|
|
|
|
# Check all relevant fields
|
|
for field in IMPORTANT_FIELDS:
|
|
val = get_field(data, field)
|
|
if not val or not isinstance(val, str):
|
|
continue
|
|
|
|
plain = strip_html(val).strip()
|
|
|
|
# 1. Check truncation (strict)
|
|
if looks_truncated_strict(val):
|
|
pdf_context = search_pdf(val)
|
|
issues.append({
|
|
"file": rel,
|
|
"field": field,
|
|
"issue": "truncated",
|
|
"item_name": item_name,
|
|
"current_end": f"...{plain[-100:]}",
|
|
"current_full_preview": f"{plain[:200]}",
|
|
"correct_continuation": pdf_context,
|
|
})
|
|
|
|
# 2. Check bad newlines in HTML strings
|
|
if has_bad_newlines(val):
|
|
issues.append({
|
|
"file": rel,
|
|
"field": field,
|
|
"issue": "unwanted_newlines",
|
|
"item_name": item_name,
|
|
"current_text": val[:300],
|
|
"correct_continuation": None,
|
|
})
|
|
|
|
# 3. Check malformed HTML
|
|
html_errors = check_unclosed_tags(val)
|
|
if html_errors:
|
|
issues.append({
|
|
"file": rel,
|
|
"field": field,
|
|
"issue": "malformed_html",
|
|
"item_name": item_name,
|
|
"html_errors": html_errors,
|
|
"current_text": val[:300],
|
|
"correct_continuation": None,
|
|
})
|
|
|
|
# 4. Check system.style (plain text field, can also be truncated)
|
|
style_val = get_field(data, "system.style")
|
|
if style_val and isinstance(style_val, str):
|
|
plain_style = style_val.strip()
|
|
if plain_style and plain_style[-1] not in '.!?»)':
|
|
pdf_context = search_pdf(plain_style)
|
|
issues.append({
|
|
"file": rel,
|
|
"field": "system.style",
|
|
"issue": "truncated",
|
|
"item_name": item_name,
|
|
"current_end": f"...{plain_style[-100:]}",
|
|
"current_full_preview": f"{plain_style[:200]}",
|
|
"correct_continuation": pdf_context,
|
|
})
|
|
|
|
# 5. Bleeding content: look for HTML tags in non-HTML fields
|
|
for field in ["system.style", "system.reference", "system.speciality"]:
|
|
val = get_field(data, field)
|
|
if val and isinstance(val, str) and '<' in val:
|
|
issues.append({
|
|
"file": rel,
|
|
"field": field,
|
|
"issue": "html_in_plain_field",
|
|
"item_name": item_name,
|
|
"current_text": val[:300],
|
|
"correct_continuation": None,
|
|
})
|
|
|
|
# 6. Check for text outside HTML tags in description-like fields (bleeding)
|
|
for field in ["system.description", "system.effects", "system.examples", "system.components", "system.notes"]:
|
|
val = get_field(data, field)
|
|
if not val or not isinstance(val, str):
|
|
continue
|
|
# Strip all HTML and check if leading text is outside tags
|
|
# e.g., "<p>foo</p> some leaked text <p>bar</p>"
|
|
# Check if there's text before the first tag
|
|
stripped = val.strip()
|
|
if stripped and not stripped.startswith('<'):
|
|
issues.append({
|
|
"file": rel,
|
|
"field": field,
|
|
"issue": "text_outside_html_tags",
|
|
"item_name": item_name,
|
|
"current_text": val[:300],
|
|
"correct_continuation": None,
|
|
})
|
|
|
|
# 7. Check technique fields for bleeding (multiple paragraphs that shouldn't be there)
|
|
for tkey in ["technique1", "technique2", "technique3"]:
|
|
tech = get_field(data, f"system.techniques.{tkey}")
|
|
if not tech:
|
|
continue
|
|
tech_text = tech.get("technique", "")
|
|
if tech_text:
|
|
plain = strip_html(tech_text).strip()
|
|
# Check for suspiciously long techniques that might have bled content
|
|
# Techniques with multiple <p> blocks may be fine, but flag very long ones
|
|
p_count = tech_text.count('</p>')
|
|
if p_count > 3:
|
|
issues.append({
|
|
"file": rel,
|
|
"field": f"system.techniques.{tkey}.technique",
|
|
"issue": "possible_bleeding_content",
|
|
"item_name": item_name,
|
|
"paragraph_count": p_count,
|
|
"current_text": tech_text[:400],
|
|
"correct_continuation": None,
|
|
})
|
|
|
|
print(f"Found {len(issues)} potential issues.", flush=True)
|
|
|
|
# ---------- output ----------
|
|
|
|
out_json = BASE / "compendium-issues.json"
|
|
out_txt = BASE / "compendium-issues.txt"
|
|
|
|
with open(out_json, 'w', encoding='utf-8') as f:
|
|
json.dump(issues, f, ensure_ascii=False, indent=2)
|
|
|
|
# Group by issue type for summary
|
|
from collections import defaultdict
|
|
by_type = defaultdict(list)
|
|
by_file = defaultdict(list)
|
|
for issue in issues:
|
|
by_type[issue['issue']].append(issue)
|
|
by_file[issue['file']].append(issue)
|
|
|
|
with open(out_txt, 'w', encoding='utf-8') as f:
|
|
f.write("=" * 80 + "\n")
|
|
f.write("COMPENDIUM TEXT QUALITY REPORT\n")
|
|
f.write("Les Chroniques de l'Étrange — FoundryVTT\n")
|
|
f.write("=" * 80 + "\n\n")
|
|
|
|
f.write(f"Total files scanned: {len(json_files)}\n")
|
|
f.write(f"Total issues found: {len(issues)}\n\n")
|
|
|
|
f.write("SUMMARY BY ISSUE TYPE:\n")
|
|
for itype, ilist in sorted(by_type.items()):
|
|
f.write(f" {itype}: {len(ilist)}\n")
|
|
f.write("\n")
|
|
|
|
f.write("=" * 80 + "\n")
|
|
f.write("DETAILED ISSUES BY FILE\n")
|
|
f.write("=" * 80 + "\n\n")
|
|
|
|
for fpath in sorted(by_file.keys()):
|
|
f.write(f"\n--- {fpath} ---\n")
|
|
for issue in by_file[fpath]:
|
|
f.write(f" FIELD: {issue['field']}\n")
|
|
f.write(f" ISSUE: {issue['issue']}\n")
|
|
if issue.get('item_name'):
|
|
f.write(f" ITEM: {issue['item_name']}\n")
|
|
if issue.get('current_end'):
|
|
f.write(f" END: {issue['current_end']}\n")
|
|
if issue.get('current_full_preview'):
|
|
f.write(f" TEXT: {issue['current_full_preview'][:200]}\n")
|
|
if issue.get('current_text'):
|
|
f.write(f" TEXT: {issue['current_text'][:200]}\n")
|
|
if issue.get('html_errors'):
|
|
f.write(f" HTML ERRORS: {issue['html_errors']}\n")
|
|
if issue.get('correct_continuation'):
|
|
f.write(f" PDF: {issue['correct_continuation'][:300]}\n")
|
|
f.write("\n")
|
|
|
|
print(f"Reports written to:\n {out_json}\n {out_txt}", flush=True)
|