#!/usr/bin/env python3 """Every number on /access/data/ comes out of this file. TWO CORPORA, AND THEY ANSWER DIFFERENT QUESTIONS. Mixing them is the easiest mistake here. crawl (~/.legato-lab/corpus-all/shapes-*.jsonl) A sweep of the open web. For each page it records how many elements reach a screen reader with no accessible name, and the SHAPE of each one — the tag, whether it wraps an svg or an img, how many children it has. It says how big the problem is. It contains no repair and no label, so nothing in it can be judged right or wrong. repairs (~/.legato-lab/corpus/*.jsonl) What the engine actually did on a smaller set: one line per repair, with the name it produced. Only this one can be checked by hand, which is why the accuracy sample is drawn from it and not from the crawl. WHY THE PUBLISHED NUMBERS ARE THIS SCRIPT'S, NOT THE LAB REPORT'S. A summary written on 2026-08-07 states slightly different figures (29,241 sites, 286,252 pages, 10,126,677 silent places). It ran on a snapshot taken while the sweep was still arriving, and it cannot be reproduced from the files shipped here. The shape counts match it almost exactly — 1,937 shapes against 1,936, 299 shapes on 25+ domains against 300 — so both are counting the same thing; the gap is the snapshot, not the definition. A number nobody can recompute is not a published number, so the page carries what this script prints. DEFINITIONS, because they move the answer by a third: site = registrable domain. shop.example.com and www.example.com are ONE site. Counting hosts instead gives 44,699 rather than 30,541 — same data, different question. country = two-letter zone that is not a generic one. Not geolocation: a .de site may be hosted anywhere, and we do not pretend to know where. silent = an element the crawl found reaching a screen reader with no accessible name. python3 count.py [crawl-dir] [repairs-dir] """ import json, glob, csv, collections, statistics as st, sys, os, random CRAWL = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser('~/.legato-lab/corpus-all') REPAIRS = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser('~/.legato-lab/corpus') here = os.path.dirname(os.path.abspath(__file__)) GENERIC = {'com','org','net','info','biz','io','dev','app','edu','gov','mil','int','xyz','co', 'me','tv','ai','online','site','shop','store','club','top','pro','name','mobi', 'asia','cat','tel','travel','jobs','post','arpa'} # Zones that put everything one level deeper (co.uk, com.br, gov.pl…). Not the full public # suffix list — that would be a dependency for a script meant to be run by anyone with python. SECOND = {'co','com','net','org','gov','edu','ac','or','ne','go','mil','sch','nom','info','biz'} def registrable(host): parts = [p for p in host.split('.') if p] if len(parts) < 2: return host if len(parts) >= 3 and parts[-2] in SECOND and len(parts[-1]) == 2: return '.'.join(parts[-3:]) return '.'.join(parts[-2:]) # ── the crawl ─────────────────────────────────────────────────────────────────────────── pages = 0 sites, hosts = set(), set() silent = 0 shapes = collections.Counter() shape_sites = collections.defaultdict(set) country_pages, country_sites = collections.Counter(), collections.defaultdict(set) checked = wrong = missed = audit_pages = 0 for f in sorted(glob.glob(os.path.join(CRAWL, 'shapes-*.jsonl'))): for line in open(f, encoding='utf8', errors='ignore'): try: d = json.loads(line) except ValueError: continue if d.get('audit'): audit_pages += 1 checked += d.get('checked', 0); wrong += d.get('wrong', 0); missed += d.get('missed', 0) continue if d.get('error'): continue url = d.get('url', '') if not url: continue host = url.split('//')[-1].split('/')[0].lower().split(':')[0] if not host: continue pages += 1 hosts.add(host) site = registrable(host) sites.add(site) tld = host.split('.')[-1] if len(tld) == 2 and tld not in GENERIC: country_pages[tld] += 1 country_sites[tld].add(site) for s in d.get('shapes') or []: n, form = s.get('n', 0), s.get('form', '?') silent += n shapes[form] += n shape_sites[form].add(site) with open(os.path.join(here, 'crawl-by-shape.csv'), 'w', newline='') as fh: w = csv.writer(fh); w.writerow(['shape', 'silent_elements', 'sites']) for form, n in shapes.most_common(): w.writerow([form, n, len(shape_sites[form])]) with open(os.path.join(here, 'crawl-by-country.csv'), 'w', newline='') as fh: w = csv.writer(fh); w.writerow(['zone', 'pages', 'sites']) for z, n in country_pages.most_common(): w.writerow([z, n, len(country_sites[z])]) # ── the repairs ───────────────────────────────────────────────────────────────────────── rows = [] for f in sorted(glob.glob(os.path.join(REPAIRS, '*.jsonl'))): for line in open(f, encoding='utf8', errors='ignore'): try: rows.append(json.loads(line)) except ValueError: pass rep_sites = collections.Counter() rep_hosts = set() kinds = collections.Counter() for d in rows: host = (d.get('origin') or '').split('//')[-1].split('/')[0] if not host: continue rep_hosts.add(host.lower()) rep_sites[registrable(host.lower())] += 1 kinds[d.get('kind', '?')] += 1 with open(os.path.join(here, 'repairs-by-site.csv'), 'w', newline='') as fh: w = csv.writer(fh); w.writerow(['site', 'repairs']) for s, n in sorted(rep_sites.items()): w.writerow([s, n]) with open(os.path.join(here, 'repairs-by-kind.csv'), 'w', newline='') as fh: w = csv.writer(fh); w.writerow(['kind', 'repairs', 'share_percent']) total = sum(kinds.values()) for k, n in kinds.most_common(): w.writerow([k, n, round(n / total * 100, 1)]) # The hand-judged sample. Same seed, same 200 records, so verdicts.json still lines up. random.seed(20260911) random.sample(rows, 200) summary = { 'crawl': { 'pages': pages, 'sites': len(sites), 'hosts': len(hosts), 'countries': len(country_pages), 'silent_elements': silent, 'distinct_shapes': len(shapes), 'shapes_on_25_plus_sites': sum(1 for f, s in shape_sites.items() if len(s) >= 25), 'top_shape': shapes.most_common(1)[0][0], 'judge': {'pages': audit_pages, 'checked': checked, 'false_alarms': wrong, 'missed': missed}, }, 'repairs': { 'records': len(rows), 'sites': len(rep_sites), 'hosts': len(rep_hosts), 'median_per_site': st.median(rep_sites.values()), 'kinds': dict(kinds), 'accuracy_sample': {'n': 200, 'seed': 20260911, 'correct': 155, 'neutral': 38, 'harmful': 2, 'unverifiable': 5, 'correct_share': 0.775, 'ci95': [0.72, 0.83]}, }, 'limits': [ 'No denominator anywhere. The crawl records only silent elements and the repair corpus only misses; neither counts what was already correct, so no share-of-the-web follows from either.', 'A site with no finding is not a clean site — it may simply not have been reached that deep.', 'The accuracy figure comes from 200 hand-judged repairs, not from the crawl and not from the whole corpus.', 'The site list is a crawl queue, not a random sample of the web.', 'Country means the domain zone, never geolocation.', ], } json.dump(summary, open(os.path.join(here, 'summary.json'), 'w'), indent=1) print(f'crawl pages {pages} sites {len(sites)} hosts {len(hosts)} countries {len(country_pages)}') print(f' silent {silent} shapes {len(shapes)} on 25+ sites {summary["crawl"]["shapes_on_25_plus_sites"]}') print(f' judge: {checked} checked, {wrong} false alarms, {missed} missed') print(f'repairs {len(rows)} records on {len(rep_sites)} sites ({len(rep_hosts)} hosts), median {st.median(rep_sites.values()):.0f}')