1
0
Fork 0
Everything2Everything/docs/research/2026-11-04/collect-b.py

44 lines
2.9 KiB
Python

import concurrent.futures, hashlib, json, pathlib, re
from urllib.parse import urljoin, urlparse, urldefrag
import requests
from bs4 import BeautifulSoup
ROOT=pathlib.Path(__file__).parent
OUT=ROOT/'bodies-b'; OUT.mkdir(exist_ok=True)
SEEDS=['https://handbrake.fr/docs/en/latest/','https://www.naps2.com/doc/','https://www.shutterencoder.com/documentation/','https://www.shutterencoder.com/faq/','https://help.pdf24.org/en/','https://www.pdf24.org/en/creator-features.html','https://helpx.adobe.com/acrobat/using/exporting-pdfs-file-formats.html']
def allowed(u):
p=urlparse(u)
return (p.netloc=='handbrake.fr' and p.path.startswith('/docs/en/latest/')) or (p.netloc=='www.naps2.com' and p.path.startswith('/doc/')) or (p.netloc=='help.pdf24.org' and p.path.startswith('/en/')) or (p.netloc=='www.pdf24.org' and p.path.startswith('/en/') and p.path.endswith('.html')) or (p.netloc=='helpx.adobe.com' and p.path.startswith('/acrobat/using/') and p.path.endswith('.html')) or u in SEEDS
def fetch(u):
try:
r=requests.get(u,timeout=25,headers={'User-Agent':'Mozilla/5.0 Research audit'}); r.raise_for_status()
if 'text/html' not in r.headers.get('content-type',''): return None
s=BeautifulSoup(r.content,'html.parser'); links=[urldefrag(urljoin(u,a['href']))[0] for a in s.select('a[href]')]
title=s.title.get_text(' ',strip=True) if s.title else u
body=s.select_one('main') or s.select_one('article') or s.select_one('#content') or s.body
if not body:return None
for x in body.select('script,style,nav,header,footer,aside'):x.decompose()
txt=body.get_text('\n',strip=True)
if len(txt)<450:return None
return {'url':u,'title':title,'text':txt,'links':links,'sha256':hashlib.sha256(txt.encode()).hexdigest()}
except Exception:return None
prior=ROOT/'raw-b.json'
results=json.loads(prior.read_text(encoding='utf-8')) if prior.exists() else []
seen={x['url'] for x in results}; hashes={x['sha256'] for x in results}
pending=json.loads((ROOT/'discovered-b.json').read_text(encoding='utf-8')) if (ROOT/'discovered-b.json').exists() else SEEDS[:]
while pending and len(results)<165:
batch=[u for u in dict.fromkeys(pending) if u not in seen][:30];pending=[];seen.update(batch)
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
for a in ex.map(fetch,batch):
if not a:continue
pending.extend(u for u in a.pop('links') if allowed(u) and u not in seen)
if a['sha256'] in hashes:continue
hashes.add(a['sha256']); results.append(a)
(OUT/(str(len(results)).zfill(3)+'.txt')).write_text(a['text'],encoding='utf-8')
# retain discovered but unprocessed links across rounds
allknown=ROOT/'discovered-b.json'
old=json.loads(allknown.read_text()) if allknown.exists() else []
old=list(dict.fromkeys(old+pending));allknown.write_text(json.dumps(old),encoding='utf-8')
pending=[u for u in old if u not in seen]
print('Fetched',len(results),'Remaining',len(pending),flush=True)
(ROOT/'raw-b.json').write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf-8')