import concurrent.futures, hashlib, json, pathlib, re from urllib.parse import urljoin, urlparse, urldefrag import requests from bs4 import BeautifulSoup ROOT=pathlib.Path(__file__).parent OUT=ROOT/'bodies-b'; OUT.mkdir(exist_ok=True) SEEDS=['https://handbrake.fr/docs/en/latest/','https://www.naps2.com/doc/','https://www.shutterencoder.com/documentation/','https://www.shutterencoder.com/faq/','https://help.pdf24.org/en/','https://www.pdf24.org/en/creator-features.html','https://helpx.adobe.com/acrobat/using/exporting-pdfs-file-formats.html'] def allowed(u): p=urlparse(u) return (p.netloc=='handbrake.fr' and p.path.startswith('/docs/en/latest/')) or (p.netloc=='www.naps2.com' and p.path.startswith('/doc/')) or (p.netloc=='help.pdf24.org' and p.path.startswith('/en/')) or (p.netloc=='www.pdf24.org' and p.path.startswith('/en/') and p.path.endswith('.html')) or (p.netloc=='helpx.adobe.com' and p.path.startswith('/acrobat/using/') and p.path.endswith('.html')) or u in SEEDS def fetch(u): try: r=requests.get(u,timeout=25,headers={'User-Agent':'Mozilla/5.0 Research audit'}); r.raise_for_status() if 'text/html' not in r.headers.get('content-type',''): return None s=BeautifulSoup(r.content,'html.parser'); links=[urldefrag(urljoin(u,a['href']))[0] for a in s.select('a[href]')] title=s.title.get_text(' ',strip=True) if s.title else u body=s.select_one('main') or s.select_one('article') or s.select_one('#content') or s.body if not body:return None for x in body.select('script,style,nav,header,footer,aside'):x.decompose() txt=body.get_text('\n',strip=True) if len(txt)<450:return None return {'url':u,'title':title,'text':txt,'links':links,'sha256':hashlib.sha256(txt.encode()).hexdigest()} except Exception:return None prior=ROOT/'raw-b.json' results=json.loads(prior.read_text(encoding='utf-8')) if prior.exists() else [] seen={x['url'] for x in results}; hashes={x['sha256'] for x in results} pending=json.loads((ROOT/'discovered-b.json').read_text(encoding='utf-8')) if (ROOT/'discovered-b.json').exists() else SEEDS[:] while pending and len(results)<165: batch=[u for u in dict.fromkeys(pending) if u not in seen][:30];pending=[];seen.update(batch) with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex: for a in ex.map(fetch,batch): if not a:continue pending.extend(u for u in a.pop('links') if allowed(u) and u not in seen) if a['sha256'] in hashes:continue hashes.add(a['sha256']); results.append(a) (OUT/(str(len(results)).zfill(3)+'.txt')).write_text(a['text'],encoding='utf-8') # retain discovered but unprocessed links across rounds allknown=ROOT/'discovered-b.json' old=json.loads(allknown.read_text()) if allknown.exists() else [] old=list(dict.fromkeys(old+pending));allknown.write_text(json.dumps(old),encoding='utf-8') pending=[u for u in old if u not in seen] print('Fetched',len(results),'Remaining',len(pending),flush=True) (ROOT/'raw-b.json').write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf-8')