44 lines
2.9 KiB
Python
44 lines
2.9 KiB
Python
import concurrent.futures, hashlib, json, pathlib, re
|
|
from urllib.parse import urljoin, urlparse, urldefrag
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
ROOT=pathlib.Path(__file__).parent
|
|
OUT=ROOT/'bodies-b'; OUT.mkdir(exist_ok=True)
|
|
SEEDS=['https://handbrake.fr/docs/en/latest/','https://www.naps2.com/doc/','https://www.shutterencoder.com/documentation/','https://www.shutterencoder.com/faq/','https://help.pdf24.org/en/','https://www.pdf24.org/en/creator-features.html','https://helpx.adobe.com/acrobat/using/exporting-pdfs-file-formats.html']
|
|
def allowed(u):
|
|
p=urlparse(u)
|
|
return (p.netloc=='handbrake.fr' and p.path.startswith('/docs/en/latest/')) or (p.netloc=='www.naps2.com' and p.path.startswith('/doc/')) or (p.netloc=='help.pdf24.org' and p.path.startswith('/en/')) or (p.netloc=='www.pdf24.org' and p.path.startswith('/en/') and p.path.endswith('.html')) or (p.netloc=='helpx.adobe.com' and p.path.startswith('/acrobat/using/') and p.path.endswith('.html')) or u in SEEDS
|
|
def fetch(u):
|
|
try:
|
|
r=requests.get(u,timeout=25,headers={'User-Agent':'Mozilla/5.0 Research audit'}); r.raise_for_status()
|
|
if 'text/html' not in r.headers.get('content-type',''): return None
|
|
s=BeautifulSoup(r.content,'html.parser'); links=[urldefrag(urljoin(u,a['href']))[0] for a in s.select('a[href]')]
|
|
title=s.title.get_text(' ',strip=True) if s.title else u
|
|
body=s.select_one('main') or s.select_one('article') or s.select_one('#content') or s.body
|
|
if not body:return None
|
|
for x in body.select('script,style,nav,header,footer,aside'):x.decompose()
|
|
txt=body.get_text('\n',strip=True)
|
|
if len(txt)<450:return None
|
|
return {'url':u,'title':title,'text':txt,'links':links,'sha256':hashlib.sha256(txt.encode()).hexdigest()}
|
|
except Exception:return None
|
|
prior=ROOT/'raw-b.json'
|
|
results=json.loads(prior.read_text(encoding='utf-8')) if prior.exists() else []
|
|
seen={x['url'] for x in results}; hashes={x['sha256'] for x in results}
|
|
pending=json.loads((ROOT/'discovered-b.json').read_text(encoding='utf-8')) if (ROOT/'discovered-b.json').exists() else SEEDS[:]
|
|
while pending and len(results)<165:
|
|
batch=[u for u in dict.fromkeys(pending) if u not in seen][:30];pending=[];seen.update(batch)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
|
|
for a in ex.map(fetch,batch):
|
|
if not a:continue
|
|
pending.extend(u for u in a.pop('links') if allowed(u) and u not in seen)
|
|
if a['sha256'] in hashes:continue
|
|
hashes.add(a['sha256']); results.append(a)
|
|
(OUT/(str(len(results)).zfill(3)+'.txt')).write_text(a['text'],encoding='utf-8')
|
|
# retain discovered but unprocessed links across rounds
|
|
allknown=ROOT/'discovered-b.json'
|
|
old=json.loads(allknown.read_text()) if allknown.exists() else []
|
|
old=list(dict.fromkeys(old+pending));allknown.write_text(json.dumps(old),encoding='utf-8')
|
|
pending=[u for u in old if u not in seen]
|
|
print('Fetched',len(results),'Remaining',len(pending),flush=True)
|
|
(ROOT/'raw-b.json').write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf-8')
|