"""Offline private-capture verification. No target requests.
python3 publish.py --private-output runner/data/crawlee-public-30-20260930 --canary runner/data/crawlee-canary-20260930
"""
import argparse,ast,hashlib,json
from collections import Counter
from datetime import datetime,timezone
from pathlib import Path
HERE=Path(__file__).resolve().parent
sha=lambda data:hashlib.sha256(data).hexdigest()
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--private-output',type=Path,required=True);p.add_argument('--canary',type=Path,required=True);args=p.parse_args()
manifest=json.loads((HERE.parent/'public-30-v1.json').read_text())
def nodes(path):
 tree=ast.parse(path.read_text());return [n for n in tree.body if getattr(n,'name',None) in ['PageText','evaluate','result_class'] or isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id in ['BLOCK_PHRASES','GATE_PHRASES'] for t in n.targets)]
reference=nodes(HERE.parent/'scrapling-public-30-20260929/run.py')
assert [ast.dump(n,include_attributes=False) for n in nodes(HERE/'run.py')]==[ast.dump(n,include_attributes=False) for n in reference]
ns={};exec('from html.parser import HTMLParser\nimport re',ns);exec(compile(ast.Module(body=reference,type_ignores=[]),'unchanged-checker','exec'),ns)
def verify(a,t,folder):
 body=(folder/(a['target_id']+'.body')).read_bytes()
 assert sha(body)==a['raw_response_sha256'] and len(body)==a['response_bytes']
 assert ns['evaluate'](body,t)==a['checks'] and ns['result_class'](a)==a['classification']
 assert a['usable_content']==(a['classification']=='usable') and a['empty_content']==(a['checks']['text_chars']==0)
 assert a['attempt_process_exit']==0 and not a['cleanup_surviving_owned_pids'] and a['request_started_at']
 readings=json.loads((folder/(a['target_id']+'-pss.json')).read_text())
 assert len(readings)==a['pss_samples'] and max(s['pss_mb'] for s in readings)==a['peak_sampled_tree_pss_mb']
 assert all(s['host_available_mb']>=3072 and s['pss_mb']<=1024 for s in readings)
 log=(folder/(a['target_id']+'-transport.json')).read_bytes();assert sha(log)==a['transport_log_sha256'];log=json.loads(log)
 requests=[{**r,'headers':{k:v for k,v in r['headers'].items() if k.lower() not in ['cookie','authorization','proxy-authorization']}} for r in log['requests']]
 assert requests==a['observed_request_headers'] and len(requests)==a['downloader_request_count'] and 1<=len(requests)<=4
 assert requests[0]['url']==t['url'] and all(r['method']=='GET' for r in requests)
 assert not any(k.lower()=='cookie' for r in log['requests'] for k in r['headers'])
 assert log['redirects']==a['redirect_history'] and len(log['redirects'])<=3
 for i,h in enumerate(log['redirects']):assert h['previous']==requests[i]['url'] and h['url']==requests[i+1]['url'] and h['status'] in [301,302,303,307,308]
 assert [{k:r.get(k) for k in ['url','status']} for r in log['responses']]==[{k:r[k] for k in ['url','status']} for r in a['document_responses']]
 if log['responses']:
  assert log['responses'][-1]['url']==a['final_url'] and log['responses'][-1]['status']==a['status']
 assert sha((folder/(a['target_id']+'-driver.log')).read_bytes())==a['driver_log_sha256']
 a.pop('screenshot_private_path',None);a.pop('raw_response_private_path',None)
 a['inspection_note']=('Complete HTTP source body met frozen checks; no rendered visibility, browser interaction or complete field extraction verified.' if a['usable_content'] else 'Saved HTTP body rechecked with unchanged rules: '+a['classification'].replace('_',' ')+'.')
 if a['status']==200 and a['empty_content']:a['inspection_note']+=' HTTP 200 has no checked text.'
 if a['target_id']=='webscrapingdev':a['inspection_note']+=' HTTP mode does not execute JavaScript to populate reviews.'
 if a['target_id']=='target':a['inspection_note']+=' Observed HTTP 429 and a complete application/octet-stream body. CheerioCrawler rejected its unsupported MIME type; the frozen error-first classifier retains request_error, rather than silently changing it to http_rate_limited.'
 if a['target_id']=='bestbuy':a['inspection_note']+=' Source asks the visitor to choose a country; required listings were absent.'
 if a['target_id']=='example':a['inspection_note']+=' Documentation-use wording did not match the frozen Example Domain phrase. No browser/CSS visibility was inspected.'
 a['request_header_observation_basis']='got beforeRequest hook after built-in header generation; includes HTTP redirect requests. Not independent wire capture; DNS/TLS/ALPN probes and lower-level network recovery are not counted.'
 return readings
run=json.loads((args.private_output/'results.json').read_text());assert len(run['attempts'])==30 and run.get('finished_at') and not run.get('blocker')
assert [(a['target_id'],a['original_url']) for a in run['attempts']]==[(t['id'],t['url']) for t in manifest['targets']]
for field,file in [('script_sha256','run.py'),('worker_sha256','worker.mjs'),('lock_sha256','package-lock.json'),('package_sha256','package.json')]:assert run[field]==sha((HERE/file).read_bytes())
assert run['manifest_sha256']==sha((HERE.parent/'public-30-v1.json').read_bytes())
for name,digest in run['preserved_evidence_sha256'].items():assert sha(Path(name).read_bytes())==digest,name
samples={a['target_id']:verify(a,t,args.private_output) for a,t in zip(run['attempts'],manifest['targets'])}
assert run['summary']['classifications']==dict(Counter(a['classification'] for a in run['attempts']))
canary=json.loads((args.canary/'results.json').read_text());assert len(canary['attempts'])==1 and canary['script_sha256']==run['script_sha256'] and canary['worker_sha256']==run['worker_sha256'] and canary['config']==run['config']
c=canary['attempts'][0];verify(c,next(t for t in manifest['targets'] if t['id']==c['target_id']),args.canary)
startup={'canary':c,'config_same_as_batch':True,'runner_sha256':run['script_sha256'],'worker_sha256':run['worker_sha256'],'executed_command':canary['executed_command'],'counts_relation':'Separate HTTP 200 canary failed frozen reviews checks; excluded. 31 explicit root crawler Requests total, plus ordinary redirects. No browser, local smoke, other-tool run or per-target mode change.'}
(HERE/'startup-checks.json').write_text(json.dumps(startup,indent=2)+'\n')
run['summary'].update(http_200_soft_blocks=sum(a['status']==200 and a['classification']=='soft_block' for a in run['attempts']),http_200_empty_content=sum(a['status']==200 and a['empty_content'] for a in run['attempts']),downloader_requests_including_redirects=sum(a['downloader_request_count'] for a in run['attempts']))
run['publication_validation']={'checked_at':datetime.now(timezone.utc).isoformat(),'unchanged_validator_ast_verified':True,'body_hashes_and_checks_verified':30,'pss_peaks_verified':30,'transport_metadata_and_log_hashes_verified':30,'driver_log_hashes_verified':30,'preserved_file_count':len(run['preserved_evidence_sha256']),'additional_target_requests':0,'surviving_owned_processes':0}
run['canary_relation']=startup['counts_relation']
run['latency_basis']='preNavigationHook clock before the initial private JSON checkpoint through requestHandler entry, including that checkpoint, redirects, body transfer and Cheerio parsing. On failure, ends after crawler.run returns and cleanup/logging completes. Excludes Node imports/crawler/storage startup, final body/log/record writes and Python frozen checks.'
(HERE/'pss-samples.json').write_text(json.dumps(samples,indent=2)+'\n');run['pss_samples_sha256']=sha((HERE/'pss-samples.json').read_bytes())
(HERE/'results.json').write_text(json.dumps(run,indent=2)+'\n');print('Recomputed every private body/check/hash/PSS and transport/log record:',run['summary'])