{ "run_id": "scrapling-public-30-20260929", "tool": "scrapling", "tool_version": "0.4.15", "curl_cffi_version": "0.16.3", "mode": "Fetcher.get HTTP only", "python_version": "3.12.3", "platform": "Linux-6.17.0-22-generic-x86_64-with-glibc2.39", "network": "Same local host direct outbound; http_proxy/https_proxy/all_proxy removed case-insensitively; no configured proxies. Different UTC window from prior tools.", "manifest_id": "public-30-v1", "manifest_sha256": "6ce519c4f7df3c32f5dfda379a8d3b280b7723bdeb48ddf64a1ae10ce6a7914c", "script_sha256": "c68cf304a92e4519eb7174ba9dc86dd1597b7bce5b75422a961833f95cc81215", "config": { "mode": "Fetcher.get HTTP only", "proxy_mode": "none", "login": false, "captcha_solving": false, "concurrency": 1, "gap_seconds": 2, "application_retries": 0, "fresh_process_and_session_per_target": true, "impersonate": "chrome150", "stealthy_headers": true, "follow_redirects": true, "max_redirects": 3, "timeout": 20, "attempt_timeout_seconds": 30, "max_capture_bytes": 5242880, "network_response_byte_cap": null, "cookies": "Fresh temporary curl session; redirect cookies possible; no imported cookies or cross-target persistence", "resources": "HTTP response only; no browser, JavaScript, images, stylesheets, iframes or workers", "verify": true, "http3": false, "selector_config": { "adaptive": false }, "pss_sample_interval_seconds": 0.1, "min_host_available_mb": 3072, "max_attempt_tree_pss_mb": 1024, "pss_scope": "Isolated Python attempt and descendants, including HTTP client and selector parse; excludes batch parent" }, "executed_command": "/tmp/scrapingevals-scrapling-public-30-20260929/bin/python public/evidence/scrapling-public-30-20260929/run.py --manifest runner/corpora/public-30-v1.json --output runner/data/scrapling-public-30-20260929", "started_at": "2026-09-29T20:06:38.296462+00:00", "host_ram_preflight": { "MemTotal": 32049.69921875, "MemAvailable": 13458.60546875, "SwapTotal": 8191.99609375, "SwapFree": 10.91015625, "unit": "MiB", "measured_at": "2026-09-29T20:06:38.293591+00:00" }, "attempts": [ { "target_id": "kohls", "site": "Kohl\u2019s", "category": "commerce", "url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "original_url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "final_url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "expected_required_content": "Men\u2019s T-shirt product listing with prices", "started_at": "2026-09-29T20:06:38.487591+00:00", "request_started_at": "2026-09-29T20:06:38.487612+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "6d0779a028ac0c2e3da6442ee6ad21dd4502e378c3e053b5e632cf2c1470a618", "response_bytes": 2501, "downloaded_body_bytes": 2501, "body_complete": true, "attempt_seconds": 0.224, "latency_seconds": 0.223, "finished_at": "2026-09-29T20:06:38.711394+00:00", "checks": { "text_chars": 32, "required_text_checks": [ { "alternatives": [ "t-shirt", "tees" ], "matched": [] }, { "alternatives": [ "men" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.455, "pss_samples": 5, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13458.60546875, "SwapTotal": 8191.99609375, "SwapFree": 10.91015625, "unit": "MiB", "measured_at": "2026-09-29T20:06:38.296542+00:00" }, "inspection_note": "HTTP 200 with challenge-provider/footer text, not clothing listings. The fixed soft-block phrase detector does not name this surface; strict required-content checks still fail." }, { "target_id": "amazon", "site": "Amazon", "category": "commerce", "url": "https://www.amazon.com/s?k=laptop", "original_url": "https://www.amazon.com/s?k=laptop", "final_url": "https://www.amazon.com/s?k=laptop", "expected_required_content": "Laptop search results with product prices", "started_at": "2026-09-29T20:06:40.990158+00:00", "request_started_at": "2026-09-29T20:06:40.990176+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html;charset=UTF-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "ebe8327a91039a6c172272664ae4bd118dee39d5b08e94edfb5f814c92980185", "response_bytes": 1326048, "downloaded_body_bytes": 1326048, "body_complete": true, "attempt_seconds": 1.148, "latency_seconds": 1.144, "finished_at": "2026-09-29T20:06:42.138217+00:00", "checks": { "text_chars": 25011, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [ "laptop" ] }, { "alternatives": [ "results" ], "matched": [ "results" ] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 47.646, "pss_samples": 15, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13316.66015625, "SwapTotal": 8191.99609375, "SwapFree": 10.9921875, "unit": "MiB", "measured_at": "2026-09-29T20:06:40.807930+00:00" }, "inspection_note": "HTTP 200 contains laptop/results text and a substantial listing body, unlike the earlier Camoufox/Patchright HTTP 503s. The required currency-price regex has zero matches, so the unchanged strict validator fails. No cause or complete product extraction is claimed." }, { "target_id": "walmart", "site": "Walmart", "category": "commerce", "url": "https://www.walmart.com/search?q=laptop", "original_url": "https://www.walmart.com/search?q=laptop", "final_url": "https://www.walmart.com/search?q=laptop", "expected_required_content": "Laptop search results with product prices", "started_at": "2026-09-29T20:06:44.600813+00:00", "request_started_at": "2026-09-29T20:06:44.600831+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "3ac5131de22bc0d339cf5a53dbd0f226a083c93400ff181b078ed2cab95e0c88", "response_bytes": 1495634, "downloaded_body_bytes": 1495634, "body_complete": true, "attempt_seconds": 3.644, "latency_seconds": 3.641, "finished_at": "2026-09-29T20:06:48.245252+00:00", "checks": { "text_chars": 7035, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [ "laptop" ] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 39 } ], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 46.771, "pss_samples": 39, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13300.87109375, "SwapTotal": 8191.99609375, "SwapFree": 11.01171875, "unit": "MiB", "measured_at": "2026-09-29T20:06:44.414919+00:00" }, "inspection_note": "HTTP 200 body contains laptop listings and prices meeting the fixed checks, unlike the earlier Camoufox/Patchright robot-or-human pages. The same network host but different time, fingerprint and cookie policies prevent attributing a cause." }, { "target_id": "ebay", "site": "eBay", "category": "commerce", "url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "original_url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "final_url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "expected_required_content": "Laptop marketplace listings with prices", "started_at": "2026-09-29T20:06:50.598767+00:00", "request_started_at": "2026-09-29T20:06:50.598785+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "07d548f16384dbda65516894fe09720ea9d0778b596d3ad91f57cb30e16c7fb4", "response_bytes": 1897798, "downloaded_body_bytes": 1897798, "body_complete": true, "attempt_seconds": 2.143, "latency_seconds": 2.139, "finished_at": "2026-09-29T20:06:52.741559+00:00", "checks": { "text_chars": 30278, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [ "laptop" ] }, { "alternatives": [ "results" ], "matched": [ "results" ] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 151 } ], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 55.873, "pss_samples": 25, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13431.00390625, "SwapTotal": 8191.99609375, "SwapFree": 11.01953125, "unit": "MiB", "measured_at": "2026-09-29T20:06:50.407811+00:00" }, "inspection_note": "HTTP 200 source body contains laptop search results and repeated prices. Passes the same fixed checks as Camoufox; earlier Patchright, Lightpanda and wreq returned HTTP 403. No browser interaction or product-detail coverage tested." }, { "target_id": "target", "site": "Target", "category": "commerce", "url": "https://www.target.com/s?searchTerm=laptop", "original_url": "https://www.target.com/s?searchTerm=laptop", "final_url": "https://www.target.com/s?searchTerm=laptop", "expected_required_content": "Laptop search listing with prices", "started_at": "2026-09-29T20:06:55.246371+00:00", "request_started_at": "2026-09-29T20:06:55.246392+00:00", "status": 429, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html, charset=UTF-8", "content_encoding": null, "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "ca695ba08b03ffa8c3f37a91382432933a2f1b3889ee87d0e88a3ed085b01a4a", "response_bytes": 2327, "downloaded_body_bytes": 2327, "body_complete": true, "attempt_seconds": 0.16, "latency_seconds": 0.159, "finished_at": "2026-09-29T20:06:55.405932+00:00", "checks": { "text_chars": 83, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_rate_limited", "usable_content": false, "peak_sampled_tree_pss_mb": 36.287, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13283.46484375, "SwapTotal": 8191.99609375, "SwapFree": 11.1015625, "unit": "MiB", "measured_at": "2026-09-29T20:06:55.054933+00:00" }, "inspection_note": "HTTP 429 busy/wait page rather than a laptop listing; recorded as rate limited, with no retry." }, { "target_id": "etsy", "site": "Etsy", "category": "commerce", "url": "https://www.etsy.com/search?q=ceramic+mug", "original_url": "https://www.etsy.com/search?q=ceramic+mug", "final_url": "https://www.etsy.com/search?q=ceramic+mug", "expected_required_content": "Ceramic mug listings with prices", "started_at": "2026-09-29T20:06:57.660685+00:00", "request_started_at": "2026-09-29T20:06:57.660703+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": null, "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "8dfc84737cadbc441fa6f25f2da4adb68b7893d648aa3224c5290402c15c6130", "response_bytes": 821, "downloaded_body_bytes": 821, "body_complete": true, "attempt_seconds": 0.226, "latency_seconds": 0.226, "finished_at": "2026-09-29T20:06:57.886941+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "ceramic" ], "matched": [] }, { "alternatives": [ "mug" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "peak_sampled_tree_pss_mb": 36.529, "pss_samples": 5, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13271.60546875, "SwapTotal": 8191.99609375, "SwapFree": 11.1015625, "unit": "MiB", "measured_at": "2026-09-29T20:06:57.464876+00:00" }, "inspection_note": "HTTP 403 JS/ad-blocker prompt, with no ceramic-mug listing. No challenge or browser fallback attempted." }, { "target_id": "bestbuy", "site": "Best Buy", "category": "commerce", "url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "original_url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "final_url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "expected_required_content": "Laptop product listing with prices", "started_at": "2026-09-29T20:07:00.167875+00:00", "request_started_at": "2026-09-29T20:07:00.167894+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "a8cbf80219f398fb87ffa6502865acd958845e5d87a3479c6b061ed76fe59a71", "response_bytes": 7056, "downloaded_body_bytes": 7056, "body_complete": true, "attempt_seconds": 0.128, "latency_seconds": 0.128, "finished_at": "2026-09-29T20:07:00.295882+00:00", "checks": { "text_chars": 687, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.586, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13298.359375, "SwapTotal": 8191.99609375, "SwapFree": 11.1015625, "unit": "MiB", "measured_at": "2026-09-29T20:06:59.975889+00:00" }, "inspection_note": "HTTP 200 country-choice page, not laptop results. No country chosen; required text and price patterns fail." }, { "target_id": "linkedin", "site": "LinkedIn", "category": "organization", "url": "https://www.linkedin.com/company/nasa/", "original_url": "https://www.linkedin.com/company/nasa/", "final_url": "https://www.linkedin.com/company/nasa/", "expected_required_content": "Public NASA organization description and industry/topic", "started_at": "2026-09-29T20:07:02.580472+00:00", "request_started_at": "2026-09-29T20:07:02.580495+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "8ff1309594ab130baac70e39033ee7b4059927d126a5a65c785522076d9ebeeb", "response_bytes": 356271, "downloaded_body_bytes": 356271, "body_complete": true, "attempt_seconds": 1.187, "latency_seconds": 1.185, "finished_at": "2026-09-29T20:07:03.767740+00:00", "checks": { "text_chars": 12645, "required_text_checks": [ { "alternatives": [ "nasa", "national aeronautics" ], "matched": [ "nasa", "national aeronautics" ] }, { "alternatives": [ "space", "research", "aviation" ], "matched": [ "space", "research", "aviation" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 40.062, "pss_samples": 15, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13408.7421875, "SwapTotal": 8191.99609375, "SwapFree": 11.1015625, "unit": "MiB", "measured_at": "2026-09-29T20:07:02.385434+00:00" }, "inspection_note": "Public NASA organization identity and overview meet the body checks alongside cookie/account prompts. No login or consent action. Source checks do not prove unobstructed rendering, tab coverage or interaction." }, { "target_id": "instagram", "site": "Instagram", "category": "organization", "url": "https://www.instagram.com/nasa/", "original_url": "https://www.instagram.com/nasa/", "final_url": "https://www.instagram.com/nasa/", "expected_required_content": "NASA organization profile with follower and post information", "started_at": "2026-09-29T20:07:06.144805+00:00", "request_started_at": "2026-09-29T20:07:06.144824+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=\"utf-8\"", "content_encoding": "zstd", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "108857091bdc1847f9d5f25d16685dc0e74b4110a4a02572b1c952160335ad7b", "response_bytes": 755250, "downloaded_body_bytes": 755250, "body_complete": true, "attempt_seconds": 0.776, "latency_seconds": 0.774, "finished_at": "2026-09-29T20:07:06.920916+00:00", "checks": { "text_chars": 0, "required_text_checks": [ { "alternatives": [ "nasa" ], "matched": [] }, { "alternatives": [ "followers" ], "matched": [] }, { "alternatives": [ "posts" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 150, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": true, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 39.903, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13432.921875, "SwapTotal": 8191.99609375, "SwapFree": 11.109375, "unit": "MiB", "measured_at": "2026-09-29T20:07:05.942015+00:00" }, "inspection_note": "HTTP 200 returns a large HTML body with zero checked body text. Script/head content is excluded by the unchanged validator. No browser rendering or login attempted." }, { "target_id": "tiktok", "site": "TikTok", "category": "organization", "url": "https://www.tiktok.com/@nasa", "original_url": "https://www.tiktok.com/@nasa", "final_url": "https://www.tiktok.com/@nasa", "expected_required_content": "NASA organization profile with video and audience information", "started_at": "2026-09-29T20:07:09.257166+00:00", "request_started_at": "2026-09-29T20:07:09.257186+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "9ec3a94f086de2c381dac5bfe9776a35ba32585ec4befb9f5138782ff7d4f209", "response_bytes": 389904, "downloaded_body_bytes": 389904, "body_complete": true, "attempt_seconds": 0.802, "latency_seconds": 0.8, "finished_at": "2026-09-29T20:07:10.058821+00:00", "checks": { "text_chars": 66, "required_text_checks": [ { "alternatives": [ "nasa" ], "matched": [ "nasa" ] }, { "alternatives": [ "followers" ], "matched": [] }, { "alternatives": [ "videos", "likes" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 150, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 40.072, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13437.53125, "SwapTotal": 8191.99609375, "SwapFree": 11.1171875, "unit": "MiB", "measured_at": "2026-09-29T20:07:09.065028+00:00" }, "inspection_note": "Source contains NASA identity and numbers but no required follower/like labels, and checked text is below the frozen minimum. No JavaScript/profile hydration or video playback tested." }, { "target_id": "g2", "site": "G2", "category": "reviews", "url": "https://www.g2.com/products/asana/reviews", "original_url": "https://www.g2.com/products/asana/reviews", "final_url": "https://www.g2.com/products/asana/reviews", "expected_required_content": "Asana review listing with review and rating information", "started_at": "2026-09-29T20:07:12.412973+00:00", "request_started_at": "2026-09-29T20:07:12.412999+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "cbff503002f0842c0726df078553738235871322f1c41e13f4b1d4cfeb1e0eb2", "response_bytes": 1707, "downloaded_body_bytes": 1707, "body_complete": true, "attempt_seconds": 0.16, "latency_seconds": 0.159, "finished_at": "2026-09-29T20:07:12.573077+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "asana" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] }, { "alternatives": [ "rating", "rated", "stars" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "peak_sampled_tree_pss_mb": 37.002, "pss_samples": 5, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13446.7109375, "SwapTotal": 8191.99609375, "SwapFree": 11.17578125, "unit": "MiB", "measured_at": "2026-09-29T20:07:12.186940+00:00" }, "inspection_note": "HTTP 403 JS/ad-blocker prompt, with no Asana reviews or ratings." }, { "target_id": "trustpilot", "site": "Trustpilot", "category": "reviews", "url": "https://www.trustpilot.com/review/www.amazon.com", "original_url": "https://www.trustpilot.com/review/www.amazon.com", "final_url": "https://www.trustpilot.com/review/www.amazon.com", "expected_required_content": "Amazon company review listing with ratings", "started_at": "2026-09-29T20:07:14.894226+00:00", "request_started_at": "2026-09-29T20:07:14.894244+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html", "content_encoding": null, "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "92d97ed45d6cc267c657b864c4deee1c2d896cda361adc660bc16538a75700f2", "response_bytes": 991, "downloaded_body_bytes": 991, "body_complete": true, "attempt_seconds": 0.085, "latency_seconds": 0.085, "finished_at": "2026-09-29T20:07:14.979445+00:00", "checks": { "text_chars": 164, "required_text_checks": [ { "alternatives": [ "amazon" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] }, { "alternatives": [ "trustscore", "rating", "rated" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "peak_sampled_tree_pss_mb": 36.807, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13497.86328125, "SwapTotal": 8191.99609375, "SwapFree": 11.17578125, "unit": "MiB", "measured_at": "2026-09-29T20:07:14.700964+00:00" }, "inspection_note": "HTTP 403 verification content without Amazon reviews or rating; no verification action or retry. Patchright/Lightpanda earlier passed, but this does not identify a cause." }, { "target_id": "tripadvisor", "site": "Tripadvisor", "category": "reviews", "url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "original_url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "final_url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "expected_required_content": "Paris restaurant listing with reviews", "started_at": "2026-09-29T20:07:17.294466+00:00", "request_started_at": "2026-09-29T20:07:17.294484+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "9b415370f510f939d9a1dc6baf25be7a7e4bbc6e7f3677f6873792ef572b7d9a", "response_bytes": 775, "downloaded_body_bytes": 775, "body_complete": true, "attempt_seconds": 0.236, "latency_seconds": 0.236, "finished_at": "2026-09-29T20:07:17.530757+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "paris" ], "matched": [] }, { "alternatives": [ "restaurants" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "peak_sampled_tree_pss_mb": 36.55, "pss_samples": 5, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13514.5546875, "SwapTotal": 8191.99609375, "SwapFree": 11.17578125, "unit": "MiB", "measured_at": "2026-09-29T20:07:17.111927+00:00" }, "inspection_note": "HTTP 403 JS/ad-blocker prompt, with no Paris restaurant reviews." }, { "target_id": "indeed", "site": "Indeed", "category": "jobs", "url": "https://www.indeed.com/jobs?q=python&l=Remote", "original_url": "https://www.indeed.com/jobs?q=python&l=Remote", "final_url": "https://www.indeed.com/jobs?q=python&l=Remote", "expected_required_content": "Remote Python job listings", "started_at": "2026-09-29T20:07:19.825364+00:00", "request_started_at": "2026-09-29T20:07:19.825381+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "f241301c82e397eae2dd390ae10665db7f2b63e0bfba6428fd7f36361246693d", "response_bytes": 1637081, "downloaded_body_bytes": 1637081, "body_complete": true, "attempt_seconds": 1.303, "latency_seconds": 1.3, "finished_at": "2026-09-29T20:07:21.128689+00:00", "checks": { "text_chars": 17678, "required_text_checks": [ { "alternatives": [ "python" ], "matched": [ "python" ] }, { "alternatives": [ "remote" ], "matched": [ "remote" ] }, { "alternatives": [ "jobs" ], "matched": [ "jobs" ] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 87 } ], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 49.743, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13507.67578125, "SwapTotal": 8191.99609375, "SwapFree": 11.17578125, "unit": "MiB", "measured_at": "2026-09-29T20:07:19.623661+00:00" }, "inspection_note": "HTTP 200 source contains Python remote job listings and employer/details text meeting the fixed checks, as in Camoufox. Earlier Patchright returned HTTP 403; no causal claim or application action." }, { "target_id": "remoteok", "site": "Remote OK", "category": "jobs", "url": "https://remoteok.com/remote-dev-jobs", "original_url": "https://remoteok.com/remote-dev-jobs", "final_url": "https://remoteok.com/remote-dev-jobs", "expected_required_content": "Remote developer job listings", "started_at": "2026-09-29T20:07:23.489960+00:00", "request_started_at": "2026-09-29T20:07:23.489978+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "zstd", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "8b2571581cb734e2a527317934dce23b0f0b3225efe1d61c998cf40cb86b646b", "response_bytes": 59406, "downloaded_body_bytes": 59406, "body_complete": true, "attempt_seconds": 0.38, "latency_seconds": 0.38, "finished_at": "2026-09-29T20:07:23.870230+00:00", "checks": { "text_chars": 5486, "required_text_checks": [ { "alternatives": [ "remote" ], "matched": [ "remote" ] }, { "alternatives": [ "developer", "engineer" ], "matched": [ "developer" ] }, { "alternatives": [ "jobs" ], "matched": [ "jobs" ] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 2 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 37.479, "pss_samples": 7, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13435.34375, "SwapTotal": 8191.99609375, "SwapFree": 11.1796875, "unit": "MiB", "measured_at": "2026-09-29T20:07:23.279928+00:00" }, "inspection_note": "HTTP 200 shell meets required text groups, but the strict developer/engineer/programmer regex has two matches where three are required. Earlier browser DOMs passed. No threshold change or JS fallback." }, { "target_id": "weworkremotely", "site": "We Work Remotely", "category": "jobs", "url": "https://weworkremotely.com/categories/remote-programming-jobs", "original_url": "https://weworkremotely.com/categories/remote-programming-jobs", "final_url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "expected_required_content": "Remote programming jobs with actual role descriptions", "started_at": "2026-09-29T20:07:26.185478+00:00", "request_started_at": "2026-09-29T20:07:26.185497+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [ { "previous": "https://weworkremotely.com/categories/remote-programming-jobs", "url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "status": 302 } ], "redirect_outcome": "followed", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "96d3caa100a3f6e180aaedfcd093a6cc2f889bfa08e46b34fce106a0c62c355d", "response_bytes": 144404, "downloaded_body_bytes": 144404, "body_complete": true, "attempt_seconds": 1.173, "latency_seconds": 1.173, "finished_at": "2026-09-29T20:07:27.358838+00:00", "checks": { "text_chars": 9661, "required_text_checks": [ { "alternatives": [ "remote" ], "matched": [ "remote" ] }, { "alternatives": [ "programming" ], "matched": [ "programming" ] }, { "alternatives": [ "developer", "engineer" ], "matched": [ "developer", "engineer" ] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 26 } ], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 37.899, "pss_samples": 15, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13447.4765625, "SwapTotal": 8191.99609375, "SwapFree": 11.18359375, "unit": "MiB", "measured_at": "2026-09-29T20:07:25.997877+00:00" }, "inspection_note": "Ordinary HTTP 302 to the canonical full-stack programming category followed successfully. Job-listing content passes; the canonical redirect is not counted as a failure." }, { "target_id": "wikipedia", "site": "Wikipedia", "category": "content", "url": "https://en.wikipedia.org/wiki/Web_scraping", "original_url": "https://en.wikipedia.org/wiki/Web_scraping", "final_url": "https://en.wikipedia.org/wiki/Web_scraping", "expected_required_content": "Web scraping article with substantive explanation", "started_at": "2026-09-29T20:07:29.721218+00:00", "request_started_at": "2026-09-29T20:07:29.721236+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "e305b5ea4c4ca2a210ae297f17ad11f2ce4201fd8f5b6247adcfb98f41d017af", "response_bytes": 235975, "downloaded_body_bytes": 235975, "body_complete": true, "attempt_seconds": 0.266, "latency_seconds": 0.265, "finished_at": "2026-09-29T20:07:29.987326+00:00", "checks": { "text_chars": 30790, "required_text_checks": [ { "alternatives": [ "web scraping" ], "matched": [ "web scraping" ] }, { "alternatives": [ "web crawler", "data extraction" ], "matched": [ "web crawler", "data extraction" ] } ], "text_regex_checks": [], "min_text_chars": 3000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 39.122, "pss_samples": 6, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13399.734375, "SwapTotal": 8191.99609375, "SwapFree": 11.19140625, "unit": "MiB", "measured_at": "2026-09-29T20:07:29.539030+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "hackernews", "site": "Hacker News", "category": "content", "url": "https://news.ycombinator.com/", "original_url": "https://news.ycombinator.com/", "final_url": "https://news.ycombinator.com/", "expected_required_content": "News story listing with points and comment links", "started_at": "2026-09-29T20:07:32.368696+00:00", "request_started_at": "2026-09-29T20:07:32.368719+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "23f96f5bdb9bde7c5681b5f21affe2845d7b5839fa771de14ea6441dd9f04143", "response_bytes": 34805, "downloaded_body_bytes": 34805, "body_complete": true, "attempt_seconds": 0.635, "latency_seconds": 0.635, "finished_at": "2026-09-29T20:07:33.004050+00:00", "checks": { "text_chars": 3948, "required_text_checks": [ { "alternatives": [ "hacker news" ], "matched": [ "hacker news" ] }, { "alternatives": [ "points" ], "matched": [ "points" ] }, { "alternatives": [ "comments" ], "matched": [ "comments" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 36.481, "pss_samples": 9, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13399.6484375, "SwapTotal": 8191.99609375, "SwapFree": 11.1953125, "unit": "MiB", "measured_at": "2026-09-29T20:07:32.173069+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "bbc", "site": "BBC", "category": "content", "url": "https://www.bbc.com/news", "original_url": "https://www.bbc.com/news", "final_url": "https://www.bbc.com/news", "expected_required_content": "Public news headlines and topic navigation", "started_at": "2026-09-29T20:07:35.306851+00:00", "request_started_at": "2026-09-29T20:07:35.306869+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "ed694ff76253f26a8b0b563dfee9c58046daefe2b4e6f334aceddd8df7fd6306", "response_bytes": 381206, "downloaded_body_bytes": 381206, "body_complete": true, "attempt_seconds": 0.25, "latency_seconds": 0.248, "finished_at": "2026-09-29T20:07:35.556464+00:00", "checks": { "text_chars": 9642, "required_text_checks": [ { "alternatives": [ "bbc" ], "matched": [ "bbc" ] }, { "alternatives": [ "news" ], "matched": [ "news" ] }, { "alternatives": [ "world", "business" ], "matched": [ "world", "business" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 38.813, "pss_samples": 6, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13391.53515625, "SwapTotal": 8191.99609375, "SwapFree": 11.20703125, "unit": "MiB", "measured_at": "2026-09-29T20:07:35.098879+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "arxiv", "site": "arXiv", "category": "content", "url": "https://arxiv.org/list/cs/recent", "original_url": "https://arxiv.org/list/cs/recent", "final_url": "https://arxiv.org/list/cs/recent", "expected_required_content": "Recent Computer Science papers with subject metadata", "started_at": "2026-09-29T20:07:37.935649+00:00", "request_started_at": "2026-09-29T20:07:37.935672+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "b1de1feefecddc4ea113b631684531e8e98358cc55f85f080167f7d0fe78a275", "response_bytes": 97480, "downloaded_body_bytes": 97480, "body_complete": true, "attempt_seconds": 0.173, "latency_seconds": 0.172, "finished_at": "2026-09-29T20:07:38.108696+00:00", "checks": { "text_chars": 16898, "required_text_checks": [ { "alternatives": [ "computer science" ], "matched": [ "computer science" ] }, { "alternatives": [ "subjects:" ], "matched": [ "subjects:" ] } ], "text_regex_checks": [], "min_text_chars": 1500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 37.628, "pss_samples": 5, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13429.88671875, "SwapTotal": 8191.99609375, "SwapFree": 11.22265625, "unit": "MiB", "measured_at": "2026-09-29T20:07:37.734919+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "gutenberg", "site": "Project Gutenberg", "category": "content", "url": "https://www.gutenberg.org/ebooks/1342", "original_url": "https://www.gutenberg.org/ebooks/1342", "final_url": "https://www.gutenberg.org/ebooks/1342", "expected_required_content": "Pride and Prejudice book description with author", "started_at": "2026-09-29T20:07:40.454371+00:00", "request_started_at": "2026-09-29T20:07:40.454389+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "764ca589880f2890c381cb1146e7f663a3d2f3c9731bce9fe662e67e0ab31f36", "response_bytes": 24862, "downloaded_body_bytes": 24862, "body_complete": true, "attempt_seconds": 0.539, "latency_seconds": 0.538, "finished_at": "2026-09-29T20:07:40.993241+00:00", "checks": { "text_chars": 2588, "required_text_checks": [ { "alternatives": [ "pride and prejudice" ], "matched": [ "pride and prejudice" ] }, { "alternatives": [ "jane austen" ], "matched": [ "jane austen" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 36.29, "pss_samples": 8, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13474.6484375, "SwapTotal": 8191.99609375, "SwapFree": 11.2265625, "unit": "MiB", "measured_at": "2026-09-29T20:07:40.260024+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "govuk", "site": "GOV.UK", "category": "content", "url": "https://www.gov.uk/bank-holidays", "original_url": "https://www.gov.uk/bank-holidays", "final_url": "https://www.gov.uk/bank-holidays", "expected_required_content": "UK bank holiday dates for England and Wales", "started_at": "2026-09-29T20:07:43.276087+00:00", "request_started_at": "2026-09-29T20:07:43.276105+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "ecef2d2dee39942eac84b14b13157fcd88c81f99f97842b08c991365db6bd04d", "response_bytes": 175904, "downloaded_body_bytes": 175904, "body_complete": true, "attempt_seconds": 0.157, "latency_seconds": 0.156, "finished_at": "2026-09-29T20:07:43.432946+00:00", "checks": { "text_chars": 17613, "required_text_checks": [ { "alternatives": [ "uk bank holidays" ], "matched": [ "uk bank holidays" ] }, { "alternatives": [ "england and wales" ], "matched": [ "england and wales" ] }, { "alternatives": [ "christmas day" ], "matched": [ "christmas day" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 36.384, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13381.9296875, "SwapTotal": 8191.99609375, "SwapFree": 11.39453125, "unit": "MiB", "measured_at": "2026-09-29T20:07:43.086914+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "python", "site": "Python documentation", "category": "content", "url": "https://docs.python.org/3/library/urllib.request.html", "original_url": "https://docs.python.org/3/library/urllib.request.html", "final_url": "https://docs.python.org/3/library/urllib.request.html", "expected_required_content": "urllib.request documentation with API explanation", "started_at": "2026-09-29T20:07:45.719734+00:00", "request_started_at": "2026-09-29T20:07:45.719754+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html", "content_encoding": "gzip", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "7690190ba74d1d6c2e247423f3fe194a5101667392c9de03bbe53296bd7a45b7", "response_bytes": 216525, "downloaded_body_bytes": 216525, "body_complete": true, "attempt_seconds": 0.25, "latency_seconds": 0.249, "finished_at": "2026-09-29T20:07:45.969996+00:00", "checks": { "text_chars": 53833, "required_text_checks": [ { "alternatives": [ "urllib.request" ], "matched": [ "urllib.request" ] }, { "alternatives": [ "urlopen" ], "matched": [ "urlopen" ] } ], "text_regex_checks": [], "min_text_chars": 3000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 39.86, "pss_samples": 6, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13370.7890625, "SwapTotal": 8191.99609375, "SwapFree": 11.40234375, "unit": "MiB", "measured_at": "2026-09-29T20:07:45.523910+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "quotes", "site": "Quotes to Scrape", "category": "javascript", "url": "https://quotes.toscrape.com/js/", "original_url": "https://quotes.toscrape.com/js/", "final_url": "https://quotes.toscrape.com/js/", "expected_required_content": "Quote text and author after JavaScript rendering", "started_at": "2026-09-29T20:07:48.361832+00:00", "request_started_at": "2026-09-29T20:07:48.361849+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "a9067d4438044d45a1a895ed6939df4fa4d0487e17b82053dd694ad425466074", "response_bytes": 5808, "downloaded_body_bytes": 5808, "body_complete": true, "attempt_seconds": 0.493, "latency_seconds": 0.492, "finished_at": "2026-09-29T20:07:48.854468+00:00", "checks": { "text_chars": 74, "required_text_checks": [ { "alternatives": [ "the world as we have created it" ], "matched": [] }, { "alternatives": [ "albert einstein" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.998, "pss_samples": 8, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13383.3046875, "SwapTotal": 8191.99609375, "SwapFree": 11.40625, "unit": "MiB", "measured_at": "2026-09-29T20:07:48.171901+00:00" }, "inspection_note": "HTTP 200 source is a JavaScript-dependent shell without quote/author content. The fixed required-content checks fail; no browser fallback." }, { "target_id": "scrapingcourse", "site": "ScrapingCourse", "category": "javascript", "url": "https://www.scrapingcourse.com/javascript-rendering", "original_url": "https://www.scrapingcourse.com/javascript-rendering", "final_url": "https://www.scrapingcourse.com/javascript-rendering", "expected_required_content": "JavaScript product listing with named products and prices", "started_at": "2026-09-29T20:07:51.173520+00:00", "request_started_at": "2026-09-29T20:07:51.173547+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "zstd", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "b9daac7e24122000dbc457ed0e4ed24b43b89593db88eac038e98499db93292a", "response_bytes": 29640, "downloaded_body_bytes": 29640, "body_complete": true, "attempt_seconds": 0.868, "latency_seconds": 0.867, "finished_at": "2026-09-29T20:07:52.041082+00:00", "checks": { "text_chars": 72, "required_text_checks": [ { "alternatives": [ "chaz kangeroo hoodie", "teton pullover hoodie" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.512, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13381.95703125, "SwapTotal": 8191.99609375, "SwapFree": 11.46484375, "unit": "MiB", "measured_at": "2026-09-29T20:07:50.991321+00:00" }, "inspection_note": "HTTP 200 source contains the JavaScript challenge instruction but no product/price listing. The unchanged checks fail; no browser fallback." }, { "target_id": "webscrapingdev", "site": "Web Scraping Dev", "category": "javascript", "url": "https://web-scraping.dev/reviews", "original_url": "https://web-scraping.dev/reviews", "final_url": "https://web-scraping.dev/reviews", "expected_required_content": "Review text loaded by the JavaScript review interface", "started_at": "2026-09-29T20:07:54.307134+00:00", "request_started_at": "2026-09-29T20:07:54.307151+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": null, "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "77a5e13ddd2d8d9d65ba531340293d16870224866f7d52bbf7377f113b514665", "response_bytes": 14727, "downloaded_body_bytes": 14727, "body_complete": true, "attempt_seconds": 0.78, "latency_seconds": 0.78, "finished_at": "2026-09-29T20:07:55.087536+00:00", "checks": { "text_chars": 244, "required_text_checks": [ { "alternatives": [ "unique flavor and great energy boost", "excellent energy drink for gamers" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.957, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13335.921875, "SwapTotal": 8191.99609375, "SwapFree": 11.47265625, "unit": "MiB", "measured_at": "2026-09-29T20:07:54.113549+00:00" }, "inspection_note": "HTTP 200 source contains review navigation and load-more shell but no required review content. No JavaScript execution; browser captures earlier passed." }, { "target_id": "example", "site": "Example Domain", "category": "control", "url": "https://example.com/", "original_url": "https://example.com/", "final_url": "https://example.com/", "expected_required_content": "Example Domain heading and documentation explanation", "started_at": "2026-09-29T20:07:57.450980+00:00", "request_started_at": "2026-09-29T20:07:57.451009+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962", "response_bytes": 713, "downloaded_body_bytes": 713, "body_complete": true, "attempt_seconds": 0.119, "latency_seconds": 0.119, "finished_at": "2026-09-29T20:07:57.570062+00:00", "checks": { "text_chars": 167, "required_text_checks": [ { "alternatives": [ "example domain" ], "matched": [] }, { "alternatives": [ "documentation examples" ], "matched": [ "documentation examples" ] } ], "text_regex_checks": [], "min_text_chars": 100, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "peak_sampled_tree_pss_mb": 36.106, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13202.7109375, "SwapTotal": 8191.99609375, "SwapFree": 11.47265625, "unit": "MiB", "measured_at": "2026-09-29T20:07:57.235890+00:00" }, "inspection_note": "HTTP 200 explanatory text is present, but the required Example Domain heading is absent from checked body text. This is a strict content failure, not a claimed access block; no validator change." }, { "target_id": "httpbingo", "site": "HTTPBingo", "category": "control", "url": "https://httpbingo.org/html", "original_url": "https://httpbingo.org/html", "final_url": "https://httpbingo.org/html", "expected_required_content": "Moby-Dick HTML sample", "started_at": "2026-09-29T20:07:59.847921+00:00", "request_started_at": "2026-09-29T20:07:59.847940+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "zstd", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "e250a7975d53801ed8c35271780a15a1da36a8a5383327cb706dea252b340d48", "response_bytes": 3742, "downloaded_body_bytes": 3742, "body_complete": true, "attempt_seconds": 0.292, "latency_seconds": 0.291, "finished_at": "2026-09-29T20:08:00.139778+00:00", "checks": { "text_chars": 3594, "required_text_checks": [ { "alternatives": [ "herman melville" ], "matched": [ "herman melville" ] }, { "alternatives": [ "moby-dick" ], "matched": [ "moby-dick" ] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 36.926, "pss_samples": 6, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13209.42578125, "SwapTotal": 8191.99609375, "SwapFree": 11.60546875, "unit": "MiB", "measured_at": "2026-09-29T20:07:59.647950+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "scrapethissite", "site": "Scrape This Site", "category": "control", "url": "https://www.scrapethissite.com/pages/simple/", "original_url": "https://www.scrapethissite.com/pages/simple/", "final_url": "https://www.scrapethissite.com/pages/simple/", "expected_required_content": "Country records including Andorra and population", "started_at": "2026-09-29T20:08:02.458672+00:00", "request_started_at": "2026-09-29T20:08:02.458690+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "zstd", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "4c68c2a54000607a0fa51ea5a57a20bbe4324dee74c442559185a1da364bd87b", "response_bytes": 203361, "downloaded_body_bytes": 203361, "body_complete": true, "attempt_seconds": 0.959, "latency_seconds": 0.959, "finished_at": "2026-09-29T20:08:03.418040+00:00", "checks": { "text_chars": 18132, "required_text_checks": [ { "alternatives": [ "countries of the world" ], "matched": [ "countries of the world" ] }, { "alternatives": [ "andorra" ], "matched": [ "andorra" ] }, { "alternatives": [ "population" ], "matched": [ "population" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 39.24, "pss_samples": 13, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13274.7734375, "SwapTotal": 8191.99609375, "SwapFree": 11.609375, "unit": "MiB", "measured_at": "2026-09-29T20:08:02.260919+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." }, { "target_id": "iana", "site": "IANA", "category": "control", "url": "https://www.iana.org/domains/reserved", "original_url": "https://www.iana.org/domains/reserved", "final_url": "https://www.iana.org/domains/reserved", "expected_required_content": "Reserved-domain reference and example domains", "started_at": "2026-09-29T20:08:05.825455+00:00", "request_started_at": "2026-09-29T20:08:05.825482+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "explicit_request_headers": { "referer": "https://www.google.com/" }, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "br", "cookie_values": "private/not published" }, "error": null, "capture_error": null, "capture_complete": true, "raw_response_sha256": "720e5cce9016212d2d28777356d2844fb0ca9e19a32636eb232f8653360dc450", "response_bytes": 10499, "downloaded_body_bytes": 10499, "body_complete": true, "attempt_seconds": 0.081, "latency_seconds": 0.08, "finished_at": "2026-09-29T20:08:05.906170+00:00", "checks": { "text_chars": 2727, "required_text_checks": [ { "alternatives": [ "iana-managed reserved domains" ], "matched": [ "iana-managed reserved domains" ] }, { "alternatives": [ "example.com" ], "matched": [ "example.com" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "peak_sampled_tree_pss_mb": 37.031, "pss_samples": 4, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13245.58203125, "SwapTotal": 8191.99609375, "SwapFree": 12.1953125, "unit": "MiB", "measured_at": "2026-09-29T20:08:05.621967+00:00" }, "inspection_note": "Private source body inspected: target-specific groups and listing patterns are present, with no checked soft block. Source-text checks do not establish full extraction or interaction." } ], "mode_selection": "Documented basic Fetcher.get is a practical bounded HTTP-only slice comparable in workload to wreq. It does not evaluate Scrapling browser modes, crawling or adaptive extraction; no per-target switching.", "body_hash_basis": "Original client-decompressed HTTP bytes, not serialized browser DOM. Partial/absent captures explicit. Full bodies/logs private in ignored runner/data.", "pss_caveat": "100 ms samples may miss peaks or changing processes. Includes isolated Python import, Scrapling/curl HTTP request and selector parse; excludes batch parent. Different workload from browser PSS.", "policy_differences": [ "Fresh temporary Fetcher curl session per isolated process; redirect cookies possible, no cross-target state. wreq cookie storage is disabled; browser baselines have fresh in-session cookies.", "Normal redirects followed, cap 3 like corrected wreq/Camoufox; Lightpanda cap 10 and Patchright cap 20. Curl redirect history has statuses/URLs but not intermediate response bodies.", "Chrome150 TLS/HTTP impersonation and default stealthy_headers (Google Referer); wreq emulates Chrome134. Implicit curl-generated browser headers are not exposed in Scrapling request_headers; only explicit headers recorded.", "No JavaScript or browser resource loads. 20-second transfer deadline; latency also includes Scrapling selector construction. 30-second whole-attempt watchdog plus RAM/PSS guards includes interpreter/import work.", "5 MiB saved-body cap after fetch returns, not a network byte cap. Download/parse memory bounded by whole-attempt time and sampled 1 GiB process-tree guard; host available RAM must remain above 3 GiB.", "Proxy variables removed; Scrapling per-request API does not accept curl Session trust_env. TLS verify=True is separate from proxy environment behavior. No login, CAPTCHA solving, paid service, browser mode or application retry.", "One dated observation per target; time/network responses, TLS/header fingerprint, cookie and JS policies differ. No winner, reliability estimate or causal claim." ], "summary": { "attempted_distinct_sites": 30, "usable_results": 15, "classifications": { "missing_required_content": 10, "usable": 15, "http_rate_limited": 1, "http_access_denied": 4 } }, "finished_at": "2026-09-29T20:08:06.033481+00:00", "baseline_sha256": { "wreq": "269132e9697d084941827877de584d1db11f6f6cd8d8fcfb41f4ab69646d54b8", "lightpanda": "bd0deb3fd9ee7f2a7a67ae95e3458503c47fa7069edd584cc051655b9ae3affa", "patchright": "cc7a70f7b5cd87e8a2e3f66e015730e4f258f9295159d6049cae70705147de14", "camoufox": "736930b15c3e677a5a651a397bbdf1e9a9076b3bf1a1458e3a0bb8716d653eda" }, "installed_api_source_sha256": { "engines/static.py": "1af4694ca56715095b5b7b02949690b6ba4ec57b878040b413a63291762446c1", "engines/toolbelt/convertor.py": "776028110aa2c0f9b802a982b591885081fb0dd4fd03d1a3f96e5bb08c0dcb3e", "engines/toolbelt/custom.py": "dff32ca463700e8f79d974ee4a164366c912a5efe883174dc184520b7f70d8f1", "fetchers/requests.py": "44adec41c8f6c97cefde2462eefd850c4416fa845a8f53c8426215f65e194de5" }, "official_versioned_http_source_matches_installed": true, "installed_packages": { "patchright": "1.63.0", "w3lib": "2.4.1", "click": "8.5.0", "playwright": "1.63.0", "pycparser": "3.0", "scrapling": "0.4.15", "msgspec": "0.22.0", "cssselect": "1.5.0", "anyio": "4.15.1", "lxml": "6.1.3", "cffi": "2.1.1", "apify_fingerprint_datapoints": "0.15.0", "orjson": "3.12.0", "Protego": "0.7.0", "tld": "0.13.2", "certifi": "2026.7.22", "typing_extensions": "4.16.0", "idna": "3.20", "greenlet": "3.5.6", "curl_cffi": "0.16.3", "pyee": "13.0.1", "browserforge": "1.2.4" }, "api_sources": [ "https://github.com/D4Vinci/Scrapling", "https://scrapling.readthedocs.io/en/latest/fetching/static.html", "https://github.com/D4Vinci/Scrapling/blob/v0.4.15/scrapling/engines/static.py", "https://pypi.org/project/scrapling/0.4.15/" ], "verification": { "exact_ordered_distinct_targets": 30, "dated_five_tool_records": 150, "private_body_checks_hashes_and_pss_recomputed": 30, "no_application_retries": true }, "capture_limitations": [ "Client-decompressed final body and limited response headers retained. No complete wire/header trace or browser resource trace.", "Curl history exposes redirect status/URL but intermediate bodies are not retained.", "Implicit curl impersonation headers are not exposed by Scrapling request_headers; the returned dictionary contains explicit Google Referer only.", "No browser, screenshot, JavaScript, adaptive selector storage, login, consent action or fallback mode used." ], "requirements_sha256": "05bb5ccf14a97d57b0f4858052c51eec7093801b74dc848c5fedd88add4fdb51" }