{ "run_id": "scrapy-public-30-20260930", "tool": "scrapy", "tool_version": "2.19.0", "twisted_version": "26.4.0", "requirements_sha256": "14eb994a6e29e328c2dbedab34252e9c1c98d091be28b7682e2667cfa76c5022", "documentation": { "repository": "https://github.com/scrapy/scrapy", "installed_release": "2.19.0", "release_url": "https://pypi.org/project/Scrapy/2.19.0/", "wheel_url": "https://files.pythonhosted.org/packages/35/64/8507dcab59f2d4920152020dc348879fd8eed1dd17f152cf65d9fddd3ff2/scrapy-2.19.0-py3-none-any.whl", "wheel_sha256": "44c1ad4b008f1976e946c75b2adce0e448cbfe470c584b1b59469abf270e8013", "verified_installed_sources": [ { "path": "scrapy/crawler.py", "sha256": "b5d249df5e92f3329e9fef460955b8b1657ab6f837805d314c7beee42d0af212" }, { "path": "scrapy/spiders/__init__.py", "sha256": "3c7ac26b9ceec2f7a6e3a87a73ad297a11c5eaac8cde6c14768365260e8e4b9e" }, { "path": "scrapy/settings/default_settings.py", "sha256": "e7fd642b2e829b8f8a1cc31e7c87504f7c4de6810f24c61b0894a2ea4f3e764f" }, { "path": "scrapy/core/downloader/handlers/http11.py", "sha256": "b1b2bb82e71811828cf81c10886dac102625d92ee7ae71e7376d4883eda09750" }, { "path": "scrapy/core/downloader/contextfactory.py", "sha256": "915374add80adcdfe5f9d62e2d3b70319865bd2b87191fb9e7a42b9e3a9ae817" }, { "path": "scrapy/core/downloader/middleware.py", "sha256": "05d1f524d93fe4b0283a342804acd047831d7fa15be9e4ff9e52b43f883789f9" }, { "path": "scrapy/downloadermiddlewares/redirect.py", "sha256": "756c084cbe3b86176e2b413a7ed80158d70b4d1731b1852d576e6dcd9d614992" }, { "path": "scrapy/downloadermiddlewares/cookies.py", "sha256": "38d77087617dcf0791f713c0e4f79bc2cb1d4551acf0edad8cc606e5a0f1fce2" }, { "path": "scrapy/downloadermiddlewares/retry.py", "sha256": "27a6b2feeb5d76c468641a12d618c0bbc6e7e0fb7ad71a1c1561fe4715e3a447" }, { "path": "scrapy/downloadermiddlewares/robotstxt.py", "sha256": "e3b3bc32ee5fb5dae4f52be217ae86908039056067aa2cc969d2d389700bbfa0" }, { "path": "scrapy/downloadermiddlewares/httpproxy.py", "sha256": "3a0248e71f460a0acf7015b1b59d49124398b679f5f758698ba0729a6a9ef47c" }, { "path": "scrapy/downloadermiddlewares/httpcompression.py", "sha256": "055c47baf01af4f330f08bac2f82f968030e4b7ce86d40c39235b62b7f641fbd" }, { "path": "scrapy/downloadermiddlewares/useragent.py", "sha256": "e1a1cf87513424b41ef1fb02cb5be8045212de4d40c3539cdd30879b953f6eb3" }, { "path": "scrapy/spidermiddlewares/httperror.py", "sha256": "3e1c9b4156c5b47b3fdca94b088c3aebac93d52a7dd0eae62f23464f8ec1305b" }, { "path": "scrapy/spidermiddlewares/referer.py", "sha256": "f06737f7b87d1dc598bd7a6ebc3fa9b8197ec36754f1332946f4b39297e7a9b2" } ], "deepwiki": "https://deepwiki.com/search/for-a-minimal-python-crawlerpr_c4296abd-da90-4507-b9cb-89b63ca6b927", "source_conclusions": [ "CrawlerProcess with async Spider.start yields one Request; parse/errback yields no further pages.", "HTTPERROR_ALLOW_ALL setting retains non-2xx in parse without handle_httpstatus_all request meta, which would disable normal redirect handling.", "REDIRECT_MAX_TIMES=3 follows normal HTTP redirects; METAREFRESH_ENABLED=False disables HTML refresh navigation.", "ROBOTSTXT_OBEY=False and RETRY_ENABLED=False; HTTPPROXY_ENABLED=False and HttpProxyMiddleware disabled, proxy env removed.", "COOKIES_ENABLED=False disables cookie storage/sending and avoids cookie public-suffix fetches; no cross-target state.", "Stock HTTP11DownloadHandler uses Twisted HTTP/1.1, without Chrome TLS impersonation. DOWNLOAD_VERIFY_CERTIFICATES=True explicitly enables cert verification.", "Middleware response order is reverse priority: observer at 900 records HTTP redirects before RedirectMiddleware 600. Private bodies/logs only; final Spider body is checked after HTTP compression handling." ] }, "mode": "Single-page Spider / HTTP11 downloader", "python_version": "3.12.3", "platform": "Linux-6.17.0-22-generic-x86_64-with-glibc2.39", "network": "Same local host direct outbound; http_proxy/https_proxy/all_proxy removed case-insensitively; no configured proxies. Different UTC window from prior tools.", "manifest_id": "public-30-v1", "manifest_sha256": "6ce519c4f7df3c32f5dfda379a8d3b280b7723bdeb48ddf64a1ae10ce6a7914c", "script_sha256": "efb97b78cb6850fb006afd13011c83a4cedac57eecf0611b17dbcc4c578078fb", "config": { "mode": "Single-page Spider / HTTP11 downloader", "proxy_mode": "none", "login": false, "captcha_solving": false, "concurrency": 1, "gap_seconds": 2, "application_retries": 0, "fresh_process_and_session_per_target": true, "user_agent": "Scrapy/2.19.0 (+https://scrapy.org)", "robots": false, "cookies_enabled": false, "follow_redirects": true, "max_redirects": 3, "meta_refresh": false, "timeout": 20, "attempt_timeout_seconds": 35, "slice_timeout_seconds": 1200, "max_capture_bytes": 5242880, "download_maxsize_bytes": 5242880, "download_size_basis": "Scrapy handler bounds response download; compression middleware also bounds decompressed body; early wire body and final decompressed body saved privately", "resources": "One response chain only; no browser, JavaScript or linked resource loads", "verify": true, "http_version": "HTTP/1.1 stock Twisted handler; no browser TLS impersonation", "pss_sample_interval_seconds": 0.1, "min_host_available_mb": 3072, "max_attempt_tree_pss_mb": 1024, "pss_scope": "Isolated Python worker/descendants, including imports, crawler startup, scheduling, downloader, middleware/capture/checks and reactor shutdown; excludes batch parent" }, "executed_command": "/tmp/scrapingevals-scrapy-20260930/bin/python public/evidence/scrapy-public-30-20260929/run.py --manifest runner/corpora/public-30-v1.json --output runner/data/scrapy-public-30-20260930", "started_at": "2026-09-29T22:07:46.152304+00:00", "host_ram_preflight": { "MemTotal": 32049.69921875, "MemAvailable": 13718.77734375, "SwapTotal": 8191.99609375, "SwapFree": 39.60546875, "unit": "MiB", "measured_at": "2026-09-29T22:07:46.148316+00:00" }, "attempts": [ { "target_id": "kohls", "site": "Kohl\u2019s", "category": "commerce", "url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "original_url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "final_url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "expected_required_content": "Men\u2019s T-shirt product listing with prices", "started_at": "2026-09-29T22:07:46.424696+00:00", "request_started_at": "2026-09-29T22:07:47.025116+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 421, "wire_body_sha256": "7df32d50a80bf340df0c3f09d0ea11ae6a5020c31b9a78647dabf28d4d026be9" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "7df32d50a80bf340df0c3f09d0ea11ae6a5020c31b9a78647dabf28d4d026be9", "response_bytes": 421, "downloaded_body_bytes": 421, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.717, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.113, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 309, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 12489, "response_received_count": 1, "elapsed_time_seconds": 0.11608717706985772, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.kohls.com/catalog/mens-tshirts-clothing.jsp?CN=Gender:Mens+Product:T-Shirts+Department:Clothing", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "9333a2fcf89a5cf30a2a28c76812fc0f929e6d27e132bbb136cd9184077bb60d", "finished_at": "2026-09-29T22:07:47.141527+00:00", "checks": { "text_chars": 227, "required_text_checks": [ { "alternatives": [ "t-shirt", "tees" ], "matched": [] }, { "alternatives": [ "men" ], "matched": [ "men" ] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [ "access denied" ] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "66c5209e5d08704ed2b402f9feac2ccaa1e564b4873ca08e8cc842635409e5f9", "peak_sampled_tree_pss_mb": 62.033, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13716.78125, "SwapTotal": 8191.99609375, "SwapFree": 39.60546875, "unit": "MiB", "measured_at": "2026-09-29T22:07:46.159477+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "amazon", "site": "Amazon", "category": "commerce", "url": "https://www.amazon.com/s?k=laptop", "original_url": "https://www.amazon.com/s?k=laptop", "final_url": "https://www.amazon.com/s?k=laptop", "expected_required_content": "Laptop search results with product prices", "started_at": "2026-09-29T22:07:49.528292+00:00", "request_started_at": "2026-09-29T22:07:50.114598+00:00", "status": 503, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.amazon.com/s?k=laptop", "status": 503, "protocol": "HTTP/1.1", "wire_body_bytes": 2671, "wire_body_sha256": "470de24561f8fdb4c965acb40542fe9fad14e1843793a497a3bacd906d05544c" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "470de24561f8fdb4c965acb40542fe9fad14e1843793a497a3bacd906d05544c", "response_bytes": 2671, "downloaded_body_bytes": 2671, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.846, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.252, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 235, "downloader/response_count": 1, "downloader/response_status_count/503": 1, "downloader/response_bytes": 3234, "response_received_count": 1, "elapsed_time_seconds": 0.25546971103176475, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.amazon.com/s?k=laptop", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "72c5bdf4bb6b1509acb585dd95012c1006ed83d660c1d23f340135994ff2e812", "finished_at": "2026-09-29T22:07:50.374175+00:00", "checks": { "text_chars": 0, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] }, { "alternatives": [ "results" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": true, "classification": "http_server_error", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "fb4b0598f52c14a775addffbd66dcb1ae3cd9e379ed60e0f550f66dd3d8fb57d", "peak_sampled_tree_pss_mb": 61.903, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13728.5, "SwapTotal": 8191.99609375, "SwapFree": 39.6328125, "unit": "MiB", "measured_at": "2026-09-29T22:07:49.285018+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http server error." }, { "target_id": "walmart", "site": "Walmart", "category": "commerce", "url": "https://www.walmart.com/search?q=laptop", "original_url": "https://www.walmart.com/search?q=laptop", "final_url": "https://www.walmart.com/search?q=laptop", "expected_required_content": "Laptop search results with product prices", "started_at": "2026-09-29T22:07:52.769277+00:00", "request_started_at": "2026-09-29T22:07:53.357537+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.walmart.com/search?q=laptop", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 245, "wire_body_sha256": "dcac58b4d1b46fdd754348227932639b72769ea5ad2aff009db95ce46decf883" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "dcac58b4d1b46fdd754348227932639b72769ea5ad2aff009db95ce46decf883", "response_bytes": 245, "downloaded_body_bytes": 245, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.846, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.251, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 241, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 1289, "response_received_count": 1, "elapsed_time_seconds": 0.25299134897068143, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.walmart.com/search?q=laptop", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "aa4d363085a1209f6d8c514a5c275116647c64932f181ffe802e2308182aecad", "finished_at": "2026-09-29T22:07:53.615796+00:00", "checks": { "text_chars": 125, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "8aae9c8122b54d269b328b46b5f5e07881026283481e491fb05f10e884d7ecb0", "peak_sampled_tree_pss_mb": 61.362, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13683.37890625, "SwapTotal": 8191.99609375, "SwapFree": 39.734375, "unit": "MiB", "measured_at": "2026-09-29T22:07:52.510917+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content. Saved checked text says the requested URL was rejected. This observed rejection is not in the frozen soft-block phrases; classification remains missing_required_content." }, { "target_id": "ebay", "site": "eBay", "category": "commerce", "url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "original_url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "final_url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "expected_required_content": "Laptop marketplace listings with prices", "started_at": "2026-09-29T22:07:56.007459+00:00", "request_started_at": "2026-09-29T22:07:56.629897+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 1832, "wire_body_sha256": "8994587507195f2f9f5d019171ef97e7e75ab6227286ff307481adbb5170f128" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "8994587507195f2f9f5d019171ef97e7e75ab6227286ff307481adbb5170f128", "response_bytes": 1832, "downloaded_body_bytes": 1832, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.718, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.086, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 245, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 3492, "response_received_count": 1, "elapsed_time_seconds": 0.08846732694655657, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.ebay.com/sch/i.html?_nkw=laptop", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "edbb13bd4f23473e52afb44b935d4c5f7973249e8492f19885494b3a15003bdb", "finished_at": "2026-09-29T22:07:56.725917+00:00", "checks": { "text_chars": 189, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] }, { "alternatives": [ "results" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "fde0784265a9a870755783c7f06ce9e170b18a8a383360083ed36ad5474e237a", "peak_sampled_tree_pss_mb": 61.251, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13679.578125, "SwapTotal": 8191.99609375, "SwapFree": 39.7578125, "unit": "MiB", "measured_at": "2026-09-29T22:07:55.734925+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "target", "site": "Target", "category": "commerce", "url": "https://www.target.com/s?searchTerm=laptop", "original_url": "https://www.target.com/s?searchTerm=laptop", "final_url": "https://www.target.com/s?searchTerm=laptop", "expected_required_content": "Laptop search listing with prices", "started_at": "2026-09-29T22:07:59.112592+00:00", "request_started_at": "2026-09-29T22:07:59.716441+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.target.com/s?searchTerm=laptop", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 34069, "wire_body_sha256": "e98e9faffa8361b4c1ac257355496bb1dab67878656d996b2229923a06ba1463" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "037d967f84b9e5535a88be103e7b2290993a06adf4d493552b8719e8f7d347ae", "response_bytes": 123422, "downloaded_body_bytes": 123422, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.345, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.739, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 244, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 37180, "httpcompression/response_bytes": 123422, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.7409346851054579, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.target.com/s?searchTerm=laptop", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "9e3d05bb6066d0f7cbfbb34d0abcec99aac1556c6cd7879dea4f4f4651684683", "finished_at": "2026-09-29T22:08:00.457651+00:00", "checks": { "text_chars": 264, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "937dfc14a180b15f6927164640809593d7ae140a48f6a2e151df645238633f67", "peak_sampled_tree_pss_mb": 61.938, "pss_samples": 17, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13623.5859375, "SwapTotal": 8191.99609375, "SwapFree": 39.76171875, "unit": "MiB", "measured_at": "2026-09-29T22:07:58.858011+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content. Saved source text contains header/footer navigation without the required laptop listings; no rendered page was inspected." }, { "target_id": "etsy", "site": "Etsy", "category": "commerce", "url": "https://www.etsy.com/search?q=ceramic+mug", "original_url": "https://www.etsy.com/search?q=ceramic+mug", "final_url": "https://www.etsy.com/search?q=ceramic+mug", "expected_required_content": "Ceramic mug listings with prices", "started_at": "2026-09-29T22:08:02.873759+00:00", "request_started_at": "2026-09-29T22:08:03.473928+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.etsy.com/search?q=ceramic+mug", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 793, "wire_body_sha256": "af68d56cf7b5e1dc7be2a62d1d81e028acc2a0e7486a8f05fa6fb563822b80ca" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "af68d56cf7b5e1dc7be2a62d1d81e028acc2a0e7486a8f05fa6fb563822b80ca", "response_bytes": 793, "downloaded_body_bytes": 793, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.777, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.147, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 243, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 2003, "response_received_count": 1, "elapsed_time_seconds": 0.14889281708747149, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.etsy.com/search?q=ceramic+mug", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "fd5eb6c47bdccbb54e367136de62154af301f4abf35b3b7b2c4f332f07e7051e", "finished_at": "2026-09-29T22:08:03.650552+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "ceramic" ], "matched": [] }, { "alternatives": [ "mug" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "5a25f9de6380448cfdb559ac95608e758c1e029986e984f5b8dff5d5d837a864", "peak_sampled_tree_pss_mb": 61.273, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13584.26953125, "SwapTotal": 8191.99609375, "SwapFree": 39.99609375, "unit": "MiB", "measured_at": "2026-09-29T22:08:02.593923+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "bestbuy", "site": "Best Buy", "category": "commerce", "url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "original_url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "final_url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "expected_required_content": "Laptop product listing with prices", "started_at": "2026-09-29T22:08:06.082323+00:00", "request_started_at": "2026-09-29T22:08:06.705768+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 1478, "wire_body_sha256": "61f35435335f6fdc0b371af8b50c24d4eeadfde951839491816dc298afc1df65" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "9355eccafcabd59ba19efe7d8c0937d442baaabfa976edd5db254d0f98607e9e", "response_bytes": 6684, "downloaded_body_bytes": 6684, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 8.725, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 8.096, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 255, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 5202, "httpcompression/response_bytes": 6684, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 8.098405225900933, "finish_reason": "finished", "responses_per_minute": 7.5, "items_per_minute": 0.0 }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.bestbuy.com/site/searchpage.jsp?st=laptop", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "9d3ec8beca1c113b783d7888c75818b64cfd776be9cedef38ba4154853e82112", "finished_at": "2026-09-29T22:08:14.807203+00:00", "checks": { "text_chars": 687, "required_text_checks": [ { "alternatives": [ "laptop" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "5c92e70e83aad09cc39d6822204edc427bebf12eef39b1a1ad592d0c5fb75b96", "peak_sampled_tree_pss_mb": 61.51, "pss_samples": 90, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13579.47265625, "SwapTotal": 8191.99609375, "SwapFree": 39.99609375, "unit": "MiB", "measured_at": "2026-09-29T22:08:05.819947+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content. Saved checked text asks the visitor to choose a country; it does not meet the frozen listing rules." }, { "target_id": "linkedin", "site": "LinkedIn", "category": "organization", "url": "https://www.linkedin.com/company/nasa/", "original_url": "https://www.linkedin.com/company/nasa/", "final_url": "https://www.linkedin.com/company/nasa/", "expected_required_content": "Public NASA organization description and industry/topic", "started_at": "2026-09-29T22:08:17.246657+00:00", "request_started_at": "2026-09-29T22:08:17.860818+00:00", "status": 999, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.linkedin.com/company/nasa/", "status": 999, "protocol": "HTTP/1.1", "wire_body_bytes": 1530, "wire_body_sha256": "644031a68bde879af85bcc9cb3e6fa1e9a6b0f61d49307581974b5dbc09d3de8" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "644031a68bde879af85bcc9cb3e6fa1e9a6b0f61d49307581974b5dbc09d3de8", "response_bytes": 1530, "downloaded_body_bytes": 1530, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.832, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.211, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 240, "downloader/response_count": 1, "downloader/response_status_count/999": 1, "downloader/response_bytes": 2832, "response_received_count": 1, "elapsed_time_seconds": 0.21295110299251974, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.linkedin.com/company/nasa/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "61c5af2691d2e9198f1977f949e9dfb560b4cf22384cc5bdd8904e5071566378", "finished_at": "2026-09-29T22:08:18.078188+00:00", "checks": { "text_chars": 0, "required_text_checks": [ { "alternatives": [ "nasa", "national aeronautics" ], "matched": [] }, { "alternatives": [ "space", "research", "aviation" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": true, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "9c100fecd7a4f4c76eb9d908de7d9a626c79597f8f64d7939d4f08bddb95ff5e", "peak_sampled_tree_pss_mb": 61.223, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13633.0234375, "SwapTotal": 8191.99609375, "SwapFree": 40.04296875, "unit": "MiB", "measured_at": "2026-09-29T22:08:16.992918+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "instagram", "site": "Instagram", "category": "organization", "url": "https://www.instagram.com/nasa/", "original_url": "https://www.instagram.com/nasa/", "final_url": "https://www.instagram.com/nasa/", "expected_required_content": "NASA organization profile with follower and post information", "started_at": "2026-09-29T22:08:20.472058+00:00", "request_started_at": "2026-09-29T22:08:21.067195+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.instagram.com/nasa/", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 124669, "wire_body_sha256": "477b4403f10d05ecd87e4bf33baef7a2d4b1cc812741d6124050e20e6e969492" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=\"utf-8\"", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "dc9b062727af44552c5be5c33a60592236c0711156169bc38f560b70d12a0004", "response_bytes": 737943, "downloaded_body_bytes": 737943, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.172, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.568, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 233, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 131458, "httpcompression/response_bytes": 737943, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.5698576830327511, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.instagram.com/nasa/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "400039ef82e90767a3b7e4c00a16e0b65e51a5dac608d53ee697815a3d121013", "finished_at": "2026-09-29T22:08:21.643702+00:00", "checks": { "text_chars": 0, "required_text_checks": [ { "alternatives": [ "nasa" ], "matched": [] }, { "alternatives": [ "followers" ], "matched": [] }, { "alternatives": [ "posts" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 150, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": true, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "8a82bd76faa8945bd41d414166dd511316684a8fdd816086027cdc47597b8df1", "peak_sampled_tree_pss_mb": 64.241, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13598.69921875, "SwapTotal": 8191.99609375, "SwapFree": 40.26953125, "unit": "MiB", "measured_at": "2026-09-29T22:08:20.222046+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "HTTP 200 had no checked source text; empty checked text is not usable content." }, { "target_id": "tiktok", "site": "TikTok", "category": "organization", "url": "https://www.tiktok.com/@nasa", "original_url": "https://www.tiktok.com/@nasa", "final_url": "https://www.tiktok.com/@nasa", "expected_required_content": "NASA organization profile with video and audience information", "started_at": "2026-09-29T22:08:24.122079+00:00", "request_started_at": "2026-09-29T22:08:24.709052+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.tiktok.com/@nasa", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 82650, "wire_body_sha256": "57a957dad484219fb9626a97a27e4c5dd1fa97e5b815d62af760c8624dc67785" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "174f80c0d1dc5f07e76e542e080ec4bb259829a595678d86b334bb88fb3c5cb8", "response_bytes": 387790, "downloaded_body_bytes": 387790, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.417, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.759, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 230, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 89284, "httpcompression/response_bytes": 387790, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.7613596469163895, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.tiktok.com/@nasa", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "601171e9ba54b196da3e5b5e7d135159411d2f81ff5f532744ed238d0aa10325", "finished_at": "2026-09-29T22:08:25.538860+00:00", "checks": { "text_chars": 66, "required_text_checks": [ { "alternatives": [ "nasa" ], "matched": [ "nasa" ] }, { "alternatives": [ "followers" ], "matched": [] }, { "alternatives": [ "videos", "likes" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 150, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "1a012893c3aeb7e5d5465134593b1f7cfcb57ecaa15741cf10082c7d64d58eda", "peak_sampled_tree_pss_mb": 62.163, "pss_samples": 18, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13615.08984375, "SwapTotal": 8191.99609375, "SwapFree": 40.26953125, "unit": "MiB", "measured_at": "2026-09-29T22:08:23.858929+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content." }, { "target_id": "g2", "site": "G2", "category": "reviews", "url": "https://www.g2.com/products/asana/reviews", "original_url": "https://www.g2.com/products/asana/reviews", "final_url": "https://www.g2.com/products/asana/reviews", "expected_required_content": "Asana review listing with review and rating information", "started_at": "2026-09-29T22:08:27.986184+00:00", "request_started_at": "2026-09-29T22:08:28.580219+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.g2.com/products/asana/reviews", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 914, "wire_body_sha256": "b058f381107d6346d8887b3e5e82d6a0e6b7712609ed8e5d3a9c2cfea8b27d6d" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "ae3d7a5de5d97d8b5523f0b4267f7fb10b726935b09963f4475ba20f5dfefab5", "response_bytes": 1704, "downloaded_body_bytes": 1704, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.745, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.143, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 243, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 2257, "httpcompression/response_bytes": 1704, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.14486526302061975, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.g2.com/products/asana/reviews", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "bfe77c47db27ce7db80e44b1ca56f325f2d58b6e5ea3c2f1db6f4f56b3d1b80a", "finished_at": "2026-09-29T22:08:28.730893+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "asana" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] }, { "alternatives": [ "rating", "rated", "stars" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "7dc9098e3dd729f080f70150263bf8a09ab83df872adc1cac5ede102e9dec43a", "peak_sampled_tree_pss_mb": 61.534, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13606.03125, "SwapTotal": 8191.99609375, "SwapFree": 40.3203125, "unit": "MiB", "measured_at": "2026-09-29T22:08:27.697931+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "trustpilot", "site": "Trustpilot", "category": "reviews", "url": "https://www.trustpilot.com/review/www.amazon.com", "original_url": "https://www.trustpilot.com/review/www.amazon.com", "final_url": "https://www.trustpilot.com/review/www.amazon.com", "expected_required_content": "Amazon company review listing with ratings", "started_at": "2026-09-29T22:08:31.186656+00:00", "request_started_at": "2026-09-29T22:08:31.776891+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.trustpilot.com/review/www.amazon.com", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 991, "wire_body_sha256": "92d97ed45d6cc267c657b864c4deee1c2d896cda361adc660bc16538a75700f2" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "92d97ed45d6cc267c657b864c4deee1c2d896cda361adc660bc16538a75700f2", "response_bytes": 991, "downloaded_body_bytes": 991, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.682, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.084, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 250, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 1318, "response_received_count": 1, "elapsed_time_seconds": 0.08714020601473749, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.trustpilot.com/review/www.amazon.com", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "855ca450e3a49c3fa540f9a151b4b20c4163b8004151665b61632120fcf85955", "finished_at": "2026-09-29T22:08:31.868956+00:00", "checks": { "text_chars": 164, "required_text_checks": [ { "alternatives": [ "amazon" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] }, { "alternatives": [ "trustscore", "rating", "rated" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "5a95f7b73a8c07f59b52c0bbdfff5403185b73548b446c711442eda0bdff00f3", "peak_sampled_tree_pss_mb": 61.297, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13587.0, "SwapTotal": 8191.99609375, "SwapFree": 40.3359375, "unit": "MiB", "measured_at": "2026-09-29T22:08:30.931921+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "tripadvisor", "site": "Tripadvisor", "category": "reviews", "url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "original_url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "final_url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "expected_required_content": "Paris restaurant listing with reviews", "started_at": "2026-09-29T22:08:34.308655+00:00", "request_started_at": "2026-09-29T22:08:34.906832+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 515, "wire_body_sha256": "d1e0390da57c1ef1c0cf7740b20d38609537e839f85e4a0bc7c3258ae8e92af5" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "93747d2cdce03a16c98e9ebd05e6f2a57312de3f67e3b9469a666e7d00e55eff", "response_bytes": 775, "downloaded_body_bytes": 775, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.813, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.208, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 274, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 1848, "httpcompression/response_bytes": 775, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.21046219789423048, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.tripadvisor.com/Restaurants-g187147-Paris_Ile_de_France.html", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "e389622b2fb768c3d88a451f2a4cce3b04383f908d117b0374e5e1bf9419e070", "finished_at": "2026-09-29T22:08:35.121314+00:00", "checks": { "text_chars": 43, "required_text_checks": [ { "alternatives": [ "paris" ], "matched": [] }, { "alternatives": [ "restaurants" ], "matched": [] }, { "alternatives": [ "reviews" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "ab830084aa188af88448ad92c56e0e28fff8afa29281e5db430198cbe8217cf3", "peak_sampled_tree_pss_mb": 61.332, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13615.6640625, "SwapTotal": 8191.99609375, "SwapFree": 40.3359375, "unit": "MiB", "measured_at": "2026-09-29T22:08:34.056933+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "indeed", "site": "Indeed", "category": "jobs", "url": "https://www.indeed.com/jobs?q=python&l=Remote", "original_url": "https://www.indeed.com/jobs?q=python&l=Remote", "final_url": "https://www.indeed.com/jobs?q=python&l=Remote", "expected_required_content": "Remote Python job listings", "started_at": "2026-09-29T22:08:37.538956+00:00", "request_started_at": "2026-09-29T22:08:38.134926+00:00", "status": 403, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.indeed.com/jobs?q=python&l=Remote", "status": 403, "protocol": "HTTP/1.1", "wire_body_bytes": 9704, "wire_body_sha256": "b9e3986fd992170d3f845d295516f8c931d612d6c2cf6255ba167cf3adcb93e9" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "ec78a4a26f57120b0ddeb86b3d997c39d79f1cfff8b654352c0d2ffa38c98420", "response_bytes": 25740, "downloaded_body_bytes": 25740, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.731, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.128, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 247, "downloader/response_count": 1, "downloader/response_status_count/403": 1, "downloader/response_bytes": 10555, "httpcompression/response_bytes": 25740, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.13095287093892694, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.indeed.com/jobs?q=python&l=Remote", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "a3eafd7d845716b924e1b75b42737bdf40a73714004273a12e04a42b6ee69eac", "finished_at": "2026-09-29T22:08:38.270251+00:00", "checks": { "text_chars": 20, "required_text_checks": [ { "alternatives": [ "python" ], "matched": [] }, { "alternatives": [ "remote" ], "matched": [] }, { "alternatives": [ "jobs" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "http_access_denied", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "929186f484aa8e32e2144dfb286d9e8861ce56312a512c42076c4c39a93222a8", "peak_sampled_tree_pss_mb": 62.134, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13698.0703125, "SwapTotal": 8191.99609375, "SwapFree": 40.3359375, "unit": "MiB", "measured_at": "2026-09-29T22:08:37.284383+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: http access denied." }, { "target_id": "remoteok", "site": "Remote OK", "category": "jobs", "url": "https://remoteok.com/remote-dev-jobs", "original_url": "https://remoteok.com/remote-dev-jobs", "final_url": "https://remoteok.com/", "expected_required_content": "Remote developer job listings", "started_at": "2026-09-29T22:08:40.661455+00:00", "request_started_at": "2026-09-29T22:08:41.229827+00:00", "status": 302, "request_outcome": "error", "redirect_history": [ { "previous": "https://remoteok.com/remote-dev-jobs", "url": "https://remoteok.com/", "status": 302 }, { "previous": "https://remoteok.com/", "url": "https://remoteok.com/", "status": 302 }, { "previous": "https://remoteok.com/", "url": "https://remoteok.com/", "status": 302 } ], "redirect_outcome": "limit_or_loop_error", "document_responses": [ { "url": "https://remoteok.com/remote-dev-jobs", "status": 302, "protocol": "HTTP/1.1", "wire_body_bytes": 0, "wire_body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "url": "https://remoteok.com/", "status": 302, "protocol": "HTTP/1.1", "wire_body_bytes": 0, "wire_body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "url": "https://remoteok.com/", "status": 302, "protocol": "HTTP/1.1", "wire_body_bytes": 0, "wire_body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "url": "https://remoteok.com/", "status": 302, "protocol": "HTTP/1.1", "wire_body_bytes": 0, "wire_body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Last early middleware wire response; terminal downloader error; never usable" }, "error": { "type": "IgnoreRequest", "message": "max redirections reached", "classification": "redirect_limit_or_loop" }, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": false, "raw_response_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "response_bytes": 0, "downloaded_body_bytes": 0, "body_complete": false, "navigation_seconds": null, "attempt_seconds": 1.186, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.61, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 4, "scheduler/enqueued": 4, "scheduler/dequeued/memory": 4, "scheduler/dequeued": 4, "downloader/request_count": 4, "downloader/request_method_count/GET": 4, "downloader/request_bytes": 1018, "downloader/response_count": 4, "downloader/response_status_count/302": 4, "downloader/response_bytes": 2240, "elapsed_time_seconds": 0.6123004108667374, "finish_reason": "finished" }, "downloader_request_count": 4, "observed_request_headers": [ { "url": "https://remoteok.com/remote-dev-jobs", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] }, { "url": "https://remoteok.com/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] }, { "name": "Referer", "values": [ "https://remoteok.com/remote-dev-jobs" ] } ] }, { "url": "https://remoteok.com/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] }, { "name": "Referer", "values": [ "https://remoteok.com/" ] } ] }, { "url": "https://remoteok.com/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] }, { "name": "Referer", "values": [ "https://remoteok.com/" ] } ] } ], "transport_log_sha256": "7c433223fb04a1ce57e50b47a002ea94b2e4c9ecffa2083c058c3d25c0592bd0", "finished_at": "2026-09-29T22:08:41.846995+00:00", "checks": { "text_chars": 0, "required_text_checks": [ { "alternatives": [ "remote" ], "matched": [] }, { "alternatives": [ "developer", "engineer" ], "matched": [] }, { "alternatives": [ "jobs" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 0 } ], "min_text_chars": 500, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": true, "classification": "redirect_limit_or_loop", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "e71a6f36d555889df7f79574b53864b4c17f1234ab6cae8f48b720b275616e23", "peak_sampled_tree_pss_mb": 61.301, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13688.54296875, "SwapTotal": 8191.99609375, "SwapFree": 40.34375, "unit": "MiB", "measured_at": "2026-09-29T22:08:40.410871+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: redirect limit or loop. Three redirects were followed; the fourth response was HTTP 302. Scrapy raised IgnoreRequest: max redirections reached. The saved final early body is empty and marked incomplete." }, { "target_id": "weworkremotely", "site": "We Work Remotely", "category": "jobs", "url": "https://weworkremotely.com/categories/remote-programming-jobs", "original_url": "https://weworkremotely.com/categories/remote-programming-jobs", "final_url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "expected_required_content": "Remote programming jobs with actual role descriptions", "started_at": "2026-09-29T22:08:44.303906+00:00", "request_started_at": "2026-09-29T22:08:44.919927+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [ { "previous": "https://weworkremotely.com/categories/remote-programming-jobs", "url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "status": 302 } ], "redirect_outcome": "followed", "document_responses": [ { "url": "https://weworkremotely.com/categories/remote-programming-jobs", "status": 302, "protocol": "HTTP/1.1", "wire_body_bytes": 0, "wire_body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 28669, "wire_body_sha256": "c33e0250a9fd9f786967b021753b56e75d235d7acdc20df0a1b43d52e2687244" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "72334233695ef3de0340301f435abed03e15750ba2ec72b798f2234ac2f6220e", "response_bytes": 144404, "downloaded_body_bytes": 144404, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.175, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.55, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 2, "scheduler/enqueued": 2, "scheduler/dequeued/memory": 2, "scheduler/dequeued": 2, "downloader/request_count": 2, "downloader/request_method_count/GET": 2, "downloader/request_bytes": 609, "downloader/response_count": 2, "downloader/response_status_count/302": 1, "downloader/response_bytes": 35640, "downloader/response_status_count/200": 1, "httpcompression/response_bytes": 144404, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.5521810140926391, "finish_reason": "finished" }, "downloader_request_count": 2, "observed_request_headers": [ { "url": "https://weworkremotely.com/categories/remote-programming-jobs", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] }, { "url": "https://weworkremotely.com/categories/remote-full-stack-programming-jobs", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] }, { "name": "Referer", "values": [ "https://weworkremotely.com/categories/remote-programming-jobs" ] } ] } ], "transport_log_sha256": "03984c6ae7640487f63d879499096b225694342c54fb5e30eaab272db5acc316", "finished_at": "2026-09-29T22:08:45.478535+00:00", "checks": { "text_chars": 9661, "required_text_checks": [ { "alternatives": [ "remote" ], "matched": [ "remote" ] }, { "alternatives": [ "programming" ], "matched": [ "programming" ] }, { "alternatives": [ "developer", "engineer" ], "matched": [ "developer", "engineer" ] } ], "text_regex_checks": [ { "pattern": "\\b(?:developer|engineer|programmer)\\b", "min_matches": 3, "matches": 26 } ], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "e158d874f3a890fb678f19eaab9a9ba0229ad80cfbec8408a9c41af98bb12ca4", "peak_sampled_tree_pss_mb": 62.14, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13645.12109375, "SwapTotal": 8191.99609375, "SwapFree": 40.36328125, "unit": "MiB", "measured_at": "2026-09-29T22:08:44.047881+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "wikipedia", "site": "Wikipedia", "category": "content", "url": "https://en.wikipedia.org/wiki/Web_scraping", "original_url": "https://en.wikipedia.org/wiki/Web_scraping", "final_url": "https://en.wikipedia.org/wiki/Web_scraping", "expected_required_content": "Web scraping article with substantive explanation", "started_at": "2026-09-29T22:08:47.959028+00:00", "request_started_at": "2026-09-29T22:08:48.578673+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://en.wikipedia.org/wiki/Web_scraping", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 46763, "wire_body_sha256": "801f6740dd38c20ee390d34166ad39d22c2cfd19fb1402d19f599de482d40540" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "e305b5ea4c4ca2a210ae297f17ad11f2ce4201fd8f5b6247adcfb98f41d017af", "response_bytes": 235975, "downloaded_body_bytes": 235975, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.988, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.328, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 244, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 53189, "httpcompression/response_bytes": 235975, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.3298750310204923, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://en.wikipedia.org/wiki/Web_scraping", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "572ef3f03b97462c6ddfa5570355f9b78852499f2111d0d5ce59b9046b8acb87", "finished_at": "2026-09-29T22:08:48.947381+00:00", "checks": { "text_chars": 30790, "required_text_checks": [ { "alternatives": [ "web scraping" ], "matched": [ "web scraping" ] }, { "alternatives": [ "web crawler", "data extraction" ], "matched": [ "web crawler", "data extraction" ] } ], "text_regex_checks": [], "min_text_chars": 3000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "d9b6450b7977782b493b65768c076b1c18df6874b0e41bf5fca175dd66710d87", "peak_sampled_tree_pss_mb": 61.451, "pss_samples": 14, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13660.3125, "SwapTotal": 8191.99609375, "SwapFree": 40.36328125, "unit": "MiB", "measured_at": "2026-09-29T22:08:47.697030+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "hackernews", "site": "Hacker News", "category": "content", "url": "https://news.ycombinator.com/", "original_url": "https://news.ycombinator.com/", "final_url": "https://news.ycombinator.com/", "expected_required_content": "News story listing with points and comment links", "started_at": "2026-09-29T22:08:51.412407+00:00", "request_started_at": "2026-09-29T22:08:51.988056+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://news.ycombinator.com/", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 5331, "wire_body_sha256": "7a83991673a65775fd116bc4f265a59a02e3dbf23a766c2100ffad49ec41666b" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "9ba29a8a4650c17178c7328a873d90b73ab124daba91c5422c3b8073aad51a16", "response_bytes": 34036, "downloaded_body_bytes": 34036, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.36, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.601, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 231, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 6020, "httpcompression/response_bytes": 34036, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.6028042370453477, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://news.ycombinator.com/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "523ffc22f1c81d31c59ffffef2264dc561c3ea658ba7f9ea51100fe4572930df", "finished_at": "2026-09-29T22:08:52.772026+00:00", "checks": { "text_chars": 3746, "required_text_checks": [ { "alternatives": [ "hacker news" ], "matched": [ "hacker news" ] }, { "alternatives": [ "points" ], "matched": [ "points" ] }, { "alternatives": [ "comments" ], "matched": [ "comments" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "f2b0e3a3fc6818b45645fffac8b2c000788574bf5bb57742090de2cf46af490e", "peak_sampled_tree_pss_mb": 61.396, "pss_samples": 17, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13586.421875, "SwapTotal": 8191.99609375, "SwapFree": 40.36328125, "unit": "MiB", "measured_at": "2026-09-29T22:08:51.151021+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "bbc", "site": "BBC", "category": "content", "url": "https://www.bbc.com/news", "original_url": "https://www.bbc.com/news", "final_url": "https://www.bbc.com/news", "expected_required_content": "Public news headlines and topic navigation", "started_at": "2026-09-29T22:08:55.245804+00:00", "request_started_at": "2026-09-29T22:08:55.840092+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.bbc.com/news", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 63367, "wire_body_sha256": "8561f770a2dc60f6058bc8c0a2e39f443fa1c90df2f2694fcff316dffe56055f" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "964c067ea02bfaaadaaff4d91868193cd47b3c7bb22ebc04ce662886dafeff63", "response_bytes": 376312, "downloaded_body_bytes": 376312, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.822, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.195, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 226, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 64996, "httpcompression/response_bytes": 376312, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.1975599869620055, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.bbc.com/news", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "d56f0cf9e5115e8315d802211f4f12835c184b2aed335726fda8bffe11b9c2d6", "finished_at": "2026-09-29T22:08:56.068259+00:00", "checks": { "text_chars": 9605, "required_text_checks": [ { "alternatives": [ "bbc" ], "matched": [ "bbc" ] }, { "alternatives": [ "news" ], "matched": [ "news" ] }, { "alternatives": [ "world", "business" ], "matched": [ "world", "business" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "e6012234104ad68582d258ccea3bce0be95de00885db10a6c81105228a6de8a8", "peak_sampled_tree_pss_mb": 61.296, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13678.6171875, "SwapTotal": 8191.99609375, "SwapFree": 40.3671875, "unit": "MiB", "measured_at": "2026-09-29T22:08:54.991032+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "arxiv", "site": "arXiv", "category": "content", "url": "https://arxiv.org/list/cs/recent", "original_url": "https://arxiv.org/list/cs/recent", "final_url": "https://arxiv.org/list/cs/recent", "expected_required_content": "Recent Computer Science papers with subject metadata", "started_at": "2026-09-29T22:08:58.484575+00:00", "request_started_at": "2026-09-29T22:08:59.051887+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://arxiv.org/list/cs/recent", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 15996, "wire_body_sha256": "a91e7df93e474de3aeca618472f13abbf2ec9ae9ad7f4977be6c95a27df8aeca" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "b1de1feefecddc4ea113b631684531e8e98358cc55f85f080167f7d0fe78a275", "response_bytes": 97480, "downloaded_body_bytes": 97480, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.782, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.175, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 234, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 16595, "httpcompression/response_bytes": 97480, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.1771072440315038, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://arxiv.org/list/cs/recent", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "eddf21e6bd2d6cba1053d89282c22f60bb074173e9f500a6bd9f940bc2c5a74e", "finished_at": "2026-09-29T22:08:59.266143+00:00", "checks": { "text_chars": 16898, "required_text_checks": [ { "alternatives": [ "computer science" ], "matched": [ "computer science" ] }, { "alternatives": [ "subjects:" ], "matched": [ "subjects:" ] } ], "text_regex_checks": [], "min_text_chars": 1500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "5277509d6ee150f02725d5efd919e6cc632de1c91d6e3cbb8fd0a2537191d379", "peak_sampled_tree_pss_mb": 61.241, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13634.53515625, "SwapTotal": 8191.99609375, "SwapFree": 40.4296875, "unit": "MiB", "measured_at": "2026-09-29T22:08:58.230934+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "gutenberg", "site": "Project Gutenberg", "category": "content", "url": "https://www.gutenberg.org/ebooks/1342", "original_url": "https://www.gutenberg.org/ebooks/1342", "final_url": "https://www.gutenberg.org/ebooks/1342", "expected_required_content": "Pride and Prejudice book description with author", "started_at": "2026-09-29T22:09:01.725290+00:00", "request_started_at": "2026-09-29T22:09:02.318673+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.gutenberg.org/ebooks/1342", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 6596, "wire_body_sha256": "c0ba29059fe73ec80532a9ca162b159f0f4ae089cd9aecb23964978f26db3fbd" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html;charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "c45c33ed150dc069a8c30f684d607fb55d6d9aff6bf3af1107604eca943245ff", "response_bytes": 24862, "downloaded_body_bytes": 24862, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.262, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.528, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 239, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 6927, "httpcompression/response_bytes": 24862, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.5300092960242182, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.gutenberg.org/ebooks/1342", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "a866833b032ce35b26d13b30ab4b2d4aea4e5c6371bbd197afedf1af30288c7b", "finished_at": "2026-09-29T22:09:02.987577+00:00", "checks": { "text_chars": 2588, "required_text_checks": [ { "alternatives": [ "pride and prejudice" ], "matched": [ "pride and prejudice" ] }, { "alternatives": [ "jane austen" ], "matched": [ "jane austen" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "44e5541114769e75940ce28c59aa5c3f04e2394a9b9c3737e2dd90b2972a0f6a", "peak_sampled_tree_pss_mb": 61.826, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13658.15234375, "SwapTotal": 8191.99609375, "SwapFree": 40.4296875, "unit": "MiB", "measured_at": "2026-09-29T22:09:01.468194+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "govuk", "site": "GOV.UK", "category": "content", "url": "https://www.gov.uk/bank-holidays", "original_url": "https://www.gov.uk/bank-holidays", "final_url": "https://www.gov.uk/bank-holidays", "expected_required_content": "UK bank holiday dates for England and Wales", "started_at": "2026-09-29T22:09:05.378277+00:00", "request_started_at": "2026-09-29T22:09:05.989562+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.gov.uk/bank-holidays", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 13491, "wire_body_sha256": "24b553a0082e3fbfbf8dd44b34990d1fec68b71fba4bcfdbff88f12aaf7b8969" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "4d698a82e0b74198f5d031a5d704b7e8a4bbdf7b8b1293b6c0b5c822f397d97f", "response_bytes": 175904, "downloaded_body_bytes": 175904, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.816, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.175, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 234, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 16953, "httpcompression/response_bytes": 175904, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.17711183708161116, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.gov.uk/bank-holidays", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "e6759a8c215f7790bd0bec11b059239cd8467a60fb86454bcf5c7b8544141a22", "finished_at": "2026-09-29T22:09:06.194698+00:00", "checks": { "text_chars": 17613, "required_text_checks": [ { "alternatives": [ "uk bank holidays" ], "matched": [ "uk bank holidays" ] }, { "alternatives": [ "england and wales" ], "matched": [ "england and wales" ] }, { "alternatives": [ "christmas day" ], "matched": [ "christmas day" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "5b866f2f7c6a913202d6314710445fa6bb0e89d2a19d2e9a38dbeb9fb95964ce", "peak_sampled_tree_pss_mb": 62.056, "pss_samples": 12, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13695.4765625, "SwapTotal": 8191.99609375, "SwapFree": 40.4296875, "unit": "MiB", "measured_at": "2026-09-29T22:09:05.107992+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "python", "site": "Python documentation", "category": "content", "url": "https://docs.python.org/3/library/urllib.request.html", "original_url": "https://docs.python.org/3/library/urllib.request.html", "final_url": "https://docs.python.org/3/library/urllib.request.html", "expected_required_content": "urllib.request documentation with API explanation", "started_at": "2026-09-29T22:09:08.629285+00:00", "request_started_at": "2026-09-29T22:09:09.226144+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://docs.python.org/3/library/urllib.request.html", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 29338, "wire_body_sha256": "7c354c742245a23b5a4e07d9ea0afbaf5316bdee4d567ba99ed025f220298d69" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "7690190ba74d1d6c2e247423f3fe194a5101667392c9de03bbe53296bd7a45b7", "response_bytes": 216525, "downloaded_body_bytes": 216525, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.847, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.209, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 255, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 29896, "httpcompression/response_bytes": 216525, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.21101348684169352, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://docs.python.org/3/library/urllib.request.html", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "c1ea5423153d2b14a8a5f656264573a94e1e65619bc289d5f7511fdd0ad30daf", "finished_at": "2026-09-29T22:09:09.476579+00:00", "checks": { "text_chars": 53833, "required_text_checks": [ { "alternatives": [ "urllib.request" ], "matched": [ "urllib.request" ] }, { "alternatives": [ "urlopen" ], "matched": [ "urlopen" ] } ], "text_regex_checks": [], "min_text_chars": 3000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "a113ca286627fa9f6a8342ae29fd607df4037cbeadc2480023074666eb717b58", "peak_sampled_tree_pss_mb": 62.699, "pss_samples": 13, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13724.546875, "SwapTotal": 8191.99609375, "SwapFree": 40.43359375, "unit": "MiB", "measured_at": "2026-09-29T22:09:08.352929+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "quotes", "site": "Quotes to Scrape", "category": "javascript", "url": "https://quotes.toscrape.com/js/", "original_url": "https://quotes.toscrape.com/js/", "final_url": "https://quotes.toscrape.com/js/", "expected_required_content": "Quote text and author after JavaScript rendering", "started_at": "2026-09-29T22:09:11.969859+00:00", "request_started_at": "2026-09-29T22:09:12.594439+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://quotes.toscrape.com/js/", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 1750, "wire_body_sha256": "3f2b0a0890740ba874dd83d9a4547c1b039b71cf1f1e3acdcdd87ff7caf31c2e" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "a9067d4438044d45a1a895ed6939df4fa4d0487e17b82053dd694ad425466074", "response_bytes": 5808, "downloaded_body_bytes": 5808, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.207, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.453, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 233, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 1934, "httpcompression/response_bytes": 5808, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.4551540969405323, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://quotes.toscrape.com/js/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "d922e6b3edb32d8f8ed2f8d56086926245a7ec4b96b2c29573d0ae16bde85d51", "finished_at": "2026-09-29T22:09:13.177237+00:00", "checks": { "text_chars": 74, "required_text_checks": [ { "alternatives": [ "the world as we have created it" ], "matched": [] }, { "alternatives": [ "albert einstein" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "4e99f91c2997ba8522e6127b47e02d58f910ca18fc0e6627f78a6a3960500efd", "peak_sampled_tree_pss_mb": 61.906, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13607.2421875, "SwapTotal": 8191.99609375, "SwapFree": 40.51171875, "unit": "MiB", "measured_at": "2026-09-29T22:09:11.717898+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content." }, { "target_id": "scrapingcourse", "site": "ScrapingCourse", "category": "javascript", "url": "https://www.scrapingcourse.com/javascript-rendering", "original_url": "https://www.scrapingcourse.com/javascript-rendering", "final_url": "https://www.scrapingcourse.com/javascript-rendering", "expected_required_content": "JavaScript product listing with named products and prices", "started_at": "2026-09-29T22:09:15.615151+00:00", "request_started_at": "2026-09-29T22:09:16.202791+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.scrapingcourse.com/javascript-rendering", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 2692, "wire_body_sha256": "f4f1cd86c3bd596638deb1e9a1a073f3f468cfdeab6a3ca11b63e16c3f381684" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "b9daac7e24122000dbc457ed0e4ed24b43b89593db88eac038e98499db93292a", "response_bytes": 29640, "downloaded_body_bytes": 29640, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.415, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.821, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 253, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 4623, "httpcompression/response_bytes": 29640, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.8229826018214226, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.scrapingcourse.com/javascript-rendering", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "6489b07dfe625bc0b4d222a94dc65625196a81d806eff775c510a68ab5d3754d", "finished_at": "2026-09-29T22:09:17.030178+00:00", "checks": { "text_chars": 72, "required_text_checks": [ { "alternatives": [ "chaz kangeroo hoodie", "teton pullover hoodie" ], "matched": [] } ], "text_regex_checks": [ { "pattern": "[$\u00a3\u20ac]\\s?\\d[\\d,.]*", "min_matches": 3, "matches": 0 } ], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "caa0aeb9120867ac0d337a428913a66ba3fd3e6096322116c60be62953cc678a", "peak_sampled_tree_pss_mb": 62.289, "pss_samples": 18, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13676.453125, "SwapTotal": 8191.99609375, "SwapFree": 40.515625, "unit": "MiB", "measured_at": "2026-09-29T22:09:15.359634+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content." }, { "target_id": "webscrapingdev", "site": "Web Scraping Dev", "category": "javascript", "url": "https://web-scraping.dev/reviews", "original_url": "https://web-scraping.dev/reviews", "final_url": "https://web-scraping.dev/reviews", "expected_required_content": "Review text loaded by the JavaScript review interface", "started_at": "2026-09-29T22:09:19.456860+00:00", "request_started_at": "2026-09-29T22:09:20.107491+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://web-scraping.dev/reviews", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 14727, "wire_body_sha256": "77a5e13ddd2d8d9d65ba531340293d16870224866f7d52bbf7377f113b514665" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "77a5e13ddd2d8d9d65ba531340293d16870224866f7d52bbf7377f113b514665", "response_bytes": 14727, "downloaded_body_bytes": 14727, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.583, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.756, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 234, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 15523, "response_received_count": 1, "elapsed_time_seconds": 0.7581391930580139, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://web-scraping.dev/reviews", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "ed865954fb2cd4005f358cbddb7915722f6a77665970dd56af72e51b98767964", "finished_at": "2026-09-29T22:09:21.039665+00:00", "checks": { "text_chars": 244, "required_text_checks": [ { "alternatives": [ "unique flavor and great energy boost", "excellent energy drink for gamers" ], "matched": [] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "3770c167fe72381736ba3e02d30276f01732e194fd5887d368e0e8ece6ed2928", "peak_sampled_tree_pss_mb": 61.281, "pss_samples": 19, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13662.859375, "SwapTotal": 8191.99609375, "SwapFree": 40.51953125, "unit": "MiB", "measured_at": "2026-09-29T22:09:19.201928+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content. This HTTP-only pass did not execute JavaScript to populate the reviews route." }, { "target_id": "example", "site": "Example Domain", "category": "control", "url": "https://example.com/", "original_url": "https://example.com/", "final_url": "https://example.com/", "expected_required_content": "Example Domain heading and documentation explanation", "started_at": "2026-09-29T22:09:23.511273+00:00", "request_started_at": "2026-09-29T22:09:24.120347+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://example.com/", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 409, "wire_body_sha256": "a7a2b5dd0a9356acf1139ffa36b6413eb898336f3e69ca2f1c26329d020611ec" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "7d3e61f8f627c8cc640c209ba0777db95f198a64e766f47484c321d5eb5e0962", "response_bytes": 713, "downloaded_body_bytes": 713, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.697, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.081, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 222, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 674, "httpcompression/response_bytes": 713, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.08323241909965873, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://example.com/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "a89ee83449c1d02d431907a52e27dc7d6479a79ba9ff3a65cda3952874bcb815", "finished_at": "2026-09-29T22:09:24.208589+00:00", "checks": { "text_chars": 167, "required_text_checks": [ { "alternatives": [ "example domain" ], "matched": [] }, { "alternatives": [ "documentation examples" ], "matched": [ "documentation examples" ] } ], "text_regex_checks": [], "min_text_chars": 100, "has_required_content": false, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "missing_required_content", "usable_content": false, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "3b99efef5db97edbeb71416568b71cda7f61bddd913191551e90761b62dc04b1", "peak_sampled_tree_pss_mb": 61.461, "pss_samples": 11, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13616.71875, "SwapTotal": 8191.99609375, "SwapFree": 40.5234375, "unit": "MiB", "measured_at": "2026-09-29T22:09:23.244894+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Saved response was rechecked with unchanged rules: missing required content. The frozen source parser/phrase rule is unchanged; only its actual checked text and matches are reported, without a browser inspection." }, { "target_id": "httpbingo", "site": "HTTPBingo", "category": "control", "url": "https://httpbingo.org/html", "original_url": "https://httpbingo.org/html", "final_url": "https://httpbingo.org/html", "expected_required_content": "Moby-Dick HTML sample", "started_at": "2026-09-29T22:09:26.627317+00:00", "request_started_at": "2026-09-29T22:09:27.273683+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://httpbingo.org/html", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 2081, "wire_body_sha256": "1748e6a05ca014eb56271c1b5807172fae0b8c51a4084ecd55bd53d3dd289fc7" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "e250a7975d53801ed8c35271780a15a1da36a8a5383327cb706dea252b340d48", "response_bytes": 3742, "downloaded_body_bytes": 3742, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 0.989, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.299, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 228, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 2386, "httpcompression/response_bytes": 3742, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.30087419506162405, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://httpbingo.org/html", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "bc7bc3bf1c7337f9a6e64a62f4653ad004f526cc5d67393e81a91eeb1dc6bb72", "finished_at": "2026-09-29T22:09:27.616259+00:00", "checks": { "text_chars": 3594, "required_text_checks": [ { "alternatives": [ "herman melville" ], "matched": [ "herman melville" ] }, { "alternatives": [ "moby-dick" ], "matched": [ "moby-dick" ] } ], "text_regex_checks": [], "min_text_chars": 300, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "0fecffac22a2590cf28fe67b651bfb1e9677adc14ad21fbeec299e970b221dfd", "peak_sampled_tree_pss_mb": 62.198, "pss_samples": 14, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13624.9140625, "SwapTotal": 8191.99609375, "SwapFree": 40.73046875, "unit": "MiB", "measured_at": "2026-09-29T22:09:26.369399+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "scrapethissite", "site": "Scrape This Site", "category": "control", "url": "https://www.scrapethissite.com/pages/simple/", "original_url": "https://www.scrapethissite.com/pages/simple/", "final_url": "https://www.scrapethissite.com/pages/simple/", "expected_required_content": "Country records including Andorra and population", "started_at": "2026-09-29T22:09:30.061579+00:00", "request_started_at": "2026-09-29T22:09:30.654358+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.scrapethissite.com/pages/simple/", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 12048, "wire_body_sha256": "7703608ddb03f0d073ea36f98c3fbfb4cd223c927b3f9ce3daf8b98aea179eff" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=utf-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "4c68c2a54000607a0fa51ea5a57a20bbe4324dee74c442559185a1da364bd87b", "response_bytes": 203361, "downloaded_body_bytes": 203361, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.523, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.924, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 246, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 12813, "httpcompression/response_bytes": 203361, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.9266630220226943, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.scrapethissite.com/pages/simple/", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "df72ebed3bf114e703849a0f7baf602746be38b87c5dab77ff52bfab1a7000cc", "finished_at": "2026-09-29T22:09:31.585082+00:00", "checks": { "text_chars": 18132, "required_text_checks": [ { "alternatives": [ "countries of the world" ], "matched": [ "countries of the world" ] }, { "alternatives": [ "andorra" ], "matched": [ "andorra" ] }, { "alternatives": [ "population" ], "matched": [ "population" ] } ], "text_regex_checks": [], "min_text_chars": 1000, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "735d4a44e3bc664184fd12c40087c3bf01433d47d85f46cf118d3fcf9631842c", "peak_sampled_tree_pss_mb": 62.177, "pss_samples": 19, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13688.61328125, "SwapTotal": 8191.99609375, "SwapFree": 40.734375, "unit": "MiB", "measured_at": "2026-09-29T22:09:29.804026+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." }, { "target_id": "iana", "site": "IANA", "category": "control", "url": "https://www.iana.org/domains/reserved", "original_url": "https://www.iana.org/domains/reserved", "final_url": "https://www.iana.org/domains/reserved", "expected_required_content": "Reserved-domain reference and example domains", "started_at": "2026-09-29T22:09:34.039987+00:00", "request_started_at": "2026-09-29T22:09:34.656872+00:00", "status": 200, "request_outcome": "returned", "redirect_history": [], "redirect_outcome": "none", "document_responses": [ { "url": "https://www.iana.org/domains/reserved", "status": 200, "protocol": "HTTP/1.1", "wire_body_bytes": 2404, "wire_body_sha256": "5d3ecedc3ab466dd29929814587af2afe28a2aada8d5625eb944f4a726c6b83f" } ], "explicit_request_headers": {}, "response_metadata": { "content_type": "text/html; charset=UTF-8", "content_encoding": "", "protocol": "HTTP/1.1", "flags": [], "body_basis": "Final Spider response after compression middleware" }, "error": null, "navigation_error": null, "capture_error": null, "screenshot_error": null, "screenshot_sha256": null, "capture_complete": true, "raw_response_sha256": "720e5cce9016212d2d28777356d2844fb0ca9e19a32636eb232f8653360dc450", "response_bytes": 10499, "downloaded_body_bytes": 10499, "body_complete": true, "navigation_seconds": null, "attempt_seconds": 1.191, "effective_settings": { "USER_AGENT": "Scrapy/2.19.0 (+https://scrapy.org)", "CONCURRENT_REQUESTS": 1, "CONCURRENT_REQUESTS_PER_DOMAIN": 1, "ROBOTSTXT_OBEY": false, "COOKIES_ENABLED": false, "RETRY_ENABLED": false, "RETRY_TIMES": 0, "HTTPPROXY_ENABLED": false, "HTTPERROR_ALLOW_ALL": true, "REDIRECT_ENABLED": true, "REDIRECT_MAX_TIMES": 3, "METAREFRESH_ENABLED": false, "DOWNLOAD_TIMEOUT": 20, "DOWNLOAD_MAXSIZE": 5242880, "DOWNLOAD_WARNSIZE": 5242880, "DOWNLOAD_VERIFY_CERTIFICATES": true, "DOWNLOAD_FAIL_ON_DATALOSS": true, "HTTPCACHE_ENABLED": false, "TELNETCONSOLE_ENABLED": false, "MEMUSAGE_ENABLED": false, "LOG_LEVEL": "INFO", "DEFAULT_REQUEST_HEADERS": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en" }, "DOWNLOADER_MIDDLEWARES": { "__main__.ResponseObserver": 900, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": null }, "DOWNLOAD_HANDLERS": { "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" } }, "latency_seconds": 0.568, "scrapy_stats": { "log_count/INFO": 3, "scheduler/enqueued/memory": 1, "scheduler/enqueued": 1, "scheduler/dequeued/memory": 1, "scheduler/dequeued": 1, "downloader/request_count": 1, "downloader/request_method_count/GET": 1, "downloader/request_bytes": 239, "downloader/response_count": 1, "downloader/response_status_count/200": 1, "downloader/response_bytes": 3509, "httpcompression/response_bytes": 10499, "httpcompression/response_count": 1, "response_received_count": 1, "elapsed_time_seconds": 0.5701963710598648, "finish_reason": "finished" }, "downloader_request_count": 1, "observed_request_headers": [ { "url": "https://www.iana.org/domains/reserved", "method": "GET", "headers": [ { "name": "Accept", "values": [ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" ] }, { "name": "Accept-Language", "values": [ "en" ] }, { "name": "User-Agent", "values": [ "Scrapy/2.19.0 (+https://scrapy.org)" ] }, { "name": "Accept-Encoding", "values": [ "gzip, deflate, br, zstd" ] } ] } ], "transport_log_sha256": "023187611abadcb8968faf151a2b3c2630767375fec74fd22834f2ed2757a311", "finished_at": "2026-09-29T22:09:35.231097+00:00", "checks": { "text_chars": 2727, "required_text_checks": [ { "alternatives": [ "iana-managed reserved domains" ], "matched": [ "iana-managed reserved domains" ] }, { "alternatives": [ "example.com" ], "matched": [ "example.com" ] } ], "text_regex_checks": [], "min_text_chars": 500, "has_required_content": true, "soft_block_checks": { "phrases_checked": [ "verify you are human", "robot or human", "robot check", "access denied", "pardon our interruption", "please complete the captcha", "enter the characters", "additional verification required", "unusual traffic", "checking your browser", "enable javascript and cookies to continue", "just a moment" ], "matched": [] }, "gate_checks": { "phrases_checked": [ "log in to continue", "sign in to continue", "sign in to linkedin", "consent required", "before you continue" ], "matched": [] } }, "empty_content": false, "classification": "usable", "usable_content": true, "cleanup_surviving_owned_pids": [], "driver_log_sha256": "0256f22e0d531e2423f5e39269693d25e572665d534c44855588bb2b9fe6ad82", "peak_sampled_tree_pss_mb": 62.182, "pss_samples": 16, "attempt_process_exit": 0, "host_ram_before": { "MemTotal": 32049.69921875, "MemAvailable": 13692.296875, "SwapTotal": 8191.99609375, "SwapFree": 40.734375, "unit": "MiB", "measured_at": "2026-09-29T22:09:33.775476+00:00" }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "inspection_note": "Complete client-decompressed Spider response met every unchanged word group, listing pattern and minimum text. No rendered visibility or complete field extraction was verified." } ], "mode_selection": "Documented CrawlerProcess plus one async Spider.start Request. Stock HTTP11 downloader, identified Scrapy User-Agent; parse and errback schedule no further pages. This tests one-page downloader behavior, not Scrapy crawl/scheduling strengths.", "body_hash_basis": "Final client-decompressed Spider response bytes, not browser DOM or re-encoded text. Early wire responses, bodies and logs private. Terminal errors retain the last early response where present; incomplete/error state explicit.", "latency_basis": "From initial Spider.start Request scheduling to delivered final response or errback; includes redirects, scheduling/middleware and body transfer/decompression. Excludes initial crawler setup, later saving/checks and reactor shutdown. Downloader hop latency is separately recorded privately.", "pss_caveat": "100 ms worker-tree samples include interpreter/imports, crawler setup, scheduling, HTTP download, middleware/capture/checks and reactor shutdown. Batch parent excluded; brief peaks may be missed. Different work from direct clients, selectors and browsers.", "baseline_sha256": { "botasaurus": "f8a1e95d8b5e55bb320234a27cbf8605b4184587509d80c73c02c6c0aacd297b", "camoufox": "736930b15c3e677a5a651a397bbdf1e9a9076b3bf1a1458e3a0bb8716d653eda", "curl_cffi": "72b4f31c91fef067d72d19cdb5cd669a56a41a54122309dd89188b04f26a47be", "lightpanda": "bd0deb3fd9ee7f2a7a67ae95e3458503c47fa7069edd584cc051655b9ae3affa", "patchright": "cc7a70f7b5cd87e8a2e3f66e015730e4f258f9295159d6049cae70705147de14", "scrapling": "610770a368ccc61098a781aab1fe24721e93992b71c1940f3bd6f36a193fe4f7", "seleniumbase": "a35988565e4c57e9785ecd2af72a9eea3ae28b99c80a63c80267d1bb99d931aa", "wreq": "269132e9697d084941827877de584d1db11f6f6cd8d8fcfb41f4ab69646d54b8" }, "policy_differences": [ "Scrapy/2.19.0 bot User-Agent, default Accept/Accept-Language=en and compression headers; stock Twisted HTTP/1.1 with certificate verification enabled, no Chrome TLS impersonation.", "Cookies and robots disabled; no robots URL or public-suffix cookie fetch. Fresh isolated process per target. Other tools differ in cookie/header/JS and loaded-resource work.", "Normal redirects cap 3; meta-refresh disabled. No per-request handle_httpstatus_all because it disables redirects in this release; global HTTPERROR_ALLOW_ALL retains error pages. Standard redirect Referer handling retained.", "Scrapy retry middleware disabled; stock Twisted cached-connection recovery not separately disabled or instrumented. No proxy, browser, JavaScript, login, CAPTCHA interaction, link discovery, feeds or HTTP cache. HttpProxyMiddleware disabled and environment proxies removed.", "20-second per-download timeout, 35-second whole-worker watchdog, 3 GiB host RAM floor and 1 GiB worker-tree PSS guard. 5 MiB handler/compression limit differs from clients that cap saved bytes only.", "One dated URL test measures downloader behavior, not crawling/scheduling scale or reliability. Different time windows and policies establish no global winner or feature cause." ], "summary": { "attempted_distinct_sites": 30, "usable_results": 11, "classifications": { "http_access_denied": 8, "http_server_error": 1, "missing_required_content": 9, "redirect_limit_or_loop": 1, "usable": 11 }, "http_200_soft_blocks": 0, "http_200_empty_content": 1, "downloader_requests_including_redirects": 34 }, "finished_at": "2026-09-29T22:09:35.418492+00:00", "installed_packages": { "aiohappyeyeballs": "2.7.1", "aiohttp": "3.14.3", "aiosignal": "1.4.0", "attrs": "26.1.0", "automat": "25.4.16", "backports-zstd": "1.7.0", "brotli": "1.2.0", "certifi": "2026.7.22", "cffi": "2.1.1", "charset-normalizer": "3.5.1", "constantly": "23.10.4", "cryptography": "50.0.1", "cssselect": "1.5.0", "defusedxml": "0.7.1", "filelock": "4.0.6", "frozenlist": "1.8.0", "hyperlink": "21.0.0", "idna": "3.20", "incremental": "24.11.0", "itemadapter": "0.13.1", "itemloaders": "1.5.0", "jmespath": "1.1.0", "lxml": "6.1.3", "multidict": "6.9.1", "packaging": "26.3", "parsel": "1.12.1", "platformdirs": "4.12.2", "propcache": "0.5.4", "protego": "0.7.0", "pycparser": "3.0", "pydispatcher": "2.0.7", "pyopenssl": "26.4.0", "queuelib": "1.10.0", "requests": "2.34.2", "requests-file": "3.0.1", "scrapy": "2.19.0", "service-identity": "26.1.0", "tldextract": "5.3.2", "twisted": "26.4.0", "typing-extensions": "4.16.0", "urllib3": "2.8.0", "w3lib": "2.4.1", "yarl": "1.25.1", "zope-interface": "8.6" }, "additional_versioned_verification": { "installed_release": "26.4.0", "wheel_url": "https://files.pythonhosted.org/packages/a6/57/bcf4e2370dd218c9aa68a9140a65d86729c73f1d529f7e94786c2766fc72/twisted-26.4.0-py3-none-any.whl", "wheel_sha256": "dc25ea0ebf6511c24f03232ee9f4afa54b291c5d897990e3a39cc4d14a1ef4c0", "source_path": "twisted/web/client.py", "source_sha256": "d62f714b733e0073f892ff9904ec94a5c64c3d9dff3039e4e594c58c1db9bc33", "official_wheel_matches_installed": true, "finding": "HTTPConnectionPool.retryAutomatically=True can recover one failed idempotent request on a cached persistent connection. Scrapy RetryMiddleware is disabled, but this stock native recovery was not separately disabled or instrumented; observed middleware request count is not an independently verified wire-attempt count." }, "downloader_count_basis": "Observed downloader middleware Requests, including ordinary redirects; not an independently captured count of wire attempts. Stock Twisted cached-connection recovery may happen below this observer.", "publication_validation": { "checked_at": "2026-09-29T22:12:13.697724+00:00", "unchanged_validator_ast_verified": true, "body_hashes_and_checks_verified": 30, "pss_peaks_verified": 30, "private_transport_logs_and_wire_hashes_verified": 30, "response_status_and_redirects_verified": 30, "driver_log_hashes_verified": 30, "additional_target_requests": 0, "surviving_owned_processes": 0 }, "request_header_observation_basis": "Downloader middleware before the native handler; implicit Host/connection/transport headers are not a complete wire dump. Cookies/auth values excluded.", "startup_checks_url": "/evidence/scrapy-public-30-20260929/startup-checks.json", "canary_relation": "One complete HTTP 200 canary body failed frozen reviews checks. It establishes returned downloader plumbing, not usable review extraction. Excluded from the full 30-target batch; 31 explicit root Spider Requests total, plus ordinary redirects. No local smoke, browser or failed setup navigation.", "pss_samples_sha256": "108780412c92672423eec1cc8091bb211c82c56a27aced3e4e72e560e199049a" }