Fix research pipeline budget tracking
This commit is contained in:
@@ -11,6 +11,7 @@ import pytest
|
||||
|
||||
from nsct.crawler.extraction import extract_main_content
|
||||
from nsct.crawler.fetcher import AsyncFetcher, FetchResult, FetchStatus
|
||||
from nsct.crawler.manager import CrawlerManager
|
||||
from nsct.crawler.normalize import NormalizedDocument
|
||||
from nsct.crawler.policy import (
|
||||
CrawlerPolicyError,
|
||||
@@ -209,6 +210,15 @@ async def test_fetcher_unreachable_domain() -> None:
|
||||
await fetcher.close()
|
||||
|
||||
|
||||
def test_error_document_has_hash_for_empty_content() -> None:
|
||||
"""A failed fetch must still yield a valid normalized document."""
|
||||
doc = CrawlerManager._error_doc("https://example.com/unavailable", "connection refused")
|
||||
|
||||
assert doc.text == ""
|
||||
assert doc.metadata["error"] == "connection refused"
|
||||
assert doc.content_hash == hashlib.sha256(b"").hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NormalizedDocument has all required fields
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user