Fix research pipeline budget tracking

This commit is contained in:
faligam
2026-09-07 11:56:13 +02:00
parent a5324d3971
commit 5f237d535f
7 changed files with 148 additions and 17 deletions

View File

@@ -11,6 +11,7 @@ import pytest
from nsct.crawler.extraction import extract_main_content
from nsct.crawler.fetcher import AsyncFetcher, FetchResult, FetchStatus
from nsct.crawler.manager import CrawlerManager
from nsct.crawler.normalize import NormalizedDocument
from nsct.crawler.policy import (
CrawlerPolicyError,
@@ -209,6 +210,15 @@ async def test_fetcher_unreachable_domain() -> None:
await fetcher.close()
def test_error_document_has_hash_for_empty_content() -> None:
"""A failed fetch must still yield a valid normalized document."""
doc = CrawlerManager._error_doc("https://example.com/unavailable", "connection refused")
assert doc.text == ""
assert doc.metadata["error"] == "connection refused"
assert doc.content_hash == hashlib.sha256(b"").hexdigest()
# ---------------------------------------------------------------------------
# NormalizedDocument has all required fields
# ---------------------------------------------------------------------------