fix(crawler): prioritize usable source retrieval over equal byte splits

Fetch candidate pages sequentially with the full remaining download budget instead of dividing a quick run's 500 KB equally across every URL. This avoids aborting otherwise readable pages at roughly 33 KB before claim extraction can inspect them.

Account for bytes consumed by failed downloads, preserve fetch errors in source responses, and fail runs with no successfully extracted document using an actionable error. Keep attempted sources on failed runs for diagnostics and cover the allocation and no-usable-source paths with regression tests.
This commit is contained in:
faligam
2026-09-07 14:04:48 +02:00
parent 610c716a97
commit 56d6415e68
5 changed files with 109 additions and 20 deletions

View File

@@ -163,15 +163,16 @@ def test_fetching_uses_actual_bytes_and_respects_remaining_source_budget() -> No
"https://example.org/two", "two", metadata={"download_bytes": 180}
)
crawler = Mock()
crawler.fetch_and_extract_many = AsyncMock(return_value=[doc_one, doc_two])
crawler.fetch_and_extract_many = AsyncMock(side_effect=[[doc_one], [doc_two]])
orchestrator._crawler = crawler
response = await orchestrator._step_fetching()
assert response["success"] is True
crawler.fetch_and_extract_many.assert_awaited_once_with(
["https://example.org/one", "https://example.org/two"], max_size=500
)
assert crawler.fetch_and_extract_many.await_args_list[0].args == (["https://example.org/one"],)
assert crawler.fetch_and_extract_many.await_args_list[0].kwargs == {"max_size": 1_000}
assert crawler.fetch_and_extract_many.await_args_list[1].args == (["https://example.org/two"],)
assert crawler.fetch_and_extract_many.await_args_list[1].kwargs == {"max_size": 880}
usage = orchestrator.budget_tracker.get_usage()
assert usage["max_sources"]["usage"] == 2
assert usage["max_total_download_bytes"]["usage"] == 300
@@ -179,6 +180,30 @@ def test_fetching_uses_actual_bytes_and_respects_remaining_source_budget() -> No
asyncio.run(run())
def test_fetching_fails_with_source_errors_when_no_document_is_usable() -> None:
async def run() -> None:
orchestrator = ResearchOrchestrator(
_config(), uuid4(), "test", budget_config=HardBudgetConfig(max_sources=2, max_total_download_bytes=1_000)
)
orchestrator._search_results = [{"url": "https://example.org/blocked"}]
failed_doc = NormalizedDocument.from_text(
"https://example.org/blocked",
"",
metadata={"error": "Download exceeded 1000 bytes", "download_bytes": 1_000},
)
crawler = Mock()
crawler.fetch_and_extract_many = AsyncMock(return_value=[failed_doc])
orchestrator._crawler = crawler
response = await orchestrator._step_fetching()
assert response["success"] is False
assert "Download exceeded 1000 bytes" in response["error"]
assert response["sources"][0]["error"] == "Download exceeded 1000 bytes"
asyncio.run(run())
def test_hard_budget_allows_exactly_the_configured_limit() -> None:
tracker = BudgetTracker(HardBudgetConfig(max_sources=2))
tracker.increment_sources(2)