fix(crawler): prioritize usable source retrieval over equal byte splits
Fetch candidate pages sequentially with the full remaining download budget instead of dividing a quick run's 500 KB equally across every URL. This avoids aborting otherwise readable pages at roughly 33 KB before claim extraction can inspect them. Account for bytes consumed by failed downloads, preserve fetch errors in source responses, and fail runs with no successfully extracted document using an actionable error. Keep attempted sources on failed runs for diagnostics and cover the allocation and no-usable-source paths with regression tests.
This commit is contained in:
@@ -163,15 +163,16 @@ def test_fetching_uses_actual_bytes_and_respects_remaining_source_budget() -> No
|
||||
"https://example.org/two", "two", metadata={"download_bytes": 180}
|
||||
)
|
||||
crawler = Mock()
|
||||
crawler.fetch_and_extract_many = AsyncMock(return_value=[doc_one, doc_two])
|
||||
crawler.fetch_and_extract_many = AsyncMock(side_effect=[[doc_one], [doc_two]])
|
||||
orchestrator._crawler = crawler
|
||||
|
||||
response = await orchestrator._step_fetching()
|
||||
|
||||
assert response["success"] is True
|
||||
crawler.fetch_and_extract_many.assert_awaited_once_with(
|
||||
["https://example.org/one", "https://example.org/two"], max_size=500
|
||||
)
|
||||
assert crawler.fetch_and_extract_many.await_args_list[0].args == (["https://example.org/one"],)
|
||||
assert crawler.fetch_and_extract_many.await_args_list[0].kwargs == {"max_size": 1_000}
|
||||
assert crawler.fetch_and_extract_many.await_args_list[1].args == (["https://example.org/two"],)
|
||||
assert crawler.fetch_and_extract_many.await_args_list[1].kwargs == {"max_size": 880}
|
||||
usage = orchestrator.budget_tracker.get_usage()
|
||||
assert usage["max_sources"]["usage"] == 2
|
||||
assert usage["max_total_download_bytes"]["usage"] == 300
|
||||
@@ -179,6 +180,30 @@ def test_fetching_uses_actual_bytes_and_respects_remaining_source_budget() -> No
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_fetching_fails_with_source_errors_when_no_document_is_usable() -> None:
|
||||
async def run() -> None:
|
||||
orchestrator = ResearchOrchestrator(
|
||||
_config(), uuid4(), "test", budget_config=HardBudgetConfig(max_sources=2, max_total_download_bytes=1_000)
|
||||
)
|
||||
orchestrator._search_results = [{"url": "https://example.org/blocked"}]
|
||||
failed_doc = NormalizedDocument.from_text(
|
||||
"https://example.org/blocked",
|
||||
"",
|
||||
metadata={"error": "Download exceeded 1000 bytes", "download_bytes": 1_000},
|
||||
)
|
||||
crawler = Mock()
|
||||
crawler.fetch_and_extract_many = AsyncMock(return_value=[failed_doc])
|
||||
orchestrator._crawler = crawler
|
||||
|
||||
response = await orchestrator._step_fetching()
|
||||
|
||||
assert response["success"] is False
|
||||
assert "Download exceeded 1000 bytes" in response["error"]
|
||||
assert response["sources"][0]["error"] == "Download exceeded 1000 bytes"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_hard_budget_allows_exactly_the_configured_limit() -> None:
|
||||
tracker = BudgetTracker(HardBudgetConfig(max_sources=2))
|
||||
tracker.increment_sources(2)
|
||||
|
||||
Reference in New Issue
Block a user