diff --git a/HANDOFF.md b/HANDOFF.md index cca2975..948a4a7 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -7,6 +7,40 @@ ## Aktueller Stand — 2026-09-07 +### Browser-E2E: Fehlerpfade und Quick-Budgets weiter repariert — 2026-09-07 + +Die nachfolgenden echten Browser-/Proxy-Tests erreichten erstmals die Crawler- +und Analysephasen. Drei weitere Fehler sind lokal repariert, durch gezielte +Regressionstests abgedeckt und mit `docker compose up -d --build nsct-api` +deployed: + +1. Der Crawler erzeugte für einzelne Fetch-Fehler ein `NormalizedDocument` mit + leerem `content_hash`. Der Fehlerpfad verwendet nun `from_text()`, das auch + für leeren Inhalt einen gültigen SHA-256-Hash erzeugt. Einzelne kaputte + Quellen brechen damit nicht mehr die gesamte Recherche ab. +2. Quick-Runs buchten beim Start einen nicht existierenden Planner-Request und + anschließend eine Claim-LLM-Anfrage für jede Quelle, einschließlich + Fehlerquellen. Bei 15 Quellen standen so vor der Analyse 17 statt maximal + 15 Requests im Budget. Der Phantom-Eintrag entfällt; nur erfolgreiche + Quellen werden für Claims verarbeitet und ein Request bleibt für die + Synthese reserviert. Ein voll ausgelasteter Quick-Run nutzt damit höchstens + `1 Planung + 13 Claims + 1 Synthese = 15` LLM-Requests. +3. `BudgetTracker.record_time_elapsed()` addierte bei jedem Aufruf erneut die + gesamte Zeit seit Run-Start. Der Tracker verbucht nun ausschließlich das + Intervall seit der letzten Messung. Für das lokale 35B-Modell mit ca. + 8 Tokens/s gelten jetzt Zeitlimits von 30 Minuten (Quick), 60 Minuten + (Normal) und 120 Minuten (Deep). + +Validierung: 68 gezielte Tests (`test_backend_pipeline_repairs.py`, +`test_crawler.py`, `test_search.py`) erfolgreich; Backend-Container neu +gebaut und `/health` liefert HTTP 200. Der vollständige neue Research-Run +wurde nicht abgewartet, weil die lokale 35B-Inferenz absichtlich mehrere +Minuten dauern kann. Beim nächsten Test eine frische Recherche starten und +insbesondere Quellen, Claims und Bericht nach Abschluss prüfen. + +Für die laufende Testphase ist es akzeptiert, dass Research-Runs nur im +Arbeitsspeicher liegen und ein API-Neubau vorhandene Run-IDs verwirft. + ### Browser-E2E: Such- und Crawler-Pipeline repariert — 2026-09-07 Während des Browser-E2E-Tests traten drei aufeinanderfolgende Backend-Fehler @@ -26,7 +60,8 @@ auf. Alle sind lokal repariert, durch gezielte Tests abgedeckt und mit die tatsächlichen Downloads deutlich kleiner waren. Abrufe werden jetzt auf verbleibende Quellen- und Bytebudgets begrenzt; die Buchung erfolgt nach tatsächlich abgerufenen Response-Bytes. Budgetobergrenzen sind inklusiv. - Das Quick-Zeitlimit beträgt nun 300 Sekunden (API-Dokumentation angepasst). + Das Quick-Zeitlimit betrug in diesem Zwischenstand 300 Sekunden und wurde + im späteren Budget-Fix für die lokale 35B-Inferenz auf 30 Minuten erhöht. Validierung: 63 gezielte Tests (`test_backend_pipeline_repairs.py`, `test_crawler.py`, `test_search.py`) erfolgreich; Docker-Container und diff --git a/src/nsct/api/rest_research.py b/src/nsct/api/rest_research.py index d992cb9..783354d 100644 --- a/src/nsct/api/rest_research.py +++ b/src/nsct/api/rest_research.py @@ -66,7 +66,7 @@ DEPTH_CONFIGS: dict[str, DepthConfig] = { max_pages_per_domain=2, max_total_download_bytes=500_000, max_llm_requests=15, - max_research_duration_seconds=300, + max_research_duration_seconds=1_800, max_context_per_llm_call=16_000, ), "normal": DepthConfig( @@ -76,7 +76,7 @@ DEPTH_CONFIGS: dict[str, DepthConfig] = { max_pages_per_domain=5, max_total_download_bytes=2_000_000, max_llm_requests=30, - max_research_duration_seconds=300, + max_research_duration_seconds=3_600, max_context_per_llm_call=24_000, ), "deep": DepthConfig( @@ -86,7 +86,7 @@ DEPTH_CONFIGS: dict[str, DepthConfig] = { max_pages_per_domain=10, max_total_download_bytes=5_000_000, max_llm_requests=60, - max_research_duration_seconds=600, + max_research_duration_seconds=7_200, max_context_per_llm_call=32_000, ), } diff --git a/src/nsct/crawler/manager.py b/src/nsct/crawler/manager.py index e28e62c..9ba10ea 100644 --- a/src/nsct/crawler/manager.py +++ b/src/nsct/crawler/manager.py @@ -167,7 +167,10 @@ class CrawlerManager: @staticmethod def _error_doc(url: str, error: str) -> NormalizedDocument: """Create a NormalizedDocument representing an error.""" - doc = NormalizedDocument( + # Keep the error result structurally identical to a successfully + # normalized document. ``from_text`` hashes the (empty) content, so + # downstream consumers can safely rely on content_hash being present. + doc = NormalizedDocument.from_text( url=url, text="", title="", @@ -176,7 +179,6 @@ class CrawlerManager: "error": error, "content_type": "error", }, - content_hash="", extraction_tool="", ) return doc diff --git a/src/nsct/orchestration/budget.py b/src/nsct/orchestration/budget.py index 22c4f0c..6928eb2 100644 --- a/src/nsct/orchestration/budget.py +++ b/src/nsct/orchestration/budget.py @@ -74,6 +74,7 @@ class BudgetTracker: def __init__(self, config: HardBudgetConfig) -> None: self._config = config self._start_time = time.monotonic() + self._last_time_recorded = self._start_time # internal counters self._counters: dict[str, int | float] = { "search_queries": 0, @@ -132,12 +133,11 @@ class BudgetTracker: def record_time_elapsed(self) -> None: """Tick the internal elapsed-time clock. - Adds the time since the tracker was created (or the last call) - to the elapsed duration. + Adds only the time since the previous call to the elapsed duration. """ - self.update_research_duration( - time.monotonic() - self._start_time - ) + now = time.monotonic() + self.update_research_duration(now - self._last_time_recorded) + self._last_time_recorded = now def increment_context_tokens(self, n: int) -> None: """Add *n* tokens to the current LLM context window counter.""" diff --git a/src/nsct/orchestration/orchestrator.py b/src/nsct/orchestration/orchestrator.py index cdd6dc3..ab54c37 100644 --- a/src/nsct/orchestration/orchestrator.py +++ b/src/nsct/orchestration/orchestrator.py @@ -345,8 +345,8 @@ class ResearchOrchestrator: ResearchRun Das erstellte Run-Objekt. """ - # Budget init - self._budget_tracker.increment_llm_requests(1) # planner call + # The planner call itself is accounted for in ``_step_planning``. + # Starting a run performs no LLM request. self._budget_tracker.increment_search(1) # initial search plan # ResearchRun erstellen @@ -724,7 +724,26 @@ class ResearchOrchestrator: self._claims = [] return {"success": True, "data": {"claims": []}, "claim_count": 0} - self._budget_tracker.increment_llm_requests(len(self._sources)) + # Error documents are retained in ``_sources`` for transparent + # reporting, but they must never consume a claim-extraction LLM + # request. Reserve one request for the synthesis step as well, + # otherwise a quick run can use its entire LLM budget here and + # fail deterministically before synthesis. + extractable_sources = [source for source in self._sources if not source.get("error")] + usage = self._budget_tracker.get_usage() + remaining_llm_requests = self._budget_config.max_llm_requests - int( + usage["max_llm_requests"]["usage"] + ) + synthesis_reserve = 1 + max_sources_for_extraction = max(0, remaining_llm_requests - synthesis_reserve) + extractable_sources = extractable_sources[:max_sources_for_extraction] + + if not extractable_sources: + logger.warning("No successful sources or LLM budget for claim extraction") + self._claims = [] + return {"success": True, "data": {"claims": []}, "claim_count": 0} + + self._budget_tracker.increment_llm_requests(len(extractable_sources)) # Stage 5: Claim Extraction try: @@ -734,7 +753,7 @@ class ResearchOrchestrator: llm_provider=llm_provider, config=self._config, research_run_id=self._run.id if self._run else uuid4(), - sources=self._sources, + sources=extractable_sources, ) self._claims = await extractor.extract() except NameError: diff --git a/tests/test_backend_pipeline_repairs.py b/tests/test_backend_pipeline_repairs.py index f56e2ff..47dcca6 100644 --- a/tests/test_backend_pipeline_repairs.py +++ b/tests/test_backend_pipeline_repairs.py @@ -6,17 +6,18 @@ import asyncio from uuid import uuid4 import httpx -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch from nsct.config import AppSettings, AudioConfig, DatabaseConfig, LLMConfig, VisionConfig from nsct.orchestration.orchestrator import ResearchOrchestrator from nsct.orchestration.budget import BudgetExhaustedError, BudgetTracker, HardBudgetConfig -from nsct.orchestration.state import ResearchRunState +from nsct.orchestration.state import ResearchRunState, StateMachine from nsct.providers.abstract import NormalizedResult from nsct.providers.llm import _LLMProviderImpl from nsct.providers.metrics import ProviderMetrics from nsct.providers.searxng import SearXNGProvider from nsct.crawler.normalize import NormalizedDocument +from nsct.api.rest_research import DEPTH_CONFIGS def _config(*, searxng_base_url: str | None = "http://searxng:8080") -> AppSettings: @@ -149,6 +150,70 @@ def test_hard_budget_allows_exactly_the_configured_limit() -> None: raise AssertionError("Source usage above the limit must exhaust the budget") +def test_time_tracking_records_only_each_new_elapsed_interval() -> None: + with patch("nsct.orchestration.budget.time.monotonic", side_effect=[100.0, 110.0, 125.0]): + tracker = BudgetTracker(HardBudgetConfig()) + tracker.record_time_elapsed() + tracker.record_time_elapsed() + + usage = tracker.get_usage() + assert usage["max_research_duration_seconds"]["usage"] == 25.0 + + +def test_depth_time_budgets_allow_local_35b_inference() -> None: + assert DEPTH_CONFIGS["quick"].max_research_duration_seconds == 1_800 + assert DEPTH_CONFIGS["normal"].max_research_duration_seconds == 3_600 + assert DEPTH_CONFIGS["deep"].max_research_duration_seconds == 7_200 + + +def test_start_does_not_charge_a_phantom_planner_request() -> None: + async def run() -> None: + orchestrator = ResearchOrchestrator(_config(), uuid4(), "test") + + await orchestrator.start() + + usage = orchestrator.budget_tracker.get_usage() + assert usage["max_llm_requests"]["usage"] == 0 + + asyncio.run(run()) + + +def test_claim_extraction_reserves_synthesis_budget_and_skips_failed_sources() -> None: + async def run() -> None: + budget = HardBudgetConfig(max_sources=15, max_llm_requests=15) + orchestrator = ResearchOrchestrator(_config(), uuid4(), "test", budget_config=budget) + await orchestrator.start() + orchestrator._state_machine = StateMachine(ResearchRunState.FETCHING) + orchestrator._budget_tracker.increment_llm_requests() # completed planning call + orchestrator._llm_provider = Mock() + orchestrator._sources = [ + {"id": str(index), "url": f"https://example.org/{index}", "error": ""} + for index in range(14) + ] + [{"id": "failed", "url": "https://example.org/failed", "error": "timeout"}] + captured: dict[str, object] = {} + + class FakeExtractor: + def __init__(self, **kwargs: object) -> None: + captured["sources"] = kwargs["sources"] + + async def extract(self) -> list[object]: + return [] + + with patch("nsct.stages.stage5_extract_claims.Stage5Extractor", FakeExtractor): + response = await orchestrator._step_extracting() + + assert response["success"] is True + assert len(captured["sources"]) == 13 + usage = orchestrator.budget_tracker.get_usage() + assert usage["max_llm_requests"]["usage"] == 14 + + # The reserved final request reaches, but does not exceed, the quick limit. + orchestrator._budget_tracker.increment_llm_requests() + orchestrator.budget_tracker.check_budget() + + asyncio.run(run()) + + def test_pipeline_executes_extracting_between_fetching_and_analyzing() -> None: async def run() -> None: orchestrator = ResearchOrchestrator(_config(), uuid4(), "test") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 19b5f7d..8d4ab34 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -11,6 +11,7 @@ import pytest from nsct.crawler.extraction import extract_main_content from nsct.crawler.fetcher import AsyncFetcher, FetchResult, FetchStatus +from nsct.crawler.manager import CrawlerManager from nsct.crawler.normalize import NormalizedDocument from nsct.crawler.policy import ( CrawlerPolicyError, @@ -209,6 +210,15 @@ async def test_fetcher_unreachable_domain() -> None: await fetcher.close() +def test_error_document_has_hash_for_empty_content() -> None: + """A failed fetch must still yield a valid normalized document.""" + doc = CrawlerManager._error_doc("https://example.com/unavailable", "connection refused") + + assert doc.text == "" + assert doc.metadata["error"] == "connection refused" + assert doc.content_hash == hashlib.sha256(b"").hexdigest() + + # --------------------------------------------------------------------------- # NormalizedDocument has all required fields # ---------------------------------------------------------------------------