diff --git a/HANDOFF.md b/HANDOFF.md index eddba17..d16795e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -7,6 +7,35 @@ ## Aktueller Stand — 2026-09-07 +### Evidenzbewertung und sichtbarer Fallback-Bericht in die Pipeline integriert — 2026-09-07 + +Ein Browser-Run hatte bereits abgerufene Quellen, aber keine Evidenz, keinen +sichtbaren Bericht und keine Methodik. Ursache war kein Frontend-Vertrag: +`analyzing` und `comparing` im Orchestrator waren noch Platzhalter. Damit +blieben Stage 6 (Quellenunabhängigkeit) und Stage 8 (Evidenzscores) außerhalb +des produktiven Pipeline-Pfads; Stage 9 erhielt folglich auch kein Evidence +Package. + +- `analyzing` führt jetzt Stage 6 aus. Fehler in der optionalen + Ähnlichkeitsanalyse degradieren konservativ, statt vorhandene Claims zu + verwerfen. +- `comparing` erzeugt mit Stage 8 je Claim transparente Evidenzscores + (Quellenunabhängigkeit, Quellennähe, Unterstützung, Widerspruch, + Direktheit und Aktualität). Nicht vorhandene Claim-Relationen werden + ausdrücklich als fehlende Korroboration behandelt, nicht erfunden. +- Die Scores werden an Stage 9 übergeben und sind damit sowohl über + `/evidence` als auch im LLM-Evidence-Package verfügbar. +- Der Fallback-Bericht erfüllt nun ebenfalls den REST-/UI-Vertrag: Er liefert + Zusammenfassung, sichtbar als unsicher markierte Claims, Quellenstatistik + und eine Methodik, wenn keine LLM-Synthese möglich ist. + +Validierung und Deployment: 18 gezielte Pipeline-Regressionstests, +Syntaxprüfung und Diff-Check erfolgreich; `docker compose up -d --build +nsct-api` durchgeführt, Container healthy und `/health` HTTP 200. Der +Rebuild verwirft erwartungsgemäß zuvor nur im Speicher gehaltene Run-IDs. +Für die End-to-End-Prüfung jetzt eine frische Recherche starten und nach +Abschluss Quellen, Claims, Evidenz, Bericht und Methodik kontrollieren. + ### Browser-E2E: Leeren Abschluss auf Suchausfall zurückgeführt und repariert — 2026-09-07 Der abgeschlossene Browser-Run `33328b0b-18df-48df-9f2d-8af077987ccc` hatte diff --git a/src/nsct/orchestration/orchestrator.py b/src/nsct/orchestration/orchestrator.py index 3aa0350..275fa46 100644 --- a/src/nsct/orchestration/orchestrator.py +++ b/src/nsct/orchestration/orchestrator.py @@ -47,6 +47,9 @@ try: except ImportError: SynthesisStage = None # type: ignore[misc,assignment] +from nsct.stages.stage6_source_independence import SourceIndependenceAnalyzer +from nsct.stages.stage8_evidence_scoring import Stage8EvidenceScoring + logger = logging.getLogger(__name__) @@ -125,6 +128,12 @@ class ResearchOrchestrator: self._search_results: list[dict[str, Any]] = [] # URLs self._sources: list[dict[str, Any]] = [] self._claims: list[Claim] = [] + self._source_independence_data: dict[str, dict[str, Any]] = {} + self._comparison_data: dict[str, Any] = { + "total_claims": 0, + "comparisons": [], + "evidence_scores": {}, + } # Sub-Components (lazy init) self._planner: ResearchPlanner | MockResearchPlanner | None = None @@ -824,13 +833,37 @@ class ResearchOrchestrator: return {"success": False, "error": f"Claim extraction failed: {exc}", "data": {}} async def _step_analyzing(self) -> dict[str, Any]: - """ANALYZING: Platzhalter — Stage 6/7/8 werden später eingebunden. - - Derzeit: Validiere Claims und sammle Metadaten. - """ + """ANALYZING: Quellenunabhängigkeit und Claim-Metadaten ermitteln.""" self._transition_to("analyzing") - logger.info("ANALYZING step: placeholder — Stage 6/7/8 pending integration") + # Stage 6 is deliberately executed before scoring. It detects exact + # duplicates without an LLM and only consults one for genuinely similar + # source pairs; a failure there must not erase otherwise usable claims. + self._source_independence_data = {} + usable_sources = [source for source in self._sources if not source.get("error")] + if usable_sources: + try: + analysis = await SourceIndependenceAnalyzer( + llm_provider=self._get_llm_provider(), + config=self._config, + sources=usable_sources, + research_run_id=self._run.id if self._run else self._research_id, + ).analyze() + source_by_id = {str(source.get("id")): source for source in usable_sources} + for score in analysis.source_scores: + source = source_by_id.get(str(score.source_id), {}) + metadata = source.get("metadata", {}) if isinstance(source, dict) else {} + self._source_independence_data[str(score.source_id)] = { + "independence_score": score.independence_score, + "parent_source_id": str(score.primary_source_id) if score.primary_source_id else None, + "syndication_group_id": str(score.syndication_group_id) if score.syndication_group_id else None, + "publication_date": metadata.get("publication_date"), + "shared_urls": score.shared_urls, + } + except Exception as exc: + # Scoring has conservative defaults for missing independence + # data, so this is a degradable analysis error, not a failed run. + logger.warning("Source-independence analysis degraded: %s", exc) # Grundlegende Claim-Validierung claim_stats = { @@ -851,6 +884,7 @@ class ResearchOrchestrator: else: claim_stats["by_type"] = {} + claim_stats["source_independence_scored"] = len(self._source_independence_data) self._run_metadata = claim_stats if self._run is not None: self._run = self._run.model_copy( @@ -867,19 +901,26 @@ class ResearchOrchestrator: } async def _step_comparing(self) -> dict[str, Any]: - """COMPARING: Platzhalter — Stage 8 Evidence Scoring. - - Derzeit: Leere Comparison, nur Logging. - """ + """COMPARING: pro Claim transparente, mehrdimensionale Evidenzscores.""" self._transition_to("comparing") - logger.info("COMPARING step: placeholder — Stage 8 Evidence Scoring pending integration") - - # Placeholder: keine Comparison-Daten - self._comparison_data: dict[str, Any] = { + # Stage 8 is deterministic and preserves all component scores. Claim + # relations remain empty until a bounded Stage-7 relation pass is run; + # the resulting 0 cross-source support is therefore explicitly a lack + # of corroboration, never an invented negative finding. + scoring = Stage8EvidenceScoring( + research_run_id=self._run.id if self._run else self._research_id, + claims=self._claims, + cluster_data={"clusters": [], "relations": []}, + source_data_map=self._source_independence_data, + ).run() + scores = scoring["scores"] + self._comparison_data = { "total_claims": len(self._claims), "comparisons": [], - "evidence_scores": {}, + "evidence_scores": {score["claim_id"]: score for score in scores}, + "summary": scoring["summary"], + "errors": scoring["errors"], } if self._run is not None: @@ -893,6 +934,7 @@ class ResearchOrchestrator: return { "success": True, "data": self._comparison_data, + "evidence_count": len(scores), } async def _step_synthesizing(self) -> dict[str, Any]: @@ -922,6 +964,7 @@ class ResearchOrchestrator: llm_provider=llm_provider, config=self._config, claims=claims_dicts, + evidence_scores=list(self._comparison_data.get("evidence_scores", {}).values()), ) result = await stage.execute(topic=self._query) @@ -1010,9 +1053,39 @@ class ResearchOrchestrator: def _get_report(self) -> dict[str, Any]: """Erzeuge den finalen Report aus allen Zwischenspeichern.""" + # This is also the public fallback-report contract when the synthesis + # model is unavailable. It must remain useful to the REST/UI layer, + # rather than exposing only technical pipeline internals. + uncertain_findings = [ + { + "claim_text": claim.claim_text, + "source_id": str(claim.source_id), + "source_url": claim.source_url, + "evidence_span": claim.evidence_span, + "confidence": claim.confidence, + } + for claim in self._claims + ] report: dict[str, Any] = { "run_id": str(self._run.id) if self._run else None, "query": self._query, + "summary": ( + "Die automatische Synthese war nicht verfügbar. " + f"{len(self._claims)} extrahierte Claims werden mit ihren " + "Quellen als unsicher ausgewiesen." + ), + "confident_findings": [], + "uncertain_areas": uncertain_findings, + "contradictions": [], + "source_statistics": {"total": len(self._sources)}, + "methodology": ( + "NSCT-Fallback-Bericht: Quellen wurden gesucht und abgerufen; " + "Claims wurden mit ihren Textbelegen erhalten. Evidenzscores " + "berücksichtigen Quellenunabhängigkeit, Quellennähe, " + "quellenübergreifende Unterstützung, Widersprüche, Direktheit " + "und zeitliche Relevanz. Ohne LLM-Synthese werden alle Claims " + "als unsicher dargestellt." + ), "state": self._state_machine.current_state.value, "plan": self._plan, "search_results": self._search_results, diff --git a/tests/test_backend_pipeline_repairs.py b/tests/test_backend_pipeline_repairs.py index d8bdcef..d867e2a 100644 --- a/tests/test_backend_pipeline_repairs.py +++ b/tests/test_backend_pipeline_repairs.py @@ -252,16 +252,81 @@ def test_orchestrator_synthesis_returns_a_report_for_uuid_claims() -> None: "contradictions": [], })) orchestrator._llm_provider = provider + orchestrator._comparison_data = { + "evidence_scores": { + str(orchestrator._claims[0].id): { + "claim_id": str(orchestrator._claims[0].id), + "evidence_type": "direct_observation", + "source_independence_score": 1.0, + "cross_source_support": 0.0, + "contradiction_level": 1.0, + "evidence_directness": 1.0, + "date_relevance_score": 0.5, + "primary_source_proximity": 0.3, + "relation_links": [], + } + } + } response = await orchestrator._step_synthesizing() assert response["success"] is True assert response["report"]["summary"] == "Neutraler Bericht." + assert response["report"]["methodology"] + assert '"evidence_type": "direct_observation"' in provider.complete.await_args.kwargs["messages"][1]["content"] provider.complete.assert_awaited_once() asyncio.run(run()) +def test_comparing_creates_evidence_scores_for_extracted_claims() -> None: + async def run() -> None: + orchestrator = ResearchOrchestrator(_config(), uuid4(), "Testthema") + await orchestrator.start() + orchestrator._state_machine = StateMachine(ResearchRunState.ANALYZING) + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Die Untersuchung beobachtete einen Anstieg um 20 Prozent.", + evidence_span="beobachtete einen Anstieg um 20 Prozent", + claim_type=ClaimType.FACT, + source_url="https://example.org/source", + ) + orchestrator._claims = [claim] + orchestrator._source_independence_data = { + str(claim.source_id): {"independence_score": 1.0} + } + + response = await orchestrator._step_comparing() + + assert response["success"] is True + assert response["evidence_count"] == 1 + score = orchestrator._comparison_data["evidence_scores"][str(claim.id)] + assert score["source_independence_score"] == 1.0 + assert score["claim_id"] == str(claim.id) + + asyncio.run(run()) + + +def test_fallback_report_has_visible_findings_and_methodology() -> None: + orchestrator = ResearchOrchestrator(_config(), uuid4(), "Testthema") + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Eine überprüfbare Behauptung.", + evidence_span="Die belegende Passage.", + claim_type=ClaimType.CLAIM, + source_url="https://example.org/source", + ) + orchestrator._claims = [claim] + + report = orchestrator._get_report() + + assert report["methodology"] + assert report["summary"] + assert report["uncertain_areas"][0]["claim_text"] == claim.claim_text + + def test_start_does_not_charge_a_phantom_planner_request() -> None: async def run() -> None: orchestrator = ResearchOrchestrator(_config(), uuid4(), "test")