feat(pipeline): produce evidence scores and complete fallback reports
Connect source-independence analysis and Stage 8 evidence scoring to the research orchestrator so completed runs retain a transparent score for every extracted claim. Forward those scores into Stage 9's evidence package, allowing synthesis to reason from actual provenance and exposing them through the research API. Replace the previously technical-only synthesis fallback with the report contract consumed by the UI: a summary, source statistics, methodology, and provenance-preserving uncertain findings. Add regression coverage for score generation, score hand-off to Stage 9, report methodology, and visible fallback findings; record the healthy container deployment in the hand-off.
This commit is contained in:
@@ -252,16 +252,81 @@ def test_orchestrator_synthesis_returns_a_report_for_uuid_claims() -> None:
|
||||
"contradictions": [],
|
||||
}))
|
||||
orchestrator._llm_provider = provider
|
||||
orchestrator._comparison_data = {
|
||||
"evidence_scores": {
|
||||
str(orchestrator._claims[0].id): {
|
||||
"claim_id": str(orchestrator._claims[0].id),
|
||||
"evidence_type": "direct_observation",
|
||||
"source_independence_score": 1.0,
|
||||
"cross_source_support": 0.0,
|
||||
"contradiction_level": 1.0,
|
||||
"evidence_directness": 1.0,
|
||||
"date_relevance_score": 0.5,
|
||||
"primary_source_proximity": 0.3,
|
||||
"relation_links": [],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = await orchestrator._step_synthesizing()
|
||||
|
||||
assert response["success"] is True
|
||||
assert response["report"]["summary"] == "Neutraler Bericht."
|
||||
assert response["report"]["methodology"]
|
||||
assert '"evidence_type": "direct_observation"' in provider.complete.await_args.kwargs["messages"][1]["content"]
|
||||
provider.complete.assert_awaited_once()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_comparing_creates_evidence_scores_for_extracted_claims() -> None:
|
||||
async def run() -> None:
|
||||
orchestrator = ResearchOrchestrator(_config(), uuid4(), "Testthema")
|
||||
await orchestrator.start()
|
||||
orchestrator._state_machine = StateMachine(ResearchRunState.ANALYZING)
|
||||
claim = Claim(
|
||||
research_run_id=uuid4(),
|
||||
source_id=uuid4(),
|
||||
claim_text="Die Untersuchung beobachtete einen Anstieg um 20 Prozent.",
|
||||
evidence_span="beobachtete einen Anstieg um 20 Prozent",
|
||||
claim_type=ClaimType.FACT,
|
||||
source_url="https://example.org/source",
|
||||
)
|
||||
orchestrator._claims = [claim]
|
||||
orchestrator._source_independence_data = {
|
||||
str(claim.source_id): {"independence_score": 1.0}
|
||||
}
|
||||
|
||||
response = await orchestrator._step_comparing()
|
||||
|
||||
assert response["success"] is True
|
||||
assert response["evidence_count"] == 1
|
||||
score = orchestrator._comparison_data["evidence_scores"][str(claim.id)]
|
||||
assert score["source_independence_score"] == 1.0
|
||||
assert score["claim_id"] == str(claim.id)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_fallback_report_has_visible_findings_and_methodology() -> None:
|
||||
orchestrator = ResearchOrchestrator(_config(), uuid4(), "Testthema")
|
||||
claim = Claim(
|
||||
research_run_id=uuid4(),
|
||||
source_id=uuid4(),
|
||||
claim_text="Eine überprüfbare Behauptung.",
|
||||
evidence_span="Die belegende Passage.",
|
||||
claim_type=ClaimType.CLAIM,
|
||||
source_url="https://example.org/source",
|
||||
)
|
||||
orchestrator._claims = [claim]
|
||||
|
||||
report = orchestrator._get_report()
|
||||
|
||||
assert report["methodology"]
|
||||
assert report["summary"]
|
||||
assert report["uncertain_areas"][0]["claim_text"] == claim.claim_text
|
||||
|
||||
|
||||
def test_start_does_not_charge_a_phantom_planner_request() -> None:
|
||||
async def run() -> None:
|
||||
orchestrator = ResearchOrchestrator(_config(), uuid4(), "test")
|
||||
|
||||
Reference in New Issue
Block a user