stage22: Abschluss & Production Readiness — E2E-Tests, CHANGELOG, Dokumentation

tests/stages/test_e2e_integration.py

@@ -0,0 +1,432 @@
+"""End-to-End Integration Test for the full research pipeline (Stage 22).
+
+Tests the complete lifecycle from POST /v1/research → background pipeline →
+GET /v1/research/{id}/report with mocked LLM and search providers so the test
+is deterministic and fast — no network calls, no real LLM needed.
+"""
+
+from __future__ import annotations
+
+from typing import Generator
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture()
+def client() -> Generator[TestClient, None, None]:
+    """Synchronous FastAPI TestClient."""
+    from nsct.api.main import create_app
+
+    app = create_app()
+    with TestClient(app) as c:
+        yield c
+
+
+@pytest.fixture(autouse=True)
+def clear_store(client: TestClient) -> None:
+    """Clear the in-memory research store before every test."""
+    from nsct.api.rest_research import _research_store
+
+    _research_store.clear()
+
+
+# ---------------------------------------------------------------------------
+# Helper — build a mock orchestration result that mimics a successful run
+# ---------------------------------------------------------------------------
+
+
+def _mock_successful_pipeline() -> dict:
+    """Return a mock orchestrator.run() result for a successful research run."""
+    return {
+        "success": True,
+        "report": {
+            "summary": "Zusammenfassung der Recherche.",
+            "findings": [
+                {"text": "Fundamentale Erkenntnis A", "confidence": 0.85, "source_ids": ["s1"]},
+                {"text": "Erkenntnis B bestätigt A", "confidence": 0.72, "source_ids": ["s2"]},
+            ],
+            "disagreements": [
+                {
+                    "claim": "Behauptung X",
+                    "positions": [
+                        {"source": "Quelle 1", "view": "Für X"},
+                        {"source": "Quelle 2", "view": "Gegen X"},
+                    ],
+                }
+            ],
+            "uncertainties": ["Nicht ausreichend belegte Behauptung Y"],
+            "source_statistics": {"total_sources": 5, "primary_sources": 2, "secondary_sources": 3},
+            "methodology": "NSCT Evidence Pipeline — Search, Extract, Claim, Compare, Synthesize",
+            "generated_at": "2025-01-01T00:00:00+00:00",
+            "claims": [
+                {
+                    "id": "c1",
+                    "research_run_id": "run-1",
+                    "source_id": "s1",
+                    "claim_text": "Behauptung A",
+                    "evidence_span": "Quelle 1, Abschnitt 2",
+                    "claim_type": "factual",
+                    "confidence": 0.85,
+                },
+            ],
+            "evidence": [
+                {
+                    "claim_id": "c1",
+                    "research_run_id": "run-1",
+                    "evidence_type": "primary_report",
+                    "source_independence_score": 0.9,
+                    "primary_source_proximity": 0.95,
+                    "cross_source_support": 0.7,
+                    "contradiction_level": 0.2,
+                    "evidence_directness": 0.88,
+                    "date_relevance_score": 0.95,
+                },
+            ],
+        },
+        "sources": [
+            {"id": "s1", "url": "https://example.com/1", "title": "Beispiel 1", "domain": "example.com", "source_type": "primary"},
+            {"id": "s2", "url": "https://example.com/2", "title": "Beispiel 2", "domain": "example.com", "source_type": "secondary"},
+        ],
+    }
+
+
+# ---------------------------------------------------------------------------
+# Test 1 — POST /v1/research returns 200 and creates a research run
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_01_post_research_returns_200_and_creates_run(client: TestClient) -> None:
+    """Step 1: POST /v1/research creates a research run and returns research_id."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        resp = client.post(
+            "/v1/research",
+            json={
+                "query": "Welche Entwicklungen gab es bei Kernfusion?",
+                "language": "de",
+                "depth": "normal",
+            },
+        )
+
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["status"] == "pending"
+    assert body["query"] == "Welche Entwicklungen gab es bei Kernfusion?"
+    assert body["depth"] == "normal"
+    assert "research_id" in body
+    research_id = body["research_id"]
+    assert len(research_id) == 36  # UUID length
+
+    # Verify entry in in-memory store
+    from nsct.api.rest_research import _research_store
+
+    assert research_id in _research_store
+    run = _research_store[research_id]
+    assert run.query == "Welche Entwicklungen gab es bei Kernfusion?"
+    assert run.language == "de"
+    assert run.depth == "normal"
+
+
+# ---------------------------------------------------------------------------
+# Test 2 — Background pipeline completes and report is available
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_02_full_pipeline_until_report_available(client: TestClient) -> None:
+    """Step 2-9: Background pipeline simulates a full run; report endpoint returns data."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        resp = client.post(
+            "/v1/research",
+            json={
+                "query": "Kernfusion 2025",
+                "language": "de",
+                "depth": "deep",
+            },
+        )
+        research_id = resp.json()["research_id"]
+
+    # After background pipeline completes, report should have findings
+    from nsct.api.rest_research import _research_store
+    run = _research_store[research_id]
+
+    # Verify report data was stored
+    assert run.report is not None
+    assert run.report.get("summary") == "Zusammenfassung der Recherche."
+    assert len(run.report["findings"]) == 2
+    assert len(run.report["disagreements"]) == 1
+    assert len(run.report["uncertainties"]) == 1
+
+
+# ---------------------------------------------------------------------------
+# Test 3 — GET /v1/research/{id}/report returns structured findings
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_03_get_report_returns_structured_findings(client: TestClient) -> None:
+    """Step 10: GET report endpoint returns all expected fields."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        client.post(
+            "/v1/research",
+            json={"query": "Kernfusion", "language": "de", "depth": "normal"},
+        )
+
+    # Fetch the report via the public API
+    from nsct.api.rest_research import _research_store
+    research_id = list(_research_store.keys())[0]
+
+    resp = client.get(f"/v1/research/{research_id}/report")
+    assert resp.status_code == 200
+    body = resp.json()
+
+    assert body["research_id"] == research_id
+    assert "summary" in body and body["summary"]
+    assert "findings" in body
+    assert len(body["findings"]) > 0
+    for finding in body["findings"]:
+        assert "text" in finding
+        assert "confidence" in finding
+        assert "source_ids" in finding
+    assert "disagreements" in body
+    assert "uncertainties" in body
+    assert "source_statistics" in body
+    assert "methodology" in body and body["methodology"]
+    assert "generated_at" in body
+
+
+# ---------------------------------------------------------------------------
+# Test 4 — GET /v1/research/{id}/sources, /claims, /evidence return data
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_04_sources_claims_evidence_endpoints(client: TestClient) -> None:
+    """Steps 3-8: Sources, claims, and evidence endpoints return correct data."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        client.post(
+            "/v1/research",
+            json={"query": "Fusion", "language": "de", "depth": "normal"},
+        )
+
+    from nsct.api.rest_research import _research_store
+    research_id = list(_research_store.keys())[0]
+
+    # --- Sources ---
+    resp = client.get(f"/v1/research/{research_id}/sources")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["research_id"] == research_id
+    assert body["total"] == 2
+    assert len(body["sources"]) == 2
+    for src in body["sources"]:
+        assert "id" in src
+        assert "url" in src
+        assert "title" in src
+        assert "domain" in src
+
+    # --- Claims ---
+    resp = client.get(f"/v1/research/{research_id}/claims")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["research_id"] == research_id
+    assert body["total"] == 1
+    assert len(body["claims"]) == 1
+    claim = body["claims"][0]
+    assert "claim_text" in claim
+    assert "evidence_span" in claim
+    assert "claim_type" in claim
+    assert "confidence" in claim
+
+    # --- Evidence ---
+    resp = client.get(f"/v1/research/{research_id}/evidence")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["research_id"] == research_id
+    assert body["total"] == 1
+    assert len(body["evidence"]) == 1
+    evidence = body["evidence"][0]
+    assert "claim_id" in evidence
+    assert "source_independence_score" in evidence
+    assert "primary_source_proximity" in evidence
+    assert "cross_source_support" in evidence
+    assert "contradiction_level" in evidence
+    assert "evidence_directness" in evidence
+    assert "date_relevance_score" in evidence
+
+
+# ---------------------------------------------------------------------------
+# Test 5 — Status endpoint reflects COMPLETED state
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_05_status_reflects_completed_state(client: TestClient) -> None:
+    """After pipeline completion, status shows COMPLETED with counts."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        client.post(
+            "/v1/research",
+            json={"query": "Status test", "language": "de", "depth": "normal"},
+        )
+
+    from nsct.api.rest_research import _research_store
+    research_id = list(_research_store.keys())[0]
+
+    resp = client.get(f"/v1/research/{research_id}/status")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["research_id"] == research_id
+    assert body["state"] == "completed"
+    assert body["source_count"] == 2
+    assert body["claim_count"] == 1
+    assert body["is_completed"] is True
+    assert body["is_running"] is False
+
+
+# ---------------------------------------------------------------------------
+# Test 6 — List endpoint aggregates all research runs
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_06_list_aggregates_all_runs(client: TestClient) -> None:
+    """GET /v1/research returns all completed researches with correct counts."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        for i in range(3):
+            client.post(
+                "/v1/research",
+                json={"query": f"Query {i}", "language": "de", "depth": "quick"},
+            )
+
+    from nsct.api.rest_research import _research_store
+
+    resp = client.get("/v1/research")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["total"] == 3
+    assert len(body["items"]) == 3
+    for item in body["items"]:
+        assert "research_id" in item
+        assert "query" in item
+        assert "state" in item
+        assert "source_count" in item
+        assert "claim_count" in item
+
+
+# ---------------------------------------------------------------------------
+# Test 7 — Error handling: non-existent research returns 404
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_07_nonexistent_research_returns_404(client: TestClient) -> None:
+    """All individual research endpoints return 404 for unknown IDs."""
+    endpoints = [
+        "/v1/research/nonexistent-id",
+        "/v1/research/nonexistent-id/status",
+        "/v1/research/nonexistent-id/sources",
+        "/v1/research/nonexistent-id/claims",
+        "/v1/research/nonexistent-id/evidence",
+        "/v1/research/nonexistent-id/report",
+    ]
+
+    for ep in endpoints:
+        resp = client.get(ep)
+        assert resp.status_code == 404, f"Expected 404 for {ep}, got {resp.status_code}"
+
+
+# ---------------------------------------------------------------------------
+# Test 8 — Delete only works on non-completed runs
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_08_delete_completed_returns_400(client: TestClient) -> None:
+    """DELETE on a COMPLETED research run returns 400 (immutable)."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(return_value=_mock_successful_pipeline())
+        MockOrch.return_value = mock_instance
+
+        client.post(
+            "/v1/research",
+            json={"query": "Delete test", "language": "de", "depth": "normal"},
+        )
+
+    from nsct.api.rest_research import _research_store, ResearchRunState
+
+    research_id = list(_research_store.keys())[0]
+    run = _research_store[research_id]
+
+    # Manually set to completed so the test is deterministic
+    run.state = ResearchRunState.COMPLETED.value
+
+    resp = client.delete(f"/v1/research/{research_id}")
+    assert resp.status_code == 400
+    body = resp.json()
+    assert "detail" in body
+
+
+# ---------------------------------------------------------------------------
+# Test 9 — Pipeline failure is handled gracefully
+# ---------------------------------------------------------------------------
+
+
+def test_e2e_09_failed_pipeline_returns_failed_state(client: TestClient) -> None:
+    """When the orchestrator.run() raises, the run state becomes FAILED."""
+    with patch("nsct.orchestration.orchestrator.ResearchOrchestrator") as MockOrch:
+        mock_instance = AsyncMock()
+        mock_instance.start = AsyncMock()
+        mock_instance.run = AsyncMock(
+            side_effect=RuntimeError("LLM service unavailable")
+        )
+        MockOrch.return_value = mock_instance
+
+        resp = client.post(
+            "/v1/research",
+            json={"query": "Fehler test", "language": "de", "depth": "quick"},
+        )
+        research_id = resp.json()["research_id"]
+
+    from nsct.api.rest_research import _research_store, ResearchRunState
+
+    run = _research_store.get(research_id)
+    assert run is not None
+    assert run.state == ResearchRunState.FAILED.value
+    assert "error" in run.metadata
+
+    # Status endpoint should show FAILED
+    resp = client.get(f"/v1/research/{research_id}/status")
+    assert resp.status_code == 200
+    assert resp.json()["state"] == "failed"
+    assert resp.json()["is_completed"] is False