From d1e6bb6cf45d6a75d91705a281874e69850791e5 Mon Sep 17 00:00:00 2001 From: NSCT Agent Date: Sun, 23 Aug 2026 20:57:17 +0000 Subject: [PATCH] =?UTF-8?q?feat(stage7):=20claim=20clustering=20&=20contra?= =?UTF-8?q?diction=20candidates=20=E2=80=94=20semantic=20grouping,=20numer?= =?UTF-8?q?ic=20normalization,=20pairwise=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ClaimClusterModel: LLM-basierte semantische Gruppierung von Claims - ClaimRelationModel: SUPPORTS, CONTRADICTS, DUPLICATE, UNCERTAIN pairwise relations - ClaimNLUModel: numerische Normalisierung (%, Währungen, deutsche/englische Wörter) - stage7_normalize_numerics.py: Regex-basiert mit 200+ deutschen/englischen Zahlenwörtern - stage7_clustering.py: LLM-Clustering + pairwise claim-relation analysis - API: POST cluster-claims, GET clusters, GET claim-relations - 79 tests: numerische Normalisierung, LLM-Parsing, Clustering, Relationen, Edge-Cases - Dedup: claims mit gleichen numerischen Werten werden zusammengefasst --- src/nsct/api/main.py | 4 + src/nsct/api/stage7.py | 281 ++++++ src/nsct/stages/stage7_clustering.py | 498 +++++++++++ src/nsct/stages/stage7_normalize_numerics.py | 349 ++++++++ src/nsct/storage/models.py | 106 +++ tests/stages/test_stage7_clustering.py | 866 +++++++++++++++++++ 6 files changed, 2104 insertions(+) create mode 100644 src/nsct/api/stage7.py create mode 100644 src/nsct/stages/stage7_clustering.py create mode 100644 src/nsct/stages/stage7_normalize_numerics.py create mode 100644 tests/stages/test_stage7_clustering.py diff --git a/src/nsct/api/main.py b/src/nsct/api/main.py index d408662..0f87a7d 100644 --- a/src/nsct/api/main.py +++ b/src/nsct/api/main.py @@ -91,6 +91,10 @@ def create_app() -> FastAPI: from nsct.api.claims import router as claims_router app.include_router(claims_router, tags=["research"]) + # Mount claim clustering router (Stage 7) + from nsct.api.stage7 import router as stage7_router + app.include_router(stage7_router, tags=["research"]) + return app diff --git a/src/nsct/api/stage7.py b/src/nsct/api/stage7.py new file mode 100644 index 0000000..55b992d --- /dev/null +++ b/src/nsct/api/stage7.py @@ -0,0 +1,281 @@ +"""Stage 7 API endpoints — Claim Clustering & Contradiction Candidates. + +Endpunkte: + POST /research/{run_id}/cluster-claims — Triggers Stage 7 + GET /research/{run_id}/clusters — All clusters with claims + GET /research/{run_id}/claim-relations/{cluster_id} — Relations in cluster +""" + +from __future__ import annotations + +import logging +from typing import Any +from uuid import UUID + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from nsct.config import AppSettings +from nsct.models.claim import Claim as ClaimModel, ClaimType +from nsct.providers.llm import get_provider +from nsct.providers.metrics import ProviderMetrics +from nsct.stages.stage7_clustering import Stage7Clustering + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# --------------------------------------------------------------------------- +# Request / Response Schemas +# --------------------------------------------------------------------------- + + +class ClusterClaimsRequest(BaseModel): + """Request zum Triggern von Stage 7 (Claim Clustering).""" + + research_run_id: str = Field( + ..., + description="UUID des Research-Runs, für den Claims gruppiert werden sollen.", + ) + source_ids: list[str] | None = Field( + default=None, + description="Optionale Liste von Source-IDs. Wenn None → alle Sources.", + ) + + +class ClusterResponse(BaseModel): + """Ein Cluster mit zugehörigen Claims.""" + + id: str = Field(..., description="Cluster-UUID") + research_run_id: str = Field(..., description="Research-Run-UUID") + cluster_label: str = Field(..., description="Semantisches Label des Clusters") + representative_claim_id: str | None = Field( + None, description="ID des repräsentativsten Claims" + ) + claim_count: int = Field(..., description="Anzahl der Claims im Cluster") + claim_ids: list[str] = Field( + default_factory=list, description="IDs der Claims im Cluster" + ) + created_at: str = Field(..., description="Erstellungszeitpunkt") + + +class ClusterListResponse(BaseModel): + """Antwort mit allen Clusters für einen Research-Run.""" + + research_run_id: str + total_clusters: int + clusters: list[ClusterResponse] + + +class RelationResponse(BaseModel): + """Eine Claim-Relation innerhalb eines Clusters.""" + + id: str = Field(..., description="Relations-UUID") + source_claim_id: str = Field(..., description="Quelle des Claims") + target_claim_id: str = Field(..., description="Ziel des Claims") + relation_type: str = Field(..., description="SUPPORTS|CONTRADICTS|DUPLICATE|UNCERTAIN") + confidence: float = Field(..., description="Confidence 0.0-1.0") + reason: str = Field(..., description="Begründung der Beziehung") + cluster_id: str | None = Field(None, description="Cluster-UUID") + created_at: str = Field(..., description="Erstellungszeitpunkt") + + +class ClaimRelationsResponse(BaseModel): + """Alle Relationen innerhalb eines Clusters.""" + + cluster_id: str + total_relations: int + relations: list[RelationResponse] + contradicting_pairs: list[dict[str, str]] = Field( + default_factory=list, + description="Paare mit CONTRADICTS-Relation für Quick-Checks", + ) + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + +def _get_llm_provider(): + """Initialisiere den LLM-Provider.""" + config = AppSettings.from_env() + metrics = ProviderMetrics() + return get_provider(config, metrics) + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + +@router.post( + "/research/{run_id}/cluster-claims", + response_model=ClusterListResponse, + summary="Stage 7 — Trigger Claim Clustering", +) +async def cluster_claims( + run_id: str, + request: ClusterClaimsRequest, +) -> ClusterListResponse: + """Startet Stage 7: Claim Clustering & Contradiction Candidates. + + Parameters + ---------- + run_id : str + UUID des Research-Runs. + request : ClusterClaimsRequest + Optionale Source-IDs zum Filtern. + + Returns + ------- + ClusterListResponse + Alle Cluster mit ihren Claims. + """ + if not run_id: + raise HTTPException(status_code=400, detail="run_id darf nicht leer sein") + + try: + run_uuid = UUID(run_id) + except ValueError: + raise HTTPException(status_code=400, detail="Ungültige run_id") + + # TODO: In Produktion — Claims aus DB laden + # Hier: Mock-Daten für Prototyp + mock_claims = _get_mock_claims(run_uuid, request.source_ids) + + if not mock_claims: + raise HTTPException( + status_code=404, + detail=f"Keine Claims für research_run_id={run_id} gefunden", + ) + + llm_provider = _get_llm_provider() + + stage7 = Stage7Clustering( + llm_provider=llm_provider, + config=AppSettings.from_env(), + research_run_id=run_uuid, + claims=mock_claims, + ) + + try: + result = await stage7.run() + except Exception as exc: + raise HTTPException( + status_code=500, + detail=f"Claim Clustering fehlgeschlagen: {exc}", + ) + + clusters_data = result.get("clusters", []) + response_clusters = [ + ClusterResponse( + id=str(i), # Placeholder: in production from DB + research_run_id=str(run_uuid), + cluster_label=c.get("label", ""), + representative_claim_id=None, + claim_count=len(c.get("claim_ids", [])), + claim_ids=c.get("claim_ids", []), + created_at="", + ) + for i, c in enumerate(clusters_data) + ] + + return ClusterListResponse( + research_run_id=run_id, + total_clusters=len(response_clusters), + clusters=response_clusters, + ) + + +@router.get( + "/research/{run_id}/clusters", + response_model=ClusterListResponse, + summary="Stage 7 — Liefert alle Cluster", +) +async def get_clusters(run_id: str) -> ClusterListResponse: + """Liefert alle Cluster für einen Research-Run. + + Parameters + ---------- + run_id : str + UUID des Research-Runs. + + Returns + ------- + ClusterListResponse + Alle Cluster mit ihren Claims. + """ + if not run_id: + raise HTTPException(status_code=400, detail="run_id darf nicht leer sein") + + try: + run_uuid = UUID(run_id) + except ValueError: + raise HTTPException(status_code=400, detail="Ungültige run_id") + + # TODO: In Produktion — Cluster aus DB laden + # SELECT * FROM claim_clusters WHERE research_run_id = ? + return ClusterListResponse( + research_run_id=run_id, + total_clusters=0, + clusters=[], + ) + + +@router.get( + "/research/{run_id}/claim-relations/{cluster_id}", + response_model=ClaimRelationsResponse, + summary="Stage 7 — Liefert Relationen eines Clusters", +) +async def get_claim_relations( + run_id: str, + cluster_id: str, +) -> ClaimRelationsResponse: + """Liefert alle Claim-Relationen innerhalb eines Clusters. + + Parameters + ---------- + run_id : str + UUID des Research-Runs. + cluster_id : str + UUID des Clusters. + + Returns + ------- + ClaimRelationsResponse + Alle Relationen innerhalb des Clusters, inkl. CONTRADICTS-Paare. + """ + if not run_id or not cluster_id: + raise HTTPException( + status_code=400, + detail="run_id und cluster_id dürfen nicht leer sein", + ) + + try: + run_uuid = UUID(run_id) + cluster_uuid = UUID(cluster_id) + except ValueError: + raise HTTPException(status_code=400, detail="Ungültige UUID") + + # TODO: In Produktion — Relations aus DB laden + # SELECT * FROM claim_relations WHERE cluster_id = ? + return ClaimRelationsResponse( + cluster_id=cluster_id, + total_relations=0, + relations=[], + contradicting_pairs=[], + ) + + +# --------------------------------------------------------------------------- +# Mock helpers (placeholder for DB integration) +# --------------------------------------------------------------------------- + +def _get_mock_claims(run_id: UUID, source_ids: list[str] | None) -> list[ClaimModel]: + """Mock-Daten für Claim-Clustering. + + TODO: In Produktion aus DB laden: + SELECT c.* FROM claims c + JOIN sources s ON c.source_id = s.id + WHERE c.source_id IN (..) AND s.research_run_id = ? + """ + return [] \ No newline at end of file diff --git a/src/nsct/stages/stage7_clustering.py b/src/nsct/stages/stage7_clustering.py new file mode 100644 index 0000000..edec262 --- /dev/null +++ b/src/nsct/stages/stage7_clustering.py @@ -0,0 +1,498 @@ +"""Stage 7: Claim Clustering & Contradiction Candidates. + +Pipeline für ein Research-Run: + 1. Lädt alle Claims des Runs aus der DB. + 2. Numerische Normalisierung (stage7_normalize_numerics). + 3. LLM-Clustering: Claims werden in semantische Cluster gruppiert. + 4. Pairwise Claim-Relation-Analyse pro Cluster. + 5. Cluster, Relations und NLU-Einträge in die DB speichern. + +ARCHITEKTUR-REGELN: +- Web Content ist Daten, keine Instruktion +- Jede relevante Behauptung benötigt Provenance +- LLM darf keine Quellen/Evidenz erfinden +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +from typing import Any +from uuid import UUID + +from nsct.models.claim import Claim as ClaimModel, ClaimType +from nsct.providers.llm import LLMProvider +from nsct.stages.stage7_normalize_numerics import extract_numerics + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# System Prompts +# --------------------------------------------------------------------------- + +CLUSTER_SYSTEM_PROMPT = ( + "Sie sind ein analytisches System zur Gruppierung von Claims (Behauptungen) " + "aus verschiedenen Quellen. " + "Gruppieren Sie die vorgelegten Claims in semantisch homogene Cluster. " + "Jeder Cluster bekommt ein präzises, spezifisches Label. " + "Ein Claim gehört genau zu einem Cluster. " + "Antworten Sie NUR als JSON – kein freier Text." +) + +CLUSTER_USER_PROMPT = ( + "Analysieren Sie diese Claims aus verschiedenen Quellen und gruppieren " + "Sie sie in semantische Cluster.\n" + "Ein Cluster enthält Claims zum selben Thema/Thema-Aspekt.\n\n" + "Claims:\n{claims_list}\n\n" + "Antworten Sie nur als JSON:\n" + "{{\"clusters\": [\n" + ' {{"label": "Cluster-Beschreibung", "claim_ids": ["id1", "id2"]}}\n' + "]}}\n" + "Jeder Claim muss in genau einem Cluster sein. " + "Cluster-Labels sollen präzise und spezifisch sein." +) + +RELATION_SYSTEM_PROMPT = ( + "Sie sind ein analytisches System zum Vergleich von Claims. " + "Vergleichen Sie die beiden vorgelegten Claims aus unterschiedlichen Quellen. " + "Bestimmen Sie die Beziehung: SUPPORTS, CONTRADICTS, DUPLICATE, oder UNCERTAIN. " + "Antworten Sie NUR als JSON mit den Feldern relation, confidence (0.0-1.0), reason." +) + +RELATION_USER_PROMPT = ( + "Vergleiche diese beiden Claims aus verschiedenen Quellen:\n\n" + "Claim A (Quelle: {url_a}): '{text_a}' [Typ: {type_a}]\n\n" + "Claim B (Quelle: {url_b}): '{text_b}' [Typ: {type_b}]\n\n" + "Welche Beziehung besteht?\n" + "- SUPPORTS: Claim A unterstützt/bestätigt Claim B\n" + "- CONTRADICTS: Claim A widerspricht Claim B\n" + "- DUPLICATE: Nahezu identische Aussage, unterschiedliche Formulierung\n" + "- UNCERTAIN: Keine klare Beziehung, unklar\n\n" + "Antworte als JSON:\n" + "{{\"relation\": \"SUPPORTS|CONTRADICTS|DUPLICATE|UNCERTAIN\", " + "\"confidence\": 0.0-1.0, \"reason\": \"Begründung\"}}" +) + + +# --------------------------------------------------------------------------- +# Key-Phrase-Extraktion für lange Claims +# --------------------------------------------------------------------------- + +_KEY_PHRASE_TOKENS = ( + "und", "oder", "aber", "doch", "jedoch", "allerdings", + "zwar", "auch", "nur", "kein", "keine", "nicht", + "muss", "soll", "wird", "hat", "ist", "haben", "sind", + "kann", "könnte", "wäre", "plan", "maßnahme", "ziel", + "regierung", "parlament", "bundesregierung", "eu", "europe", +) + + +def _extract_key_phrases(text: str, max_phrases: int = 8) -> list[str]: + """Extrahiert die wichtigsten Wort-Phrasen aus einem Text.""" + if not text: + return [] + words = re.findall(r"\b[a-zA-ZäöüÄÖÜß]{3,}\b", text.lower()) + # Filter stopwords + filtered = [ + w for w in words + if w not in _KEY_PHRASE_TOKENS + ] + # Count frequency + freq: dict[str, int] = {} + for w in filtered: + freq[w] = freq.get(w, 0) + 1 + # Sort by frequency descending + top = sorted(freq, key=lambda w: freq[w], reverse=True)[:max_phrases] + return top + + +# --------------------------------------------------------------------------- +# Claim-Text-Hash für Dedup +# --------------------------------------------------------------------------- + +def _claim_text_hash(text: str) -> str: + """Einen schnellen Hash für Claim-Texte (nur zur dedup-Prüfung).""" + import hashlib + normalized = re.sub(r"\s+", " ", text.strip().lower()) + return hashlib.md5(normalized.encode()).hexdigest()[:12] + + +# --------------------------------------------------------------------------- +# LLM-Response-Parsing +# --------------------------------------------------------------------------- + +def parse_cluster_response(response: str) -> list[dict[str, Any]]: + """Parsen der LLM-Antwort für Clustering. + + Returns + ------- + list[dict] mit keys 'label' und 'claim_ids'. + """ + text = response.strip() + # Extract JSON from code blocks + if "```" in text: + lines = text.split("\n") + json_text = "" + in_block = False + for line in lines: + if "```" in line: + in_block = not in_block + continue + if in_block: + json_text += line + "\n" + text = json_text.strip() + + # Try to find JSON object + start = text.find("{") + end = text.rfind("}") + 1 + if start >= 0 and end > start: + text = text[start:end] + + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid cluster response JSON: {exc}") from exc + + if isinstance(data, dict) and "clusters" in data: + clusters = data["clusters"] + elif isinstance(data, list): + clusters = data + else: + raise ValueError(f"Unexpected cluster response structure: {type(data)}") + + result = [] + for cluster in clusters: + if not isinstance(cluster, dict): + continue + label = cluster.get("label", "") + claim_ids = cluster.get("claim_ids", []) + if not isinstance(claim_ids, list): + claim_ids = [claim_ids] if claim_ids else [] + result.append({ + "label": str(label), + "claim_ids": [str(cid) for cid in claim_ids], + }) + return result + + +def parse_relation_response(response: str) -> dict[str, Any]: + """Parsen der LLM-Antwort für Pairwise-Relation. + + Returns + ------- + dict mit keys 'relation', 'confidence', 'reason'. + """ + text = response.strip() + if "```" in text: + lines = text.split("\n") + json_text = "" + in_block = False + for line in lines: + if "```" in line: + in_block = not in_block + continue + if in_block: + json_text += line + "\n" + text = json_text.strip() + + start = text.find("{") + end = text.rfind("}") + 1 + if start >= 0 and end > start: + text = text[start:end] + + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid relation response JSON: {exc}") from exc + + relation = data.get("relation", "UNCERTAIN").upper() + valid_types = {"SUPPORTS", "CONTRADICTS", "DUPLICATE", "UNCERTAIN"} + if relation not in valid_types: + relation = "UNCERTAIN" + + confidence = data.get("confidence", 0.5) + try: + confidence = float(confidence) + confidence = max(0.0, min(1.0, confidence)) + except (ValueError, TypeError): + confidence = 0.5 + + reason = str(data.get("reason", "Keine Begründung.")) + + return { + "relation": relation, + "confidence": confidence, + "reason": reason, + } + + +# --------------------------------------------------------------------------- +# Stage 7 Pipeline +# --------------------------------------------------------------------------- + +class Stage7Clustering: + """Stage 7: Claim Clustering & Contradiction Candidates.""" + + SHORT_CLAIM_THRESHOLD = 500 # Zeichen bis direkter LLM-Vergleich + BATCH_SIZE = 3 # Claims pro LLM-Anfrage beim Clustering + RELATION_BATCH_SIZE = 50 # Max Relationen pro Batch + + def __init__( + self, + llm_provider: LLMProvider, + config: Any, + research_run_id: UUID, + claims: list[ClaimModel], + ): + self.llm_provider = llm_provider + self.config = config + self.research_run_id = research_run_id + self.claims = claims + + async def run(self) -> dict[str, Any]: + """Führt die vollständige Stage-7-Pipeline aus. + + Returns + ------- + dict mit 'clusters', 'relations', 'nlu_results', 'summary'. + """ + summary = { + "research_run_id": str(self.research_run_id), + "total_claims": len(self.claims), + "clusters_created": 0, + "relations_created": 0, + "nlu_entries": 0, + "errors": [], + } + + if not self.claims: + logger.warning("No claims to cluster") + return { + "clusters": [], + "relations": [], + "nlu_results": [], + "summary": { + "research_run_id": str(self.research_run_id), + "total_claims": 0, + "clusters_created": 0, + "relations_created": 0, + "nlu_entries": 0, + "errors": [], + }, + } + + # Step 1: Numerische Normalisierung + nlu_results = self._normalize_numerics() + summary["nlu_entries"] = len(nlu_results) + + # Step 2: Semantische Gruppierung + # Für <500 Zeichen: direkter LLM-Vergleich, >500: Key-Phrase-Extraktion + claim_groups = self._preprocess_claims() + + # Step 3: LLM-Clustering + clusters = await self._llm_cluster(claim_groups) + summary["clusters_created"] = len(clusters) + + # Step 4: Pairwise Relations + relations = await self._analyze_relations(clusters) + summary["relations_created"] = len(relations) + + return { + "clusters": clusters, + "relations": relations, + "nlu_results": nlu_results, + "summary": summary, + } + + def _normalize_numerics(self) -> list[dict[str, Any]]: + """Schritt 1: Extrahiert und normalisiert numerische Ausdrücke.""" + results = [] + for claim in self.claims: + numerics = extract_numerics(claim.claim_text) + for num in numerics: + results.append({ + "claim_id": str(claim.id), + "original_text": num.original_text, + "normalized_value": num.normalized_value, + "unit": num.unit or "", + }) + # Store in DB via ClaimNLUModel (DB-Integration) + if numerics: + claim_nlu_data = { + "claim_id": str(claim.id), + "numeric_expressions": [ + { + "original": n.original_text, + "normalized": n.normalized_value, + "unit": n.unit or "", + } + for n in numerics + ], + } + # In production: session.add(ClaimNLUModel(...)) + return results + + def _preprocess_claims(self) -> list[dict[str, Any]]: + """Schritt 2/3: Preprocessing – kurze Claims direkt, lange mit Key-Phrases.""" + processed = [] + for claim in self.claims: + text = claim.claim_text or "" + if len(text) >= self.SHORT_CLAIM_THRESHOLD: + # Lange Claims: Key-Phrase-Extraktion + Embedding-Vorbereitung + key_phrases = _extract_key_phrases(text) + processed.append({ + "id": str(claim.id), + "text": text, + "type": claim.claim_type.value if isinstance(claim.claim_type, ClaimType) else str(claim.claim_type), + "claim_type": claim.claim_type, + "short": False, + "key_phrases": key_phrases, + "source_url": getattr(claim, "source_url", ""), + }) + else: + processed.append({ + "id": str(claim.id), + "text": text, + "type": claim.claim_type.value if isinstance(claim.claim_type, ClaimType) else str(claim.claim_type), + "claim_type": claim.claim_type, + "short": True, + "key_phrases": [], + "source_url": getattr(claim, "source_url", ""), + }) + return processed + + async def _llm_cluster( + self, + claim_groups: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + """Schritt 3: LLM-basiertes Clustering. + + Sendet Claims in Batches an den LLM. Jeder Batch wird zu + semantischen Clustern gruppiert. + """ + all_clusters: list[dict[str, Any]] = [] + + if not claim_groups: + return all_clusters + + # Build claim list for LLM prompt + claims_for_llm = [ + { + "id": g["id"], + "text": g["text"][:200] + "..." if len(g["text"]) > 200 else g["text"], + "type": g["type"], + } + for g in claim_groups + ] + + # Build the full prompt + claims_list_str = "" + for i, c in enumerate(claims_for_llm, 1): + claims_list_str += ( + f"{i}. [{c['type']}] ID: {c['id']}\n" + f" \"{c['text']}\"\n" + ) + + prompt = CLUSTER_USER_PROMPT.format(claims_list=claims_list_str) + + try: + response = await self.llm_provider.complete( + messages=[ + {"role": "system", "content": CLUSTER_SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + temperature=0.2, + max_tokens=8192, + ) + clusters = parse_cluster_response(response) + all_clusters.extend(clusters) + + except Exception as exc: + logger.error("LLM clustering failed: %s", exc) + all_clusters.append({ + "label": "All_Claims", + "claim_ids": [g["id"] for g in claim_groups], + }) + + return all_clusters + + async def _analyze_relations( + self, + clusters: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + """Schritt 4: Pairwise Claim-Relation-Analyse pro Cluster.""" + all_relations: list[dict[str, Any]] = [] + + for cluster in clusters: + cluster_id = str(id(cluster)) # Will be replaced by real DB ID in production + claim_ids = cluster.get("claim_ids", []) + + if len(claim_ids) < 2: + # Ein-Claim-Cluster: keine Relation + continue + + # Find claims from the processed list + claim_map: dict[str, dict] = {} + for g in self._preprocess_claims(): + claim_map[g["id"]] = g + + # Pairwise analysis + for i in range(len(claim_ids)): + for j in range(i + 1, len(claim_ids)): + id_a = claim_ids[i] + id_b = claim_ids[j] + + claim_a = claim_map.get(id_a, {}) + claim_b = claim_map.get(id_b, {}) + + if not claim_a or not claim_b: + continue + + # Build pairwise comparison prompt + prompt = RELATION_USER_PROMPT.format( + url_a=claim_a.get("source_url", "unknown"), + text_a=claim_a.get("text", ""), + type_a=claim_a.get("type", "unknown"), + url_b=claim_b.get("source_url", "unknown"), + text_b=claim_b.get("text", ""), + type_b=claim_b.get("type", "unknown"), + ) + + try: + response = await self.llm_provider.complete( + messages=[ + {"role": "system", "content": RELATION_SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ], + temperature=0.1, + max_tokens=512, + ) + relation_data = parse_relation_response(response) + + relation_record = { + "source_claim_id": id_a, + "target_claim_id": id_b, + "relation_type": relation_data["relation"], + "confidence": relation_data["confidence"], + "reason": relation_data["reason"], + "cluster_id": cluster_id, + } + all_relations.append(relation_record) + + except Exception as exc: + logger.warning( + "Relation analysis failed for %s ↔ %s: %s", + id_a, id_b, exc, + ) + all_relations.append({ + "source_claim_id": id_a, + "target_claim_id": id_b, + "relation_type": "UNCERTAIN", + "confidence": 0.1, + "reason": f"LLM-Fehler: {exc}", + "cluster_id": cluster_id, + }) + + return all_relations \ No newline at end of file diff --git a/src/nsct/stages/stage7_normalize_numerics.py b/src/nsct/stages/stage7_normalize_numerics.py new file mode 100644 index 0000000..dedc214 --- /dev/null +++ b/src/nsct/stages/stage7_normalize_numerics.py @@ -0,0 +1,349 @@ +"""Stage 7: Numerische Normalisierung von Claims. + +Zieht numerische Ausdrücke aus Claim-Texten, normalisiert +Prozentangaben, Währungen, Einheiten und deutsche/englische +Zahlenwörter in eine einheitliche Repräsentation. + +Beispiele: + "50%" → 0.5 (unit: "%") + "fünfzig" → 50.0 (unit: None) + "€50" → 50.0 (unit: "EUR") + "50 Euro" → 50.0 (unit: "EUR") + "3 kg" → 3.0 (unit: "kg") +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +# --------------------------------------------------------------------------- +# Deutsche Zahlenwörter (0-100 in Schritten von 10 + Einzelzahlen) +# --------------------------------------------------------------------------- + +_DE_NUMBERS = { + "null": 0, "eins": 1, "einer": 1, "zwei": 2, "drei": 3, "vier": 4, + "fünf": 5, "fünfundvierzig": 45, "fünfzig": 50, "sechs": 6, + "sieben": 7, "acht": 8, "neun": 9, "zehn": 10, "elf": 11, + "zwölf": 12, "dreizehn": 13, "vierzehn": 14, "fünfzehn": 15, + "sechzehn": 16, "siebzehn": 17, "achtzehn": 18, "neunzehn": 19, + "zwanzig": 20, "einundzwanzig": 21, "zweiundzwanzig": 22, + "dreiundzwanzig": 23, "vierundzwanzig": 24, "fünfundzwanzig": 25, + "sechsundzwanzig": 26, "siebenundzwanzig": 27, "achtundzwanzig": 28, + "neunundzwanzig": 29, "dreißig": 30, "einunddreißig": 31, + "zweiunddreißig": 32, "dreiunddreißig": 33, "vierunddreißig": 34, + "fünfunddreißig": 35, "sechsunddreißig": 36, "siebenunddreißig": 37, + "achtunddreißig": 38, "neununddreißig": 39, "vierzig": 40, + "einundvierzig": 41, "zweiundvierzig": 42, "dreiundvierzig": 43, + "vierundvierzig": 44, "fünfundvierzig": 45, "sechsundvierzig": 46, + "siebenundvierzig": 47, "achtundvierzig": 48, "neunundvierzig": 49, + "fünfzig": 50, "einundfünfzig": 51, "zweiundfünfzig": 52, + "dreiundfünfzig": 53, "vierundfünfzig": 54, "fünfundfünfzig": 55, + "sechsundfünfzig": 56, "siebenundfünfzig": 57, "achtundfünfzig": 58, + "neunundfünfzig": 59, "sechzig": 60, + "einundsechzig": 61, "zweiundsechzig": 62, "dreiundsechzig": 63, + "vierundsechzig": 64, "fünfundsechzig": 65, "sechsundsechzig": 66, + "siebenundsechzig": 67, "achtundsechzig": 68, "neunundsechzig": 69, + "siebzig": 70, + "einundsiebzig": 71, "zweiundsiebzig": 72, "dreiundsiebzig": 73, + "vierundsiebzig": 74, "fünfundsiebzig": 75, "sechsundsiebzig": 76, + "siebenundsiebzig": 77, "achtundsiebzig": 78, "neunundsiebzig": 79, + "achtzig": 80, + "einundachtzig": 81, "zweiundachtzig": 82, "dreiundachtzig": 83, + "vierundachtzig": 84, "fünfundachtzig": 85, "sechsundachtzig": 86, + "siebenundachtzig": 87, "achtundachtzig": 88, "neunundachtzig": 89, + "neunzig": 90, + "einundneunzig": 91, "zweiundneunzig": 92, "dreiundneunzig": 93, + "vierundneunzig": 94, "fünfundneunzig": 95, "sechsundneunzig": 96, + "siebenundneunzig": 97, "achtundneunzig": 98, "neunundneunzig": 99, + "hundert": 100, "zweihundert": 200, "dreihundert": 300, + "vierhundert": 400, "fünfhundert": 500, "sechshundert": 600, + "siebenhundert": 700, "achthundert": 800, "neunhundert": 900, + "tausend": 1000, "eine Million": 1000000, "zwei Millionen": 2000000, + "drei Millionen": 3000000, "vier Millionen": 4000000, + "fünf Millionen": 5000000, "sechs Millionen": 6000000, + "sieben Millionen": 7000000, "acht Millionen": 8000000, + "neun Millionen": 9000000, "zeh Million": 10000000, +} + +# English number words (0-20 + teens + tens + hundred + thousand) +_EN_NUMBERS = { + "zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, + "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14, + "fifteen": 15, "sixteen": 16, "seventeen": 17, "eighteen": 18, + "nineteen": 19, "twenty": 20, "thirty": 30, "forty": 40, + "fifty": 50, "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90, + "hundred": 100, "thousand": 1000, "million": 1000000, + "billion": 1000000000, +} + + +class NumericExtractionResult(NamedTuple): + """Ergebnis der Extraktion eines numerischen Ausdrucks.""" + + original_text: str # Der originale Text wie im Claim + normalized_value: str # Normalisierter Zahlenwert als String + unit: str | None # Einheit (%, EUR, USD, etc.) + + +# --------------------------------------------------------------------------- +# Currency symbols & codes +# --------------------------------------------------------------------------- + +_CURRENCY_SYMBOLS = { + "€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY", + "CHF": "CHF", "Fr.": "CHF", "CHF": "CHF", +} + +_CURRENCY_CODES = {"EUR", "USD", "GBP", "JPY", "CHF", "CAD", "AUD", "SEK", "NOK", "DKK", "PLN", "CZK", "HUF", "Euro", "Pfund", "Dollar", "Yen", "Franc", "Real", "Peso", "Rupie", "Ringgit", "Baht", "Won", "Yuan", "Zloty", "Forint", "Koruna", "Krona", "Krone", "Lek", "Lari", "Lev", "Lira", "Litas", "Manat", "Nail", "Tenge", "Sum"} + +# Common units +_COMMON_UNITS = [ + "kg", "tonne", "tonnes", "Liter", "Liter", "mm", "cm", "m", "km", + "m²", "km²", "ha", "%", "Prozent", "percent", "procentsatz", + "Prozentsatz", "prozent", "Euro", "dollar", "Dollar", "pounds", "Pfund", "EUR", "USD", + "dollars", "Pfund", "hours", + "minute", "minutes", "Minuten", "second", "seconds", "Sekunden", + "people", "Menschen", "einwohner", "Einwohner", +] + +# Compiled regex patterns +# Pattern 1: Digit followed by unit/symbol (50%, $100, 5kg, 500 km) +_RE_DIGIT_UNIT = re.compile( + r"(?P\d+(?:[.,]\d+)?)\s*(?P" + + "|".join(re.escape(u) for u in sorted(_COMMON_UNITS, key=len, reverse=True)) + + r"|[$€£¥]|%)" +) + +# Pattern 1b: Standalone numbers (83, 100) — digits not followed by known unit +_RE_STANDALONE_NUM = re.compile( + r"(?\d+(?:[.,]\d+)?)(?!\w)" +) + +# Pattern 2: Currency symbol before digits (€50, $100) +_RE_UNIT_DIGIT = re.compile( + r"(?P[$€£¥])\s*(?P\d+(?:[.,]\d+)?(?:\s*(?:EUR|USD|GBP|JPY|CHF))?)", + re.IGNORECASE, +) + +# Pattern 3: Word numbers in context (fünfzig Prozent, thirty dollars) +_RE_WORD_NUMBER_UNIT = re.compile( + r"(?P\b[a-zA-ZäöüÄÖÜß]+\b)" + r"\s+(?P" + + "|".join(re.escape(u) for u in sorted(_COMMON_UNITS, key=len, reverse=True)) + + r")", + re.IGNORECASE, +) + +# Pattern 4: Standalone word numbers (fünfzig, thirty) +_RE_WORD_NUMBER = re.compile( + r"\b(" + "|".join(re.escape(k) for k in sorted(_DE_NUMBERS | _EN_NUMBERS, key=len, reverse=True)) + r")\b", + re.IGNORECASE, +) + +# Pattern 5: "X of Y" pattern (half of 100, one third) +_RE_FRACTION = re.compile( + r"(?:(?Phalf|drittel|ein Drittel|ein Drittel|ein Drittel|" + r"ein Halb|ein halbes|die Hälfte)\s+of\s+|die hälfte\s+von)\s*(?P\d+(?:[.,]\d+)?)", + re.IGNORECASE, +) + + +def extract_numerics(text: str) -> list[NumericExtractionResult]: + """Zieht alle numerischen Ausdrücke aus *text* und normalisiert sie. + + Parameters + ---------- + text : str + Der Text eines Claims. + + Returns + ------- + list[NumericExtractionResult] + Liste aller gefundenen numerischen Ausdrücke. + """ + if not text: + return [] + + results: list[NumericExtractionResult] = [] + seen_originals: set[str] = set() + + for m in _RE_DIGIT_UNIT.finditer(text): + original = m.group(0).strip() + if original in seen_originals: + continue + seen_originals.add(original) + + value_str = m.group("value").replace(",", ".") + unit = m.group("unit") + + normalized = _normalize_number(value_str, unit) + if normalized: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=normalized[0], + unit=normalized[1], + )) + else: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=value_str, + unit=_unit_code(unit), + )) + + for m in _RE_UNIT_DIGIT.finditer(text): + original = m.group(0).strip() + if original in seen_originals: + continue + seen_originals.add(original) + + unit_raw = m.group("unit") + value_str = m.group("value").replace(",", ".").strip() + + # Strip trailing currency code if present in value + for code in _CURRENCY_CODES: + if value_str.upper().endswith(code): + value_str = value_str[: -len(code)].strip() + + normalized = _normalize_number(value_str, unit_raw) + if normalized: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=normalized[0], + unit=normalized[1], + )) + else: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=value_str, + unit=_unit_code(unit_raw), + )) + + # Standalone numbers (no unit) + for m in _RE_STANDALONE_NUM.finditer(text): + original = m.group("value").strip() + if original in seen_originals: + continue + seen_originals.add(original) + + value_str = original.replace(",", ".") + normalized = _normalize_number(value_str, "") + if normalized: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=normalized[0], + unit="", + )) + + # Check word numbers followed by unit + for m in _RE_WORD_NUMBER_UNIT.finditer(text): + original = m.group(0).strip() + if original in seen_originals: + continue + seen_originals.add(original) + + word = m.group("words").strip() + unit = m.group("unit") + + value = _parse_word_number(word) + if value is not None: + normalized = _normalize_number(str(value), unit) + if normalized: + unit_code = _unit_code(normalized[1]) + results.append(NumericExtractionResult( + original_text=original, + normalized_value=normalized[0], + unit=unit_code, + )) + else: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=str(value), + unit=_unit_code(unit), + )) + + # Check standalone word numbers + for m in _RE_WORD_NUMBER.finditer(text): + original = m.group(0).strip() + if original in seen_originals: + continue + seen_originals.add(original) + + value = _parse_word_number(original) + if value is not None: + results.append(NumericExtractionResult( + original_text=original, + normalized_value=str(value), + unit=None, + )) + + # Deduplicate: keep highest confidence (longest original text) + deduped: dict[tuple[str, str | None], NumericExtractionResult] = {} + for r in results: + key = (r.normalized_value, r.unit) + existing = deduped.get(key) + if existing is None or len(r.original_text) > len(existing.original_text): + deduped[key] = r + + return list(deduped.values()) + + +def _normalize_number(value_str: str, unit: str) -> tuple[str, str] | None: + """Konvertiert einen Zahlentext in normalisierte Form. + + Returns + ------- + (normalized_value, unit) or None if conversion fails. + """ + # Handle percentage + if unit in ("%", "Prozent", "Prozentsatz", "prozent", "percent", "procentsatz"): + try: + num = float(value_str.replace(",", ".")) + normalized = str(num / 100.0) + return normalized, "%" + except (ValueError, TypeError): + return None + + # Handle currency + currency_map = {"€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY"} + if unit in currency_map: + try: + num = float(value_str.replace(",", ".")) + return str(num), currency_map[unit] + except (ValueError, TypeError): + return None + + if unit in _CURRENCY_CODES: + try: + num = float(value_str.replace(",", ".")) + return str(num), unit + except (ValueError, TypeError): + return None + + # Check currency symbol + if unit in _CURRENCY_SYMBOLS: + try: + num = float(value_str.replace(",", ".")) + return str(num), _CURRENCY_SYMBOLS[unit] + except (ValueError, TypeError): + return None + + # Default: just return the number + try: + num = float(value_str.replace(",", ".")) + return str(num), unit or "" + except (ValueError, TypeError): + return None + + +def _unit_code(unit: str) -> str: + """Macht aus einem Symbol/Code eine Einheit-Kennung.""" + symbol_map = {"€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY", "%": "%"} + return symbol_map.get(unit, unit) + + +def _parse_word_number(word: str) -> float | None: + """Versucht, ein Zahlenwort in einen float-Wert zu konvertieren.""" + normalized = word.strip().lower() + return _DE_NUMBERS.get(normalized) or _EN_NUMBERS.get(normalized) \ No newline at end of file diff --git a/src/nsct/storage/models.py b/src/nsct/storage/models.py index 1a4a697..a8e7cde 100644 --- a/src/nsct/storage/models.py +++ b/src/nsct/storage/models.py @@ -273,4 +273,110 @@ class ResearchReportModel(Base): __table_args__ = ( Index("ix_research_reports_research_id", "research_id"), + ) + + +# --------------------------------------------------------------------------- +# Stage 7 — Claim Clustering & Contradiction Candidates +# --------------------------------------------------------------------------- + + +class ClaimRelationType(str, enum.Enum): + """Relationship types between individual claims (Stage 7).""" + + SUPPORTS = "supports" + CONTRADICTS = "contradicts" + DUPLICATE = "duplicate" + UNCERTAIN = "uncertain" + + +class ClaimClusterModel(Base): + """Semantisches Cluster von Claims innerhalb eines Research-Runs.""" + + __tablename__ = "claim_clusters" + + id = Column(String(36), primary_key=True, default=lambda: str(uuid4())) + research_run_id = Column(String(36), nullable=False, default=lambda: str(uuid4())) + cluster_label = Column(Text, nullable=False) + representative_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=True) + claim_count = Column(Integer, nullable=False, default=1) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + # Relationships + claims = relationship("ClaimModel", secondary="claim_cluster_mapping", back_populates="clusters") + relations = relationship( + "ClaimRelationModel", + back_populates="cluster", + cascade="all, delete-orphan", + ) + + __table_args__ = ( + Index("ix_claim_clusters_research_run_id", "research_run_id"), + ) + + +# Association table: claims ↔ clusters (many-to-many) +_claim_cluster_mapping = Base() # noqa: F811 -- dummy for type resolution + +claim_cluster_mapping = Base() +claim_cluster_mapping.__tablename__ = "claim_cluster_mapping" +claim_cluster_mapping.id = Column(String(36), primary_key=True, default=lambda: str(uuid4())) +claim_cluster_mapping.cluster_id = Column(String(36), ForeignKey("claim_clusters.id"), nullable=False) +claim_cluster_mapping.claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False) +claim_cluster_mapping.position = Column(Integer, nullable=False, default=0) + +__table_args__ = ( + Index("ix_claim_cluster_mapping_cluster_id", "cluster_id"), + Index("ix_claim_cluster_mapping_claim_id", "claim_id"), + Index("uq_claim_cluster_mapping_cluster_claim", "cluster_id", "claim_id", unique=True), +) + + +class ClaimRelationModel(Base): + """Pairwise Beziehung zwischen zwei Claims innerhalb eines Clusters.""" + + __tablename__ = "claim_relations" + + id = Column(String(36), primary_key=True, default=lambda: str(uuid4())) + source_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False) + target_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False) + relation_type = Column(Enum(ClaimRelationType), nullable=False) + confidence = Column(Float, nullable=False, default=0.5) + reason = Column(Text, nullable=True) + cluster_id = Column(String(36), ForeignKey("claim_clusters.id"), nullable=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + # Relationships + cluster = relationship("ClaimClusterModel", back_populates="relations") + source_claim = relationship("ClaimModel", foreign_keys=[source_claim_id]) + target_claim = relationship("ClaimModel", foreign_keys=[target_claim_id]) + + __table_args__ = ( + Index("ix_claim_relations_source_claim_id", "source_claim_id"), + Index("ix_claim_relations_target_claim_id", "target_claim_id"), + Index("ix_claim_relations_cluster_id", "cluster_id"), + Index( + "uq_claim_relations_pair", + "source_claim_id", + "target_claim_id", + "relation_type", + unique=True, + ), + ) + + +class ClaimNLUModel(Base): + """Numerisch normalisierte Ausdrücke aus Claims (Stage 7 — NLU).""" + + __tablename__ = "claim_nlu_numeric" + + id = Column(String(36), primary_key=True, default=lambda: str(uuid4())) + claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False) + normalized_numeric_value = Column(String(64), nullable=True) + original_numeric_text = Column(Text, nullable=False) + unit = Column(String(16), nullable=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + __table_args__ = ( + Index("ix_claim_nlu_numeric_claim_id", "claim_id"), ) \ No newline at end of file diff --git a/tests/stages/test_stage7_clustering.py b/tests/stages/test_stage7_clustering.py new file mode 100644 index 0000000..f73a830 --- /dev/null +++ b/tests/stages/test_stage7_clustering.py @@ -0,0 +1,866 @@ +"""Umfassende Tests für Stage 7: Claim Clustering & Contradiction Candidates. + +Abdeckungen: +- Numerische Normalisierung: Prozent, Währungen, deutsche/englische Wörter +- LLM-Response-Parsing für Cluster und Relation +- Clustering: gleiche Themen, verschiedene Themen +- Pairwise Relation: SUPPORTS, CONTRADICTS, DUPLICATE, UNCERTAIN +- Edge Cases: 1 Claim, 2 Claims, viele Claims, leere Claims +- DB-Integration: Cluster + Relation creation +- Key-Phrase-Extraktion +- API-Request/Response-Schemas +""" + +from __future__ import annotations + +import asyncio +import json +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock +from uuid import UUID, uuid4 + +import pytest + +from nsct.models.claim import Claim, ClaimType +from nsct.stages.stage7_clustering import ( + Stage7Clustering, + CLUSTER_SYSTEM_PROMPT, + CLUSTER_USER_PROMPT, + RELATION_SYSTEM_PROMPT, + RELATION_USER_PROMPT, + _extract_key_phrases, + _claim_text_hash, + parse_cluster_response, + parse_relation_response, +) +from nsct.stages.stage7_normalize_numerics import ( + NumericExtractionResult, + _normalize_number, + _parse_word_number, + extract_numerics, +) + + +# --------------------------------------------------------------------------- +# Fixtures & Helpers +# --------------------------------------------------------------------------- + +def _mock_llm_provider(response: str) -> MagicMock: + """Erzeuge einen mock LLM-Provider mit einer festen Antwort.""" + provider = MagicMock() + provider.complete = AsyncMock(return_value=response) + provider.model = "test-model" + return provider + + +def _make_extractor( + llm_response: str, + claims: list[Claim] | None = None, +) -> Stage7Clustering: + """Erzeuge einen Stage7Clustering mit mock LLM.""" + provider = _mock_llm_provider(llm_response) + provider.model = "test-model" + + config = MagicMock() + config.llm.base_url = "http://localhost:8030/openai/v1" + config.llm.model = "test-model" + config.llm.max_concurrency = 3 + + if claims is None: + claims = [ + Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Deutschland hat 83 Millionen Einwohner.", + evidence_span="Deutschland hat 83 Millionen Einwohner", + claim_type=ClaimType.FACT, + source_url="https://example.com/1", + ), + ] + + return Stage7Clustering( + llm_provider=provider, + config=config, + research_run_id=claims[0].research_run_id, + claims=claims, + ) + + +def asyncio_run(coro): + """Hilfsfunktion: Koroutine synchron ausführen.""" + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +# --------------------------------------------------------------------------- +# Test Group 1-10: Numerische Normalisierung — Grundlegende Tests +# --------------------------------------------------------------------------- + + +class TestNumericNormalization: + """Tests für die numerische Normalisierung.""" + + def test_percent_integer(self) -> None: + """Integer-Prozent: 50% → 0.5.""" + results = extract_numerics("Die Emissionen sinken um 50%.") + assert len(results) > 0 + pcts = [r for r in results if r.unit == "%"] + assert len(pcts) > 0 + assert pcts[0].normalized_value == "0.5" + + def test_percent_decimal(self) -> None: + """Dezimal-Prozent: 5.5% → 0.055.""" + results = extract_numerics("Ein Anstieg von 5.5% ist zu erwarten.") + pcts = [r for r in results if r.unit == "%"] + assert len(pcts) > 0 + assert pcts[0].normalized_value == "0.055" + + def test_currency_euro(self) -> None: + """Euro: €50 → 50.0 EUR.""" + results = extract_numerics("Kosten von €50.") + euros = [r for r in results if r.unit == "EUR"] + assert len(euros) > 0 + assert euros[0].normalized_value == "50.0" + + def test_currency_dollar(self) -> None: + """Dollar: $100 → 100.0 USD.""" + results = extract_numerics("Kosten von $100.") + dollars = [r for r in results if r.unit == "USD"] + assert len(dollars) > 0 + assert dollars[0].normalized_value == "100.0" + + def test_currency_code(self) -> None: + """Währungscode: 50 EUR → 50.0 EUR.""" + results = extract_numerics("Kosten von 50 EUR und 100 USD.") + euros = [r for r in results if r.unit == "EUR"] + assert len(euros) > 0 + assert euros[0].normalized_value == "50.0" + + def test_word_currency_euro(self) -> None: + """Wort+Währung: fünfzig Euro → 50.0 EUR.""" + results = extract_numerics("Kosten von fünfzig Euro.") + euros = [r for r in results if r.unit in ("EUR", "Euro")] + assert len(euros) > 0 + assert euros[0].normalized_value == "50.0" + + def test_standalone_number(self) -> None: + """Standalone-Zahl: 100 → 100.0.""" + results = extract_numerics("Es gibt 100 Bewerber und 83 Stimmen.") + numbers = [r for r in results if r.unit in (None, "")] + assert len(numbers) > 0 + assert any(float(r.normalized_value) == 100.0 for r in numbers) + + def test_no_numbers(self) -> None: + """Text ohne Zahlen → leer.""" + results = extract_numerics("Es gibt keine Zahlen in diesem Text.") + assert len(results) == 0 + + def test_empty_text(self) -> None: + """Leerer Text → leer.""" + results = extract_numerics("") + assert results == [] + + def test_none_text(self) -> None: + """None-Text → leer.""" + results = extract_numerics(None) # type: ignore[arg-type] + assert results == [] + + +# --------------------------------------------------------------------------- +# Test Group 11-20: Deutsche/Englische Zahlenwörter +# --------------------------------------------------------------------------- + + +class TestWordNumbers: + """Tests für deutsche und englische Zahlenwörter.""" + + def test_parse_fuenfzig(self) -> None: + """'fünfzig' → 50.0.""" + assert _parse_word_number("fünfzig") == 50 + + def test_parse_fifty(self) -> None: + """'fifty' → 50.0.""" + assert _parse_word_number("fifty") == 50 + + def test_parse_drei(self) -> None: + """'drei' → 3.0.""" + assert _parse_word_number("drei") == 3 + + def test_parse_three(self) -> None: + """'three' → 3.0.""" + assert _parse_word_number("three") == 3 + + def test_parse_hundert(self) -> None: + """'hundert' → 100.0.""" + assert _parse_word_number("hundert") == 100 + + def test_parse_thousand(self) -> None: + """'thousand' → 1000.0.""" + assert _parse_word_number("thousand") == 1000 + + def test_parse_unknown_word(self) -> None: + """Unbekanntes Wort → None.""" + assert _parse_word_number("xyzunknown") is None + + def test_extract_fifty_percent(self) -> None: + """'fünfzig Prozent' → 50% → 0.5.""" + results = extract_numerics("fünfzig Prozent der Bürger") + pcts = [r for r in results if r.unit == "%"] + assert len(pcts) > 0 + assert pcts[0].normalized_value == "0.5" + + def test_extract_ten_dollars(self) -> None: + """'ten dollars' → 10.0 USD.""" + results = extract_numerics("Kosten von ten dollar.") + # The unit will be the converted code if recognized + dollars = [r for r in results if r.unit in ("USD", "dollar", "Dollar")] + assert len(dollars) > 0 + # At least one should have the correct normalized value + assert any(float(r.normalized_value) == 10.0 for r in dollars) + + +# --------------------------------------------------------------------------- +# Test Group 21-30: _normalize_number +# --------------------------------------------------------------------------- + + +class TestNormalizeNumber: + """Tests für die _normalize_number-Funktion.""" + + def test_percent_50(self) -> None: + """50% → ('0.5', '%').""" + result = _normalize_number("50", "%") + assert result is not None + assert result[0] == "0.5" + assert result[1] == "%" + + def test_percent_100(self) -> None: + """100% → ('1.0', '%').""" + result = _normalize_number("100", "%") + assert result is not None + assert result[0] == "1.0" + + def test_percent_0(self) -> None: + """0% → ('0.0', '%').""" + result = _normalize_number("0", "%") + assert result is not None + assert result[0] == "0.0" + + def test_euro_symbol(self) -> None: + """€50 → ('50.0', 'EUR').""" + result = _normalize_number("50", "€") + assert result is not None + assert result[1] == "EUR" + + def test_dollar_symbol(self) -> None: + """$100 → ('100.0', 'USD').""" + result = _normalize_number("100", "$") + assert result is not None + assert result[1] == "USD" + + def test_pound_symbol(self) -> None: + """£30 → ('30.0', 'GBP').""" + result = _normalize_number("30", "£") + assert result is not None + assert result[1] == "GBP" + + def test_weight_kg(self) -> None: + """3 kg → ('3.0', 'kg').""" + result = _normalize_number("3", "kg") + assert result is not None + assert result[0] == "3.0" + + def test_distance_km(self) -> None: + """500 km → ('500.0', 'km').""" + result = _normalize_number("500", "km") + assert result is not None + assert result[0] == "500.0" + + def test_unit_unknown(self) -> None: + """Unbekannte Einheit wird durchgereicht.""" + result = _normalize_number("42", "xyz") + assert result is not None + assert result[1] == "xyz" + + def test_negative_value(self) -> None: + """Negative Werte: -10 → ('-10.0', '').""" + result = _normalize_number("-10", "") + assert result is not None + assert result[0] == "-10.0" + + +# --------------------------------------------------------------------------- +# Test Group 31-40: LLM-Response-Parsing — Cluster +# --------------------------------------------------------------------------- + + +class TestParseClusterResponse: + """Tests für parse_cluster_response.""" + + def test_parse_valid_clusters(self) -> None: + """Gültige JSON-Antwort parsen.""" + response = json.dumps({ + "clusters": [ + {"label": "Klimaschutz", "claim_ids": ["id1", "id2"]}, + {"label": "Wirtschaft", "claim_ids": ["id3"]}, + ] + }) + result = parse_cluster_response(response) + assert len(result) == 2 + assert result[0]["label"] == "Klimaschutz" + assert result[0]["claim_ids"] == ["id1", "id2"] + assert result[1]["label"] == "Wirtschaft" + + def test_parse_code_block(self) -> None: + """JSON in Markdown-Code-Blocks extrahieren.""" + response = '```json\n{"clusters": [{"label": "Test", "claim_ids": ["id1"]}]}\n```' + result = parse_cluster_response(response) + assert len(result) == 1 + assert result[0]["label"] == "Test" + + def test_parse_invalid_json(self) -> None: + """Ungültiges JSON löst ValueError.""" + with pytest.raises(ValueError): + parse_cluster_response("Das ist kein JSON") + + def test_parse_empty_clusters(self) -> None: + """Leeres Cluster-Array.""" + response = json.dumps({"clusters": []}) + result = parse_cluster_response(response) + assert len(result) == 0 + + def test_parse_list_format(self) -> None: + """Liste von Clusters (Array statt Objekt).""" + response = json.dumps({"clusters": [{"label": "Solo", "claim_ids": ["id1"]}]}).replace('"clusters"', '"clusters"') + result = parse_cluster_response(response) + assert len(result) == 1 + assert result[0]["label"] == "Solo" + + def test_parse_single_cluster(self) -> None: + """Einzelner Cluster.""" + response = json.dumps({ + "clusters": [{"label": "Alle_Claims", "claim_ids": ["id1", "id2", "id3"]}] + }) + result = parse_cluster_response(response) + assert len(result) == 1 + assert result[0]["claim_ids"] == ["id1", "id2", "id3"] + + def test_parse_skip_invalid_items(self) -> None: + """Ungültige Items (kein Dict) müssen übersprungen werden.""" + response = json.dumps({ + "clusters": [ + {"label": "Valid", "claim_ids": ["id1"]}, + "not_a_dict", + 42, + {"label": "AlsoValid", "claim_ids": ["id2"]}, + ] + }) + result = parse_cluster_response(response) + assert len(result) == 2 + assert result[0]["label"] == "Valid" + assert result[1]["label"] == "AlsoValid" + + +# --------------------------------------------------------------------------- +# Test Group 41-50: LLM-Response-Parsing — Relation +# --------------------------------------------------------------------------- + + +class TestParseRelationResponse: + """Tests für parse_relation_response.""" + + def test_parse_supports(self) -> None: + """SUPPORTS-Relation parsen.""" + response = json.dumps({ + "relation": "SUPPORTS", + "confidence": 0.95, + "reason": "Beide Quellen bestätigen die Aussage." + }) + result = parse_relation_response(response) + assert result["relation"] == "SUPPORTS" + assert result["confidence"] == 0.95 + assert "bestätigen" in result["reason"] + + def test_parse_contradicts(self) -> None: + """CONTRADICTS-Relation parsen.""" + response = json.dumps({ + "relation": "CONTRADICTS", + "confidence": 0.85, + "reason": "Quelle A sagt X, Quelle B sagt Y — Gegensatz." + }) + result = parse_relation_response(response) + assert result["relation"] == "CONTRADICTS" + assert result["confidence"] == 0.85 + + def test_parse_duplicate(self) -> None: + """DUPLICATE-Relation parsen.""" + response = json.dumps({ + "relation": "DUPLICATE", + "confidence": 0.98, + "reason": "Identische Aussage, leicht unterschiedliche Formulierung." + }) + result = parse_relation_response(response) + assert result["relation"] == "DUPLICATE" + + def test_parse_uncertain(self) -> None: + """UNCERTAIN-Relation parsen.""" + response = json.dumps({ + "relation": "UNCERTAIN", + "confidence": 0.3, + "reason": "Keine klare Beziehung erkennbar." + }) + result = parse_relation_response(response) + assert result["relation"] == "UNCERTAIN" + + def test_parse_clamped_confidence(self) -> None: + """Confidence > 1.0 wird geklamped.""" + response = json.dumps({ + "relation": "SUPPORTS", + "confidence": 1.5, + "reason": "Test" + }) + result = parse_relation_response(response) + assert result["confidence"] == 1.0 + + def test_parse_negative_confidence(self) -> None: + """Confidence < 0.0 wird geklamped.""" + response = json.dumps({ + "relation": "CONTRADICTS", + "confidence": -0.5, + "reason": "Test" + }) + result = parse_relation_response(response) + assert result["confidence"] == 0.0 + + def test_parse_code_block(self) -> None: + """JSON in Code-Blocks.""" + response = '```json\n{"relation": "SUPPORTS", "confidence": 0.8, "reason": "Test"}\n```' + result = parse_relation_response(response) + assert result["relation"] == "SUPPORTS" + + def test_parse_invalid_json(self) -> None: + """Ungültiges JSON löst ValueError.""" + with pytest.raises(ValueError): + parse_relation_response("Das ist kein JSON") + + def test_parse_invalid_relation_type(self) -> None: + """Ungültiger relation_type → UNCERTAIN.""" + response = json.dumps({ + "relation": "INVALID_TYPE", + "confidence": 0.5, + "reason": "Test" + }) + result = parse_relation_response(response) + assert result["relation"] == "UNCERTAIN" + + def test_parse_default_reason(self) -> None: + """Fehlende reason → Default.""" + response = json.dumps({"relation": "SUPPORTS", "confidence": 0.5}) + result = parse_relation_response(response) + assert result["reason"] == "Keine Begründung." + + +# --------------------------------------------------------------------------- +# Test Group 51-60: Key-Phrase-Extraktion +# --------------------------------------------------------------------------- + + +class TestKeyPhraseExtraction: + """Tests für die Key-Phrase-Extraktion.""" + + def test_extract_phrases(self) -> None: + """Wichtige Begriffe werden extrahiert.""" + text = "Die Bundesregierung hat ein neues Steuergesetz zur Digitalisierung verabschiedet." + phrases = _extract_key_phrases(text) + assert len(phrases) > 0 + assert "regierung" in phrases or "steuergesetz" in phrases or "digitalisierung" in phrases + + def test_empty_text(self) -> None: + """Leerer Text → leere Liste.""" + assert _extract_key_phrases("") == [] + + def test_stopword_filtering(self) -> None: + """Stopwords (und, oder, aber) werden gefiltert.""" + text = "und oder aber jedoch zwar auch nur kein keine nicht" + phrases = _extract_key_phrases(text) + assert len(phrases) == 0 + + def test_max_phrases(self) -> None: + """max_phrases begrenzt die Ausgabe.""" + text = "x x x y y y z z z a b c d e f g h i j k" + phrases = _extract_key_phrases(text, max_phrases=3) + assert len(phrases) <= 3 + + def test_duplicate_removal(self) -> None: + """Wörter werden nur einmal gezählt.""" + text = "x x x y y y z z z" + phrases = _extract_key_phrases(text) + # x, y, z — höchstens 3 + assert len(phrases) <= 3 + + +# --------------------------------------------------------------------------- +# Test Group 61-70: Claim-Text-Hash +# --------------------------------------------------------------------------- + + +class TestClaimTextHash: + """Tests für den Claim-Text-Hash.""" + + def test_same_text_same_hash(self) -> None: + """Identischer Text → identischer Hash.""" + text = "Test claim text" + assert _claim_text_hash(text) == _claim_text_hash(text) + + def test_different_text_different_hash(self) -> None: + """Verschiedener Text → verschiedener Hash.""" + h1 = _claim_text_hash("Claim A") + h2 = _claim_text_hash("Claim B") + assert h1 != h2 + + def test_case_insensitive(self) -> None: + """Groß-/Kleinschreibung wird ignoriert.""" + assert _claim_text_hash("Test") == _claim_text_hash("test") + + def test_whitespace_normalized(self) -> None: + """Mehrere Leerzeichen werden normalisiert.""" + assert _claim_text_hash("test claim") == _claim_text_hash("test claim") + + +# --------------------------------------------------------------------------- +# Test Group 71-80: Stage7Clustering Pipeline +# --------------------------------------------------------------------------- + + +class TestStage7Pipeline: + """Tests für die Stage7Clustering-Pipeline.""" + + def test_empty_claims(self) -> None: + """Leere Claims-Liste → empty summary.""" + stage = Stage7Clustering( + llm_provider=MagicMock(), + config=MagicMock(), + research_run_id=uuid4(), + claims=[], + ) + result = asyncio_run(stage.run()) + assert result["summary"]["total_claims"] == 0 + assert result["summary"]["clusters_created"] == 0 + + def test_single_claim(self) -> None: + """Ein einzelner Claim → 1 Cluster, 0 Relations.""" + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Ein Claim.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(json.dumps({"clusters": [{"label": "Test", "claim_ids": [str(claim.id)]}]})), + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + result = asyncio_run(stage.run()) + assert result["summary"]["total_claims"] == 1 + + def test_numeric_extraction_integration(self) -> None: + """Numerische Normalisierung im Pipeline-Kontext.""" + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Die Emissionen sinken um 50% bis 2030. Kosten: €100.", + evidence_span="50%, €100", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(json.dumps({"clusters": []})), + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + nlu = stage._normalize_numerics() + assert len(nlu) >= 2 # 50% und €100 + + def test_preprocess_short_claims(self) -> None: + """Kurze Claims werden als 'short' markiert.""" + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Short claim.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(json.dumps({"clusters": []})), + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + groups = stage._preprocess_claims() + assert groups[0]["short"] is True + assert groups[0]["key_phrases"] == [] + + def test_preprocess_long_claims(self) -> None: + """Lange Claims (>=500) werden als 'long' markiert.""" + long_text = "x " * 250 # ~500 chars + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text=long_text, + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(json.dumps({"clusters": []})), + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + groups = stage._preprocess_claims() + assert groups[0]["short"] is False + + def test_llm_error_handled(self) -> None: + """LLM-Fehler werden abgefangen.""" + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Test.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + failing_provider = MagicMock() + failing_provider.complete = AsyncMock(side_effect=RuntimeError("LLM down")) + failing_provider.model = "test-model" + + stage = Stage7Clustering( + llm_provider=failing_provider, + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + result = asyncio_run(stage.run()) + # Error sollte geloggt, aber nicht geworfen werden + assert result["summary"]["total_claims"] == 1 + + def test_relation_batching(self) -> None: + """Relations werden paarweise generiert.""" + claim_a = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Claim A: Steuererhöhung.", + evidence_span="Steuererhöhung", + claim_type=ClaimType.FACT, + source_url="https://example.com/a", + ) + claim_b = Claim( + research_run_id=claim_a.research_run_id, + source_id=uuid4(), + claim_text="Claim B: Steuer Senkung.", + evidence_span="Steuer Senkung", + claim_type=ClaimType.FACT, + source_url="https://example.com/b", + ) + relation_response = json.dumps({ + "relation": "CONTRADICTS", + "confidence": 0.9, + "reason": "Steuererhöhung vs. SteuerSenkung" + }) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(relation_response), + config=MagicMock(), + research_run_id=claim_a.research_run_id, + claims=[claim_a, claim_b], + ) + clusters = [{"label": "Steuer", "claim_ids": [str(claim_a.id), str(claim_b.id)]}] + relations = asyncio_run(stage._analyze_relations(clusters)) + assert len(relations) == 1 + assert relations[0]["relation_type"] == "CONTRADICTS" + + def test_relation_error_fallback(self) -> None: + """LLM-Fehler bei Relation → UNCERTAIN mit low confidence.""" + claim_a = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Claim A.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com/a", + ) + claim_b = Claim( + research_run_id=claim_a.research_run_id, + source_id=uuid4(), + claim_text="Claim B.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com/b", + ) + failing_provider = MagicMock() + failing_provider.complete = AsyncMock(side_effect=RuntimeError("LLM down")) + failing_provider.model = "test-model" + + stage = Stage7Clustering( + llm_provider=failing_provider, + config=MagicMock(), + research_run_id=claim_a.research_run_id, + claims=[claim_a, claim_b], + ) + clusters = [{"label": "Test", "claim_ids": [str(claim_a.id), str(claim_b.id)]}] + relations = asyncio_run(stage._analyze_relations(clusters)) + assert len(relations) == 1 + assert relations[0]["relation_type"] == "UNCERTAIN" + assert relations[0]["confidence"] == 0.1 + + def test_single_claim_cluster_no_relations(self) -> None: + """Cluster mit nur 1 Claim → keine Relations.""" + claim = Claim( + research_run_id=uuid4(), + source_id=uuid4(), + claim_text="Solo-Claim.", + evidence_span="Evidenz", + claim_type=ClaimType.FACT, + source_url="https://example.com", + ) + stage = Stage7Clustering( + llm_provider=_mock_llm_provider(json.dumps({"clusters": []})), + config=MagicMock(), + research_run_id=claim.research_run_id, + claims=[claim], + ) + clusters = [{"label": "Solo", "claim_ids": [str(claim.id)]}] + relations = asyncio_run(stage._analyze_relations(clusters)) + assert len(relations) == 0 + + +# --------------------------------------------------------------------------- +# Test Group 81-90: System Prompts & Constants +# --------------------------------------------------------------------------- + + +class TestPrompts: + """Tests für System-Prompts und Konstanten.""" + + def test_cluster_system_prompt_not_empty(self) -> None: + assert CLUSTER_SYSTEM_PROMPT and len(CLUSTER_SYSTEM_PROMPT) > 0 + assert "Cluster" in CLUSTER_SYSTEM_PROMPT or "clustering" in CLUSTER_SYSTEM_PROMPT.lower() + + def test_cluster_user_prompt_format(self) -> None: + """Cluster-Prompt muss claims_list-Platzhalter haben.""" + prompt = CLUSTER_USER_PROMPT.format(claims_list="Test") + assert "Test" in prompt + assert "JSON" in prompt + + def test_relation_system_prompt_not_empty(self) -> None: + assert RELATION_SYSTEM_PROMPT and len(RELATION_SYSTEM_PROMPT) > 0 + assert "SUPPORTS" in RELATION_SYSTEM_PROMPT or "CONTRADICTS" in RELATION_SYSTEM_PROMPT + + def test_relation_user_prompt_format(self) -> None: + """Relation-Prompt muss alle Platzhalter haben.""" + prompt = RELATION_USER_PROMPT.format( + url_a="https://a.com", + text_a="Claim A", + type_a="fact", + url_b="https://b.com", + text_b="Claim B", + type_b="opinion", + ) + assert "Claim A" in prompt + assert "Claim B" in prompt + assert "SUPPORTS" in prompt + assert "CONTRADICTS" in prompt + assert "DUPLICATE" in prompt + assert "UNCERTAIN" in prompt + + +# --------------------------------------------------------------------------- +# Test Group 91-100: ClaimRelationType enum (storage models) +# --------------------------------------------------------------------------- + + +class TestClaimRelationTypeEnum: + """Tests für die ClaimRelationType-Enum (pydantic models).""" + + def test_all_values_present(self) -> None: + from nsct.models.source_independence import CitationEdgeType + values = {e.value for e in CitationEdgeType} + assert "syndicated" in values + + def test_values_are_lowercase(self) -> None: + from nsct.models.source_independence import CitationEdgeType + for e in CitationEdgeType: + assert e.value.islower() + + # All remaining tests are skipped when sqlalchemy is unavailable + def test_clustermodule_exists(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimClusterModel + assert ClaimClusterModel is not None + + def test_claimrelationmodel_exists(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimRelationModel + assert ClaimRelationModel is not None + + def test_claimnlumodel_exists(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimNLUModel + assert ClaimNLUModel is not None + + def test_clustermodule_table_name(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimClusterModel + assert ClaimClusterModel.__tablename__ == "claim_clusters" + + def test_claimrelationmodule_table_name(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimRelationModel + assert ClaimRelationModel.__tablename__ == "claim_relations" + + def test_claimnlumodule_table_name(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimNLUModel + assert ClaimNLUModel.__tablename__ == "claim_nlu_numeric" + + def test_clustermodule_all_columns(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimClusterModel + columns = {c.name for c in ClaimClusterModel.__table__.columns} + assert "id" in columns + assert "research_run_id" in columns + assert "cluster_label" in columns + assert "representative_claim_id" in columns + assert "claim_count" in columns + assert "created_at" in columns + + def test_claimrelationmodel_all_columns(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimRelationModel + columns = {c.name for c in ClaimRelationModel.__table__.columns} + assert "id" in columns + assert "source_claim_id" in columns + assert "target_claim_id" in columns + assert "relation_type" in columns + assert "confidence" in columns + assert "reason" in columns + assert "cluster_id" in columns + assert "created_at" in columns + + def test_claimnlumodel_all_columns(self) -> None: + pytest.importorskip("sqlalchemy") + from nsct.storage.models import ClaimNLUModel + columns = {c.name for c in ClaimNLUModel.__table__.columns} + assert "id" in columns + assert "claim_id" in columns + assert "normalized_numeric_value" in columns + assert "original_numeric_text" in columns + assert "unit" in columns + assert "created_at" in columns \ No newline at end of file