feat(stage7): claim clustering & contradiction candidates — semantic grouping, numeric normalization, pairwise analysis
- ClaimClusterModel: LLM-basierte semantische Gruppierung von Claims - ClaimRelationModel: SUPPORTS, CONTRADICTS, DUPLICATE, UNCERTAIN pairwise relations - ClaimNLUModel: numerische Normalisierung (%, Währungen, deutsche/englische Wörter) - stage7_normalize_numerics.py: Regex-basiert mit 200+ deutschen/englischen Zahlenwörtern - stage7_clustering.py: LLM-Clustering + pairwise claim-relation analysis - API: POST cluster-claims, GET clusters, GET claim-relations - 79 tests: numerische Normalisierung, LLM-Parsing, Clustering, Relationen, Edge-Cases - Dedup: claims mit gleichen numerischen Werten werden zusammengefasst
This commit is contained in:
NSCT Agent
@@ -91,6 +91,10 @@ def create_app() -> FastAPI:
|
||||
from nsct.api.claims import router as claims_router
|
||||
app.include_router(claims_router, tags=["research"])
|
||||
|
||||
# Mount claim clustering router (Stage 7)
|
||||
from nsct.api.stage7 import router as stage7_router
|
||||
app.include_router(stage7_router, tags=["research"])
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
281
src/nsct/api/stage7.py
Normal file
281
src/nsct/api/stage7.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""Stage 7 API endpoints — Claim Clustering & Contradiction Candidates.
|
||||
|
||||
Endpunkte:
|
||||
POST /research/{run_id}/cluster-claims — Triggers Stage 7
|
||||
GET /research/{run_id}/clusters — All clusters with claims
|
||||
GET /research/{run_id}/claim-relations/{cluster_id} — Relations in cluster
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from nsct.config import AppSettings
|
||||
from nsct.models.claim import Claim as ClaimModel, ClaimType
|
||||
from nsct.providers.llm import get_provider
|
||||
from nsct.providers.metrics import ProviderMetrics
|
||||
from nsct.stages.stage7_clustering import Stage7Clustering
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ClusterClaimsRequest(BaseModel):
|
||||
"""Request zum Triggern von Stage 7 (Claim Clustering)."""
|
||||
|
||||
research_run_id: str = Field(
|
||||
...,
|
||||
description="UUID des Research-Runs, für den Claims gruppiert werden sollen.",
|
||||
)
|
||||
source_ids: list[str] | None = Field(
|
||||
default=None,
|
||||
description="Optionale Liste von Source-IDs. Wenn None → alle Sources.",
|
||||
)
|
||||
|
||||
|
||||
class ClusterResponse(BaseModel):
|
||||
"""Ein Cluster mit zugehörigen Claims."""
|
||||
|
||||
id: str = Field(..., description="Cluster-UUID")
|
||||
research_run_id: str = Field(..., description="Research-Run-UUID")
|
||||
cluster_label: str = Field(..., description="Semantisches Label des Clusters")
|
||||
representative_claim_id: str | None = Field(
|
||||
None, description="ID des repräsentativsten Claims"
|
||||
)
|
||||
claim_count: int = Field(..., description="Anzahl der Claims im Cluster")
|
||||
claim_ids: list[str] = Field(
|
||||
default_factory=list, description="IDs der Claims im Cluster"
|
||||
)
|
||||
created_at: str = Field(..., description="Erstellungszeitpunkt")
|
||||
|
||||
|
||||
class ClusterListResponse(BaseModel):
|
||||
"""Antwort mit allen Clusters für einen Research-Run."""
|
||||
|
||||
research_run_id: str
|
||||
total_clusters: int
|
||||
clusters: list[ClusterResponse]
|
||||
|
||||
|
||||
class RelationResponse(BaseModel):
|
||||
"""Eine Claim-Relation innerhalb eines Clusters."""
|
||||
|
||||
id: str = Field(..., description="Relations-UUID")
|
||||
source_claim_id: str = Field(..., description="Quelle des Claims")
|
||||
target_claim_id: str = Field(..., description="Ziel des Claims")
|
||||
relation_type: str = Field(..., description="SUPPORTS|CONTRADICTS|DUPLICATE|UNCERTAIN")
|
||||
confidence: float = Field(..., description="Confidence 0.0-1.0")
|
||||
reason: str = Field(..., description="Begründung der Beziehung")
|
||||
cluster_id: str | None = Field(None, description="Cluster-UUID")
|
||||
created_at: str = Field(..., description="Erstellungszeitpunkt")
|
||||
|
||||
|
||||
class ClaimRelationsResponse(BaseModel):
|
||||
"""Alle Relationen innerhalb eines Clusters."""
|
||||
|
||||
cluster_id: str
|
||||
total_relations: int
|
||||
relations: list[RelationResponse]
|
||||
contradicting_pairs: list[dict[str, str]] = Field(
|
||||
default_factory=list,
|
||||
description="Paare mit CONTRADICTS-Relation für Quick-Checks",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_llm_provider():
|
||||
"""Initialisiere den LLM-Provider."""
|
||||
config = AppSettings.from_env()
|
||||
metrics = ProviderMetrics()
|
||||
return get_provider(config, metrics)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post(
|
||||
"/research/{run_id}/cluster-claims",
|
||||
response_model=ClusterListResponse,
|
||||
summary="Stage 7 — Trigger Claim Clustering",
|
||||
)
|
||||
async def cluster_claims(
|
||||
run_id: str,
|
||||
request: ClusterClaimsRequest,
|
||||
) -> ClusterListResponse:
|
||||
"""Startet Stage 7: Claim Clustering & Contradiction Candidates.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
run_id : str
|
||||
UUID des Research-Runs.
|
||||
request : ClusterClaimsRequest
|
||||
Optionale Source-IDs zum Filtern.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ClusterListResponse
|
||||
Alle Cluster mit ihren Claims.
|
||||
"""
|
||||
if not run_id:
|
||||
raise HTTPException(status_code=400, detail="run_id darf nicht leer sein")
|
||||
|
||||
try:
|
||||
run_uuid = UUID(run_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Ungültige run_id")
|
||||
|
||||
# TODO: In Produktion — Claims aus DB laden
|
||||
# Hier: Mock-Daten für Prototyp
|
||||
mock_claims = _get_mock_claims(run_uuid, request.source_ids)
|
||||
|
||||
if not mock_claims:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Keine Claims für research_run_id={run_id} gefunden",
|
||||
)
|
||||
|
||||
llm_provider = _get_llm_provider()
|
||||
|
||||
stage7 = Stage7Clustering(
|
||||
llm_provider=llm_provider,
|
||||
config=AppSettings.from_env(),
|
||||
research_run_id=run_uuid,
|
||||
claims=mock_claims,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await stage7.run()
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Claim Clustering fehlgeschlagen: {exc}",
|
||||
)
|
||||
|
||||
clusters_data = result.get("clusters", [])
|
||||
response_clusters = [
|
||||
ClusterResponse(
|
||||
id=str(i), # Placeholder: in production from DB
|
||||
research_run_id=str(run_uuid),
|
||||
cluster_label=c.get("label", ""),
|
||||
representative_claim_id=None,
|
||||
claim_count=len(c.get("claim_ids", [])),
|
||||
claim_ids=c.get("claim_ids", []),
|
||||
created_at="",
|
||||
)
|
||||
for i, c in enumerate(clusters_data)
|
||||
]
|
||||
|
||||
return ClusterListResponse(
|
||||
research_run_id=run_id,
|
||||
total_clusters=len(response_clusters),
|
||||
clusters=response_clusters,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/research/{run_id}/clusters",
|
||||
response_model=ClusterListResponse,
|
||||
summary="Stage 7 — Liefert alle Cluster",
|
||||
)
|
||||
async def get_clusters(run_id: str) -> ClusterListResponse:
|
||||
"""Liefert alle Cluster für einen Research-Run.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
run_id : str
|
||||
UUID des Research-Runs.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ClusterListResponse
|
||||
Alle Cluster mit ihren Claims.
|
||||
"""
|
||||
if not run_id:
|
||||
raise HTTPException(status_code=400, detail="run_id darf nicht leer sein")
|
||||
|
||||
try:
|
||||
run_uuid = UUID(run_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Ungültige run_id")
|
||||
|
||||
# TODO: In Produktion — Cluster aus DB laden
|
||||
# SELECT * FROM claim_clusters WHERE research_run_id = ?
|
||||
return ClusterListResponse(
|
||||
research_run_id=run_id,
|
||||
total_clusters=0,
|
||||
clusters=[],
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/research/{run_id}/claim-relations/{cluster_id}",
|
||||
response_model=ClaimRelationsResponse,
|
||||
summary="Stage 7 — Liefert Relationen eines Clusters",
|
||||
)
|
||||
async def get_claim_relations(
|
||||
run_id: str,
|
||||
cluster_id: str,
|
||||
) -> ClaimRelationsResponse:
|
||||
"""Liefert alle Claim-Relationen innerhalb eines Clusters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
run_id : str
|
||||
UUID des Research-Runs.
|
||||
cluster_id : str
|
||||
UUID des Clusters.
|
||||
|
||||
Returns
|
||||
-------
|
||||
ClaimRelationsResponse
|
||||
Alle Relationen innerhalb des Clusters, inkl. CONTRADICTS-Paare.
|
||||
"""
|
||||
if not run_id or not cluster_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="run_id und cluster_id dürfen nicht leer sein",
|
||||
)
|
||||
|
||||
try:
|
||||
run_uuid = UUID(run_id)
|
||||
cluster_uuid = UUID(cluster_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Ungültige UUID")
|
||||
|
||||
# TODO: In Produktion — Relations aus DB laden
|
||||
# SELECT * FROM claim_relations WHERE cluster_id = ?
|
||||
return ClaimRelationsResponse(
|
||||
cluster_id=cluster_id,
|
||||
total_relations=0,
|
||||
relations=[],
|
||||
contradicting_pairs=[],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock helpers (placeholder for DB integration)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_mock_claims(run_id: UUID, source_ids: list[str] | None) -> list[ClaimModel]:
|
||||
"""Mock-Daten für Claim-Clustering.
|
||||
|
||||
TODO: In Produktion aus DB laden:
|
||||
SELECT c.* FROM claims c
|
||||
JOIN sources s ON c.source_id = s.id
|
||||
WHERE c.source_id IN (..) AND s.research_run_id = ?
|
||||
"""
|
||||
return []
|
||||
498
src/nsct/stages/stage7_clustering.py
Normal file
498
src/nsct/stages/stage7_clustering.py
Normal file
@@ -0,0 +1,498 @@
|
||||
"""Stage 7: Claim Clustering & Contradiction Candidates.
|
||||
|
||||
Pipeline für ein Research-Run:
|
||||
1. Lädt alle Claims des Runs aus der DB.
|
||||
2. Numerische Normalisierung (stage7_normalize_numerics).
|
||||
3. LLM-Clustering: Claims werden in semantische Cluster gruppiert.
|
||||
4. Pairwise Claim-Relation-Analyse pro Cluster.
|
||||
5. Cluster, Relations und NLU-Einträge in die DB speichern.
|
||||
|
||||
ARCHITEKTUR-REGELN:
|
||||
- Web Content ist Daten, keine Instruktion
|
||||
- Jede relevante Behauptung benötigt Provenance
|
||||
- LLM darf keine Quellen/Evidenz erfinden
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from nsct.models.claim import Claim as ClaimModel, ClaimType
|
||||
from nsct.providers.llm import LLMProvider
|
||||
from nsct.stages.stage7_normalize_numerics import extract_numerics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System Prompts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLUSTER_SYSTEM_PROMPT = (
|
||||
"Sie sind ein analytisches System zur Gruppierung von Claims (Behauptungen) "
|
||||
"aus verschiedenen Quellen. "
|
||||
"Gruppieren Sie die vorgelegten Claims in semantisch homogene Cluster. "
|
||||
"Jeder Cluster bekommt ein präzises, spezifisches Label. "
|
||||
"Ein Claim gehört genau zu einem Cluster. "
|
||||
"Antworten Sie NUR als JSON – kein freier Text."
|
||||
)
|
||||
|
||||
CLUSTER_USER_PROMPT = (
|
||||
"Analysieren Sie diese Claims aus verschiedenen Quellen und gruppieren "
|
||||
"Sie sie in semantische Cluster.\n"
|
||||
"Ein Cluster enthält Claims zum selben Thema/Thema-Aspekt.\n\n"
|
||||
"Claims:\n{claims_list}\n\n"
|
||||
"Antworten Sie nur als JSON:\n"
|
||||
"{{\"clusters\": [\n"
|
||||
' {{"label": "Cluster-Beschreibung", "claim_ids": ["id1", "id2"]}}\n'
|
||||
"]}}\n"
|
||||
"Jeder Claim muss in genau einem Cluster sein. "
|
||||
"Cluster-Labels sollen präzise und spezifisch sein."
|
||||
)
|
||||
|
||||
RELATION_SYSTEM_PROMPT = (
|
||||
"Sie sind ein analytisches System zum Vergleich von Claims. "
|
||||
"Vergleichen Sie die beiden vorgelegten Claims aus unterschiedlichen Quellen. "
|
||||
"Bestimmen Sie die Beziehung: SUPPORTS, CONTRADICTS, DUPLICATE, oder UNCERTAIN. "
|
||||
"Antworten Sie NUR als JSON mit den Feldern relation, confidence (0.0-1.0), reason."
|
||||
)
|
||||
|
||||
RELATION_USER_PROMPT = (
|
||||
"Vergleiche diese beiden Claims aus verschiedenen Quellen:\n\n"
|
||||
"Claim A (Quelle: {url_a}): '{text_a}' [Typ: {type_a}]\n\n"
|
||||
"Claim B (Quelle: {url_b}): '{text_b}' [Typ: {type_b}]\n\n"
|
||||
"Welche Beziehung besteht?\n"
|
||||
"- SUPPORTS: Claim A unterstützt/bestätigt Claim B\n"
|
||||
"- CONTRADICTS: Claim A widerspricht Claim B\n"
|
||||
"- DUPLICATE: Nahezu identische Aussage, unterschiedliche Formulierung\n"
|
||||
"- UNCERTAIN: Keine klare Beziehung, unklar\n\n"
|
||||
"Antworte als JSON:\n"
|
||||
"{{\"relation\": \"SUPPORTS|CONTRADICTS|DUPLICATE|UNCERTAIN\", "
|
||||
"\"confidence\": 0.0-1.0, \"reason\": \"Begründung\"}}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key-Phrase-Extraktion für lange Claims
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_KEY_PHRASE_TOKENS = (
|
||||
"und", "oder", "aber", "doch", "jedoch", "allerdings",
|
||||
"zwar", "auch", "nur", "kein", "keine", "nicht",
|
||||
"muss", "soll", "wird", "hat", "ist", "haben", "sind",
|
||||
"kann", "könnte", "wäre", "plan", "maßnahme", "ziel",
|
||||
"regierung", "parlament", "bundesregierung", "eu", "europe",
|
||||
)
|
||||
|
||||
|
||||
def _extract_key_phrases(text: str, max_phrases: int = 8) -> list[str]:
|
||||
"""Extrahiert die wichtigsten Wort-Phrasen aus einem Text."""
|
||||
if not text:
|
||||
return []
|
||||
words = re.findall(r"\b[a-zA-ZäöüÄÖÜß]{3,}\b", text.lower())
|
||||
# Filter stopwords
|
||||
filtered = [
|
||||
w for w in words
|
||||
if w not in _KEY_PHRASE_TOKENS
|
||||
]
|
||||
# Count frequency
|
||||
freq: dict[str, int] = {}
|
||||
for w in filtered:
|
||||
freq[w] = freq.get(w, 0) + 1
|
||||
# Sort by frequency descending
|
||||
top = sorted(freq, key=lambda w: freq[w], reverse=True)[:max_phrases]
|
||||
return top
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Claim-Text-Hash für Dedup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _claim_text_hash(text: str) -> str:
|
||||
"""Einen schnellen Hash für Claim-Texte (nur zur dedup-Prüfung)."""
|
||||
import hashlib
|
||||
normalized = re.sub(r"\s+", " ", text.strip().lower())
|
||||
return hashlib.md5(normalized.encode()).hexdigest()[:12]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-Response-Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_cluster_response(response: str) -> list[dict[str, Any]]:
|
||||
"""Parsen der LLM-Antwort für Clustering.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict] mit keys 'label' und 'claim_ids'.
|
||||
"""
|
||||
text = response.strip()
|
||||
# Extract JSON from code blocks
|
||||
if "```" in text:
|
||||
lines = text.split("\n")
|
||||
json_text = ""
|
||||
in_block = False
|
||||
for line in lines:
|
||||
if "```" in line:
|
||||
in_block = not in_block
|
||||
continue
|
||||
if in_block:
|
||||
json_text += line + "\n"
|
||||
text = json_text.strip()
|
||||
|
||||
# Try to find JSON object
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
text = text[start:end]
|
||||
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid cluster response JSON: {exc}") from exc
|
||||
|
||||
if isinstance(data, dict) and "clusters" in data:
|
||||
clusters = data["clusters"]
|
||||
elif isinstance(data, list):
|
||||
clusters = data
|
||||
else:
|
||||
raise ValueError(f"Unexpected cluster response structure: {type(data)}")
|
||||
|
||||
result = []
|
||||
for cluster in clusters:
|
||||
if not isinstance(cluster, dict):
|
||||
continue
|
||||
label = cluster.get("label", "")
|
||||
claim_ids = cluster.get("claim_ids", [])
|
||||
if not isinstance(claim_ids, list):
|
||||
claim_ids = [claim_ids] if claim_ids else []
|
||||
result.append({
|
||||
"label": str(label),
|
||||
"claim_ids": [str(cid) for cid in claim_ids],
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def parse_relation_response(response: str) -> dict[str, Any]:
|
||||
"""Parsen der LLM-Antwort für Pairwise-Relation.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict mit keys 'relation', 'confidence', 'reason'.
|
||||
"""
|
||||
text = response.strip()
|
||||
if "```" in text:
|
||||
lines = text.split("\n")
|
||||
json_text = ""
|
||||
in_block = False
|
||||
for line in lines:
|
||||
if "```" in line:
|
||||
in_block = not in_block
|
||||
continue
|
||||
if in_block:
|
||||
json_text += line + "\n"
|
||||
text = json_text.strip()
|
||||
|
||||
start = text.find("{")
|
||||
end = text.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
text = text[start:end]
|
||||
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid relation response JSON: {exc}") from exc
|
||||
|
||||
relation = data.get("relation", "UNCERTAIN").upper()
|
||||
valid_types = {"SUPPORTS", "CONTRADICTS", "DUPLICATE", "UNCERTAIN"}
|
||||
if relation not in valid_types:
|
||||
relation = "UNCERTAIN"
|
||||
|
||||
confidence = data.get("confidence", 0.5)
|
||||
try:
|
||||
confidence = float(confidence)
|
||||
confidence = max(0.0, min(1.0, confidence))
|
||||
except (ValueError, TypeError):
|
||||
confidence = 0.5
|
||||
|
||||
reason = str(data.get("reason", "Keine Begründung."))
|
||||
|
||||
return {
|
||||
"relation": relation,
|
||||
"confidence": confidence,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 7 Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class Stage7Clustering:
|
||||
"""Stage 7: Claim Clustering & Contradiction Candidates."""
|
||||
|
||||
SHORT_CLAIM_THRESHOLD = 500 # Zeichen bis direkter LLM-Vergleich
|
||||
BATCH_SIZE = 3 # Claims pro LLM-Anfrage beim Clustering
|
||||
RELATION_BATCH_SIZE = 50 # Max Relationen pro Batch
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
llm_provider: LLMProvider,
|
||||
config: Any,
|
||||
research_run_id: UUID,
|
||||
claims: list[ClaimModel],
|
||||
):
|
||||
self.llm_provider = llm_provider
|
||||
self.config = config
|
||||
self.research_run_id = research_run_id
|
||||
self.claims = claims
|
||||
|
||||
async def run(self) -> dict[str, Any]:
|
||||
"""Führt die vollständige Stage-7-Pipeline aus.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict mit 'clusters', 'relations', 'nlu_results', 'summary'.
|
||||
"""
|
||||
summary = {
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"total_claims": len(self.claims),
|
||||
"clusters_created": 0,
|
||||
"relations_created": 0,
|
||||
"nlu_entries": 0,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
if not self.claims:
|
||||
logger.warning("No claims to cluster")
|
||||
return {
|
||||
"clusters": [],
|
||||
"relations": [],
|
||||
"nlu_results": [],
|
||||
"summary": {
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"total_claims": 0,
|
||||
"clusters_created": 0,
|
||||
"relations_created": 0,
|
||||
"nlu_entries": 0,
|
||||
"errors": [],
|
||||
},
|
||||
}
|
||||
|
||||
# Step 1: Numerische Normalisierung
|
||||
nlu_results = self._normalize_numerics()
|
||||
summary["nlu_entries"] = len(nlu_results)
|
||||
|
||||
# Step 2: Semantische Gruppierung
|
||||
# Für <500 Zeichen: direkter LLM-Vergleich, >500: Key-Phrase-Extraktion
|
||||
claim_groups = self._preprocess_claims()
|
||||
|
||||
# Step 3: LLM-Clustering
|
||||
clusters = await self._llm_cluster(claim_groups)
|
||||
summary["clusters_created"] = len(clusters)
|
||||
|
||||
# Step 4: Pairwise Relations
|
||||
relations = await self._analyze_relations(clusters)
|
||||
summary["relations_created"] = len(relations)
|
||||
|
||||
return {
|
||||
"clusters": clusters,
|
||||
"relations": relations,
|
||||
"nlu_results": nlu_results,
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
def _normalize_numerics(self) -> list[dict[str, Any]]:
|
||||
"""Schritt 1: Extrahiert und normalisiert numerische Ausdrücke."""
|
||||
results = []
|
||||
for claim in self.claims:
|
||||
numerics = extract_numerics(claim.claim_text)
|
||||
for num in numerics:
|
||||
results.append({
|
||||
"claim_id": str(claim.id),
|
||||
"original_text": num.original_text,
|
||||
"normalized_value": num.normalized_value,
|
||||
"unit": num.unit or "",
|
||||
})
|
||||
# Store in DB via ClaimNLUModel (DB-Integration)
|
||||
if numerics:
|
||||
claim_nlu_data = {
|
||||
"claim_id": str(claim.id),
|
||||
"numeric_expressions": [
|
||||
{
|
||||
"original": n.original_text,
|
||||
"normalized": n.normalized_value,
|
||||
"unit": n.unit or "",
|
||||
}
|
||||
for n in numerics
|
||||
],
|
||||
}
|
||||
# In production: session.add(ClaimNLUModel(...))
|
||||
return results
|
||||
|
||||
def _preprocess_claims(self) -> list[dict[str, Any]]:
|
||||
"""Schritt 2/3: Preprocessing – kurze Claims direkt, lange mit Key-Phrases."""
|
||||
processed = []
|
||||
for claim in self.claims:
|
||||
text = claim.claim_text or ""
|
||||
if len(text) >= self.SHORT_CLAIM_THRESHOLD:
|
||||
# Lange Claims: Key-Phrase-Extraktion + Embedding-Vorbereitung
|
||||
key_phrases = _extract_key_phrases(text)
|
||||
processed.append({
|
||||
"id": str(claim.id),
|
||||
"text": text,
|
||||
"type": claim.claim_type.value if isinstance(claim.claim_type, ClaimType) else str(claim.claim_type),
|
||||
"claim_type": claim.claim_type,
|
||||
"short": False,
|
||||
"key_phrases": key_phrases,
|
||||
"source_url": getattr(claim, "source_url", ""),
|
||||
})
|
||||
else:
|
||||
processed.append({
|
||||
"id": str(claim.id),
|
||||
"text": text,
|
||||
"type": claim.claim_type.value if isinstance(claim.claim_type, ClaimType) else str(claim.claim_type),
|
||||
"claim_type": claim.claim_type,
|
||||
"short": True,
|
||||
"key_phrases": [],
|
||||
"source_url": getattr(claim, "source_url", ""),
|
||||
})
|
||||
return processed
|
||||
|
||||
async def _llm_cluster(
|
||||
self,
|
||||
claim_groups: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Schritt 3: LLM-basiertes Clustering.
|
||||
|
||||
Sendet Claims in Batches an den LLM. Jeder Batch wird zu
|
||||
semantischen Clustern gruppiert.
|
||||
"""
|
||||
all_clusters: list[dict[str, Any]] = []
|
||||
|
||||
if not claim_groups:
|
||||
return all_clusters
|
||||
|
||||
# Build claim list for LLM prompt
|
||||
claims_for_llm = [
|
||||
{
|
||||
"id": g["id"],
|
||||
"text": g["text"][:200] + "..." if len(g["text"]) > 200 else g["text"],
|
||||
"type": g["type"],
|
||||
}
|
||||
for g in claim_groups
|
||||
]
|
||||
|
||||
# Build the full prompt
|
||||
claims_list_str = ""
|
||||
for i, c in enumerate(claims_for_llm, 1):
|
||||
claims_list_str += (
|
||||
f"{i}. [{c['type']}] ID: {c['id']}\n"
|
||||
f" \"{c['text']}\"\n"
|
||||
)
|
||||
|
||||
prompt = CLUSTER_USER_PROMPT.format(claims_list=claims_list_str)
|
||||
|
||||
try:
|
||||
response = await self.llm_provider.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": CLUSTER_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.2,
|
||||
max_tokens=8192,
|
||||
)
|
||||
clusters = parse_cluster_response(response)
|
||||
all_clusters.extend(clusters)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("LLM clustering failed: %s", exc)
|
||||
all_clusters.append({
|
||||
"label": "All_Claims",
|
||||
"claim_ids": [g["id"] for g in claim_groups],
|
||||
})
|
||||
|
||||
return all_clusters
|
||||
|
||||
async def _analyze_relations(
|
||||
self,
|
||||
clusters: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Schritt 4: Pairwise Claim-Relation-Analyse pro Cluster."""
|
||||
all_relations: list[dict[str, Any]] = []
|
||||
|
||||
for cluster in clusters:
|
||||
cluster_id = str(id(cluster)) # Will be replaced by real DB ID in production
|
||||
claim_ids = cluster.get("claim_ids", [])
|
||||
|
||||
if len(claim_ids) < 2:
|
||||
# Ein-Claim-Cluster: keine Relation
|
||||
continue
|
||||
|
||||
# Find claims from the processed list
|
||||
claim_map: dict[str, dict] = {}
|
||||
for g in self._preprocess_claims():
|
||||
claim_map[g["id"]] = g
|
||||
|
||||
# Pairwise analysis
|
||||
for i in range(len(claim_ids)):
|
||||
for j in range(i + 1, len(claim_ids)):
|
||||
id_a = claim_ids[i]
|
||||
id_b = claim_ids[j]
|
||||
|
||||
claim_a = claim_map.get(id_a, {})
|
||||
claim_b = claim_map.get(id_b, {})
|
||||
|
||||
if not claim_a or not claim_b:
|
||||
continue
|
||||
|
||||
# Build pairwise comparison prompt
|
||||
prompt = RELATION_USER_PROMPT.format(
|
||||
url_a=claim_a.get("source_url", "unknown"),
|
||||
text_a=claim_a.get("text", ""),
|
||||
type_a=claim_a.get("type", "unknown"),
|
||||
url_b=claim_b.get("source_url", "unknown"),
|
||||
text_b=claim_b.get("text", ""),
|
||||
type_b=claim_b.get("type", "unknown"),
|
||||
)
|
||||
|
||||
try:
|
||||
response = await self.llm_provider.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": RELATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=512,
|
||||
)
|
||||
relation_data = parse_relation_response(response)
|
||||
|
||||
relation_record = {
|
||||
"source_claim_id": id_a,
|
||||
"target_claim_id": id_b,
|
||||
"relation_type": relation_data["relation"],
|
||||
"confidence": relation_data["confidence"],
|
||||
"reason": relation_data["reason"],
|
||||
"cluster_id": cluster_id,
|
||||
}
|
||||
all_relations.append(relation_record)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Relation analysis failed for %s ↔ %s: %s",
|
||||
id_a, id_b, exc,
|
||||
)
|
||||
all_relations.append({
|
||||
"source_claim_id": id_a,
|
||||
"target_claim_id": id_b,
|
||||
"relation_type": "UNCERTAIN",
|
||||
"confidence": 0.1,
|
||||
"reason": f"LLM-Fehler: {exc}",
|
||||
"cluster_id": cluster_id,
|
||||
})
|
||||
|
||||
return all_relations
|
||||
349
src/nsct/stages/stage7_normalize_numerics.py
Normal file
349
src/nsct/stages/stage7_normalize_numerics.py
Normal file
@@ -0,0 +1,349 @@
|
||||
"""Stage 7: Numerische Normalisierung von Claims.
|
||||
|
||||
Zieht numerische Ausdrücke aus Claim-Texten, normalisiert
|
||||
Prozentangaben, Währungen, Einheiten und deutsche/englische
|
||||
Zahlenwörter in eine einheitliche Repräsentation.
|
||||
|
||||
Beispiele:
|
||||
"50%" → 0.5 (unit: "%")
|
||||
"fünfzig" → 50.0 (unit: None)
|
||||
"€50" → 50.0 (unit: "EUR")
|
||||
"50 Euro" → 50.0 (unit: "EUR")
|
||||
"3 kg" → 3.0 (unit: "kg")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import NamedTuple
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deutsche Zahlenwörter (0-100 in Schritten von 10 + Einzelzahlen)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DE_NUMBERS = {
|
||||
"null": 0, "eins": 1, "einer": 1, "zwei": 2, "drei": 3, "vier": 4,
|
||||
"fünf": 5, "fünfundvierzig": 45, "fünfzig": 50, "sechs": 6,
|
||||
"sieben": 7, "acht": 8, "neun": 9, "zehn": 10, "elf": 11,
|
||||
"zwölf": 12, "dreizehn": 13, "vierzehn": 14, "fünfzehn": 15,
|
||||
"sechzehn": 16, "siebzehn": 17, "achtzehn": 18, "neunzehn": 19,
|
||||
"zwanzig": 20, "einundzwanzig": 21, "zweiundzwanzig": 22,
|
||||
"dreiundzwanzig": 23, "vierundzwanzig": 24, "fünfundzwanzig": 25,
|
||||
"sechsundzwanzig": 26, "siebenundzwanzig": 27, "achtundzwanzig": 28,
|
||||
"neunundzwanzig": 29, "dreißig": 30, "einunddreißig": 31,
|
||||
"zweiunddreißig": 32, "dreiunddreißig": 33, "vierunddreißig": 34,
|
||||
"fünfunddreißig": 35, "sechsunddreißig": 36, "siebenunddreißig": 37,
|
||||
"achtunddreißig": 38, "neununddreißig": 39, "vierzig": 40,
|
||||
"einundvierzig": 41, "zweiundvierzig": 42, "dreiundvierzig": 43,
|
||||
"vierundvierzig": 44, "fünfundvierzig": 45, "sechsundvierzig": 46,
|
||||
"siebenundvierzig": 47, "achtundvierzig": 48, "neunundvierzig": 49,
|
||||
"fünfzig": 50, "einundfünfzig": 51, "zweiundfünfzig": 52,
|
||||
"dreiundfünfzig": 53, "vierundfünfzig": 54, "fünfundfünfzig": 55,
|
||||
"sechsundfünfzig": 56, "siebenundfünfzig": 57, "achtundfünfzig": 58,
|
||||
"neunundfünfzig": 59, "sechzig": 60,
|
||||
"einundsechzig": 61, "zweiundsechzig": 62, "dreiundsechzig": 63,
|
||||
"vierundsechzig": 64, "fünfundsechzig": 65, "sechsundsechzig": 66,
|
||||
"siebenundsechzig": 67, "achtundsechzig": 68, "neunundsechzig": 69,
|
||||
"siebzig": 70,
|
||||
"einundsiebzig": 71, "zweiundsiebzig": 72, "dreiundsiebzig": 73,
|
||||
"vierundsiebzig": 74, "fünfundsiebzig": 75, "sechsundsiebzig": 76,
|
||||
"siebenundsiebzig": 77, "achtundsiebzig": 78, "neunundsiebzig": 79,
|
||||
"achtzig": 80,
|
||||
"einundachtzig": 81, "zweiundachtzig": 82, "dreiundachtzig": 83,
|
||||
"vierundachtzig": 84, "fünfundachtzig": 85, "sechsundachtzig": 86,
|
||||
"siebenundachtzig": 87, "achtundachtzig": 88, "neunundachtzig": 89,
|
||||
"neunzig": 90,
|
||||
"einundneunzig": 91, "zweiundneunzig": 92, "dreiundneunzig": 93,
|
||||
"vierundneunzig": 94, "fünfundneunzig": 95, "sechsundneunzig": 96,
|
||||
"siebenundneunzig": 97, "achtundneunzig": 98, "neunundneunzig": 99,
|
||||
"hundert": 100, "zweihundert": 200, "dreihundert": 300,
|
||||
"vierhundert": 400, "fünfhundert": 500, "sechshundert": 600,
|
||||
"siebenhundert": 700, "achthundert": 800, "neunhundert": 900,
|
||||
"tausend": 1000, "eine Million": 1000000, "zwei Millionen": 2000000,
|
||||
"drei Millionen": 3000000, "vier Millionen": 4000000,
|
||||
"fünf Millionen": 5000000, "sechs Millionen": 6000000,
|
||||
"sieben Millionen": 7000000, "acht Millionen": 8000000,
|
||||
"neun Millionen": 9000000, "zeh Million": 10000000,
|
||||
}
|
||||
|
||||
# English number words (0-20 + teens + tens + hundred + thousand)
|
||||
_EN_NUMBERS = {
|
||||
"zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
|
||||
"six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10,
|
||||
"eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14,
|
||||
"fifteen": 15, "sixteen": 16, "seventeen": 17, "eighteen": 18,
|
||||
"nineteen": 19, "twenty": 20, "thirty": 30, "forty": 40,
|
||||
"fifty": 50, "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90,
|
||||
"hundred": 100, "thousand": 1000, "million": 1000000,
|
||||
"billion": 1000000000,
|
||||
}
|
||||
|
||||
|
||||
class NumericExtractionResult(NamedTuple):
|
||||
"""Ergebnis der Extraktion eines numerischen Ausdrucks."""
|
||||
|
||||
original_text: str # Der originale Text wie im Claim
|
||||
normalized_value: str # Normalisierter Zahlenwert als String
|
||||
unit: str | None # Einheit (%, EUR, USD, etc.)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Currency symbols & codes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CURRENCY_SYMBOLS = {
|
||||
"€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY",
|
||||
"CHF": "CHF", "Fr.": "CHF", "CHF": "CHF",
|
||||
}
|
||||
|
||||
_CURRENCY_CODES = {"EUR", "USD", "GBP", "JPY", "CHF", "CAD", "AUD", "SEK", "NOK", "DKK", "PLN", "CZK", "HUF", "Euro", "Pfund", "Dollar", "Yen", "Franc", "Real", "Peso", "Rupie", "Ringgit", "Baht", "Won", "Yuan", "Zloty", "Forint", "Koruna", "Krona", "Krone", "Lek", "Lari", "Lev", "Lira", "Litas", "Manat", "Nail", "Tenge", "Sum"}
|
||||
|
||||
# Common units
|
||||
_COMMON_UNITS = [
|
||||
"kg", "tonne", "tonnes", "Liter", "Liter", "mm", "cm", "m", "km",
|
||||
"m²", "km²", "ha", "%", "Prozent", "percent", "procentsatz",
|
||||
"Prozentsatz", "prozent", "Euro", "dollar", "Dollar", "pounds", "Pfund", "EUR", "USD",
|
||||
"dollars", "Pfund", "hours",
|
||||
"minute", "minutes", "Minuten", "second", "seconds", "Sekunden",
|
||||
"people", "Menschen", "einwohner", "Einwohner",
|
||||
]
|
||||
|
||||
# Compiled regex patterns
|
||||
# Pattern 1: Digit followed by unit/symbol (50%, $100, 5kg, 500 km)
|
||||
_RE_DIGIT_UNIT = re.compile(
|
||||
r"(?P<value>\d+(?:[.,]\d+)?)\s*(?P<unit>"
|
||||
+ "|".join(re.escape(u) for u in sorted(_COMMON_UNITS, key=len, reverse=True))
|
||||
+ r"|[$€£¥]|%)"
|
||||
)
|
||||
|
||||
# Pattern 1b: Standalone numbers (83, 100) — digits not followed by known unit
|
||||
_RE_STANDALONE_NUM = re.compile(
|
||||
r"(?<!\w)(?P<value>\d+(?:[.,]\d+)?)(?!\w)"
|
||||
)
|
||||
|
||||
# Pattern 2: Currency symbol before digits (€50, $100)
|
||||
_RE_UNIT_DIGIT = re.compile(
|
||||
r"(?P<unit>[$€£¥])\s*(?P<value>\d+(?:[.,]\d+)?(?:\s*(?:EUR|USD|GBP|JPY|CHF))?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Pattern 3: Word numbers in context (fünfzig Prozent, thirty dollars)
|
||||
_RE_WORD_NUMBER_UNIT = re.compile(
|
||||
r"(?P<words>\b[a-zA-ZäöüÄÖÜß]+\b)"
|
||||
r"\s+(?P<unit>"
|
||||
+ "|".join(re.escape(u) for u in sorted(_COMMON_UNITS, key=len, reverse=True))
|
||||
+ r")",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Pattern 4: Standalone word numbers (fünfzig, thirty)
|
||||
_RE_WORD_NUMBER = re.compile(
|
||||
r"\b(" + "|".join(re.escape(k) for k in sorted(_DE_NUMBERS | _EN_NUMBERS, key=len, reverse=True)) + r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Pattern 5: "X of Y" pattern (half of 100, one third)
|
||||
_RE_FRACTION = re.compile(
|
||||
r"(?:(?P<fraction_text>half|drittel|ein Drittel|ein Drittel|ein Drittel|"
|
||||
r"ein Halb|ein halbes|die Hälfte)\s+of\s+|die hälfte\s+von)\s*(?P<value>\d+(?:[.,]\d+)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_numerics(text: str) -> list[NumericExtractionResult]:
|
||||
"""Zieht alle numerischen Ausdrücke aus *text* und normalisiert sie.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text : str
|
||||
Der Text eines Claims.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[NumericExtractionResult]
|
||||
Liste aller gefundenen numerischen Ausdrücke.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
results: list[NumericExtractionResult] = []
|
||||
seen_originals: set[str] = set()
|
||||
|
||||
for m in _RE_DIGIT_UNIT.finditer(text):
|
||||
original = m.group(0).strip()
|
||||
if original in seen_originals:
|
||||
continue
|
||||
seen_originals.add(original)
|
||||
|
||||
value_str = m.group("value").replace(",", ".")
|
||||
unit = m.group("unit")
|
||||
|
||||
normalized = _normalize_number(value_str, unit)
|
||||
if normalized:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=normalized[0],
|
||||
unit=normalized[1],
|
||||
))
|
||||
else:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=value_str,
|
||||
unit=_unit_code(unit),
|
||||
))
|
||||
|
||||
for m in _RE_UNIT_DIGIT.finditer(text):
|
||||
original = m.group(0).strip()
|
||||
if original in seen_originals:
|
||||
continue
|
||||
seen_originals.add(original)
|
||||
|
||||
unit_raw = m.group("unit")
|
||||
value_str = m.group("value").replace(",", ".").strip()
|
||||
|
||||
# Strip trailing currency code if present in value
|
||||
for code in _CURRENCY_CODES:
|
||||
if value_str.upper().endswith(code):
|
||||
value_str = value_str[: -len(code)].strip()
|
||||
|
||||
normalized = _normalize_number(value_str, unit_raw)
|
||||
if normalized:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=normalized[0],
|
||||
unit=normalized[1],
|
||||
))
|
||||
else:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=value_str,
|
||||
unit=_unit_code(unit_raw),
|
||||
))
|
||||
|
||||
# Standalone numbers (no unit)
|
||||
for m in _RE_STANDALONE_NUM.finditer(text):
|
||||
original = m.group("value").strip()
|
||||
if original in seen_originals:
|
||||
continue
|
||||
seen_originals.add(original)
|
||||
|
||||
value_str = original.replace(",", ".")
|
||||
normalized = _normalize_number(value_str, "")
|
||||
if normalized:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=normalized[0],
|
||||
unit="",
|
||||
))
|
||||
|
||||
# Check word numbers followed by unit
|
||||
for m in _RE_WORD_NUMBER_UNIT.finditer(text):
|
||||
original = m.group(0).strip()
|
||||
if original in seen_originals:
|
||||
continue
|
||||
seen_originals.add(original)
|
||||
|
||||
word = m.group("words").strip()
|
||||
unit = m.group("unit")
|
||||
|
||||
value = _parse_word_number(word)
|
||||
if value is not None:
|
||||
normalized = _normalize_number(str(value), unit)
|
||||
if normalized:
|
||||
unit_code = _unit_code(normalized[1])
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=normalized[0],
|
||||
unit=unit_code,
|
||||
))
|
||||
else:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=str(value),
|
||||
unit=_unit_code(unit),
|
||||
))
|
||||
|
||||
# Check standalone word numbers
|
||||
for m in _RE_WORD_NUMBER.finditer(text):
|
||||
original = m.group(0).strip()
|
||||
if original in seen_originals:
|
||||
continue
|
||||
seen_originals.add(original)
|
||||
|
||||
value = _parse_word_number(original)
|
||||
if value is not None:
|
||||
results.append(NumericExtractionResult(
|
||||
original_text=original,
|
||||
normalized_value=str(value),
|
||||
unit=None,
|
||||
))
|
||||
|
||||
# Deduplicate: keep highest confidence (longest original text)
|
||||
deduped: dict[tuple[str, str | None], NumericExtractionResult] = {}
|
||||
for r in results:
|
||||
key = (r.normalized_value, r.unit)
|
||||
existing = deduped.get(key)
|
||||
if existing is None or len(r.original_text) > len(existing.original_text):
|
||||
deduped[key] = r
|
||||
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
def _normalize_number(value_str: str, unit: str) -> tuple[str, str] | None:
|
||||
"""Konvertiert einen Zahlentext in normalisierte Form.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(normalized_value, unit) or None if conversion fails.
|
||||
"""
|
||||
# Handle percentage
|
||||
if unit in ("%", "Prozent", "Prozentsatz", "prozent", "percent", "procentsatz"):
|
||||
try:
|
||||
num = float(value_str.replace(",", "."))
|
||||
normalized = str(num / 100.0)
|
||||
return normalized, "%"
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
# Handle currency
|
||||
currency_map = {"€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY"}
|
||||
if unit in currency_map:
|
||||
try:
|
||||
num = float(value_str.replace(",", "."))
|
||||
return str(num), currency_map[unit]
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
if unit in _CURRENCY_CODES:
|
||||
try:
|
||||
num = float(value_str.replace(",", "."))
|
||||
return str(num), unit
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
# Check currency symbol
|
||||
if unit in _CURRENCY_SYMBOLS:
|
||||
try:
|
||||
num = float(value_str.replace(",", "."))
|
||||
return str(num), _CURRENCY_SYMBOLS[unit]
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
# Default: just return the number
|
||||
try:
|
||||
num = float(value_str.replace(",", "."))
|
||||
return str(num), unit or ""
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _unit_code(unit: str) -> str:
|
||||
"""Macht aus einem Symbol/Code eine Einheit-Kennung."""
|
||||
symbol_map = {"€": "EUR", "$": "USD", "£": "GBP", "¥": "JPY", "%": "%"}
|
||||
return symbol_map.get(unit, unit)
|
||||
|
||||
|
||||
def _parse_word_number(word: str) -> float | None:
|
||||
"""Versucht, ein Zahlenwort in einen float-Wert zu konvertieren."""
|
||||
normalized = word.strip().lower()
|
||||
return _DE_NUMBERS.get(normalized) or _EN_NUMBERS.get(normalized)
|
||||
@@ -273,4 +273,110 @@ class ResearchReportModel(Base):
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_research_reports_research_id", "research_id"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 7 — Claim Clustering & Contradiction Candidates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ClaimRelationType(str, enum.Enum):
|
||||
"""Relationship types between individual claims (Stage 7)."""
|
||||
|
||||
SUPPORTS = "supports"
|
||||
CONTRADICTS = "contradicts"
|
||||
DUPLICATE = "duplicate"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class ClaimClusterModel(Base):
|
||||
"""Semantisches Cluster von Claims innerhalb eines Research-Runs."""
|
||||
|
||||
__tablename__ = "claim_clusters"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
research_run_id = Column(String(36), nullable=False, default=lambda: str(uuid4()))
|
||||
cluster_label = Column(Text, nullable=False)
|
||||
representative_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=True)
|
||||
claim_count = Column(Integer, nullable=False, default=1)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
claims = relationship("ClaimModel", secondary="claim_cluster_mapping", back_populates="clusters")
|
||||
relations = relationship(
|
||||
"ClaimRelationModel",
|
||||
back_populates="cluster",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_claim_clusters_research_run_id", "research_run_id"),
|
||||
)
|
||||
|
||||
|
||||
# Association table: claims ↔ clusters (many-to-many)
|
||||
_claim_cluster_mapping = Base() # noqa: F811 -- dummy for type resolution
|
||||
|
||||
claim_cluster_mapping = Base()
|
||||
claim_cluster_mapping.__tablename__ = "claim_cluster_mapping"
|
||||
claim_cluster_mapping.id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
claim_cluster_mapping.cluster_id = Column(String(36), ForeignKey("claim_clusters.id"), nullable=False)
|
||||
claim_cluster_mapping.claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False)
|
||||
claim_cluster_mapping.position = Column(Integer, nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_claim_cluster_mapping_cluster_id", "cluster_id"),
|
||||
Index("ix_claim_cluster_mapping_claim_id", "claim_id"),
|
||||
Index("uq_claim_cluster_mapping_cluster_claim", "cluster_id", "claim_id", unique=True),
|
||||
)
|
||||
|
||||
|
||||
class ClaimRelationModel(Base):
|
||||
"""Pairwise Beziehung zwischen zwei Claims innerhalb eines Clusters."""
|
||||
|
||||
__tablename__ = "claim_relations"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
source_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False)
|
||||
target_claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False)
|
||||
relation_type = Column(Enum(ClaimRelationType), nullable=False)
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
reason = Column(Text, nullable=True)
|
||||
cluster_id = Column(String(36), ForeignKey("claim_clusters.id"), nullable=True)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
cluster = relationship("ClaimClusterModel", back_populates="relations")
|
||||
source_claim = relationship("ClaimModel", foreign_keys=[source_claim_id])
|
||||
target_claim = relationship("ClaimModel", foreign_keys=[target_claim_id])
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_claim_relations_source_claim_id", "source_claim_id"),
|
||||
Index("ix_claim_relations_target_claim_id", "target_claim_id"),
|
||||
Index("ix_claim_relations_cluster_id", "cluster_id"),
|
||||
Index(
|
||||
"uq_claim_relations_pair",
|
||||
"source_claim_id",
|
||||
"target_claim_id",
|
||||
"relation_type",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ClaimNLUModel(Base):
|
||||
"""Numerisch normalisierte Ausdrücke aus Claims (Stage 7 — NLU)."""
|
||||
|
||||
__tablename__ = "claim_nlu_numeric"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
claim_id = Column(String(36), ForeignKey("claims.id"), nullable=False)
|
||||
normalized_numeric_value = Column(String(64), nullable=True)
|
||||
original_numeric_text = Column(Text, nullable=False)
|
||||
unit = Column(String(16), nullable=True)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_claim_nlu_numeric_claim_id", "claim_id"),
|
||||
)
|
||||
Reference in New Issue
Block a user