feat(stage6): source independence & citation graph — detect syndication, shared origins, text similarity
- SourceIndependenceModel: per-source independence_score (0.0-1.0), syndication_group_id, primary_source_id, content_hash, shared_urls - CitationGraphEdgeModel: directed edges (SYNDICATED, QUOTES, LINKS_TO, REPOST, SIMILAR_CONTENT) with confidence + evidence - Content-Hash (SHA-256): instant syndication detection for identical content - difflib Vorfilterung: >60% → LLM, >80% → high confidence, 100% → immediate syndication - LLM-Pairwise-Analysis: two-text-comparison for suspicious pairs only (bounded concurrency) - independence_score: 1.0 base, -0.4 for syndicated, -0.1 per high-similarity pair - Pydantic schemas: SourceIndependenceScore, CitationGraphEdge, SyntacticSimilarityResult, LlmSyndicationAnalysis, SourceIndependenceAnalysisResult - LLM response parser: handles JSON, markdown code blocks, partial/invalid JSON - 43 tests: content hash, similarity thresholds, LLM parsing, analyzer integration, edge cases, prompt templates
This commit is contained in:
NSCT Agent
172
src/nsct/models/source_independence.py
Normal file
172
src/nsct/models/source_independence.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""Pydantic v2 schemas — Source Independence & Citation Graph (Stage 6).
|
||||
|
||||
Jede Quelle erhält einen independence_score (0.0–1.0).
|
||||
Quellen mit gemeinsamem Ursprung werden im Citation Graph verknüpft.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class CitationEdgeType(str, Enum):
|
||||
"""Kanten-Typen im Source-Citation-Graph (Stage 6)."""
|
||||
|
||||
SYNDICATED = "syndicated"
|
||||
QUOTES = "quotes"
|
||||
LINKS_TO = "links_to"
|
||||
REPOST = "repost"
|
||||
SIMILAR_CONTENT = "similar_content"
|
||||
|
||||
|
||||
class SyndicationDirection(str, Enum):
|
||||
"""Richtung der Syndication zwischen zwei Quellen."""
|
||||
|
||||
SOURCE_B = "B->A"
|
||||
SOURCE_A = "A->B"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
class SourceIndependenceScore(BaseModel):
|
||||
"""Independence-Score für eine einzelne Quelle."""
|
||||
|
||||
source_id: UUID = Field(..., description="UUID der Quelle.")
|
||||
independence_score: float = Field(
|
||||
..., ge=0.0, le=1.0, description="Unabhängigkeits-Score (0.0=Duplikat, 1.0=vollständig unabhängig)"
|
||||
)
|
||||
syndication_group_id: UUID | None = Field(
|
||||
default=None, description="Gruppe syndizierter Quellen."
|
||||
)
|
||||
primary_source_id: UUID | None = Field(
|
||||
default=None, description="Primärquelle wenn diese Quelle syndiziert wurde."
|
||||
)
|
||||
content_hash: str | None = Field(
|
||||
default=None, description="SHA-256 Hash des Inhalts."
|
||||
)
|
||||
text_similarity_high_count: int = Field(
|
||||
default=0, description="Anzahl Quellen mit >80% Text-Ähnlichkeit."
|
||||
)
|
||||
shared_urls: list[str] = Field(
|
||||
default_factory=list, description="URLs die auf andere Quellen verweisen."
|
||||
)
|
||||
similarity_pairs: list[dict[str, Any]] = Field(
|
||||
default_factory=list, description="Paare mit >60% Similarität und LLM-Ergebnis."
|
||||
)
|
||||
|
||||
|
||||
class CitationGraphEdge(BaseModel):
|
||||
"""Gerichtete Kante zwischen zwei Quellen."""
|
||||
|
||||
id: UUID = Field(default_factory=uuid4)
|
||||
source_id: UUID = Field(..., description="Quelle die die Beziehung aufweist.")
|
||||
target_source_id: UUID = Field(..., description="Quelle die referenziert wird.")
|
||||
edge_type: CitationEdgeType = Field(..., description="Typ der Beziehung.")
|
||||
confidence: float = Field(..., ge=0.0, le=1.0, description="Vertrauen in die Kante.")
|
||||
evidence: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Begründung, shared_urls, similarity_score",
|
||||
)
|
||||
created_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
@field_validator("evidence", mode="before")
|
||||
@classmethod
|
||||
def _ensure_dict(cls, v):
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
return json.loads(v)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return {"reason": v}
|
||||
if v is None:
|
||||
return {"reason": ""}
|
||||
return v
|
||||
|
||||
|
||||
class SyntacticSimilarityResult(BaseModel):
|
||||
"""Ergebnis einer textuellen Ähnlichkeitsanalyse."""
|
||||
|
||||
source_a: UUID
|
||||
source_b: UUID
|
||||
similarity_ratio: float = Field(ge=0.0, le=1.0)
|
||||
identical_hash: bool = False
|
||||
llm_result: dict[str, Any] | None = None
|
||||
edge_type: CitationEdgeType | None = None
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
class SourceIndependenceAnalysisResult(BaseModel):
|
||||
"""Gesamtes Ergebnis der Source Independence Analyse."""
|
||||
|
||||
research_run_id: UUID = Field(default_factory=uuid4)
|
||||
source_scores: list[SourceIndependenceScore] = Field(default_factory=list)
|
||||
citation_edges: list[CitationGraphEdge] = Field(default_factory=list)
|
||||
syndication_groups: dict[str, list[UUID]] = Field(default_factory=dict)
|
||||
total_sources: int = 0
|
||||
unique_sources: int = 0
|
||||
llm_calls_made: int = 0
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-Parsing-Helfer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LlmSyndicationAnalysis(BaseModel):
|
||||
"""LLM-Antwort für Syndication-Analyse (Zwei-Text-Vergleich)."""
|
||||
|
||||
syndicated: bool = Field(default=False)
|
||||
syndication_direction: str = Field(default="none")
|
||||
similarity_score: float = Field(default=0.0, ge=0.0, le=1.0)
|
||||
shared_urls: list[str] = Field(default_factory=list)
|
||||
common_origins: list[str] = Field(default_factory=list)
|
||||
confidence: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
reason: str = Field(default="", description="Kurze Begründung.")
|
||||
|
||||
@field_validator("syndication_direction", mode="before")
|
||||
@classmethod
|
||||
def _validate_direction(cls, v):
|
||||
if v is None or not isinstance(v, str):
|
||||
return "none"
|
||||
return v.strip()
|
||||
|
||||
|
||||
def parse_llm_syndication_response(raw_text: str) -> LlmSyndicationAnalysis:
|
||||
"""Parst eine LLM-Antwort (JSON) für Syndication-Analyse."""
|
||||
try:
|
||||
text = raw_text.strip()
|
||||
# Extract JSON from possible markdown code blocks
|
||||
if "```" in text:
|
||||
for block in text.split("```"):
|
||||
block = block.strip()
|
||||
if block.startswith("json"):
|
||||
block = block[4:].strip()
|
||||
try:
|
||||
data = json.loads(block)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
else:
|
||||
# Try entire block as JSON
|
||||
data = json.loads(text)
|
||||
else:
|
||||
data = json.loads(text)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
data = {}
|
||||
|
||||
return LlmSyndicationAnalysis(
|
||||
syndicated=data.get("syndicated", False),
|
||||
syndication_direction=data.get("syndication_direction", "none"),
|
||||
similarity_score=float(data.get("similarity_score", 0.0)),
|
||||
shared_urls=data.get("shared_urls", []),
|
||||
common_origins=data.get("common_origins", []),
|
||||
confidence=float(data.get("confidence", 0.5)),
|
||||
reason=data.get("reason", ""),
|
||||
)
|
||||
Reference in New Issue
Block a user