feat(stage6): source independence & citation graph — detect syndication, shared origins, text similarity

- SourceIndependenceModel: per-source independence_score (0.0-1.0), syndication_group_id, primary_source_id, content_hash, shared_urls
- CitationGraphEdgeModel: directed edges (SYNDICATED, QUOTES, LINKS_TO, REPOST, SIMILAR_CONTENT) with confidence + evidence
- Content-Hash (SHA-256): instant syndication detection for identical content
- difflib Vorfilterung: >60% → LLM, >80% → high confidence, 100% → immediate syndication
- LLM-Pairwise-Analysis: two-text-comparison for suspicious pairs only (bounded concurrency)
- independence_score: 1.0 base, -0.4 for syndicated, -0.1 per high-similarity pair
- Pydantic schemas: SourceIndependenceScore, CitationGraphEdge, SyntacticSimilarityResult, LlmSyndicationAnalysis, SourceIndependenceAnalysisResult
- LLM response parser: handles JSON, markdown code blocks, partial/invalid JSON
- 43 tests: content hash, similarity thresholds, LLM parsing, analyzer integration, edge cases, prompt templates
This commit is contained in:
NSCT Agent
2026-08-23 18:47:24 +00:00
parent 27e494d161
commit a60cf21a2c
4 changed files with 1234 additions and 1 deletions

View File

@@ -16,6 +16,7 @@ from sqlalchemy import (
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
)
@@ -58,6 +59,16 @@ class EdgeRelation(str, enum.Enum):
SUPPLEMENTS = "supplements"
class CitationEdgeType(str, enum.Enum):
"""Kanten-Typen im Source-Citation-Graph (Stage 6: Source Independence)."""
SYNDICATED = "syndicated"
QUOTES = "quotes"
LINKS_TO = "links_to"
REPOST = "repost"
SIMILAR_CONTENT = "similar_content"
# ---------------------------------------------------------------------------
# Base
# ---------------------------------------------------------------------------
@@ -112,6 +123,12 @@ class SourceModel(Base):
# Relationships
claims = relationship("ClaimModel", back_populates="source", cascade="all, delete-orphan")
independence = relationship(
"SourceIndependenceModel",
back_populates="source",
uselist=False,
cascade="all, delete-orphan",
)
__table_args__ = (
Index("ix_sources_domain", "domain"),
@@ -167,7 +184,7 @@ class EvidenceRelationModel(Base):
# ---------------------------------------------------------------------------
# CitationEdge
# CitationEdge (Stage 0 — legacy claim-level edges)
# ---------------------------------------------------------------------------
@@ -186,6 +203,55 @@ class CitationEdgeModel(Base):
)
# ---------------------------------------------------------------------------
# SourceIndependence (Stage 6 — per-source independence metadata)
# ---------------------------------------------------------------------------
class SourceIndependenceModel(Base):
"""Speichert den Independence-Score und Syndication-Informationen pro Quelle (Stage 6)."""
__tablename__ = "source_independence"
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False, unique=True)
independence_score = Column(Float, nullable=False, default=1.0)
syndication_group_id = Column(String(36), nullable=True)
primary_source_id = Column(String(36), ForeignKey("sources.id"), nullable=True)
content_hash = Column(String(64), nullable=True)
text_similarity_high = Column(Integer, nullable=False, default=0)
shared_urls = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
# Relationships
source = relationship("SourceModel", back_populates="independence")
# ---------------------------------------------------------------------------
# CitationGraphEdge (Stage 6 — source-to-source edges)
# ---------------------------------------------------------------------------
class CitationGraphEdgeModel(Base):
"""Directed edge zwischen zwei Quellen im Source-Citation-Graph (Stage 6)."""
__tablename__ = "citation_graph_edges"
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
target_source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
edge_type = Column(Enum(CitationEdgeType), nullable=False)
confidence = Column(Float, nullable=False, default=1.0)
evidence = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
__table_args__ = (
Index("ix_citation_graph_edge_source_id", "source_id"),
Index("ix_citation_graph_edge_target_id", "target_source_id"),
)
# ---------------------------------------------------------------------------
# ResearchReport
# ---------------------------------------------------------------------------