feat(stage6): source independence & citation graph — detect syndication, shared origins, text similarity
- SourceIndependenceModel: per-source independence_score (0.0-1.0), syndication_group_id, primary_source_id, content_hash, shared_urls - CitationGraphEdgeModel: directed edges (SYNDICATED, QUOTES, LINKS_TO, REPOST, SIMILAR_CONTENT) with confidence + evidence - Content-Hash (SHA-256): instant syndication detection for identical content - difflib Vorfilterung: >60% → LLM, >80% → high confidence, 100% → immediate syndication - LLM-Pairwise-Analysis: two-text-comparison for suspicious pairs only (bounded concurrency) - independence_score: 1.0 base, -0.4 for syndicated, -0.1 per high-similarity pair - Pydantic schemas: SourceIndependenceScore, CitationGraphEdge, SyntacticSimilarityResult, LlmSyndicationAnalysis, SourceIndependenceAnalysisResult - LLM response parser: handles JSON, markdown code blocks, partial/invalid JSON - 43 tests: content hash, similarity thresholds, LLM parsing, analyzer integration, edge cases, prompt templates
This commit is contained in:
@@ -16,6 +16,7 @@ from sqlalchemy import (
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
)
|
||||
@@ -58,6 +59,16 @@ class EdgeRelation(str, enum.Enum):
|
||||
SUPPLEMENTS = "supplements"
|
||||
|
||||
|
||||
class CitationEdgeType(str, enum.Enum):
|
||||
"""Kanten-Typen im Source-Citation-Graph (Stage 6: Source Independence)."""
|
||||
|
||||
SYNDICATED = "syndicated"
|
||||
QUOTES = "quotes"
|
||||
LINKS_TO = "links_to"
|
||||
REPOST = "repost"
|
||||
SIMILAR_CONTENT = "similar_content"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -112,6 +123,12 @@ class SourceModel(Base):
|
||||
|
||||
# Relationships
|
||||
claims = relationship("ClaimModel", back_populates="source", cascade="all, delete-orphan")
|
||||
independence = relationship(
|
||||
"SourceIndependenceModel",
|
||||
back_populates="source",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_sources_domain", "domain"),
|
||||
@@ -167,7 +184,7 @@ class EvidenceRelationModel(Base):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CitationEdge
|
||||
# CitationEdge (Stage 0 — legacy claim-level edges)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -186,6 +203,55 @@ class CitationEdgeModel(Base):
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SourceIndependence (Stage 6 — per-source independence metadata)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SourceIndependenceModel(Base):
|
||||
"""Speichert den Independence-Score und Syndication-Informationen pro Quelle (Stage 6)."""
|
||||
|
||||
__tablename__ = "source_independence"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False, unique=True)
|
||||
independence_score = Column(Float, nullable=False, default=1.0)
|
||||
syndication_group_id = Column(String(36), nullable=True)
|
||||
primary_source_id = Column(String(36), ForeignKey("sources.id"), nullable=True)
|
||||
content_hash = Column(String(64), nullable=True)
|
||||
text_similarity_high = Column(Integer, nullable=False, default=0)
|
||||
shared_urls = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
source = relationship("SourceModel", back_populates="independence")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CitationGraphEdge (Stage 6 — source-to-source edges)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CitationGraphEdgeModel(Base):
|
||||
"""Directed edge zwischen zwei Quellen im Source-Citation-Graph (Stage 6)."""
|
||||
|
||||
__tablename__ = "citation_graph_edges"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
target_source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
edge_type = Column(Enum(CitationEdgeType), nullable=False)
|
||||
confidence = Column(Float, nullable=False, default=1.0)
|
||||
evidence = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_citation_graph_edge_source_id", "source_id"),
|
||||
Index("ix_citation_graph_edge_target_id", "target_source_id"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResearchReport
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user