feat(stage6): source independence & citation graph — detect syndication, shared origins, text similarity
- SourceIndependenceModel: per-source independence_score (0.0-1.0), syndication_group_id, primary_source_id, content_hash, shared_urls - CitationGraphEdgeModel: directed edges (SYNDICATED, QUOTES, LINKS_TO, REPOST, SIMILAR_CONTENT) with confidence + evidence - Content-Hash (SHA-256): instant syndication detection for identical content - difflib Vorfilterung: >60% → LLM, >80% → high confidence, 100% → immediate syndication - LLM-Pairwise-Analysis: two-text-comparison for suspicious pairs only (bounded concurrency) - independence_score: 1.0 base, -0.4 for syndicated, -0.1 per high-similarity pair - Pydantic schemas: SourceIndependenceScore, CitationGraphEdge, SyntacticSimilarityResult, LlmSyndicationAnalysis, SourceIndependenceAnalysisResult - LLM response parser: handles JSON, markdown code blocks, partial/invalid JSON - 43 tests: content hash, similarity thresholds, LLM parsing, analyzer integration, edge cases, prompt templates
This commit is contained in:
NSCT Agent
172
src/nsct/models/source_independence.py
Normal file
172
src/nsct/models/source_independence.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""Pydantic v2 schemas — Source Independence & Citation Graph (Stage 6).
|
||||
|
||||
Jede Quelle erhält einen independence_score (0.0–1.0).
|
||||
Quellen mit gemeinsamem Ursprung werden im Citation Graph verknüpft.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class CitationEdgeType(str, Enum):
|
||||
"""Kanten-Typen im Source-Citation-Graph (Stage 6)."""
|
||||
|
||||
SYNDICATED = "syndicated"
|
||||
QUOTES = "quotes"
|
||||
LINKS_TO = "links_to"
|
||||
REPOST = "repost"
|
||||
SIMILAR_CONTENT = "similar_content"
|
||||
|
||||
|
||||
class SyndicationDirection(str, Enum):
|
||||
"""Richtung der Syndication zwischen zwei Quellen."""
|
||||
|
||||
SOURCE_B = "B->A"
|
||||
SOURCE_A = "A->B"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
class SourceIndependenceScore(BaseModel):
|
||||
"""Independence-Score für eine einzelne Quelle."""
|
||||
|
||||
source_id: UUID = Field(..., description="UUID der Quelle.")
|
||||
independence_score: float = Field(
|
||||
..., ge=0.0, le=1.0, description="Unabhängigkeits-Score (0.0=Duplikat, 1.0=vollständig unabhängig)"
|
||||
)
|
||||
syndication_group_id: UUID | None = Field(
|
||||
default=None, description="Gruppe syndizierter Quellen."
|
||||
)
|
||||
primary_source_id: UUID | None = Field(
|
||||
default=None, description="Primärquelle wenn diese Quelle syndiziert wurde."
|
||||
)
|
||||
content_hash: str | None = Field(
|
||||
default=None, description="SHA-256 Hash des Inhalts."
|
||||
)
|
||||
text_similarity_high_count: int = Field(
|
||||
default=0, description="Anzahl Quellen mit >80% Text-Ähnlichkeit."
|
||||
)
|
||||
shared_urls: list[str] = Field(
|
||||
default_factory=list, description="URLs die auf andere Quellen verweisen."
|
||||
)
|
||||
similarity_pairs: list[dict[str, Any]] = Field(
|
||||
default_factory=list, description="Paare mit >60% Similarität und LLM-Ergebnis."
|
||||
)
|
||||
|
||||
|
||||
class CitationGraphEdge(BaseModel):
|
||||
"""Gerichtete Kante zwischen zwei Quellen."""
|
||||
|
||||
id: UUID = Field(default_factory=uuid4)
|
||||
source_id: UUID = Field(..., description="Quelle die die Beziehung aufweist.")
|
||||
target_source_id: UUID = Field(..., description="Quelle die referenziert wird.")
|
||||
edge_type: CitationEdgeType = Field(..., description="Typ der Beziehung.")
|
||||
confidence: float = Field(..., ge=0.0, le=1.0, description="Vertrauen in die Kante.")
|
||||
evidence: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Begründung, shared_urls, similarity_score",
|
||||
)
|
||||
created_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
@field_validator("evidence", mode="before")
|
||||
@classmethod
|
||||
def _ensure_dict(cls, v):
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
return json.loads(v)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return {"reason": v}
|
||||
if v is None:
|
||||
return {"reason": ""}
|
||||
return v
|
||||
|
||||
|
||||
class SyntacticSimilarityResult(BaseModel):
|
||||
"""Ergebnis einer textuellen Ähnlichkeitsanalyse."""
|
||||
|
||||
source_a: UUID
|
||||
source_b: UUID
|
||||
similarity_ratio: float = Field(ge=0.0, le=1.0)
|
||||
identical_hash: bool = False
|
||||
llm_result: dict[str, Any] | None = None
|
||||
edge_type: CitationEdgeType | None = None
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
class SourceIndependenceAnalysisResult(BaseModel):
|
||||
"""Gesamtes Ergebnis der Source Independence Analyse."""
|
||||
|
||||
research_run_id: UUID = Field(default_factory=uuid4)
|
||||
source_scores: list[SourceIndependenceScore] = Field(default_factory=list)
|
||||
citation_edges: list[CitationGraphEdge] = Field(default_factory=list)
|
||||
syndication_groups: dict[str, list[UUID]] = Field(default_factory=dict)
|
||||
total_sources: int = 0
|
||||
unique_sources: int = 0
|
||||
llm_calls_made: int = 0
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-Parsing-Helfer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LlmSyndicationAnalysis(BaseModel):
|
||||
"""LLM-Antwort für Syndication-Analyse (Zwei-Text-Vergleich)."""
|
||||
|
||||
syndicated: bool = Field(default=False)
|
||||
syndication_direction: str = Field(default="none")
|
||||
similarity_score: float = Field(default=0.0, ge=0.0, le=1.0)
|
||||
shared_urls: list[str] = Field(default_factory=list)
|
||||
common_origins: list[str] = Field(default_factory=list)
|
||||
confidence: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
reason: str = Field(default="", description="Kurze Begründung.")
|
||||
|
||||
@field_validator("syndication_direction", mode="before")
|
||||
@classmethod
|
||||
def _validate_direction(cls, v):
|
||||
if v is None or not isinstance(v, str):
|
||||
return "none"
|
||||
return v.strip()
|
||||
|
||||
|
||||
def parse_llm_syndication_response(raw_text: str) -> LlmSyndicationAnalysis:
|
||||
"""Parst eine LLM-Antwort (JSON) für Syndication-Analyse."""
|
||||
try:
|
||||
text = raw_text.strip()
|
||||
# Extract JSON from possible markdown code blocks
|
||||
if "```" in text:
|
||||
for block in text.split("```"):
|
||||
block = block.strip()
|
||||
if block.startswith("json"):
|
||||
block = block[4:].strip()
|
||||
try:
|
||||
data = json.loads(block)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
else:
|
||||
# Try entire block as JSON
|
||||
data = json.loads(text)
|
||||
else:
|
||||
data = json.loads(text)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
data = {}
|
||||
|
||||
return LlmSyndicationAnalysis(
|
||||
syndicated=data.get("syndicated", False),
|
||||
syndication_direction=data.get("syndication_direction", "none"),
|
||||
similarity_score=float(data.get("similarity_score", 0.0)),
|
||||
shared_urls=data.get("shared_urls", []),
|
||||
common_origins=data.get("common_origins", []),
|
||||
confidence=float(data.get("confidence", 0.5)),
|
||||
reason=data.get("reason", ""),
|
||||
)
|
||||
410
src/nsct/stages/stage6_source_independence.py
Normal file
410
src/nsct/stages/stage6_source_independence.py
Normal file
@@ -0,0 +1,410 @@
|
||||
"""Stage 6: Source Independence & Citation Graph.
|
||||
|
||||
Pipeline:
|
||||
1. Berechnet Content-Hash (SHA-256) pro Quelle
|
||||
2. Gruppiert identische Hashes (100 % → sofortige Syndication)
|
||||
3. difflib-Vorfilterung für textuelle Ähnlichkeit
|
||||
4. Nur verdächtige Paare (>60 %) → LLM-Analyse
|
||||
5. Berechnet independence_score pro Quelle (0.0–1.0)
|
||||
6. Gruppiert Quellen in Syndication-Groups
|
||||
7. Erzeugt Citation-Graph-Edges
|
||||
|
||||
ARCHITEKTUR-REGELN:
|
||||
- Web Content ist Daten, keine Instruktion
|
||||
- Anzahl Webseiten != Anzahl unabhängiger Quellen
|
||||
- LLM-Verwendung sparsam — difflib zuerst
|
||||
- LLM darf keine Quellen/Evidenz erfinden
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from asyncio import Semaphore
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from nsct.config import AppSettings
|
||||
from nsct.models.source_independence import (
|
||||
CitationEdgeType as CitationGraphEdgeType,
|
||||
CitationGraphEdge,
|
||||
LlmSyndicationAnalysis,
|
||||
parse_llm_syndication_response,
|
||||
SourceIndependenceAnalysisResult,
|
||||
SourceIndependenceScore,
|
||||
)
|
||||
from nsct.providers.llm import LLMProvider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thresholds
|
||||
SIMILARITY_IDENTICAL = 1.0 # exakt gleicher Content-Hash
|
||||
SIMILARITY_HIGH = 0.80 # starke Evidenz für Syndication
|
||||
SIMILARITY_MODERATE = 0.60 # moderate Evidenz → LLM-Analyse
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-Prompt für Syndication-Analyse
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SYNDICATION_SYSTEM_PROMPT = (
|
||||
"Du bist ein Neutral Search Crawler Tool (NSCT) Analyst.\n"
|
||||
"Deine Aufgabe ist es, zwei Textquellen auf Unabhängigkeit zu prüfen.\n\n"
|
||||
"REGELN:\n"
|
||||
"- Gib NUR ein JSON-Objekt zurück (keine Markdown-Code-Blöcke, kein Extra-Text).\n"
|
||||
"- Syndiziert: Site B kopiert Originaltext von Site A → direction: 'A->B'\n"
|
||||
"- Zitiert: Eine Quelle zitiert explizit die andere mit Anführungszeichen.\n"
|
||||
"- Redaktions-Schleife: Nahezu identischer Text ohne Quellenangabe.\n"
|
||||
"- Unabhängig: Eigener Content, eigene Recherche.\n"
|
||||
"- Gib KEINE Zusammenfassung, prüfe NUR die Beziehung.\n"
|
||||
)
|
||||
|
||||
SYNDICATION_USER_PROMPT_TEMPLATE = (
|
||||
"Analysiere die Unabhängigkeit dieser beiden Quellen.\n\n"
|
||||
"=== Quelle A ===\n"
|
||||
"URL: {url_a}\n"
|
||||
"Titel: {title_a}\n"
|
||||
"Domain: {domain_a}\n"
|
||||
"Text (erste 3000 Zeichen): {text_a}\n\n"
|
||||
"=== Quelle B ===\n"
|
||||
"URL: {url_b}\n"
|
||||
"Titel: {title_b}\n"
|
||||
"Domain: {domain_b}\n"
|
||||
"Text (erste 3000 Zeichen): {text_b}\n\n"
|
||||
"Antworte als JSON: {{"
|
||||
'"syndicated": bool, '
|
||||
'"syndication_direction": "A->B | B->A | none", '
|
||||
'"similarity_score": 0.0-1.0, '
|
||||
'"shared_urls": [...], '
|
||||
'"common_origins": [...], '
|
||||
'"confidence": 0.0-1.0, '
|
||||
'"reason": "kurze Begründung" '
|
||||
"}}\n"
|
||||
"Wenn keine Syndication: syndicated=false, direction=none, similarity_score basierend auf textueller Übereinstimmung."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_content_hash(content: str) -> str:
|
||||
"""SHA-256 Hash des extrahierten Inhalts (stripped, normalized)."""
|
||||
normalized = content.strip().replace("\r\n", "\n").replace("\r", "\n")
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def compute_similarity_ratio(text_a: str, text_b: str) -> float:
|
||||
"""difflib.SequenceMatcher similarity ratio."""
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
return SequenceMatcher(
|
||||
None,
|
||||
text_a.strip()[:50000],
|
||||
text_b.strip()[:50000],
|
||||
).ratio()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Syndication-LLM-Analyse (async)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _analyze_syndication_pair(
|
||||
llm_provider: LLMProvider,
|
||||
config: AppSettings,
|
||||
source_a: dict,
|
||||
source_b: dict,
|
||||
similarity_ratio: float,
|
||||
semaphore: Semaphore | None = None,
|
||||
) -> LlmSyndicationAnalysis:
|
||||
"""LLM-Analyse für ein Quellen-Paar."""
|
||||
if len(source_a.get("content", "")) < 100:
|
||||
return LlmSyndicationAnalysis(
|
||||
syndicated=False, syndication_direction="none",
|
||||
similarity_score=0.0, reason="Source A content too short",
|
||||
)
|
||||
if len(source_b.get("content", "")) < 100:
|
||||
return LlmSyndicationAnalysis(
|
||||
syndicated=False, syndication_direction="none",
|
||||
similarity_score=0.0, reason="Source B content too short",
|
||||
)
|
||||
|
||||
content_a = source_a.get("content", "")[:3000]
|
||||
content_b = source_b.get("content", "")[:3000]
|
||||
|
||||
prompt = SYNDICATION_USER_PROMPT_TEMPLATE.format(
|
||||
url_a=source_a.get("url", ""),
|
||||
title_a=source_a.get("title", ""),
|
||||
domain_a=source_a.get("domain", ""),
|
||||
text_a=content_a,
|
||||
url_b=source_b.get("url", ""),
|
||||
title_b=source_b.get("title", ""),
|
||||
domain_b=source_b.get("domain", ""),
|
||||
text_b=content_b,
|
||||
)
|
||||
|
||||
if semaphore:
|
||||
async with semaphore:
|
||||
llm_response = await llm_provider.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": SYNDICATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=config.llm.model,
|
||||
temperature=0.1,
|
||||
max_tokens=1024,
|
||||
)
|
||||
else:
|
||||
llm_response = await llm_provider.complete(
|
||||
messages=[
|
||||
{"role": "system", "content": SYNDICATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
model=config.llm.model,
|
||||
temperature=0.1,
|
||||
max_tokens=1024,
|
||||
)
|
||||
|
||||
return parse_llm_syndication_response(llm_response)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 6 Extractor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SourceIndependenceAnalyzer:
|
||||
"""Analyse der Quellunabhängigkeit (Stage 6)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
llm_provider: LLMProvider,
|
||||
config: AppSettings,
|
||||
sources: list[dict],
|
||||
research_run_id: UUID,
|
||||
):
|
||||
self.llm_provider = llm_provider
|
||||
self.config = config
|
||||
self.sources = sources
|
||||
self.research_run_id = research_run_id
|
||||
self.llm_calls_made = 0
|
||||
|
||||
async def analyze(self) -> SourceIndependenceAnalysisResult:
|
||||
"""Führe die komplette Source Independence Analyse durch."""
|
||||
if not self.sources:
|
||||
return SourceIndependenceAnalysisResult(
|
||||
research_run_id=self.research_run_id,
|
||||
total_sources=0, unique_sources=0, llm_calls_made=0,
|
||||
)
|
||||
|
||||
# Phase 1: Content-Hash pro Quelle
|
||||
content_hashes: dict[str, str] = {}
|
||||
for src in self.sources:
|
||||
sid = src.get("id", str(uuid4()))
|
||||
content = src.get("content", "")
|
||||
content_hashes[sid] = compute_content_hash(content)
|
||||
|
||||
# Phase 2: Identische Hashes → sofortige Syndication
|
||||
hash_groups: dict[str, list[str]] = {}
|
||||
for sid, h in content_hashes.items():
|
||||
hash_groups.setdefault(h, []).append(sid)
|
||||
|
||||
identical_groups = {h: sids for h, sids in hash_groups.items() if len(sids) > 1}
|
||||
|
||||
# Phase 3: Text-Ähnlichkeit über difflib
|
||||
source_map = {s.get("id", str(uuid4())): s for s in self.sources}
|
||||
|
||||
# Nur Paare mit >SIMILARITY_MODERATE similarity → LLM
|
||||
llm_candidates: list[tuple[str, str, float]] = []
|
||||
|
||||
# Identische Paare sofort markieren
|
||||
edges: list[CitationGraphEdge] = []
|
||||
all_scores: dict[str, SourceIndependenceScore] = {}
|
||||
|
||||
for h, sids in identical_groups.items():
|
||||
for i in range(len(sids)):
|
||||
for j in range(i + 1, len(sids)):
|
||||
sid_a = sids[i]
|
||||
sid_b = sids[j]
|
||||
edges.append(CitationGraphEdge(
|
||||
source_id=sid_a,
|
||||
target_source_id=sid_b,
|
||||
edge_type=CitationGraphEdgeType.SYNDICATED,
|
||||
confidence=1.0,
|
||||
evidence={
|
||||
"reason": "Identischer Content-Hash (SHA-256)",
|
||||
"shared_urls": [],
|
||||
"similarity_score": SIMILARITY_IDENTICAL,
|
||||
"identical_hash": True,
|
||||
},
|
||||
))
|
||||
|
||||
# difflib-Vorfilterung für alle nicht-identischen Paare
|
||||
source_ids = list(source_map.keys())
|
||||
pairwise_results: dict[str, Any] = {}
|
||||
|
||||
for i in range(len(source_ids)):
|
||||
for j in range(i + 1, len(source_ids)):
|
||||
sid_a = source_ids[i]
|
||||
sid_b = source_ids[j]
|
||||
|
||||
if sid_a in content_hashes and sid_b in content_hashes:
|
||||
if content_hashes[sid_a] == content_hashes[sid_b]:
|
||||
continue # bereits verarbeitet
|
||||
|
||||
content_a = source_map[sid_a].get("content", "")
|
||||
content_b = source_map[sid_b].get("content", "")
|
||||
ratio = compute_similarity_ratio(content_a, content_b)
|
||||
|
||||
if ratio >= SIMILARITY_MODERATE:
|
||||
llm_candidates.append((sid_a, sid_b, ratio))
|
||||
pairwise_results[f"{sid_a}:{sid_b}"] = ratio
|
||||
|
||||
# Phase 4: LLM-Analyse nur für verdächtige Paare
|
||||
concurrency = min(getattr(self.config.llm, "max_concurrency", 3), 5)
|
||||
semaphore = Semaphore(concurrency)
|
||||
|
||||
llm_tasks = []
|
||||
llm_results: list[tuple[str, str, float, LlmSyndicationAnalysis]] = []
|
||||
|
||||
async def _process_pair(sid_a: str, sid_b: str, ratio: float):
|
||||
try:
|
||||
result = await _analyze_syndication_pair(
|
||||
self.llm_provider, self.config,
|
||||
source_map[sid_a], source_map[sid_b],
|
||||
ratio, semaphore,
|
||||
)
|
||||
self.llm_calls_made += 1
|
||||
llm_results.append((sid_a, sid_b, ratio, result))
|
||||
except Exception:
|
||||
logger.exception("LLM analysis failed for pair %s <-> %s", sid_a, sid_b)
|
||||
|
||||
for sid_a, sid_b, ratio in llm_candidates:
|
||||
llm_tasks.append(_process_pair(sid_a, sid_b, ratio))
|
||||
|
||||
if llm_tasks:
|
||||
await asyncio.gather(*llm_tasks, return_exceptions=True)
|
||||
|
||||
# Phase 5: Ergebnisse aggregieren
|
||||
for sid_a, sid_b, ratio, llm_result in llm_results:
|
||||
evidence = {
|
||||
"reason": llm_result.reason,
|
||||
"shared_urls": llm_result.shared_urls,
|
||||
"similarity_score": ratio,
|
||||
}
|
||||
|
||||
if llm_result.similarity_score >= SIMILARITY_HIGH or llm_result.syndicated:
|
||||
edge_type = CitationGraphEdgeType.SYNDICATED
|
||||
confidence = max(llm_result.confidence, ratio)
|
||||
elif ratio >= SIMILARITY_HIGH:
|
||||
edge_type = CitationGraphEdgeType.SIMILAR_CONTENT
|
||||
confidence = ratio
|
||||
elif ratio >= SIMILARITY_MODERATE:
|
||||
if llm_result.syndicated:
|
||||
edge_type = CitationGraphEdgeType.SYNDICATED
|
||||
confidence = llm_result.confidence
|
||||
else:
|
||||
edge_type = CitationGraphEdgeType.SIMILAR_CONTENT
|
||||
confidence = ratio * 0.8
|
||||
else:
|
||||
continue # zu niedrig für Kante
|
||||
|
||||
edges.append(CitationGraphEdge(
|
||||
source_id=sid_a,
|
||||
target_source_id=sid_b,
|
||||
edge_type=edge_type,
|
||||
confidence=confidence,
|
||||
evidence=evidence,
|
||||
))
|
||||
|
||||
if llm_result.syndicated and llm_result.syndication_direction != "none":
|
||||
# Syndication-Direktionalität setzen
|
||||
if llm_result.syndication_direction == "A->B":
|
||||
evidence["direction"] = "A->B"
|
||||
edges.append(CitationGraphEdge(
|
||||
source_id=sid_b,
|
||||
target_source_id=sid_a,
|
||||
edge_type=CitationGraphEdgeType.REPOST,
|
||||
confidence=confidence * 0.9,
|
||||
evidence={**evidence, "reason": f"Syndication: {source_map[sid_a].get('title', '')} → {source_map[sid_b].get('title', '')}"},
|
||||
))
|
||||
|
||||
# Phase 6: independence_score berechnen
|
||||
syndicated_set: set[str] = set()
|
||||
syndication_graph: dict[str, str] = {} # source_id → primary_source_id
|
||||
|
||||
for edge in edges:
|
||||
if edge.edge_type in (CitationGraphEdgeType.SYNDICATED, CitationGraphEdgeType.REPOST):
|
||||
syndicated_set.add(edge.target_source_id)
|
||||
syndication_graph[edge.target_source_id] = edge.source_id
|
||||
|
||||
# Group identical-hash sources
|
||||
syndication_groups: dict[str, list[str]] = {}
|
||||
for h, sids in identical_groups.items():
|
||||
gid = str(uuid4())
|
||||
syndication_groups[gid] = sids
|
||||
|
||||
# Calculate per-source scores
|
||||
for src in self.sources:
|
||||
sid = src.get("id", str(uuid4()))
|
||||
content_hash = content_hashes.get(sid, "")
|
||||
|
||||
# Start with 1.0, deduct for syndication
|
||||
score = 1.0
|
||||
if sid in syndicated_set:
|
||||
score -= 0.4 # heavy penalty for syndication
|
||||
# Count high-similarity pairs
|
||||
high_count = sum(
|
||||
1 for sid_a, sid_b, ratio, _ in llm_results
|
||||
if (sid_a == sid or sid_b == sid) and ratio >= SIMILARITY_HIGH
|
||||
)
|
||||
if high_count > 0:
|
||||
score -= 0.1 * high_count
|
||||
score = max(0.0, min(1.0, score))
|
||||
|
||||
# Find primary source
|
||||
primary_source_id = None
|
||||
for target, source in syndication_graph.items():
|
||||
if target == sid:
|
||||
primary_source_id = source
|
||||
break
|
||||
|
||||
# Find syndication group
|
||||
group_id = None
|
||||
for gid, members in syndication_groups.items():
|
||||
if sid in members:
|
||||
group_id = gid
|
||||
break
|
||||
|
||||
all_scores[sid] = SourceIndependenceScore(
|
||||
source_id=UUID(sid),
|
||||
independence_score=round(score, 3),
|
||||
syndication_group_id=UUID(group_id) if group_id else None,
|
||||
primary_source_id=UUID(primary_source_id) if primary_source_id else None,
|
||||
content_hash=content_hash,
|
||||
text_similarity_high_count=high_count,
|
||||
shared_urls=[],
|
||||
)
|
||||
|
||||
# Count unique sources (exclude syndicated from count)
|
||||
unique_count = len([s for s in source_map.keys() if s not in syndicated_set])
|
||||
|
||||
result = SourceIndependenceAnalysisResult(
|
||||
research_run_id=self.research_run_id,
|
||||
source_scores=list(all_scores.values()),
|
||||
citation_edges=edges,
|
||||
syndication_groups={
|
||||
gid: [UUID(s) for s in members]
|
||||
for gid, members in syndication_groups.items()
|
||||
},
|
||||
total_sources=len(self.sources),
|
||||
unique_sources=unique_count,
|
||||
llm_calls_made=self.llm_calls_made,
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -16,6 +16,7 @@ from sqlalchemy import (
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
)
|
||||
@@ -58,6 +59,16 @@ class EdgeRelation(str, enum.Enum):
|
||||
SUPPLEMENTS = "supplements"
|
||||
|
||||
|
||||
class CitationEdgeType(str, enum.Enum):
|
||||
"""Kanten-Typen im Source-Citation-Graph (Stage 6: Source Independence)."""
|
||||
|
||||
SYNDICATED = "syndicated"
|
||||
QUOTES = "quotes"
|
||||
LINKS_TO = "links_to"
|
||||
REPOST = "repost"
|
||||
SIMILAR_CONTENT = "similar_content"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -112,6 +123,12 @@ class SourceModel(Base):
|
||||
|
||||
# Relationships
|
||||
claims = relationship("ClaimModel", back_populates="source", cascade="all, delete-orphan")
|
||||
independence = relationship(
|
||||
"SourceIndependenceModel",
|
||||
back_populates="source",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_sources_domain", "domain"),
|
||||
@@ -167,7 +184,7 @@ class EvidenceRelationModel(Base):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CitationEdge
|
||||
# CitationEdge (Stage 0 — legacy claim-level edges)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -186,6 +203,55 @@ class CitationEdgeModel(Base):
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SourceIndependence (Stage 6 — per-source independence metadata)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SourceIndependenceModel(Base):
|
||||
"""Speichert den Independence-Score und Syndication-Informationen pro Quelle (Stage 6)."""
|
||||
|
||||
__tablename__ = "source_independence"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False, unique=True)
|
||||
independence_score = Column(Float, nullable=False, default=1.0)
|
||||
syndication_group_id = Column(String(36), nullable=True)
|
||||
primary_source_id = Column(String(36), ForeignKey("sources.id"), nullable=True)
|
||||
content_hash = Column(String(64), nullable=True)
|
||||
text_similarity_high = Column(Integer, nullable=False, default=0)
|
||||
shared_urls = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
source = relationship("SourceModel", back_populates="independence")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CitationGraphEdge (Stage 6 — source-to-source edges)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CitationGraphEdgeModel(Base):
|
||||
"""Directed edge zwischen zwei Quellen im Source-Citation-Graph (Stage 6)."""
|
||||
|
||||
__tablename__ = "citation_graph_edges"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
target_source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
edge_type = Column(Enum(CitationEdgeType), nullable=False)
|
||||
confidence = Column(Float, nullable=False, default=1.0)
|
||||
evidence = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_citation_graph_edge_source_id", "source_id"),
|
||||
Index("ix_citation_graph_edge_target_id", "target_source_id"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResearchReport
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user