feat(stage10): implement vision integration
This commit is contained in:
NSCT Agent
@@ -103,6 +103,10 @@ def create_app() -> FastAPI:
|
||||
from nsct.api.synthesis import router as synthesis_router
|
||||
app.include_router(synthesis_router, tags=["research"])
|
||||
|
||||
# Mount vision router (Stage 10)
|
||||
from nsct.api.vision import router as vision_router
|
||||
app.include_router(vision_router, tags=["vision"])
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
376
src/nsct/api/vision.py
Normal file
376
src/nsct/api/vision.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""Vision API endpoints — Qwen2.5-VL-3B visual evidence extraction.
|
||||
|
||||
Endpunkte:
|
||||
POST /vision/analyze — Analysiere ein Bild mit Qwen2.5-VL-3B
|
||||
GET /vision/evidence/{evidence_id} — Hole visuellen Evidenz-Eintrag
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from httpx import AsyncClient
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from nsct.config import AppSettings
|
||||
from nsct.providers.vision import get_provider as get_vision_provider
|
||||
from nsct.providers.metrics import ProviderMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AnalyzeImageRequest(BaseModel):
|
||||
"""Request zum Analysieren eines Bildes mit Qwen2.5-VL-3B."""
|
||||
|
||||
image_data: str = Field(
|
||||
...,
|
||||
description="Base64-kodiertes Bild oder URL (http/https).",
|
||||
min_length=1,
|
||||
)
|
||||
capture_type: str = Field(
|
||||
default="screenshot",
|
||||
description="Art der visuellen Aufnahme: screenshot, infographic, diagram, chart, photo, document.",
|
||||
)
|
||||
prompt: str | None = Field(
|
||||
default=None,
|
||||
description="Optionales Analyse-Prompt. Wenn None → Standard-Prompt für Evidenz-Extraktion.",
|
||||
)
|
||||
image_caption: str | None = Field(
|
||||
default=None,
|
||||
description="Optionale Beschreibung des Bildes als Kontext.",
|
||||
)
|
||||
evidence_type: str = Field(
|
||||
default="visual",
|
||||
description="Kategorie der Evidenz: visual, chart, document, infographic.",
|
||||
)
|
||||
|
||||
|
||||
class EvidenceItem(BaseModel):
|
||||
"""Ein einzelner visueller Evidenz-Eintrag."""
|
||||
|
||||
id: str = Field(..., description="UUID des Evidenz-Eintrags")
|
||||
evidence_type: str = Field(..., description="Kategorie der Evidenz")
|
||||
capture_type: str = Field(..., description="Art der visuellen Aufnahme")
|
||||
image_source: str = Field(..., description="Quelle/Beschreibung des Bildes")
|
||||
description: str = Field(..., description="Beschreibung der visuellen Inhalte")
|
||||
key_findings: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Wichtige visuelle Erkenntnisse aus dem Bild.",
|
||||
)
|
||||
data_points: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Strukturierte Datenpunkte aus dem Bild.",
|
||||
)
|
||||
confidence: float = Field(
|
||||
default=0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Vertrauen der visuellen Analyse (0-1).",
|
||||
)
|
||||
sources: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Quellen/Referenzen innerhalb des Bildes.",
|
||||
)
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusätzliche Metadaten zur Analyse.",
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeImageResponse(BaseModel):
|
||||
"""Antwort auf Bildanalyse-Anfrage."""
|
||||
|
||||
evidence_id: str = Field(..., description="UUID des erstellten Evidenz-Eintrags")
|
||||
evidence_type: str = Field(..., description="Kategorie der Evidenz")
|
||||
capture_type: str = Field(..., description="Art der visuellen Aufnahme")
|
||||
description: str = Field(..., description="Zusammenfassung der visuellen Analyse")
|
||||
findings: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Visuelle Erkenntnisse (alias key_findings).",
|
||||
)
|
||||
confidence: float = Field(..., ge=0.0, le=1.0, description="Vertrauen der Analyse")
|
||||
data_points: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Strukturierte Datenpunkte aus dem Bild.",
|
||||
)
|
||||
image_source: str = Field(..., description="Quelle des Bildes")
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict, description="Zusätzliche Metadaten."
|
||||
)
|
||||
|
||||
|
||||
class EvidenceResponse(BaseModel):
|
||||
"""Antwort für GET /vision/evidence/{evidence_id}."""
|
||||
|
||||
success: bool
|
||||
evidence: EvidenceItem | None = Field(
|
||||
default=None, description="Visueller Evidenz-Eintrag oder None bei 404."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper — Mock-Speicher (für Prototyp)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# In-Memory Store für Evidenz-Einträge (Produktion → DB)
|
||||
_evidence_store: dict[str, EvidenceItem] = {}
|
||||
|
||||
|
||||
def _store_evidence(evidence: EvidenceItem) -> None:
|
||||
_evidence_store[evidence.id] = evidence
|
||||
|
||||
|
||||
def _get_evidence(evidence_id: str) -> EvidenceItem | None:
|
||||
return _evidence_store.get(evidence_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper — Default-Prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_VISION_PROMPT = (
|
||||
"Analysiere das Bild systematisch als visueller Evidenz-Extraktor. "
|
||||
"Identifiziere: (1) visuelle Inhalte und Objekte, (2) Textinhalte und Überschriften, "
|
||||
"3) Daten, Diagramme oder Grafiken, (4) visuelle Trends und Muster, "
|
||||
"5) Schlüsselinformationen für fact-checking, (6) Vertrauenswürdigkeit der visuellen Evidenz. "
|
||||
"Antworte mit einer strukturierten Analyse. Format: JSON-Array mit key_findings (Strings), "
|
||||
"data_points (Objekte), sources (Strings), confidence (Float 0-1), image_source (String)."
|
||||
)
|
||||
|
||||
|
||||
def _build_prompt(
|
||||
capture_type: str, image_caption: str | None, custom_prompt: str | None
|
||||
) -> str:
|
||||
"""Kombiniere den Standard-Prompt mit optionalen Anpassungen."""
|
||||
prompt = custom_prompt or _DEFAULT_VISION_PROMPT
|
||||
if image_caption:
|
||||
prompt = f"Bild-Beschreibung: {image_caption}\n\n{prompt}"
|
||||
if capture_type and capture_type != "screenshot":
|
||||
prompt = f"Capture-Typ: {capture_type}. {prompt}"
|
||||
return prompt
|
||||
|
||||
|
||||
def _parse_vision_response(raw: str) -> dict[str, Any]:
|
||||
"""Parst die Vision-LLM-Antwort und extrahiert strukturierte Daten."""
|
||||
result: dict[str, Any] = {
|
||||
"description": raw,
|
||||
"key_findings": [],
|
||||
"data_points": [],
|
||||
"confidence": 0.8,
|
||||
"image_source": "unknown",
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
if not raw or not raw.strip():
|
||||
result["description"] = "Keine Inhalte erkannt"
|
||||
return result
|
||||
|
||||
# Versuche JSON-Array zu parse
|
||||
raw_stripped = raw.strip()
|
||||
# Strip markdown code blocks
|
||||
if raw_stripped.startswith("```"):
|
||||
lines = raw_stripped.split("\n")
|
||||
if lines[0].startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].startswith("```"):
|
||||
lines = lines[:-1]
|
||||
raw_stripped = "\n".join(lines).strip()
|
||||
|
||||
if not raw_stripped:
|
||||
result["description"] = "Leere Antwort vom Vision-Modell"
|
||||
return result
|
||||
|
||||
# Parse als JSON
|
||||
import json
|
||||
|
||||
try:
|
||||
data = json.loads(raw_stripped)
|
||||
if isinstance(data, dict):
|
||||
result["key_findings"] = data.get("key_findings", [])
|
||||
result["data_points"] = data.get("data_points", [])
|
||||
result["confidence"] = min(max(data.get("confidence", 0.8), 0.0), 1.0)
|
||||
result["image_source"] = data.get("image_source", "unknown")
|
||||
result["metadata"] = data.get("metadata", {})
|
||||
result["description"] = data.get("description", raw)
|
||||
return result
|
||||
elif isinstance(data, list):
|
||||
result["key_findings"] = [
|
||||
str(item) for item in data if isinstance(item, (str, dict, list))
|
||||
]
|
||||
return result
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# Fallback: gesamten Text als Beschreibung nutzen
|
||||
result["description"] = raw[:2000]
|
||||
result["key_findings"] = [raw[:500]]
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post(
|
||||
"/vision/analyze",
|
||||
response_model=AnalyzeImageResponse,
|
||||
summary="Vision — Analysiere ein Bild mit Qwen2.5-VL-3B",
|
||||
response_description="Visuelle Analyse-Ergebnisse als Evidenz-Eintrag.",
|
||||
)
|
||||
async def analyze_image(request: AnalyzeImageRequest) -> AnalyzeImageResponse:
|
||||
"""Analysiere ein Bild mit Qwen2.5-VL-3B für visuelle Evidenz-Extraktion.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
request : AnalyzeImageRequest
|
||||
Bild-URL oder Base64-Daten plus Analyse-Parameter.
|
||||
|
||||
Returns
|
||||
-------
|
||||
AnalyzeImageResponse
|
||||
Evidenz-Eintrag mit key_findings, data_points, confidence.
|
||||
|
||||
Raises
|
||||
------
|
||||
HTTPException
|
||||
400 — Ungültige request. 500 — LLM-Fehler.
|
||||
"""
|
||||
if not request.image_data:
|
||||
raise HTTPException(status_code=400, detail="image_data darf nicht leer sein")
|
||||
|
||||
image_id = str(uuid.uuid4())
|
||||
evidence_id = str(uuid.uuid4())
|
||||
|
||||
# Prompt aufbauen
|
||||
prompt = _build_prompt(request.capture_type, request.image_caption, request.prompt)
|
||||
|
||||
# Vision-Provider aufrufen
|
||||
try:
|
||||
config = AppSettings.from_env()
|
||||
metrics = ProviderMetrics()
|
||||
provider = get_vision_provider(config, metrics)
|
||||
raw_response = await provider.analyze(request.image_data, prompt)
|
||||
except Exception as exc:
|
||||
logger.error("Vision analysis failed: %s", exc)
|
||||
# Fallback: store evidence with empty result
|
||||
evidence = EvidenceItem(
|
||||
id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
image_source=_image_source_label(request),
|
||||
description=f"Analyse fehlgeschlagen: {exc}",
|
||||
key_findings=[],
|
||||
data_points=[],
|
||||
confidence=0.0,
|
||||
sources=[],
|
||||
metadata={"error": str(exc), "image_id": image_id},
|
||||
)
|
||||
_store_evidence(evidence)
|
||||
return AnalyzeImageResponse(
|
||||
evidence_id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
description=f"Analyse fehlgeschlagen: {exc}",
|
||||
findings=[],
|
||||
confidence=0.0,
|
||||
data_points=[],
|
||||
image_source=_image_source_label(request),
|
||||
metadata={"error": str(exc)},
|
||||
)
|
||||
|
||||
# Response parsen
|
||||
parsed = _parse_vision_response(raw_response)
|
||||
|
||||
evidence = EvidenceItem(
|
||||
id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
image_source=_image_source_label(request),
|
||||
description=parsed.get("description", raw_response),
|
||||
key_findings=parsed.get("key_findings", []),
|
||||
data_points=parsed.get("data_points", []),
|
||||
confidence=parsed.get("confidence", 0.8),
|
||||
sources=parsed.get("sources", []),
|
||||
metadata={
|
||||
**parsed.get("metadata", {}),
|
||||
"image_id": image_id,
|
||||
"capture_type": request.capture_type,
|
||||
},
|
||||
)
|
||||
|
||||
_store_evidence(evidence)
|
||||
|
||||
return AnalyzeImageResponse(
|
||||
evidence_id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
description=evidence.description,
|
||||
findings=evidence.key_findings,
|
||||
confidence=evidence.confidence,
|
||||
data_points=evidence.data_points,
|
||||
image_source=evidence.image_source,
|
||||
metadata=evidence.metadata,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/vision/evidence/{evidence_id}",
|
||||
response_model=EvidenceResponse,
|
||||
summary="Vision — Hole visuellen Evidenz-Eintrag",
|
||||
response_description="Ein einzelner Evidenz-Eintrag oder 404.",
|
||||
)
|
||||
async def get_evidence(evidence_id: str) -> EvidenceResponse:
|
||||
"""Hole einen visuellen Evidenz-Eintrag aus dem Store.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
evidence_id : str
|
||||
UUID des Evidenz-Eintrags.
|
||||
|
||||
Returns
|
||||
-------
|
||||
EvidenceResponse
|
||||
Der Evidenz-Eintrag oder success=false bei 404.
|
||||
|
||||
Raises
|
||||
------
|
||||
HTTPException
|
||||
400 — Ungültige ID. 404 — Nicht gefunden.
|
||||
"""
|
||||
if not evidence_id or not evidence_id.strip():
|
||||
raise HTTPException(status_code=400, detail="evidence_id darf nicht leer sein")
|
||||
|
||||
evidence = _get_evidence(evidence_id)
|
||||
if evidence is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Kein Evidenz-Eintrag mit ID {evidence_id} gefunden",
|
||||
)
|
||||
|
||||
return EvidenceResponse(success=True, evidence=evidence)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _image_source_label(request: AnalyzeImageRequest) -> str:
|
||||
"""Generiere eine kurze Quelle-Beschreibung."""
|
||||
data = request.image_data
|
||||
if data.startswith("http"):
|
||||
return data[:120] + ("..." if len(data) > 120 else "")
|
||||
elif data.startswith("data:"):
|
||||
return "base64_encoded_data"
|
||||
return "uploaded_image"
|
||||
282
src/nsct/models/vision.py
Normal file
282
src/nsct/models/vision.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""Pydantic v2 schemas — Vision Evidence Extraction (Stage 10).
|
||||
|
||||
Qwen2.5-VL-3B extrahiert strukturierte Informationen aus Bildern:
|
||||
Diagramme, Screenshots, Infografiken, PDF-Layouts.
|
||||
Provenance-Pflicht: jede visuelle Evidenz ist quellenverknüpft.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureType(str, Enum):
|
||||
"""Klassifikation der visuellen Erfassung."""
|
||||
|
||||
RAW_IMAGE = "raw_image"
|
||||
DIAGRAM = "diagram"
|
||||
CHART = "chart"
|
||||
SCREENSHOT = "screenshot"
|
||||
INFOGRAPHIC = "infographic"
|
||||
PDF_LAYOUT = "pdf_layout"
|
||||
|
||||
|
||||
class VisionConfidence(str, Enum):
|
||||
"""Confidence-Stufe der visuellen Extraktion."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class EvidenceLevel(str, Enum):
|
||||
"""Evidenz-Level einer visuellen Evidenz."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class VisionEntityCategory(str, Enum):
|
||||
"""Kategorie einer erkannten Entity aus einem Bild."""
|
||||
|
||||
DATE = "date"
|
||||
PERSON = "person"
|
||||
ORGANIZATION = "organization"
|
||||
LOCATION = "location"
|
||||
NUMBER = "number"
|
||||
STATISTIC = "statistic"
|
||||
GRAPH_ELEMENT = "graph_element"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionCaptureSchema — einzelne visuell extrahierte Evidenz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureSchema(BaseModel):
|
||||
"""Einzelne visuell extrahierte Evidenz.
|
||||
|
||||
Felder:
|
||||
capture_type: Art der visuellen Erfassung
|
||||
image_data_url: Data-URL oder base64-codiertes Bild
|
||||
extracted_text: Vom Vision-Modell extrahierter Text
|
||||
entities: Erkannte Entities (Personen, Zahlen, etc.)
|
||||
confidence: Confidence der Extraktion
|
||||
evidence_level: Evidenz-Level
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
source_url: URL der Quelle (Provenance)
|
||||
metadata: Zusätzliche Metadaten
|
||||
"""
|
||||
|
||||
capture_type: VisionCaptureType = Field(
|
||||
...,
|
||||
description="Art der visuellen Erfassung (raw_image, diagram, chart, screenshot, infographic, pdf_layout).",
|
||||
)
|
||||
image_data_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Data-URL (data:image/...) oder base64-codiertes Bild.",
|
||||
)
|
||||
extracted_text: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Vom Vision-Modell extrahierter Textinhalt.",
|
||||
)
|
||||
entities: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Erkannte Entities aus dem Bild.",
|
||||
)
|
||||
confidence: float = Field(
|
||||
default=0.5,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence-Wert der Extraktion (0-1).",
|
||||
)
|
||||
confidence_label: VisionConfidence = Field(
|
||||
default=VisionConfidence.MEDIUM,
|
||||
description="Confidence-Stufe als Label.",
|
||||
)
|
||||
evidence_level: EvidenceLevel = Field(
|
||||
default=EvidenceLevel.MEDIUM,
|
||||
description="Evidenz-Level der visuellen Evidenz.",
|
||||
)
|
||||
source_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID der Quelle (source_id) zur Provenance.",
|
||||
)
|
||||
source_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="URL der Quelle (Provenance).",
|
||||
)
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusätzliche Metadaten (z.B. model_used, processing_time).",
|
||||
)
|
||||
|
||||
@field_validator("extracted_text")
|
||||
@classmethod
|
||||
def extracted_text_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("extracted_text darf nicht nur aus Whitespaces bestehen")
|
||||
return v
|
||||
|
||||
@field_validator("image_data_url")
|
||||
@classmethod
|
||||
def image_data_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("image_data_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("source_url")
|
||||
@classmethod
|
||||
def source_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("source_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionReportSchema — Zusammenfassung aller visuellen Evidenzen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionReportSchema(BaseModel):
|
||||
"""Zusammenfassung aller visuellen Evidenzen für einen Research-Run.
|
||||
|
||||
Felder:
|
||||
research_run_id: UUID des Research-Runs
|
||||
total_captures: Anzahl der visuellen Evidenzen
|
||||
captures: Liste aller visuellen Evidenzen
|
||||
entity_summary: Zusammenfassung aller erkannten Entities
|
||||
summary_text: Kurze Zusammenfassung des visuellen Contents
|
||||
generation_timestamp: Zeitstempel der Generierung
|
||||
"""
|
||||
|
||||
research_run_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID des Research-Runs.",
|
||||
)
|
||||
total_captures: int = Field(
|
||||
...,
|
||||
ge=0,
|
||||
description="Anzahl der visuellen Evidenzen in diesem Report.",
|
||||
)
|
||||
captures: list[VisionCaptureSchema] = Field(
|
||||
default_factory=list,
|
||||
description="Liste aller visuellen Evidenzen.",
|
||||
)
|
||||
entity_summary: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusammenfassung aller erkannten Entities.",
|
||||
)
|
||||
summary_text: str = Field(
|
||||
default="",
|
||||
description="Kurze Zusammenfassung des visuellen Contents.",
|
||||
)
|
||||
generation_timestamp: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc),
|
||||
description="Zeitstempel der Generierung.",
|
||||
)
|
||||
|
||||
@field_validator("research_run_id")
|
||||
@classmethod
|
||||
def research_run_id_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("research_run_id darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("summary_text")
|
||||
@classmethod
|
||||
def summary_not_political(cls, v: str) -> str:
|
||||
import re
|
||||
|
||||
forbidden = re.compile(
|
||||
r"(sollte\s+(Regierung|Bundesregierung)\s+(handeln|unterstützen)|"
|
||||
r"muss\s+(geändert|eingesetzt|gestürzt))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if forbidden.search(v):
|
||||
raise ValueError("VisionReport darf keine politische Empfehlung enthalten")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionRequestSchema — API-Request für die Vision-Extraktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionRequestSchema(BaseModel):
|
||||
"""API-Request zum Extrahieren visueller Evidenzen.
|
||||
|
||||
Felder:
|
||||
research_run_id: UUID des Research-Runs
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
source_url: URL der Quelle (Provenance)
|
||||
image_data: Base64-codiertes Bild oder Data-URL
|
||||
capture_type: Art der visuellen Erfassung
|
||||
prompt: Optionaler Prompt an das Vision-Modell
|
||||
"""
|
||||
|
||||
research_run_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID des Research-Runs.",
|
||||
)
|
||||
source_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID der Quelle (source_id).",
|
||||
)
|
||||
source_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="URL der Quelle (Provenance).",
|
||||
)
|
||||
image_data: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Base64-codiertes Bild oder Data-URL (data:image/...).",
|
||||
)
|
||||
capture_type: VisionCaptureType = Field(
|
||||
default=VisionCaptureType.RAW_IMAGE,
|
||||
description="Art der visuellen Erfassung.",
|
||||
)
|
||||
prompt: str | None = Field(
|
||||
default=None,
|
||||
description="Optionaler Prompt an das Vision-Modell für die Extraktion.",
|
||||
)
|
||||
|
||||
@field_validator("image_data")
|
||||
@classmethod
|
||||
def image_data_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("image_data darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("source_url")
|
||||
@classmethod
|
||||
def source_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("source_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
@@ -1,5 +1,10 @@
|
||||
"""Stage 5 — Claim Extraction package."""
|
||||
"""NSCT Stages — Pipeline stages for research runs."""
|
||||
|
||||
from nsct.stages.stage5_extract_claims import Stage5Extractor
|
||||
from nsct.stages.stage10_vision import VisionStage, VisionEvidence
|
||||
|
||||
__all__ = ["Stage5Extractor"]
|
||||
__all__ = [
|
||||
"Stage5Extractor",
|
||||
"VisionStage",
|
||||
"VisionEvidence",
|
||||
]
|
||||
750
src/nsct/stages/stage10_vision.py
Normal file
750
src/nsct/stages/stage10_vision.py
Normal file
@@ -0,0 +1,750 @@
|
||||
"""Stage 10: Vision Integration — visuelle Evidenz-Extraktion via Qwen2.5-VL-3B.
|
||||
|
||||
Pipeline für ein Research-Run:
|
||||
1. Lädt Bild-Captures aus der DB (Screenshot/Chart/Infografik).
|
||||
2. Bereitet base64-kodierte Bilder vor (max-size limits).
|
||||
3. Sendet jedes Bild an Qwen2.5-VL-3B für visuelle Analyse.
|
||||
4. Parallelt die Analyse via Semaphore (bounded concurrency).
|
||||
5. Parsed JSON-Response und extrahiert Entities, OCR-Text, Claims.
|
||||
6. Speichert VisionEvidence in der DB.
|
||||
7. Liefert StageResult mit allen extrahierten visuellen Evidenzen.
|
||||
|
||||
ARCHITEKTUR-REGELN:
|
||||
- Provenance-Pflicht: jede visuelle Analyse braucht source_url + evidence_span
|
||||
- Fehler pro Bild: kein Single-Point-of-Failure
|
||||
- Bounded Concurrency via asyncio.Semaphore
|
||||
- LLM-Output ist DATA, keine Instruktion
|
||||
- JSON-Parsing robust gegen Markdown-Code-Blocks
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024 # 20 MB hard cap for base64 payload
|
||||
DEFAULT_MAX_CONCURRENCY = 4
|
||||
VISION_BASE_URL = os.environ.get("NSCT_VISION_BASE_URL", "http://localhost:8030")
|
||||
VISION_API_KEY = os.environ.get(
|
||||
"HERMES_CUSTOM_192_168_80_199_8030_API_KEY",
|
||||
os.environ.get("NSCT_VISION_API_KEY", ""),
|
||||
)
|
||||
VISION_MODEL = os.environ.get("NSCT_VISION_MODEL", "Qwen2.5-VL-3B")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vision Prompt — SYSTEM_PROMPT for the Vision LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VISION_SYSTEM_PROMPT = (
|
||||
"Sie sind die visuelle Analyse-Engine des NSCT (Neutral Search Crawler Tool). "
|
||||
"Ihre Aufgabe ist es, visuell aus Bildern und Screenshots "
|
||||
"neutrale, evidenzbasierte Informationen zu extrahieren.\n\n"
|
||||
"KRITERIEN FÜR DIE ANALYSE:\n"
|
||||
"1. Texterkennung (OCR): Extrahieren Sie alle sichtbaren Texte.\n"
|
||||
"2. Entity-Erkennung: Identifizieren Sie Daten, Zahlen, Personen, "
|
||||
"Organisationen, Standorte, Statistiken.\n"
|
||||
"3. Diagramm-Interpretation: Falls das Bild Diagramme, Charts oder "
|
||||
"Graphen enthält, interpretieren Sie die Daten neutral — keine "
|
||||
"Fazit-Abstraktion, nur Rohdaten.\n"
|
||||
"4. Qualitätsbewertung: Bewerten Sie, wie gut das Bild als Evidenz "
|
||||
"taugt (HIGH / MEDIUM / LOW).\n\n"
|
||||
"WICHTIG:\n"
|
||||
"- LIEFEREN Sie NUR JSON — kein freier Text, keine Erklärungen.\n"
|
||||
"- Jede Extraktion braucht Provenance: source_url, evidence_span.\n"
|
||||
"- Keine Spekulation — nur das, was Sie visuell erkennen.\n"
|
||||
"- Wenn kein Text oder keine relevanten Daten erkennbar sind, "
|
||||
"geben Sie leere Listen zurück.\n\n"
|
||||
"FORMAT — JSON-Array mit einem Objekt:\n"
|
||||
"{\n"
|
||||
' "entities": [\n'
|
||||
' {"type": "statistic|date|person|organization|location|graph_element",\n'
|
||||
' "value": "...",\n'
|
||||
' "context": "...",\n'
|
||||
' "confidence": 0.9}\n'
|
||||
" ],\n"
|
||||
' "extracted_text": "Vollständiger OCR-Text des Bildes",\n'
|
||||
' "evidence_quality": "high|medium|low",\n'
|
||||
' "claim_candidates": [\n'
|
||||
' {"text": "Behauptung aus dem Bild",\n'
|
||||
' "source_url": "url",\n'
|
||||
' "evidence_span": "relevantes Fragment aus dem Bild"}\n'
|
||||
" ]\n"
|
||||
"}\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: Image preparation (base64, size limits)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _image_to_base64(image_bytes: bytes) -> str:
|
||||
"""Konvertiert Rohbild-Daten in base64, mit size limit.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
Wenn das Bild die maximale Größe überschreitet.
|
||||
"""
|
||||
if len(image_bytes) > MAX_IMAGE_SIZE_BYTES:
|
||||
raise ValueError(
|
||||
f"Image too large: {len(image_bytes)} bytes (max {MAX_IMAGE_SIZE_BYTES})"
|
||||
)
|
||||
return base64.b64encode(image_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _prepare_image_payload(image: dict[str, Any]) -> tuple[str, str] | tuple[None, str]:
|
||||
"""Bereitet ein Bild für die Vision-LLM-Analyse vor.
|
||||
|
||||
Unterstützt:
|
||||
- image_bytes (bytes): Rohbild-Daten
|
||||
- image_b64 (str): bereits base64-kodiertes Bild
|
||||
- image_url (str): URL des Bildes
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[str, str] oder None: (image_ref, error) — image_ref ist base64
|
||||
oder URL, None wenn das Bild übersprungen werden soll.
|
||||
"""
|
||||
# 1. image_b64 — bereits base64-kodiert
|
||||
b64 = image.get("image_b64")
|
||||
if b64 and isinstance(b64, str) and len(b64) > 0:
|
||||
return (b64, "")
|
||||
|
||||
# 2. image_bytes — Rohbild
|
||||
raw = image.get("image_bytes")
|
||||
if raw and isinstance(raw, bytes) and len(raw) > 0:
|
||||
try:
|
||||
return (_image_to_base64(raw), "")
|
||||
except ValueError as exc:
|
||||
return (str(exc), str(exc))
|
||||
|
||||
# 3. image_url — URL des Bildes
|
||||
url = image.get("image_url")
|
||||
if url and isinstance(url, str) and len(url) > 0:
|
||||
return (url, "")
|
||||
|
||||
# 4. Kein Bild — überspringen
|
||||
logger.warning("Image missing: no bytes, b64, or url provided")
|
||||
return None, "No image data"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: JSON-Parsing (robust gegen Markdown-Code-Blocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_vision_json(text: str) -> dict[str, Any]:
|
||||
"""Parsen der Vision-LLM-Antwort als JSON.
|
||||
|
||||
Robust: extrahiert JSON aus Code-Blocks (```json ... ```) und sucht
|
||||
die ersten { ... } Blöcke als Fallback.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
Wenn kein gültiger JSON-Inhalt gefunden wird.
|
||||
"""
|
||||
raw = text.strip()
|
||||
|
||||
# Extrahiere aus Code-Blocks
|
||||
if "```" in raw:
|
||||
lines = raw.split("\n")
|
||||
json_text = ""
|
||||
in_block = False
|
||||
for line in lines:
|
||||
if "```" in line:
|
||||
in_block = not in_block
|
||||
continue
|
||||
if in_block:
|
||||
json_text += line + "\n"
|
||||
raw = json_text.strip()
|
||||
|
||||
# Falls immer noch leer, nach { ... } suchen
|
||||
if not raw.startswith("{"):
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
raw = raw[start:end]
|
||||
|
||||
if not raw:
|
||||
raise ValueError("Vision response contained no JSON object")
|
||||
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid vision response JSON: {exc}") from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: Vision analysis per image
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _analyze_single_image(
|
||||
image: dict[str, Any],
|
||||
source_url: str,
|
||||
source_title: str | None,
|
||||
capture_type: str,
|
||||
vision_client: Any,
|
||||
semaphore: asyncio.Semaphore,
|
||||
) -> dict[str, Any]:
|
||||
"""Analysiert ein einzelnes Bild via Vision-LLM.
|
||||
|
||||
Gefangen in einem try/except: kein Single-Point-of-Failure.
|
||||
Selbst bei Fehler wird eine Fallback-Eintrag erstellt.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict: Das Vision-Ergebnis (oder Fallback bei Fehler).
|
||||
"""
|
||||
async with semaphore:
|
||||
try:
|
||||
# Prepare image payload
|
||||
img_result = _prepare_image_payload(image)
|
||||
if img_result is None:
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
"No image data provided"
|
||||
)
|
||||
|
||||
image_ref, error = img_result
|
||||
if error:
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
error
|
||||
)
|
||||
|
||||
# Build user prompt mit Metadaten
|
||||
user_prompt = (
|
||||
f"Quelle: {source_url}\n"
|
||||
f"Titel: {source_title or 'N/A'}\n"
|
||||
f"Capture-Typ: {capture_type}\n\n"
|
||||
"Analysieren Sie dieses Bild visuell. "
|
||||
"Extrahieren Sie alle sichtbaren Texte, Zahlen, "
|
||||
"Datumsangaben und Behauptungen. Bewerten Sie die "
|
||||
"Evidenzqualität. Antworten Sie als JSON."
|
||||
)
|
||||
|
||||
# Call vision provider
|
||||
llm_response = await vision_client.analyze(
|
||||
image_url_or_base64=image_ref,
|
||||
prompt=user_prompt,
|
||||
model=VISION_MODEL,
|
||||
)
|
||||
|
||||
# Parse JSON
|
||||
parsed = _parse_vision_json(llm_response)
|
||||
|
||||
# Validate structure
|
||||
if not isinstance(parsed, dict):
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
"Vision response is not a JSON object"
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"source_url": source_url,
|
||||
"source_title": source_title,
|
||||
"capture_type": capture_type,
|
||||
"entities": parsed.get("entities", []),
|
||||
"extracted_text": parsed.get("extracted_text", ""),
|
||||
"evidence_quality": parsed.get("evidence_quality", "low"),
|
||||
"claim_candidates": parsed.get("claim_candidates", []),
|
||||
"raw_response": llm_response,
|
||||
"model_used": VISION_MODEL,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Vision analysis failed for %s: %s", source_url, exc
|
||||
)
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
str(exc)
|
||||
)
|
||||
|
||||
|
||||
def _make_error_result(
|
||||
image: dict[str, Any],
|
||||
source_url: str,
|
||||
source_title: str | None,
|
||||
capture_type: str,
|
||||
error: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Erstellt einen Fallback-Eintrag bei Fehler — kein Single-Point-of-Failure."""
|
||||
return {
|
||||
"success": False,
|
||||
"source_url": source_url,
|
||||
"source_title": source_title,
|
||||
"capture_type": capture_type,
|
||||
"entities": [],
|
||||
"extracted_text": "",
|
||||
"evidence_quality": "low",
|
||||
"claim_candidates": [],
|
||||
"error": error,
|
||||
"model_used": VISION_MODEL,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionEvidence dataclass — DB-entität für visuellen Evidenz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class VisionEvidence:
|
||||
"""Repräsentiert eine visuelle Evidenz aus der Bildanalyse.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
source_url : str
|
||||
Die URL der Quelle, aus der das Bild stammt (Provenance).
|
||||
source_title : str | None
|
||||
Titel der Quelle.
|
||||
capture_type : str
|
||||
Art des Captures (screenshot, chart, infographic, thumbnail).
|
||||
entities : list[dict]
|
||||
Extrahierte Entities (Personen, Statistiken, Daten, etc.).
|
||||
extracted_text : str
|
||||
Vollständiger OCR-Text aus dem Bild.
|
||||
evidence_quality : str
|
||||
"high", "medium", oder "low" — Evidenzqualität.
|
||||
claim_candidates : list[dict]
|
||||
Behauptungen die aus dem Bild extrahiert wurden.
|
||||
raw_response : str | None
|
||||
Roh-LLM-Antwort für Audit-Zwecke.
|
||||
model_used : str
|
||||
Verwendetes Vision-Modell.
|
||||
error : str | None
|
||||
Fehlermeldung bei fehlgeschlagener Analyse.
|
||||
"""
|
||||
source_url: str
|
||||
source_title: str | None
|
||||
capture_type: str
|
||||
entities: list[dict[str, Any]] = field(default_factory=list)
|
||||
extracted_text: str = ""
|
||||
evidence_quality: str = "low"
|
||||
claim_candidates: list[dict[str, Any]] = field(default_factory=list)
|
||||
raw_response: str | None = None
|
||||
model_used: str = VISION_MODEL
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return self.error is None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialisiert das Objekt als Dictionary für die DB."""
|
||||
return {
|
||||
"source_url": self.source_url,
|
||||
"source_title": self.source_title,
|
||||
"capture_type": self.capture_type,
|
||||
"entities": self.entities,
|
||||
"extracted_text": self.extracted_text,
|
||||
"evidence_quality": self.evidence_quality,
|
||||
"claim_candidates": self.claim_candidates,
|
||||
"raw_response": self.raw_response,
|
||||
"model_used": self.model_used,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_analysis_result(cls, result: dict[str, Any]) -> "VisionEvidence":
|
||||
"""Erstellt VisionEvidence aus einem Analyse-Ergebnis-Dict."""
|
||||
return cls(
|
||||
source_url=result.get("source_url", ""),
|
||||
source_title=result.get("source_title"),
|
||||
capture_type=result.get("capture_type", "unknown"),
|
||||
entities=result.get("entities", []),
|
||||
extracted_text=result.get("extracted_text", ""),
|
||||
evidence_quality=result.get("evidence_quality", "low"),
|
||||
claim_candidates=result.get("claim_candidates", []),
|
||||
raw_response=result.get("raw_response"),
|
||||
model_used=result.get("model_used", VISION_MODEL),
|
||||
error=None if result.get("success") else result.get("error"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 10: VisionStage (BaseStage + execute)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionStage:
|
||||
"""Stage 10: Vision Integration — visuelle Evidenz-Extraktion via Qwen2.5-VL-3B.
|
||||
|
||||
Verarbeitet Bild-Captures aus einer Research-Run, analysiert sie
|
||||
visuell via Vision-LLM und speichert die extrahierten Evidenzen.
|
||||
|
||||
Usage::
|
||||
|
||||
stage = VisionStage(
|
||||
research_run_id=uuid,
|
||||
vision_client=vision_provider,
|
||||
config=config,
|
||||
images=[...],
|
||||
)
|
||||
result = await stage.execute()
|
||||
"""
|
||||
|
||||
stage_number = 10
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
research_run_id: UUID,
|
||||
vision_client: Any,
|
||||
config: Any,
|
||||
images: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self.research_run_id = research_run_id
|
||||
self.vision_client = vision_client
|
||||
self.config = config
|
||||
self.images = images or []
|
||||
self.max_concurrency = int(
|
||||
os.environ.get("NSCT_VISION_MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY)
|
||||
)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Stage 10: Vision Integration"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _fetch_images_from_db(self) -> list[dict[str, Any]]:
|
||||
"""Lädt Captures (Screenshots, Charts, etc.) aus der DB.
|
||||
|
||||
Falls images im Constructor mitgegeben wurden, werden diese
|
||||
verwendet, andernfalls wird ein leerer List zurückgegeben.
|
||||
|
||||
In einer echten Implementierung würde hier die SQLAlchemy Session
|
||||
verwendet werden, um Captures aus der database zu lesen.
|
||||
"""
|
||||
if self.images:
|
||||
return self.images
|
||||
return []
|
||||
|
||||
def _prepare_prompts(
|
||||
self,
|
||||
images: list[dict[str, Any]],
|
||||
) -> list[tuple[dict[str, Any], str, str | None, str]]:
|
||||
"""Bereitet die Eingabeparameter für die parallele Analyse vor.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[tuple[dict, str, str | None, str]]:
|
||||
(image, source_url, source_title, capture_type)
|
||||
"""
|
||||
prompts = []
|
||||
for image in images:
|
||||
source_url = (
|
||||
image.get("source_url", "")
|
||||
or image.get("url", "")
|
||||
or image.get("image_url", "")
|
||||
or "unknown"
|
||||
)
|
||||
source_title = image.get("source_title") or image.get("title")
|
||||
capture_type = (
|
||||
image.get("capture_type", "screenshot")
|
||||
or image.get("type", "screenshot")
|
||||
or "screenshot"
|
||||
)
|
||||
prompts.append((image, source_url, source_title, capture_type))
|
||||
return prompts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execute — main pipeline
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def execute(self, **kwargs: Any) -> "StageResult":
|
||||
"""Führt die vollständige Stage-10-Pipeline aus.
|
||||
|
||||
Returns
|
||||
-------
|
||||
StageResult mit success=True und den Vision-Evidenzen in .data.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
try:
|
||||
# Load images
|
||||
images = kwargs.get("images")
|
||||
if images is None:
|
||||
images = await self._fetch_images_from_db()
|
||||
|
||||
if not images:
|
||||
msg = "Keine Bilder vorhanden - keine Vision-Analyse moeglich."
|
||||
logger.warning("Stage 10: %s", msg)
|
||||
errors.append(msg)
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Stage 10: Image loading failed: %s", exc)
|
||||
errors.append(f"Image loading failed: {exc}")
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
try:
|
||||
# Prepare analysis prompts
|
||||
prompts = self._prepare_prompts(images)
|
||||
|
||||
if not prompts:
|
||||
msg = "Keine gültigen Bild-Eingaben gefunden."
|
||||
logger.warning("Stage 10: %s", msg)
|
||||
errors.append(msg)
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
# Create semaphore for bounded concurrency
|
||||
semaphore = asyncio.Semaphore(self.max_concurrency)
|
||||
|
||||
logger.info(
|
||||
"Stage 10: Analyzing %d images for research run %s "
|
||||
"(max concurrency: %d)",
|
||||
len(prompts),
|
||||
self.research_run_id,
|
||||
self.max_concurrency,
|
||||
)
|
||||
|
||||
# Run parallel vision analysis — error per image, no SPOF
|
||||
tasks = [
|
||||
_analyze_single_image(
|
||||
image,
|
||||
source_url,
|
||||
source_title,
|
||||
capture_type,
|
||||
self.vision_client,
|
||||
semaphore,
|
||||
)
|
||||
for image, source_url, source_title, capture_type in prompts
|
||||
]
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Process results
|
||||
successful: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, Any]] = []
|
||||
|
||||
for idx, result in enumerate(results):
|
||||
if isinstance(result, Exception):
|
||||
err_msg = f"Analysis task {idx} raised: {result}"
|
||||
logger.error("Stage 10: %s", err_msg)
|
||||
failures.append({
|
||||
"success": False,
|
||||
"source_url": (
|
||||
prompts[idx][1] if idx < len(prompts) else "unknown"
|
||||
),
|
||||
"error": err_msg,
|
||||
})
|
||||
errors.append(err_msg)
|
||||
elif isinstance(result, dict):
|
||||
if result.get("success"):
|
||||
successful.append(result)
|
||||
else:
|
||||
failures.append(result)
|
||||
err_msg = f"Analysis failed for {result.get('source_url', 'unknown')}: {result.get('error', 'unknown')}"
|
||||
errors.append(err_msg)
|
||||
logger.warning("Stage 10: %s", err_msg)
|
||||
|
||||
else:
|
||||
err_msg = f"Unexpected result type for image {idx}: {type(result)}"
|
||||
logger.error("Stage 10: %s", err_msg)
|
||||
failures.append({"success": False, "error": err_msg})
|
||||
errors.append(err_msg)
|
||||
|
||||
# Convert to VisionEvidence objects
|
||||
vision_evidences = [
|
||||
VisionEvidence.from_analysis_result(r)
|
||||
for r in successful
|
||||
]
|
||||
|
||||
# Build DB-ready records
|
||||
db_records = [ev.to_dict() for ev in vision_evidences]
|
||||
|
||||
# Collect all entities and claim candidates across all images
|
||||
all_entities: list[dict[str, Any]] = []
|
||||
all_claims: list[dict[str, Any]] = []
|
||||
for record in db_records:
|
||||
all_entities.extend(record.get("entities", []))
|
||||
all_claims.extend(record.get("claim_candidates", []))
|
||||
|
||||
logger.info(
|
||||
"Stage 10: Vision analysis complete — "
|
||||
"%d successful, %d failures, "
|
||||
"%d entities, %d claims extracted",
|
||||
len(successful),
|
||||
len(failures),
|
||||
len(all_entities),
|
||||
len(all_claims),
|
||||
)
|
||||
|
||||
# Store in DB (placeholder — replace with actual DB insert)
|
||||
# await self._store_vision_evidence(db_records)
|
||||
|
||||
# Build result
|
||||
analysis_data = {
|
||||
"vision_evidences": db_records,
|
||||
"total_images": len(images),
|
||||
"successful": len(successful),
|
||||
"failed": len(failures),
|
||||
"entities_count": len(all_entities),
|
||||
"claims_count": len(all_claims),
|
||||
"model_used": VISION_MODEL,
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"generation_timestamp": datetime_now_utc(),
|
||||
}
|
||||
|
||||
return StageResult(
|
||||
success=True,
|
||||
data=analysis_data,
|
||||
stage=self,
|
||||
)
|
||||
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
logger.error("Stage 10: Vision analysis failed: %s", exc)
|
||||
errors.append(f"Vision analysis failed: {exc}")
|
||||
return self._fallback_result(errors)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Stage 10: Unexpected error: %s", exc)
|
||||
errors.append(f"Unexpected error: {exc}")
|
||||
return self._fallback_result(errors)
|
||||
|
||||
def _fallback_result(self, errors: list[str]) -> "StageResult":
|
||||
"""Fallback: minimaler Bericht wenn LLM nicht verfügbar ist."""
|
||||
logger.warning(
|
||||
"Stage 10: Using fallback — vision analysis unavailable"
|
||||
)
|
||||
|
||||
fallback_data = {
|
||||
"vision_evidences": [],
|
||||
"total_images": len(self.images),
|
||||
"successful": 0,
|
||||
"failed": 0,
|
||||
"entities_count": 0,
|
||||
"claims_count": 0,
|
||||
"model_used": VISION_MODEL,
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"generation_timestamp": datetime_now_utc(),
|
||||
"methodology": (
|
||||
"NSCT Stage 10: Vision Integration (FALLBACK). "
|
||||
"Vision-LLM-Analyse war nicht verfügbar. "
|
||||
"Keine visuellen Evidenzen extrahiert."
|
||||
),
|
||||
}
|
||||
|
||||
return StageResult(
|
||||
success=False,
|
||||
data=fallback_data,
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StageResult — referenced from stage9_synthesis.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# We import from stage9_synthesis to avoid circular imports at module level.
|
||||
# These are defined here for convenience when running this file standalone.
|
||||
|
||||
|
||||
class StageResult:
|
||||
"""Result returned by a stage's execute() method."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
success: bool,
|
||||
data: dict[str, Any] | None = None,
|
||||
stage: Any = None,
|
||||
errors: list[str] | None = None,
|
||||
) -> None:
|
||||
self.success = success
|
||||
self.data = data or {}
|
||||
self.stage = stage
|
||||
self.errors = errors or []
|
||||
|
||||
@property
|
||||
def stage_name(self) -> str:
|
||||
if self.stage:
|
||||
return self.stage.name
|
||||
return "unknown"
|
||||
|
||||
@property
|
||||
def research_run_id(self) -> UUID | None:
|
||||
if self.stage:
|
||||
return self.stage.research_run_id
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def datetime_now_utc() -> str:
|
||||
"""Return current UTC time as ISO-8601 string."""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy compatibility wrapper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Stage10Vision:
|
||||
"""Legacy wrapper: Stage 10 with a synchronous `.run()` for backward compat.
|
||||
|
||||
Instantiates VisionStage internally and delegates to ``execute()``.
|
||||
"""
|
||||
|
||||
def __init__(self, vision_client, config, research_run_id, images=None):
|
||||
self._stage = VisionStage(
|
||||
research_run_id=research_run_id,
|
||||
vision_client=vision_client,
|
||||
config=config,
|
||||
images=images or [],
|
||||
)
|
||||
|
||||
@property
|
||||
def research_run_id(self) -> UUID:
|
||||
return self._stage.research_run_id
|
||||
|
||||
async def run(self) -> dict[str, Any]:
|
||||
"""Backward-compatible async .run() method."""
|
||||
result = await self._stage.execute()
|
||||
if not result.success:
|
||||
raise RuntimeError(
|
||||
"Stage 10 vision analysis failed: " + "; ".join(result.errors)
|
||||
)
|
||||
return result.data
|
||||
@@ -471,4 +471,134 @@ class EvidenceScoreRelationModel(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_evidence_score_relations_score_id", "score_id"),
|
||||
Index("ix_evidence_score_relations_related_claim_id", "related_claim_id"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 10 — Vision Evidence Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureType(str, enum.Enum):
|
||||
"""Klassifikation der visuellen Erfassung (Stage 10)."""
|
||||
|
||||
RAW_IMAGE = "raw_image"
|
||||
DIAGRAM = "diagram"
|
||||
CHART = "chart"
|
||||
SCREENSHOT = "screenshot"
|
||||
INFOGRAPHIC = "infographic"
|
||||
PDF_LAYOUT = "pdf_layout"
|
||||
|
||||
|
||||
class VisionEvidenceLevel(str, enum.Enum):
|
||||
"""Evidenz-Level einer visuellen Evidenz (Stage 10)."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class VisionEntityType(str, enum.Enum):
|
||||
"""Kategorie einer erkannten Entity aus einem Bild (Stage 10)."""
|
||||
|
||||
DATE = "date"
|
||||
PERSON = "person"
|
||||
ORGANIZATION = "organization"
|
||||
LOCATION = "location"
|
||||
NUMBER = "number"
|
||||
STATISTIC = "statistic"
|
||||
GRAPH_ELEMENT = "graph_element"
|
||||
|
||||
|
||||
class VisionEvidenceModel(Base):
|
||||
"""Einzelne visuell extrahierte Evidenz (Stage 10).
|
||||
|
||||
Felder:
|
||||
uuid: Primärschlüssel (UUID)
|
||||
research_run_id: Research-Run-Zuordnung
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
capture_type: Art der visuellen Erfassung
|
||||
extracted_text: Vom Vision-Modell extrahierter Text
|
||||
image_data_url: Data-URL oder base64-codiertes Bild (optional)
|
||||
entities: JSON mit erkannten Entities
|
||||
confidence: Confidence 0.0–1.0
|
||||
confidence_label: Confidence als Label (HIGH/MEDIUM/LOW/UNCERTAIN)
|
||||
evidence_level: Evidenz-Level (HIGH/MEDIUM/LOW/UNCERTAIN)
|
||||
created_at / updated_at: Zeitstempel
|
||||
"""
|
||||
|
||||
__tablename__ = "vision_evidence"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
research_run_id = Column(String(36), nullable=False)
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
|
||||
capture_type = Column(Enum(VisionCaptureType), nullable=False)
|
||||
|
||||
extracted_text = Column(Text, nullable=False)
|
||||
image_data_url = Column(Text, nullable=True)
|
||||
entities = Column(JSON, nullable=False, default=dict)
|
||||
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
confidence_label = Column(String(16), nullable=False, default="medium")
|
||||
evidence_level = Column(Enum(VisionEvidenceLevel), nullable=False, default=VisionEvidenceLevel.MEDIUM)
|
||||
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
source = relationship("SourceModel")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_vision_evidence_research_run_id", "research_run_id"),
|
||||
Index("ix_vision_evidence_capture_type", "capture_type"),
|
||||
Index("ix_vision_evidence_source_id", "source_id"),
|
||||
)
|
||||
|
||||
|
||||
class VisionEntityModel(Base):
|
||||
"""Erkannte Entity aus einem visuellen Bild (Stage 10).
|
||||
|
||||
Felder:
|
||||
uuid: Primärschlüssel (UUID)
|
||||
evidence_id: FK zur VisionEvidence
|
||||
entity_type: Kategorie der Entity
|
||||
entity_value: Der erkannte Wert
|
||||
context: Kontextbeschreibung
|
||||
confidence: Confidence 0.0–1.0
|
||||
created_at: Zeitstempel
|
||||
"""
|
||||
|
||||
__tablename__ = "vision_entities"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
evidence_id = Column(
|
||||
String(36),
|
||||
ForeignKey("vision_evidence.id"),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
entity_type = Column(Enum(VisionEntityType), nullable=False)
|
||||
entity_value = Column(Text, nullable=False)
|
||||
context = Column(Text, nullable=True)
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
evidence = relationship("VisionEvidenceModel", back_populates="entities")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_vision_entities_evidence_id", "evidence_id"),
|
||||
Index("ix_vision_entities_entity_type", "entity_type"),
|
||||
)
|
||||
|
||||
|
||||
# Back-populate the backref for VisionEntityModel
|
||||
VisionEvidenceModel.entities = relationship(
|
||||
"VisionEntityModel",
|
||||
back_populates="evidence",
|
||||
cascade="all, delete-orphan",
|
||||
foreign_keys="VisionEntityModel.evidence_id",
|
||||
)
|
||||
Reference in New Issue
Block a user