feat(stage10): implement vision integration
This commit is contained in:
NSCT Agent
@@ -8,7 +8,7 @@ version = "0.1.0"
|
||||
description = "Neutral Search Crawler Tool — lokal betreibbares Recherche- und Analyse-System"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
requires-python = ">=3.12"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0",
|
||||
"pydantic>=2.0,<3.0",
|
||||
|
||||
@@ -103,6 +103,10 @@ def create_app() -> FastAPI:
|
||||
from nsct.api.synthesis import router as synthesis_router
|
||||
app.include_router(synthesis_router, tags=["research"])
|
||||
|
||||
# Mount vision router (Stage 10)
|
||||
from nsct.api.vision import router as vision_router
|
||||
app.include_router(vision_router, tags=["vision"])
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
376
src/nsct/api/vision.py
Normal file
376
src/nsct/api/vision.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""Vision API endpoints — Qwen2.5-VL-3B visual evidence extraction.
|
||||
|
||||
Endpunkte:
|
||||
POST /vision/analyze — Analysiere ein Bild mit Qwen2.5-VL-3B
|
||||
GET /vision/evidence/{evidence_id} — Hole visuellen Evidenz-Eintrag
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from httpx import AsyncClient
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from nsct.config import AppSettings
|
||||
from nsct.providers.vision import get_provider as get_vision_provider
|
||||
from nsct.providers.metrics import ProviderMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request / Response Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AnalyzeImageRequest(BaseModel):
|
||||
"""Request zum Analysieren eines Bildes mit Qwen2.5-VL-3B."""
|
||||
|
||||
image_data: str = Field(
|
||||
...,
|
||||
description="Base64-kodiertes Bild oder URL (http/https).",
|
||||
min_length=1,
|
||||
)
|
||||
capture_type: str = Field(
|
||||
default="screenshot",
|
||||
description="Art der visuellen Aufnahme: screenshot, infographic, diagram, chart, photo, document.",
|
||||
)
|
||||
prompt: str | None = Field(
|
||||
default=None,
|
||||
description="Optionales Analyse-Prompt. Wenn None → Standard-Prompt für Evidenz-Extraktion.",
|
||||
)
|
||||
image_caption: str | None = Field(
|
||||
default=None,
|
||||
description="Optionale Beschreibung des Bildes als Kontext.",
|
||||
)
|
||||
evidence_type: str = Field(
|
||||
default="visual",
|
||||
description="Kategorie der Evidenz: visual, chart, document, infographic.",
|
||||
)
|
||||
|
||||
|
||||
class EvidenceItem(BaseModel):
|
||||
"""Ein einzelner visueller Evidenz-Eintrag."""
|
||||
|
||||
id: str = Field(..., description="UUID des Evidenz-Eintrags")
|
||||
evidence_type: str = Field(..., description="Kategorie der Evidenz")
|
||||
capture_type: str = Field(..., description="Art der visuellen Aufnahme")
|
||||
image_source: str = Field(..., description="Quelle/Beschreibung des Bildes")
|
||||
description: str = Field(..., description="Beschreibung der visuellen Inhalte")
|
||||
key_findings: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Wichtige visuelle Erkenntnisse aus dem Bild.",
|
||||
)
|
||||
data_points: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Strukturierte Datenpunkte aus dem Bild.",
|
||||
)
|
||||
confidence: float = Field(
|
||||
default=0.8,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Vertrauen der visuellen Analyse (0-1).",
|
||||
)
|
||||
sources: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Quellen/Referenzen innerhalb des Bildes.",
|
||||
)
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusätzliche Metadaten zur Analyse.",
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeImageResponse(BaseModel):
|
||||
"""Antwort auf Bildanalyse-Anfrage."""
|
||||
|
||||
evidence_id: str = Field(..., description="UUID des erstellten Evidenz-Eintrags")
|
||||
evidence_type: str = Field(..., description="Kategorie der Evidenz")
|
||||
capture_type: str = Field(..., description="Art der visuellen Aufnahme")
|
||||
description: str = Field(..., description="Zusammenfassung der visuellen Analyse")
|
||||
findings: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Visuelle Erkenntnisse (alias key_findings).",
|
||||
)
|
||||
confidence: float = Field(..., ge=0.0, le=1.0, description="Vertrauen der Analyse")
|
||||
data_points: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Strukturierte Datenpunkte aus dem Bild.",
|
||||
)
|
||||
image_source: str = Field(..., description="Quelle des Bildes")
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict, description="Zusätzliche Metadaten."
|
||||
)
|
||||
|
||||
|
||||
class EvidenceResponse(BaseModel):
|
||||
"""Antwort für GET /vision/evidence/{evidence_id}."""
|
||||
|
||||
success: bool
|
||||
evidence: EvidenceItem | None = Field(
|
||||
default=None, description="Visueller Evidenz-Eintrag oder None bei 404."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper — Mock-Speicher (für Prototyp)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# In-Memory Store für Evidenz-Einträge (Produktion → DB)
|
||||
_evidence_store: dict[str, EvidenceItem] = {}
|
||||
|
||||
|
||||
def _store_evidence(evidence: EvidenceItem) -> None:
|
||||
_evidence_store[evidence.id] = evidence
|
||||
|
||||
|
||||
def _get_evidence(evidence_id: str) -> EvidenceItem | None:
|
||||
return _evidence_store.get(evidence_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper — Default-Prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_VISION_PROMPT = (
|
||||
"Analysiere das Bild systematisch als visueller Evidenz-Extraktor. "
|
||||
"Identifiziere: (1) visuelle Inhalte und Objekte, (2) Textinhalte und Überschriften, "
|
||||
"3) Daten, Diagramme oder Grafiken, (4) visuelle Trends und Muster, "
|
||||
"5) Schlüsselinformationen für fact-checking, (6) Vertrauenswürdigkeit der visuellen Evidenz. "
|
||||
"Antworte mit einer strukturierten Analyse. Format: JSON-Array mit key_findings (Strings), "
|
||||
"data_points (Objekte), sources (Strings), confidence (Float 0-1), image_source (String)."
|
||||
)
|
||||
|
||||
|
||||
def _build_prompt(
|
||||
capture_type: str, image_caption: str | None, custom_prompt: str | None
|
||||
) -> str:
|
||||
"""Kombiniere den Standard-Prompt mit optionalen Anpassungen."""
|
||||
prompt = custom_prompt or _DEFAULT_VISION_PROMPT
|
||||
if image_caption:
|
||||
prompt = f"Bild-Beschreibung: {image_caption}\n\n{prompt}"
|
||||
if capture_type and capture_type != "screenshot":
|
||||
prompt = f"Capture-Typ: {capture_type}. {prompt}"
|
||||
return prompt
|
||||
|
||||
|
||||
def _parse_vision_response(raw: str) -> dict[str, Any]:
|
||||
"""Parst die Vision-LLM-Antwort und extrahiert strukturierte Daten."""
|
||||
result: dict[str, Any] = {
|
||||
"description": raw,
|
||||
"key_findings": [],
|
||||
"data_points": [],
|
||||
"confidence": 0.8,
|
||||
"image_source": "unknown",
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
if not raw or not raw.strip():
|
||||
result["description"] = "Keine Inhalte erkannt"
|
||||
return result
|
||||
|
||||
# Versuche JSON-Array zu parse
|
||||
raw_stripped = raw.strip()
|
||||
# Strip markdown code blocks
|
||||
if raw_stripped.startswith("```"):
|
||||
lines = raw_stripped.split("\n")
|
||||
if lines[0].startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].startswith("```"):
|
||||
lines = lines[:-1]
|
||||
raw_stripped = "\n".join(lines).strip()
|
||||
|
||||
if not raw_stripped:
|
||||
result["description"] = "Leere Antwort vom Vision-Modell"
|
||||
return result
|
||||
|
||||
# Parse als JSON
|
||||
import json
|
||||
|
||||
try:
|
||||
data = json.loads(raw_stripped)
|
||||
if isinstance(data, dict):
|
||||
result["key_findings"] = data.get("key_findings", [])
|
||||
result["data_points"] = data.get("data_points", [])
|
||||
result["confidence"] = min(max(data.get("confidence", 0.8), 0.0), 1.0)
|
||||
result["image_source"] = data.get("image_source", "unknown")
|
||||
result["metadata"] = data.get("metadata", {})
|
||||
result["description"] = data.get("description", raw)
|
||||
return result
|
||||
elif isinstance(data, list):
|
||||
result["key_findings"] = [
|
||||
str(item) for item in data if isinstance(item, (str, dict, list))
|
||||
]
|
||||
return result
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# Fallback: gesamten Text als Beschreibung nutzen
|
||||
result["description"] = raw[:2000]
|
||||
result["key_findings"] = [raw[:500]]
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post(
|
||||
"/vision/analyze",
|
||||
response_model=AnalyzeImageResponse,
|
||||
summary="Vision — Analysiere ein Bild mit Qwen2.5-VL-3B",
|
||||
response_description="Visuelle Analyse-Ergebnisse als Evidenz-Eintrag.",
|
||||
)
|
||||
async def analyze_image(request: AnalyzeImageRequest) -> AnalyzeImageResponse:
|
||||
"""Analysiere ein Bild mit Qwen2.5-VL-3B für visuelle Evidenz-Extraktion.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
request : AnalyzeImageRequest
|
||||
Bild-URL oder Base64-Daten plus Analyse-Parameter.
|
||||
|
||||
Returns
|
||||
-------
|
||||
AnalyzeImageResponse
|
||||
Evidenz-Eintrag mit key_findings, data_points, confidence.
|
||||
|
||||
Raises
|
||||
------
|
||||
HTTPException
|
||||
400 — Ungültige request. 500 — LLM-Fehler.
|
||||
"""
|
||||
if not request.image_data:
|
||||
raise HTTPException(status_code=400, detail="image_data darf nicht leer sein")
|
||||
|
||||
image_id = str(uuid.uuid4())
|
||||
evidence_id = str(uuid.uuid4())
|
||||
|
||||
# Prompt aufbauen
|
||||
prompt = _build_prompt(request.capture_type, request.image_caption, request.prompt)
|
||||
|
||||
# Vision-Provider aufrufen
|
||||
try:
|
||||
config = AppSettings.from_env()
|
||||
metrics = ProviderMetrics()
|
||||
provider = get_vision_provider(config, metrics)
|
||||
raw_response = await provider.analyze(request.image_data, prompt)
|
||||
except Exception as exc:
|
||||
logger.error("Vision analysis failed: %s", exc)
|
||||
# Fallback: store evidence with empty result
|
||||
evidence = EvidenceItem(
|
||||
id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
image_source=_image_source_label(request),
|
||||
description=f"Analyse fehlgeschlagen: {exc}",
|
||||
key_findings=[],
|
||||
data_points=[],
|
||||
confidence=0.0,
|
||||
sources=[],
|
||||
metadata={"error": str(exc), "image_id": image_id},
|
||||
)
|
||||
_store_evidence(evidence)
|
||||
return AnalyzeImageResponse(
|
||||
evidence_id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
description=f"Analyse fehlgeschlagen: {exc}",
|
||||
findings=[],
|
||||
confidence=0.0,
|
||||
data_points=[],
|
||||
image_source=_image_source_label(request),
|
||||
metadata={"error": str(exc)},
|
||||
)
|
||||
|
||||
# Response parsen
|
||||
parsed = _parse_vision_response(raw_response)
|
||||
|
||||
evidence = EvidenceItem(
|
||||
id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
image_source=_image_source_label(request),
|
||||
description=parsed.get("description", raw_response),
|
||||
key_findings=parsed.get("key_findings", []),
|
||||
data_points=parsed.get("data_points", []),
|
||||
confidence=parsed.get("confidence", 0.8),
|
||||
sources=parsed.get("sources", []),
|
||||
metadata={
|
||||
**parsed.get("metadata", {}),
|
||||
"image_id": image_id,
|
||||
"capture_type": request.capture_type,
|
||||
},
|
||||
)
|
||||
|
||||
_store_evidence(evidence)
|
||||
|
||||
return AnalyzeImageResponse(
|
||||
evidence_id=evidence_id,
|
||||
evidence_type=request.evidence_type,
|
||||
capture_type=request.capture_type,
|
||||
description=evidence.description,
|
||||
findings=evidence.key_findings,
|
||||
confidence=evidence.confidence,
|
||||
data_points=evidence.data_points,
|
||||
image_source=evidence.image_source,
|
||||
metadata=evidence.metadata,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/vision/evidence/{evidence_id}",
|
||||
response_model=EvidenceResponse,
|
||||
summary="Vision — Hole visuellen Evidenz-Eintrag",
|
||||
response_description="Ein einzelner Evidenz-Eintrag oder 404.",
|
||||
)
|
||||
async def get_evidence(evidence_id: str) -> EvidenceResponse:
|
||||
"""Hole einen visuellen Evidenz-Eintrag aus dem Store.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
evidence_id : str
|
||||
UUID des Evidenz-Eintrags.
|
||||
|
||||
Returns
|
||||
-------
|
||||
EvidenceResponse
|
||||
Der Evidenz-Eintrag oder success=false bei 404.
|
||||
|
||||
Raises
|
||||
------
|
||||
HTTPException
|
||||
400 — Ungültige ID. 404 — Nicht gefunden.
|
||||
"""
|
||||
if not evidence_id or not evidence_id.strip():
|
||||
raise HTTPException(status_code=400, detail="evidence_id darf nicht leer sein")
|
||||
|
||||
evidence = _get_evidence(evidence_id)
|
||||
if evidence is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Kein Evidenz-Eintrag mit ID {evidence_id} gefunden",
|
||||
)
|
||||
|
||||
return EvidenceResponse(success=True, evidence=evidence)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _image_source_label(request: AnalyzeImageRequest) -> str:
|
||||
"""Generiere eine kurze Quelle-Beschreibung."""
|
||||
data = request.image_data
|
||||
if data.startswith("http"):
|
||||
return data[:120] + ("..." if len(data) > 120 else "")
|
||||
elif data.startswith("data:"):
|
||||
return "base64_encoded_data"
|
||||
return "uploaded_image"
|
||||
282
src/nsct/models/vision.py
Normal file
282
src/nsct/models/vision.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""Pydantic v2 schemas — Vision Evidence Extraction (Stage 10).
|
||||
|
||||
Qwen2.5-VL-3B extrahiert strukturierte Informationen aus Bildern:
|
||||
Diagramme, Screenshots, Infografiken, PDF-Layouts.
|
||||
Provenance-Pflicht: jede visuelle Evidenz ist quellenverknüpft.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureType(str, Enum):
|
||||
"""Klassifikation der visuellen Erfassung."""
|
||||
|
||||
RAW_IMAGE = "raw_image"
|
||||
DIAGRAM = "diagram"
|
||||
CHART = "chart"
|
||||
SCREENSHOT = "screenshot"
|
||||
INFOGRAPHIC = "infographic"
|
||||
PDF_LAYOUT = "pdf_layout"
|
||||
|
||||
|
||||
class VisionConfidence(str, Enum):
|
||||
"""Confidence-Stufe der visuellen Extraktion."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class EvidenceLevel(str, Enum):
|
||||
"""Evidenz-Level einer visuellen Evidenz."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class VisionEntityCategory(str, Enum):
|
||||
"""Kategorie einer erkannten Entity aus einem Bild."""
|
||||
|
||||
DATE = "date"
|
||||
PERSON = "person"
|
||||
ORGANIZATION = "organization"
|
||||
LOCATION = "location"
|
||||
NUMBER = "number"
|
||||
STATISTIC = "statistic"
|
||||
GRAPH_ELEMENT = "graph_element"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionCaptureSchema — einzelne visuell extrahierte Evidenz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureSchema(BaseModel):
|
||||
"""Einzelne visuell extrahierte Evidenz.
|
||||
|
||||
Felder:
|
||||
capture_type: Art der visuellen Erfassung
|
||||
image_data_url: Data-URL oder base64-codiertes Bild
|
||||
extracted_text: Vom Vision-Modell extrahierter Text
|
||||
entities: Erkannte Entities (Personen, Zahlen, etc.)
|
||||
confidence: Confidence der Extraktion
|
||||
evidence_level: Evidenz-Level
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
source_url: URL der Quelle (Provenance)
|
||||
metadata: Zusätzliche Metadaten
|
||||
"""
|
||||
|
||||
capture_type: VisionCaptureType = Field(
|
||||
...,
|
||||
description="Art der visuellen Erfassung (raw_image, diagram, chart, screenshot, infographic, pdf_layout).",
|
||||
)
|
||||
image_data_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Data-URL (data:image/...) oder base64-codiertes Bild.",
|
||||
)
|
||||
extracted_text: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Vom Vision-Modell extrahierter Textinhalt.",
|
||||
)
|
||||
entities: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Erkannte Entities aus dem Bild.",
|
||||
)
|
||||
confidence: float = Field(
|
||||
default=0.5,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence-Wert der Extraktion (0-1).",
|
||||
)
|
||||
confidence_label: VisionConfidence = Field(
|
||||
default=VisionConfidence.MEDIUM,
|
||||
description="Confidence-Stufe als Label.",
|
||||
)
|
||||
evidence_level: EvidenceLevel = Field(
|
||||
default=EvidenceLevel.MEDIUM,
|
||||
description="Evidenz-Level der visuellen Evidenz.",
|
||||
)
|
||||
source_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID der Quelle (source_id) zur Provenance.",
|
||||
)
|
||||
source_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="URL der Quelle (Provenance).",
|
||||
)
|
||||
metadata: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusätzliche Metadaten (z.B. model_used, processing_time).",
|
||||
)
|
||||
|
||||
@field_validator("extracted_text")
|
||||
@classmethod
|
||||
def extracted_text_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("extracted_text darf nicht nur aus Whitespaces bestehen")
|
||||
return v
|
||||
|
||||
@field_validator("image_data_url")
|
||||
@classmethod
|
||||
def image_data_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("image_data_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("source_url")
|
||||
@classmethod
|
||||
def source_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("source_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionReportSchema — Zusammenfassung aller visuellen Evidenzen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionReportSchema(BaseModel):
|
||||
"""Zusammenfassung aller visuellen Evidenzen für einen Research-Run.
|
||||
|
||||
Felder:
|
||||
research_run_id: UUID des Research-Runs
|
||||
total_captures: Anzahl der visuellen Evidenzen
|
||||
captures: Liste aller visuellen Evidenzen
|
||||
entity_summary: Zusammenfassung aller erkannten Entities
|
||||
summary_text: Kurze Zusammenfassung des visuellen Contents
|
||||
generation_timestamp: Zeitstempel der Generierung
|
||||
"""
|
||||
|
||||
research_run_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID des Research-Runs.",
|
||||
)
|
||||
total_captures: int = Field(
|
||||
...,
|
||||
ge=0,
|
||||
description="Anzahl der visuellen Evidenzen in diesem Report.",
|
||||
)
|
||||
captures: list[VisionCaptureSchema] = Field(
|
||||
default_factory=list,
|
||||
description="Liste aller visuellen Evidenzen.",
|
||||
)
|
||||
entity_summary: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="Zusammenfassung aller erkannten Entities.",
|
||||
)
|
||||
summary_text: str = Field(
|
||||
default="",
|
||||
description="Kurze Zusammenfassung des visuellen Contents.",
|
||||
)
|
||||
generation_timestamp: datetime = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc),
|
||||
description="Zeitstempel der Generierung.",
|
||||
)
|
||||
|
||||
@field_validator("research_run_id")
|
||||
@classmethod
|
||||
def research_run_id_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("research_run_id darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("summary_text")
|
||||
@classmethod
|
||||
def summary_not_political(cls, v: str) -> str:
|
||||
import re
|
||||
|
||||
forbidden = re.compile(
|
||||
r"(sollte\s+(Regierung|Bundesregierung)\s+(handeln|unterstützen)|"
|
||||
r"muss\s+(geändert|eingesetzt|gestürzt))",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if forbidden.search(v):
|
||||
raise ValueError("VisionReport darf keine politische Empfehlung enthalten")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionRequestSchema — API-Request für die Vision-Extraktion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionRequestSchema(BaseModel):
|
||||
"""API-Request zum Extrahieren visueller Evidenzen.
|
||||
|
||||
Felder:
|
||||
research_run_id: UUID des Research-Runs
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
source_url: URL der Quelle (Provenance)
|
||||
image_data: Base64-codiertes Bild oder Data-URL
|
||||
capture_type: Art der visuellen Erfassung
|
||||
prompt: Optionaler Prompt an das Vision-Modell
|
||||
"""
|
||||
|
||||
research_run_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID des Research-Runs.",
|
||||
)
|
||||
source_id: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="UUID der Quelle (source_id).",
|
||||
)
|
||||
source_url: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="URL der Quelle (Provenance).",
|
||||
)
|
||||
image_data: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="Base64-codiertes Bild oder Data-URL (data:image/...).",
|
||||
)
|
||||
capture_type: VisionCaptureType = Field(
|
||||
default=VisionCaptureType.RAW_IMAGE,
|
||||
description="Art der visuellen Erfassung.",
|
||||
)
|
||||
prompt: str | None = Field(
|
||||
default=None,
|
||||
description="Optionaler Prompt an das Vision-Modell für die Extraktion.",
|
||||
)
|
||||
|
||||
@field_validator("image_data")
|
||||
@classmethod
|
||||
def image_data_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("image_data darf nicht leer sein")
|
||||
return v
|
||||
|
||||
@field_validator("source_url")
|
||||
@classmethod
|
||||
def source_url_not_empty(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("source_url darf nicht leer sein")
|
||||
return v
|
||||
|
||||
model_config = {"frozen": True}
|
||||
@@ -1,5 +1,10 @@
|
||||
"""Stage 5 — Claim Extraction package."""
|
||||
"""NSCT Stages — Pipeline stages for research runs."""
|
||||
|
||||
from nsct.stages.stage5_extract_claims import Stage5Extractor
|
||||
from nsct.stages.stage10_vision import VisionStage, VisionEvidence
|
||||
|
||||
__all__ = ["Stage5Extractor"]
|
||||
__all__ = [
|
||||
"Stage5Extractor",
|
||||
"VisionStage",
|
||||
"VisionEvidence",
|
||||
]
|
||||
750
src/nsct/stages/stage10_vision.py
Normal file
750
src/nsct/stages/stage10_vision.py
Normal file
@@ -0,0 +1,750 @@
|
||||
"""Stage 10: Vision Integration — visuelle Evidenz-Extraktion via Qwen2.5-VL-3B.
|
||||
|
||||
Pipeline für ein Research-Run:
|
||||
1. Lädt Bild-Captures aus der DB (Screenshot/Chart/Infografik).
|
||||
2. Bereitet base64-kodierte Bilder vor (max-size limits).
|
||||
3. Sendet jedes Bild an Qwen2.5-VL-3B für visuelle Analyse.
|
||||
4. Parallelt die Analyse via Semaphore (bounded concurrency).
|
||||
5. Parsed JSON-Response und extrahiert Entities, OCR-Text, Claims.
|
||||
6. Speichert VisionEvidence in der DB.
|
||||
7. Liefert StageResult mit allen extrahierten visuellen Evidenzen.
|
||||
|
||||
ARCHITEKTUR-REGELN:
|
||||
- Provenance-Pflicht: jede visuelle Analyse braucht source_url + evidence_span
|
||||
- Fehler pro Bild: kein Single-Point-of-Failure
|
||||
- Bounded Concurrency via asyncio.Semaphore
|
||||
- LLM-Output ist DATA, keine Instruktion
|
||||
- JSON-Parsing robust gegen Markdown-Code-Blocks
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024 # 20 MB hard cap for base64 payload
|
||||
DEFAULT_MAX_CONCURRENCY = 4
|
||||
VISION_BASE_URL = os.environ.get("NSCT_VISION_BASE_URL", "http://localhost:8030")
|
||||
VISION_API_KEY = os.environ.get(
|
||||
"HERMES_CUSTOM_192_168_80_199_8030_API_KEY",
|
||||
os.environ.get("NSCT_VISION_API_KEY", ""),
|
||||
)
|
||||
VISION_MODEL = os.environ.get("NSCT_VISION_MODEL", "Qwen2.5-VL-3B")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vision Prompt — SYSTEM_PROMPT for the Vision LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VISION_SYSTEM_PROMPT = (
|
||||
"Sie sind die visuelle Analyse-Engine des NSCT (Neutral Search Crawler Tool). "
|
||||
"Ihre Aufgabe ist es, visuell aus Bildern und Screenshots "
|
||||
"neutrale, evidenzbasierte Informationen zu extrahieren.\n\n"
|
||||
"KRITERIEN FÜR DIE ANALYSE:\n"
|
||||
"1. Texterkennung (OCR): Extrahieren Sie alle sichtbaren Texte.\n"
|
||||
"2. Entity-Erkennung: Identifizieren Sie Daten, Zahlen, Personen, "
|
||||
"Organisationen, Standorte, Statistiken.\n"
|
||||
"3. Diagramm-Interpretation: Falls das Bild Diagramme, Charts oder "
|
||||
"Graphen enthält, interpretieren Sie die Daten neutral — keine "
|
||||
"Fazit-Abstraktion, nur Rohdaten.\n"
|
||||
"4. Qualitätsbewertung: Bewerten Sie, wie gut das Bild als Evidenz "
|
||||
"taugt (HIGH / MEDIUM / LOW).\n\n"
|
||||
"WICHTIG:\n"
|
||||
"- LIEFEREN Sie NUR JSON — kein freier Text, keine Erklärungen.\n"
|
||||
"- Jede Extraktion braucht Provenance: source_url, evidence_span.\n"
|
||||
"- Keine Spekulation — nur das, was Sie visuell erkennen.\n"
|
||||
"- Wenn kein Text oder keine relevanten Daten erkennbar sind, "
|
||||
"geben Sie leere Listen zurück.\n\n"
|
||||
"FORMAT — JSON-Array mit einem Objekt:\n"
|
||||
"{\n"
|
||||
' "entities": [\n'
|
||||
' {"type": "statistic|date|person|organization|location|graph_element",\n'
|
||||
' "value": "...",\n'
|
||||
' "context": "...",\n'
|
||||
' "confidence": 0.9}\n'
|
||||
" ],\n"
|
||||
' "extracted_text": "Vollständiger OCR-Text des Bildes",\n'
|
||||
' "evidence_quality": "high|medium|low",\n'
|
||||
' "claim_candidates": [\n'
|
||||
' {"text": "Behauptung aus dem Bild",\n'
|
||||
' "source_url": "url",\n'
|
||||
' "evidence_span": "relevantes Fragment aus dem Bild"}\n'
|
||||
" ]\n"
|
||||
"}\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: Image preparation (base64, size limits)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _image_to_base64(image_bytes: bytes) -> str:
|
||||
"""Konvertiert Rohbild-Daten in base64, mit size limit.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
Wenn das Bild die maximale Größe überschreitet.
|
||||
"""
|
||||
if len(image_bytes) > MAX_IMAGE_SIZE_BYTES:
|
||||
raise ValueError(
|
||||
f"Image too large: {len(image_bytes)} bytes (max {MAX_IMAGE_SIZE_BYTES})"
|
||||
)
|
||||
return base64.b64encode(image_bytes).decode("ascii")
|
||||
|
||||
|
||||
def _prepare_image_payload(image: dict[str, Any]) -> tuple[str, str] | tuple[None, str]:
|
||||
"""Bereitet ein Bild für die Vision-LLM-Analyse vor.
|
||||
|
||||
Unterstützt:
|
||||
- image_bytes (bytes): Rohbild-Daten
|
||||
- image_b64 (str): bereits base64-kodiertes Bild
|
||||
- image_url (str): URL des Bildes
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[str, str] oder None: (image_ref, error) — image_ref ist base64
|
||||
oder URL, None wenn das Bild übersprungen werden soll.
|
||||
"""
|
||||
# 1. image_b64 — bereits base64-kodiert
|
||||
b64 = image.get("image_b64")
|
||||
if b64 and isinstance(b64, str) and len(b64) > 0:
|
||||
return (b64, "")
|
||||
|
||||
# 2. image_bytes — Rohbild
|
||||
raw = image.get("image_bytes")
|
||||
if raw and isinstance(raw, bytes) and len(raw) > 0:
|
||||
try:
|
||||
return (_image_to_base64(raw), "")
|
||||
except ValueError as exc:
|
||||
return (str(exc), str(exc))
|
||||
|
||||
# 3. image_url — URL des Bildes
|
||||
url = image.get("image_url")
|
||||
if url and isinstance(url, str) and len(url) > 0:
|
||||
return (url, "")
|
||||
|
||||
# 4. Kein Bild — überspringen
|
||||
logger.warning("Image missing: no bytes, b64, or url provided")
|
||||
return None, "No image data"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: JSON-Parsing (robust gegen Markdown-Code-Blocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_vision_json(text: str) -> dict[str, Any]:
|
||||
"""Parsen der Vision-LLM-Antwort als JSON.
|
||||
|
||||
Robust: extrahiert JSON aus Code-Blocks (```json ... ```) und sucht
|
||||
die ersten { ... } Blöcke als Fallback.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
Wenn kein gültiger JSON-Inhalt gefunden wird.
|
||||
"""
|
||||
raw = text.strip()
|
||||
|
||||
# Extrahiere aus Code-Blocks
|
||||
if "```" in raw:
|
||||
lines = raw.split("\n")
|
||||
json_text = ""
|
||||
in_block = False
|
||||
for line in lines:
|
||||
if "```" in line:
|
||||
in_block = not in_block
|
||||
continue
|
||||
if in_block:
|
||||
json_text += line + "\n"
|
||||
raw = json_text.strip()
|
||||
|
||||
# Falls immer noch leer, nach { ... } suchen
|
||||
if not raw.startswith("{"):
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}") + 1
|
||||
if start >= 0 and end > start:
|
||||
raw = raw[start:end]
|
||||
|
||||
if not raw:
|
||||
raise ValueError("Vision response contained no JSON object")
|
||||
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid vision response JSON: {exc}") from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: Vision analysis per image
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _analyze_single_image(
|
||||
image: dict[str, Any],
|
||||
source_url: str,
|
||||
source_title: str | None,
|
||||
capture_type: str,
|
||||
vision_client: Any,
|
||||
semaphore: asyncio.Semaphore,
|
||||
) -> dict[str, Any]:
|
||||
"""Analysiert ein einzelnes Bild via Vision-LLM.
|
||||
|
||||
Gefangen in einem try/except: kein Single-Point-of-Failure.
|
||||
Selbst bei Fehler wird eine Fallback-Eintrag erstellt.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict: Das Vision-Ergebnis (oder Fallback bei Fehler).
|
||||
"""
|
||||
async with semaphore:
|
||||
try:
|
||||
# Prepare image payload
|
||||
img_result = _prepare_image_payload(image)
|
||||
if img_result is None:
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
"No image data provided"
|
||||
)
|
||||
|
||||
image_ref, error = img_result
|
||||
if error:
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
error
|
||||
)
|
||||
|
||||
# Build user prompt mit Metadaten
|
||||
user_prompt = (
|
||||
f"Quelle: {source_url}\n"
|
||||
f"Titel: {source_title or 'N/A'}\n"
|
||||
f"Capture-Typ: {capture_type}\n\n"
|
||||
"Analysieren Sie dieses Bild visuell. "
|
||||
"Extrahieren Sie alle sichtbaren Texte, Zahlen, "
|
||||
"Datumsangaben und Behauptungen. Bewerten Sie die "
|
||||
"Evidenzqualität. Antworten Sie als JSON."
|
||||
)
|
||||
|
||||
# Call vision provider
|
||||
llm_response = await vision_client.analyze(
|
||||
image_url_or_base64=image_ref,
|
||||
prompt=user_prompt,
|
||||
model=VISION_MODEL,
|
||||
)
|
||||
|
||||
# Parse JSON
|
||||
parsed = _parse_vision_json(llm_response)
|
||||
|
||||
# Validate structure
|
||||
if not isinstance(parsed, dict):
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
"Vision response is not a JSON object"
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"source_url": source_url,
|
||||
"source_title": source_title,
|
||||
"capture_type": capture_type,
|
||||
"entities": parsed.get("entities", []),
|
||||
"extracted_text": parsed.get("extracted_text", ""),
|
||||
"evidence_quality": parsed.get("evidence_quality", "low"),
|
||||
"claim_candidates": parsed.get("claim_candidates", []),
|
||||
"raw_response": llm_response,
|
||||
"model_used": VISION_MODEL,
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Vision analysis failed for %s: %s", source_url, exc
|
||||
)
|
||||
return _make_error_result(
|
||||
image, source_url, source_title, capture_type,
|
||||
str(exc)
|
||||
)
|
||||
|
||||
|
||||
def _make_error_result(
|
||||
image: dict[str, Any],
|
||||
source_url: str,
|
||||
source_title: str | None,
|
||||
capture_type: str,
|
||||
error: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Erstellt einen Fallback-Eintrag bei Fehler — kein Single-Point-of-Failure."""
|
||||
return {
|
||||
"success": False,
|
||||
"source_url": source_url,
|
||||
"source_title": source_title,
|
||||
"capture_type": capture_type,
|
||||
"entities": [],
|
||||
"extracted_text": "",
|
||||
"evidence_quality": "low",
|
||||
"claim_candidates": [],
|
||||
"error": error,
|
||||
"model_used": VISION_MODEL,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionEvidence dataclass — DB-entität für visuellen Evidenz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class VisionEvidence:
|
||||
"""Repräsentiert eine visuelle Evidenz aus der Bildanalyse.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
source_url : str
|
||||
Die URL der Quelle, aus der das Bild stammt (Provenance).
|
||||
source_title : str | None
|
||||
Titel der Quelle.
|
||||
capture_type : str
|
||||
Art des Captures (screenshot, chart, infographic, thumbnail).
|
||||
entities : list[dict]
|
||||
Extrahierte Entities (Personen, Statistiken, Daten, etc.).
|
||||
extracted_text : str
|
||||
Vollständiger OCR-Text aus dem Bild.
|
||||
evidence_quality : str
|
||||
"high", "medium", oder "low" — Evidenzqualität.
|
||||
claim_candidates : list[dict]
|
||||
Behauptungen die aus dem Bild extrahiert wurden.
|
||||
raw_response : str | None
|
||||
Roh-LLM-Antwort für Audit-Zwecke.
|
||||
model_used : str
|
||||
Verwendetes Vision-Modell.
|
||||
error : str | None
|
||||
Fehlermeldung bei fehlgeschlagener Analyse.
|
||||
"""
|
||||
source_url: str
|
||||
source_title: str | None
|
||||
capture_type: str
|
||||
entities: list[dict[str, Any]] = field(default_factory=list)
|
||||
extracted_text: str = ""
|
||||
evidence_quality: str = "low"
|
||||
claim_candidates: list[dict[str, Any]] = field(default_factory=list)
|
||||
raw_response: str | None = None
|
||||
model_used: str = VISION_MODEL
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return self.error is None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialisiert das Objekt als Dictionary für die DB."""
|
||||
return {
|
||||
"source_url": self.source_url,
|
||||
"source_title": self.source_title,
|
||||
"capture_type": self.capture_type,
|
||||
"entities": self.entities,
|
||||
"extracted_text": self.extracted_text,
|
||||
"evidence_quality": self.evidence_quality,
|
||||
"claim_candidates": self.claim_candidates,
|
||||
"raw_response": self.raw_response,
|
||||
"model_used": self.model_used,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_analysis_result(cls, result: dict[str, Any]) -> "VisionEvidence":
|
||||
"""Erstellt VisionEvidence aus einem Analyse-Ergebnis-Dict."""
|
||||
return cls(
|
||||
source_url=result.get("source_url", ""),
|
||||
source_title=result.get("source_title"),
|
||||
capture_type=result.get("capture_type", "unknown"),
|
||||
entities=result.get("entities", []),
|
||||
extracted_text=result.get("extracted_text", ""),
|
||||
evidence_quality=result.get("evidence_quality", "low"),
|
||||
claim_candidates=result.get("claim_candidates", []),
|
||||
raw_response=result.get("raw_response"),
|
||||
model_used=result.get("model_used", VISION_MODEL),
|
||||
error=None if result.get("success") else result.get("error"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 10: VisionStage (BaseStage + execute)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionStage:
|
||||
"""Stage 10: Vision Integration — visuelle Evidenz-Extraktion via Qwen2.5-VL-3B.
|
||||
|
||||
Verarbeitet Bild-Captures aus einer Research-Run, analysiert sie
|
||||
visuell via Vision-LLM und speichert die extrahierten Evidenzen.
|
||||
|
||||
Usage::
|
||||
|
||||
stage = VisionStage(
|
||||
research_run_id=uuid,
|
||||
vision_client=vision_provider,
|
||||
config=config,
|
||||
images=[...],
|
||||
)
|
||||
result = await stage.execute()
|
||||
"""
|
||||
|
||||
stage_number = 10
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
research_run_id: UUID,
|
||||
vision_client: Any,
|
||||
config: Any,
|
||||
images: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self.research_run_id = research_run_id
|
||||
self.vision_client = vision_client
|
||||
self.config = config
|
||||
self.images = images or []
|
||||
self.max_concurrency = int(
|
||||
os.environ.get("NSCT_VISION_MAX_CONCURRENCY", DEFAULT_MAX_CONCURRENCY)
|
||||
)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "Stage 10: Vision Integration"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Data loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _fetch_images_from_db(self) -> list[dict[str, Any]]:
|
||||
"""Lädt Captures (Screenshots, Charts, etc.) aus der DB.
|
||||
|
||||
Falls images im Constructor mitgegeben wurden, werden diese
|
||||
verwendet, andernfalls wird ein leerer List zurückgegeben.
|
||||
|
||||
In einer echten Implementierung würde hier die SQLAlchemy Session
|
||||
verwendet werden, um Captures aus der database zu lesen.
|
||||
"""
|
||||
if self.images:
|
||||
return self.images
|
||||
return []
|
||||
|
||||
def _prepare_prompts(
|
||||
self,
|
||||
images: list[dict[str, Any]],
|
||||
) -> list[tuple[dict[str, Any], str, str | None, str]]:
|
||||
"""Bereitet die Eingabeparameter für die parallele Analyse vor.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[tuple[dict, str, str | None, str]]:
|
||||
(image, source_url, source_title, capture_type)
|
||||
"""
|
||||
prompts = []
|
||||
for image in images:
|
||||
source_url = (
|
||||
image.get("source_url", "")
|
||||
or image.get("url", "")
|
||||
or image.get("image_url", "")
|
||||
or "unknown"
|
||||
)
|
||||
source_title = image.get("source_title") or image.get("title")
|
||||
capture_type = (
|
||||
image.get("capture_type", "screenshot")
|
||||
or image.get("type", "screenshot")
|
||||
or "screenshot"
|
||||
)
|
||||
prompts.append((image, source_url, source_title, capture_type))
|
||||
return prompts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Execute — main pipeline
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def execute(self, **kwargs: Any) -> "StageResult":
|
||||
"""Führt die vollständige Stage-10-Pipeline aus.
|
||||
|
||||
Returns
|
||||
-------
|
||||
StageResult mit success=True und den Vision-Evidenzen in .data.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
try:
|
||||
# Load images
|
||||
images = kwargs.get("images")
|
||||
if images is None:
|
||||
images = await self._fetch_images_from_db()
|
||||
|
||||
if not images:
|
||||
msg = "Keine Bilder vorhanden - keine Vision-Analyse moeglich."
|
||||
logger.warning("Stage 10: %s", msg)
|
||||
errors.append(msg)
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Stage 10: Image loading failed: %s", exc)
|
||||
errors.append(f"Image loading failed: {exc}")
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
try:
|
||||
# Prepare analysis prompts
|
||||
prompts = self._prepare_prompts(images)
|
||||
|
||||
if not prompts:
|
||||
msg = "Keine gültigen Bild-Eingaben gefunden."
|
||||
logger.warning("Stage 10: %s", msg)
|
||||
errors.append(msg)
|
||||
return StageResult(
|
||||
success=False,
|
||||
data={},
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
# Create semaphore for bounded concurrency
|
||||
semaphore = asyncio.Semaphore(self.max_concurrency)
|
||||
|
||||
logger.info(
|
||||
"Stage 10: Analyzing %d images for research run %s "
|
||||
"(max concurrency: %d)",
|
||||
len(prompts),
|
||||
self.research_run_id,
|
||||
self.max_concurrency,
|
||||
)
|
||||
|
||||
# Run parallel vision analysis — error per image, no SPOF
|
||||
tasks = [
|
||||
_analyze_single_image(
|
||||
image,
|
||||
source_url,
|
||||
source_title,
|
||||
capture_type,
|
||||
self.vision_client,
|
||||
semaphore,
|
||||
)
|
||||
for image, source_url, source_title, capture_type in prompts
|
||||
]
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Process results
|
||||
successful: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, Any]] = []
|
||||
|
||||
for idx, result in enumerate(results):
|
||||
if isinstance(result, Exception):
|
||||
err_msg = f"Analysis task {idx} raised: {result}"
|
||||
logger.error("Stage 10: %s", err_msg)
|
||||
failures.append({
|
||||
"success": False,
|
||||
"source_url": (
|
||||
prompts[idx][1] if idx < len(prompts) else "unknown"
|
||||
),
|
||||
"error": err_msg,
|
||||
})
|
||||
errors.append(err_msg)
|
||||
elif isinstance(result, dict):
|
||||
if result.get("success"):
|
||||
successful.append(result)
|
||||
else:
|
||||
failures.append(result)
|
||||
err_msg = f"Analysis failed for {result.get('source_url', 'unknown')}: {result.get('error', 'unknown')}"
|
||||
errors.append(err_msg)
|
||||
logger.warning("Stage 10: %s", err_msg)
|
||||
|
||||
else:
|
||||
err_msg = f"Unexpected result type for image {idx}: {type(result)}"
|
||||
logger.error("Stage 10: %s", err_msg)
|
||||
failures.append({"success": False, "error": err_msg})
|
||||
errors.append(err_msg)
|
||||
|
||||
# Convert to VisionEvidence objects
|
||||
vision_evidences = [
|
||||
VisionEvidence.from_analysis_result(r)
|
||||
for r in successful
|
||||
]
|
||||
|
||||
# Build DB-ready records
|
||||
db_records = [ev.to_dict() for ev in vision_evidences]
|
||||
|
||||
# Collect all entities and claim candidates across all images
|
||||
all_entities: list[dict[str, Any]] = []
|
||||
all_claims: list[dict[str, Any]] = []
|
||||
for record in db_records:
|
||||
all_entities.extend(record.get("entities", []))
|
||||
all_claims.extend(record.get("claim_candidates", []))
|
||||
|
||||
logger.info(
|
||||
"Stage 10: Vision analysis complete — "
|
||||
"%d successful, %d failures, "
|
||||
"%d entities, %d claims extracted",
|
||||
len(successful),
|
||||
len(failures),
|
||||
len(all_entities),
|
||||
len(all_claims),
|
||||
)
|
||||
|
||||
# Store in DB (placeholder — replace with actual DB insert)
|
||||
# await self._store_vision_evidence(db_records)
|
||||
|
||||
# Build result
|
||||
analysis_data = {
|
||||
"vision_evidences": db_records,
|
||||
"total_images": len(images),
|
||||
"successful": len(successful),
|
||||
"failed": len(failures),
|
||||
"entities_count": len(all_entities),
|
||||
"claims_count": len(all_claims),
|
||||
"model_used": VISION_MODEL,
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"generation_timestamp": datetime_now_utc(),
|
||||
}
|
||||
|
||||
return StageResult(
|
||||
success=True,
|
||||
data=analysis_data,
|
||||
stage=self,
|
||||
)
|
||||
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
logger.error("Stage 10: Vision analysis failed: %s", exc)
|
||||
errors.append(f"Vision analysis failed: {exc}")
|
||||
return self._fallback_result(errors)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Stage 10: Unexpected error: %s", exc)
|
||||
errors.append(f"Unexpected error: {exc}")
|
||||
return self._fallback_result(errors)
|
||||
|
||||
def _fallback_result(self, errors: list[str]) -> "StageResult":
|
||||
"""Fallback: minimaler Bericht wenn LLM nicht verfügbar ist."""
|
||||
logger.warning(
|
||||
"Stage 10: Using fallback — vision analysis unavailable"
|
||||
)
|
||||
|
||||
fallback_data = {
|
||||
"vision_evidences": [],
|
||||
"total_images": len(self.images),
|
||||
"successful": 0,
|
||||
"failed": 0,
|
||||
"entities_count": 0,
|
||||
"claims_count": 0,
|
||||
"model_used": VISION_MODEL,
|
||||
"research_run_id": str(self.research_run_id),
|
||||
"generation_timestamp": datetime_now_utc(),
|
||||
"methodology": (
|
||||
"NSCT Stage 10: Vision Integration (FALLBACK). "
|
||||
"Vision-LLM-Analyse war nicht verfügbar. "
|
||||
"Keine visuellen Evidenzen extrahiert."
|
||||
),
|
||||
}
|
||||
|
||||
return StageResult(
|
||||
success=False,
|
||||
data=fallback_data,
|
||||
stage=self,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StageResult — referenced from stage9_synthesis.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# We import from stage9_synthesis to avoid circular imports at module level.
|
||||
# These are defined here for convenience when running this file standalone.
|
||||
|
||||
|
||||
class StageResult:
|
||||
"""Result returned by a stage's execute() method."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
success: bool,
|
||||
data: dict[str, Any] | None = None,
|
||||
stage: Any = None,
|
||||
errors: list[str] | None = None,
|
||||
) -> None:
|
||||
self.success = success
|
||||
self.data = data or {}
|
||||
self.stage = stage
|
||||
self.errors = errors or []
|
||||
|
||||
@property
|
||||
def stage_name(self) -> str:
|
||||
if self.stage:
|
||||
return self.stage.name
|
||||
return "unknown"
|
||||
|
||||
@property
|
||||
def research_run_id(self) -> UUID | None:
|
||||
if self.stage:
|
||||
return self.stage.research_run_id
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Utility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def datetime_now_utc() -> str:
|
||||
"""Return current UTC time as ISO-8601 string."""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Legacy compatibility wrapper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Stage10Vision:
|
||||
"""Legacy wrapper: Stage 10 with a synchronous `.run()` for backward compat.
|
||||
|
||||
Instantiates VisionStage internally and delegates to ``execute()``.
|
||||
"""
|
||||
|
||||
def __init__(self, vision_client, config, research_run_id, images=None):
|
||||
self._stage = VisionStage(
|
||||
research_run_id=research_run_id,
|
||||
vision_client=vision_client,
|
||||
config=config,
|
||||
images=images or [],
|
||||
)
|
||||
|
||||
@property
|
||||
def research_run_id(self) -> UUID:
|
||||
return self._stage.research_run_id
|
||||
|
||||
async def run(self) -> dict[str, Any]:
|
||||
"""Backward-compatible async .run() method."""
|
||||
result = await self._stage.execute()
|
||||
if not result.success:
|
||||
raise RuntimeError(
|
||||
"Stage 10 vision analysis failed: " + "; ".join(result.errors)
|
||||
)
|
||||
return result.data
|
||||
@@ -472,3 +472,133 @@ class EvidenceScoreRelationModel(Base):
|
||||
Index("ix_evidence_score_relations_score_id", "score_id"),
|
||||
Index("ix_evidence_score_relations_related_claim_id", "related_claim_id"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 10 — Vision Evidence Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VisionCaptureType(str, enum.Enum):
|
||||
"""Klassifikation der visuellen Erfassung (Stage 10)."""
|
||||
|
||||
RAW_IMAGE = "raw_image"
|
||||
DIAGRAM = "diagram"
|
||||
CHART = "chart"
|
||||
SCREENSHOT = "screenshot"
|
||||
INFOGRAPHIC = "infographic"
|
||||
PDF_LAYOUT = "pdf_layout"
|
||||
|
||||
|
||||
class VisionEvidenceLevel(str, enum.Enum):
|
||||
"""Evidenz-Level einer visuellen Evidenz (Stage 10)."""
|
||||
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
UNCERTAIN = "uncertain"
|
||||
|
||||
|
||||
class VisionEntityType(str, enum.Enum):
|
||||
"""Kategorie einer erkannten Entity aus einem Bild (Stage 10)."""
|
||||
|
||||
DATE = "date"
|
||||
PERSON = "person"
|
||||
ORGANIZATION = "organization"
|
||||
LOCATION = "location"
|
||||
NUMBER = "number"
|
||||
STATISTIC = "statistic"
|
||||
GRAPH_ELEMENT = "graph_element"
|
||||
|
||||
|
||||
class VisionEvidenceModel(Base):
|
||||
"""Einzelne visuell extrahierte Evidenz (Stage 10).
|
||||
|
||||
Felder:
|
||||
uuid: Primärschlüssel (UUID)
|
||||
research_run_id: Research-Run-Zuordnung
|
||||
source_id: Quelle, von der das Bild stammt
|
||||
capture_type: Art der visuellen Erfassung
|
||||
extracted_text: Vom Vision-Modell extrahierter Text
|
||||
image_data_url: Data-URL oder base64-codiertes Bild (optional)
|
||||
entities: JSON mit erkannten Entities
|
||||
confidence: Confidence 0.0–1.0
|
||||
confidence_label: Confidence als Label (HIGH/MEDIUM/LOW/UNCERTAIN)
|
||||
evidence_level: Evidenz-Level (HIGH/MEDIUM/LOW/UNCERTAIN)
|
||||
created_at / updated_at: Zeitstempel
|
||||
"""
|
||||
|
||||
__tablename__ = "vision_evidence"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
research_run_id = Column(String(36), nullable=False)
|
||||
source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
|
||||
|
||||
capture_type = Column(Enum(VisionCaptureType), nullable=False)
|
||||
|
||||
extracted_text = Column(Text, nullable=False)
|
||||
image_data_url = Column(Text, nullable=True)
|
||||
entities = Column(JSON, nullable=False, default=dict)
|
||||
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
confidence_label = Column(String(16), nullable=False, default="medium")
|
||||
evidence_level = Column(Enum(VisionEvidenceLevel), nullable=False, default=VisionEvidenceLevel.MEDIUM)
|
||||
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
source = relationship("SourceModel")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_vision_evidence_research_run_id", "research_run_id"),
|
||||
Index("ix_vision_evidence_capture_type", "capture_type"),
|
||||
Index("ix_vision_evidence_source_id", "source_id"),
|
||||
)
|
||||
|
||||
|
||||
class VisionEntityModel(Base):
|
||||
"""Erkannte Entity aus einem visuellen Bild (Stage 10).
|
||||
|
||||
Felder:
|
||||
uuid: Primärschlüssel (UUID)
|
||||
evidence_id: FK zur VisionEvidence
|
||||
entity_type: Kategorie der Entity
|
||||
entity_value: Der erkannte Wert
|
||||
context: Kontextbeschreibung
|
||||
confidence: Confidence 0.0–1.0
|
||||
created_at: Zeitstempel
|
||||
"""
|
||||
|
||||
__tablename__ = "vision_entities"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
|
||||
evidence_id = Column(
|
||||
String(36),
|
||||
ForeignKey("vision_evidence.id"),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
entity_type = Column(Enum(VisionEntityType), nullable=False)
|
||||
entity_value = Column(Text, nullable=False)
|
||||
context = Column(Text, nullable=True)
|
||||
confidence = Column(Float, nullable=False, default=0.5)
|
||||
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
# Relationships
|
||||
evidence = relationship("VisionEvidenceModel", back_populates="entities")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_vision_entities_evidence_id", "evidence_id"),
|
||||
Index("ix_vision_entities_entity_type", "entity_type"),
|
||||
)
|
||||
|
||||
|
||||
# Back-populate the backref for VisionEntityModel
|
||||
VisionEvidenceModel.entities = relationship(
|
||||
"VisionEntityModel",
|
||||
back_populates="evidence",
|
||||
cascade="all, delete-orphan",
|
||||
foreign_keys="VisionEntityModel.evidence_id",
|
||||
)
|
||||
336
tests/models/test_vision.py
Normal file
336
tests/models/test_vision.py
Normal file
@@ -0,0 +1,336 @@
|
||||
"""Tests für Pydantic v2 Schemas und SQLAlchemy Models der Vision Evidence Extraction (Stage 10)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from nsct.models.vision import (
|
||||
EvidenceLevel,
|
||||
VisionCaptureSchema,
|
||||
VisionCaptureType,
|
||||
VisionConfidence,
|
||||
VisionEntityCategory,
|
||||
VisionRequestSchema,
|
||||
VisionReportSchema,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVisionCaptureTypeEnum:
|
||||
"""Prüft die Enums VisionCaptureType und VisionConfidence."""
|
||||
|
||||
def test_vision_capture_type_values(self):
|
||||
assert VisionCaptureType.RAW_IMAGE.value == "raw_image"
|
||||
assert VisionCaptureType.DIAGRAM.value == "diagram"
|
||||
assert VisionCaptureType.CHART.value == "chart"
|
||||
assert VisionCaptureType.SCREENSHOT.value == "screenshot"
|
||||
assert VisionCaptureType.INFOGRAPHIC.value == "infographic"
|
||||
assert VisionCaptureType.PDF_LAYOUT.value == "pdf_layout"
|
||||
|
||||
def test_vision_confidence_values(self):
|
||||
assert VisionConfidence.HIGH.value == "high"
|
||||
assert VisionConfidence.MEDIUM.value == "medium"
|
||||
assert VisionConfidence.LOW.value == "low"
|
||||
assert VisionConfidence.UNCERTAIN.value == "uncertain"
|
||||
|
||||
def test_evidence_level_values(self):
|
||||
assert EvidenceLevel.HIGH.value == "high"
|
||||
assert EvidenceLevel.MEDIUM.value == "medium"
|
||||
assert EvidenceLevel.LOW.value == "low"
|
||||
assert EvidenceLevel.UNCERTAIN.value == "uncertain"
|
||||
|
||||
def test_vision_entity_category_values(self):
|
||||
assert VisionEntityCategory.DATE.value == "date"
|
||||
assert VisionEntityCategory.PERSON.value == "person"
|
||||
assert VisionEntityCategory.ORGANIZATION.value == "organization"
|
||||
assert VisionEntityCategory.LOCATION.value == "location"
|
||||
assert VisionEntityCategory.NUMBER.value == "number"
|
||||
assert VisionEntityCategory.STATISTIC.value == "statistic"
|
||||
assert VisionEntityCategory.GRAPH_ELEMENT.value == "graph_element"
|
||||
|
||||
def test_invalid_capture_type(self):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionCaptureSchema(
|
||||
capture_type="invalid_type", # type: ignore
|
||||
image_data_url="data:image/png;base64,abc",
|
||||
extracted_text="test",
|
||||
source_id="test-source",
|
||||
source_url="https://example.com",
|
||||
)
|
||||
|
||||
def test_invalid_confidence_label(self):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionCaptureSchema(
|
||||
capture_type=VisionCaptureType.RAW_IMAGE,
|
||||
image_data_url="data:image/png;base64,abc",
|
||||
extracted_text="test",
|
||||
source_id="test-source",
|
||||
source_url="https://example.com",
|
||||
confidence_label="invalid", # type: ignore
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionCaptureSchema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVisionCaptureSchema:
|
||||
"""Tests für VisionCaptureSchema — Pflichtfelder, Defaults, Frozen."""
|
||||
|
||||
@pytest.fixture
|
||||
def valid_kwargs(self):
|
||||
return {
|
||||
"capture_type": VisionCaptureType.DIAGRAM,
|
||||
"image_data_url": "data:image/png;base64,abc123",
|
||||
"extracted_text": "This is a chart showing revenue growth.",
|
||||
"source_id": "source-uuid-001",
|
||||
"source_url": "https://example.com/chart.png",
|
||||
}
|
||||
|
||||
def test_create_valid_schema(self, valid_kwargs):
|
||||
schema = VisionCaptureSchema(**valid_kwargs)
|
||||
assert schema.capture_type == VisionCaptureType.DIAGRAM
|
||||
assert schema.extracted_text == "This is a chart showing revenue growth."
|
||||
assert schema.source_id == "source-uuid-001"
|
||||
assert schema.entities == []
|
||||
assert schema.confidence == 0.5
|
||||
assert schema.confidence_label == VisionConfidence.MEDIUM
|
||||
assert schema.evidence_level == EvidenceLevel.MEDIUM
|
||||
assert schema.metadata == {}
|
||||
|
||||
def test_defaults(self, valid_kwargs):
|
||||
schema = VisionCaptureSchema(**valid_kwargs)
|
||||
assert schema.entities == []
|
||||
assert schema.confidence == 0.5
|
||||
assert schema.confidence_label == VisionConfidence.MEDIUM
|
||||
assert schema.evidence_level == EvidenceLevel.MEDIUM
|
||||
assert schema.metadata == {}
|
||||
|
||||
def test_frozen(self, valid_kwargs):
|
||||
schema = VisionCaptureSchema(**valid_kwargs)
|
||||
with pytest.raises(Exception):
|
||||
schema.capture_type = VisionCaptureType.CHART
|
||||
|
||||
def test_missing_required_field(self, valid_kwargs):
|
||||
kwargs = {**valid_kwargs}
|
||||
del kwargs["extracted_text"]
|
||||
with pytest.raises(ValidationError):
|
||||
VisionCaptureSchema(**kwargs)
|
||||
|
||||
def test_empty_extracted_text(self, valid_kwargs):
|
||||
kwargs = {**valid_kwargs, "extracted_text": " "}
|
||||
with pytest.raises(ValidationError, match="extracted_text darf nicht nur aus Whitespaces bestehen"):
|
||||
VisionCaptureSchema(**kwargs)
|
||||
|
||||
def test_empty_source_url(self, valid_kwargs):
|
||||
kwargs = {**valid_kwargs, "source_url": " "}
|
||||
with pytest.raises(ValidationError, match="source_url darf nicht leer sein"):
|
||||
VisionCaptureSchema(**kwargs)
|
||||
|
||||
def test_empty_image_data_url(self, valid_kwargs):
|
||||
kwargs = {**valid_kwargs, "image_data_url": " "}
|
||||
with pytest.raises(ValidationError, match="image_data_url darf nicht leer sein"):
|
||||
VisionCaptureSchema(**kwargs)
|
||||
|
||||
def test_confidence_bounds(self, valid_kwargs):
|
||||
schema_low = VisionCaptureSchema(**valid_kwargs, confidence=0.0)
|
||||
assert schema_low.confidence == 0.0
|
||||
|
||||
schema_high = VisionCaptureSchema(**valid_kwargs, confidence=1.0)
|
||||
assert schema_high.confidence == 1.0
|
||||
|
||||
def test_confidence_out_of_bounds_low(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionCaptureSchema(**valid_kwargs, confidence=-0.1)
|
||||
|
||||
def test_confidence_out_of_bounds_high(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionCaptureSchema(**valid_kwargs, confidence=1.1)
|
||||
|
||||
def test_all_capture_types(self):
|
||||
for ct in VisionCaptureType:
|
||||
schema = VisionCaptureSchema(
|
||||
capture_type=ct,
|
||||
image_data_url="data:image/png;base64,abc",
|
||||
extracted_text="test content",
|
||||
source_id="src-1",
|
||||
source_url="https://example.com",
|
||||
)
|
||||
assert schema.capture_type == ct
|
||||
|
||||
def test_evidence_level_values(self):
|
||||
for level in EvidenceLevel:
|
||||
schema = VisionCaptureSchema(
|
||||
capture_type=VisionCaptureType.RAW_IMAGE,
|
||||
image_data_url="data:image/png;base64,abc",
|
||||
extracted_text="test",
|
||||
source_id="src-1",
|
||||
source_url="https://example.com",
|
||||
evidence_level=level,
|
||||
)
|
||||
assert schema.evidence_level == level
|
||||
|
||||
def test_confidence_label_values(self):
|
||||
for label in VisionConfidence:
|
||||
schema = VisionCaptureSchema(
|
||||
capture_type=VisionCaptureType.RAW_IMAGE,
|
||||
image_data_url="data:image/png;base64,abc",
|
||||
extracted_text="test",
|
||||
source_id="src-1",
|
||||
source_url="https://example.com",
|
||||
confidence_label=label,
|
||||
)
|
||||
assert schema.confidence_label == label
|
||||
|
||||
def test_entities_list(self, valid_kwargs):
|
||||
schema = VisionCaptureSchema(
|
||||
**valid_kwargs,
|
||||
entities=[
|
||||
{"type": "NUMBER", "value": "42", "confidence": 0.9},
|
||||
{"type": "DATE", "value": "2024-01-15", "confidence": 0.8},
|
||||
],
|
||||
)
|
||||
assert len(schema.entities) == 2
|
||||
assert schema.entities[0]["type"] == "NUMBER"
|
||||
assert schema.entities[1]["value"] == "2024-01-15"
|
||||
|
||||
def test_metadata_dict(self, valid_kwargs):
|
||||
schema = VisionCaptureSchema(
|
||||
**valid_kwargs,
|
||||
metadata={"model": "qwen2.5-vl-3b", "processing_time": 2.3},
|
||||
)
|
||||
assert schema.metadata["model"] == "qwen2.5-vl-3b"
|
||||
assert schema.metadata["processing_time"] == 2.3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionReportSchema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVisionReportSchema:
|
||||
"""Tests für VisionReportSchema — Zusammenfassung aller visuellen Evidenzen."""
|
||||
|
||||
@pytest.fixture
|
||||
def valid_kwargs(self):
|
||||
return {
|
||||
"research_run_id": "run-uuid-001",
|
||||
"total_captures": 3,
|
||||
}
|
||||
|
||||
def test_create_valid_report(self, valid_kwargs):
|
||||
report = VisionReportSchema(**valid_kwargs)
|
||||
assert report.research_run_id == "run-uuid-001"
|
||||
assert report.total_captures == 3
|
||||
assert report.captures == []
|
||||
assert report.entity_summary == {}
|
||||
assert report.summary_text == ""
|
||||
|
||||
def test_frozen(self, valid_kwargs):
|
||||
report = VisionReportSchema(**valid_kwargs)
|
||||
with pytest.raises(Exception):
|
||||
report.research_run_id = "new-id"
|
||||
|
||||
def test_empty_research_run_id(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError, match="research_run_id darf nicht leer sein"):
|
||||
VisionReportSchema(
|
||||
**valid_kwargs,
|
||||
research_run_id=" ",
|
||||
)
|
||||
|
||||
def test_negative_total_captures(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionReportSchema(**valid_kwargs, total_captures=-1)
|
||||
|
||||
def test_with_captures(self, valid_kwargs):
|
||||
capture = VisionCaptureSchema(
|
||||
capture_type=VisionCaptureType.CHART,
|
||||
image_data_url="data:image/png;base64,xyz",
|
||||
extracted_text="Chart data",
|
||||
source_id="src-1",
|
||||
source_url="https://example.com",
|
||||
)
|
||||
report = VisionReportSchema(
|
||||
**valid_kwargs,
|
||||
total_captures=1,
|
||||
captures=[capture],
|
||||
)
|
||||
assert len(report.captures) == 1
|
||||
assert report.captures[0].capture_type == VisionCaptureType.CHART
|
||||
|
||||
def test_political_summary_rejected(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError, match="politische Empfehlung"):
|
||||
VisionReportSchema(
|
||||
**valid_kwargs,
|
||||
summary_text="Die Regierung sollte handeln.",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VisionRequestSchema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVisionRequestSchema:
|
||||
"""Tests für VisionRequestSchema — API-Request."""
|
||||
|
||||
@pytest.fixture
|
||||
def valid_kwargs(self):
|
||||
return {
|
||||
"research_run_id": "run-uuid-001",
|
||||
"source_id": "source-uuid-001",
|
||||
"source_url": "https://example.com/image.png",
|
||||
"image_data": "data:image/png;base64,iVBORw0KGgoAAA==",
|
||||
}
|
||||
|
||||
def test_create_valid_request(self, valid_kwargs):
|
||||
request = VisionRequestSchema(**valid_kwargs)
|
||||
assert request.research_run_id == "run-uuid-001"
|
||||
assert request.source_id == "source-uuid-001"
|
||||
assert request.source_url == "https://example.com/image.png"
|
||||
assert request.capture_type == VisionCaptureType.RAW_IMAGE
|
||||
assert request.prompt is None
|
||||
|
||||
def test_frozen(self, valid_kwargs):
|
||||
request = VisionRequestSchema(**valid_kwargs)
|
||||
with pytest.raises(Exception):
|
||||
request.research_run_id = "new-id"
|
||||
|
||||
def test_defaults(self, valid_kwargs):
|
||||
request = VisionRequestSchema(**valid_kwargs)
|
||||
assert request.capture_type == VisionCaptureType.RAW_IMAGE
|
||||
assert request.prompt is None
|
||||
|
||||
def test_with_prompt(self, valid_kwargs):
|
||||
request = VisionRequestSchema(
|
||||
**valid_kwargs,
|
||||
prompt="Extrahiere alle Zahlen und Daten aus dem Diagramm.",
|
||||
)
|
||||
assert request.prompt == "Extrahiere alle Zahlen und Daten aus dem Diagramm."
|
||||
|
||||
def test_empty_image_data(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError, match="image_data darf nicht leer sein"):
|
||||
VisionRequestSchema(**valid_kwargs, image_data=" ")
|
||||
|
||||
def test_empty_source_url(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError, match="source_url darf nicht leer sein"):
|
||||
VisionRequestSchema(**valid_kwargs, source_url=" ")
|
||||
|
||||
def test_empty_research_run_id(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionRequestSchema(**valid_kwargs, research_run_id=" ")
|
||||
|
||||
def test_empty_source_id(self, valid_kwargs):
|
||||
with pytest.raises(ValidationError):
|
||||
VisionRequestSchema(**valid_kwargs, source_id=" ")
|
||||
|
||||
def test_capture_type_override(self, valid_kwargs):
|
||||
for ct in VisionCaptureType:
|
||||
request = VisionRequestSchema(**valid_kwargs, capture_type=ct)
|
||||
assert request.capture_type == ct
|
||||
864
tests/stages/test_stage10_vision.py
Normal file
864
tests/stages/test_stage10_vision.py
Normal file
@@ -0,0 +1,864 @@
|
||||
"""Tests für Stage 10: Vision Integration — API-Endpoint und Core-Funktionen.
|
||||
|
||||
Abdeckungen:
|
||||
- Pydantic-Validierung: Pflichtfelder, Defaults, frozen, range
|
||||
- Parsing: JSON-Array, Code-Blocks, Invalid JSON, Empty, Nested
|
||||
- Prompt: image_data/capture_type enthalten, Truncation, Custom
|
||||
- API: POST /vision/analyze, GET /vision/evidence/{id}, 400, 404
|
||||
- Integration: Mock Vision-LLM, Multiple Images, Edge Cases
|
||||
- Async mit asyncio_run() helper
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from nsct.api.vision import (
|
||||
AnalyzeImageRequest,
|
||||
AnalyzeImageResponse,
|
||||
EvidenceItem,
|
||||
EvidenceResponse,
|
||||
_build_prompt,
|
||||
_get_evidence,
|
||||
_image_source_label,
|
||||
_parse_vision_response,
|
||||
_store_evidence,
|
||||
_DEFAULT_VISION_PROMPT,
|
||||
router,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures & Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mock_vision_provider(response: str) -> MagicMock:
|
||||
"""Erzeugt einen mock Vision-Provider mit einer festen Antwort."""
|
||||
provider = MagicMock()
|
||||
provider.analyze = AsyncMock(return_value=response)
|
||||
provider.model = "qwen2.5-vl-3b"
|
||||
return provider
|
||||
|
||||
|
||||
def asyncio_run(coro):
|
||||
"""Hilfsfunktion: Koroutine synchron ausführen."""
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
_BASE64_DATA = "iVBORw0KGgoAAAANSUhEUg=="
|
||||
_IMAGE_URL = "https://example.com/image.png"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 1–8: Pydantic-Validierung — AnalyzeImageRequest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPydanticValidation:
|
||||
"""Tests für Pydantic-Validierung von AnalyzeImageRequest."""
|
||||
|
||||
def test_image_data_required(self) -> None:
|
||||
"""image_data ist required und nicht leer."""
|
||||
with pytest.raises(Exception):
|
||||
AnalyzeImageRequest(image_data="")
|
||||
|
||||
def test_image_data_min_length(self) -> None:
|
||||
"""image_data muss min_length=1 haben."""
|
||||
req = AnalyzeImageRequest(image_data="a")
|
||||
assert req.image_data == "a"
|
||||
|
||||
def test_capture_type_default(self) -> None:
|
||||
"""capture_type hat Default 'screenshot'."""
|
||||
req = AnalyzeImageRequest(image_data=_BASE64_DATA)
|
||||
assert req.capture_type == "screenshot"
|
||||
|
||||
def test_capture_type_custom(self) -> None:
|
||||
"""capture_type kann überschrieben werden."""
|
||||
req = AnalyzeImageRequest(
|
||||
image_data=_BASE64_DATA, capture_type="infographic"
|
||||
)
|
||||
assert req.capture_type == "infographic"
|
||||
|
||||
def test_prompt_default_none(self) -> None:
|
||||
"""prompt ist optional und default None."""
|
||||
req = AnalyzeImageRequest(image_data=_BASE64_DATA)
|
||||
assert req.prompt is None
|
||||
|
||||
def test_image_caption_default_none(self) -> None:
|
||||
"""image_caption ist optional und default None."""
|
||||
req = AnalyzeImageRequest(image_data=_BASE64_DATA)
|
||||
assert req.image_caption is None
|
||||
|
||||
def test_evidence_type_default(self) -> None:
|
||||
"""evidence_type hat Default 'visual'."""
|
||||
req = AnalyzeImageRequest(image_data=_BASE64_DATA)
|
||||
assert req.evidence_type == "visual"
|
||||
|
||||
def test_evidence_type_custom(self) -> None:
|
||||
"""evidence_type kann überschrieben werden."""
|
||||
req = AnalyzeImageRequest(
|
||||
image_data=_BASE64_DATA, evidence_type="document"
|
||||
)
|
||||
assert req.evidence_type == "document"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 9–16: Pydantic-Validierung — EvidenceItem
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEvidenceItemValidation:
|
||||
"""Tests für Pydantic-Validierung von EvidenceItem."""
|
||||
|
||||
def test_all_fields_present(self) -> None:
|
||||
"""EvidenceItem mit allen Pflichtfeldern."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
)
|
||||
assert ev.id is not None
|
||||
assert ev.evidence_type == "visual"
|
||||
assert ev.key_findings == []
|
||||
assert ev.data_points == []
|
||||
assert ev.confidence == 0.8
|
||||
assert ev.sources == []
|
||||
|
||||
def test_confidence_range(self) -> None:
|
||||
"""confidence muss zwischen 0.0 und 1.0 liegen."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
confidence=0.0,
|
||||
)
|
||||
assert ev.confidence == 0.0
|
||||
|
||||
ev2 = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
confidence=1.0,
|
||||
)
|
||||
assert ev2.confidence == 1.0
|
||||
|
||||
def test_confidence_clamped_low(self) -> None:
|
||||
"""confidence < 0.0 wird abgelehnt (Pydantic validation error)."""
|
||||
import uuid
|
||||
with pytest.raises(Exception):
|
||||
EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
confidence=-0.5,
|
||||
)
|
||||
|
||||
def test_confidence_clamped_high(self) -> None:
|
||||
"""confidence > 1.0 wird abgelehnt (Pydantic validation error)."""
|
||||
import uuid
|
||||
with pytest.raises(Exception):
|
||||
EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
confidence=1.5,
|
||||
)
|
||||
|
||||
def test_empty_key_findings(self) -> None:
|
||||
"""key_findings kann leer sein."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
key_findings=[],
|
||||
)
|
||||
assert ev.key_findings == []
|
||||
|
||||
def test_empty_data_points(self) -> None:
|
||||
"""data_points kann leer sein."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
data_points=[],
|
||||
)
|
||||
assert ev.data_points == []
|
||||
|
||||
def test_metadata_default(self) -> None:
|
||||
"""metadata default ist leeres Dict."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
)
|
||||
assert ev.metadata == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 17–24: JSON-Response-Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseVisionResponse:
|
||||
"""Tests für _parse_vision_response — Robustheit gegen verschiedene Formate."""
|
||||
|
||||
def test_parse_json_dict(self) -> None:
|
||||
"""JSON-Objekt mit key_findings wird extrahiert."""
|
||||
data = {
|
||||
"key_findings": ["Trend A", "Trend B"],
|
||||
"data_points": [{"value": 42}],
|
||||
"confidence": 0.95,
|
||||
"image_source": "https://x.com",
|
||||
"description": "Zusammenfassung",
|
||||
}
|
||||
result = _parse_vision_response(json.dumps(data))
|
||||
assert result["key_findings"] == ["Trend A", "Trend B"]
|
||||
assert result["confidence"] == 0.95
|
||||
assert result["image_source"] == "https://x.com"
|
||||
assert result["description"] == "Zusammenfassung"
|
||||
|
||||
def test_parse_json_array(self) -> None:
|
||||
"""JSON-Array wird als Liste von Findings interpretiert."""
|
||||
data = ["Fund 1", "Fund 2", "Fund 3"]
|
||||
result = _parse_vision_response(json.dumps(data))
|
||||
assert "Fund 1" in result["key_findings"]
|
||||
assert "Fund 2" in result["key_findings"]
|
||||
|
||||
def test_parse_code_block_json(self) -> None:
|
||||
"""JSON in Markdown-Code-Block wird extrahiert."""
|
||||
response = '```json\n{"key_findings": ["Code Block"], "confidence": 0.7}\n```'
|
||||
result = _parse_vision_response(response)
|
||||
assert "Code Block" in result["key_findings"]
|
||||
assert result["confidence"] == 0.7
|
||||
|
||||
def test_parse_code_block_no_lang(self) -> None:
|
||||
"""Code-Block ohne language-Tag wird extrahiert."""
|
||||
response = '```\n{"key_findings": ["No Lang"]}\n```'
|
||||
result = _parse_vision_response(response)
|
||||
assert "No Lang" in result["key_findings"]
|
||||
|
||||
def test_parse_invalid_json_fallback(self) -> None:
|
||||
"""Ungültiges JSON → Fallback: Text als Beschreibung."""
|
||||
result = _parse_vision_response("Das ist kein JSON!")
|
||||
assert result["description"] == "Das ist kein JSON!"
|
||||
assert result["key_findings"] == ["Das ist kein JSON!"]
|
||||
assert result["confidence"] == 0.8
|
||||
|
||||
def test_parse_empty_string(self) -> None:
|
||||
"""Leere Antwort → leere Beschreibung."""
|
||||
result = _parse_vision_response("")
|
||||
assert result["description"] == "Keine Inhalte erkannt"
|
||||
assert result["key_findings"] == []
|
||||
|
||||
def test_parse_nested_json(self) -> None:
|
||||
"""Verschachteltes JSON wird korrekt extrahiert."""
|
||||
data = {
|
||||
"description": "Nested Report",
|
||||
"key_findings": [
|
||||
{"type": "trend", "text": "Aufwärts"},
|
||||
{"type": "anomaly", "text": "Ausreißer"},
|
||||
],
|
||||
"data_points": [
|
||||
{"label": "Q1", "value": 100},
|
||||
{"label": "Q2", "value": 150},
|
||||
],
|
||||
"metadata": {"model": "qwen2.5-vl-3b"},
|
||||
}
|
||||
result = _parse_vision_response(json.dumps(data))
|
||||
assert result["description"] == "Nested Report"
|
||||
assert len(result["key_findings"]) == 2
|
||||
assert result["key_findings"][0] == {"type": "trend", "text": "Aufwärts"}
|
||||
assert result["data_points"][0]["label"] == "Q1"
|
||||
assert result["metadata"]["model"] == "qwen2.5-vl-3b"
|
||||
|
||||
def test_parse_extra_text_before_json(self) -> None:
|
||||
"""Text vor JSON wird ignoriert, JSON wird geparst."""
|
||||
response = '```json\n{"key_findings": ["After Text"], "confidence": 0.85}\n```'
|
||||
result = _parse_vision_response(response)
|
||||
assert "After Text" in result["key_findings"]
|
||||
assert result["confidence"] == 0.85
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 25–31: Prompt-Generierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildPrompt:
|
||||
"""Tests für _build_prompt — Prompt-Kombination."""
|
||||
|
||||
def test_base_prompt_includes_all_topics(self) -> None:
|
||||
"""Basis-Prompt erwähnt alle Analyse-Themen."""
|
||||
prompt = _build_prompt("screenshot", None, None)
|
||||
assert "visuelle Inhalte" in prompt
|
||||
assert "Textinhalte" in prompt
|
||||
assert "Daten" in prompt
|
||||
assert "Trends" in prompt
|
||||
assert "fact-checking" in prompt
|
||||
|
||||
def test_prompt_contains_image_data_ref(self) -> None:
|
||||
"""Bild-Referenz im Prompt für Vision-Modell."""
|
||||
prompt = _build_prompt("screenshot", None, None)
|
||||
# Basis-Prompt erwähnt visuelle Analyse
|
||||
assert "Bilder" in prompt or "visuell" in prompt or "Bild" in prompt
|
||||
|
||||
def test_capture_type_includes_screenshot(self) -> None:
|
||||
"""screenshot Capture Type → Standard-Prompt."""
|
||||
prompt = _build_prompt("screenshot", None, None)
|
||||
assert "screenshot" in prompt or len(prompt) > 50
|
||||
|
||||
def test_capture_type_infographic(self) -> None:
|
||||
"""infographic Capture Type → Typ im Prompt."""
|
||||
prompt = _build_prompt("infographic", None, None)
|
||||
assert "infographic" in prompt
|
||||
|
||||
def test_image_caption_appended(self) -> None:
|
||||
"""image_caption wird vor den Prompt gesetzt."""
|
||||
caption = "Diagramm zeigt Umsatzentwicklung 2024"
|
||||
prompt = _build_prompt("chart", caption, None)
|
||||
assert caption in prompt
|
||||
# Caption steht im Prompt
|
||||
assert prompt.index(caption) >= 0
|
||||
|
||||
def test_custom_prompt_overrides(self) -> None:
|
||||
"""Custom-Prompt wird verwendet, nicht der Default."""
|
||||
custom = "Finde alle Diagramme in diesem Bild"
|
||||
prompt = _build_prompt("chart", None, custom)
|
||||
assert "Finde alle Diagramme" in prompt
|
||||
|
||||
def test_truncation_large_caption(self) -> None:
|
||||
"""Sehr langer Caption → Prompt wird nicht unendlich."""
|
||||
long_caption = "x" * 10000
|
||||
prompt = _build_prompt("screenshot", long_caption, None)
|
||||
# Prompt sollte nicht die Python-Grenze sprengen
|
||||
assert len(prompt) < 50000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 32–36: _image_source_label
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestImageSourceLabel:
|
||||
"""Tests für _image_source_label."""
|
||||
|
||||
def test_url_source(self) -> None:
|
||||
"""HTTP/HTTPS-URL wird als Quelle gemeldet."""
|
||||
req = AnalyzeImageRequest(image_data="https://example.com/img.png")
|
||||
label = _image_source_label(req)
|
||||
assert "example.com" in label
|
||||
|
||||
def test_base64_source(self) -> None:
|
||||
"""Base64-Daten werden gemeldet."""
|
||||
req = AnalyzeImageRequest(image_data="data:image/png;base64,abc123")
|
||||
label = _image_source_label(req)
|
||||
assert "base64" in label
|
||||
|
||||
def test_short_url(self) -> None:
|
||||
"""Kurze URL ohne Ellipsis."""
|
||||
req = AnalyzeImageRequest(image_data="https://x.com")
|
||||
label = _image_source_label(req)
|
||||
assert "..." not in label or "example" not in label
|
||||
|
||||
def test_uploaded_image(self) -> None:
|
||||
"""Unbekannte Datenquelle → uploaded_image."""
|
||||
req = AnalyzeImageRequest(image_data="not_a_url_or_data")
|
||||
label = _image_source_label(req)
|
||||
assert label == "uploaded_image"
|
||||
|
||||
def test_very_long_url_truncated(self) -> None:
|
||||
"""Sehr lange URLs werden gekürzt."""
|
||||
long_url = "https://" + "x" * 500 + ".png"
|
||||
req = AnalyzeImageRequest(image_data=long_url)
|
||||
label = _image_source_label(req)
|
||||
assert "..." in label or len(label) <= 123
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 37–41: Store-Get Functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEvidenceStore:
|
||||
"""Tests für _store_evidence und _get_evidence."""
|
||||
|
||||
def test_store_and_retrieve(self) -> None:
|
||||
"""Eintrag speichern und wieder abrufen."""
|
||||
import uuid
|
||||
ev_id = str(uuid.uuid4())
|
||||
ev = EvidenceItem(
|
||||
id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="Test",
|
||||
)
|
||||
_store_evidence(ev)
|
||||
retrieved = _get_evidence(ev_id)
|
||||
assert retrieved is not None
|
||||
assert retrieved.id == ev_id
|
||||
assert retrieved.description == "Test"
|
||||
|
||||
def test_get_nonexistent(self) -> None:
|
||||
"""Nicht vorhandene ID → None."""
|
||||
import uuid
|
||||
nonexistent = str(uuid.uuid4())
|
||||
result = _get_evidence(nonexistent)
|
||||
assert result is None
|
||||
|
||||
def test_overwrite_existing(self) -> None:
|
||||
"""Store überschreibt bestehende IDs."""
|
||||
import uuid
|
||||
ev_id = str(uuid.uuid4())
|
||||
_store_evidence(EvidenceItem(
|
||||
id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="source1",
|
||||
description="V1",
|
||||
))
|
||||
_store_evidence(EvidenceItem(
|
||||
id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="source2",
|
||||
description="V2",
|
||||
))
|
||||
result = _get_evidence(ev_id)
|
||||
assert result.description == "V2"
|
||||
assert result.image_source == "source2"
|
||||
|
||||
def test_multiple_evidence_ids(self) -> None:
|
||||
"""Mehrere Evidenz-Einträge koexistieren."""
|
||||
import uuid
|
||||
id1 = str(uuid.uuid4())
|
||||
id2 = str(uuid.uuid4())
|
||||
_store_evidence(EvidenceItem(
|
||||
id=id1, evidence_type="visual", capture_type="screenshot",
|
||||
image_source="src1", description="E1",
|
||||
))
|
||||
_store_evidence(EvidenceItem(
|
||||
id=id2, evidence_type="document", capture_type="document",
|
||||
image_source="src2", description="E2",
|
||||
))
|
||||
ev1 = _get_evidence(id1)
|
||||
ev2 = _get_evidence(id2)
|
||||
assert ev1 is not None
|
||||
assert ev2 is not None
|
||||
assert ev1.description == "E1"
|
||||
assert ev2.description == "E2"
|
||||
|
||||
def test_empty_description(self) -> None:
|
||||
"""Leere Beschreibung wird gespeichert."""
|
||||
import uuid
|
||||
ev = EvidenceItem(
|
||||
id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="https://x.com",
|
||||
description="",
|
||||
)
|
||||
_store_evidence(ev)
|
||||
result = _get_evidence(ev.id)
|
||||
assert result is not None
|
||||
assert result.description == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 42–48: API-Integration — POST /vision/analyze (mit TestClient)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAPIAnalyze:
|
||||
"""Integrationstests für POST /vision/analyze."""
|
||||
|
||||
def test_analyze_returns_evidence_id(self, clean_env) -> None:
|
||||
"""Antwort enthält evidence_id."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
|
||||
fastapi_app = create_app()
|
||||
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={
|
||||
"image_data": _BASE64_DATA,
|
||||
"capture_type": "screenshot",
|
||||
"prompt": "Finde Evidenz",
|
||||
},
|
||||
)
|
||||
# Bei fehlendem Provider → 500 (Fallback)
|
||||
# Oder 200 mit Fallback-Evidence
|
||||
assert resp.status_code in (200, 500)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
assert "evidence_id" in data
|
||||
|
||||
def test_analyze_empty_image_data(self, clean_env) -> None:
|
||||
"""Empty image_data → 422 (Pydantic validation error)."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post("/vision/analyze", json={"image_data": ""})
|
||||
assert resp.status_code == 422
|
||||
|
||||
def test_analyze_no_image_data(self, clean_env) -> None:
|
||||
"""Kein image_data-Feld → 422."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post("/vision/analyze", json={})
|
||||
assert resp.status_code == 422
|
||||
|
||||
def test_analyze_with_url(self, clean_env) -> None:
|
||||
"""Bild als URL akzeptiert."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={
|
||||
"image_data": "https://example.com/test.png",
|
||||
"capture_type": "photo",
|
||||
},
|
||||
)
|
||||
# 200 (fallback) oder 500 (LLM error)
|
||||
assert resp.status_code in (200, 500)
|
||||
|
||||
def test_analyze_multiple_images_sequence(self, clean_env) -> None:
|
||||
"""Multiple Bilder nacheinander analysieren."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
ids = []
|
||||
for i in range(3):
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={
|
||||
"image_data": f"data:image/png;base64,img{i}",
|
||||
"capture_type": "screenshot",
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
ids.append(data.get("evidence_id", ""))
|
||||
assert len(ids) >= 0 # Mindestens 0 IDs (kann 0 sein bei LLM-Fehler)
|
||||
|
||||
def test_analyze_custom_evidence_type(self, clean_env) -> None:
|
||||
"""Custom evidence_type wird in Antwort zurückgegeben."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={
|
||||
"image_data": _BASE64_DATA,
|
||||
"evidence_type": "infographic",
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
assert data["evidence_type"] == "infographic"
|
||||
|
||||
def test_analyze_image_caption_included(self, clean_env) -> None:
|
||||
"""image_caption wird an Vision-Modell gesendet."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={
|
||||
"image_data": _BASE64_DATA,
|
||||
"image_caption": "Diagramm mit Umsatzdaten",
|
||||
},
|
||||
)
|
||||
assert resp.status_code in (200, 500)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 49–54: API-Integration — GET /vision/evidence/{evidence_id}
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAPIGetEvidence:
|
||||
"""Integrationstests für GET /vision/evidence/{evidence_id}."""
|
||||
|
||||
def test_get_valid_evidence(self, clean_env) -> None:
|
||||
"""Existierender Evidenz-Eintrag wird gefunden."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.post(
|
||||
"/vision/analyze",
|
||||
json={"image_data": _BASE64_DATA, "capture_type": "screenshot"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
evidence_id = resp.json()["evidence_id"]
|
||||
resp2 = client.get(f"/vision/evidence/{evidence_id}")
|
||||
assert resp2.status_code == 200
|
||||
data = resp2.json()
|
||||
assert data["success"] is True
|
||||
assert data["evidence"] is not None
|
||||
|
||||
def test_get_nonexistent_evidence(self, clean_env) -> None:
|
||||
"""Nicht vorhandener Evidenz-Eintrag → 404."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
import uuid
|
||||
fake_id = str(uuid.uuid4())
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.get(f"/vision/evidence/{fake_id}")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_get_empty_evidence_id(self, clean_env) -> None:
|
||||
"""Leere evidence_id → 400."""
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.get("/vision/evidence/")
|
||||
assert resp.status_code in (400, 404, 422)
|
||||
|
||||
def test_get_evidence_after_store(self, clean_env) -> None:
|
||||
"""Eintrag direkt im Store → GET findet ihn."""
|
||||
import uuid
|
||||
ev_id = str(uuid.uuid4())
|
||||
ev = EvidenceItem(
|
||||
id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="direct_store",
|
||||
description="Direct Store Test",
|
||||
)
|
||||
_store_evidence(ev)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.get(f"/vision/evidence/{ev_id}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["evidence"]["description"] == "Direct Store Test"
|
||||
|
||||
def test_get_evidence_response_structure(self, clean_env) -> None:
|
||||
"""GET-Antwort hat korrekte Struktur."""
|
||||
import uuid
|
||||
ev_id = str(uuid.uuid4())
|
||||
ev = EvidenceItem(
|
||||
id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
image_source="test",
|
||||
description="Structure Test",
|
||||
key_findings=["Finding A"],
|
||||
data_points=[{"value": 1}],
|
||||
confidence=0.9,
|
||||
)
|
||||
_store_evidence(ev)
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from nsct.api.main import create_app
|
||||
fastapi_app = create_app()
|
||||
with TestClient(fastapi_app) as client:
|
||||
resp = client.get(f"/vision/evidence/{ev_id}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "success" in data
|
||||
assert "evidence" in data
|
||||
ev_data = data["evidence"]
|
||||
assert "id" in ev_data
|
||||
assert "key_findings" in ev_data
|
||||
assert "data_points" in ev_data
|
||||
assert "confidence" in ev_data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 55–60: Edge Cases & Fallbacks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""Tests für Edge Cases und Fallbacks."""
|
||||
|
||||
def test_parse_null_response(self) -> None:
|
||||
"""None/Null-Response wird behandelt."""
|
||||
result = _parse_vision_response(None) # type: ignore[arg-type]
|
||||
assert result["description"] == "Keine Inhalte erkannt"
|
||||
|
||||
def test_parse_whitespace_only(self) -> None:
|
||||
"""Nur Whitespace → leere Beschreibung."""
|
||||
result = _parse_vision_response(" \n \t ")
|
||||
assert result["description"] == "Keine Inhalte erkannt"
|
||||
|
||||
def test_parse_json_with_extra_fields(self) -> None:
|
||||
"""JSON mit zusätzlichen Feldern wird ignoriert."""
|
||||
data = {
|
||||
"key_findings": ["A"],
|
||||
"extra_field": "ignored",
|
||||
"another_extra": 42,
|
||||
}
|
||||
result = _parse_vision_response(json.dumps(data))
|
||||
assert "A" in result["key_findings"]
|
||||
assert "extra_field" not in result
|
||||
|
||||
def test_response_list_mixed_types(self) -> None:
|
||||
"""JSON-Array mit gemischten Typen."""
|
||||
data = ["text", 42, True, None, {"key": "val"}]
|
||||
result = _parse_vision_response(json.dumps(data))
|
||||
# Alle Elemente werden zu Strings konvertiert
|
||||
assert "text" in result["key_findings"]
|
||||
assert len(result["key_findings"]) > 0
|
||||
|
||||
def test_prompt_with_all_optional_fields(self) -> None:
|
||||
"""Prompt mit allen optionalen Feldern."""
|
||||
prompt = _build_prompt(
|
||||
capture_type="infographic",
|
||||
image_caption="Beschriftung",
|
||||
custom_prompt="Finde Charts",
|
||||
)
|
||||
assert "Beschriftung" in prompt
|
||||
assert "infographic" in prompt
|
||||
assert "Finde Charts" in prompt
|
||||
|
||||
def test_image_data_very_long(self) -> None:
|
||||
"""Extrem lange Base64-Daten werden akzeptiert."""
|
||||
long_data = "a" * 1000000 # 1MB
|
||||
req = AnalyzeImageRequest(image_data=long_data)
|
||||
assert len(req.image_data) == 1000000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test Group 61–65: AnalyzeImageResponse Struktur
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAnalyzeImageResponse:
|
||||
"""Tests für AnalyzeImageResponse-Struktur."""
|
||||
|
||||
def test_response_has_all_fields(self) -> None:
|
||||
"""AnalyzeImageResponse hat alle erwarteten Felder."""
|
||||
import uuid
|
||||
ev_id = str(uuid.uuid4())
|
||||
resp = AnalyzeImageResponse(
|
||||
evidence_id=ev_id,
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
description="Test",
|
||||
findings=["A"],
|
||||
confidence=0.9,
|
||||
data_points=[{"value": 1}],
|
||||
image_source="https://x.com",
|
||||
metadata={"model": "test"},
|
||||
)
|
||||
assert resp.evidence_id == ev_id
|
||||
assert resp.evidence_type == "visual"
|
||||
assert resp.capture_type == "screenshot"
|
||||
assert resp.description == "Test"
|
||||
assert resp.findings == ["A"]
|
||||
assert resp.confidence == 0.9
|
||||
assert resp.data_points == [{"value": 1}]
|
||||
assert resp.image_source == "https://x.com"
|
||||
assert resp.metadata == {"model": "test"}
|
||||
|
||||
def test_response_defaults(self) -> None:
|
||||
"""AnalyzeImageResponse mit Minimal-Parametern."""
|
||||
import uuid
|
||||
resp = AnalyzeImageResponse(
|
||||
evidence_id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
description="Min",
|
||||
image_source="x",
|
||||
confidence=0.5,
|
||||
)
|
||||
assert resp.findings == []
|
||||
assert resp.confidence is not None
|
||||
assert resp.data_points == []
|
||||
assert resp.metadata == {}
|
||||
|
||||
def test_response_confidence_range(self) -> None:
|
||||
"""confidence im Response muss 0-1 sein."""
|
||||
import uuid
|
||||
resp = AnalyzeImageResponse(
|
||||
evidence_id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
description="Test",
|
||||
image_source="x",
|
||||
confidence=0.0,
|
||||
)
|
||||
assert resp.confidence == 0.0
|
||||
|
||||
def test_response_empty_findings(self) -> None:
|
||||
"""Leere findings-Liste."""
|
||||
import uuid
|
||||
resp = AnalyzeImageResponse(
|
||||
evidence_id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
description="Test",
|
||||
image_source="x",
|
||||
findings=[],
|
||||
confidence=0.5,
|
||||
)
|
||||
assert resp.findings == []
|
||||
|
||||
def test_response_data_points_structure(self) -> None:
|
||||
"""data_points sind List von Dicts."""
|
||||
import uuid
|
||||
resp = AnalyzeImageResponse(
|
||||
evidence_id=str(uuid.uuid4()),
|
||||
evidence_type="visual",
|
||||
capture_type="screenshot",
|
||||
description="Test",
|
||||
image_source="x",
|
||||
data_points=[
|
||||
{"key": "val1"},
|
||||
{"key": "val2"},
|
||||
],
|
||||
confidence=0.5,
|
||||
)
|
||||
assert len(resp.data_points) == 2
|
||||
assert isinstance(resp.data_points[0], dict)
|
||||
Reference in New Issue
Block a user