Stage 0: Repository und Architekturgrundlage

- Pyproject.toml mit FastAPI, Pydantic v2, SQLAlchemy, httpx, asyncio, BeautifulSoup4, selectolax, trafilatura, uvicorn, pytest-asyncio - Multi-stage Dockerfile (Python 3.12-slim, Non-Root-User nsct) - docker-compose.yml (nsct-api + postgres + optional searxng) - .env.example mit allen Config-Parametern - Config-System: AppSettings mit LLMConfig, VisionConfig, AudioConfig, DatabaseConfig — komplett aus Environment, keine Hardcodes - Strukturiertes Logging mit research_id/llm_request_id Tracking - Pydantic v2 Schemas: SearchQuery, Source, Claim, EvidenceRelation, CitationEdge, ResearchReport - SQLAlchemy 2.0 Declarative Models + async Engine Factory - SSRF-Schutz: URL-Validation, IP-Blocklist (RFC1918, Cloud Metadata, file://, ftp://) - Provider-Interfaces: LLMProvider, VisionProvider, AudioProvider, SearchProvider, ContentFetcher als ABCs - Health-Endpoints: /health, /ready (LLM-Connect-Test), /providers - FastAPI App mit CORS, lifespan (LLM Pre-Flight) - CLI-Stub mit Entry-Points: nsct, nsct-core, nsct-api - 6 Test-Cases: /health, /ready, /providers + No-Secrets-Test - Vollständige Dokumentation: README, ARCHITECTURE, SECURITY, METHODOLOGY, API, DEPLOYMENT - .gitignore (Python, Docker, IDE, .env)
2026-08-23 11:33:45 +00:00
commit e9410be941
28 changed files with 4192 additions and 0 deletions
--- a/src/nsct/storage/init.py
+++ b/src/nsct/storage/init.py
@@ -0,0 +1 @@
+"""NSCT — storage package init."""
--- a/src/nsct/storage/engine.py
+++ b/src/nsct/storage/engine.py
@@ -0,0 +1,83 @@
+"""Async SQLAlchemy engine factory — creates and manages the connection pool."""
+
+from __future__ import annotations
+
+import logging
+from typing import AsyncGenerator
+
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from nsct.config import AppSettings
+from nsct.storage.models import Base  # noqa: F401 — Base is needed for create_all
+
+logger = logging.getLogger(__name__)
+
+_engine = None
+_session_factory = None
+
+
+async def get_engine(config: AppSettings) -> None:
+    """Initialise the global async engine and session factory.
+
+    Must be called once during application startup (lifespan) before any
+    request is handled.
+    """
+    global _engine, _session_factory
+
+    if _engine is not None:
+        logger.warning("Engine already initialised; skipping create_engine.")
+        return
+
+    db_url = config.postgres.url
+
+    _engine = create_async_engine(
+        db_url,
+        echo=config.debug,
+        pool_size=10,
+        max_overflow=20,
+        pool_recycle=1800,
+        pool_pre_ping=True,
+    )
+
+    # Ensure tables exist (schema migration not handled here — that is a
+    # separate migration step; this creates missing tables only).
+    async with _engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+
+    _session_factory = async_sessionmaker(
+        _engine,
+        class_=AsyncSession,
+        expire_on_commit=False,
+        autocommit=False,
+        autoflush=False,
+    )
+
+    logger.info("Async SQLAlchemy engine initialised. DB: %s", db_url)
+
+
+async def get_session() -> AsyncGenerator[AsyncSession, None]:
+    """Yield an async session for request-scoped database access."""
+    global _session_factory
+    if _session_factory is None:
+        raise RuntimeError("Engine not initialised. Call get_engine() first.")
+
+    session = _session_factory()
+    try:
+        yield session
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
+    finally:
+        await session.close()
+
+
+async def close_engine() -> None:
+    """Close the engine and release all pooled connections."""
+    global _engine, _session_factory
+    if _engine is not None:
+        await _engine.dispose()
+        logger.info("SQLAlchemy engine disposed.")
+        _engine = None
+        _session_factory = None
--- a/src/nsct/storage/models.py
+++ b/src/nsct/storage/models.py
@@ -0,0 +1,210 @@
+"""SQLAlchemy 2.0 Declarative models — mapping of NSCT domain objects to tables."""
+
+from __future__ import annotations
+
+import enum
+from datetime import datetime
+from typing import Any
+from uuid import UUID, uuid4
+
+from sqlalchemy import (
+    BigInteger,
+    Column,
+    DateTime,
+    Enum,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+)
+from sqlalchemy.orm import DeclarativeBase, relationship
+
+
+class ClaimType(str, enum.Enum):
+    FACTUAL = "factual"
+    OPINION = "opinion"
+    PREDICTION = "prediction"
+    EVALUATION = "evaluation"
+    COMPARISON = "comparison"
+
+
+class SourceType(str, enum.Enum):
+    NEWS = "news"
+    ACADEMIC = "academic"
+    BLOG = "blog"
+    GOVERNMENT = "government"
+    CORPORATE = "corporate"
+    SOCIAL_MEDIA = "social_media"
+    DOCUMENT = "document"
+    OTHER = "other"
+
+
+class EvidenceRelationType(str, enum.Enum):
+    AGREES = "agrees"
+    DISAGREES = "disagrees"
+    NEUTRAL = "neutral"
+    PARTIALLY_AGREES = "partially_agrees"
+    PARTIALLY_DISAGREES = "partially_disagrees"
+    CONTRADICTS = "contradicts"
+
+
+class EdgeRelation(str, enum.Enum):
+    CITATION = "citation"
+    CORROBORATION = "corroboration"
+    CONTRADICTION = "contradiction"
+    DEPENDS_ON = "depends_on"
+    SUPPLEMENTS = "supplements"
+
+
+# ---------------------------------------------------------------------------
+# Base
+# ---------------------------------------------------------------------------
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+# ---------------------------------------------------------------------------
+# SearchQuery
+# ---------------------------------------------------------------------------
+
+
+class SearchQueryModel(Base):
+    __tablename__ = "search_queries"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    research_id = Column(String(36), nullable=False, default=lambda: str(uuid4()))
+    query = Column(Text, nullable=False)
+    purpose = Column(Text, nullable=True)
+    language = Column(String(16), nullable=False, default="de")
+    category = Column(String(64), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    __table_args__ = (Index("ix_search_queries_research_id", "research_id"),)
+
+
+# ---------------------------------------------------------------------------
+# Source
+# ---------------------------------------------------------------------------
+
+
+class SourceModel(Base):
+    __tablename__ = "sources"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text, nullable=True)
+    domain = Column(String(255), nullable=False)
+    title = Column(Text, nullable=True)
+    author = Column(Text, nullable=True)
+    publisher = Column(Text, nullable=True)
+    publication_date = Column(DateTime, nullable=True)
+    retrieved_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    content_type = Column(String(128), nullable=True)
+    source_type = Column(Enum(SourceType), nullable=True)
+    language = Column(String(16), nullable=False, default="unknown")
+    content_hash = Column(String(64), nullable=True)
+    parent_source_id = Column(String(36), ForeignKey("sources.id"), nullable=True)
+    content = Column(Text, nullable=True)
+
+    # Relationships
+    claims = relationship("ClaimModel", back_populates="source", cascade="all, delete-orphan")
+
+    __table_args__ = (
+        Index("ix_sources_domain", "domain"),
+        Index("ix_sources_parent_source_id", "parent_source_id"),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Claim
+# ---------------------------------------------------------------------------
+
+
+class ClaimModel(Base):
+    __tablename__ = "claims"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    claim = Column(Text, nullable=False)
+    normalized_claim = Column(Text, nullable=True)
+    claim_type = Column(Enum(ClaimType), nullable=False, default=ClaimType.FACTUAL)
+    subject = Column(Text, nullable=True)
+    predicate = Column(Text, nullable=True)
+    object = Column(Text, nullable=True)
+    evidence_span = Column(Text, nullable=True)
+    confidence = Column(Float, nullable=False, default=1.0)
+    event_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    source = relationship("SourceModel", back_populates="claims")
+
+
+# ---------------------------------------------------------------------------
+# EvidenceRelation
+# ---------------------------------------------------------------------------
+
+
+class EvidenceRelationModel(Base):
+    __tablename__ = "evidence_relations"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    claim_a = Column(String(36), ForeignKey("claims.id"), nullable=False)
+    claim_b = Column(String(36), ForeignKey("claims.id"), nullable=False)
+    relation = Column(Enum(EvidenceRelationType), nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    reason = Column(Text, nullable=True)
+
+    __table_args__ = (
+        Index("ix_evidence_relations_claim_a", "claim_a"),
+        Index("ix_evidence_relations_claim_b", "claim_b"),
+        # Ensure uniqueness of the (a, b, relation) triple
+        Index("uq_evidence_relations_ab", "claim_a", "claim_b", "relation", unique=True),
+    )
+
+
+# ---------------------------------------------------------------------------
+# CitationEdge
+# ---------------------------------------------------------------------------
+
+
+class CitationEdgeModel(Base):
+    __tablename__ = "citation_edges"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    source_from = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    source_to = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    relation = Column(Enum(EdgeRelation), nullable=False)
+    confidence = Column(Float, nullable=False, default=1.0)
+
+    __table_args__ = (
+        Index("ix_citation_edges_source_from", "source_from"),
+        Index("ix_citation_edges_source_to", "source_to"),
+    )
+
+
+# ---------------------------------------------------------------------------
+# ResearchReport
+# ---------------------------------------------------------------------------
+
+
+class ResearchReportModel(Base):
+    __tablename__ = "research_reports"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    research_id = Column(String(36), ForeignKey("search_queries.research_id"), nullable=False)
+    query = Column(Text, nullable=False)
+    summary = Column(Text, nullable=False, default="")
+    findings = Column(Text, nullable=False, default="[]")
+    disagreements = Column(Text, nullable=False, default="[]")
+    uncertainties = Column(Text, nullable=False, default="[]")
+    source_statistics = Column(Text, nullable=False, default="{}")
+    methodology = Column(Text, nullable=False, default="")
+    generated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    __table_args__ = (
+        Index("ix_research_reports_research_id", "research_id"),
+    )