Stage 0: Repository und Architekturgrundlage

- Pyproject.toml mit FastAPI, Pydantic v2, SQLAlchemy, httpx, asyncio, BeautifulSoup4, selectolax, trafilatura, uvicorn, pytest-asyncio - Multi-stage Dockerfile (Python 3.12-slim, Non-Root-User nsct) - docker-compose.yml (nsct-api + postgres + optional searxng) - .env.example mit allen Config-Parametern - Config-System: AppSettings mit LLMConfig, VisionConfig, AudioConfig, DatabaseConfig — komplett aus Environment, keine Hardcodes - Strukturiertes Logging mit research_id/llm_request_id Tracking - Pydantic v2 Schemas: SearchQuery, Source, Claim, EvidenceRelation, CitationEdge, ResearchReport - SQLAlchemy 2.0 Declarative Models + async Engine Factory - SSRF-Schutz: URL-Validation, IP-Blocklist (RFC1918, Cloud Metadata, file://, ftp://) - Provider-Interfaces: LLMProvider, VisionProvider, AudioProvider, SearchProvider, ContentFetcher als ABCs - Health-Endpoints: /health, /ready (LLM-Connect-Test), /providers - FastAPI App mit CORS, lifespan (LLM Pre-Flight) - CLI-Stub mit Entry-Points: nsct, nsct-core, nsct-api - 6 Test-Cases: /health, /ready, /providers + No-Secrets-Test - Vollständige Dokumentation: README, ARCHITECTURE, SECURITY, METHODOLOGY, API, DEPLOYMENT - .gitignore (Python, Docker, IDE, .env)
2026-08-23 11:33:45 +00:00
commit e9410be941
28 changed files with 4192 additions and 0 deletions
--- a/src/nsct/init.py
+++ b/src/nsct/init.py
@@ -0,0 +1,3 @@
+"""NSCT — Neutral Search Crawler Tool."""
+
+__version__ = "0.1.0"
--- a/src/nsct/api/init.py
+++ b/src/nsct/api/init.py
@@ -0,0 +1 @@
+"""NSCT — package init."""
--- a/src/nsct/api/health.py
+++ b/src/nsct/api/health.py
@@ -0,0 +1,84 @@
+"""Health check router for the FastAPI application."""
+
+from __future__ import annotations
+
+import httpx
+import logging
+
+from fastapi import APIRouter
+from nsct.config import AppSettings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health_check():
+    """Basic health check — always returns ok when the process is alive."""
+    from nsct import __version__
+
+    return {
+        "status": "ok",
+        "version": __version__,
+    }
+
+
+@router.get("/ready")
+async def readiness_check(config: AppSettings | None = None):
+    """Readiness check — probes downstream services."""
+    result: dict[str, str | bool] = {
+        "status": "not_ready",
+        "llm": "error",
+        "database": "unknown",
+    }
+
+    # LLM connectivity check
+    llm_base = config.llm.base_url if config and config.llm and config.llm.base_url else ""
+    if llm_base:
+        try:
+            async with httpx.AsyncClient(timeout=3.0) as client:
+                resp = await client.get(f"{llm_base}/models")
+                if resp.status_code == 200:
+                    result["llm"] = "ok"
+                    result["llm_model"] = config.llm.model if config else "unknown"
+                else:
+                    result["llm"] = f"error:{resp.status_code}"
+        except Exception as exc:
+            result["llm"] = f"error:{type(exc).__name__}"
+
+    if result.get("llm") == "ok":
+        result["status"] = "ready"
+
+    return result
+
+
+@router.get("/providers")
+def providers_info(config: AppSettings | None = None):
+    """List available providers — NO secrets returned."""
+    if not config:
+        return {
+            "llm": {"available": False},
+            "vision": {"available": False},
+            "audio": {"available": False},
+        }
+
+    llm_available = bool(config.llm.base_url)
+    vision_available = bool(config.vision.base_url)
+    audio_available = bool(config.audio.base_url)
+
+    return {
+        "llm": {
+            "available": llm_available,
+            "model": config.llm.model if llm_available else None,
+            "max_concurrency": config.llm.max_concurrency if llm_available else None,
+        },
+        "vision": {
+            "available": vision_available,
+            "model": config.vision.model if vision_available else None,
+        },
+        "audio": {
+            "available": audio_available,
+            "model": config.audio.model if audio_available else None,
+        },
+    }
--- a/src/nsct/api/main.py
+++ b/src/nsct/api/main.py
@@ -0,0 +1,81 @@
+"""FastAPI application — entry point for the API server."""
+
+from __future__ import annotations
+
+import logging
+from contextlib import asynccontextmanager
+
+import httpx
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from nsct.config import AppSettings
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Lifespan
+# ---------------------------------------------------------------------------
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan — startup and shutdown hooks."""
+    config: AppSettings = app.state.config
+
+    logger.info("NSCT API starting up — version %s", app.state.version)
+
+    # Pre-flight: validate LLM connectivity
+    llm_base = config.llm.base_url if config and config.llm and config.llm.base_url else ""
+    if llm_base:
+        try:
+            async with httpx.AsyncClient(timeout=5.0) as client:
+                resp = await client.get(f"{llm_base}/models")
+                if resp.status_code == 200:
+                    logger.info("LLM provider reachable — model endpoint returned 200")
+                else:
+                    logger.warning("LLM provider returned %s", resp.status_code)
+        except Exception as exc:
+            logger.warning("LLM connectivity check failed: %s", exc)
+
+    yield
+
+
+# ---------------------------------------------------------------------------
+# App factory
+# ---------------------------------------------------------------------------
+
+
+def create_app() -> FastAPI:
+    """Create the FastAPI application with all routes and middleware."""
+    config = AppSettings.from_env()
+
+    app = FastAPI(
+        title="NSCT API",
+        description="Neutral Search Crawler Tool — API server",
+        version="0.1.0",
+        docs_url="/docs" if config.debug else None,
+        redoc_url="/redoc" if config.debug else None,
+        lifespan=lifespan,
+    )
+    app.state.config = config
+    app.state.version = "0.1.0"
+
+    # CORS — allow local development
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:3000", "http://localhost:5173", "http://127.0.0.1:3000"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Mount health router
+    from nsct.api.health import router as health_router
+    app.include_router(health_router, tags=["system"])
+
+    return app
+
+
+app = create_app()
--- a/src/nsct/cli.py
+++ b/src/nsct/cli.py
@@ -0,0 +1,65 @@
+"""CLI entry-point for NSCT — stub for future CLI commands."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from nsct import __version__
+
+
+def main() -> int:
+    """Main CLI entry-point."""
+    parser = argparse.ArgumentParser(
+        prog="nsct",
+        description="NSCT — Neutral Search Crawler Tool",
+    )
+    parser.add_argument("--version", action="version", version=f"NSCT v{__version__}")
+    parser.add_argument(
+        "command",
+        nargs="?",
+        default="help",
+        help="Command to run (help, search, report, etc.)",
+    )
+    args = parser.parse_args()
+
+    if args.command == "help":
+        print("NSCT — Neutral Search Crawler Tool")
+        print(f"Version: {__version__}")
+        print()
+        print("Commands:")
+        print("  help      Show this help message")
+        print("  search    Search web sources (coming in Stage 1)")
+        print("  report    Generate a research report (coming in Stage 2)")
+        print("  version   Show version")
+        return 0
+
+    print(f"Unknown command: {args.command}")
+    return 1
+
+
+def main_api() -> int:
+    """Start the FastAPI server directly from CLI."""
+    import uvicorn
+
+    from nsct.config import AppSettings
+
+    config = AppSettings.from_env()
+    host = "0.0.0.0"
+    port = 8080
+
+    print(f"Starting NSCT API server on {host}:{port}")
+    print(f"LLM model: {config.llm.model}")
+    print(f"Debug mode: {config.debug}")
+
+    uvicorn.run(
+        "nsct.api.main:app",
+        host=host,
+        port=port,
+        log_level="info" if not config.debug else "debug",
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/src/nsct/config.py
+++ b/src/nsct/config.py
@@ -0,0 +1,116 @@
+"""Central configuration — all values from environment, zero hard-coded secrets."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class LLMConfig(BaseModel):
+    """LLM worker configuration."""
+
+    base_url: str = Field(..., description="Base URL of the LLM provider (OpenAI-compatible).")
+    model: str = Field(..., description="Model identifier to use for LLM calls.")
+    max_concurrency: int = Field(default=3, ge=1, description="Max parallel LLM requests.")
+
+    def get_secret(self) -> str:
+        """Return the raw API key string if present."""
+        key = os.environ.get("NSCT_LLM_API_KEY", "")
+        return key if key else ""
+
+
+class VisionConfig(BaseModel):
+    """Vision (image/video) model configuration."""
+
+    base_url: str = Field(..., description="Base URL of the vision provider endpoint.")
+    model: str = Field(..., description="Vision model identifier.")
+
+    def get_secret(self) -> str:
+        key = os.environ.get("NSCT_VISION_API_KEY", "")
+        return key if key else ""
+
+
+class AudioConfig(BaseModel):
+    """Audio / speech model configuration."""
+
+    base_url: str = Field(..., description="Base URL of the audio provider endpoint.")
+    model: str = Field(default="default", description="Audio model identifier.")
+
+    def get_secret(self) -> str:
+        key = os.environ.get("NSCT_AUDIO_API_KEY", "")
+        return key if key else ""
+
+
+class DatabaseConfig(BaseModel):
+    """Database connection configuration."""
+
+    url: str = Field(
+        ...,
+        description="Database URL (postgresql+asyncpg://...) or a default SQLite fallback.",
+    )
+
+
+class AppSettings(BaseModel):
+    """Top-level application settings."""
+
+    llm: LLMConfig = Field(default_factory=LLMConfig)
+    vision: VisionConfig = Field(default_factory=VisionConfig)
+    audio: AudioConfig = Field(default_factory=AudioConfig)
+    postgres: DatabaseConfig = Field(default_factory=DatabaseConfig)
+    debug: bool = Field(default=False)
+    searxng_base_url: str | None = Field(default=None, description="SearXNG instance URL.")
+
+    @staticmethod
+    def from_env() -> "AppSettings":
+        """Build AppSettings entirely from environment variables."""
+        llm_base_url = os.environ.get("NSCT_LLM_BASE_URL", "")
+        llm_model = os.environ.get("NSCT_LLM_MODEL", "")
+        llm_concurrency = int(os.environ.get("NSCT_LLM_MAX_CONCURRENCY", "3"))
+
+        vision_base_url = os.environ.get("NSCT_VISION_BASE_URL", "")
+        vision_model = os.environ.get("NSCT_VISION_MODEL", "")
+
+        audio_base_url = os.environ.get("NSCT_AUDIO_BASE_URL", "")
+        audio_model = os.environ.get("NSCT_AUDIO_MODEL", "default")
+
+        db_url = os.environ.get("NSCT_DB_URL", "")
+
+        llm_cfg = LLMConfig(
+            base_url=llm_base_url,
+            model=llm_model,
+            max_concurrency=llm_concurrency,
+        )
+
+        vision_cfg = VisionConfig(
+            base_url=vision_base_url,
+            model=vision_model,
+        )
+
+        audio_cfg = AudioConfig(
+            base_url=audio_base_url,
+            model=audio_model,
+        )
+
+        postgres_cfg = DatabaseConfig(url=db_url)
+
+        debug = os.environ.get("NSCT_DEBUG", "false").lower() == "true"
+        searxng_url = os.environ.get("NSCT_SEARXNG_BASE_URL", None)
+
+        return AppSettings(
+            llm=llm_cfg,
+            vision=vision_cfg,
+            audio=audio_cfg,
+            postgres=postgres_cfg,
+            debug=debug,
+            searxng_base_url=searxng_url,
+        )
+
+    @property
+    def config_dir(self) -> Path:
+        """Return a config directory path for persisting runtime data."""
+        data_dir = Path.home() / ".nsct"
+        data_dir.mkdir(parents=True, exist_ok=True)
+        return data_dir
--- a/src/nsct/logging_config.py
+++ b/src/nsct/logging_config.py
@@ -0,0 +1,126 @@
+"""Structured logging with research_id and llm_request_id tracking.
+
+Uses the standard library logging module with a JSON formatter so that every
+log record is machine-parseable and traceable across distributed components.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+import time
+import uuid
+from typing import Any
+
+from nsct import __version__
+
+# ---------------------------------------------------------------------------
+# Global request-context helpers — all structured log calls pick them up.
+# ---------------------------------------------------------------------------
+
+_request_ctx: dict[str, str] = {}
+
+
+def set_request_ctx(**kwargs: str) -> None:
+    """Set (merge) key-value pairs into the per-request context dict."""
+    _request_ctx.update(kwargs)
+
+
+def clear_request_ctx() -> None:
+    """Clear all per-request context entries."""
+    _request_ctx.clear()
+
+
+# ---------------------------------------------------------------------------
+# JSON Formatter
+# ---------------------------------------------------------------------------
+
+
+class JSONFormatter(logging.Formatter):
+    """Emit log records as a single JSON object per line."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._start_ts = time.monotonic()
+
+    # no default format — we build the dict ourselves
+    default_fmt: str = ""
+
+    def format(self, record: logging.LogRecord) -> str:
+        ts = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
+        elapsed_ms = round((time.monotonic() - self._start_ts) * 1000, 2)
+
+        extra = {
+            "_ts": ts,
+            "_ms": elapsed_ms,
+            "_pid": record.process,
+            "_thread": record.thread,
+            "_level": record.levelname,
+            "_module": record.module,
+            "_function": record.funcName,
+            "_line": record.lineno,
+            "_version": __version__,
+        }
+
+        # pull in the request context
+        for key, val in _request_ctx.items():
+            extra[f"ctx.{key}"] = val
+
+        # merge with standard fields
+        data: dict[str, Any] = dict(extra)
+        data["msg"] = record.getMessage()
+        data["name"] = record.name
+
+        # exception info
+        if record.exc_info and record.exc_info[0] is not None:
+            data["exc_info"] = self.formatException(record.exc_info)
+
+        return json.dumps(data, default=str, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------------------
+# Bootstrap
+# ---------------------------------------------------------------------------
+
+
+def setup_logging(
+    level: str = "INFO",
+    *,
+    json_format: bool = True,
+) -> None:
+    """Configure the root logger with a single console handler.
+
+    Args:
+        level: Log level string (DEBUG, INFO, WARNING, ERROR, CRITICAL).
+        json_format: If True, use JSONFormatter; else use a human-friendly format.
+    """
+    log_level = getattr(logging, level.upper(), logging.INFO)
+
+    root = logging.getLogger()
+    root.setLevel(log_level)
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(log_level)
+
+    if json_format:
+        handler.setFormatter(JSONFormatter())
+    else:
+        handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s [%(levelname)-8s] %(name)s:%(funcName)s:%(lineno)d — %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+
+    root.addHandler(handler)
+
+
+# ---------------------------------------------------------------------------
+# Convenience logger factory
+# ---------------------------------------------------------------------------
+
+
+def get_logger(name: str = __name__) -> logging.Logger:
+    """Return a pre-configured child logger."""
+    return logging.getLogger(name)
--- a/src/nsct/models/schemas.py
+++ b/src/nsct/models/schemas.py
@@ -0,0 +1,184 @@
+"""Pydantic v2 schemas for NSCT data objects."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from uuid import UUID, uuid4
+
+from pydantic import BaseModel, Field
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+class ClaimType(str, Enum):
+    """Types of claims that can be extracted from sources."""
+
+    FACTUAL = "factual"
+    OPINION = "opinion"
+    PREDICTION = "prediction"
+    EVALUATION = "evaluation"
+    COMPARISON = "comparison"
+
+
+class EvidenceRelationType(str, Enum):
+    """Relationship types between two claims."""
+
+    AGREES = "agrees"
+    DISAGREES = "disagrees"
+    NEUTRAL = "neutral"
+    PARTIALLY_AGREES = "partially_agrees"
+    PARTIALLY_DISAGREES = "partially_disagrees"
+    CONTRADICTS = "contradicts"
+
+
+class SourceType(str, Enum):
+    """Classification of a source."""
+
+    NEWS = "news"
+    ACADEMIC = "academic"
+    BLOG = "blog"
+    GOVERNMENT = "government"
+    CORPORATE = "corporate"
+    SOCIAL_MEDIA = "social_media"
+    DOCUMENT = "document"
+    OTHER = "other"
+
+
+class EdgeRelation(str, Enum):
+    """Relationship between two sources."""
+
+    CITATION = "citation"
+    CORROBORATION = "corroboration"
+    CONTRADICTION = "contradiction"
+    DEPENDS_ON = "depends_on"
+    SUPPLEMENTS = "supplements"
+
+
+# ---------------------------------------------------------------------------
+# SearchQuery
+# ---------------------------------------------------------------------------
+
+
+class SearchQuery(BaseModel):
+    """Represents a user-initiated search/research query."""
+
+    id: UUID = Field(default_factory=uuid4)
+    research_id: UUID = Field(default_factory=uuid4)
+    query: str = Field(..., min_length=1, description="The search query text.")
+    purpose: str | None = Field(default=None, description="Intent behind the query.")
+    language: str = Field(default="de", description="Language code, e.g. 'de', 'en'.")
+    category: str | None = Field(default=None, description="Optional category for grouping.")
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+    model_config = {"frozen": True}
+
+
+# ---------------------------------------------------------------------------
+# Source
+# ---------------------------------------------------------------------------
+
+
+class Source(BaseModel):
+    """A single retrieved source (web page, document, etc.)."""
+
+    id: UUID = Field(default_factory=uuid4)
+    url: str = Field(..., description="Original URL.")
+    canonical_url: str | None = Field(default=None, description="Canonical URL after redirect resolution.")
+    domain: str = Field(..., description="Extracted domain.")
+    title: str | None = Field(default=None)
+    author: str | None = Field(default=None)
+    publisher: str | None = Field(default=None)
+    publication_date: datetime | None = Field(default=None)
+    retrieved_at: datetime = Field(default_factory=datetime.utcnow)
+    content_type: str | None = Field(default=None, description="MIME type.")
+    source_type: SourceType | None = Field(default=None)
+    language: str = Field(default="unknown")
+    content_hash: str | None = Field(default=None, description="SHA-256 hash of raw content.")
+    parent_source_id: UUID | None = Field(default=None, description="Parent source for mirrors/canonical pairs.")
+    content: str | None = Field(default=None, description="Extracted text content.")
+
+    model_config = {"frozen": True}
+
+
+# ---------------------------------------------------------------------------
+# Claim
+# ---------------------------------------------------------------------------
+
+
+class Claim(BaseModel):
+    """A factual assertion extracted from a Source."""
+
+    id: UUID = Field(default_factory=uuid4)
+    source_id: UUID = Field(..., description="UUID of the Source this claim came from.")
+    claim: str = Field(..., description="Original claim text.")
+    normalized_claim: str | None = Field(default=None, description="De-biased / neutral claim wording.")
+    claim_type: ClaimType = Field(default=ClaimType.FACTUAL)
+    subject: str | None = Field(default=None)
+    predicate: str | None = Field(default=None)
+    object: str | None = Field(default=None)
+    evidence_span: str | None = Field(default=None, description="Span of text in the source that supports this claim.")
+    confidence: float = Field(default=1.0, ge=0.0, le=1.0, description="Extraction confidence (0-1).")
+    event_date: datetime | None = Field(default=None)
+
+    model_config = {"frozen": True}
+
+
+# ---------------------------------------------------------------------------
+# EvidenceRelation
+# ---------------------------------------------------------------------------
+
+
+class EvidenceRelation(BaseModel):
+    """Relationship between two claims."""
+
+    claim_a: UUID = Field(..., description="UUID of the first claim.")
+    claim_b: UUID = Field(..., description="UUID of the second claim.")
+    relation: EvidenceRelationType = Field(...)
+    confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    reason: str | None = Field(default=None, description="Free-text explanation of the relation.")
+
+
+# ---------------------------------------------------------------------------
+# CitationEdge
+# ---------------------------------------------------------------------------
+
+
+class CitationEdge(BaseModel):
+    """Directed edge between two sources indicating a relationship."""
+
+    source_from: UUID = Field(..., description="UUID of the citing source.")
+    source_to: UUID = Field(..., description="UUID of the cited source.")
+    relation: EdgeRelation = Field(...)
+    confidence: float = Field(default=1.0, ge=0.0, le=1.0)
+
+
+# ---------------------------------------------------------------------------
+# ResearchReport
+# ---------------------------------------------------------------------------
+
+
+class ResearchReport(BaseModel):
+    """Final aggregated research report."""
+
+    research_id: UUID = Field(default_factory=uuid4)
+    query: str = Field(...)
+    summary: str = Field(default="", description="Human-readable summary.")
+    findings: list[str] = Field(default_factory=list, description="List of key findings.")
+    disagreements: list[dict[str, Any]] = Field(
+        default_factory=list, description="Conflicting claims with details."
+    )
+    uncertainties: list[str] = Field(default_factory=list, description="Known uncertainties.")
+    source_statistics: dict[str, Any] = Field(
+        default_factory=dict, description="Stats: counts per source type, language, etc."
+    )
+    methodology: str = Field(
+        default="", description="Description of the methodology used to produce the report."
+    )
+    generated_at: datetime = Field(default_factory=datetime.utcnow)
+
+    model_config = {"frozen": True}
--- a/src/nsct/providers/init.py
+++ b/src/nsct/providers/init.py
@@ -0,0 +1 @@
+"""NSCT — providers package init."""
--- a/src/nsct/security/init.py
+++ b/src/nsct/security/init.py
@@ -0,0 +1 @@
+"""NSCT — security package init."""
--- a/src/nsct/security/policy.py
+++ b/src/nsct/security/policy.py
@@ -0,0 +1,139 @@
+"""Security policy — SSRF protection, URL validation, and IP blocklisting.
+
+All outbound HTTP requests from NSCT MUST pass through this module.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import re
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Blocked address ranges
+# ---------------------------------------------------------------------------
+
+_BLOCKED_NETWORKS: list[ipaddress.IPv4Network] = [
+    ipaddress.IPv4Network("127.0.0.0/8"),       # Loopback
+    ipaddress.IPv4Network("10.0.0.0/8"),        # Private
+    ipaddress.IPv4Network("172.16.0.0/12"),     # Private
+    ipaddress.IPv4Network("192.168.0.0/16"),    # Private
+    ipaddress.IPv4Network("169.254.0.0/16"),    # Link-local
+    ipaddress.IPv4Network("0.0.0.0/8"),         # "This" network
+    ipaddress.IPv4Network("100.64.0.0/10"),     # Shared (RFC 6598)
+    ipaddress.IPv4Network("192.0.0.0/24"),      # IETF protocol assignments
+    ipaddress.IPv4Network("192.0.2.0/24"),      # TEST-NET-1
+    ipaddress.IPv4Network("198.51.100.0/24"),   # TEST-NET-2
+    ipaddress.IPv4Network("203.0.113.0/24"),    # TEST-NET-3
+]
+
+# IPv6 equivalents (loopback, unique-local, link-local)
+_BLOCKED_IPV6_NETWORKS: list[ipaddress.IPv6Network] = [
+    ipaddress.IPv6Network("::1/128"),          # Loopback
+    ipaddress.IPv6Network("::1/128"),
+    ipaddress.IPv6Network("fc00::/7"),         # Unique-local
+    ipaddress.IPv6Network("fe80::/10"),        # Link-local
+]
+
+# Blocked schemes
+_BLOCKED_SCHEMES = frozenset(["file", "ftp", "gopher", "ldap", "mailto", "data", "blob"])
+
+# Cloud metadata endpoint pattern
+_METADATA_PATTERNS = [
+    "169.254.169.254",
+    "metadata.google.internal",
+    "metadata.aws.internal",
+    "169.254.170.2",
+    "169.254.169.254",
+    "instance-data",
+    "amazonaws.com",
+]
+
+# ---------------------------------------------------------------------------
+# Validation
+# ---------------------------------------------------------------------------
+
+
+class SSRFError(Exception):
+    """Raised when a URL is deemed unsafe for outbound access."""
+
+
+class SSRFValidationError(SSRFError):
+    """Specifically raised for SSRF-related validation failures."""
+
+
+class InvalidURLError(SSRFError):
+    """Raised when the URL is structurally invalid."""
+
+
+def validate_url(url: str) -> None:
+    """Validate a URL against SSRF rules.
+
+    Raises SSRFError / InvalidURLError on violation.
+    """
+    if not url or not isinstance(url, str):
+        raise InvalidURLError("URL must be a non-empty string")
+
+    # --- Scheme check ---
+    parsed = urlparse(url)
+    scheme = parsed.scheme.lower()
+
+    if scheme not in ("http", "https"):
+        raise InvalidURLError(f"Scheme '{scheme}' is not allowed (only http/https)")
+
+    # --- Blocked schemes ---
+    if scheme in _BLOCKED_SCHEMES:
+        raise InvalidURLError(f"Scheme '{scheme}' is explicitly blocked")
+
+    # --- Host check ---
+    host = parsed.hostname or ""
+    if not host:
+        raise InvalidURLError("URL has no hostname")
+
+    # --- Check for localhost / private IPs ---
+    try:
+        addr = ipaddress.ip_address(host)
+    except ValueError:
+        # Hostname — do DNS-based resolution (caller must handle)
+        pass
+    else:
+        # IPv4 blocked networks
+        if isinstance(addr, ipaddress.IPv4Address):
+            for net in _BLOCKED_NETWORKS:
+                if addr in net:
+                    raise SSRFValidationError(
+                        f"IP {addr} is in a blocked range"
+                    )
+
+        # IPv6 blocked networks
+        if isinstance(addr, ipaddress.IPv6Address):
+            for net in _BLOCKED_IPV6_NETWORKS:
+                if addr in net:
+                    raise SSRFValidationError(
+                        f"IPv6 address {addr} is in a blocked range"
+                    )
+
+    # --- Cloud metadata detection ---
+    host_lower = host.lower()
+    for pattern in _METADATA_PATTERNS:
+        if pattern in host_lower:
+            raise SSRFValidationError(
+                f"Hostname '{host}' matches cloud metadata pattern"
+            )
+
+    logger.info("URL validated: %s", url)
+
+
+def is_safe_host(host: str) -> bool:
+    """Quick hostname check — returns False for obviously unsafe hosts.
+
+    This is a first-pass filter; callers should still call validate_url().
+    """
+    lower = host.lower()
+    for pattern in _METADATA_PATTERNS:
+        if pattern in lower:
+            return False
+    return True
--- a/src/nsct/storage/init.py
+++ b/src/nsct/storage/init.py
@@ -0,0 +1 @@
+"""NSCT — storage package init."""
--- a/src/nsct/storage/engine.py
+++ b/src/nsct/storage/engine.py
@@ -0,0 +1,83 @@
+"""Async SQLAlchemy engine factory — creates and manages the connection pool."""
+
+from __future__ import annotations
+
+import logging
+from typing import AsyncGenerator
+
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from nsct.config import AppSettings
+from nsct.storage.models import Base  # noqa: F401 — Base is needed for create_all
+
+logger = logging.getLogger(__name__)
+
+_engine = None
+_session_factory = None
+
+
+async def get_engine(config: AppSettings) -> None:
+    """Initialise the global async engine and session factory.
+
+    Must be called once during application startup (lifespan) before any
+    request is handled.
+    """
+    global _engine, _session_factory
+
+    if _engine is not None:
+        logger.warning("Engine already initialised; skipping create_engine.")
+        return
+
+    db_url = config.postgres.url
+
+    _engine = create_async_engine(
+        db_url,
+        echo=config.debug,
+        pool_size=10,
+        max_overflow=20,
+        pool_recycle=1800,
+        pool_pre_ping=True,
+    )
+
+    # Ensure tables exist (schema migration not handled here — that is a
+    # separate migration step; this creates missing tables only).
+    async with _engine.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+
+    _session_factory = async_sessionmaker(
+        _engine,
+        class_=AsyncSession,
+        expire_on_commit=False,
+        autocommit=False,
+        autoflush=False,
+    )
+
+    logger.info("Async SQLAlchemy engine initialised. DB: %s", db_url)
+
+
+async def get_session() -> AsyncGenerator[AsyncSession, None]:
+    """Yield an async session for request-scoped database access."""
+    global _session_factory
+    if _session_factory is None:
+        raise RuntimeError("Engine not initialised. Call get_engine() first.")
+
+    session = _session_factory()
+    try:
+        yield session
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
+    finally:
+        await session.close()
+
+
+async def close_engine() -> None:
+    """Close the engine and release all pooled connections."""
+    global _engine, _session_factory
+    if _engine is not None:
+        await _engine.dispose()
+        logger.info("SQLAlchemy engine disposed.")
+        _engine = None
+        _session_factory = None
--- a/src/nsct/storage/models.py
+++ b/src/nsct/storage/models.py
@@ -0,0 +1,210 @@
+"""SQLAlchemy 2.0 Declarative models — mapping of NSCT domain objects to tables."""
+
+from __future__ import annotations
+
+import enum
+from datetime import datetime
+from typing import Any
+from uuid import UUID, uuid4
+
+from sqlalchemy import (
+    BigInteger,
+    Column,
+    DateTime,
+    Enum,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+)
+from sqlalchemy.orm import DeclarativeBase, relationship
+
+
+class ClaimType(str, enum.Enum):
+    FACTUAL = "factual"
+    OPINION = "opinion"
+    PREDICTION = "prediction"
+    EVALUATION = "evaluation"
+    COMPARISON = "comparison"
+
+
+class SourceType(str, enum.Enum):
+    NEWS = "news"
+    ACADEMIC = "academic"
+    BLOG = "blog"
+    GOVERNMENT = "government"
+    CORPORATE = "corporate"
+    SOCIAL_MEDIA = "social_media"
+    DOCUMENT = "document"
+    OTHER = "other"
+
+
+class EvidenceRelationType(str, enum.Enum):
+    AGREES = "agrees"
+    DISAGREES = "disagrees"
+    NEUTRAL = "neutral"
+    PARTIALLY_AGREES = "partially_agrees"
+    PARTIALLY_DISAGREES = "partially_disagrees"
+    CONTRADICTS = "contradicts"
+
+
+class EdgeRelation(str, enum.Enum):
+    CITATION = "citation"
+    CORROBORATION = "corroboration"
+    CONTRADICTION = "contradiction"
+    DEPENDS_ON = "depends_on"
+    SUPPLEMENTS = "supplements"
+
+
+# ---------------------------------------------------------------------------
+# Base
+# ---------------------------------------------------------------------------
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+# ---------------------------------------------------------------------------
+# SearchQuery
+# ---------------------------------------------------------------------------
+
+
+class SearchQueryModel(Base):
+    __tablename__ = "search_queries"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    research_id = Column(String(36), nullable=False, default=lambda: str(uuid4()))
+    query = Column(Text, nullable=False)
+    purpose = Column(Text, nullable=True)
+    language = Column(String(16), nullable=False, default="de")
+    category = Column(String(64), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    __table_args__ = (Index("ix_search_queries_research_id", "research_id"),)
+
+
+# ---------------------------------------------------------------------------
+# Source
+# ---------------------------------------------------------------------------
+
+
+class SourceModel(Base):
+    __tablename__ = "sources"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    url = Column(Text, nullable=False)
+    canonical_url = Column(Text, nullable=True)
+    domain = Column(String(255), nullable=False)
+    title = Column(Text, nullable=True)
+    author = Column(Text, nullable=True)
+    publisher = Column(Text, nullable=True)
+    publication_date = Column(DateTime, nullable=True)
+    retrieved_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    content_type = Column(String(128), nullable=True)
+    source_type = Column(Enum(SourceType), nullable=True)
+    language = Column(String(16), nullable=False, default="unknown")
+    content_hash = Column(String(64), nullable=True)
+    parent_source_id = Column(String(36), ForeignKey("sources.id"), nullable=True)
+    content = Column(Text, nullable=True)
+
+    # Relationships
+    claims = relationship("ClaimModel", back_populates="source", cascade="all, delete-orphan")
+
+    __table_args__ = (
+        Index("ix_sources_domain", "domain"),
+        Index("ix_sources_parent_source_id", "parent_source_id"),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Claim
+# ---------------------------------------------------------------------------
+
+
+class ClaimModel(Base):
+    __tablename__ = "claims"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    source_id = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    claim = Column(Text, nullable=False)
+    normalized_claim = Column(Text, nullable=True)
+    claim_type = Column(Enum(ClaimType), nullable=False, default=ClaimType.FACTUAL)
+    subject = Column(Text, nullable=True)
+    predicate = Column(Text, nullable=True)
+    object = Column(Text, nullable=True)
+    evidence_span = Column(Text, nullable=True)
+    confidence = Column(Float, nullable=False, default=1.0)
+    event_date = Column(DateTime, nullable=True)
+
+    # Relationships
+    source = relationship("SourceModel", back_populates="claims")
+
+
+# ---------------------------------------------------------------------------
+# EvidenceRelation
+# ---------------------------------------------------------------------------
+
+
+class EvidenceRelationModel(Base):
+    __tablename__ = "evidence_relations"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    claim_a = Column(String(36), ForeignKey("claims.id"), nullable=False)
+    claim_b = Column(String(36), ForeignKey("claims.id"), nullable=False)
+    relation = Column(Enum(EvidenceRelationType), nullable=False)
+    confidence = Column(Float, nullable=False, default=0.5)
+    reason = Column(Text, nullable=True)
+
+    __table_args__ = (
+        Index("ix_evidence_relations_claim_a", "claim_a"),
+        Index("ix_evidence_relations_claim_b", "claim_b"),
+        # Ensure uniqueness of the (a, b, relation) triple
+        Index("uq_evidence_relations_ab", "claim_a", "claim_b", "relation", unique=True),
+    )
+
+
+# ---------------------------------------------------------------------------
+# CitationEdge
+# ---------------------------------------------------------------------------
+
+
+class CitationEdgeModel(Base):
+    __tablename__ = "citation_edges"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    source_from = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    source_to = Column(String(36), ForeignKey("sources.id"), nullable=False)
+    relation = Column(Enum(EdgeRelation), nullable=False)
+    confidence = Column(Float, nullable=False, default=1.0)
+
+    __table_args__ = (
+        Index("ix_citation_edges_source_from", "source_from"),
+        Index("ix_citation_edges_source_to", "source_to"),
+    )
+
+
+# ---------------------------------------------------------------------------
+# ResearchReport
+# ---------------------------------------------------------------------------
+
+
+class ResearchReportModel(Base):
+    __tablename__ = "research_reports"
+
+    id = Column(String(36), primary_key=True, default=lambda: str(uuid4()))
+    research_id = Column(String(36), ForeignKey("search_queries.research_id"), nullable=False)
+    query = Column(Text, nullable=False)
+    summary = Column(Text, nullable=False, default="")
+    findings = Column(Text, nullable=False, default="[]")
+    disagreements = Column(Text, nullable=False, default="[]")
+    uncertainties = Column(Text, nullable=False, default="[]")
+    source_statistics = Column(Text, nullable=False, default="{}")
+    methodology = Column(Text, nullable=False, default="")
+    generated_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    __table_args__ = (
+        Index("ix_research_reports_research_id", "research_id"),
+    )