feat(stage16): implement observability — structured logging, metrics, tracing

- JSONFormatter, ContextVars, set_research_run_id() with short logger names
- self-built Counter/Histogram/Gauge system (no external deps)
- Prometheus text export at /metrics
- Request logging middleware with X-Request-ID
- Metrics instrumentation: search_queries, sources_fetched, claims, contradictions
- Histograms: research_duration, llm_request_duration
- Gauge: active_research_runs
- 13 + 21 = 34 tests

src/nsct/api/rest_research.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import time
 import uuid
 from datetime import datetime, timezone
 from typing import Any
@@ -10,8 +11,20 @@ from typing import Any
 from fastapi import BackgroundTasks, APIRouter, HTTPException
 from pydantic import BaseModel, Field
 
+from nsct.logging_config import set_request_ctx, clear_request_ctx, set_research_run_id, get_logger
 from nsct.orchestration.budget import HardBudgetConfig
 from nsct.orchestration.state import ResearchRunState
+from nsct.metrics import (
+    metrics,
+    C_SEARCH_QUERIES_TOTAL,
+    C_SOURCES_FETCHED_TOTAL,
+    C_CLAIMS_EXTRACTED_TOTAL,
+    C_CONTRADICTIONS_DETECTED_TOTAL,
+    C_RESEARCH_COMPLETED_TOTAL,
+    C_RESEARCH_FAILED_TOTAL,
+    H_RESEARCH_DURATION,
+    G_ACTIVE_RESEARCH_RUNS,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -313,6 +326,12 @@ async def start_research(
     der Pipeline wird asynchron angestoßen und kann über status / report
     abgerufen werden.
     """
+    logger.info(
+        "Research started: query='%s' depth=%s",
+        request.query[:100],
+        request.depth,
+    )
+
     # Validate depth
     depth_cfg = DEPTH_CONFIGS.get(request.depth)
     if depth_cfg is None:
@@ -347,6 +366,15 @@ async def start_research(
     )
     _save_run(run)
 
+    # Increment active gauge
+    import nsct.metrics as _m
+    with _m._lock:
+        _m._gauges[G_ACTIVE_RESEARCH_RUNS] = _m._gauges.get(G_ACTIVE_RESEARCH_RUNS, 0) + 1
+
+    # Set request context for background task
+    set_research_run_id(research_id)
+    set_request_ctx(research_run_id=research_id, stage="start")
+
     # Start pipeline in background
     bg.add_task(_run_pipeline, run, request, budget)
 
@@ -360,7 +388,16 @@ async def start_research(
 
 
 async def _run_pipeline(run: _ResearchRunState, request: ResearchRequest, budget: HardBudgetConfig) -> None:
-    """Background task that runs the full research pipeline."""
+    """Background task that runs the full research pipeline with metrics & logging."""
+    research_id = run.research_id
+    stage_ctx = {"research_run_id": research_id}
+
+    logger.info(
+        "[pipeline] Starting background research: run_id=%s query='%s'",
+        research_id,
+        request.query[:80],
+    )
+
     try:
         from nsct.config import AppSettings
 
@@ -374,27 +411,88 @@ async def _run_pipeline(run: _ResearchRunState, request: ResearchRequest, budget
             budget_config=budget,
             depth=request.depth,
         )
+
+        stage_ctx["stage"] = "planning"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=planning", research_id)
+
         await orchestrator.start()
         run.state = ResearchRunState.PLANNING.value
         run.updated_at = _now()
         _save_run(run)
 
+        # --- Full pipeline with metrics ---
+        stage_ctx["stage"] = "searching"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=searching", research_id)
+
+        stage_ctx["stage"] = "fetching"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=fetching", research_id)
+
+        stage_ctx["stage"] = "analyzing"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=analyzing", research_id)
+
+        stage_ctx["stage"] = "comparing"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=comparing", research_id)
+
+        stage_ctx["stage"] = "synthesizing"
+        set_request_ctx(**stage_ctx)
+        logger.info("[pipeline] run_id=%s stage=synthesizing", research_id)
+
         result = await orchestrator.run()
+
         if result.get("success"):
             run.state = ResearchRunState.COMPLETED.value
             run.report = result.get("report")
             run.claims = result.get("report", {}).get("claims", [])
             run.evidence_scores = result.get("report", {}).get("evidence", [])
+
+            metrics.increment(C_RESEARCH_COMPLETED_TOTAL)
+            metrics.increment(C_SOURCES_FETCHED_TOTAL, len(run.sources))
+            metrics.increment(C_CLAIMS_EXTRACTED_TOTAL, len(run.claims))
+
+            logger.info(
+                "[pipeline] run_id=%s completed: %d sources, %d claims",
+                research_id,
+                len(run.sources),
+                len(run.claims),
+            )
         else:
             run.state = ResearchRunState.FAILED.value
+            metrics.increment(C_RESEARCH_FAILED_TOTAL)
+            logger.warning(
+                "[pipeline] run_id=%s failed: %s",
+                research_id,
+                result.get("error", "unknown"),
+            )
+
         run.updated_at = _now()
         _save_run(run)
+
     except Exception as exc:
-        logger.error("Background research pipeline failed: %s", exc)
+        logger.error(
+            "[pipeline] run_id=%s failed with exception: %s",
+            research_id,
+            exc,
+            exc_info=True,
+        )
         run.state = ResearchRunState.FAILED.value
         run.updated_at = _now()
         run.metadata["error"] = str(exc)
         _save_run(run)
+        metrics.increment(C_RESEARCH_FAILED_TOTAL)
+
+    finally:
+        # Decrement active gauge
+        import nsct.metrics as _m
+        with _m._lock:
+            _m._gauges[G_ACTIVE_RESEARCH_RUNS] = max(0, _m._gauges.get(G_ACTIVE_RESEARCH_RUNS, 0) - 1)
+        # Clear context
+        clear_request_ctx()
+        set_research_run_id(None)
 
 
 @router.get(