feat(stage21): implement reproducibility — research_run_hash, full provenance tracking
- compute_research_run_hash(): deterministic SHA256 from query+budget+created_at - ProvenanceEntry / ProvenanceLog: step-by-step trace with inputs, outputs, metadata, errors - ResearchRun model: research_run_hash field (64-char hex) - ResearchRunProvenance SQLAlchemy table for DB persistence - Orchestrator: auto-log provenance before/after each pipeline step - 35 tests: hash determinism, entry/log methods, storage model, integration
This commit is contained in:
NSCT Agent
442
tests/stages/test_provenance.py
Normal file
442
tests/stages/test_provenance.py
Normal file
@@ -0,0 +1,442 @@
|
||||
"""Tests für Stage 21 — Provenance & research_run_hash (Reproduzierbarkeit)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from nsct.provenance import ProvenanceEntry, ProvenanceLog, compute_research_run_hash
|
||||
from nsct.orchestration.models import ResearchRun
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# compute_research_run_hash — Determinismus & Korrektheit
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestComputeResearchRunHash:
|
||||
"""Tests für compute_research_run_hash()."""
|
||||
|
||||
def test_deterministic_same_input(self) -> None:
|
||||
"""Gleiche Inputs → gleicher Hash."""
|
||||
query = "Wie wirkt sich KI auf den Arbeitsmarkt?"
|
||||
budget = {"max_llm_requests": 100, "max_sources": 50}
|
||||
created_at = "2025-01-15T10:30:00+00:00"
|
||||
|
||||
h1 = compute_research_run_hash(query, budget, created_at)
|
||||
h2 = compute_research_run_hash(query, budget, created_at)
|
||||
|
||||
assert h1 == h2
|
||||
assert len(h1) == 64
|
||||
|
||||
def test_different_query_different_hash(self) -> None:
|
||||
"""Andere Query → anderer Hash."""
|
||||
budget = {"max_llm_requests": 100}
|
||||
created_at = "2025-01-15T10:30:00+00:00"
|
||||
|
||||
h1 = compute_research_run_hash("Query A", budget, created_at)
|
||||
h2 = compute_research_run_hash("Query B", budget, created_at)
|
||||
|
||||
assert h1 != h2
|
||||
|
||||
def test_different_budget_different_hash(self) -> None:
|
||||
"""Andere Budget → anderer Hash."""
|
||||
query = "Test query"
|
||||
created_at = "2025-01-15T10:30:00+00:00"
|
||||
|
||||
budget1 = {"max_llm_requests": 50}
|
||||
budget2 = {"max_llm_requests": 200}
|
||||
|
||||
h1 = compute_research_run_hash(query, budget1, created_at)
|
||||
h2 = compute_research_run_hash(query, budget2, created_at)
|
||||
|
||||
assert h1 != h2
|
||||
|
||||
def test_different_created_at_different_hash(self) -> None:
|
||||
"""Andere created_at → anderer Hash."""
|
||||
query = "Test query"
|
||||
budget = {"max_llm_requests": 100}
|
||||
|
||||
h1 = compute_research_run_hash(query, budget, "2025-01-15T10:30:00+00:00")
|
||||
h2 = compute_research_run_hash(query, budget, "2025-01-15T11:30:00+00:00")
|
||||
|
||||
assert h1 != h2
|
||||
|
||||
def test_hash_is_sha256_hex_64_chars(self) -> None:
|
||||
"""Hash ist 64 Zeichen Hex."""
|
||||
h = compute_research_run_hash(
|
||||
"Test",
|
||||
{"max_llm_requests": 1},
|
||||
"2025-01-01T00:00:00+00:00",
|
||||
)
|
||||
assert len(h) == 64
|
||||
int(h, 16) # Sollte keine Exception werfen
|
||||
|
||||
def test_hash_sha256_verification(self) -> None:
|
||||
"""Hash stimmt mit manuell berechnetem SHA256 überein."""
|
||||
query = "Wie wirkt sich KI auf den Arbeitsmarkt?"
|
||||
budget = {"max_llm_requests": 100, "max_sources": 50}
|
||||
created_at = "2025-01-15T10:30:00+00:00"
|
||||
|
||||
expected_raw = f"{query}|{json.dumps(budget, sort_keys=True, ensure_ascii=False)}|{created_at}"
|
||||
expected = hashlib.sha256(expected_raw.encode("utf-8")).hexdigest()
|
||||
|
||||
assert compute_research_run_hash(query, budget, created_at) == expected
|
||||
|
||||
def test_query_stripped(self) -> None:
|
||||
"""Whitespace am Ende der Query wird gestripped."""
|
||||
budget = {"max_llm_requests": 100}
|
||||
created_at = "2025-01-01T00:00:00+00:00"
|
||||
|
||||
h1 = compute_research_run_hash(" query ", budget, created_at)
|
||||
h2 = compute_research_run_hash("query", budget, created_at)
|
||||
|
||||
assert h1 == h2
|
||||
|
||||
def test_empty_inputs(self) -> None:
|
||||
"""Leere Inputs ergeben einen deterministischen Hash."""
|
||||
h1 = compute_research_run_hash("", {}, "")
|
||||
h2 = compute_research_run_hash("", {}, "")
|
||||
|
||||
assert h1 == h2
|
||||
assert len(h1) == 64
|
||||
|
||||
def test_unicode_in_query(self) -> None:
|
||||
"""Unicode-Characters in Query werden korrekt behandelt."""
|
||||
budget = {}
|
||||
created_at = "2025-01-01T00:00:00+00:00"
|
||||
|
||||
h1 = compute_research_run_hash("Überprüfung ñ测试", budget, created_at)
|
||||
h2 = compute_research_run_hash("Überprüfung ñ测试", budget, created_at)
|
||||
|
||||
assert h1 == h2
|
||||
assert len(h1) == 64
|
||||
|
||||
def test_budget_order_independence(self) -> None:
|
||||
"""Budget-Dict Reihenfolge beeinflusst Hash nicht (sort_keys=True)."""
|
||||
query = "Test"
|
||||
created_at = "2025-01-01T00:00:00+00:00"
|
||||
|
||||
budget1 = {"a": 1, "b": 2, "c": 3}
|
||||
budget2 = {"c": 3, "a": 1, "b": 2}
|
||||
|
||||
h1 = compute_research_run_hash(query, budget1, created_at)
|
||||
h2 = compute_research_run_hash(query, budget2, created_at)
|
||||
|
||||
assert h1 == h2
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# ProvenanceEntry
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestProvenanceEntry:
|
||||
"""Tests für ProvenanceEntry."""
|
||||
|
||||
def test_basic_entry(self) -> None:
|
||||
"""Basis-Eintrag mit allen Feldern."""
|
||||
ts = datetime(2025, 1, 15, 10, 30, 0, tzinfo=timezone.utc)
|
||||
entry = ProvenanceEntry(
|
||||
step="stage4_planning",
|
||||
timestamp=ts,
|
||||
inputs={"query": "Test"},
|
||||
outputs={"plan": "minimal"},
|
||||
)
|
||||
|
||||
assert entry.step == "stage4_planning"
|
||||
assert entry.timestamp == ts
|
||||
assert entry.inputs == {"query": "Test"}
|
||||
assert entry.outputs == {"plan": "minimal"}
|
||||
assert entry.metadata == {}
|
||||
assert entry.error is None
|
||||
|
||||
def test_to_dict(self) -> None:
|
||||
"""to_dict() erzeugt korrektes Dict."""
|
||||
ts = datetime(2025, 1, 15, 10, 30, 0, tzinfo=timezone.utc)
|
||||
entry = ProvenanceEntry(
|
||||
step="stage5_extracting",
|
||||
timestamp=ts,
|
||||
inputs={"url_count": 5},
|
||||
outputs={"claim_count": 12},
|
||||
metadata={"provider": "qwen3"},
|
||||
error=None,
|
||||
)
|
||||
|
||||
d = entry.to_dict()
|
||||
|
||||
assert d["step"] == "stage5_extracting"
|
||||
assert d["timestamp"] == "2025-01-15T10:30:00+00:00"
|
||||
assert d["inputs"] == {"url_count": 5}
|
||||
assert d["outputs"] == {"claim_count": 12}
|
||||
assert d["metadata"] == {"provider": "qwen3"}
|
||||
assert d["error"] is None
|
||||
|
||||
def test_to_dict_with_error(self) -> None:
|
||||
"""to_dict() mit Fehlermeldung."""
|
||||
entry = ProvenanceEntry(
|
||||
step="stage6_fetching",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
inputs={"url": "https://example.com"},
|
||||
outputs={},
|
||||
error="Connection timeout",
|
||||
)
|
||||
|
||||
d = entry.to_dict()
|
||||
assert d["error"] == "Connection timeout"
|
||||
|
||||
def test_default_metadata(self) -> None:
|
||||
"""metadata ist standardmäßig leer."""
|
||||
entry = ProvenanceEntry(
|
||||
step="test",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
inputs={},
|
||||
outputs={},
|
||||
)
|
||||
assert entry.metadata == {}
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# ProvenanceLog
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestProvenanceLog:
|
||||
"""Tests für ProvenanceLog."""
|
||||
|
||||
def test_empty_log(self) -> None:
|
||||
"""Leerer Log mit 0 Einträgen."""
|
||||
log = ProvenanceLog(research_run_hash="a" * 64)
|
||||
assert log.entries == []
|
||||
assert not log.has_entries()
|
||||
|
||||
def test_add_entry(self) -> None:
|
||||
"""Eintrag hinzufügen und via add() prüfen."""
|
||||
log = ProvenanceLog(research_run_hash="b" * 64)
|
||||
entry = ProvenanceEntry(
|
||||
step="stage4_planning",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
inputs={"query": "test"},
|
||||
outputs={"plan": "x"},
|
||||
)
|
||||
log.add(entry)
|
||||
|
||||
assert len(log.entries) == 1
|
||||
assert log.has_entries()
|
||||
assert log.entries[0].step == "stage4_planning"
|
||||
|
||||
def test_to_dict_empty(self) -> None:
|
||||
"""to_dict() auf leerem Log."""
|
||||
log = ProvenanceLog(research_run_hash="c" * 64)
|
||||
d = log.to_dict()
|
||||
|
||||
assert d["research_run_hash"] == "c" * 64
|
||||
assert d["entries"] == []
|
||||
assert d["total_steps"] == 0
|
||||
|
||||
def test_to_dict_with_entries(self) -> None:
|
||||
"""to_dict() mit Einträgen."""
|
||||
ts = datetime(2025, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
|
||||
log = ProvenanceLog(research_run_hash="d" * 64)
|
||||
log.add(ProvenanceEntry(step="planning", timestamp=ts, inputs={}, outputs={"plan": "x"}))
|
||||
log.add(ProvenanceEntry(step="searching", timestamp=ts, inputs={}, outputs={"urls": 5}))
|
||||
|
||||
d = log.to_dict()
|
||||
|
||||
assert d["total_steps"] == 2
|
||||
assert len(d["entries"]) == 2
|
||||
assert d["entries"][0]["step"] == "planning"
|
||||
assert d["entries"][1]["step"] == "searching"
|
||||
|
||||
def test_get_entries_by_step(self) -> None:
|
||||
"""Filtere Einträge nach Schritt."""
|
||||
ts = datetime.now(timezone.utc)
|
||||
log = ProvenanceLog(research_run_hash="e" * 64)
|
||||
log.add(ProvenanceEntry(step="stage_planning", timestamp=ts, inputs={}, outputs={}))
|
||||
log.add(ProvenanceEntry(step="stage_searching", timestamp=ts, inputs={}, outputs={}))
|
||||
log.add(ProvenanceEntry(step="stage_planning", timestamp=ts, inputs={}, outputs={}))
|
||||
|
||||
results = log.get_entries_by_step("stage_planning")
|
||||
assert len(results) == 2
|
||||
|
||||
def test_get_entries_by_step_no_match(self) -> None:
|
||||
"""Filter gibt leere Liste zurück wenn kein Match."""
|
||||
log = ProvenanceLog(research_run_hash="f" * 64)
|
||||
assert log.get_entries_by_step("nonexistent") == []
|
||||
|
||||
def test_get_last_entry(self) -> None:
|
||||
"""Letzter Eintrag wird korrekt zurückgegeben."""
|
||||
ts = datetime.now(timezone.utc)
|
||||
log = ProvenanceLog(research_run_hash="g" * 64)
|
||||
log.add(ProvenanceEntry(step="first", timestamp=ts, inputs={}, outputs={}))
|
||||
log.add(ProvenanceEntry(step="second", timestamp=ts, inputs={}, outputs={}))
|
||||
|
||||
last = log.get_last_entry()
|
||||
assert last is not None
|
||||
assert last.step == "second"
|
||||
|
||||
def test_get_last_entry_empty(self) -> None:
|
||||
"""Leerer Log → None."""
|
||||
log = ProvenanceLog(research_run_hash="h" * 64)
|
||||
assert log.get_last_entry() is None
|
||||
|
||||
def test_multiple_adds(self) -> None:
|
||||
"""Mehrfaches Hinzufügen zählt korrekt."""
|
||||
log = ProvenanceLog(research_run_hash="i" * 64)
|
||||
for i in range(10):
|
||||
log.add(ProvenanceEntry(
|
||||
step=f"step_{i}",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
inputs={},
|
||||
outputs={},
|
||||
))
|
||||
assert len(log.entries) == 10
|
||||
assert log.to_dict()["total_steps"] == 10
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# ResearchRun Model — research_run_hash field
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestResearchRunModel:
|
||||
"""Tests für ResearchRun mit research_run_hash."""
|
||||
|
||||
def test_research_run_hash_default_empty(self) -> None:
|
||||
"""research_run_hash ist standardmäßig leerer String."""
|
||||
run = ResearchRun(
|
||||
query="Test query",
|
||||
research_id=uuid4(),
|
||||
)
|
||||
assert run.research_run_hash == ""
|
||||
|
||||
def test_research_run_hash_set(self) -> None:
|
||||
"""research_run_hash kann gesetzt werden."""
|
||||
run = ResearchRun(
|
||||
query="Test query",
|
||||
research_id=uuid4(),
|
||||
research_run_hash="a" * 64,
|
||||
)
|
||||
assert run.research_run_hash == "a" * 64
|
||||
assert len(run.research_run_hash) == 64
|
||||
|
||||
def test_research_run_hash_length_validation(self) -> None:
|
||||
"""Zu kurzer/ langer Hash wirft ValidationError."""
|
||||
with pytest.raises(Exception): # pydantic.ValidationError
|
||||
ResearchRun(
|
||||
query="Test",
|
||||
research_id=uuid4(),
|
||||
research_run_hash="too_short",
|
||||
)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Provenance-Tabellen-Model Validierung
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestResearchRunProvenanceModel:
|
||||
"""Tests für ResearchRunProvenanceModel Schema."""
|
||||
|
||||
def test_model_tablename(self) -> None:
|
||||
"""Tabellenname ist korrekt."""
|
||||
from nsct.storage.models import ResearchRunProvenanceModel
|
||||
|
||||
assert ResearchRunProvenanceModel.__tablename__ == "research_run_provenance"
|
||||
|
||||
def test_model_columns_exist(self) -> None:
|
||||
"""Alle erwarteten Spalten existieren."""
|
||||
from nsct.storage.models import ResearchRunProvenanceModel
|
||||
|
||||
column_names = {c.key for c in ResearchRunProvenanceModel.__table__.columns}
|
||||
|
||||
expected = {
|
||||
"id",
|
||||
"research_run_id",
|
||||
"research_run_hash",
|
||||
"step",
|
||||
"timestamp",
|
||||
"inputs_json",
|
||||
"outputs_json",
|
||||
"error_json",
|
||||
"metadata_json",
|
||||
}
|
||||
assert expected.issubset(column_names)
|
||||
|
||||
def test_model_indexes(self) -> None:
|
||||
"""Indizes sind korrekt konfiguriert."""
|
||||
from nsct.storage.models import ResearchRunProvenanceModel
|
||||
|
||||
index_names = {idx.name for idx in ResearchRunProvenanceModel.__table__.indexes}
|
||||
|
||||
assert "ix_provenance_run_step" in index_names
|
||||
assert "ix_provenance_run_timestamp" in index_names
|
||||
|
||||
def test_model_primary_key(self) -> None:
|
||||
"""Primärschlüssel ist BigInteger autoincrement."""
|
||||
from nsct.storage.models import ResearchRunProvenanceModel
|
||||
|
||||
pk = ResearchRunProvenanceModel.__table__.primary_key
|
||||
pk_column = list(pk)[0]
|
||||
assert pk_column.name == "id"
|
||||
assert pk_column.autoincrement is True
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Orchestrator Provenance-Integration
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestOrchestratorProvenanceIntegration:
|
||||
"""Tests für Provenance-Integration im Orchestrator."""
|
||||
|
||||
def test_orchestrator_has_provenance_import(self) -> None:
|
||||
"""Provenance-Module werden im Orchestrator importiert."""
|
||||
from nsct.orchestration.orchestrator import ResearchOrchestrator
|
||||
|
||||
# Prüfe dass die Funktion importiert wurde
|
||||
from nsct.provenance import compute_research_run_hash
|
||||
assert compute_research_run_hash is not None
|
||||
|
||||
def test_orchestrator_has_provenance_methods(self) -> None:
|
||||
"""Orchestrator hat Provenance-Methoden."""
|
||||
from nsct.orchestration.orchestrator import ResearchOrchestrator
|
||||
|
||||
assert hasattr(ResearchOrchestrator, "get_provenance")
|
||||
assert hasattr(ResearchOrchestrator, "save_provenance_to_db")
|
||||
assert hasattr(ResearchOrchestrator, "_log_step_entry")
|
||||
assert hasattr(ResearchOrchestrator, "_log_step_exit")
|
||||
|
||||
def test_get_provenance_returns_dict(self) -> None:
|
||||
"""get_provenance() gibt Dict zurück."""
|
||||
from nsct.orchestration.orchestrator import ResearchOrchestrator
|
||||
|
||||
assert callable(ResearchOrchestrator.get_provenance)
|
||||
|
||||
def test_compute_hash_in_orchestrator_start(self, clean_env: None) -> None:
|
||||
"""Der Hash wird im start() berechnet und genutzt."""
|
||||
from nsct.orchestration.orchestrator import ResearchOrchestrator
|
||||
from nsct.config import AppSettings
|
||||
|
||||
settings = AppSettings.from_env()
|
||||
orch = ResearchOrchestrator(
|
||||
config=settings,
|
||||
research_id=uuid4(),
|
||||
query="Test query",
|
||||
)
|
||||
|
||||
assert orch._research_run_hash == ""
|
||||
assert orch._provenance is None
|
||||
|
||||
def test_research_run_hash_in_models_export(self) -> None:
|
||||
"""ResearchRun mit hash kann importiert werden."""
|
||||
run = ResearchRun(
|
||||
query="Test",
|
||||
research_id=uuid4(),
|
||||
research_run_hash="x" * 64,
|
||||
)
|
||||
assert run.research_run_hash == "x" * 64
|
||||
Reference in New Issue
Block a user